DRAFT: Sharded Kube-state-metrics

kube-state-metrics/common_config.libsonnet

@@ -0,0 +1,129 @@
+local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet';
+
+{
+  local this = self,
+
+  image:: error 'image is required',
+
+  server_info:: {
+    port: 8080,
+    telemetry_host: '0.0.0.0',
+    telemetry_port: 8081,
+  },
+
+  local container = kausal.core.v1.container,
+  local containerPort = kausal.core.v1.containerPort,
+  container:: container.new('kube-state-metrics', this.image)
+              + container.withPorts([
+                containerPort.new('ksm', 8080),
+                containerPort.new('self-metrics', 8081),
+              ])
+              + kausal.util.resourcesRequests('50m', '50Mi')
+              + kausal.util.resourcesLimits('250m', '150Mi'),
+
+  local policyRule = kausal.rbac.v1.policyRule,
+  rbac:
+    kausal.util.rbac('kube-state-metrics', [
+      policyRule.new()
+      + policyRule.withApiGroups([''])
+      + policyRule.withResources([
+        'configmaps',
+        'secrets',
+        'nodes',
+        'pods',
+        'services',
+        'resourcequotas',
+        'replicationcontrollers',
+        'limitranges',
+        'persistentvolumeclaims',
+        'persistentvolumes',
+        'namespaces',
+        'endpoints',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['extensions'])
+      + policyRule.withResources([
+        'daemonsets',
+        'deployments',
+        'replicasets',
+        'ingresses',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['apps'])
+      + policyRule.withResources([
+        'daemonsets',
+        'deployments',
+        'replicasets',
+        'statefulsets',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['batch'])
+      + policyRule.withResources([
+        'cronjobs',
+        'jobs',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['autoscaling'])
+      + policyRule.withResources([
+        'horizontalpodautoscalers',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['authorization.k8s.io'])
+      + policyRule.withResources(['subjectaccessreviews'])
+      + policyRule.withVerbs(['create']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['ingresses'])
+      + policyRule.withResources(['ingress'])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['policy'])
+      + policyRule.withResources(['poddisruptionbudgets'])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['certificates.k8s.io'])
+      + policyRule.withResources(['certificatesigningrequests'])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['storage.k8s.io'])
+      + policyRule.withResources([
+        'storageclasses',
+        'volumeattachments',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['admissionregistration.k8s.io'])
+      + policyRule.withResources([
+        'mutatingwebhookconfigurations',
+        'validatingwebhookconfigurations',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['networking.k8s.io'])
+      + policyRule.withResources([
+        'networkpolicies',
+        'ingresses',
+      ])
+      + policyRule.withVerbs(['list', 'watch']),
+
+      policyRule.new()
+      + policyRule.withApiGroups(['coordination.k8s.io'])
+      + policyRule.withResources(['leases'])
+      + policyRule.withVerbs(['list', 'watch']),
+    ]),
+}

kube-state-metrics/main.libsonnet

@@ -3,142 +3,31 @@ local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libso
 {
   new(
     namespace,
+    name='kube-state-metrics',
     image='registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.1.0',
+    sharded=false,
+    replicas=3,
   ):: {
     local k = kausal {
       _config+: {
         namespace: namespace,
       },
     },
-
-    local container = k.core.v1.container,
-    local containerPort = k.core.v1.containerPort,
-    container::
-      container.new('kube-state-metrics', image)
-      + container.withArgs([
-        '--port=8080',
-        '--telemetry-host=0.0.0.0',
-        '--telemetry-port=8081',
-      ])
-      + container.withPorts([
-        containerPort.new('ksm', 8080),
-        containerPort.new('self-metrics', 8081),
-      ])
-      + k.util.resourcesRequests('50m', '50Mi')
-      + k.util.resourcesLimits('250m', '150Mi'),
-
-    local deployment = k.apps.v1.deployment,
-    deployment:
-      deployment.new('kube-state-metrics', 1, [self.container])
-      + deployment.mixin.spec.template.spec.withServiceAccountName(self.rbac.service_account.metadata.name)
-      + deployment.mixin.spec.template.spec.securityContext.withRunAsUser(65534)
-      + deployment.mixin.spec.template.spec.securityContext.withRunAsGroup(65534),
-
-    local policyRule = k.rbac.v1.policyRule,
-    rbac:
-      k.util.rbac('kube-state-metrics', [
-        policyRule.new()
-        + policyRule.withApiGroups([''])
-        + policyRule.withResources([
-          'configmaps',
-          'secrets',
-          'nodes',
-          'pods',
-          'services',
-          'resourcequotas',
-          'replicationcontrollers',
-          'limitranges',
-          'persistentvolumeclaims',
-          'persistentvolumes',
-          'namespaces',
-          'endpoints',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['extensions'])
-        + policyRule.withResources([
-          'daemonsets',
-          'deployments',
-          'replicasets',
-          'ingresses',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['apps'])
-        + policyRule.withResources([
-          'daemonsets',
-          'deployments',
-          'replicasets',
-          'statefulsets',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['batch'])
-        + policyRule.withResources([
-          'cronjobs',
-          'jobs',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['autoscaling'])
-        + policyRule.withResources([
-          'horizontalpodautoscalers',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['authorization.k8s.io'])
-        + policyRule.withResources(['subjectaccessreviews'])
-        + policyRule.withVerbs(['create']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['ingresses'])
-        + policyRule.withResources(['ingress'])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['policy'])
-        + policyRule.withResources(['poddisruptionbudgets'])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['certificates.k8s.io'])
-        + policyRule.withResources(['certificatesigningrequests'])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['storage.k8s.io'])
-        + policyRule.withResources([
-          'storageclasses',
-          'volumeattachments',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['admissionregistration.k8s.io'])
-        + policyRule.withResources([
-          'mutatingwebhookconfigurations',
-          'validatingwebhookconfigurations',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['networking.k8s.io'])
-        + policyRule.withResources([
-          'networkpolicies',
-          'ingresses',
-        ])
-        + policyRule.withVerbs(['list', 'watch']),
-
-        policyRule.new()
-        + policyRule.withApiGroups(['coordination.k8s.io'])
-        + policyRule.withResources(['leases'])
-        + policyRule.withVerbs(['list', 'watch']),
-      ]),
+    local common_config = {
+      image:: image,
+    } + import './common_config.libsonnet',
+    local singleton_ksm = common_config + import './singleton.libsonnet',
+    local sharded_ksm = common_config {
+      replicas:: replicas,
+      name:: name,
+      namespace:: namespace,
+    } + import './sharded.libsonnet',
+
+    deployment: if sharded then {} else singleton_ksm.deployment,
+    [if sharded then 'statefulset']: sharded_ksm.statefulset,
+    [if sharded then 'configmaps']: sharded_ksm.configmaps,
+
+    rbac: common_config.rbac,
   },
 
   scrape_config: (import './scrape_config.libsonnet'),

kube-state-metrics/sharded.libsonnet

@@ -0,0 +1,68 @@
+local shared_config = import './shared_config.libsonnet';
+local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet';
+
+{
+  local this = self,
+
+  replicas:: error 'replicas are required',
+  name:: error 'name is required',
+  namespace:: error 'namespace is required',
+
+  config_file:: '$(POD_NAME).yml',
+  config_path:: '/etc/kube-state-metrics',
+
+  rbac:: shared_config.rbac,
+  ksm_config:: {
+    port: shared_config.server_info.port,
+    telemetry_host: shared_config.server_info.telemetry_host,
+    telemetry_port: shared_config.server_info.telemetry_port,
+  },
+
+  local configMap = kausal.core.v1.configMap,
+  config_map: configMap.new('%s-config' % this.name)
+              + configMap.withData({
+                ['%s-%s.yml' % [this.name, i]]:
+                  kausal.util.manifestYaml(
+                    this.ksm_config
+                    {
+                      total_shards: this.replicas,
+                      shard: i,
+                    }
+                  )
+                for i in std.range(0, this.replicas - 1)
+              }),
+
+  local container = kausal.core.v1.container,
+  local containerPort = kausal.core.v1.containerPort,
+  local envVar = kausal.core.v1.envVar,
+  container:: shared_config.container
+              + container.withArgs([
+                '--config=%s' % std.join('/', [self.config_path, self.config_file]),
+              ])
+              + container.withEnvMixin([
+                envVar.fromFieldPath('POD_NAME', 'metadata.name'),
+              ]),
+
+  local statefulSet = kausal.apps.v1.statefulSet,
+  statefulset:
+    statefulSet.new(this.name, this.replicas, [self.container])
+    + statefulSet.spec.withServiceName(this.name)
+    + statefulSet.mixin.spec.template.spec.withServiceAccountName(self.rbac.service_account.metadata.name)
+    + statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(65534)
+    + statefulSet.mixin.spec.template.spec.securityContext.withRunAsGroup(65534)
+    + statefulSet.configMapVolumeMount(self.config_map, self.config_path)
+    + statefulSet.spec.template.spec.affinity.podAntiAffinity.withRequiredDuringSchedulingIgnoredDuringExecution(
+      kausal.core.v1.podAffinityTerm.withTopologyKey('kubernetes.io/hostname')
+      + kausal.core.v1.podAffinityTerm.labelSelector.withMatchLabels({
+        name: this.name,
+      })
+    ),
+
+  local podDisruptionBudget = kausal.policy.v1.podDisruptionBudget,
+  ksm_pdb:
+    podDisruptionBudget.new(this.name) +
+    podDisruptionBudget.mixin.metadata.withNamespace(this.namespace) +
+    podDisruptionBudget.mixin.metadata.withLabels({ name: '%s-pdb' % this.name }) +
+    podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: this.name }) +
+    podDisruptionBudget.mixin.spec.withMaxUnavailable(1),
+}

kube-state-metrics/singleton.libsonnet

@@ -0,0 +1,21 @@
+local shared_config = import './shared_config.libsonnet';
+local kausal = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet';
+
+{
+  local container = kausal.core.v1.container,
+  local containerPort = kausal.core.v1.containerPort,
+  container+:: container.withArgs([
+    '--port=%s' % shared_config.server_info.port,
+    '--telemetry-host=%s' % shared_config.server_info.telemetry_host,
+    '--telemetry-port=%s' % shared_config.server_info.telemetry_port,
+  ]),
+
+  local deployment = kausal.apps.v1.deployment,
+  deployment:
+    deployment.new('kube-state-metrics', 1, [self.container])
+    + deployment.mixin.spec.template.spec.withServiceAccountName(self.rbac.service_account.metadata.name)
+    + deployment.mixin.spec.template.spec.securityContext.withRunAsUser(65534)
+    + deployment.mixin.spec.template.spec.securityContext.withRunAsGroup(65534),
+
+  rbac:: shared_config.rbac,
+}