fix: add flag to disable kube-state-metrics deployment

Author: Vibhor Dabas

manifests/monitoring/kube-state-metrics.yaml

@@ -0,0 +1,276 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - secrets
+      - nodes
+      - pods
+      - services
+      - resourcequotas
+      - replicationcontrollers
+      - limitranges
+      - persistentvolumeclaims
+      - persistentvolumes
+      - namespaces
+      - endpoints
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - extensions
+    resources:
+      - daemonsets
+      - deployments
+      - replicasets
+      - ingresses
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - daemonsets
+      - deployments
+      - replicasets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+      - jobs
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+    verbs:
+      - create
+  - apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+    verbs:
+      - create
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - certificatesigningrequests
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+    verbs:
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-state-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: monitoring
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: kube-state-metrics
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kube-state-metrics
+  template:
+    metadata:
+      labels:
+        app: kube-state-metrics
+    spec:
+      containers:
+        - args:
+            - --logtostderr
+            - --secure-listen-address=:8443
+            - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+            - --upstream=http://127.0.0.1:8081/
+          image: quay.io/brancz/kube-rbac-proxy:v0.11.0
+          name: kube-rbac-proxy-main
+          ports:
+            - containerPort: 8443
+              name: https-main
+          resources:
+            limits:
+              memory: 40Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - args:
+            - --logtostderr
+            - --secure-listen-address=:9443
+            - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+            - --upstream=http://127.0.0.1:8082/
+          image: quay.io/brancz/kube-rbac-proxy:v0.11.0
+          name: kube-rbac-proxy-self
+          ports:
+            - containerPort: 9443
+              name: https-self
+          resources:
+            limits:
+              memory: 40Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi
+        - args:
+            - --host=127.0.0.1
+            - --port=8081
+            - --telemetry-host=127.0.0.1
+            - --telemetry-port=8082
+            - --metric-annotations-allowlist=pods=[CapacityProvisioned]
+          image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.5.0
+          name: kube-state-metrics
+          resources:
+            limits:
+              cpu: 500m
+              memory: 150Mi
+            requests:
+              cpu: 100m
+              memory: 64Mi
+      nodeSelector:
+        kubernetes.io/os: linux
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: kube-state-metrics
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - extensions
+    resourceNames:
+      - kube-state-metrics
+    resources:
+      - deployments
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - apps
+    resourceNames:
+      - kube-state-metrics
+    resources:
+      - deployments
+    verbs:
+      - get
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kube-state-metrics
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    k8s-app: kube-state-metrics
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  clusterIP: None
+  ports:
+    - name: https-main
+      port: 8443
+      targetPort: https-main
+    - name: https-self
+      port: 9443
+      targetPort: https-self
+  selector:
+    app: kube-state-metrics
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: monitoring
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    k8s-app: kube-state-metrics
+  name: kube-state-metrics
+  namespace: monitoring
+spec:
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      honorLabels: true
+      interval: 30s
+      port: https-main
+      relabelings:
+        - action: labeldrop
+          regex: (pod|service|endpoint|namespace)
+      scheme: https
+      scrapeTimeout: 30s
+      tlsConfig:
+        insecureSkipVerify: true
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      interval: 30s
+      port: https-self
+      scheme: https
+      tlsConfig:
+        insecureSkipVerify: true
+  jobLabel: k8s-app
+  selector:
+    matchLabels:
+      k8s-app: kube-state-metrics

pkg/phases/monitoring/monitoring.go

@@ -35,6 +35,7 @@ var specs = []string{
 	"grafana-operator.yaml",
 	"kube-prometheus.yaml",
 	"prometheus-adapter.yaml",
+	"kube-state-metrics.yaml",
 	"pushgateway.yaml",
 	"unmanaged/alertmanager-rules.yaml.raw",
 	"unmanaged/service-monitors.yaml",
@@ -66,6 +67,15 @@ func Install(p *platform.Platform) error {
 		return nil
 	}
 
+	if p.Monitoring.DisableKubeStateMetrics {
+		for i, v := range specs {
+			if v == "kube-state-metrics.yaml" {
+				specs = append(specs[:i], specs[i+1:]...)
+				break
+			}
+		}
+	}
+
 	if p.Monitoring.Karma.Version == "" {
 		p.Monitoring.Karma.Version = "v0.63"
 	}

pkg/types/types.go

@@ -506,19 +506,21 @@ func (dns DynamicDNS) IsEnabled() bool {
 }
 
 type Monitoring struct {
-	Disabled      Boolean       `yaml:"disabled,omitempty" json:"disabled,omitempty"`
-	AlertEmail    string        `yaml:"alert_email,omitempty" json:"alert_email,omitempty"`
-	Version       string        `yaml:"version,omitempty" json:"version,omitempty"`
-	Prometheus    Prometheus    `yaml:"prometheus,omitempty" json:"prometheus,omitempty"`
-	Karma         Karma         `yaml:"karma,omitempty" json:"karma,omitempty"`
-	Grafana       Grafana       `yaml:"grafana,omitempty" json:"grafana,omitempty"`
-	AlertManager  AlertManager  `yaml:"alertmanager,omitempty" json:"alertManager,omitempty"`
-	KubeRbacProxy string        `yaml:"kubeRbacProxy,omitempty" json:"kubeRbacProxy,omitempty"`
-	NodeExporter  string        `yaml:"nodeExporter,omitempty" json:"nodeExporter,omitempty"`
-	AddonResizer  string        `yaml:"addonResizer,omitempty" json:"addonResizer,omitempty"`
-	ExcludeAlerts []string      `yaml:"excludeAlerts,omitempty" json:"excludeAlerts,omitempty"`
-	PushGateway   PushGateway   `yaml:"pushGateway,omitempty" json:"pushGateway,omitempty"`
-	E2E           MonitoringE2E `yaml:"e2e,omitempty" json:"e2e,omitempty"`
+	Disabled                Boolean       `yaml:"disabled,omitempty" json:"disabled,omitempty"`
+	AlertEmail              string        `yaml:"alert_email,omitempty" json:"alert_email,omitempty"`
+	Version                 string        `yaml:"version,omitempty" json:"version,omitempty"`
+	Prometheus              Prometheus    `yaml:"prometheus,omitempty" json:"prometheus,omitempty"`
+	Karma                   Karma         `yaml:"karma,omitempty" json:"karma,omitempty"`
+	Grafana                 Grafana       `yaml:"grafana,omitempty" json:"grafana,omitempty"`
+	AlertManager            AlertManager  `yaml:"alertmanager,omitempty" json:"alertManager,omitempty"`
+	KubeStateMetrics        string        `yaml:"kubeStateMetrics,omitempty" json:"kubeStateMetrics,omitempty"`
+	DisableKubeStateMetrics Boolean       `yaml:"disableKubeStateMetrics,omitempty" json:"disableKubeStateMetrics,omitempty"`
+	KubeRbacProxy           string        `yaml:"kubeRbacProxy,omitempty" json:"kubeRbacProxy,omitempty"`
+	NodeExporter            string        `yaml:"nodeExporter,omitempty" json:"nodeExporter,omitempty"`
+	AddonResizer            string        `yaml:"addonResizer,omitempty" json:"addonResizer,omitempty"`
+	ExcludeAlerts           []string      `yaml:"excludeAlerts,omitempty" json:"excludeAlerts,omitempty"`
+	PushGateway             PushGateway   `yaml:"pushGateway,omitempty" json:"pushGateway,omitempty"`
+	E2E                     MonitoringE2E `yaml:"e2e,omitempty" json:"e2e,omitempty"`
 }
 
 func (m Monitoring) IsDisabled() bool {