From ccb83152a0114b68421772e2ceb9045af886d6de Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Thu, 20 Jan 2022 16:11:41 -0500
Subject: [PATCH 1/8] update dashboards

---
 cortex-mixin/config.libsonnet                 | 13 ++++++--
 .../alertmanager-resources.libsonnet          |  5 +--
 .../dashboards/compactor-resources.libsonnet  |  2 +-
 cortex-mixin/dashboards/queries.libsonnet     |  3 +-
 .../dashboards/reads-resources.libsonnet      | 14 ++++++---
 cortex-mixin/dashboards/reads.libsonnet       | 31 ++++++++++++-------
 .../dashboards/rollout-progress.libsonnet     |  6 ++--
 .../dashboards/writes-resources.libsonnet     |  5 +--
 cortex-mixin/dashboards/writes.libsonnet      |  5 +--
 9 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet
index 06941b6d..fa8b005e 100644
--- a/cortex-mixin/config.libsonnet
+++ b/cortex-mixin/config.libsonnet
@@ -7,6 +7,11 @@
     // May contain 'chunks', 'blocks' or both.
     // Enables chunks- or blocks- specific panels and dashboards.
     storage_engine: ['blocks'],
+    
+    // Disable unused panels depending whether a component was installed or not
+    cortex_gw_enabled: false,
+    query_scheduler_enabled: false,
+    ruler_enabled: false,
 
     // For chunks backend, switch for chunk index type.
     // May contain 'bigtable', 'dynamodb' or 'cassandra'.
@@ -47,15 +52,17 @@
     cortex_p99_latency_threshold_seconds: 2.5,
 
     // Whether resources dashboards are enabled (based on cAdvisor metrics).
-    resources_dashboards_enabled: false,
+    resources_dashboards_enabled: true,
 
     // The label used to differentiate between different application instances (i.e. 'pod' in a kubernetes install).
     per_instance_label: 'pod',
 
     // Name selectors for different application instances, using the "per_instance_label".
     instance_names: {
-      compactor: 'compactor.*',
-      alertmanager: 'alertmanager.*',
+      alertmanager: '.*alertmanager.*',
+      compactor: '.*compactor.*',
+      ingester: '.*ingester.*',
+      store_gateway: '.*store-gateway.*',
     },
 
     // The label used to differentiate between different nodes (i.e. servers).
diff --git a/cortex-mixin/dashboards/alertmanager-resources.libsonnet b/cortex-mixin/dashboards/alertmanager-resources.libsonnet
index 41506020..63d6cca9 100644
--- a/cortex-mixin/dashboards/alertmanager-resources.libsonnet
+++ b/cortex-mixin/dashboards/alertmanager-resources.libsonnet
@@ -4,7 +4,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
   'alertmanager-resources.json':
     ($.dashboard('Cortex / Alertmanager Resources') + { uid: '68b66aed90ccab448009089544a8d6c6' })
     .addClusterSelectorTemplates(false)
-    .addRow(
+    .addRowIf(
+      $._config.cortex_gw_enabled,
       $.row('Gateway')
       .addPanel(
         $.containerCPUUsagePanel('CPU', $._config.job_names.gateway),
@@ -61,7 +62,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     .addRow(
       $.row('')
       .addPanel(
-        $.containerDiskSpaceUtilization('Disk Space Utilization', 'alertmanager'),
+        $.containerDiskSpaceUtilization('Disk Space Utilization', $._config.instance_names.alertmanager),
       )
     ),
 }
diff --git a/cortex-mixin/dashboards/compactor-resources.libsonnet b/cortex-mixin/dashboards/compactor-resources.libsonnet
index 82a6bce4..3e628e9a 100644
--- a/cortex-mixin/dashboards/compactor-resources.libsonnet
+++ b/cortex-mixin/dashboards/compactor-resources.libsonnet
@@ -34,7 +34,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.containerDiskReadsPanel('Disk Reads', 'compactor'),
       )
       .addPanel(
-        $.containerDiskSpaceUtilization('Disk Space Utilization', 'compactor'),
+        $.containerDiskSpaceUtilization('Disk Space Utilization', $._config.instance_names.compactor),
       )
     ) + {
       templating+: {
diff --git a/cortex-mixin/dashboards/queries.libsonnet b/cortex-mixin/dashboards/queries.libsonnet
index 259f5dfa..88fa99df 100644
--- a/cortex-mixin/dashboards/queries.libsonnet
+++ b/cortex-mixin/dashboards/queries.libsonnet
@@ -21,7 +21,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.queryPanel('cortex_query_frontend_queue_length{%s}' % $.jobMatcher($._config.job_names.query_frontend), '{{cluster}} / {{namespace}} / {{%s}}' % $._config.per_instance_label),
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.query_scheduler_enabled,
       $.row('Query Scheduler')
       .addPanel(
         $.panel('Queue Duration') +
diff --git a/cortex-mixin/dashboards/reads-resources.libsonnet b/cortex-mixin/dashboards/reads-resources.libsonnet
index f0750c88..08129ecc 100644
--- a/cortex-mixin/dashboards/reads-resources.libsonnet
+++ b/cortex-mixin/dashboards/reads-resources.libsonnet
@@ -4,7 +4,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
   'cortex-reads-resources.json':
     ($.dashboard('Cortex / Reads Resources') + { uid: '2fd2cda9eea8d8af9fbc0a5960425120' })
     .addClusterSelectorTemplates(false)
-    .addRow(
+    .addRowIf(
+      $._config.cortex_gw_enabled,
       $.row('Gateway')
       .addPanel(
         $.containerCPUUsagePanel('CPU', $._config.job_names.gateway),
@@ -28,7 +29,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.query_frontend),
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.query_scheduler_enabled,
       $.row('Query Scheduler')
       .addPanel(
         $.containerCPUUsagePanel('CPU', 'query-scheduler'),
@@ -64,7 +66,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.ingester),
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.ruler_enabled,
       $.row('Ruler')
       .addPanel(
         $.panel('Rules') +
@@ -77,7 +80,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.containerCPUUsagePanel('CPU', 'ruler'),
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.ruler_enabled,
       $.row('')
       .addPanel(
         $.containerMemoryWorkingSetPanel('Memory (workingset)', 'ruler'),
@@ -109,7 +113,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.containerDiskReadsPanel('Disk Reads', 'store-gateway'),
       )
       .addPanel(
-        $.containerDiskSpaceUtilization('Disk Space Utilization', 'store-gateway'),
+        $.containerDiskSpaceUtilization('Disk Space Utilization', $._config.instance_names.store_gateway),
       )
     ) + {
       templating+: {
diff --git a/cortex-mixin/dashboards/reads.libsonnet b/cortex-mixin/dashboards/reads.libsonnet
index 9bc9b7d6..d72f9c38 100644
--- a/cortex-mixin/dashboards/reads.libsonnet
+++ b/cortex-mixin/dashboards/reads.libsonnet
@@ -1,6 +1,12 @@
 local utils = import 'mixin-utils/utils.libsonnet';
 
 (import 'dashboard-utils.libsonnet') {
+  local config = {
+    gateway_routes_regex: '(prometheus|api_prom)_api_v1_.+',
+    query_routes_regex: '(prometheus|api_prom)_api_v1_query',
+    query_range_routes_regex: '(prometheus|api_prom)_api_v1_query_range',
+  },
+  
   'cortex-reads.json':
     ($.dashboard('Cortex / Reads') + { uid: '8d6ba60eccc4b6eedfa329b24b1bd339' })
     .addClusterSelectorTemplates()
@@ -42,7 +48,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
             rate(
               cortex_request_duration_seconds_count{
                 %(queryFrontend)s,
-                route=~"(prometheus|api_prom)_api_v1_query"
+                route=~"%(query_routes_regex)s"
               }[$__rate_interval]
             )
           ) +
@@ -73,7 +79,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
             rate(
               cortex_request_duration_seconds_count{
                 %(queryFrontend)s,
-                route=~"(prometheus|api_prom)_api_v1_query_range"
+                route=~"%(query_range_routes_regex)s"
               }[$__rate_interval]
             )
           )
@@ -89,20 +95,21 @@ local utils = import 'mixin-utils/utils.libsonnet';
         ),
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.cortex_gw_enabled,
       $.row('Gateway')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"(prometheus|api_prom)_api_v1_.+"}' % $.jobMatcher($._config.job_names.gateway))
+        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"%(gateway_read_routes_regex)s"}' % $.jobMatcher($._config.job_names.gateway))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.gateway) + [utils.selector.re('route', '(prometheus|api_prom)_api_v1_.+')])
+        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.gateway) + [utils.selector.re('route', '%(gateway_read_routes_regex)s')])
       )
       .addPanel(
         $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
         $.hiddenLegendQueryPanel(
-          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"(prometheus|api_prom)_api_v1_.+"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.gateway)], ''
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.gateway)], ''
         ) +
         { yaxes: $.yaxes('s') }
       )
@@ -111,16 +118,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Query Frontend')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"(prometheus|api_prom)_api_v1_.+"}' % $.jobMatcher($._config.job_names.query_frontend))
+        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"%(gateway_read_routes_regex)s"}' % $.jobMatcher($._config.job_names.query_frontend))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.query_frontend) + [utils.selector.re('route', '(prometheus|api_prom)_api_v1_.+')])
+        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.query_frontend) + [utils.selector.re('route', '%(gateway_read_routes_regex)s')])
       )
       .addPanel(
         $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
         $.hiddenLegendQueryPanel(
-          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"(prometheus|api_prom)_api_v1_.+"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.query_frontend)], ''
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.query_frontend)], ''
         ) +
         { yaxes: $.yaxes('s') }
       )
@@ -165,16 +172,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Querier')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_querier_request_duration_seconds_count{%s, route=~"(prometheus|api_prom)_api_v1_.+"}' % $.jobMatcher($._config.job_names.querier))
+        $.qpsPanel('cortex_querier_request_duration_seconds_count{%s, route=~"%(gateway_read_routes_regex)s"}' % $.jobMatcher($._config.job_names.querier))
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_querier_request_duration_seconds', $.jobSelector($._config.job_names.querier) + [utils.selector.re('route', '(prometheus|api_prom)_api_v1_.+')])
+        utils.latencyRecordingRulePanel('cortex_querier_request_duration_seconds', $.jobSelector($._config.job_names.querier) + [utils.selector.re('route', '%(gateway_read_routes_regex)s')])
       )
       .addPanel(
         $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
         $.hiddenLegendQueryPanel(
-          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_querier_request_duration_seconds_bucket{%s, route=~"(prometheus|api_prom)_api_v1_.+"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.querier)], ''
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_querier_request_duration_seconds_bucket{%s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.querier)], ''
         ) +
         { yaxes: $.yaxes('s') }
       )
diff --git a/cortex-mixin/dashboards/rollout-progress.libsonnet b/cortex-mixin/dashboards/rollout-progress.libsonnet
index 16c54095..45465328 100644
--- a/cortex-mixin/dashboards/rollout-progress.libsonnet
+++ b/cortex-mixin/dashboards/rollout-progress.libsonnet
@@ -4,9 +4,11 @@ local utils = import 'mixin-utils/utils.libsonnet';
   local config = {
     namespace_matcher: $.namespaceMatcher(),
     gateway_job_matcher: $.jobMatcher($._config.job_names.gateway),
+    writes_job_matcher: $.jobMatcher($._config.job_names.distributor),
+    reads_job_matcher: $.jobMatcher($._config.job_names.querier),
     gateway_write_routes_regex: 'api_(v1|prom)_push',
     gateway_read_routes_regex: '(prometheus|api_prom)_api_v1_.+',
-    all_services_regex: std.join('|', ['cortex-gw', 'distributor', 'ingester.*', 'query-frontend.*', 'query-scheduler.*', 'querier.*', 'compactor', 'store-gateway', 'ruler', 'alertmanager']),
+    all_services_regex: std.join('|', ['cortex-gw', '.*distributor.*', '.*ingester.*', '.*query-frontend.*', '.*query-scheduler.*', '.*querier.*', '.*compactor.*', '.*store-gateway.*', '.*ruler.*', '.*alertmanager.*', '.*memcached.*']),
   },
 
   'cortex-rollout-progress.json':
@@ -235,7 +237,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
                 count by(container, version) (
                   label_replace(
                     kube_pod_container_info{%(namespace_matcher)s,container=~"%(all_services_regex)s"},
-                    "version", "$1", "image", ".*:(.+)-.*"
+                    "version", "$1", "image", ".*:(.+)"
                   )
                 )
               ||| % config,
diff --git a/cortex-mixin/dashboards/writes-resources.libsonnet b/cortex-mixin/dashboards/writes-resources.libsonnet
index 64f83ef1..408dd335 100644
--- a/cortex-mixin/dashboards/writes-resources.libsonnet
+++ b/cortex-mixin/dashboards/writes-resources.libsonnet
@@ -4,7 +4,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
   'cortex-writes-resources.json':
     ($.dashboard('Cortex / Writes Resources') + { uid: 'c0464f0d8bd026f776c9006b0591bb0b' })
     .addClusterSelectorTemplates(false)
-    .addRow(
+    .addRowIf(
+      $._config.cortex_gw_enabled,
       $.row('Gateway')
       .addPanel(
         $.containerCPUUsagePanel('CPU', $._config.job_names.gateway),
@@ -62,7 +63,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.containerDiskReadsPanel('Disk Reads', 'ingester')
       )
       .addPanel(
-        $.containerDiskSpaceUtilization('Disk Space Utilization', 'ingester'),
+        $.containerDiskSpaceUtilization('Disk Space Utilization', $._config.instance_names.ingester),
       )
     )
     + {
diff --git a/cortex-mixin/dashboards/writes.libsonnet b/cortex-mixin/dashboards/writes.libsonnet
index e99faee4..c4a18fd7 100644
--- a/cortex-mixin/dashboards/writes.libsonnet
+++ b/cortex-mixin/dashboards/writes.libsonnet
@@ -61,10 +61,11 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Requests / sec') +
-        $.statPanel('sum(rate(cortex_request_duration_seconds_count{%s, route=~"api_(v1|prom)_push"}[5m]))' % $.jobMatcher($._config.job_names.gateway), format='reqps')
+        $.statPanel('sum(rate(cortex_request_duration_seconds_count{%s, route=~"api_(v1|prom)_push"}[5m]))' % $.jobMatcher($._config.job_names.distributor), format='reqps')
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.cortex_gw_enabled,
       $.row('Gateway')
       .addPanel(
         $.panel('Requests / sec') +

From 65a588d769ee9020cdb7a87f64e7adad37f84b5b Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Thu, 20 Jan 2022 16:46:49 -0500
Subject: [PATCH 2/8] update rollout dashboard

---
 .../dashboards/rollout-progress.libsonnet     | 62 +++++++++----------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/cortex-mixin/dashboards/rollout-progress.libsonnet b/cortex-mixin/dashboards/rollout-progress.libsonnet
index 45465328..be757cbc 100644
--- a/cortex-mixin/dashboards/rollout-progress.libsonnet
+++ b/cortex-mixin/dashboards/rollout-progress.libsonnet
@@ -85,7 +85,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           { color: 'green', value: 1 },
         ], unit='percentunit', min=0, max=1) + {
           id: 1,
-          gridPos: { h: 8, w: 10, x: 0, y: 0 },
+          gridPos: { h: 10, w: 10, x: 0, y: 0 },
         },
 
         //
@@ -93,50 +93,50 @@ local utils = import 'mixin-utils/utils.libsonnet';
         //
         $.panel('Writes - 2xx') +
         $.newStatPanel(|||
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s",status_code=~"2.+"}[$__rate_interval])) /
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}[$__rate_interval]))
+          sum(rate(cortex_request_duration_seconds_count{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s",status_code=~"2.+"}[$__rate_interval])) /
+          sum(rate(cortex_request_duration_seconds_count{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}[$__rate_interval])) or on() vector(0)
         ||| % config, thresholds=[
           { color: 'green', value: null },
         ]) + {
           id: 2,
-          gridPos: { h: 4, w: 2, x: 10, y: 0 },
+          gridPos: { h: 5, w: 2, x: 10, y: 10 },
         },
 
         $.panel('Writes - 4xx') +
         $.newStatPanel(|||
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s",status_code=~"4.+"}[$__rate_interval])) /
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}[$__rate_interval]))
+          sum(rate(cortex_request_duration_seconds_count{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s",status_code=~"4.+"}[$__rate_interval])) /
+          sum(rate(cortex_request_duration_seconds_count{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}[$__rate_interval])) or on() vector(0)
         ||| % config, thresholds=[
           { color: 'green', value: null },
           { color: 'orange', value: 0.2 },
           { color: 'red', value: 0.5 },
         ]) + {
           id: 3,
-          gridPos: { h: 4, w: 2, x: 12, y: 0 },
+          gridPos: { h: 5, w: 2, x: 12, y: 10 },
         },
 
         $.panel('Writes - 5xx') +
         $.newStatPanel(|||
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s",status_code=~"5.+"}[$__rate_interval])) /
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}[$__rate_interval]))
+          sum(rate(cortex_request_duration_seconds_count{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s",status_code=~"5.+"}[$__rate_interval])) /
+          sum(rate(cortex_request_duration_seconds_count{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}[$__rate_interval])) or on() vector(0)
         ||| % config, thresholds=[
           { color: 'green', value: null },
           { color: 'red', value: 0.01 },
         ]) + {
           id: 4,
-          gridPos: { h: 4, w: 2, x: 14, y: 0 },
+          gridPos: { h: 5, w: 2, x: 14, y: 10 },
         },
 
         $.panel('Writes 99th Latency') +
         $.newStatPanel(|||
-          histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}))
+          histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}))
         ||| % config, unit='s', thresholds=[
           { color: 'green', value: null },
           { color: 'orange', value: 0.2 },
           { color: 'red', value: 0.5 },
         ]) + {
           id: 5,
-          gridPos: { h: 4, w: 8, x: 16, y: 0 },
+          gridPos: { h: 5, w: 8, x: 16, y: 10 },
         },
 
         //
@@ -144,50 +144,50 @@ local utils = import 'mixin-utils/utils.libsonnet';
         //
         $.panel('Reads - 2xx') +
         $.newStatPanel(|||
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s",status_code=~"2.+"}[$__rate_interval])) /
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval]))
+          sum(rate(cortex_request_duration_seconds_count{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s",status_code=~"2.+"}[$__rate_interval])) /
+          sum(rate(cortex_request_duration_seconds_count{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])) or on() vector(0)
         ||| % config, thresholds=[
           { color: 'green', value: null },
         ]) + {
           id: 6,
-          gridPos: { h: 4, w: 2, x: 10, y: 4 },
+          gridPos: { h: 5, w: 2, x: 10, y: 15 },
         },
 
         $.panel('Reads - 4xx') +
         $.newStatPanel(|||
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s",status_code=~"4.+"}[$__rate_interval])) /
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval]))
+          sum(rate(cortex_request_duration_seconds_count{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s",status_code=~"4.+"}[$__rate_interval])) /
+          sum(rate(cortex_request_duration_seconds_count{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])) or on() vector(0)
         ||| % config, thresholds=[
           { color: 'green', value: null },
           { color: 'orange', value: 0.01 },
           { color: 'red', value: 0.05 },
         ]) + {
           id: 7,
-          gridPos: { h: 4, w: 2, x: 12, y: 4 },
+          gridPos: { h: 5, w: 2, x: 12, y: 15 },
         },
 
         $.panel('Reads - 5xx') +
         $.newStatPanel(|||
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s",status_code=~"5.+"}[$__rate_interval])) /
-          sum(rate(cortex_request_duration_seconds_count{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval]))
+          sum(rate(cortex_request_duration_seconds_count{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s",status_code=~"5.+"}[$__rate_interval])) /
+          sum(rate(cortex_request_duration_seconds_count{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])) or on() vector(0)
         ||| % config, thresholds=[
           { color: 'green', value: null },
           { color: 'red', value: 0.01 },
         ]) + {
           id: 8,
-          gridPos: { h: 4, w: 2, x: 14, y: 4 },
+          gridPos: { h: 5, w: 2, x: 14, y: 15 },
         },
 
         $.panel('Reads 99th Latency') +
         $.newStatPanel(|||
-          histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}))
+          histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}))
         ||| % config, unit='s', thresholds=[
           { color: 'green', value: null },
           { color: 'orange', value: 1 },
           { color: 'red', value: 2.5 },
         ]) + {
           id: 9,
-          gridPos: { h: 4, w: 8, x: 16, y: 4 },
+          gridPos: { h: 5, w: 8, x: 16, y: 15 },
         },
 
         //
@@ -220,7 +220,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
             },
           },
           id: 10,
-          gridPos: { h: 8, w: 10, x: 0, y: 8 },
+          gridPos: { h: 10, w: 6, x: 10, y: 0 },
         },
 
         //
@@ -266,7 +266,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
             {
               // Hide time.
               id: 'organize',
-              options: { excludeByName: { Time: true } },
+              options: { excludeByName: { Time: true }, indexByName: { Time: 0, container: 1 } },
             },
             {
               // Sort by container.
@@ -276,7 +276,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           ],
 
           id: 11,
-          gridPos: { h: 8, w: 6, x: 10, y: 8 },
+          gridPos: { h: 10, w: 10, x: 0, y: 10 },
         },
 
         //
@@ -285,15 +285,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.panel('Latency vs 24h ago') +
         $.queryPanel([|||
           1 - (
-            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s"} offset 24h))[1h:])
+            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s"} offset 24h))[1h:])
             /
-            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(gateway_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}))[1h:])
+            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(writes_job_matcher)s, route=~"%(gateway_write_routes_regex)s"}))[1h:])
           )
         ||| % config, |||
           1 - (
-            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s"} offset 24h))[1h:])
+            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s"} offset 24h))[1h:])
             /
-            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(gateway_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}))[1h:])
+            avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{%(reads_job_matcher)s, route=~"%(gateway_read_routes_regex)s"}))[1h:])
           )
         ||| % config], ['writes', 'reads']) + {
           yaxes: $.yaxes({
@@ -302,7 +302,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           }),
 
           id: 12,
-          gridPos: { h: 8, w: 8, x: 16, y: 8 },
+          gridPos: { h: 10, w: 8, x: 16, y: 0 },
         },
       ],
 

From 23686bf7ea057d5028eb5e5418b194709f9723f6 Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Thu, 20 Jan 2022 18:42:03 -0500
Subject: [PATCH 3/8] parametrize prometheus http prefix

---
 cortex-mixin/config.libsonnet                 |  3 +++
 cortex-mixin/dashboards/reads.libsonnet       | 27 ++++++++++---------
 .../dashboards/rollout-progress.libsonnet     |  2 +-
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet
index fa8b005e..d90b8c67 100644
--- a/cortex-mixin/config.libsonnet
+++ b/cortex-mixin/config.libsonnet
@@ -8,6 +8,9 @@
     // Enables chunks- or blocks- specific panels and dashboards.
     storage_engine: ['blocks'],
     
+    // HTTP URL prefix under which the Prometheus api is available.
+    prometheus_http_prefix: 'cortex',
+    
     // Disable unused panels depending whether a component was installed or not
     cortex_gw_enabled: false,
     query_scheduler_enabled: false,
diff --git a/cortex-mixin/dashboards/reads.libsonnet b/cortex-mixin/dashboards/reads.libsonnet
index d72f9c38..02848621 100644
--- a/cortex-mixin/dashboards/reads.libsonnet
+++ b/cortex-mixin/dashboards/reads.libsonnet
@@ -2,9 +2,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
 
 (import 'dashboard-utils.libsonnet') {
   local config = {
-    gateway_routes_regex: '(prometheus|api_prom)_api_v1_.+',
-    query_routes_regex: '(prometheus|api_prom)_api_v1_query',
-    query_range_routes_regex: '(prometheus|api_prom)_api_v1_query_range',
+    gateway_read_routes_regex: '(%s|api_prom)_api_v1_.+' % $._config.prometheus_http_prefix,
   },
   
   'cortex-reads.json':
@@ -61,6 +59,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           )
         ||| % {
           queryFrontend: $.jobMatcher($._config.job_names.query_frontend),
+          query_routes_regex: '(%s|api_prom)_api_v1_query' % $._config.prometheus_http_prefix,
           ruler: $.jobMatcher($._config.job_names.ruler),
         }, format='reqps') +
         $.panelDescription(
@@ -85,6 +84,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
           )
         ||| % {
           queryFrontend: $.jobMatcher($._config.job_names.query_frontend),
+          query_range_routes_regex: '(%s|api_prom)_api_v1_query_range' % $._config.prometheus_http_prefix,
         }, format='reqps') +
         $.panelDescription(
           'Range queries per second',
@@ -100,16 +100,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Gateway')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"%(gateway_read_routes_regex)s"}' % $.jobMatcher($._config.job_names.gateway))
+        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"%s"}' % [$.jobMatcher($._config.job_names.gateway), config.gateway_read_routes_regex])
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.gateway) + [utils.selector.re('route', '%(gateway_read_routes_regex)s')])
+        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.gateway) + [utils.selector.re('route', config.gateway_read_routes_regex)])
       )
       .addPanel(
         $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
         $.hiddenLegendQueryPanel(
-          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.gateway)], ''
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"%s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.gateway), config.gateway_read_routes_regex], ''
         ) +
         { yaxes: $.yaxes('s') }
       )
@@ -118,21 +118,22 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Query Frontend')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"%(gateway_read_routes_regex)s"}' % $.jobMatcher($._config.job_names.query_frontend))
+        $.qpsPanel('cortex_request_duration_seconds_count{%s, route=~"%s"}' % [$.jobMatcher($._config.job_names.query_frontend), config.gateway_read_routes_regex])
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.query_frontend) + [utils.selector.re('route', '%(gateway_read_routes_regex)s')])
+        utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.query_frontend) + [utils.selector.re('route', config.gateway_read_routes_regex)])
       )
       .addPanel(
         $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
         $.hiddenLegendQueryPanel(
-          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.query_frontend)], ''
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route=~"%s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.query_frontend), config.gateway_read_routes_regex], ''
         ) +
         { yaxes: $.yaxes('s') }
       )
     )
-    .addRow(
+    .addRowIf(
+      $._config.query_scheduler_enabled,
       $.row('Query Scheduler')
       .addPanel(
         $.textPanel(
@@ -172,16 +173,16 @@ local utils = import 'mixin-utils/utils.libsonnet';
       $.row('Querier')
       .addPanel(
         $.panel('Requests / sec') +
-        $.qpsPanel('cortex_querier_request_duration_seconds_count{%s, route=~"%(gateway_read_routes_regex)s"}' % $.jobMatcher($._config.job_names.querier))
+        $.qpsPanel('cortex_querier_request_duration_seconds_count{%s, route=~"%s"}' % [$.jobMatcher($._config.job_names.querier), config.gateway_read_routes_regex])
       )
       .addPanel(
         $.panel('Latency') +
-        utils.latencyRecordingRulePanel('cortex_querier_request_duration_seconds', $.jobSelector($._config.job_names.querier) + [utils.selector.re('route', '%(gateway_read_routes_regex)s')])
+        utils.latencyRecordingRulePanel('cortex_querier_request_duration_seconds', $.jobSelector($._config.job_names.querier) + [utils.selector.re('route', config.gateway_read_routes_regex)])
       )
       .addPanel(
         $.panel('Per %s p99 Latency' % $._config.per_instance_label) +
         $.hiddenLegendQueryPanel(
-          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_querier_request_duration_seconds_bucket{%s, route=~"%(gateway_read_routes_regex)s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.querier)], ''
+          'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_querier_request_duration_seconds_bucket{%s, route=~"%s"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.querier), config.gateway_read_routes_regex], ''
         ) +
         { yaxes: $.yaxes('s') }
       )
diff --git a/cortex-mixin/dashboards/rollout-progress.libsonnet b/cortex-mixin/dashboards/rollout-progress.libsonnet
index be757cbc..743c7962 100644
--- a/cortex-mixin/dashboards/rollout-progress.libsonnet
+++ b/cortex-mixin/dashboards/rollout-progress.libsonnet
@@ -7,7 +7,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     writes_job_matcher: $.jobMatcher($._config.job_names.distributor),
     reads_job_matcher: $.jobMatcher($._config.job_names.querier),
     gateway_write_routes_regex: 'api_(v1|prom)_push',
-    gateway_read_routes_regex: '(prometheus|api_prom)_api_v1_.+',
+    gateway_read_routes_regex: '(%s|api_prom)_api_v1_.+' % $._config.prometheus_http_prefix,
     all_services_regex: std.join('|', ['cortex-gw', '.*distributor.*', '.*ingester.*', '.*query-frontend.*', '.*query-scheduler.*', '.*querier.*', '.*compactor.*', '.*store-gateway.*', '.*ruler.*', '.*alertmanager.*', '.*memcached.*']),
   },
 

From 582754db36ba4d839974776839d4cba0bfc5f154 Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Thu, 20 Jan 2022 20:19:34 -0500
Subject: [PATCH 4/8] fix counter when ruler is not installed

---
 cortex-mixin/dashboards/reads.libsonnet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cortex-mixin/dashboards/reads.libsonnet b/cortex-mixin/dashboards/reads.libsonnet
index 02848621..d0cfd513 100644
--- a/cortex-mixin/dashboards/reads.libsonnet
+++ b/cortex-mixin/dashboards/reads.libsonnet
@@ -55,7 +55,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
               cortex_prometheus_rule_evaluations_total{
                 %(ruler)s
               }[$__rate_interval]
-            )
+            ) or on() vector(0)
           )
         ||| % {
           queryFrontend: $.jobMatcher($._config.job_names.query_frontend),

From 7d84d70a4e8814fa9913e5713585ba71bf84d5d4 Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Thu, 20 Jan 2022 22:47:09 -0500
Subject: [PATCH 5/8] lint fixes, remove cluster label from queries where it's
 missing

---
 cortex-mixin/config.libsonnet                     |  4 ++--
 cortex-mixin/dashboards/dashboard-utils.libsonnet |  5 ++++-
 cortex-mixin/dashboards/reads.libsonnet           |  2 +-
 cortex-mixin/dashboards/scaling.libsonnet         | 11 ++++++-----
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet
index d90b8c67..d4fa0f2a 100644
--- a/cortex-mixin/config.libsonnet
+++ b/cortex-mixin/config.libsonnet
@@ -7,10 +7,10 @@
     // May contain 'chunks', 'blocks' or both.
     // Enables chunks- or blocks- specific panels and dashboards.
     storage_engine: ['blocks'],
-    
+
     // HTTP URL prefix under which the Prometheus api is available.
     prometheus_http_prefix: 'cortex',
-    
+
     // Disable unused panels depending whether a component was installed or not
     cortex_gw_enabled: false,
     query_scheduler_enabled: false,
diff --git a/cortex-mixin/dashboards/dashboard-utils.libsonnet b/cortex-mixin/dashboards/dashboard-utils.libsonnet
index e4268192..be47b67a 100644
--- a/cortex-mixin/dashboards/dashboard-utils.libsonnet
+++ b/cortex-mixin/dashboards/dashboard-utils.libsonnet
@@ -9,6 +9,9 @@ local utils = import 'mixin-utils/utils.libsonnet';
   // - some links that propagate the selectred cluster.
   dashboard(title)::
     super.dashboard(title=title, datasource=$._config.dashboard_datasource) + {
+      refresh: '30s',
+      timezone: 'browser',
+
       addRowIf(condition, row)::
         if condition
         then self.addRow(row)
@@ -73,7 +76,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
   namespaceMatcher()::
     if $._config.singleBinary
     then 'job=~"$job"'
-    else 'cluster=~"$cluster", namespace=~"$namespace"',
+    else 'namespace=~"$namespace"',
 
   jobSelector(job)::
     if $._config.singleBinary
diff --git a/cortex-mixin/dashboards/reads.libsonnet b/cortex-mixin/dashboards/reads.libsonnet
index d0cfd513..4aa0bf91 100644
--- a/cortex-mixin/dashboards/reads.libsonnet
+++ b/cortex-mixin/dashboards/reads.libsonnet
@@ -4,7 +4,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
   local config = {
     gateway_read_routes_regex: '(%s|api_prom)_api_v1_.+' % $._config.prometheus_http_prefix,
   },
-  
+
   'cortex-reads.json':
     ($.dashboard('Cortex / Reads') + { uid: '8d6ba60eccc4b6eedfa329b24b1bd339' })
     .addClusterSelectorTemplates()
diff --git a/cortex-mixin/dashboards/scaling.libsonnet b/cortex-mixin/dashboards/scaling.libsonnet
index a01a7db3..895dd566 100644
--- a/cortex-mixin/dashboards/scaling.libsonnet
+++ b/cortex-mixin/dashboards/scaling.libsonnet
@@ -1,7 +1,6 @@
 local utils = import 'mixin-utils/utils.libsonnet';
 
 (import 'dashboard-utils.libsonnet') {
-
   'cortex-scaling.json':
     ($.dashboard('Cortex / Scaling') + { uid: '88c041017b96856c9176e07cf557bdcf' })
     .addClusterSelectorTemplates()
@@ -42,18 +41,20 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.tablePanel([
           |||
             sort_desc(
-              cluster_namespace_deployment_reason:required_replicas:count{cluster=~"$cluster", namespace=~"$namespace"}
+              cluster_namespace_deployment_reason:required_replicas:count{%(namespace_matcher)s}
                 > ignoring(reason) group_left
-              cluster_namespace_deployment:actual_replicas:count{cluster=~"$cluster", namespace=~"$namespace"}
+              cluster_namespace_deployment:actual_replicas:count{%(namespace_matcher)s}
             )
-          |||,
+          ||| % {
+            namespace_matcher: $.namespaceMatcher(),
+          },
         ], {
           __name__: { alias: 'Cluster', type: 'hidden' },
           cluster: { alias: 'Cluster' },
           namespace: { alias: 'Namespace' },
           deployment: { alias: 'Service' },
           reason: { alias: 'Reason' },
-          Value: { alias: 'Required Replicas', decimals: 0 },
+          'Value #A': { alias: 'Required Replicas', decimals: 0 },
         })
       )
     ),

From d87573302cb8432f22c7562f22dd1b78ced5dd6c Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Fri, 21 Jan 2022 17:17:40 -0500
Subject: [PATCH 6/8] fix disk dashboards

---
 .../dashboards/dashboard-utils.libsonnet      | 47 +++++--------------
 1 file changed, 13 insertions(+), 34 deletions(-)

diff --git a/cortex-mixin/dashboards/dashboard-utils.libsonnet b/cortex-mixin/dashboards/dashboard-utils.libsonnet
index be47b67a..a46d2f41 100644
--- a/cortex-mixin/dashboards/dashboard-utils.libsonnet
+++ b/cortex-mixin/dashboards/dashboard-utils.libsonnet
@@ -207,15 +207,13 @@ local utils = import 'mixin-utils/utils.libsonnet';
       |||
         sum by(%s, %s, device) (
           rate(
-            node_disk_written_bytes_total[$__rate_interval]
+            %s[$__rate_interval]
           )
         )
-        +
-        %s
       ||| % [
         $._config.per_node_label,
         $._config.per_instance_label,
-        $.filterNodeDiskContainer(containerName),
+        $.nodeDiskContainerBytesTotal(containerName, 'writes'),
       ],
       '{{%s}} - {{device}}' % $._config.per_instance_label
     ) +
@@ -228,13 +226,13 @@ local utils = import 'mixin-utils/utils.libsonnet';
       |||
         sum by(%s, %s, device) (
           rate(
-            node_disk_read_bytes_total[$__rate_interval]
+            %s[$__rate_interval]
           )
-        ) + %s
+        )
       ||| % [
         $._config.per_node_label,
         $._config.per_instance_label,
-        $.filterNodeDiskContainer(containerName),
+        $.nodeDiskContainerBytesTotal(containerName, 'reads'),
       ],
       '{{%s}} - {{device}}' % $._config.per_instance_label
     ) +
@@ -264,9 +262,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     { yaxes: $.yaxes('percentunit') },
 
   containerLabelMatcher(containerName)::
-    if containerName == 'ingester'
-    then 'label_name=~"ingester.*"'
-    else 'label_name="%s"' % containerName,
+    'persistentvolumeclaim=~"%s"' % containerName,
 
   goHeapInUsePanel(title, jobName)::
     $.panel(title) +
@@ -470,32 +466,15 @@ local utils = import 'mixin-utils/utils.libsonnet';
       { yaxes: $.yaxes('percentunit') }
     ),
 
-  filterNodeDiskContainer(containerName)::
+  nodeDiskContainerBytesTotal(containerName, op)::
     |||
-      ignoring(%s) group_right() (
-        label_replace(
-          count by(
-            %s,
-            %s,
-            device
-          )
-          (
-            container_fs_writes_bytes_total{
-              %s,
-              container="%s",
-              device!~".*sda.*"
-            }
-          ),
-          "device",
-          "$1",
-          "device",
-          "/dev/(.*)"
-        ) * 0
-      )
+      container_fs_%s_bytes_total{
+        %s,
+        container="%s",
+        device!~".*sda.*|.*nvme0.*"
+      }
     ||| % [
-      $._config.per_instance_label,
-      $._config.per_node_label,
-      $._config.per_instance_label,
+      op,
       $.namespaceMatcher(),
       containerName,
     ],

From 5bebd954bb390670c381be531dd5166947eaa05c Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Mon, 24 Jan 2022 12:40:24 -0500
Subject: [PATCH 7/8] maintain backwards compatibility with main branch

---
 cortex-mixin/config.libsonnet | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/cortex-mixin/config.libsonnet b/cortex-mixin/config.libsonnet
index d4fa0f2a..b2c7c643 100644
--- a/cortex-mixin/config.libsonnet
+++ b/cortex-mixin/config.libsonnet
@@ -9,12 +9,12 @@
     storage_engine: ['blocks'],
 
     // HTTP URL prefix under which the Prometheus api is available.
-    prometheus_http_prefix: 'cortex',
+    prometheus_http_prefix: 'prometheus',
 
     // Disable unused panels depending whether a component was installed or not
-    cortex_gw_enabled: false,
-    query_scheduler_enabled: false,
-    ruler_enabled: false,
+    cortex_gw_enabled: true,
+    query_scheduler_enabled: true,
+    ruler_enabled: true,
 
     // For chunks backend, switch for chunk index type.
     // May contain 'bigtable', 'dynamodb' or 'cassandra'.
@@ -55,7 +55,7 @@
     cortex_p99_latency_threshold_seconds: 2.5,
 
     // Whether resources dashboards are enabled (based on cAdvisor metrics).
-    resources_dashboards_enabled: true,
+    resources_dashboards_enabled: false,
 
     // The label used to differentiate between different application instances (i.e. 'pod' in a kubernetes install).
     per_instance_label: 'pod',

From 8c912e13d31b65e3f591754160391d6defb72028 Mon Sep 17 00:00:00 2001
From: Dan Constantinescu <dconstantinescu@blackberry.com>
Date: Mon, 24 Jan 2022 18:06:08 -0500
Subject: [PATCH 8/8] update CHANGELOG.md

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3135d97c..b7fff5cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
 
 ## master / unreleased
 
+* [ENHANCEMENT] Update cortex-mixin to support Cortex deployment from the Helm chart. #361
+
 ## 1.11.0 / 2021-12-30
 
 * [CHANGE] Store gateway: set `-blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency`,