diff --git a/signals/Makefile b/signals/Makefile new file mode 100644 index 000000000..a62593e33 --- /dev/null +++ b/signals/Makefile @@ -0,0 +1,36 @@ +.PHONY: generate test clean + +generate: + cd ./schema && cue exp gengotypes && mv cue_types_gen.go ../gen/go/cue_types_gen.go + +# Optional: add a clean target to remove generated files +clean: + rm -f ./gen/go/gen/cue_types_gen.go + +test: + @echo "Running CUE validation tests..." + @echo "Testing valid files (should pass):" + @for file in ./schema/tests/test_*_ok.json; do \ + if [ -f "$$file" ]; then \ + echo " Testing $$file (should pass)"; \ + if cue vet ./schema/*.cue "$$file"; then \ + echo " ✓ PASS: $$file"; \ + else \ + echo " ✗ FAIL: $$file (expected to pass)"; \ + exit 1; \ + fi; \ + fi; \ + done + @echo "Testing invalid files (should fail):" + @for file in ./schema/tests/test_*_bad.json; do \ + if [ -f "$$file" ]; then \ + echo " Testing $$file (should fail)"; \ + if cue vet ./schema/*.cue "$$file" 2>/dev/null; then \ + echo " ✗ FAIL: $$file (expected to fail but passed)"; \ + exit 1; \ + else \ + echo " ✓ PASS: $$file (correctly failed validation)"; \ + fi; \ + fi; \ + done + @echo "All tests completed successfully!" diff --git a/signals/README.md b/signals/README.md new file mode 100644 index 000000000..05327a12f --- /dev/null +++ b/signals/README.md @@ -0,0 +1,190 @@ +# Signals Framework + +A language-agnostic framework for defining, processing, and visualizing observability signals across multiple data sources and dashboard systems. + +## Overview + +The signals framework provides a standardized way to define observability metrics (signals) that can be processed into dashboard visualizations. It supports multiple data sources (Prometheus, Loki, OTEL) and can generate dashboards for various visualization systems. + +## Language-Agnostic Interface Specifications + +To ensure consistent implementation across all target languages, the framework provides formal interface specifications: + +### 1. Method Interface Specification (`signal_methods.md`) + +Defines the exact methods that must be implemented for the Signal class/object in every language: + +- **Builder Methods**: `withTopK()`, `withExprWrappersMixin()`, `withOffset()`, etc. +- **Panel Rendering**: `asTimeSeries()`, `asStat()`, `asGauge()`, `asTable()`, etc. +- **Panel Composition**: `asPanelMixin()`, `asTarget()`, `asTableColumn()`, etc. +- **Expression Generation**: `asPanelExpression()`, `asRuleExpression()` +- **Utilities**: `getVariablesMultiChoice()` + +### 2. OpenAPI Interface (`signal_interface.yaml`) + +REST API specification that can be used to: +- Generate client stubs for each language +- Validate method signatures and data structures +- Ensure consistent input/output contracts + +### 3. Core Schema (`signal.cue`, `signal_source.cue`) + +CUE schema definitions that provide: +- Type-safe signal and signal group definitions +- Validation rules for all signal properties +- Data structure contracts for cross-language compatibility + +## Implementation Strategy + +### Language-Specific Implementation + +Each target language must implement the Signal interface following these guidelines: + +**Jsonnet** (Reference Implementation) +```jsonnet +local signals = signal.init(config); +signals.mySignal.withTopK(10).asTimeSeries() +``` + +**TypeScript** +```typescript +const signal = new Signal(config); +signal.withTopK(10).asTimeSeries(); +``` + +**Python** +```python +signal = Signal(config) +signal.with_top_k(10).as_time_series() +``` + +**Go** +```go +panel := signal.WithTopK(10).AsTimeSeries() +``` + +### Consistency Requirements + +All implementations must: + +1. **Method Signatures**: Match the interface specification exactly +2. **Expression Transformation**: Follow the same auto-transformation rules +3. **Template Expansion**: Support identical variable templates +4. **Panel Structure**: Generate compatible Grafana panel objects +5. **Testing**: Include comprehensive test suites + +## Core Functions + +The framework implements these core functions consistently across languages: + +### 1. Signal Definition & Validation +``` +Input: Signal configuration (JSON/YAML/Code) +Output: Validated Signal object with transformation rules +``` + +### 2. Expression Processing +``` +Input: Signal + context (datasource, aggregation level) +Output: PromQL/LogQL expressions with auto-transformations +``` + +### 3. Panel Generation +``` +Input: Signal + panel type +Output: Complete Grafana panel configuration +``` + +### 4. Dashboard Assembly +``` +Input: Signal collection + metadata +Output: Complete dashboard with panels and variables +``` + +## Expression Transformation Rules + +All implementations follow these automatic transformations: + +- **Counter**: `rate(metric[interval])` or similar range functions +- **Histogram**: `histogram_quantile(0.95, rate(metric[interval])) by (le)` +- **Gauge/Info/Raw**: No transformation +- **Aggregation**: `sum by (labels) (expression)` when aggregation is enabled + +## Supported Languages + +- ✅ **Jsonnet** - Reference implementation (common-lib) +- 🔄 **TypeScript** - Web/Node.js environments +- 🔄 **Python** - Data science and automation +- 🔄 **Go** - Performance-critical applications + +## Schema Definitions + +- **`signal.cue`** - Core signal and signal group schema with validation +- **`signal_source.cue`** - Data source-specific configuration schema +- **`signal_interface.yaml`** - OpenAPI specification for method contracts +- **`signal_methods.md`** - Detailed method interface documentation + +## Getting Started + +### 1. Define Signals +Create signal definitions using the CUE schema: +```jsonnet +{ + name: "CPU Usage", + type: "gauge", + unit: "percent", + sources: { + prometheus: { + expr: "100 - avg(rate(cpu_idle[5m])) * 100" + } + } +} +``` + +### 2. Validate Definitions +```bash +make test # Validates against CUE schema +``` + +### 3. Implement Language Bindings +Use the interface specifications to implement Signal classes in your target language: + +- Follow `signal_methods.md` for method signatures +- Use `signal_interface.yaml` for code generation +- Implement expression transformation rules +- Add comprehensive tests + +### 4. Generate Dashboards +```typescript +const signal = new Signal(config); +const dashboard = new Dashboard() + .addPanel(signal.asTimeSeries()) + .addVariables(Signal.getVariablesMultiChoice([signal])); +``` + +## File Structure + +``` +signals/ +├── schema/ +│ ├── signal.cue # Core schema definitions +│ ├── signal_source.cue # Source-specific schema +│ ├── signal_interface.yaml # OpenAPI method contracts +│ ├── signal_methods.md # Method interface spec +│ └── tests/ # Validation test files +├── gen/ # Generated code output +├── architecture.md # Architecture documentation +├── requirements.md # Functional requirements +└── README.md # This file +``` + +## Development Workflow + +1. **Schema First**: Define/modify schemas in CUE files +2. **Validate**: Ensure test files pass validation +3. **Interface**: Update method specifications if needed +4. **Implement**: Code language-specific implementations +5. **Test**: Verify cross-language compatibility +6. **Generate**: Use for dashboard/alert generation + +This approach ensures that regardless of the implementation language, all Signal objects behave consistently and generate compatible dashboard outputs. \ No newline at end of file diff --git a/signals/architecture.md b/signals/architecture.md new file mode 100644 index 000000000..7155865ef --- /dev/null +++ b/signals/architecture.md @@ -0,0 +1,264 @@ +# Architecture: How signals work + +This document provides a high-level overview of the signals pattern, its flow in the system, and the language-agnostic implementation architecture. + +## Schema Components + +The signals schema consists of three main components: + +### 1. Signal Group (`#signalGroup`) +Top-level container that defines: +- **Datasource configuration**: `datasource`, `datasourceLabel` +- **Aggregation settings**: `aggFunction`, `aggLevel` +- **Discovery**: `discoveryMetric` for service discovery +- **Templating**: `legendCustomTemplate` for consistent labeling +- **Timing**: `interval`, `alertsInterval` for data collection +- **Signal collection**: `signals` map containing individual signal definitions + +### 2. Signal (`#signal`) +Individual metric definition that specifies: +- **Identity**: `name` (required), `nameShort` (for legends/columns), `type` (counter/gauge/histogram/info/raw/stub) +- **Metadata**: `unit`, `description`, `optional` flag +- **Aggregation**: `aggLevel`, `aggFunction` for metric rollup +- **Visualization**: `legendCustomTemplate` for display formatting +- **Sources**: `sources` map linking to different data sources + +### 3. Signal Source (`#signalSource`) +Data source-specific metric query configuration: +- **Query**: `expr` (required PromQL/LogQL expression) +- **Processing**: `exprWrappers` for query transformation +- **Functions**: `rangeFunction` (rate/irate/delta/etc), `aggFunction` +- **Filtering**: `aggKeepLabels` to preserve specific labels +- **Display**: `infoLabel`, `legendCustomTemplate` for visualization +- **Mapping**: `valueMappings` for value transformation +- **Quantiles**: `quantile` for histogram metrics (0-1 range) + +## Language-Agnostic Implementation Architecture + +The signals framework is designed for consistent implementation across multiple programming languages: + +### Interface Specifications Layer + +``` +┌─────────────────────────────────────────────────────────┐ +│ Interface Specifications │ +├─────────────────────┬─────────────────────┬─────────────┤ +│ CUE Schema │ │ Methods │ +│ │ │ Spec │ +└─────────────────────┴─────────────────────┴─────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Language Implementations │ +├──────────────┬──────────────┬──────────────┬───-────────┤ +│ Jsonnet │ TypeScript │ Python │ Go │ +│Implementation│Implementation│Implementation│Implementa. │ +└──────────────┴──────────────┴──────────────┴───────────-┘ + │ + ▼ +┌─────────────────────────────────────────────────────────┐ +│ Generated Outputs │ +├─────────────────────-─────────────────────┬─────────────┤ +│ Grafana Panels/Parts of panels | Prometheus │ +│ │ Rules │ +└─────────────────────-─────────────────────┴─────────────┘ +``` + +### Core Method Organization + +Each language implementation organizes methods into two categories: + +#### 1. **Root Signal Methods** - Core signal modifications and Prometheus output +- **Builder Methods**: `withTopK()`, `withOffset()`, `withQuantile()`, `withExprWrappersMixin()`, `withFilteringSelectorMixin()` +- **Rule Generation**: `asRuleExpression()` + +#### 2. **Grafana Methods** (`.grafana` namespace) - Dashboard and panel generation +- **Panel Rendering**: `asTimeSeries()`, `asStat()`, `asGauge()`, `asTable()`, `asStatusHistory()` +- **Panel Composition**: `asPanelMixin()`, `asTarget()`, `asTableColumn()`, `asOverride()` +- **Expression Generation**: `asPanelExpression()` +- **Utility Functions**: `getVariablesMultiChoice()` + +## Data Processing Flow + +``` +┌─────────────────┐ ┌─────────────────────┐ ┌──────────────────┐ +│ Schema Definition│ │Language Implementation│ │Expression Transform│ +│ │ │ │ │ │ +│ ┌─────────────┐ │ │ ┌─────────────────┐ │ │ ┌──────────────┐ │ +│ │signalGroup │─┼────┼▶│Signal Class/Obj │─┼────┼▶│Counter │ │ +│ └─────────────┘ │ │ └─────────────────┘ │ │ │Transform │ │ +│ ┌─────────────┐ │ │ ┌─────────────────┐ │ │ └──────────────┘ │ +│ │signal │─┼────┼▶│Expression │ │ │ ┌──────────────┐ │ +│ └─────────────┘ │ │ │Processor │ │ │ │Histogram │ │ +│ ┌─────────────┐ │ │ └─────────────────┘ │ │ │Transform │ │ +│ │signalSource │─┼────┼▶│Panel Generator │ │ │ └──────────────┘ │ +│ └─────────────┘ │ │ │ │ ┌──────────────┐ │ +└─────────────────┘ └─────────────────────┘ │ │Aggregation │ │ + │ │Engine │ │ + │ └──────────────┘ │ + │ ┌──────────────┐ │ + │ │Template │ │ + │ │Expander │ │ + │ └──────────────┘ │ + └──────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ Output Generation │ +├─────────────────────┬─────────────────────┬─────────────────────────┤ +│ Panel Objects │ Dashboard Variables │ Prometheus Rules │ +│ │ │ │ +│ • TimeSeries │ • Multi-choice vars │ • Rule expressions │ +│ • Stat panels │ • Instance filters │ • Alert conditions │ +│ • Gauge panels │ • Group filters │ • Recording rules │ +│ • Table panels │ │ │ +└─────────────────────┴─────────────────────┴─────────────────────────┘ +``` + +## Expression Transformation Pipeline + +### 1. Base Expression Processing +``` +Input: signal.sources[datasource].expr +Apply: Template variable expansion (%(queriesSelector)s, %(agg)s, etc.) +Output: Expanded base expression +``` + +### 2. Type-Specific Transformation +``` +Counter: (base_expr[]) +Histogram: histogram_quantile(, (rate(base_expr[])) by (le,)) +Gauge: base_expr (no transformation) +Info: base_expr (no transformation) +Raw: base_expr (no transformation) +Stub: "" (empty expression for missing optional signals) +``` + +### 3. Aggregation Application +``` +When aggLevel != "none": + by ( + ) () +``` + +### 4. Expression Wrappers +``` +Apply exprWrappers: [["topk(10,", ")"]] → topk(10, ) +Apply withTopK(): signal.withTopK(5) → topk(5, ) +Apply withOffset(): signal.withOffset("5m") → offset 5m +``` + +## Multi-Language Consistency + +### Implementation Pattern + +All languages follow the same pattern for method organization: + +```typescript +// Root signal methods (core modifications) +const modifiedSignal = signal + .withTopK(10) + .withOffset("5m") + .withQuantile(0.95); + +// Prometheus rule generation +const rule = modifiedSignal.asRuleExpression(); + +// Grafana panel generation +const panel = modifiedSignal.grafana.asTimeSeries(); +const target = modifiedSignal.grafana.asTarget(); +const mixin = modifiedSignal.grafana.asPanelMixin(); +``` + +### Code Generation Strategy + +``` +┌─────────────────┐ ┌─────────────────────┐ ┌──────────────────┐ +│ CUE Schema │───▶│ Type Generation │───▶│ Language Types │ +└─────────────────┘ └─────────────────────┘ └──────────────────┘ + │ │ +┌─────────────────┐ ┌─────────────────────┐ │ • TypeScript │ +│ OpenAPI Spec │───▶│ Client Generation │───▶│ • Python │ +└─────────────────┘ └─────────────────────┘ │ • Go │ + │ │ +┌─────────────────┐ ┌─────────────────────┐ │ Client Stubs │ +│Method Interface │───▶│Implementation Guide │───▶│ │ +└─────────────────┘ └─────────────────────┘ └──────────────────┘ +``` + +### Template Variable Support + +Both root signal methods and Grafana methods support template expansion: + +#### Grafana Methods (`.grafana` namespace) +Support all template variables including Grafana-specific ones: +- `%(queriesSelector)s` - Complete selector with filtering + instance + group labels +- `%(filteringSelector)s` - Filtering selector only +- `%(groupLabels)s`, `%(instanceLabels)s` - Label lists +- `%(agg)s`, `%(aggLegend)s` - Aggregation labels +- `%(aggFunction)s` - Aggregation function +- `%(interval)s` - Dashboard interval (Grafana-specific) +- `%(alertsInterval)s` - Alerts interval + +#### Root Signal Methods (Prometheus-related) +Support all template variables except Grafana-specific ones: +- All variables above except `%(interval)s` +- Use `%(alertsInterval)s` instead for Prometheus rules + +## Validation & Testing Strategy + +Each language implementation must pass: + +1. **Schema Validation Tests** - CUE schema compliance for all signal definitions +2. **Method Signature Tests** - Interface specification compliance across languages +3. **Expression Transformation Tests** - Consistent output for all signal types +4. **Template Expansion Tests** - Correct variable substitution in both contexts +5. **Panel Generation Tests** - Compatible Grafana panel structures +6. **Cross-Language Compatibility Tests** - Same inputs produce same outputs +7. **Namespace Isolation Tests** - Root methods don't affect Grafana methods + +## Key Architecture Principles + +### 1. **Method Organization** +- **Root Signal**: Core signal modifications and Prometheus output +- **Grafana Namespace**: Panel generation and dashboard-specific functionality +- **Clear Separation**: Logical grouping of functionality by target system + +### 2. **Builder Pattern** +- **Fluent Interface**: Chain modifications with immutable signal objects +- **Type Safety**: Each method returns appropriate types for further chaining +- **Flexibility**: Mix root modifications with Grafana output generation + +### 3. **Multi-Source Support** +- **Source Mapping**: Same signal can have different implementations per data source +- **Template Consistency**: Same templating system across all sources +- **Fallback Handling**: Optional signals gracefully handle missing sources + +### 4. **Expression Processing** +- **Type-Driven Transformations**: Automatic query generation based on signal type +- **Layered Processing**: Base → Type → Aggregation → Wrappers +- **Template Expansion**: Context-aware variable substitution + +### 5. **Language Agnostic Design** +- **Interface Contracts**: Formal specifications ensure compatibility +- **Code Generation**: Automated type and stub generation from schemas +- **Consistent Behavior**: Identical outputs across all language implementations + +## Data Flow Summary + +``` +Schema Definition (CUE) + ↓ +Language Implementation (Jsonnet/TS/Python/Go) + ↓ +Signal Object Creation + ↓ +Root Method Application (.withTopK(), .withOffset()) + ↓ +Expression Transformation Pipeline + ↓ +Output Generation: + ├── Prometheus Rules (.asRuleExpression()) + └── Grafana Panels (.grafana.asTimeSeries()) +``` + +This architecture ensures consistent behavior across all programming languages while providing flexibility for different output formats and use cases. diff --git a/signals/gen/go/cue_types_gen.go b/signals/gen/go/cue_types_gen.go new file mode 100644 index 000000000..c40ca6233 --- /dev/null +++ b/signals/gen/go/cue_types_gen.go @@ -0,0 +1,113 @@ +// Code generated by "cue exp gengotypes"; DO NOT EDIT. + +package signal + +type AggLevel string + +type AggFunction string + +type SignalType string + +type Signal struct { + Name string `json:"name"` + + Type string `json:"type"` + + Unit string `json:"unit,omitempty"` + + Description string `json:"description,omitempty"` + + Optional bool `json:"optional,omitempty"` + + AggLevel string `json:"aggLevel,omitempty"` + + AggFunction string `json:"aggFunction,omitempty"` + + LegendCustomTemplate any/* TODO: IncompleteKind: _|_ */ `json:"legendCustomTemplate,omitempty"` + + Sources map[string]struct { + Expr string `json:"expr"` + + ExprWrappers []string `json:"exprWrappers,omitempty"` + + RangeFunction string `json:"rangeFunction,omitempty"` + + AggFunction string `json:"aggFunction,omitempty"` + + AggKeepLabels []string `json:"aggKeepLabels,omitempty"` + + InfoLabel string `json:"infoLabel,omitempty"` + + LegendCustomTemplate string `json:"legendCustomTemplate,omitempty"` + + ValueMappings []struct { + Type string `json:"Type"` + + Options map[string]struct { + Text string `json:"Text"` + + Color string `json:"Color"` + + Index int64 `json:"Index"` + } `json:"Options"` + } `json:"valueMappings,omitempty"` + + Quantile any/* CUE number; int64 or float64 */ `json:"quantile,omitempty"` + } `json:"sources"` +} + +type SignalGroup struct { + Datasource string `json:"datasource,omitempty"` + + DatasourceLabel string `json:"datasourceLabel,omitempty"` + + // filteringSelector?: [...string] + // groupLabels?: [...string] + // instanceLabels?: [...string] + AggFunction AggFunction `json:"aggFunction,omitempty"` + + AggLevel AggLevel `json:"aggLevel,omitempty"` + + DiscoveryMetric map[string]string `json:"discoveryMetric"` + + LegendCustomTemplate string `json:"legendCustomTemplate,omitempty"` + + Interval string `json:"interval,omitempty"` + + AlertsInterval string `json:"alertsInterval,omitempty"` + + // varAdHocEnabled?: bool + // varAdHocLabels?: [...string] + // enableLokiLogs?: bool + Signals map[string]Signal `json:"signals"` +} + +type SignalSource struct { + Expr string `json:"expr"` + + ExprWrappers []string `json:"exprWrappers,omitempty"` + + RangeFunction string `json:"rangeFunction,omitempty"` + + AggFunction string `json:"aggFunction,omitempty"` + + AggKeepLabels []string `json:"aggKeepLabels,omitempty"` + + InfoLabel string `json:"infoLabel,omitempty"` + + LegendCustomTemplate string `json:"legendCustomTemplate,omitempty"` + + ValueMappings []struct { + Type string `json:"Type"` + + Options map[string]struct { + Text string `json:"Text"` + + Color string `json:"Color"` + + Index int64 `json:"Index"` + } `json:"Options"` + } `json:"valueMappings,omitempty"` + + Quantile any/* CUE number; int64 or float64 */ `json:"quantile,omitempty"` +} diff --git a/signals/requirements.md b/signals/requirements.md new file mode 100644 index 000000000..636a2260f --- /dev/null +++ b/signals/requirements.md @@ -0,0 +1,209 @@ +# Signals Framework Requirements + +This document defines the functional and technical requirements for the signals observability framework. + +## Goals + +The signals framework aims to achieve the following high-level objectives: + +### 1. **Unified Observability Interface** +- Provide a single, consistent abstraction layer for defining monitoring signals across different data sources and visualization systems + +### 2. **Multi-Stack Compatibility** +- Support seamless migration between monitoring stacks (Prometheus ↔ OpenTelemetry ↔ Loki) without redefining signals +- Enable hybrid monitoring environments where different teams use different tools but share signal definitions + +### 3. **Developer Experience Excellence** +- Offer intuitive, type-safe APIs in multiple programming languages (Jsonnet, TypeScript, Python, Go) +- Provide fluent, chainable interfaces that make common tasks simple and complex tasks possible +- Enable rapid prototyping and iteration on monitoring configurations with immediate feedback + +### 4. **Configuration as Code** +- Support version-controlled, peer-reviewed monitoring configurations through code +- Enable automated testing and validation of monitoring setups before deployment +- Provide deterministic, reproducible monitoring infrastructure across environments + +### 5. **Extensibility and Future-Proofing** +- Design plugin-friendly architecture that supports new data sources and visualization systems +- Enable custom signal types and transformations for specialized use cases +- Provide stable APIs that can evolve without breaking existing implementations + +### 7. **Operational Efficiency** +- Reduce the time-to-value for new monitoring implementations from weeks to hours +- Minimize maintenance overhead through automated configuration generation +- Enable consistent monitoring practices across teams and projects within organizations + +### 8. **Quality and Reliability** +- Ensure high confidence in monitoring configurations through comprehensive validation +- Provide consistent behavior across all supported languages and platforms +- Enable thorough testing of monitoring setups before production deployment + +These goals drive all technical decisions and prioritization within the signals framework development. + +## Functional Requirements + +### 1. Multi-Source Signal Support +- **MUST** support multiple data sources per signal (Prometheus, OpenTelemetry, Loki, etc.) +- **MUST** allow same signal definition to work across different monitoring stacks +- **MUST** support source-specific query expressions and configurations +- **MUST** handle missing sources gracefully with optional signals + +### 2. Signal Type Support +- **MUST** support counter signals with automatic rate/increase transformations +- **MUST** support gauge signals with pass-through expressions +- **MUST** support histogram signals with quantile calculations +- **MUST** support info signals for metadata extraction +- **MUST** support raw signals for custom expressions without transformation +- **MUST** support stub signals for missing optional sources + +### 3. Expression Processing +- **MUST** support template variable expansion (%(queriesSelector)s, %(agg)s, etc.) +- **MUST** apply type-specific automatic transformations +- **MUST** support configurable aggregation levels (none, instance, group) +- **MUST** support configurable aggregation functions (avg, min, max, sum) +- **MUST** support expression wrappers for custom query modifications +- **MUST** support offset and topK query modifications + +### 4. Visualization Support +- **MUST** generate Grafana TimeSeries, Stat, Gauge, Table, and StatusHistory panels +- **MUST** support panel composition with mixins and targets +- **MUST** generate dashboard variables for filtering +- **MUST** support custom legend templates and value mappings +- **MUST** handle units and display formatting automatically +- **MUST** support Grafana Scenes VizPanel generation for TypeScript implementation +- **MUST** integrate with Grafana Scenes PanelBuilders API for standard visualizations +- **MUST** support Grafana Scenes field configuration and overrides +- **MUST** support Grafana Scenes custom panel plugins and runtime registration + +### 5. Alerting Support +- **MUST** generate Prometheus-compatible rule expressions +- **MUST** support alert-specific intervals (excluding Grafana variables) +- **MUST** maintain expression consistency between dashboards and alerts + +## Technical Requirements + +### 6. Programming Language Support +- **MUST** support Jsonnet implementation +- **MUST** support TypeScript implementation +- **MUST** support Python implementation +- **MUST** support Go implementation +- **MUST** maintain identical behavior across all language implementations + +### 7. Schema Validation +- **MUST** use CUE schema for validation and type safety +- **MUST** validate signal definitions at compile time +- **MUST** provide clear error messages for schema violations +- **MUST** support schema evolution and backward compatibility + +### 8. Method Organization +- **MUST** provide root signal methods for core modifications +- **MUST** provide Grafana namespace for dashboard-specific functionality +- **MUST** maintain clear separation between modification and output generation +- **MUST** support fluent chaining interface patterns + +### 9. API Consistency +- **MUST** use identical method signatures across all languages +- **MUST** follow language-specific naming conventions (camelCase, snake_case, PascalCase) +- **MUST** return appropriate types for method chaining +- **MUST** maintain immutability in signal modifications + +### 10. Template Variable Context +- **MUST** support different template variables for Grafana vs Prometheus contexts +- **MUST** include %(interval)s for Grafana dashboard generation +- **MUST** exclude %(interval)s from Prometheus rule generation +- **MUST** support all common variables: %(queriesSelector)s, %(agg)s, %(aggFunction)s + +### 11. Grafana Scenes Integration (TypeScript) +- **MUST** generate VizPanel objects compatible with @grafana/scenes library +- **MUST** support PanelBuilders API for all standard Grafana visualizations +- **MUST** support SceneDataProvider integration for data binding +- **MUST** support field configuration with custom properties and overrides +- **MUST** support override matchers (name, regex, type, query, value, comparison) +- **MUST** support custom panel plugin registration with sceneUtils.registerRuntimePanelPlugin +- **MUST** support panel header actions and menu customization +- **MUST** support mixin functions for common visualization configurations +- **MUST** integrate with SceneQueryRunner for data source queries +- **MUST** support EmbeddedScene and SceneFlexLayout integration + +## Quality Requirements + +### 12. Testing Coverage +- **MUST** provide schema validation tests for all signal types +- **MUST** provide cross-language compatibility tests +- **MUST** provide expression transformation tests +- **MUST** provide template expansion tests +- **MUST** provide panel generation tests +- **MUST** provide namespace isolation tests + +### 13. Documentation Standards +- **MUST** provide comprehensive API documentation for all languages +- **MUST** include working examples for all signal types +- **MUST** document template variable usage and context +- **MUST** provide migration guides for schema changes + +### 14. Performance Requirements +- **MUST** generate expressions efficiently without excessive string manipulation +- **MUST** support lazy evaluation where possible +- **MUST** minimize memory allocation in method chaining +- **SHOULD** cache compiled expressions when appropriate + +## Interface Requirements + +### 15. Code Generation Support +- **MUST** support automated type generation from CUE schema +- **MUST** generate language-specific client stubs + +### 16. Extension Points +- **MUST** support custom signal types through extension +- **MUST** support custom expression wrappers +- **MUST** support custom aggregation functions +- **SHOULD** support plugin architecture for new data sources + +### 17. Configuration Management +- **MUST** support JSON/YAML configuration files +- **MUST** support environment-specific overrides +- **MUST** support signal grouping and organization +- **MUST** validate configuration against schema + +## Compatibility Requirements + +### 18. Dashboard System Agnostic +- **MUST** generate Grafana-compatible panel JSON +- **SHOULD** support extension to other dashboard systems +- **MUST** maintain clean separation between signal logic and visualization + +### 19. Monitoring Stack Agnostic +- **MUST** support Prometheus metrics and PromQL +- **MUST** support OpenTelemetry metrics +- **MUST** support Loki logs and LogQL +- **SHOULD** support extension to other monitoring systems + +### 20. Version Compatibility +- **MUST** maintain backward compatibility for schema changes +- **MUST** provide clear deprecation warnings +- **MUST** support gradual migration between versions +- **MUST** version API specifications + +## Security Requirements + +### 21. Input Validation +- **MUST** validate all user inputs against schema +- **MUST** sanitize expressions to prevent injection attacks +- **MUST** validate template variable expansion +- **MUST** provide safe defaults for all configurations + +### 22. Access Control +- **SHOULD** support role-based access to signals +- **SHOULD** support data source access controls +- **SHOULD** audit signal configuration changes + +## Future Requirements + +### 23. Planned Features +- **SHOULD** support ad-hoc variable filters +- **SHOULD** support Loki logs integration +- **SHOULD** support additional dashboard systems +- **SHOULD** support real-time signal validation +- **SHOULD** support signal dependency tracking + +These requirements ensure the signals framework provides a robust, scalable, and maintainable solution for observability across multiple programming languages and monitoring stacks. diff --git a/signals/schema/signal.cue b/signals/schema/signal.cue new file mode 100644 index 000000000..f4f115e16 --- /dev/null +++ b/signals/schema/signal.cue @@ -0,0 +1,113 @@ +package signal + +// Metrics aggregation level - determines how metrics are grouped +// - "none": no aggregation, show individual metrics +// - "instance": aggregate by instance (single entity/service instance) +// - "group": aggregate by group (job/service level) +#aggLevel: "none" | "instance" | "group" + +// Aggregation function used when aggLevel is set to "instance" or "group" +// Applied as: by () () +#aggFunction: "avg" | "min" | "max" | "sum" + +// Signal type determines automatic transformations applied to expressions: +// - "counter": wrapped with rate/increase functions, e.g. rate([]) +// - "gauge": no transformation, used as-is +// - "histogram": wrapped with histogram_quantile function +// - "info": no transformation, typically used for metadata +// - "raw": no transformation, preserves complex custom expressions +// - "stub": internal type for optional signals when source is missing +#signalType: "counter" | "gauge" | "histogram" | "info" | "raw" | "stub" + +// Individual signal definition within a signal group +#signal: close({ + // Signal name used for panel titles and descriptions + name!: string + + // Short name used for panel legends and table columns by default + nameShort?: string + + // Signal type - determines automatic query transformations + type!: #signalType + + // Optional description used for panel descriptions and tooltips + description?: string + + // Signal type - determines automatic query transformations + type!: #signalType + + // Units for the signal (bytes, seconds, percent, etc.) + unit?: string + + // Mark signal as optional - won't cause validation errors if missing + // When true, missing signals render as empty panels instead of errors + optional?: bool + + // Override aggregation level from signal group default + aggLevel?: #aggLevel + + // Override aggregation function from signal group default + aggFunction?: #aggFunction + + // Custom legend template to override automatic legend generation + // Supports Grafana templating like {{instance}}, {{job}} + legendCustomTemplate?: string + + // Signal sources - one per data source type (prometheus, opentelemetry, etc.) + // Key is the source name, value contains the expression and configuration + sources: {[string]: #signalSource} +}) + +// Signal group configuration - contains multiple related signals +#signalGroup: close({ + // Prometheus datasource name (default: "datasource") + datasource?: string + + // Human-readable datasource label (default: "Data source") + datasourceLabel?: string + + // Future: filtering selector for scoping metrics + // filteringSelector?: [...string] + + // Future: labels used to identify groups (jobs/services) + // groupLabels?: [...string] + + // Future: labels used to identify instances + // instanceLabels?: [...string] + + // Default aggregation function for all signals in this group + aggFunction?: #aggFunction + + // Default aggregation level for all signals in this group + aggLevel?: #aggLevel + + // Metrics used for variable discovery in dashboards + // Key is source type, value is metric name (e.g. {"prometheus": "up"}) + discoveryMetric: {[string]: string} + + // Default custom legend template for all signals in this group + legendCustomTemplate?: string + + // Interval used for counter/histogram transformations in dashboards + // Supports Grafana variables like $__rate_interval, $__interval + interval?: string + + // Interval used for counter/histogram transformations in alerts + // Must be static value (5m, 1h) - Grafana variables not supported + alertsInterval?: string + + // Collection of signals in this group + signals: {[string]: #signal} + + // Future: enable ad hoc variable filters + // varAdHocEnabled?: bool + + // Future: limit ad hoc filters to specific labels + // varAdHocLabels?: [...string] + + // Future: enable Loki logs integration + // enableLokiLogs?: bool +}) + +// Root schema - array of signal groups +signalGroup: [...#signalGroup] diff --git a/signals/schema/signal_interface.yaml b/signals/schema/signal_interface.yaml new file mode 100644 index 000000000..01bd480e3 --- /dev/null +++ b/signals/schema/signal_interface.yaml @@ -0,0 +1,520 @@ +openapi: 3.0.3 +info: + title: Signal Interface Specification + description: | + Language-agnostic interface specification for Signal objects and their methods. + This defines the contract that must be implemented in each target language. + version: 1.0.0 + +components: + schemas: + # Core data types + Signal: + type: object + description: Core signal object with all properties and methods + properties: + # Properties + name: + type: string + nameShort: + type: string + type: + type: string + enum: [counter, gauge, histogram, info, raw, stub] + unit: + type: string + description: + type: string + optional: + type: boolean + aggLevel: + type: string + enum: [none, instance, group] + aggFunction: + type: string + enum: [avg, min, max, sum] + sourceMaps: + type: array + items: + $ref: '#/components/schemas/SignalSource' + + SignalSource: + type: object + properties: + expr: + type: string + exprWrappers: + type: array + items: + type: array + items: + type: string + rangeFunction: + type: string + enum: [rate, irate, delta, idelta, increase] + aggKeepLabels: + type: array + items: + type: string + infoLabel: + type: string + valueMappings: + type: array + items: + type: object + legendCustomTemplate: + type: string + quantile: + type: number + + Panel: + type: object + description: Generic panel object (Grafana panel structure) + + Target: + type: object + description: Query target for panels + + Override: + type: object + description: Panel field override + + Variables: + type: array + description: Dashboard variables collection + items: + type: object + +# Interface Methods Specification +paths: + # Grafana-specific Methods + /grafana/asTimeSeries: + post: + summary: asTimeSeries() + description: Render signal as Grafana TimeSeries panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: TimeSeries panel + content: + application/json: + schema: + $ref: '#/components/schemas/Panel' + + /grafana/asStat: + post: + summary: asStat() + description: Render signal as Grafana Stat panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Stat panel + content: + application/json: + schema: + $ref: '#/components/schemas/Panel' + + /grafana/asGauge: + post: + summary: asGauge() + description: Render signal as Grafana Gauge panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Gauge panel + content: + application/json: + schema: + $ref: '#/components/schemas/Panel' + + /grafana/asStatusHistory: + post: + summary: asStatusHistory() + description: Render signal as Grafana StatusHistory panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: StatusHistory panel + content: + application/json: + schema: + $ref: '#/components/schemas/Panel' + + /grafana/asTable: + post: + summary: asTable(format='table|time_series') + description: Render signal as Grafana Table panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + format: + type: string + enum: [table, time_series] + default: table + responses: + '200': + description: Table panel + content: + application/json: + schema: + $ref: '#/components/schemas/Panel' + + /grafana/asPanelMixin: + post: + summary: asPanelMixin() + description: Add signal as target and override to existing Grafana panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + existingPanel: + $ref: '#/components/schemas/Panel' + responses: + '200': + description: Panel mixin object + content: + application/json: + schema: + type: object + properties: + targets: + type: array + items: + $ref: '#/components/schemas/Target' + overrides: + type: array + items: + $ref: '#/components/schemas/Override' + + /grafana/asTableColumn: + post: + summary: asTableColumn(format='table|time_series') + description: Add signal as column to existing Grafana table panel + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + format: + type: string + enum: [table, time_series] + default: table + responses: + '200': + description: Table column mixin + content: + application/json: + schema: + type: object + properties: + targets: + type: array + items: + $ref: '#/components/schemas/Target' + overrides: + type: array + items: + $ref: '#/components/schemas/Override' + + /grafana/asTarget: + post: + summary: asTarget() + description: Generate Grafana query target for signal + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Query target + content: + application/json: + schema: + $ref: '#/components/schemas/Target' + + /grafana/asTableTarget: + post: + summary: asTableTarget() + description: Generate Grafana table-formatted query target + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Table query target + content: + application/json: + schema: + $ref: '#/components/schemas/Target' + + /grafana/asOverride: + post: + summary: asOverride() + description: Generate Grafana panel field override for signal + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Panel override + content: + application/json: + schema: + $ref: '#/components/schemas/Override' + + /grafana/asPanelExpression: + post: + summary: asPanelExpression() + description: Generate expression for Grafana panel target + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Panel expression + content: + application/json: + schema: + type: object + properties: + expression: + type: string + + /grafana/getVariablesMultiChoice: + post: + summary: getVariablesMultiChoice() + description: Generate Grafana dashboard variables for signals + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signals: + type: array + items: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Dashboard variables + content: + application/json: + schema: + $ref: '#/components/schemas/Variables' + + # Prometheus-specific Methods + /prometheus/withTopK: + post: + summary: withTopK(limit=25) + description: Wrap signal expression into topk() function + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + limit: + type: integer + default: 25 + responses: + '200': + description: Modified signal + content: + application/json: + schema: + $ref: '#/components/schemas/Signal' + + /prometheus/withExprWrappersMixin: + post: + summary: withExprWrappersMixin(wrapper=[]) + description: Add additional expression wrappers on top of existing ones + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + wrapper: + type: array + items: + type: string + responses: + '200': + description: Modified signal + content: + application/json: + schema: + $ref: '#/components/schemas/Signal' + + /prometheus/withOffset: + post: + summary: withOffset(offset) + description: Add offset modifier to the expression + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + offset: + type: string + responses: + '200': + description: Modified signal + content: + application/json: + schema: + $ref: '#/components/schemas/Signal' + + /prometheus/withFilteringSelectorMixin: + post: + summary: withFilteringSelectorMixin(mixin) + description: Add additional selector to filteringSelector + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + mixin: + type: string + responses: + '200': + description: Modified signal + content: + application/json: + schema: + $ref: '#/components/schemas/Signal' + + /prometheus/withQuantile: + post: + summary: withQuantile(quantile=0.95) + description: Add quantile modifier for histogram signals + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + quantile: + type: number + default: 0.95 + minimum: 0 + maximum: 1 + responses: + '200': + description: Modified signal + content: + application/json: + schema: + $ref: '#/components/schemas/Signal' + + /prometheus/asRuleExpression: + post: + summary: asRuleExpression() + description: Generate Prometheus rule expression (no Grafana variables) + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + signal: + $ref: '#/components/schemas/Signal' + responses: + '200': + description: Rule expression + content: + application/json: + schema: + type: object + properties: + expression: + type: string \ No newline at end of file diff --git a/signals/schema/signal_methods.md b/signals/schema/signal_methods.md new file mode 100644 index 000000000..5ecfc56fc --- /dev/null +++ b/signals/schema/signal_methods.md @@ -0,0 +1,443 @@ +# Signal Methods Interface Specification + +This document defines the exact methods that must be implemented for the Signal class/object in every target language (Jsonnet, TypeScript, Python, Go). + +The methods are organized by their target system: +- **Root Signal Methods** - Core signal modifications and Prometheus rule generation +- **Grafana Methods** - Dashboard and panel generation (`.grafana` namespace) + +## Signal Object Properties + +``` +Signal { + name: string // Signal name for panel titles + nameShort: string // Short name for legends + type: enum // counter|gauge|histogram|info|raw|stub + unit: string // Units (bytes, seconds, etc.) + description: string // Panel description + optional: boolean // Whether signal is optional + aggLevel: enum // none|instance|group + aggFunction: enum // avg|min|max|sum + sourceMaps: SignalSource[] // Array of source configurations +} + +SignalSource { + expr: string // Base PromQL/LogQL expression + exprWrappers: string[][] // [["left", "right"]] wrapper pairs + rangeFunction: enum // rate|irate|delta|idelta|increase + aggKeepLabels: string[] // Labels to keep during aggregation + infoLabel: string // Label for info type metrics + valueMappings: object[] // Grafana value mappings + legendCustomTemplate: string // Custom legend template + quantile: number // 0.0-1.0 for histogram metrics +} +``` + +## Root Signal Methods + +### Query Modification Methods + +These methods modify the signal and return a new modified signal object (fluent interface): + +| Method | Signature | Description | +|--------|-----------|-------------| +| `withTopK` | `signal.withTopK(limit: number = 25) -> Signal` | Wrap expression with topk() | +| `withExprWrappersMixin` | `signal.withExprWrappersMixin(wrapper: string[]) -> Signal` | Add expression wrapper | +| `withOffset` | `signal.withOffset(offset: string) -> Signal` | Add offset to expression | +| `withFilteringSelectorMixin` | `signal.withFilteringSelectorMixin(mixin: string) -> Signal` | Add filtering selector | +| `withQuantile` | `signal.withQuantile(quantile: number = 0.95) -> Signal` | Set quantile for histograms | + +### Rule Generation Methods + +These methods return Prometheus-specific expressions: + +| Method | Signature | Description | +|--------|-----------|-------------| +| `asRuleExpression` | `signal.asRuleExpression() -> string` | Prometheus rule expression (no Grafana vars) | + +## Grafana Methods (.grafana namespace) + +### Panel Rendering Methods + +These methods return complete Grafana panel objects: + +| Method | Signature | Description | +|--------|-----------|-------------| +| `asTimeSeries` | `signal.grafana.asTimeSeries() -> Panel` | Render as Grafana TimeSeries panel | +| `asStat` | `signal.grafana.asStat() -> Panel` | Render as Grafana Stat panel | +| `asGauge` | `signal.grafana.asGauge() -> Panel` | Render as Grafana Gauge panel | +| `asStatusHistory` | `signal.grafana.asStatusHistory() -> Panel` | Render as Grafana StatusHistory panel | +| `asTable` | `signal.grafana.asTable(format: string = "table") -> Panel` | Render as Grafana Table panel | + +### Panel Composition Methods + +These methods return objects that can be mixed into existing Grafana panels: + +| Method | Signature | Description | +|--------|-----------|-------------| +| `asPanelMixin` | `signal.grafana.asPanelMixin() -> {targets: Target[], overrides: Override[]}` | Panel mixin with targets and overrides | +| `asTableColumn` | `signal.grafana.asTableColumn(format: string = "table") -> {targets: Target[], overrides: Override[]}` | Table column mixin | +| `asTarget` | `signal.grafana.asTarget() -> Target` | Query target object | +| `asTableTarget` | `signal.grafana.asTableTarget() -> Target` | Table-formatted target | +| `asOverride` | `signal.grafana.asOverride() -> Override` | Panel field override | + +### Expression Methods + +These methods return Grafana-specific expressions: + +| Method | Signature | Description | +|--------|-----------|-------------| +| `asPanelExpression` | `signal.grafana.asPanelExpression() -> string` | Panel query expression (with Grafana variables) | + +### Utility Methods + +Static/class methods for working with collections of signals in Grafana: + +| Method | Signature | Description | +|--------|-----------|-------------| +| `getVariablesMultiChoice` | `Signal.grafana.getVariablesMultiChoice(signals: Signal[]) -> Variable[]` | Generate Grafana dashboard variables | + +## Expression Transformation Rules + +All implementations must follow these automatic expression transformation rules: + +1. **Counter signals**: Wrap with `([])` +2. **Histogram signals**: Wrap with `histogram_quantile(, (rate([])) by (le,))` +3. **Gauge/Info/Raw signals**: No automatic transformation +4. **Aggregation**: When `aggLevel` is `group` or `instance`, wrap with ` by () ()` + +## Variable Template Expansion + +Support these template variables in expressions: + +- `%(queriesSelector)s` - filteringSelector + instanceLabels + groupLabels matchers +- `%(filteringSelector)s` - filteringSelector matchers only +- `%(groupLabels)s` - groupLabels list +- `%(instanceLabels)s` - instanceLabels list +- `%(agg)s` - aggregation labels based on aggLevel + aggKeepLabels +- `%(aggLegend)s` - aggregation labels in legend format `{{label}}` +- `%(aggFunction)s` - aggregation function name +- `%(interval)s` - interval value +- `%(alertsInterval)s` - alerts interval value + + +## Builder Pattern Flow + +The approach enables clear separation and fluent chaining: + +```typescript +// Example workflow +const baseSignal = new Signal(config); + +// Build modifications on root signal - returns new Signal for chaining +const modifiedSignal = baseSignal + .withTopK(10) + .withOffset("5m") + .withQuantile(0.99); + +// Generate outputs for different systems +const prometheusRule = modifiedSignal.asRuleExpression(); +const grafanaPanel = modifiedSignal.grafana.asTimeSeries(); +const grafanaTarget = modifiedSignal.grafana.asTarget(); + +// Original signal unmodified for different use +const basicPanel = baseSignal.grafana.asStat(); + +// Single-line builder pattern examples +const quickPanel = baseSignal.withTopK(10).withOffset("5m").grafana.asTimeSeries(); +const compositePanel = baseSignal.withQuantile(0.95).withFilteringSelectorMixin('env="prod"').grafana.asStat(); +``` + +## Testing Requirements + +Each language implementation must include: + +1. **Unit tests** for all methods in each namespace +2. **Integration tests** with sample signal configurations +3. **Expression transformation tests** for all signal types +4. **Template expansion tests** for both Grafana and Prometheus contexts +5. **Panel generation tests** comparing Grafana output structure +6. **Rule generation tests** comparing Prometheus output structure +7. **Namespace isolation tests** ensuring modifications don't affect other namespaces + +## Code Generation + +For consistent implementation across languages: + +1. Use the OpenAPI spec (`signal_interface.yaml`) to generate client stubs +2. Implement the actual logic following the transformation rules above +3. Ensure method signatures match exactly across all languages +4. Use consistent naming conventions (camelCase, snake_case, PascalCase per language standards) +5. Organize methods with core signal modifications on root object and Grafana methods in `.grafana` namespace +6. Ensure isolation - modifications create new signal instances and don't affect the original signal + +## Implementation Requirements by Language + +### Jsonnet +```jsonnet +// Constructor +local signals = signal.init(config); +signals.addSignal(name, type, ...) + +// Root signal methods (formerly prom namespace) +local signalMethods = { + withTopK(limit=25):: { /* Returns new signal with modified config */ }, + withOffset(offset):: { /* Returns new signal with modified config */ }, + withQuantile(quantile=0.95):: { /* Returns new signal with modified config */ }, + withExprWrappersMixin(wrapper):: { /* Returns new signal with modified config */ }, + withFilteringSelectorMixin(mixin):: { /* Returns new signal with modified config */ }, + + // Prometheus-specific output + asRuleExpression():: { /* Returns rule expression string */ }, +}; + +// Grafana namespace methods (unchanged) +local grafanaNamespace = { + asTimeSeries():: { /* Returns panel object */ }, + asStat():: { /* Returns panel object */ }, + asGauge():: { /* Returns panel object */ }, + asTable(format='table'):: { /* Returns panel object */ }, + asPanelMixin():: { /* Returns mixin object */ }, + asTarget():: { /* Returns target object */ }, + asPanelExpression():: { /* Returns expression string */ }, +}; + +// Usage with fluent chaining +local modifiedSignal = signals.mySignal.withTopK(10).withOffset("5m"); + +// Use modified signal for Prometheus rules +local rule = modifiedSignal.asRuleExpression(); + +// Use modified signal for Grafana panels +local panel = modifiedSignal.grafana.asTimeSeries(); + +// Or use original signal directly +local originalPanel = signals.mySignal.grafana.asTimeSeries(); + +// Builder pattern: Chain modifications with grafana rendering +local quickPanel = signals.mySignal.withTopK(10).withOffset("5m").grafana.asTimeSeries(); +local statPanel = signals.cpuSignal.withQuantile(0.99).grafana.asStat(); +local mixedPanel = signals.memorySignal.withExprWrappersMixin(['sum(', ')']).grafana.asPanelMixin(); +local tablePanel = signals.diskSignal.withFilteringSelectorMixin('env="prod"').grafana.asTable(); +``` + +### TypeScript +```typescript +// Class-based approach with Grafana namespace +class Signal { + constructor(config: SignalConfig) { + this.config = config; + this.grafana = new GrafanaNamespace(this); + } + + // Root methods (formerly prom namespace) + withTopK(limit = 25): Signal { + return new Signal({...this.config, topK: limit}); + } + + withOffset(offset: string): Signal { + return new Signal({...this.config, offset}); + } + + withQuantile(quantile = 0.95): Signal { + return new Signal({...this.config, quantile}); + } + + withExprWrappersMixin(wrapper: string[]): Signal { + return new Signal({...this.config, exprWrappers: [...(this.config.exprWrappers || []), wrapper]}); + } + + withFilteringSelectorMixin(mixin: string): Signal { + return new Signal({...this.config, filteringSelector: mixin}); + } + + asRuleExpression(): string { + // Return Prometheus rule expression + return ""; + } + + // Grafana namespace + public grafana: GrafanaNamespace; + private config: SignalConfig; +} + +class GrafanaNamespace { + constructor(private signal: Signal) {} + + asTimeSeries(): Panel { ... } + asPanelMixin(): {targets: Target[], overrides: Override[]} { ... } + asTarget(): Target { ... } + asPanelExpression(): string { ... } + + static getVariablesMultiChoice(signals: Signal[]): Variable[] { ... } +} + +// Usage with fluent chaining on root signal +const signal = new Signal(config); +const modifiedSignal = signal.withTopK(10).withOffset("5m"); + +// Use modified signal for Prometheus rules +const rule = modifiedSignal.asRuleExpression(); + +// Use modified signal for Grafana panels +const panel = modifiedSignal.grafana.asTimeSeries(); + +// Or use original signal directly +const originalPanel = signal.grafana.asTimeSeries(); + +// Builder pattern: Chain modifications with grafana rendering +const quickPanel = signal.withTopK(10).withOffset("5m").grafana.asTimeSeries(); +const statPanel = cpuSignal.withQuantile(0.99).grafana.asStat(); +const targetPanel = memorySignal.withExprWrappersMixin(['avg(', ')']).grafana.asTarget(); +const compositePanel = basePanel.withFilteringSelectorMixin('env="prod"').grafana.asPanelMixin(); +const tablePanel = diskSignal.withTopK(5).withOffset("1h").grafana.asTable("time_series"); +``` + +### Python +```python +# Class-based approach with Grafana namespace +class Signal: + def __init__(self, config: SignalConfig): + self.config = config + self.grafana = GrafanaNamespace(self) + + # Root methods (formerly prom namespace) + def with_top_k(self, limit: int = 25) -> 'Signal': + new_config = {**self.config, 'top_k': limit} + return Signal(new_config) + + def with_offset(self, offset: str) -> 'Signal': + new_config = {**self.config, 'offset': offset} + return Signal(new_config) + + def with_quantile(self, quantile: float = 0.95) -> 'Signal': + new_config = {**self.config, 'quantile': quantile} + return Signal(new_config) + + def with_expr_wrappers_mixin(self, wrapper: list[str]) -> 'Signal': + existing_wrappers = self.config.get('expr_wrappers', []) + new_config = {**self.config, 'expr_wrappers': existing_wrappers + [wrapper]} + return Signal(new_config) + + def with_filtering_selector_mixin(self, mixin: str) -> 'Signal': + new_config = {**self.config, 'filtering_selector': mixin} + return Signal(new_config) + + def as_rule_expression(self) -> str: + # Return Prometheus rule expression + return "" + +class GrafanaNamespace: + def __init__(self, signal: 'Signal'): + self._signal = signal + + def as_time_series(self) -> Panel: ... + def as_panel_mixin(self) -> dict: ... + def as_target(self) -> Target: ... + def as_panel_expression(self) -> str: ... + + @staticmethod + def get_variables_multi_choice(signals: list['Signal']) -> list[Variable]: ... + +# Usage with fluent chaining on root signal +signal = Signal(config) +modified_signal = signal.with_top_k(10).with_offset("5m") + +# Use modified signal for Prometheus rules +rule = modified_signal.as_rule_expression() + +# Use modified signal for Grafana panels +panel = modified_signal.grafana.as_time_series() + +# Or use original signal directly +original_panel = signal.grafana.as_time_series() + +# Builder pattern: Chain modifications with grafana rendering +quick_panel = signal.with_top_k(10).with_offset("5m").grafana.as_time_series() +stat_panel = cpu_signal.with_quantile(0.99).grafana.as_stat() +target_panel = memory_signal.with_expr_wrappers_mixin(['sum(', ')']).grafana.as_target() +gauge_panel = disk_signal.with_filtering_selector_mixin('instance="server1"').grafana.as_gauge() +table_panel = network_signal.with_top_k(5).with_offset("1h").grafana.as_table("time_series") +``` + +### Go +```go +// Struct with Grafana namespace +type Signal struct { + Config SignalConfig + Grafana *GrafanaNamespace +} + +func NewSignal(config SignalConfig) *Signal { + s := &Signal{Config: config} + s.Grafana = &GrafanaNamespace{signal: s} + return s +} + +// Root methods (formerly prom namespace) +func (s *Signal) WithTopK(limit int) *Signal { + newConfig := s.Config + newConfig.TopK = limit + return NewSignal(newConfig) +} + +func (s *Signal) WithOffset(offset string) *Signal { + newConfig := s.Config + newConfig.Offset = offset + return NewSignal(newConfig) +} + +func (s *Signal) WithQuantile(quantile float64) *Signal { + newConfig := s.Config + newConfig.Quantile = quantile + return NewSignal(newConfig) +} + +func (s *Signal) WithExprWrappersMixin(wrapper []string) *Signal { + newConfig := s.Config + newConfig.ExprWrappers = append(newConfig.ExprWrappers, wrapper) + return NewSignal(newConfig) +} + +func (s *Signal) WithFilteringSelectorMixin(mixin string) *Signal { + newConfig := s.Config + newConfig.FilteringSelector = mixin + return NewSignal(newConfig) +} + +func (s *Signal) AsRuleExpression() string { + // Return Prometheus rule expression + return "" +} + +type GrafanaNamespace struct { + signal *Signal +} + +func (g *GrafanaNamespace) AsTimeSeries() *Panel { ... } +func (g *GrafanaNamespace) AsPanelMixin() *PanelMixin { ... } +func (g *GrafanaNamespace) AsTarget() *Target { ... } +func (g *GrafanaNamespace) AsPanelExpression() string { ... } + +func GetVariablesMultiChoice(signals []*Signal) []*Variable { ... } + +// Usage with fluent chaining on root signal +signal := NewSignal(config) +modifiedSignal := signal.WithTopK(10).WithOffset("5m") + +// Use modified signal for Prometheus rules +rule := modifiedSignal.AsRuleExpression() + +// Use modified signal for Grafana panels +panel := modifiedSignal.Grafana.AsTimeSeries() + +// Or use original signal directly +originalPanel := signal.Grafana.AsTimeSeries() + +// Builder pattern: Chain modifications with grafana rendering +quickPanel := signal.WithTopK(10).WithOffset("5m").Grafana.AsTimeSeries() +statPanel := cpuSignal.WithQuantile(0.99).Grafana.AsStat() +targetPanel := memorySignal.WithExprWrappersMixin([]string{"sum(", ")"}).Grafana.AsTarget() +gaugePanel := diskSignal.WithFilteringSelectorMixin(`instance="server1"`).Grafana.AsGauge() +tablePanel := networkSignal.WithTopK(5).WithOffset("1h").Grafana.AsTable("time_series") +``` \ No newline at end of file diff --git a/signals/schema/signal_source.cue b/signals/schema/signal_source.cue new file mode 100644 index 000000000..09c838f8f --- /dev/null +++ b/signals/schema/signal_source.cue @@ -0,0 +1,61 @@ +package signal + +// Signal source configuration - defines how to query and transform a signal +// Each signal can have multiple sources (prometheus, opentelemetry, etc.) +#signalSource: close({ + // Base PromQL/LogQL expression in simplest form + // Supports templating variables like %(queriesSelector)s, %(groupLabels)s + // Automatically transformed based on signal type (counter, histogram, etc.) + // Example: "node_cpu_seconds_total{%(queriesSelector)s}" + expr: string + + // Additional wrapper functions applied AFTER auto-transformations + // Array of [left_part, right_part] pairs that wrap the expression + // Example: [["topk(10,", ")"]] wraps expression as topk(10, ) + exprWrappers?: [[string, string]] + + // Rate function to use for counter signal types + // Applied as: ([]) + // - "rate": average rate of increase per second + // - "irate": instantaneous rate based on last two data points + // - "delta": difference between first and last value + // - "idelta": instantaneous delta based on last two data points + // - "increase": total increase over time range + rangeFunction?: "rate" | "irate" | "delta" | "idelta" | "increase" + + // Override aggregation function for this specific source + // Takes precedence over signal and signal group defaults + aggFunction?: #aggFunction + + // Extra labels to keep when aggregating with by() clause + // Used in conjunction with aggLevel to control aggregation scope + // Example: ["pool", "level"] keeps these labels during aggregation + aggKeepLabels?: [...string] + + // Label name used to extract info from "info" type signals + // Only applicable when signal type is "info" + // Example: "version" extracts version info from info metric + infoLabel?: string + + // Custom legend template to override automatic legend generation + // Supports Grafana templating like {{instance}}, {{job}} + // Takes precedence over signal and signal group legend templates + legendCustomTemplate?: string + + // Grafana value mappings for transforming raw values to display text/colors + // Follows Grafana dashboard schema format for value mappings + // Example: map 0="Down"/red, 1="Up"/green for status signals + valueMappings?: [...{ + Type: string + Options: {[string]: { + Text: string // Display text for the value + Color: string // Color name or hex code + Index: int // Sort order in legend + }} + }] + + // Quantile value for histogram signal types (0.0 to 1.0) + // Used in histogram_quantile() function for histogram transformations + // Example: 0.95 for 95th percentile, 0.99 for 99th percentile + quantile?: number & >=0 & <=1 +}) \ No newline at end of file diff --git a/signals/schema/tests/test_signal_1_bad.json b/signals/schema/tests/test_signal_1_bad.json new file mode 100644 index 000000000..5ed79f666 --- /dev/null +++ b/signals/schema/tests/test_signal_1_bad.json @@ -0,0 +1,39 @@ +{ + "signalGroup": + [ + { + "discoveryMetric": { + "prometheus": "up", + "loki": "up" + }, + "signals": { + "signal1": + { + "name": "http_requests_total", + "type": "BADcounter", + "unit": "requests", + "description": "Total number of HTTP requests", + "optional": false, + "aggLevel": "instance", + "aggFunction": "sum", + "sources": { + "prometheus": { + "expr": "sum(rate(http_requests_total[5m]))", + "rangeFunction": "rate", + "aggFunction": "badfunction", + "infoLabel": "HTTP Requests Rate", + "legendCustomTemplate": "{{method}} {{path}}", + "quantile": 1 + }, + "loki": { + "expr": "sum(count_over_time({job=\"apiserver\"} |~ \"GET /api/v1/.*\" [5m]))", + "rangeFunction": "rate", + "aggFunction": "sum", + "infoLabel": "API Requests Rate" + } + } + } + } + } + ] +} diff --git a/signals/schema/tests/test_signal_1_ok.json b/signals/schema/tests/test_signal_1_ok.json new file mode 100644 index 000000000..ed0163e71 --- /dev/null +++ b/signals/schema/tests/test_signal_1_ok.json @@ -0,0 +1,39 @@ +{ + "signalGroup": + [ + { + "discoveryMetric": { + "prometheus": "up", + "loki": "up" + }, + "signals": { + "signal1": + { + "name": "http_requests_total", + "type": "counter", + "unit": "requests", + "description": "Total number of HTTP requests", + "optional": false, + "aggLevel": "instance", + "aggFunction": "sum", + "sources": { + "prometheus": { + "expr": "sum(rate(http_requests_total[5m]))", + "rangeFunction": "rate", + "aggFunction": "sum", + "infoLabel": "HTTP Requests Rate", + "legendCustomTemplate": "{{method}} {{path}}", + "quantile": 1 + }, + "loki": { + "expr": "sum(count_over_time({job=\"apiserver\"} |~ \"GET /api/v1/.*\" [5m]))", + "rangeFunction": "rate", + "aggFunction": "sum", + "infoLabel": "API Requests Rate" + } + } + } + } + } + ] +} diff --git a/signals/schema/tests/test_signal_golang_ok.json b/signals/schema/tests/test_signal_golang_ok.json new file mode 100644 index 000000000..2a57795e2 --- /dev/null +++ b/signals/schema/tests/test_signal_golang_ok.json @@ -0,0 +1,333 @@ +{ + "signalGroup": + [ + { + "aggLevel": "instance", + "alertsInterval": "5m", + "discoveryMetric": { + "otel": "process_runtime_go_goroutines", + "otel_with_suffixes": "process_runtime_go_goroutines", + "prometheus": "go_info" + }, + "signals": { + "cgoCalls": { + "description": "Number of cgo calls made by the current process", + "name": "CGo calls", + "sources": { + "otel": { + "expr": "process_runtime_go_cgo_calls{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_cgo_calls{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_cgo_go_to_c_calls_calls_total{%(queriesSelector)s}" + } + }, + "type": "counter" + }, + "gcDurationMax": { + "description": "Maximum amount of time in GC stop-the-world pauses.\nDuring a stop-the-world pause, all goroutines are paused and only the garbage collector can run.\n", + "name": "GC duration (max)", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_gc_duration_seconds{%(queriesSelector)s, quantile=\"1\"}", + "exprWrappers": [ + [ + "", + "*10^6" + ] + ] + } + }, + "type": "gauge", + "unit": "ns" + }, + "gcDurationMin": { + "description": "Minimum amount of time in GC stop-the-world pauses. \nDuring a stop-the-world pause, all goroutines are paused and only the garbage collector can run.\n", + "name": "GC duration (min)", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_gc_duration_seconds{%(queriesSelector)s, quantile=\"0\"}", + "exprWrappers": [ + [ + "", + "*10^6" + ] + ] + } + }, + "type": "gauge", + "unit": "ns" + }, + "gcDurationPercentile": { + "description": "Amount of time in GC stop-the-world pauses (95th percentile).\nDuring a stop-the-world pause, all goroutines are paused and only the garbage collector can run.\n", + "name": "GC duration", + "optional": true, + "sources": { + "otel": { + "expr": "process_runtime_go_gc_pause_ns_bucket{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_gc_pause_ns_bucket{%(queriesSelector)s}" + } + }, + "type": "histogram", + "unit": "ns" + }, + "goRoutines": { + "description": "Number of goroutines.", + "name": "Goroutines", + "sources": { + "otel": { + "expr": "process_runtime_go_goroutines{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_goroutines{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_goroutines{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "short" + }, + "goThreads": { + "description": "System threads used.\n", + "name": "Threads used", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_threads{%(queriesSelector)s}" + } + }, + "type": "gauge" + }, + "memBuckHash": { + "description": "Memory reserved by mcache", + "name": "Memory buck hash", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_buck_hash_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memGc": { + "description": "Memory reserved by gc metadata", + "name": "Memory gc metadata", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_gc_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memHeapAllocatedObjects": { + "description": "Number of allocated heap objects.\nThis changes as GC is performed and new objects are allocated.\n", + "name": "Heap allocated objects", + "sources": { + "otel": { + "expr": "process_runtime_go_mem_heap_objects{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_mem_heap_objects{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_memstats_heap_objects{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "short" + }, + "memHeapIdleBytes": { + "description": "Bytes in idle (unused) spans.\n", + "name": "Heap idle spans, bytes", + "sources": { + "otel": { + "expr": "process_runtime_go_mem_heap_idle{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_mem_heap_idle_bytes{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_memstats_heap_idle_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memHeapInUseBytes": { + "description": "Bytes in in-use spans.\n", + "name": "Heap in-use spans, bytes", + "sources": { + "otel": { + "expr": "process_runtime_go_mem_heap_inuse{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_mem_heap_inuse_bytes{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_memstats_heap_inuse_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memHeapObjBytes": { + "description": "Bytes of allocated heap objects.\n", + "name": "Heap objects, bytes", + "optional": true, + "sources": { + "otel": { + "expr": "process_runtime_go_mem_heap_alloc{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_mem_heap_alloc_bytes{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_memstats_heap_alloc_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memHeapReleasedBytes": { + "description": "Bytes of idle spans whose physical memory has been returned to the OS.\n", + "name": "Heap released, bytes", + "sources": { + "otel": { + "expr": "process_runtime_go_mem_heap_released{%(queriesSelector)s}" + }, + "otel_with_suffixes": { + "expr": "process_runtime_go_mem_heap_released_bytes{%(queriesSelector)s}" + }, + "prometheus": { + "expr": "go_memstats_heap_released_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memHeapReserved": { + "description": "Memory reserved from system by heap.\n", + "name": "Memory reserved from system by heap", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_heap_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memMcache": { + "description": "Memory reserved by mcache", + "name": "Memory mcache", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_mcache_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memMspan": { + "description": "Memory reserved by mspan", + "name": "Memory mspan", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_mspan_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memOther": { + "description": "Memory reserved for other needs", + "name": "Memory other", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_other_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memReserved": { + "description": "Memory reserved from system.\n", + "name": "Memory reserved from system", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memStack": { + "description": "Memory reserved by stack.\n", + "name": "Memory reserved by stack", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_stack_sys_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "memTotal": { + "name": "Memory allocated from system", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_memstats_alloc_bytes{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "uptime": { + "description": "Golang process uptime.", + "name": "Uptime", + "sources": { + "otel": { + "expr": "runtime_uptime{%(queriesSelector)s}/1000" + }, + "otel_with_suffixes": { + "expr": "runtime_uptime_milliseconds_total{%(queriesSelector)s}/1000" + }, + "prometheus": { + "expr": "time()-process_start_time_seconds{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "dtdurations" + }, + "version": { + "description": "Golang version used.", + "name": "Golang version", + "optional": true, + "sources": { + "prometheus": { + "expr": "go_info{%(queriesSelector)s}", + "infoLabel": "version" + } + }, + "type": "info" + } + } + } + +]} \ No newline at end of file diff --git a/signals/schema/tests/test_signal_kafka_topic_ok.json b/signals/schema/tests/test_signal_kafka_topic_ok.json new file mode 100644 index 000000000..2c85ee596 --- /dev/null +++ b/signals/schema/tests/test_signal_kafka_topic_ok.json @@ -0,0 +1,219 @@ +{ + "signalGroup": + [ + { + "aggFunction": "sum", + "aggLevel": "group", + "discoveryMetric": { + "bitnami": "kafka_log_log_logstartoffset", + "grafanacloud": "kafka_log_log_logstartoffset", + "prometheus": "kafka_log_log_logstartoffset" + }, + "legendCustomTemplate": "{{ topic }}", + "signals": { + "topicBytesInPerSec": { + "description": "Topic bytes in rate.", + "name": "Topic bytes in", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_server_brokertopicmetrics_bytesinpersec_count{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_server_brokertopicmetrics_bytesinpersec{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_server_brokertopicmetrics_bytesin_total{%(queriesSelector)s}" + } + }, + "type": "counter", + "unit": "Bps" + }, + "topicBytesOutPerSec": { + "description": "Topic bytes out rate.", + "name": "Topic bytes out", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_server_brokertopicmetrics_bytesoutpersec_count{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_server_brokertopicmetrics_bytesoutpersec{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_server_brokertopicmetrics_bytesout_total{%(queriesSelector)s}" + } + }, + "type": "counter", + "unit": "Bps" + }, + "topicLogEndOffset": { + "aggFunction": "max", + "description": "Topic end offset.", + "legendCustomTemplate": "{{ topic }}/{{ partition }}", + "name": "Topic end offset", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_logendoffset{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_logendoffset{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_logendoffset{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "none" + }, + "topicLogSize": { + "aggFunction": "max", + "description": "Size in bytes of the current topic-partition.", + "legendCustomTemplate": "{{ topic }}/{{ partition }}", + "name": "Topic log size", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_size{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_size{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_size{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "decbytes" + }, + "topicLogStartOffset": { + "aggFunction": "max", + "description": "Topic start offset.", + "legendCustomTemplate": "{{ topic }}/{{ partition }}", + "name": "Topic start offset", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_logstartoffset{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_logstartoffset{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_log_log_logstartoffset{%(queriesSelector)s}" + } + }, + "type": "gauge", + "unit": "none" + }, + "topicMessagesPerSec": { + "description": "Messages in per second.", + "name": "Messages in per second", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_topic_partition_current_offset{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_topic_partition_current_offset{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic" + ], + "expr": "kafka_topic_partition_current_offset{%(queriesSelector)s}" + } + }, + "type": "counter", + "unit": "mps" + }, + "topicMessagesPerSecByPartition": { + "description": "Messages in per second.", + "legendCustomTemplate": "{{ topic }}/{{ partition }}", + "name": "Messages in per second", + "sources": { + "bitnami": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_topic_partition_current_offset{%(queriesSelector)s}" + }, + "grafanacloud": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_topic_partition_current_offset{%(queriesSelector)s}" + }, + "prometheus": { + "aggKeepLabels": [ + "topic", + "partition" + ], + "expr": "kafka_topic_partition_current_offset{%(queriesSelector)s}" + } + }, + "type": "counter", + "unit": "mps" + } + } + } + + ] +} diff --git a/signals/todo.md b/signals/todo.md new file mode 100644 index 000000000..0eff94c43 --- /dev/null +++ b/signals/todo.md @@ -0,0 +1,4 @@ +# Todo + +- Define schema for signals (signal, signal source, etc.). +- (Additional tasks can be added here.) \ No newline at end of file