chore(helm,otel): update OTEL deployment (#1338)

pinglin · web-flow · commit ba2c81cd841b · 2025-08-01T22:27:35.000+01:00
Because - the `core` Helm chart needs to update on the OpenTelemetry setup. This commit - aligns Docker Compose OpenTelemetry configuration > [!NOTE] > INS-8333 (PR #1337) is based on this PR feature branch.
diff --git a/.env b/.env
@@ -210,7 +210,7 @@ MINIO_CONSOLE_HOST_PORT=19001
 
 # OpenTelemetry Collector
 OTEL_COLLECTOR_IMAGE=otel/opentelemetry-collector-contrib
-OTEL_COLLECTOR_VERSION=0.128.0
+OTEL_COLLECTOR_VERSION=0.129.0
 OTEL_COLLECTOR_HOST=otel-collector
 OTEL_COLLECTOR_PORT=4317
 OTEL_COLLECTOR_PROMETHEUS_PORT=9001
diff --git a/Makefile.helper b/Makefile.helper
@@ -32,10 +32,11 @@ define HELM
 	@helm install ${HELM_RELEASE_NAME} charts/core \
 		--namespace ${HELM_NAMESPACE} --create-namespace \
 		--set edition=$(if $(filter true,$(1)),k8s-ce:test,k8s-ce) \
-		--set mgmtBackend.instillCoreHost=http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT} \
-		--set artifactBackend.instillCoreHost=http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT} \
+		--set tags.observability=$(OBSERVE_ENABLED) \
 		--set pipelineBackend.instillCoreHost=http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT} \
+		--set artifactBackend.instillCoreHost=http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT} \
 		--set modelBackend.instillCoreHost=http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT} \
+		--set mgmtBackend.instillCoreHost=http://${INSTILL_CORE_HOST}:${API_GATEWAY_PORT} \
 		$(if $(filter true,$(1)),--set ray-cluster.head.autoscaling.enableInTreeAutoscaling=false,) \
 		$(if $(filter true,$(1)),--set ray-cluster.head.resources.requests.cpu=0,) \
 		$(if $(filter true,$(1)),--set ray-cluster.head.resources.requests.memory=0,) \
diff --git a/charts/core/Chart.lock b/charts/core/Chart.lock
@@ -16,9 +16,9 @@ dependencies:
   version: 8.5.1
 - name: opentelemetry-collector
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
-  version: 0.119.0
+  version: 0.129.0
 - name: kube-prometheus-stack
   repository: https://prometheus-community.github.io/helm-charts
   version: 70.3.0
-digest: sha256:d5ac0a0622d650e502e16aa7945f82ded2cfe62669792cb4dd9ccc8b900d87a9
-generated: "2025-07-27T00:27:03.685644+01:00"
+digest: sha256:0a00b79854d8ea609ed566b50e1ba1ad24fa6edd4e5d5701a14133f4a9533c9f
+generated: "2025-07-30T00:52:08.163458+01:00"
diff --git a/charts/core/Chart.yaml b/charts/core/Chart.yaml
@@ -40,9 +40,11 @@ dependencies:
     condition: elasticsearch.enabled
   - name: opentelemetry-collector
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
-    version: 0.119.0
-    condition: opentelemetry-collector.enabled
+    version: 0.129.0
+    tags:
+      - observability
   - name: kube-prometheus-stack
     repository: https://prometheus-community.github.io/helm-charts
     version: 70.3.0
-    condition: kube-prometheus-stack.enabled
+    tags:
+      - observability
diff --git a/charts/core/README.md b/charts/core/README.md
@@ -10,7 +10,7 @@ The Helm chart of Instill Core
 |------------|------|---------|
 | https://helm.elastic.co | elasticsearch | 8.5.1 |
 | https://helm.influxdata.com | influxdb2 | 2.1.1 |
-| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.119.0 |
+| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.129.0 |
 | https://prometheus-community.github.io/helm-charts | kube-prometheus-stack | 70.3.0 |
 | https://ray-project.github.io/kuberay-helm | kuberay-operator | 1.3.2 |
 | https://ray-project.github.io/kuberay-helm | ray-cluster | 1.3.2 |
diff --git a/charts/core/templates/_helpers.tpl b/charts/core/templates/_helpers.tpl
@@ -413,7 +413,7 @@ OpenTelemetry
 {{- end -}}
 
 {{- define "core.otel.port" -}}
-  {{- printf "8095" -}}
+  {{- printf "4317" -}}
 {{- end -}}
 
 {{/*
diff --git a/charts/core/templates/api-gateway/configmap.yaml b/charts/core/templates/api-gateway/configmap.yaml
@@ -4,8 +4,18 @@ metadata:
   name: {{ include "core.apiGateway" . }}
 data:
   .env: |-
+
     LOG_LEVEL={{ upper .Values.logLevel }}
 
+    # Enable observability stack
+    OBSERVE_ENABLED={{ .Values.tags.observability }}
+
+    # Service version
+    SERVICE_VERSION={{ .Values.apiGateway.image.tag }}
+
+    # Deploy environment
+    DEPLOY_ENV={{ .Values.edition }}
+
     # api-gateway
     API_GATEWAY_HOST={{ include "core.apiGateway" . }}
     API_GATEWAY_PORT={{ include "core.apiGateway.httpPort" . }}
diff --git a/charts/core/templates/api-gateway/deployment.yaml b/charts/core/templates/api-gateway/deployment.yaml
@@ -96,15 +96,16 @@ spec:
               path: /__health
               scheme: {{ upper (ternary "https" "http" .Values.apiGateway.tls.enabled) }}
               port: {{ ternary "https" "http" .Values.apiGateway.tls.enabled }}
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
           livenessProbe:
             httpGet:
               path: /__health
               scheme: {{ upper (ternary "https" "http" .Values.apiGateway.tls.enabled) }}
               port: {{ ternary "https" "http" .Values.apiGateway.tls.enabled }}
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           {{- if .Values.apiGateway.resources }}
           resources:
             {{- toYaml .Values.apiGateway.resources | nindent 12 }}
diff --git a/charts/core/templates/artifact-backend/configmap.yaml b/charts/core/templates/artifact-backend/configmap.yaml
@@ -53,11 +53,10 @@ data:
       https:
         cert:
         key:
-    log:
-      external: false
-      otelcollector:
-        host: {{ include "core.otel" . }}
-        port: {{ include "core.otel.port" . }}
+    otelcollector:
+      enable: {{ .Values.tags.observability }}
+      host: {{ include "core.otel" . }}
+      port: {{ include "core.otel.port" . }}
     openfga:
       host: {{ include "core.openfga" . }}
       port: 8081
diff --git a/charts/core/templates/artifact-backend/deployment.yaml b/charts/core/templates/artifact-backend/deployment.yaml
@@ -117,13 +117,17 @@ spec:
               path: /v1alpha/__readiness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           livenessProbe:
             httpGet:
               path: /v1alpha/__liveness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           {{- if .Values.artifactBackend.resources }}
           resources:
             {{- toYaml .Values.artifactBackend.resources | nindent 12 }}
diff --git a/charts/core/templates/mgmt-backend/configmap.yaml b/charts/core/templates/mgmt-backend/configmap.yaml
@@ -60,11 +60,10 @@ data:
       https:
         cert:
         key:
-    log:
-      external: false
-      otelcollector:
-        host: {{ include "core.otel" . }}
-        port: {{ include "core.otel.port" . }}
+    otelcollector:
+      enable: {{ .Values.tags.observability }}
+      host: {{ include "core.otel" . }}
+      port: {{ include "core.otel.port" . }}
     openfga:
       host: {{ include "core.openfga" . }}
       port: 8080
diff --git a/charts/core/templates/mgmt-backend/deployment.yaml b/charts/core/templates/mgmt-backend/deployment.yaml
@@ -134,15 +134,17 @@ spec:
               path: /v1beta/__readiness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           livenessProbe:
             httpGet:
               path: /v1beta/__liveness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           {{- if .Values.mgmtBackend.resources }}
           resources:
             {{- toYaml .Values.mgmtBackend.resources | nindent 12 }}
diff --git a/charts/core/templates/model-backend/configmap.yaml b/charts/core/templates/model-backend/configmap.yaml
@@ -26,11 +26,10 @@ data:
       {{- end }}
       instillcorehost: {{ .Values.modelBackend.instillCoreHost }}
       taskschemaversion: {{ .Values.modelBackend.taskSchemaVersion }}
-    log:
-      external: false
-      otelcollector:
-        host: {{ include "core.otel" . }}
-        port: {{ include "core.otel.port" . }}
+    otelcollector:
+      enable: {{ .Values.tags.observability }}
+      host: {{ include "core.otel" . }}
+      port: {{ include "core.otel.port" . }}
     mgmtbackend:
       host: {{ include "core.mgmtBackend" . }}
       publicport: {{ include "core.mgmtBackend.publicPort" . }}
diff --git a/charts/core/templates/model-backend/deployment.yaml b/charts/core/templates/model-backend/deployment.yaml
@@ -158,15 +158,17 @@ spec:
               path: /v1alpha/__readiness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           livenessProbe:
             httpGet:
               path: /v1alpha/__liveness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           {{- if .Values.modelBackend.resources }}
           resources:
             {{- toYaml .Values.modelBackend.resources | nindent 12 }}
diff --git a/charts/core/templates/pipeline-backend/configmap.yaml b/charts/core/templates/pipeline-backend/configmap.yaml
@@ -93,11 +93,10 @@ data:
       clientkey: {{ default "" .Values.pipelineBackend.temporal.clientKey }}
       insecureskipverify: {{ default "true" .Values.pipelineBackend.temporal.insecureSkipVerify }}
       metricsport: {{ include "core.temporal.metrics.port" . }}
-    log:
-      external: false
-      otelcollector:
-        host: {{ include "core.otel" . }}
-        port: {{ include "core.otel.port" . }}
+    otelcollector:
+      enable: {{ .Values.tags.observability }}
+      host: {{ include "core.otel" . }}
+      port: {{ include "core.otel.port" . }}
     openfga:
       host: {{ include "core.openfga" . }}
       port: 8081
diff --git a/charts/core/templates/pipeline-backend/deployment.yaml b/charts/core/templates/pipeline-backend/deployment.yaml
@@ -144,16 +144,17 @@ spec:
               path: /v1beta/__readiness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
-            initialDelaySeconds: 60
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           livenessProbe:
             httpGet:
               path: /v1beta/__liveness
               scheme: {{ ternary "https" "http" .Values.internalTLS.enabled | upper }}
               port: {{ ternary "https" "http" .Values.internalTLS.enabled }}-public
-            periodSeconds: 10
-            initialDelaySeconds: 60
-            timeoutSeconds: 10
+            periodSeconds: 5
+            initialDelaySeconds: 10
+            timeoutSeconds: 5
           {{- if .Values.pipelineBackend.resources }}
           resources:
             {{- toYaml .Values.pipelineBackend.resources | nindent 12 }}
diff --git a/charts/core/values.yaml b/charts/core/values.yaml
@@ -1651,19 +1651,17 @@ elasticsearch:
 opentelemetry-collector:
   enabled: true
   mode: deployment
-  # This is required and recommanded for the upgrade from 0.88.0 to 0.89.0
-  # https://github.com/open-telemetry/opentelemetry-helm-charts/blob/main/charts/opentelemetry-collector/UPGRADING.md#0880-to-0890
   image:
-    repository: "otel/opentelemetry-collector-k8s"
+    repository: "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib"
   ports:
     zipkin:
       enabled: false
     otlp-http:
       enabled: false
     otlp:
       enabled: true
-      containerPort: 8095
-      servicePort: 8095
+      containerPort: 4317
+      servicePort: 4317
     metrics:
       enabled: true
       containerPort: 9001
@@ -1676,7 +1674,7 @@ opentelemetry-collector:
       otlp:
         protocols:
           grpc:
-            endpoint: ${env:MY_POD_IP}:8095
+            endpoint: ${env:MY_POD_IP}:4317
     processors:
       memory_limiter:
         limit_percentage: 50
@@ -1687,36 +1685,32 @@ opentelemetry-collector:
     exporters:
       debug: {}
       otlp:
-        endpoint:
+        endpoint: tempo:4317
         tls:
           insecure: true
+      otlphttp/logs:
+        endpoint: "http://loki:3100/otlp"
+        tls:
+          insecure: true
+      prometheus:
+        endpoint: ${env:MY_POD_IP}:9001
+        namespace: instill-core
+        send_timestamps: true
+        enable_open_metrics: true
     service:
       pipelines:
-        logs:
-          exporters:
-            - debug
-          processors:
-            - memory_limiter
-            - batch
-          receivers:
-            - otlp
-        metrics:
-          exporters:
-            - debug
-          processors:
-            - memory_limiter
-            - batch
-          receivers:
-            - otlp
-            - prometheus
         traces:
-          exporters:
-            - debug
-          processors:
-            - memory_limiter
-            - batch
-          receivers:
-            - otlp
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [debug, otlp]
+        metrics:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [debug, prometheus]
+        logs:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [debug, otlphttp/logs]
 # -- The configuration of Prometheus Stack
 kube-prometheus-stack:
   enabled: true
@@ -1818,3 +1812,5 @@ kube-prometheus-stack:
           enabled: false
       dashboards:
         enabled: true
+tags:
+  observability: false
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -17,6 +17,8 @@ services:
     environment:
       LOG_LEVEL: INFO
       DEPLOY_ENV: ${EDITION}
+      SERVICE_VERSION: ${API_GATEWAY_VERSION}
+      OBSERVE_ENABLED: ${OBSERVE_ENABLED}
       API_GATEWAY_HOST: ${API_GATEWAY_HOST}
       API_GATEWAY_PORT: ${API_GATEWAY_PORT}
       API_GATEWAY_STATS_PORT: ${API_GATEWAY_STATS_PORT}
