Výborné je, že tohle všechno lze v AKS držet v ConfigMap a Kubernetes manifestech. Aplikace pak stále posílá na jeden OTLP endpoint a kolektor rozhoduje, co kam odejde.
Helm šablona OTEL kolektoru v AKS
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "maf-demo.fullname" . }}-otel-collector-config
labels:
{{- include "maf-demo.labels" . | nindent 4 }}
app.kubernetes.io/component: otel-collector
data:
config.yaml: |
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
batch:
timeout: 5s
send_batch_size: 1024
memory_limiter:
check_interval: 1s
limit_mib: 512
# Transform processor for anonymization - strips PII and sensitive data
transform/anonymize:
error_mode: ignore
trace_statements:
- context: span
statements:
# Pseudonymize user.id using SHA256 for consistent hashing
- set(attributes["user.id"], SHA256(attributes["user.id"])) where attributes["user.id"] != nil
# Remove VIP status - boolean flag considered PII
- delete_key(attributes, "user.is_vip")
- delete_key(attributes, "user.roles")
# Pseudonymize department for correlation while protecting identity
- set(attributes["organization.department"], SHA256(attributes["organization.department"])) where attributes["organization.department"] != nil
# Pseudonymize session and thread IDs
- set(attributes["session.id"], SHA256(attributes["session.id"])) where attributes["session.id"] != nil
- set(attributes["thread.id"], SHA256(attributes["thread.id"])) where attributes["thread.id"] != nil
# Strip tool arguments and results - may contain sensitive data
- delete_key(attributes, "tool.arguments")
- delete_key(attributes, "tool.result")
# Strip GenAI input/output messages - contains user queries and LLM responses
- delete_key(attributes, "gen_ai.input.messages")
- delete_key(attributes, "gen_ai.output.messages")
- delete_key(attributes, "gen_ai.prompt")
- delete_key(attributes, "gen_ai.completion")
- delete_key(attributes, "gen_ai.request.model")
- delete_key(attributes, "gen_ai.response.model")
# Strip any other GenAI content using pattern matching
- delete_matching_keys(attributes, "gen_ai\\..*\\.content")
- delete_matching_keys(attributes, "gen_ai\\..*messages.*")
# Redact common PII patterns in remaining string attributes
- replace_all_patterns(attributes, "value", "\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b", "EMAIL_REDACTED")
- replace_all_patterns(attributes, "value", "\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b", "PHONE_REDACTED")
- replace_all_patterns(attributes, "value", "\\b\\d{4}[\\s-]?\\d{4}[\\s-]?\\d{4}[\\s-]?\\d{4}\\b", "CARD_REDACTED")
- context: resource
statements:
# Pseudonymize user.id in resource attributes
- set(attributes["user.id"], SHA256(attributes["user.id"])) where attributes["user.id"] != nil
# Remove VIP status from resource
- delete_key(attributes, "user.is_vip")
- delete_key(attributes, "user.roles")
# Pseudonymize department in resource
- set(attributes["organization.department"], SHA256(attributes["organization.department"])) where attributes["organization.department"] != nil
# Pseudonymize session in resource
- set(attributes["session.id"], SHA256(attributes["session.id"])) where attributes["session.id"] != nil
log_statements:
- context: log
statements:
# Pseudonymize user identifiers in logs
- set(attributes["user.id"], SHA256(attributes["user.id"])) where attributes["user.id"] != nil
# Remove sensitive log attributes
- delete_key(attributes, "user_message")
- delete_key(attributes, "response")
- delete_key(attributes, "tool_name")
- delete_key(attributes, "arguments")
- delete_key(attributes, "result")
- delete_key(attributes, "user.is_vip")
- delete_key(attributes, "user.roles")
# Pseudonymize department and session
- set(attributes["organization.department"], SHA256(attributes["organization.department"])) where attributes["organization.department"] != nil
- set(attributes["session.id"], SHA256(attributes["session.id"])) where attributes["session.id"] != nil
# Redact PII patterns in log body if it's a string
- replace_all_patterns(attributes, "value", "\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b", "EMAIL_REDACTED") where IsString(body)
- replace_all_patterns(attributes, "value", "\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b", "PHONE_REDACTED") where IsString(body)
- context: resource
statements:
# Pseudonymize user.id in resource attributes for logs
- set(attributes["user.id"], SHA256(attributes["user.id"])) where attributes["user.id"] != nil
# Pseudonymize department in resource for logs
- set(attributes["organization.department"], SHA256(attributes["organization.department"])) where attributes["organization.department"] != nil
metric_statements:
- context: datapoint
statements:
# Pseudonymize user identifiers in metrics
- set(attributes["user.id"], SHA256(attributes["user.id"])) where attributes["user.id"] != nil
# Remove VIP status from metrics
- delete_key(attributes, "user.is_vip")
# Pseudonymize department in metrics
- set(attributes["organization.department"], SHA256(attributes["organization.department"])) where attributes["organization.department"] != nil
- context: resource
statements:
# Pseudonymize user.id in resource attributes for metrics
- set(attributes["user.id"], SHA256(attributes["user.id"])) where attributes["user.id"] != nil
# Remove VIP status from resource for metrics
- delete_key(attributes, "user.is_vip")
# Pseudonymize department in resource for metrics
- set(attributes["organization.department"], SHA256(attributes["organization.department"])) where attributes["organization.department"] != nil
exporters:
# Console exporter for troubleshooting
debug:
verbosity: detailed
sampling_initial: 5
sampling_thereafter: 200
# OTLP exporter for Aspire Dashboard
otlp:
endpoint: {{ include "maf-demo.fullname" . }}-aspire-dashboard:18889
tls:
insecure: true
# OTLP exporter for Anonymized Aspire Dashboard
otlp/anonymized:
endpoint: {{ include "maf-demo.fullname" . }}-aspire-dashboard-anon:18889
tls:
insecure: true
# Azure Monitor exporter for Application Insights
# Supports traces and logs (metrics not supported by App Insights OTLP ingestion)
azuremonitor:
connection_string: {{ .Values.appInsights.connectionString | quote }}
# Langfuse OTLP exporter for LLM observability
# Exports traces to Langfuse via HTTP/protobuf protocol
otlphttp/langfuse:
endpoint: {{ .Values.langfuse.endpoint | default "http://langfuse-web.langfuse.svc.cluster.local:3000/api/public/otel" | quote }}
headers:
Authorization: {{ .Values.langfuse.authorization | default "Basic " | quote }}
# Prometheus Remote Write exporter for Azure Monitor Prometheus
# Uses sidecar container for Azure AD token management
# The sidecar listens on localhost:8081 and injects the Authorization header
prometheusremotewrite:
endpoint: "http://localhost:8081/api/v1/write"
tls:
insecure: true
resource_to_telemetry_conversion:
enabled: true
service:
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp, azuremonitor, otlphttp/langfuse]
# Anonymized trace pipeline - strips PII before sending to anonymized dashboard
traces/anonymized:
receivers: [otlp]
processors: [memory_limiter, transform/anonymize, batch]
exporters: [otlp/anonymized]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp, prometheusremotewrite]
# Anonymized metrics pipeline
metrics/anonymized:
receivers: [otlp]
processors: [memory_limiter, transform/anonymize, batch]
exporters: [otlp/anonymized]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp, azuremonitor]
# Anonymized logs pipeline
logs/anonymized:
receivers: [otlp]
processors: [memory_limiter, transform/anonymize, batch]
exporters: [otlp/anonymized]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "maf-demo.fullname" . }}-otel-collector
labels:
{{- include "maf-demo.labels" . | nindent 4 }}
app.kubernetes.io/component: otel-collector
azure.workload.identity/use: "true"
annotations:
azure.workload.identity/client-id: {{ .Values.agent.clientId }}
azure.workload.identity/tenant-id: {{ .Values.agent.tenantId }}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "maf-demo.fullname" . }}-otel-collector
labels:
{{- include "maf-demo.labels" . | nindent 4 }}
app.kubernetes.io/component: otel-collector
spec:
replicas: 1
selector:
matchLabels:
{{- include "maf-demo.otelCollectorSelectorLabels" . | nindent 6 }}
template:
metadata:
labels:
{{- include "maf-demo.otelCollectorSelectorLabels" . | nindent 8 }}
azure.workload.identity/use: "true"
spec:
serviceAccountName: {{ include "maf-demo.fullname" . }}-otel-collector
containers:
- name: otel-collector
image: otel/opentelemetry-collector-contrib:{{ .Values.otelCollector.tag }}
ports:
- containerPort: 4317 # OTLP gRPC
name: otlp-grpc
protocol: TCP
- containerPort: 4318 # OTLP HTTP
name: otlp-http
protocol: TCP
- containerPort: 8888 # Metrics
name: metrics
protocol: TCP
- containerPort: 8889 # Prometheus metrics
name: prometheus
protocol: TCP
volumeMounts:
- name: config
mountPath: /etc/otelcol-contrib
readOnly: true
resources:
{{- toYaml .Values.otelCollector.resources | nindent 10 }}
# Sidecar container for Azure Monitor Prometheus remote write authentication
# This container obtains Azure AD tokens using workload identity and proxies
# Prometheus remote write requests with the Authorization header injected
- name: prom-remotewrite
image: mcr.microsoft.com/azuremonitor/containerinsights/ciprod/prometheus-remote-write/images:prom-remotewrite-20250814.1
imagePullPolicy: Always
ports:
- name: rw-port
containerPort: 8081
protocol: TCP
env:
- name: INGESTION_URL
value: {{ .Values.prometheus.remoteWriteEndpoint | quote }}
- name: LISTENING_PORT
value: "8081"
- name: IDENTITY_TYPE
value: "workloadIdentity"
- name: CLUSTER
value: {{ .Values.clusterName | default "aks-cluster" | quote }}
livenessProbe:
httpGet:
path: /health
port: rw-port
initialDelaySeconds: 10
timeoutSeconds: 10
readinessProbe:
httpGet:
path: /ready
port: rw-port
initialDelaySeconds: 10
timeoutSeconds: 10
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
volumes:
- name: config
configMap:
name: {{ include "maf-demo.fullname" . }}-otel-collector-config
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "maf-demo.fullname" . }}-otel-collector
labels:
{{- include "maf-demo.labels" . | nindent 4 }}
app.kubernetes.io/component: otel-collector
spec:
type: ClusterIP
ports:
- port: 4317
targetPort: otlp-grpc
protocol: TCP
name: otlp-grpc
- port: 4318
targetPort: otlp-http
protocol: TCP
name: otlp-http
- port: 8888
targetPort: metrics
protocol: TCP
name: metrics
selector:
{{- include "maf-demo.otelCollectorSelectorLabels" . | nindent 4 }}
