Week 07: 深度可观测性 - 监控、日志与链路追踪

scrape_configs:
- job_name: 'kubernetes-pods'
  kubernetes_sd_configs:
  - role: pod
  relabel_configs:
  # 只抓取有 prometheus.io/scrape 注解的 Pod
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  # 使用自定义端口
  - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
    action: replace
    target_label: __address__
    regex: (.+)
    replacement: ${1}
  # 添加 namespace 标签
  - source_labels: [__meta_kubernetes_namespace]
    target_label: namespace
  # 添加 pod 标签
  - source_labels: [__meta_kubernetes_pod_name]
    target_label: pod

metric_relabel_configs:
# 删除高 Cardinality 标签
- source_labels: [url]
  action: labeldrop
# 重命名指标
- source_labels: [__name__]
  regex: 'http_requests_total'
  replacement: 'http_requests'
  target_label: __name__
# 删除不需要的指标
- source_labels: [__name__]
  regex: 'go_.*'
  action: drop

# 30 天 99.9% 可用性 SLO
# Error Budget = 30 * 24 * 60 * 0.001 = 43.2 分钟

groups:
- name: slo-alerts
  rules:
  # 快速燃烧 (5% budget in 1h)
  - alert: HighErrorBudgetBurn
    expr: |
      (
        sum(rate(http_requests_total{status=~"5.."}[1h]))
        /
        sum(rate(http_requests_total[1h]))
      ) > 14.4 * 0.001
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Error budget burning too fast"

  # 慢速燃烧 (10% budget in 6h)
  - alert: ErrorBudgetBurn
    expr: |
      (
        sum(rate(http_requests_total{status=~"5.."}[6h]))
        /
        sum(rate(http_requests_total[6h]))
      ) > 2.4 * 0.001
    for: 30m
    labels:
      severity: warning

# 短窗口 (快速检测) + 长窗口 (确认趋势)
- alert: HighErrorRate
  expr: |
    (
      sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.01
    ) and (
      sum(rate(http_requests_total{status=~"5.."}[1h])) / sum(rate(http_requests_total[1h])) > 0.01
    )

<metric_name>{<label_name>=<label_value>, ...} <value> [<timestamp>]

# /etc/prometheus/rules/recording.yml
groups:
- name: recording_rules
  rules:
  - record: job:http_requests_total:rate5m
    expr: sum(rate(http_requests_total[5m])) by (job)

# /etc/prometheus/rules/alerting.yml
groups:
- name: api_alerts
  rules:
  - alert: HighErrorRate
    expr: |
      sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
      /
      sum(rate(http_requests_total[5m])) by (job)
      > 0.05
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "High 5xx error rate on {{ $labels.job }}"
      description: "Error rate is {{ $value | humanizePercentage }}"

# /etc/alertmanager/alertmanager.yml
route:
  receiver: default
  group_by: [alertname, job]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
  - match:
      severity: critical
    receiver: pagerduty
  - match:
      severity: warning
    receiver: slack

receivers:
- name: default
  email_configs:
  - to: ops@example.com
- name: pagerduty
  pagerduty_configs:
  - service_key: <key>
- name: slack
  slack_configs:
  - channel: '#alerts'
    api_url: https://hooks.slack.com/...

amtool silence add alertname="HighErrorRate" --duration=2h --comment="Planned maintenance"

# promtail-config.yaml
server:
  http_listen_port: 9080

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: kubernetes-pods
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        target_label: app
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
    pipeline_stages:
      - json:
          expressions:
            level: level
            msg: msg
      - labels:
          level:

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm install opentelemetry-operator open-telemetry/opentelemetry-operator

apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: auto-instrumentation
spec:
  exporter:
    endpoint: http://otel-collector:4317
  propagators:
    - tracecontext
    - baggage
  sampler:
    type: parentbased_traceidratio
    argument: "0.25" # 采样率 25%
  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:latest
  python:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-python:latest

# Pod 注解自动注入
metadata:
  annotations:
    instrumentation.opentelemetry.io/inject-java: "true"

# Loki Ruler 配置
groups:
- name: log_alerts
  rules:
  - alert: HighErrorLogRate
    expr: sum(rate({app="api-server"} |= "ERROR" [5m])) > 10
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "High error log rate"