Week 08: 持续交付 (GitOps) 与灾难恢复 (DR)

# 自定义差异忽略
spec:
  ignoreDifferences:
  - group: apps
    kind: Deployment
    jsonPointers:
    - /spec/replicas  # 忽略 HPA 管理的副本数

# 启用 Server-Side Apply
spec:
  syncPolicy:
    syncOptions:
    - ServerSideApply=true

# 查看数据库大小
etcdctl endpoint status --write-out=table
# +----------------+---------+---------+---------+-----------+
# |    ENDPOINT    |   ID    | VERSION | DB SIZE | IS LEADER |
# +----------------+---------+---------+---------+-----------+
# | localhost:2379 | 1234abc |  3.5.12 |  2.1 GB |      true |

# 手动压缩历史版本
ETCDCTL_API=3 etcdctl compact $(etcdctl endpoint status --write-out=json | jq -r '.[0].Status.header.revision')

# 碎片整理 (回收磁盘空间)
etcdctl defrag --endpoints=https://127.0.0.1:2379

# 查看成员状态
etcdctl member list --write-out=table

# 健康检查
etcdctl endpoint health --endpoints=https://node1:2379,https://node2:2379,https://node3:2379

# Quorum 计算
# 3 节点: 需要 2 个健康 (容忍 1 故障)
# 5 节点: 需要 3 个健康 (容忍 2 故障)

# 添加 Learner
etcdctl member add new-node --learner --peer-urls=https://new-node:2380

# 提升为 Voting 成员 (同步完成后)
etcdctl member promote <member-id>

给定: 系统正常运行, 3 副本 Deployment
当: 随机杀死 1 个 Pod
则: 
  - 服务持续可用 (HTTP 200 > 99%)
  - 新 Pod 在 30s 内启动
  - 告警在 1 分钟内触发

# 限制影响范围
spec:
  selector:
    namespaces:
    - production
    labelSelectors:
      app: api-server
  mode: one           # 只影响一个目标
  # mode: percent     # 或按百分比
  # value: "30"

apiVersion: chaos-mesh.org/v1alpha1
kind: Schedule
metadata:
  name: pod-chaos-schedule
spec:
  schedule: "@every 1h"
  historyLimit: 1      # 保留历史
  concurrencyPolicy: Forbid
  type: PodChaos
  podChaos:
    action: pod-kill
    mode: one
    duration: "30s"    # 自动恢复

kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

# 获取初始密码
kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d

# 暴露 UI
kubectl port-forward svc/argocd-server -n argocd 8080:443

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: guestbook
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/argoproj/argocd-example-apps.git
    targetRevision: HEAD
    path: guestbook
  destination:
    server: https://kubernetes.default.svc
    namespace: guestbook
  syncPolicy:
    automated:
      prune: true      # 删除不在 Git 中的资源
      selfHeal: true   # 自动修复漂移
    syncOptions:
    - CreateNamespace=true

# 先部署 ConfigMap (wave -1)
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
---
# 再部署 Deployment (wave 0, 默认)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app
  annotations:
    argocd.argoproj.io/sync-wave: "0"

apiVersion: batch/v1
kind: Job
metadata:
  name: db-migrate
  annotations:
    argocd.argoproj.io/hook: PreSync
    argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
  template:
    spec:
      containers:
      - name: migrate
        image: myapp:latest
        command: ["./migrate.sh"]
      restartPolicy: Never

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: guestbook
  namespace: argocd
spec:
  generators:
  - list:
      elements:
      - cluster: staging
        url: https://staging.k8s.example.com
      - cluster: production
        url: https://prod.k8s.example.com
  template:
    metadata:
      name: '{{cluster}}-guestbook'
    spec:
      project: default
      source:
        repoURL: https://github.com/example/guestbook.git
        targetRevision: HEAD
        path: 'overlays/{{cluster}}'
      destination:
        server: '{{url}}'
        namespace: guestbook

# 使用 etcdctl 创建快照
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-snapshot.db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# 验证快照
ETCDCTL_API=3 etcdctl snapshot status /backup/etcd-snapshot.db --write-out=table

# 1. 停止所有 etcd 和 kube-apiserver
systemctl stop etcd

# 2. 移除旧数据
mv /var/lib/etcd /var/lib/etcd.old

# 3. 恢复快照
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-snapshot.db \
  --data-dir=/var/lib/etcd \
  --name=etcd-node1 \
  --initial-cluster=etcd-node1=https://192.168.1.101:2380 \
  --initial-advertise-peer-urls=https://192.168.1.101:2380

# 4. 修复权限
chown -R etcd:etcd /var/lib/etcd

# 5. 启动 etcd
systemctl start etcd

apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-backup
  namespace: kube-system
spec:
  schedule: "0 */6 * * *" # 每 6 小时
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            image: bitnami/etcd:latest
            command:
            - /bin/sh
            - -c
            - |
              etcdctl snapshot save /backup/etcd-$(date +%Y%m%d-%H%M%S).db \
                --endpoints=$ETCD_ENDPOINT \
                --cacert=/certs/ca.crt \
                --cert=/certs/server.crt \
                --key=/certs/server.key
            volumeMounts:
            - name: backup
              mountPath: /backup
            - name: certs
              mountPath: /certs
          restartPolicy: OnFailure
          volumes:
          - name: backup
            persistentVolumeClaim:
              claimName: etcd-backup-pvc
          - name: certs
            secret:
              secretName: etcd-certs

velero install \
  --provider aws \
  --plugins velero/velero-plugin-for-aws:v1.8.0 \
  --bucket velero-backups \
  --secret-file ./credentials-velero \
  --backup-location-config region=us-east-1 \
  --snapshot-location-config region=us-east-1 \
  --use-volume-snapshots=true

# 备份整个集群
velero backup create cluster-backup

# 备份特定 Namespace
velero backup create prod-backup --include-namespaces=production

# 备份带 Label 的资源
velero backup create app-backup --selector app=myapp

# 定时备份
velero schedule create daily-backup --schedule="0 2 * * *" --include-namespaces=production

# 列出备份
velero backup get

# 恢复到原 Namespace
velero restore create --from-backup prod-backup

# 恢复到新 Namespace
velero restore create --from-backup prod-backup --namespace-mappings production:production-restored

helm repo add chaos-mesh https://charts.chaos-mesh.org
helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --create-namespace

apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: pod-kill-example
  namespace: chaos-testing
spec:
  action: pod-kill
  mode: one  # 随机杀死一个 Pod
  selector:
    namespaces:
    - production
    labelSelectors:
      app: api-server
  scheduler:
    cron: "@every 10m"  # 每 10 分钟执行一次

apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: network-delay-example
  namespace: chaos-testing
spec:
  action: delay
  mode: all
  selector:
    namespaces:
    - production
    labelSelectors:
      app: database
  delay:
    latency: "100ms"
    jitter: "10ms"
    correlation: "50"
  duration: "5m"