Week 01: 容器运行时深度剖析与高可用初始化

stat -fc %T /sys/fs/cgroup/

# 查看进程所属的 Namespace
ls -la /proc/$$/ns/
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 cgroup -> 'cgroup:[4026531835]'
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 ipc -> 'ipc:[4026531839]'
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 mnt -> 'mnt:[4026531840]'
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 net -> 'net:[4026531992]'
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 pid -> 'pid:[4026531836]'
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 user -> 'user:[4026531837]'
# lrwxrwxrwx 1 root root 0 Jan 21 10:00 uts -> 'uts:[4026531838]'

# 查看 UID 映射
cat /proc/<container_pid>/uid_map
#         0       1000          1   # 容器内 UID 0 → 宿主机 UID 1000

# 映射格式: <ns_inside_start> <host_start> <length>

# 查看可用控制器
cat /sys/fs/cgroup/cgroup.controllers
# cpu cpuset io memory hugetlb pids rdma misc

# 为子 cgroup 启用控制器
echo "+cpu +memory +io" > /sys/fs/cgroup/kubepods/cgroup.subtree_control

# 查看生效的控制器
cat /sys/fs/cgroup/kubepods/cgroup.subtree_control
# cpu memory io

cat /sys/fs/cgroup/kubepods/cpu.pressure
# some avg10=0.00 avg60=0.00 avg300=0.00 total=0
# full avg10=0.00 avg60=0.00 avg300=0.00 total=0

cat /sys/fs/cgroup/kubepods/memory.pressure
# some avg10=0.50 avg60=0.33 avg300=0.21 total=123456789
# full avg10=0.00 avg60=0.00 avg300=0.00 total=0

{
  "ociVersion": "1.0.2",
  "process": {
    "terminal": false,
    "user": { "uid": 0, "gid": 0 },
    "args": ["/bin/sh"],
    "env": ["PATH=/usr/bin"],
    "cwd": "/"
  },
  "root": { "path": "rootfs", "readonly": true },
  "linux": {
    "namespaces": [
      { "type": "pid" },
      { "type": "network" },
      { "type": "mount" }
    ],
    "resources": {
      "memory": { "limit": 536870912 },
      "cpu": { "shares": 1024, "quota": 100000, "period": 100000 }
    }
  }
}

# 查看所有证书过期时间
kubeadm certs check-expiration

# 输出示例:
# CERTIFICATE                EXPIRES                  RESIDUAL TIME
# admin.conf                 Jan 21, 2027 08:00 UTC   364d
# apiserver                  Jan 21, 2027 08:00 UTC   364d
# apiserver-etcd-client      Jan 21, 2027 08:00 UTC   364d
# ...

# 手动续期所有证书
kubeadm certs renew all

# 续期单个证书
kubeadm certs renew apiserver

# 查看证书详情
openssl x509 -in /etc/kubernetes/pki/apiserver.crt -text -noout

# 关键字段:
# Issuer: CN=kubernetes  (签发者)
# Subject: CN=kube-apiserver (主体)
# X509v3 Subject Alternative Name:
#     DNS:kubernetes, DNS:kubernetes.default, IP:10.96.0.1, IP:192.168.20.101

# 验证证书链
openssl verify -CAfile /etc/kubernetes/pki/ca.crt /etc/kubernetes/pki/apiserver.crt

hostnamectl set-hostname k8s-node1 # 分别设置为 node1, node2, node3
cat >> /etc/hosts <<EOF
192.168.20.101 k8s-node1
192.168.20.102 k8s-node2
192.168.20.103 k8s-node3
192.168.20.104 k8s-vip # 控制平面虚拟 IP
EOF

sudo sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
sudo swapoff -a
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

sudo systemctl stop firewalld && sudo systemctl disable firewalld

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF

sudo modprobe overlay
sudo modprobe br_netfilter
sudo modprobe ip_vs
sudo modprobe ip_vs_rr
sudo modprobe ip_vs_wrr
sudo modprobe ip_vs_sh
sudo modprobe nf_conntrack

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

sudo sysctl --system

# 安装 IPVS 基础工具
sudo dnf install -y ipset ipvsadm

sudo dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo
sudo dnf install -y containerd.io

# 生成默认配置并进行生产级优化 (关键步骤)
containerd config default | sudo tee /etc/containerd/config.toml >/dev/null

# 1. 启用 SystemdCgroup (K8s 官方推荐)
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml

# 2. 修改 sandbox_image 镜像仓库 (解决大陆无法访问 registry.k8s.io 问题)
# 针对 v1.35, pause 镜像版本通常为 3.10, 建议使用阿里云镜像加速
sudo sed -i 's|sandbox_image = ".*"|sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.10"|g' /etc/containerd/config.toml

sudo systemctl restart containerd && sudo systemctl enable containerd

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.35/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.35/rpm/repodata/repomd.xml.key
EOF

sudo dnf install -y kubelet kubeadm kubectl ipset ipvsadm --disableexcludes=kubernetes
sudo systemctl enable --now kubelet

sudo dnf install -y keepalived haproxy

frontend k8s-api
    bind *:16443
    mode tcp
    option tcplog
    default_backend k8s-api-backend

backend k8s-api-backend
    mode tcp
    option tcp-check
    balance roundrobin
    server k8s-node1 192.168.20.101:6443 check
    server k8s-node2 192.168.20.102:6443 check
    server k8s-node3 192.168.20.103:6443 check

vrrp_instance VI_1 {
    state MASTER          # node2/3 设置为 BACKUP
    interface eth0        # 您的网卡名称
    virtual_router_id 51
    priority 100          # node2 设为 90, node3 设为 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.20.104    # 您的 VIP
    }
}

sudo systemctl enable --now haproxy keepalived

apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
kubernetesVersion: v1.35.0
imageRepository: registry.aliyuncs.com/google_containers
controlPlaneEndpoint: "192.168.20.104:16443" # 指向负载均衡端口
networking:
  podSubnet: "10.97.0.0/16"
  serviceSubnet: "10.96.0.0/16"
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

sudo kubeadm init --config kubeadm-config.yaml --upload-certs

kubectl taint nodes --all node-role.kubernetes.io/control-plane-

kubeadm init --config kubeadm-config.yaml --upload-certs
[init] Using Kubernetes version: v1.35.0
[preflight] Running pre-flight checks
	[WARNING SystemVerification]: kernel release 5.14.0-570.55.1.el9_6.x86_64 is unsupported. Supported LTS versions from the 5.x series are 5.4, 5.10 and 5.15. Any 6.x version is also supported. For cgroups v2 support, the recommended version is 5.10 or newer
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-node1 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.20.101 192.168.20.104]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-node1 localhost] and IPs [192.168.20.101 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-node1 localhost] and IPs [192.168.20.101 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
W1225 16:44:05.324448  142317 endpoint.go:56] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
W1225 16:44:05.368717  142317 endpoint.go:56] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "super-admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
W1225 16:44:05.542045  142317 endpoint.go:56] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
W1225 16:44:05.720756  142317 endpoint.go:56] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/instance-config.yaml"
[patches] Applied patch of type "application/strategic-merge-patch+json" to target "kubeletconfiguration"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests"
[kubelet-check] Waiting for a healthy kubelet at http://127.0.0.1:10248/healthz. This can take up to 4m0s
[kubelet-check] The kubelet is healthy after 501.834926ms
[control-plane-check] Waiting for healthy control plane components. This can take up to 4m0s
[control-plane-check] Checking kube-apiserver at https://192.168.20.101:6443/livez
[control-plane-check] Checking kube-controller-manager at https://127.0.0.1:10257/healthz
[control-plane-check] Checking kube-scheduler at https://127.0.0.1:10259/livez
[control-plane-check] kube-controller-manager is healthy after 504.163478ms
[control-plane-check] kube-scheduler is healthy after 1.98641028s
[control-plane-check] kube-apiserver is healthy after 3.501668409s
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
00d5dde98a8c3b815de529fe24f658325d3b31150fae8efbd72cbc7baaf42ea2
[mark-control-plane] Marking the node k8s-node1 as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-node1 as control-plane by adding the taints [node-role.kubernetes.io/control-plane:NoSchedule]
[bootstrap-token] Using token: 7asjr6.3pocg3akvka3sscf
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
W1225 16:44:11.233493  142317 endpoint.go:56] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes running the following command on each as root:

  kubeadm join 192.168.20.104:16443 --token 7asjr6.3pocg3akvka3sscf \
	--discovery-token-ca-cert-hash sha256:a926f40b153f1e1d818be1b2a40198edc6a0dd6f8bf4e1e8f25da3fdd938067e \
	--control-plane --certificate-key 00d5dde98a8c3b815de529fe24f658325d3b31150fae8efbd72cbc7baaf42ea2

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.20.104:16443 --token 7asjr6.3pocg3akvka3sscf \
	--discovery-token-ca-cert-hash sha256:a926f40b153f1e1d818be1b2a40198edc6a0dd6f8bf4e1e8f25da3fdd938067e

# 查看 Kubelet 运行状态
journalctl -xeu kubelet

# 查看具体容器报错 (针对 Containerd)
crictl --runtime-endpoint unix:///var/run/containerd/containerd.sock ps -a
# 找到状态为 Exited 的容器 ID (如 kube-apiserver)
crictl --runtime-endpoint unix:///var/run/containerd/containerd.sock logs <ID>

sudo kubeadm reset -f
sudo rm -rf /etc/kubernetes /var/lib/etcd /var/lib/kubelet $HOME/.kube
# 重启网络插件残留 (可选)
sudo ip link delete cni0
sudo ip link delete flannel.1

curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# 验证安装
helm version

helm upgrade cilium cilium/cilium --version 1.18.5 \
  --namespace kube-system \
  --set k8sServiceHost=192.168.20.104 \
  --set k8sServicePort=16443 \
  --set kubeProxyReplacement=true \
  --set ipam.operator.clusterPoolIPv4PodCIDRList={10.97.0.0/16} \
  --set hubble.enabled=true \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true
Release "cilium" has been upgraded. Happy Helming!
NAME: cilium
LAST DEPLOYED: Fri Dec 26 09:56:53 2025
NAMESPACE: kube-system
STATUS: deployed
REVISION: 2
TEST SUITE: None
NOTES:
You have successfully installed Cilium with Hubble Relay and Hubble UI.

Your release version is 1.18.5.

For any further help, visit https://docs.cilium.io/en/v1.18/gettinghelp

helm upgrade cilium cilium/cilium --version 1.18.5 \
  --namespace kube-system \
∙ --reuse-values \
  --set prometheus.enabled=true \
  --set operator.prometheus.enabled=true \
  --set hubble.metrics.enabled="{dns,drop,tcp,flow,icmp,http}" \
  --set hubble.metrics.enableOpenMetrics=true

Release "cilium" has been upgraded. Happy Helming!
NAME: cilium
LAST DEPLOYED: Fri Dec 26 11:23:10 2025
NAMESPACE: kube-system
STATUS: deployed
REVISION: 3
TEST SUITE: None
NOTES:
You have successfully installed Cilium with Hubble Relay and Hubble UI.

Your release version is 1.18.5.

For any further help, visit https://docs.cilium.io/en/v1.18/gettinghelp

   cilium status --wait
    /¯¯\
 /¯¯\__/¯¯\    Cilium:             OK
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    OK
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disabled

DaemonSet              cilium                   Desired: 3, Ready: 3/3, Available: 3/3
DaemonSet              cilium-envoy             Desired: 3, Ready: 3/3, Available: 3/3
Deployment             cilium-operator          Desired: 2, Ready: 2/2, Available: 2/2
Deployment             hubble-relay             Desired: 1, Ready: 1/1, Available: 1/1
Deployment             hubble-ui                Desired: 1, Ready: 1/1, Available: 1/1
Containers:            cilium                   Running: 3
                       cilium-envoy             Running: 3
                       cilium-operator          Running: 2
                       clustermesh-apiserver
                       hubble-relay             Running: 1
                       hubble-ui                Running: 1
Cluster Pods:          4/4 managed by Cilium
Helm chart version:    1.18.5
Image versions         cilium             quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628: 3
                       cilium-envoy       quay.io/cilium/cilium-envoy:v1.34.12-1765374555-6a93b0bbba8d6dc75b651cbafeedb062b2997716@sha256:3108521821c6922695ff1f6ef24b09026c94b195283f8bfbfc0fa49356a156e1: 3
                       cilium-operator    quay.io/cilium/operator-generic:v1.18.5@sha256:36c3f6f14c8ced7f45b40b0a927639894b44269dd653f9528e7a0dc363a4eb99: 2
                       hubble-relay       quay.io/cilium/hubble-relay:v1.18.5@sha256:17212962c92ff52384f94e407ffe3698714fcbd35c7575f67f24032d6224e446: 1
                       hubble-ui          quay.io/cilium/hubble-ui-backend:v0.13.3@sha256:db1454e45dc39ca41fbf7cad31eec95d99e5b9949c39daaad0fa81ef29d56953: 1
                       hubble-ui          quay.io/cilium/hubble-ui:v0.13.3@sha256:661d5de7050182d495c6497ff0b007a7a1e379648e60830dd68c4d78ae21761d: 1

dnf install -y nfs-utils

helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
helm repo update

helm install nfs-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
    --namespace kube-system \
    --set nfs.server=192.168.20.14 \
    --set nfs.path=/k8sdev \
    --set storageClass.name=nfs-client \
    --set storageClass.defaultClass=true \
    --set storageClass.reclaimPolicy=Delete \
    --set storageClass.archiveOnDelete=false

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfs-test-pvc
spec:
  accessModes:
    - ReadWriteMany  # NFS 支持多点读写
  resources:
    requests:
      storage: 1Gi

kubectl apply -f test-pvc.yaml
kubectl get pvc nfs-test-pvc