M7: 硬件虚拟化与基础设施

// Guest 状态
VMCS_GUEST_RIP           // 指令指针
VMCS_GUEST_RSP           // 栈指针
VMCS_GUEST_CR3           // 页表基址
VMCS_GUEST_INTERRUPTIBILITY  // 中断状态

// 执行控制
VMCS_PIN_BASED_CONTROLS  // NMI, 外部中断
VMCS_CPU_BASED_CONTROLS  // I/O, MSR, CPUID
VMCS_EXCEPTION_BITMAP    // 哪些异常触发 Exit

// Exit 信息
VMCS_EXIT_REASON         // 退出原因代码
VMCS_EXIT_QUALIFICATION  // 附加信息 (如 I/O 端口号)
VMCS_GUEST_LINEAR_ADDR   // 引发 Exit 的线性地址

// arch/x86/include/asm/kvm_host.h

struct kvm {
    struct mm_struct *mm;                // 宿主机地址空间
    struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM];
    struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; // vCPU 数组
    atomic_t online_vcpus;
    
    struct kvm_io_bus __rcu *buses[KVM_NR_BUSES];  // I/O 总线模拟
    struct kvm_vm_stat stat;             // 统计信息
    struct kvm_arch arch;                // 架构相关 (EPT, APIC 等)
    
    refcount_t users_count;
    struct mutex lock;
    spinlock_t mmu_lock;                 // 保护 EPT/影子页表
    // ...
};

struct kvm_vcpu {
    struct kvm *kvm;                     // 所属 VM
    int vcpu_id;
    int cpu;                             // 当前运行的物理 CPU
    
    struct kvm_run *run;                 // 与用户空间共享的内存
    struct kvm_vcpu_arch arch;           // 架构状态 (寄存器, VMCS 等)
    
    struct task_struct *task;            // 对应的 QEMU 线程
    bool preempted;
    bool ready;
    
    struct kvm_dirty_ring dirty_ring;    // 脏页追踪
    // ...
};

// virt/kvm/kvm_main.c
// 用户空间通过 ioctl(KVM_RUN) 触发

static long kvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, ...)
{
    switch (ioctl) {
    case KVM_RUN:
        r = kvm_arch_vcpu_ioctl_run(vcpu);  // 进入架构相关代码
        break;
    // ...
    }
}

// arch/x86/kvm/x86.c
int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
{
    vcpu_load(vcpu);                     // 1. 加载 vCPU 上下文
    
    for (;;) {
        if (kvm_vcpu_running(vcpu)) {
            r = vcpu_run(vcpu);          // 2. 核心运行循环
        } else {
            r = vcpu_block(vcpu);        // vCPU 处于 HLT 等待
        }
        
        if (r <= 0) break;               // 需要返回用户空间
        
        if (signal_pending(current)) {   // 3. 处理信号
            r = -EINTR;
            vcpu->run->exit_reason = KVM_EXIT_INTR;
            break;
        }
    }
    
    vcpu_put(vcpu);
    return r;
}

// arch/x86/kvm/vmx/vmx.c
// Intel VMX 实现的核心入口

static int vmx_vcpu_run(struct kvm_vcpu *vcpu)
{
    struct vcpu_vmx *vmx = to_vmx(vcpu);
    
    // 1. 准备 VMCS
    vmx_vcpu_load_vmcs(vcpu, cpu);
    
    // 2. 同步 Guest 状态到 VMCS
    vmx_sync_dirty_debug_regs(vcpu);
    vmx_load_guest_msr(vmx);
    
    // 3. 执行 VM Entry
    vmx->exit_reason.full = __vmx_vcpu_run(vmx, ...);  // 汇编入口
    
    // 4. VM Exit 后回到这里
    vmx_vcpu_enter_exit(vcpu, vmx);
    
    // 5. 读取 Exit 原因并处理
    if (vmx->exit_reason.basic == EXIT_REASON_EXCEPTION_NMI)
        kvm_set_exception(vcpu, ...);
    
    return vmx_complete_atomic_exit(vcpu);
}

// arch/x86/kvm/vmx/vmenter.S (关键汇编)
SYM_FUNC_START(__vmx_vcpu_run)
    // 保存 Host 通用寄存器
    push    %rbp
    push    %rbx
    push    %r12-r15
    
    // 加载 Guest 通用寄存器
    mov     VCPU_RBX(%rdi), %rbx
    mov     VCPU_RCX(%rdi), %rcx
    // ... 其他寄存器
    
    // 执行 VMLAUNCH 或 VMRESUME
    vmlaunch  // 或 vmresume
    
    // --- Guest 在此执行 ---
    // VM Exit 后返回到这里
    
    // 保存 Guest 寄存器到 VMCS
    mov     %rbx, VCPU_RBX(%rdi)
    // ...
    
    // 恢复 Host 寄存器
    pop     %r15-%r12
    ret
SYM_FUNC_END(__vmx_vcpu_run)

// arch/x86/kvm/vmx/vmx.c

static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *) = {
    [EXIT_REASON_EXCEPTION_NMI]     = handle_exception_nmi,
    [EXIT_REASON_EXTERNAL_INTERRUPT]= handle_external_interrupt,
    [EXIT_REASON_CPUID]             = kvm_emulate_cpuid,
    [EXIT_REASON_HLT]               = kvm_emulate_halt,
    [EXIT_REASON_IO_INSTRUCTION]    = handle_io,
    [EXIT_REASON_CR_ACCESS]         = handle_cr,
    [EXIT_REASON_MSR_READ]          = kvm_emulate_rdmsr,
    [EXIT_REASON_MSR_WRITE]         = kvm_emulate_wrmsr,
    [EXIT_REASON_EPT_VIOLATION]     = handle_ept_violation,
    [EXIT_REASON_EPT_MISCONFIG]     = handle_ept_misconfig,
    [EXIT_REASON_VMCALL]            = kvm_emulate_hypercall,
    // ... 更多处理函数
};

static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
{
    struct vcpu_vmx *vmx = to_vmx(vcpu);
    u32 exit_reason = vmx->exit_reason.basic;
    
    // Fast path: 某些 Exit 可快速处理
    if (exit_reason < kvm_vmx_max_exit_handlers &&
        kvm_vmx_exit_handlers[exit_reason])
        return kvm_vmx_exit_handlers[exit_reason](vcpu);
    
    // Unknown exit - 返回用户空间
    vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
    return 0;
}

// arch/x86/kvm/vmx/vmx.c

static int handle_ept_violation(struct kvm_vcpu *vcpu)
{
    gpa_t gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
    u64 error_code = vmcs_read64(EXIT_QUALIFICATION);
    
    // 判断违规类型
    bool is_write = error_code & EPT_VIOLATION_WRITE;
    bool is_fetch = error_code & EPT_VIOLATION_INSTR_FETCH;
    
    // 调用通用 page fault 处理
    return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
}

// arch/x86/kvm/mmu/mmu.c
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code, ...)
{
    // 1. 检查是否 MMIO 区域
    if (is_mmio_fault)
        return handle_mmio_page_fault(vcpu, gpa, ...);
    
    // 2. 分配 EPT 页表并建立映射
    r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, ...);
    
    // 3. 成功则重新进入 Guest
    return r;
}

# 查看 vCPU 统计信息
cat /sys/kernel/debug/kvm/*/vcpu0/stat

# 关键指标含义
exits        # VM Exit 总次数
halt_exits   # HLT 指令触发的 Exit
io_exits     # I/O 指令触发的 Exit
mmio_exits   # MMIO 访问触发的 Exit
irq_exits    # 中断注入触发的 Exit
fpu_reload   # FPU 状态重载次数

// arch/x86/include/asm/svm.h

struct vmcb_control_area {
    u32 intercept_cr;        // CR 读写拦截
    u32 intercept_dr;        // DR 读写拦截
    u32 intercept_exceptions;// 异常拦截
    u64 tsc_offset;          // TSC 偏移
    u8  tlb_control;         // TLB 刷新控制
    // ...
};

struct vmcb_save_area {
    struct vmcb_seg es, cs, ss, ds; // 段寄存器
    u64 rip, rsp, rax;              // 通用寄存器
    u64 cr0, cr2, cr3, cr4;         // 控制寄存器
    // ...
};

struct vmcb {
    struct vmcb_control_area control;
    struct vmcb_save_area save;
};

// arch/x86/kvm/svm/svm.c

static void svm_vcpu_run(struct kvm_vcpu *vcpu)
{
    struct vcpu_svm *svm = to_svm(vcpu);
    
    // 1. 将 Guest 状态写入 VMCB
    svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
    
    // 2. 汇编入口: 核心指令 clgi; vmload; vmrun; vmsave; stgi
    __svm_vcpu_run(svm->vmcb_pa, svm->host_save_user_msrs);
    
    // 3. 从 VMCB 读取 Guest 状态
    vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
    
    // 4. 处理 Exit
    handle_exit(vcpu);
}

// arch/arm64/kvm/hyp/vhe/switch.c

static void __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
{
    struct kvm_cpu_context *host_ctxt;
    struct kvm_cpu_context *guest_ctxt;
    
    host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
    guest_ctxt = &vcpu->arch.ctxt;
    
    // 1. 保存 Host 寄存器 (Callee-saved)
    sysreg_save_host_state_vhe(host_ctxt);
    
    // 2. 加载 Guest 寄存器 & 配置 HCR_EL2
    sysreg_restore_guest_state_vhe(guest_ctxt);
    __activate_traps(vcpu);
    
    // 3. 进入 Guest (ERET 到 EL1)
    // ARM 使用异常返回 (ERET) 指令降级到 Guest
    __guest_enter(vcpu);
    
    // --- Guest 执行 ---
    
    // 4. Guest 退出 (Trap 到 EL2)
    __guest_exit(vcpu);
    
    // 5. 恢复 Host 状态
    sysreg_save_guest_state_vhe(guest_ctxt);
    sysreg_restore_host_state_vhe(host_ctxt);
}

Bit 0: Read access
Bit 1: Write access  
Bit 2: Execute access (可配置 User/Supervisor)
Bit 7: Large page (2MB/1GB)
Bit 8: Accessed (硬件置位)
Bit 9: Dirty (硬件置位, 需启用)
Bits 12-51: 下一级页表物理地址 / 最终 HPA

Control.VPID = 1  // 启用
VMCS.VPID = <unique_id>  // 每个 vCPU 唯一

struct vring_desc {
    __le64 addr;    // 缓冲区物理地址 (GPA)
    __le32 len;     // 缓冲区长度
    __le16 flags;   // VRING_DESC_F_NEXT, _WRITE, _INDIRECT
    __le16 next;    // 链表中下一个描述符索引
};

struct vring_avail {
    __le16 flags;           // 是否禁用通知
    __le16 idx;             // 下一个可用位置
    __le16 ring[QSIZE];     // 描述符链头索引
    __le16 used_event;      // 事件通知阈值 (可选)
};

struct vring_used {
    __le16 flags;
    __le16 idx;
    struct vring_used_elem ring[QSIZE];
    __le16 avail_event;
};

struct vring_used_elem {
    __le32 id;     // 描述符链头索引
    __le32 len;    // 写入的字节数
};

# 启用 VF
echo 4 > /sys/class/net/eth0/device/sriov_numvfs

# 查看 VF
lspci | grep "Virtual Function"

// 创建新 namespace
int fd = clone(child_fn, stack, CLONE_NEWPID | CLONE_NEWNS, NULL);

// 加入已有 namespace
int ns_fd = open("/proc/1234/ns/pid", O_RDONLY);
setns(ns_fd, CLONE_NEWPID);

// 取消共享当前 namespace
unshare(CLONE_NEWNET);

# 创建 cgroup
mkdir /sys/fs/cgroup/container1

# 限制 CPU 为 20% (20ms/100ms)
echo "20000 100000" > /sys/fs/cgroup/container1/cpu.max

# 限制内存为 512MB
echo "536870912" > /sys/fs/cgroup/container1/memory.max

# 添加进程
echo $PID > /sys/fs/cgroup/container1/cgroup.procs

# 挂载 OverlayFS
mount -t overlay overlay \
    -o lowerdir=/images/layer1:/images/layer2,\
       upperdir=/container/upper,\
       workdir=/container/work \
    /container/merged

{
    "ociVersion": "1.0.0",
    "process": {
        "args": ["/bin/sh"],
        "env": ["PATH=/usr/bin"],
        "cwd": "/"
    },
    "root": {
        "path": "rootfs",
        "readonly": false
    },
    "linux": {
        "namespaces": [
            {"type": "pid"},
            {"type": "network"},
            {"type": "mount"}
        ],
        "resources": {
            "memory": {"limit": 536870912}
        }
    }
}

# 创建 MicroVM
curl --unix-socket /tmp/firecracker.socket -X PUT \
    -d '{"vcpu_count": 2, "mem_size_mib": 1024}' \
    http://localhost/machine-config

# AMD SEV (KVM)
# 检查 SEV 支持
dmesg | grep -i sev

# 启动 SEV 虚拟机
qemu-system-x86_64 \
    -machine q35,confidential-guest-support=sev0 \
    -object sev-guest,id=sev0,policy=0x1

# Intel TDX (需要 TDX 内核)
# 检查 TDX 支持
cat /sys/module/kvm_intel/parameters/tdx

# 启动 TDX 虚拟机
qemu-system-x86_64 \
    -machine q35,confidential-guest-support=tdx \
    -object tdx-guest,id=tdx

# 绑定 vCPU 到物理 CPU
virsh vcpupin vm1 0 2
virsh vcpupin vm1 1 3

# 配置 NUMA 拓扑
<numatune>
    <memory mode="strict" nodeset="0"/>
</numatune>

# QEMU Live Migration 示例
virsh migrate --live --verbose vm1 qemu+ssh://dst-host/system

# 启用压缩
virsh migrate-setmaxdowntime vm1 100  # 最大停机 100ms
virsh migrate-setspeed vm1 1000       # 限速 1000 MiB/s

# 监控迁移进度
virsh domjobinfo vm1