QEMU/KVM虚拟机运行核心流程

$ ./qemu-system-x86_64 --enable-kvm -machine q35 -cpu host,+vmx -smp 1 -m 2048 -name ubuntu -hda /opt/vms/ubuntu.qcow2 -cdrom /opt/vms/ubuntu.iso

void qemu_init(int argc, char **argv, char **envp){    //...    // 对参数进行解析    for(;;) {        if (optind >= argc)            break;        if (argv[optind][0] != '-') {            loc_set_cmdline(argv, optind, 1);            drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);        } else {            const QEMUOption *popt;
            popt = lookup_opt(argc, argv, &optarg, &optind);            if (!(popt->arch_mask & arch_type)) {                error_report("Option not supported for this target");                exit(1);            }            switch(popt->index) {            case QEMU_OPTION_cpu:                /* hw initialization will check this */                cpu_option = optarg;                break;            //...            // 主要关注下面几个参数            case QEMU_OPTION_m:                opts = qemu_opts_parse_noisily(qemu_find_opts("memory"),                                               optarg, true);                if (!opts) {                    exit(EXIT_FAILURE);                }                break;            case QEMU_OPTION_enable_kvm:                qdict_put_str(machine_opts_dict, "accel", "kvm");                break;            case QEMU_OPTION_M:            case QEMU_OPTION_machine:                {                    bool help;
                    keyval_parse_into(machine_opts_dict, optarg, "type", &help, &error_fatal);                    if (help) {                        machine_help_func(machine_opts_dict);                        exit(EXIT_SUCCESS);                    }                    break;                }            case QEMU_OPTION_smp:                machine_parse_property_opt(qemu_find_opts("smp-opts"),                                           "smp", optarg);                break;            }        }    }    //...    // 根据accel设置accelerators = kvm    qemu_apply_legacy_machine_options(machine_opts_dict);    qemu_apply_machine_options(machine_opts_dict);
    // 也会根据进程名判断可用的加速类型    configure_accelerators(argv[0]);    // 内部调用了do_configure_accelerator --> accel_init_machine    // accel_init_machine --> kvm_init    // 初始化具体的accel类（这里是kvm）    // 在qemu-6.2.0/accel/kvm/kvm-all.c line 3629    // 函数kvm_accel_class_init内部找到真正的初始化函数    // ac->init_machine = kvm_init;    //...
    // 在qmp_x_exit_preconfig与虚拟cpu创建有关    if (!preconfig_requested) {        qmp_x_exit_preconfig(&error_fatal);    }    qemu_init_displays();    // 设置accel    accel_setup_post(current_machine);    os_setup_post();    resume_mux_open();}

kvm_init(MachineState * ms) (qemu-6.2.0accelkvmkvm-all.c:2308)accel_init_machine(AccelState * accel, MachineState * ms) (qemu-6.2.0accelaccel-softmmu.c:39)do_configure_accelerator(void * opaque, QemuOpts * opts, Error ** errp) (qemu-6.2.0softmmuvl.c:2348)qemu_opts_foreach(QemuOptsList * list, qemu_opts_loopfunc func, void * opaque, Error ** errp) (qemu-6.2.0utilqemu-option.c:1135)configure_accelerators(const char * progname) (qemu-6.2.0softmmuvl.c:2414)qemu_init(int argc, char ** argv, char ** envp) (qemu-6.2.0softmmuvl.c:3724)main(int argc, char ** argv, char ** envp) (qemu-6.2.0softmmumain.c:49)

static int kvm_init(MachineState *ms){    MachineClass *mc = MACHINE_GET_CLASS(ms);    static const char upgrade_note[] =        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmodn"        "(see http://sourceforge.net/projects/kvm).n";    //...    QLIST_INIT(&s->kvm_parked_vcpus);    // 开始使用kvm之前的标准流程    // 打开设备/dev/kvm，检查kvm API版本    // 保存了kvm设备描述符s->fd    s->fd = qemu_open_old("/dev/kvm", O_RDWR);    if (s->fd == -1) {        fprintf(stderr, "Could not access KVM kernel module: %mn");        ret = -errno;        goto err;    }
    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);    if (ret < KVM_API_VERSION) {        if (ret >= 0) {            ret = -EINVAL;        }        fprintf(stderr, "kvm version too oldn");        goto err;    }
    if (ret > KVM_API_VERSION) {        ret = -EINVAL;        fprintf(stderr, "kvm version not supportedn");        goto err;    }
    //...    // 创建虚拟机，保存虚拟机描述符s->vmfd    do {        ret = kvm_ioctl(s, KVM_CREATE_VM, type);    } while (ret == -EINTR);    //...    s->vmfd = ret;    //...}

void x86_cpus_init(X86MachineState *x86ms, int default_cpu_version){    //...    // 根据参数smp的值，创建对应数量的虚拟cpu    for (i = 0; i < ms->smp.cpus; i++) {        x86_cpu_new(x86ms, possible_cpus->cpus[i].arch_id, &error_fatal);    }}

void qemu_init_vcpu(CPUState *cpu){    //...    // 调用函数kvm_start_vcpu_thread创建虚拟cpu执行线程    cpus_accel->create_vcpu_thread(cpu);    //...}static void kvm_start_vcpu_thread(CPUState *cpu){    //...    // 线程函数kvm_vcpu_thread_fn    qemu_thread_create(cpu->thread, thread_name, kvm_vcpu_thread_fn,                       cpu, QEMU_THREAD_JOINABLE);    //...}static void *kvm_vcpu_thread_fn(void *arg){    //...    // kvm_init_vcpu中通过kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id)    // 获取了vcpu描述符 cpu->kvm_fd = ret;    r = kvm_init_vcpu(cpu, &error_fatal);    kvm_init_cpu_signals(cpu);
    /* signal CPU creation */    cpu_thread_signal_created(cpu);    qemu_guest_random_seed_thread_part2(cpu->random_seed);
    // do while循环执行kvm_cpu_exec    do {        if (cpu_can_run(cpu)) {            r = kvm_cpu_exec(cpu);            if (r == EXCP_DEBUG) {                cpu_handle_guest_debug(cpu);            }        }        qemu_wait_io_event(cpu);    } while (!cpu->unplug || cpu_can_run(cpu));
    kvm_destroy_vcpu(cpu);    cpu_thread_signal_destroyed(cpu);    qemu_mutex_unlock_iothread();    rcu_unregister_thread();    return NULL;}int kvm_cpu_exec(CPUState *cpu){    //...    do {        // kvm_vcpu_ioctl(cpu, KVM_RUN, 0)        // 从这里进入kvm内核阶段，开始运行虚拟机        run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);        //...        // 根据退出原因，分发处理        switch (run->exit_reason) {        case KVM_EXIT_IO:            DPRINTF("handle_ion");            /* Called outside BQL */            kvm_handle_io(run->io.port, attrs,                          (uint8_t *)run + run->io.data_offset,                          run->io.direction,                          run->io.size,                          run->io.count);            ret = 0;            break;        default:            DPRINTF("kvm_arch_handle_exitn");            ret = kvm_arch_handle_exit(cpu, run);            break;        }    } while (ret == 0);    cpu_exec_end(cpu);    //...    qatomic_set(&cpu->exit_request, 0);    return ret;}

static long kvm_vcpu_ioctl(struct file *filp,               unsigned int ioctl, unsigned long arg){    //...    switch (ioctl) {    case KVM_RUN: {        //...        // 根据KVM_RUN,调用kvm_arch_vcpu_ioctl_run        r = kvm_arch_vcpu_ioctl_run(vcpu);        trace_kvm_userspace_exit(vcpu->run->exit_reason, r);        break;    }    //...    }out:    mutex_unlock(&vcpu->mutex);    kfree(fpu);    kfree(kvm_sregs);    return r;}// arch/x86/kvm/x86.c line 10103int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu){    //...    if (kvm_run->immediate_exit)        r = -EINTR;    else        r = vcpu_run(vcpu);    //...    return r;}// arch/x86/kvm/x86.c line 9923static int vcpu_run(struct kvm_vcpu *vcpu){    int r;    struct kvm *kvm = vcpu->kvm;    //...    for (;;) {        if (kvm_vcpu_running(vcpu)) {            // 进入guest模式的入口            r = vcpu_enter_guest(vcpu);        } else {            r = vcpu_block(kvm, vcpu);        }        // 当返回值r<=0时，退出循环，一步步返回到用户层QEMU处理        // 当返回值r>0时，继续guest运行循环        if (r <= 0)            break;    }    //...    return r;}// arch/x86/kvm/x86.c line 9532static int vcpu_enter_guest(struct kvm_vcpu *vcpu){    int r;    // 一系列kvm_check_request函数调用    // 检查guest请求    //...    // guest内存管理单元    r = kvm_mmu_reload(vcpu);
    //...    // 禁用内核抢占    preempt_disable();    //...
    vcpu->mode = IN_GUEST_MODE;    // exit_fastpath = static_call(kvm_x86_run)(vcpu);    // 调用架构相关的run函数进入guest模式运行    for (;;) {        exit_fastpath = static_call(kvm_x86_run)(vcpu);        //...        break;    }    // 能走到这里标志已退出guest模式    vcpu->mode = OUTSIDE_GUEST_MODE;    //...    // 启用内核抢占    preempt_enable();
    // 调用架构相关kvm_x86_handle_exit函数    // 根据具体退出原因进行处理    r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);    return r;}

// arch/x86/kvm/vmx/vmx.c line 6628static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu){    // 检查和准备工作    //...    vmx_vcpu_enter_exit(vcpu, vmx);    //...}// arch/x86/kvm/vmx/vmx.c line 6606static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,                    struct vcpu_vmx *vmx){    //...    // arch/x86/kvm/vmx/vmenter.S汇编    vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,                   vmx->loaded_vmcs->launched);    //...}

arch/x86/kvm/vmx/vmenter.S1.保存host状态2.加载guest状态3.进入guest模式: call vmx_vmentercpu从ROOT模式切换至NON-ROOT模式，进入到guest的世界运行4.发生VM Exit时，保存guest状态，加载host状态 cpu从NON-ROOT模式切换至ROOT模式，返回到host的世界

static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath){    int ret = __vmx_handle_exit(vcpu, exit_fastpath);    //...    return ret;}static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath){    //...    exit_handler_index = array_index_nospec((u16)exit_reason.basic,                        kvm_vmx_max_exit_handlers);    return kvm_vmx_exit_handlers[exit_handler_index](vcpu);
} // 退出处理例程返回<=0，表示异常需要到用户层qemu进行进一步处理 // 退出处理例程返回值>0，表示内核层已经处理完，可继续执行static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {    [EXIT_REASON_EXCEPTION_NMI]           = handle_exception_nmi,    [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,    [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,    [EXIT_REASON_NMI_WINDOW]          = handle_nmi_window,    [EXIT_REASON_IO_INSTRUCTION]          = handle_io,    [EXIT_REASON_CR_ACCESS]               = handle_cr,    [EXIT_REASON_DR_ACCESS]               = handle_dr,    [EXIT_REASON_CPUID]                   = kvm_emulate_cpuid,    [EXIT_REASON_MSR_READ]                = kvm_emulate_rdmsr,    [EXIT_REASON_MSR_WRITE]               = kvm_emulate_wrmsr,    [EXIT_REASON_INTERRUPT_WINDOW]        = handle_interrupt_window,    [EXIT_REASON_HLT]                     = kvm_emulate_halt,    [EXIT_REASON_INVD]              = kvm_emulate_invd,    [EXIT_REASON_INVLPG]              = handle_invlpg,    [EXIT_REASON_RDPMC]                   = kvm_emulate_rdpmc,    [EXIT_REASON_VMCALL]                  = kvm_emulate_hypercall,    [EXIT_REASON_VMCLEAR]              = handle_vmx_instruction,    [EXIT_REASON_VMLAUNCH]              = handle_vmx_instruction,    [EXIT_REASON_VMPTRLD]              = handle_vmx_instruction,    [EXIT_REASON_VMPTRST]              = handle_vmx_instruction,    [EXIT_REASON_VMREAD]              = handle_vmx_instruction,    [EXIT_REASON_VMRESUME]              = handle_vmx_instruction,    [EXIT_REASON_VMWRITE]              = handle_vmx_instruction,    [EXIT_REASON_VMOFF]              = handle_vmx_instruction,    [EXIT_REASON_VMON]              = handle_vmx_instruction,    [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,    [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,    [EXIT_REASON_APIC_WRITE]              = handle_apic_write,    [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,    [EXIT_REASON_WBINVD]                  = kvm_emulate_wbinvd,    [EXIT_REASON_XSETBV]                  = kvm_emulate_xsetbv,    [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,    [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,    [EXIT_REASON_GDTR_IDTR]              = handle_desc,    [EXIT_REASON_LDTR_TR]              = handle_desc,    [EXIT_REASON_EPT_VIOLATION]          = handle_ept_violation,    [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,    [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,    [EXIT_REASON_MWAIT_INSTRUCTION]          = kvm_emulate_mwait,    [EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,    [EXIT_REASON_MONITOR_INSTRUCTION]     = kvm_emulate_monitor,    [EXIT_REASON_INVEPT]                  = handle_vmx_instruction,    [EXIT_REASON_INVVPID]                 = handle_vmx_instruction,    [EXIT_REASON_RDRAND]                  = kvm_handle_invalid_op,    [EXIT_REASON_RDSEED]                  = kvm_handle_invalid_op,    [EXIT_REASON_PML_FULL]              = handle_pml_full,    [EXIT_REASON_INVPCID]                 = handle_invpcid,    [EXIT_REASON_VMFUNC]              = handle_vmx_instruction,    [EXIT_REASON_PREEMPTION_TIMER]          = handle_preemption_timer,    [EXIT_REASON_ENCLS]              = handle_encls,    [EXIT_REASON_BUS_LOCK]                = handle_bus_lock_vmexit,};