Index: xen-3.2-testing/xen/arch/x86/hvm/vmx/vmcs.c =================================================================== --- xen-3.2-testing.orig/xen/arch/x86/hvm/vmx/vmcs.c +++ xen-3.2-testing/xen/arch/x86/hvm/vmx/vmcs.c @@ -38,6 +38,9 @@ #include #include +static int opt_vpid_enabled = 1; +boolean_param("vpid", opt_vpid_enabled); + /* Dynamic (run-time adjusted) execution control flags. */ u32 vmx_pin_based_exec_control __read_mostly; u32 vmx_cpu_based_exec_control __read_mostly; @@ -110,6 +113,8 @@ static void vmx_init_vmcs_config(void) opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | SECONDARY_EXEC_WBINVD_EXITING | SECONDARY_EXEC_ENABLE_EPT; + if ( opt_vpid_enabled ) + opt2 |= SECONDARY_EXEC_ENABLE_VPID; _vmx_secondary_exec_control = adjust_vmx_controls( min2, opt2, MSR_IA32_VMX_PROCBASED_CTLS2); @@ -315,6 +320,8 @@ int vmx_cpu_up(void) ept_sync_all(); + vpid_sync_all(); + return 1; } @@ -628,6 +635,13 @@ static int construct_vmcs(struct vcpu *v #endif } + if ( cpu_has_vmx_vpid ) + { + v->arch.hvm_vmx.vpid = v->vcpu_id + + v->domain->arch.hvm_domain.vmx_vpid_base; + __vmwrite(VIRTUAL_PROCESSOR_ID, v->arch.hvm_vmx.vpid); + } + vmx_vmcs_exit(v); paging_update_paging_modes(v); /* will update HOST & GUEST_CR3 as reqd */ @@ -821,6 +835,7 @@ void vmx_do_resume(struct vcpu *v) vmx_load_vmcs(v); hvm_migrate_timers(v); vmx_set_host_env(v); + vpid_sync_vcpu_all(v); } debug_state = v->domain->debugger_attached; @@ -975,6 +990,8 @@ void vmcs_dump_vcpu(struct vcpu *v) (uint32_t)vmr(TPR_THRESHOLD)); printk("EPT pointer = 0x%08x%08x\n", (uint32_t)vmr(EPT_POINTER_HIGH), (uint32_t)vmr(EPT_POINTER)); + printk("virtual processor ID = 0x%04x\n", + (uint32_t)vmr(VIRTUAL_PROCESSOR_ID)); vmx_vmcs_exit(v); } Index: xen-3.2-testing/xen/arch/x86/hvm/vmx/vmx.c =================================================================== --- xen-3.2-testing.orig/xen/arch/x86/hvm/vmx/vmx.c +++ xen-3.2-testing/xen/arch/x86/hvm/vmx/vmx.c @@ -58,18 +58,23 @@ static void vmx_ctxt_switch_to(struct vc static int vmx_alloc_vlapic_mapping(struct domain *d); static void vmx_free_vlapic_mapping(struct domain *d); +static int vmx_alloc_vpid(struct domain *d); +static void vmx_free_vpid(struct domain *d); static void vmx_install_vlapic_mapping(struct vcpu *v); static void vmx_update_guest_cr(struct vcpu *v, unsigned int cr); static void vmx_update_guest_efer(struct vcpu *v); static int vmx_domain_initialise(struct domain *d) { - return vmx_alloc_vlapic_mapping(d); + if ( vmx_alloc_vpid(d) == 0 ) + return vmx_alloc_vlapic_mapping(d); + return -EBUSY; } static void vmx_domain_destroy(struct domain *d) { vmx_free_vlapic_mapping(d); + vmx_free_vpid(d); } static int vmx_vcpu_initialise(struct vcpu *v) @@ -1122,6 +1127,7 @@ static void vmx_update_guest_cr(struct v vmx_load_pdptrs(v); } __vmwrite(GUEST_CR3, v->arch.hvm_vcpu.hw_cr[3]); + vpid_sync_vcpu_all(v); break; case 4: v->arch.hvm_vcpu.hw_cr[4] = HVM_CR4_HOST_MASK; @@ -1170,9 +1176,14 @@ static void vmx_update_guest_efer(struct static void vmx_flush_guest_tlbs(void) { - /* No tagged TLB support on VMX yet. The fact that we're in Xen - * at all means any guest will have a clean TLB when it's next run, - * because VMRESUME will flush it for us. */ + /* If VPID (i.e. tagged TLB support) is not enabled, the fact that + * we're in Xen at all means any guest will have a clean TLB when + * it's next run, because VMRESUME will flush it for us. + * + * If enabled, we invalidate all translations associated with all + * VPID values */ + if (0) + vpid_sync_all(); } static void vmx_inject_exception( @@ -1226,6 +1237,11 @@ static struct hvm_function_table vmx_fun .cpu_down = vmx_cpu_down, }; +static int vpid_bitmap_size; +static int vpid_bitmap_bytes; +static void *vpid_bitmap; +static spinlock_t vpid_lock; + void start_vmx(void) { static int bootstrapped; @@ -1265,6 +1281,26 @@ void start_vmx(void) vmx_function_table.hap_supported = 1; } + if ( cpu_has_vmx_vpid ) + { + printk("VMX: VPID is available.\n"); + + vpid_bitmap_size = (1 << VMCS_VPID_WIDTH) / MAX_VIRT_CPUS; + vpid_bitmap_bytes = vpid_bitmap_size / (BITS_PER_LONG / BYTES_PER_LONG); + vpid_bitmap = xmalloc_bytes(vpid_bitmap_bytes); + memset(vpid_bitmap, 0, vpid_bitmap_bytes); + if ( vpid_bitmap == NULL ) + { + printk("VMX: failed to allocate VPID bitmap.\n"); + return; + } + + /* vpid 0 is used by hypervisor itself */ + set_bit(0, vpid_bitmap); + + spin_lock_init(&vpid_lock); + } + setup_vmcs_dump(); hvm_enable(&vmx_function_table); @@ -2534,6 +2570,45 @@ static void vmx_free_vlapic_mapping(stru free_xenheap_page(mfn_to_virt(mfn)); } +static int vmx_alloc_vpid(struct domain *d) +{ + int vpid; + + if ( !cpu_has_vmx_vpid ) + return 0; + + spin_lock(&vpid_lock); + + vpid = find_first_zero_bit(vpid_bitmap, vpid_bitmap_size); + if ( vpid >= vpid_bitmap_size ) + { + spin_unlock(&vpid_lock); + printk("VPID is used up.\n"); + return -EBUSY; + } + if ( test_and_set_bit(vpid, vpid_bitmap) ) + { + spin_unlock(&vpid_lock); + printk("VPID allocation bug, fix your code!\n"); + return -EBUSY; + } + + spin_unlock(&vpid_lock); + + d->arch.hvm_domain.vmx_vpid_base = vpid * MAX_VIRT_CPUS; + return 0; +} + +static void vmx_free_vpid(struct domain *d) +{ + if ( !cpu_has_vmx_vpid ) + return; + + spin_lock(&vpid_lock); + clear_bit(d->arch.hvm_domain.vmx_vpid_base / MAX_VIRT_CPUS, vpid_bitmap); + spin_unlock(&vpid_lock); +} + static void vmx_install_vlapic_mapping(struct vcpu *v) { paddr_t virt_page_ma, apic_page_ma; Index: xen-3.2-testing/xen/arch/x86/mm/shadow/multi.c =================================================================== --- xen-3.2-testing.orig/xen/arch/x86/mm/shadow/multi.c +++ xen-3.2-testing/xen/arch/x86/mm/shadow/multi.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "private.h" #include "types.h" @@ -3106,6 +3107,7 @@ sh_invlpg(struct vcpu *v, unsigned long == SH_type_fl1_shadow ) { flush_tlb_local(); + vpid_sync_vcpu_all(v); return 0; } Index: xen-3.2-testing/xen/include/asm-x86/hvm/domain.h =================================================================== --- xen-3.2-testing.orig/xen/include/asm-x86/hvm/domain.h +++ xen-3.2-testing/xen/include/asm-x86/hvm/domain.h @@ -61,6 +61,7 @@ struct hvm_domain { uint64_t params[HVM_NR_PARAMS]; unsigned long vmx_apic_access_mfn; + unsigned long vmx_vpid_base; /* Memory ranges with pinned cache attributes. */ struct list_head pinned_cacheattr_ranges; Index: xen-3.2-testing/xen/include/asm-x86/hvm/vmx/vmcs.h =================================================================== --- xen-3.2-testing.orig/xen/include/asm-x86/hvm/vmx/vmcs.h +++ xen-3.2-testing/xen/include/asm-x86/hvm/vmx/vmcs.h @@ -90,6 +90,8 @@ struct arch_vmx_struct { u32 exec_control; u32 secondary_exec_control; + u16 vpid; + #ifdef __x86_64__ struct vmx_msr_state msr_state; unsigned long shadow_gs; @@ -156,6 +158,7 @@ extern u32 vmx_vmentry_control; #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 #define SECONDARY_EXEC_ENABLE_EPT 0x00000002 +#define SECONDARY_EXEC_ENABLE_VPID 0x00000020 #define SECONDARY_EXEC_WBINVD_EXITING 0x00000040 extern u32 vmx_secondary_exec_control; @@ -175,6 +178,8 @@ extern bool_t cpu_has_vmx_ins_outs_instr (vmx_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) #define cpu_has_vmx_ept \ (vmx_secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) +#define cpu_has_vmx_vpid \ + (vmx_secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID) /* GUEST_INTERRUPTIBILITY_INFO flags. */ #define VMX_INTR_SHADOW_STI 0x00000001 @@ -184,6 +189,7 @@ extern bool_t cpu_has_vmx_ins_outs_instr /* VMCS field encodings. */ enum vmcs_field { + VIRTUAL_PROCESSOR_ID = 0x00000000, GUEST_ES_SELECTOR = 0x00000800, GUEST_CS_SELECTOR = 0x00000802, GUEST_SS_SELECTOR = 0x00000804, @@ -323,6 +329,8 @@ enum vmcs_field { HOST_RIP = 0x00006c16, }; +#define VMCS_VPID_WIDTH (16) + void vmx_disable_intercept_for_msr(struct vcpu *v, u32 msr); int vmx_read_guest_msr(struct vcpu *v, u32 msr, u64 *val); int vmx_write_guest_msr(struct vcpu *v, u32 msr, u64 val); Index: xen-3.2-testing/xen/include/asm-x86/hvm/vmx/vmx.h =================================================================== --- xen-3.2-testing.orig/xen/include/asm-x86/hvm/vmx/vmx.h +++ xen-3.2-testing/xen/include/asm-x86/hvm/vmx/vmx.h @@ -172,6 +172,7 @@ int vmx_realmode_io_complete(void); #define VMRESUME_OPCODE ".byte 0x0f,0x01,0xc3\n" #define VMWRITE_OPCODE ".byte 0x0f,0x79\n" #define INVEPT_OPCODE ".byte 0x66,0x0f,0x38,0x80\n" /* m128,r64/32 */ +#define INVVPID_OPCODE ".byte 0x66,0x0f,0x38,0x81\n" /* m128,r64/32 */ #define VMXOFF_OPCODE ".byte 0x0f,0x01,0xc4\n" #define VMXON_OPCODE ".byte 0xf3,0x0f,0xc7\n" @@ -277,6 +278,23 @@ static inline void __invept(int ext, u64 : "memory"); } +static inline void __invvpid(int ext, u16 vpid, u64 gva) +{ + struct { + u64 vpid:16; + u64 rsvd:48; + u64 gva; + } __attribute__ ((packed)) operand = {vpid, 0, gva}; + + __asm__ __volatile__ ( INVVPID_OPCODE + MODRM_EAX_08 + /* CF==1 or ZF==1 --> rc = -1 */ + "ja 1f ; ud2 ; 1:\n" + : + : "a" (&operand), "c" (ext) + : "memory"); +} + static inline void __vmxoff(void) { asm volatile ( @@ -329,6 +347,22 @@ static inline void ept_sync_all(void) __invept(2, 0, 0); } +static inline void vpid_sync_vcpu_all(struct vcpu *v) +{ + if ( !cpu_has_vmx_vpid ) + return; + + __invvpid(1, v->arch.hvm_vmx.vpid, 0); +} + +static inline void vpid_sync_all(void) +{ + if ( !cpu_has_vmx_vpid ) + return; + + __invvpid(2, 0, 0); +} + static inline void __vmx_inject_exception( struct vcpu *v, int trap, int type, int error_code) {