# HG changeset patch
# User Keir Fraser <keir.fraser@citrix.com>
# Date 1281707886 -3600
# Node ID 3cee41690fa26853acb0be65065c52f6029ca599
# Parent  01d185dab39e9be399b203bec91870e04f576c23
x2APIC: Improve x2APIC suspend/resume
References: bnc#656369, bnc#658704

x2apic depends on interrupt remapping, so it should disable interrupt
remapping behind x2apic disabling. And also this patch wraps
__enable_x2apic to get rid of duplicated code.

Signed-off-by: Weidong Han <weidong.han@intel.com>

Index: xen-4.0.1-testing/xen/arch/x86/apic.c
===================================================================
--- xen-4.0.1-testing.orig/xen/arch/x86/apic.c
+++ xen-4.0.1-testing/xen/arch/x86/apic.c
@@ -496,9 +496,20 @@ static void apic_pm_activate(void)
     apic_pm_state.active = 1;
 }
 
+static void __enable_x2apic(void)
+{
+    u32 lo, hi;
+
+    rdmsr(MSR_IA32_APICBASE, lo, hi);
+    if ( !(lo & MSR_IA32_APICBASE_EXTD) )
+    {
+        lo |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
+        wrmsr(MSR_IA32_APICBASE, lo, 0);
+    }
+}
+
 static void resume_x2apic(void)
 {
-    uint64_t msr_content;
     struct IO_APIC_route_entry **ioapic_entries = NULL;
 
     ASSERT(x2apic_enabled);
@@ -520,14 +531,7 @@ static void resume_x2apic(void)
     mask_IO_APIC_setup(ioapic_entries);
 
     iommu_enable_IR();
-
-    rdmsrl(MSR_IA32_APICBASE, msr_content);
-    if ( !(msr_content & MSR_IA32_APICBASE_EXTD) )
-    {
-        msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
-        msr_content = (uint32_t)msr_content;
-        wrmsrl(MSR_IA32_APICBASE, msr_content);
-    }
+    __enable_x2apic();
 
     restore_IO_APIC_setup(ioapic_entries);
     unmask_8259A();
@@ -743,9 +747,10 @@ int lapic_suspend(void)
     apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
     apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
     apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
-    
+
     local_irq_save(flags);
     disable_local_APIC();
+    iommu_disable_IR();
     local_irq_restore(flags);
     return 0;
 }
@@ -1042,15 +1047,8 @@ static void enable_bsp_x2apic(void)
 
     if ( !x2apic_preenabled )
     {
-        u32 lo, hi;
-
-        rdmsr(MSR_IA32_APICBASE, lo, hi);
-        if ( !(lo & MSR_IA32_APICBASE_EXTD) )
-        {
-            lo |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
-            wrmsr(MSR_IA32_APICBASE, lo, 0);
-            printk("x2APIC mode enabled.\n");
-        }
+        __enable_x2apic();
+        printk("x2APIC mode enabled.\n");
     }
 
 restore_out:
@@ -1064,19 +1062,12 @@ out:
 
 static void enable_ap_x2apic(void)
 {
-    u32 lo, hi;
-
     ASSERT(smp_processor_id() != 0);
 
     /* APs only enable x2apic when BSP did so. */
     BUG_ON(!x2apic_enabled);
 
-    rdmsr(MSR_IA32_APICBASE, lo, hi);
-    if ( !(lo & MSR_IA32_APICBASE_EXTD) )
-    {
-        lo |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD;
-        wrmsr(MSR_IA32_APICBASE, lo, 0);
-    }
+    __enable_x2apic();
 }
 
 void enable_x2apic(void)
Index: xen-4.0.1-testing/xen/drivers/passthrough/vtd/intremap.c
===================================================================
--- xen-4.0.1-testing.orig/xen/drivers/passthrough/vtd/intremap.c
+++ xen-4.0.1-testing/xen/drivers/passthrough/vtd/intremap.c
@@ -873,6 +873,24 @@ int iommu_enable_IR(void)
 }
 
 /*
+ * This function is used to disable Interrutp remapping when
+ * suspend local apic
+ */
+void iommu_disable_IR(void)
+{
+    struct acpi_drhd_unit *drhd;
+
+    if ( !iommu_supports_eim() )
+        return;
+
+    for_each_drhd_unit ( drhd )
+        disable_intremap(drhd->iommu);
+
+    for_each_drhd_unit ( drhd )
+        disable_qinval(drhd->iommu);
+}
+
+/*
  * Check if interrupt remapping is enabled or not
  * return 1: enabled
  * return 0: not enabled
Index: xen-4.0.1-testing/xen/drivers/passthrough/vtd/iommu.c
===================================================================
--- xen-4.0.1-testing.orig/xen/drivers/passthrough/vtd/iommu.c
+++ xen-4.0.1-testing/xen/drivers/passthrough/vtd/iommu.c
@@ -2126,10 +2126,11 @@ static void vtd_suspend(void)
 
         iommu_disable_translation(iommu);
 
-        if ( iommu_intremap )
-            disable_intremap(iommu);
-
-        if ( iommu_qinval )
+        /* If interrupt remapping is enabled, queued invalidation
+         * will be disabled following interupt remapping disabling
+         * in local apic suspend
+         */
+        if ( !iommu_intremap && iommu_qinval )
             disable_qinval(iommu);
     }
 }
Index: xen-4.0.1-testing/xen/include/xen/iommu.h
===================================================================
--- xen-4.0.1-testing.orig/xen/include/xen/iommu.h
+++ xen-4.0.1-testing/xen/include/xen/iommu.h
@@ -59,6 +59,7 @@ struct iommu {
 int iommu_setup(void);
 int iommu_supports_eim(void);
 int iommu_enable_IR(void);
+void iommu_disable_IR(void);
 int intremap_enabled(void);
 
 int iommu_add_device(struct pci_dev *pdev);