xen/5583da09-x86-MSI-track-host-and-guest-masking-separately.patch

References: bsc#907514 bsc#910258 bsc#918984 bsc#923967

# Commit ad28e42bd1d28d746988ed71654e8aa670629753
# Date 2015-06-19 10:59:53 +0200
# Author Jan Beulich <jbeulich@suse.com>
# Committer Jan Beulich <jbeulich@suse.com>
x86/MSI: track host and guest masking separately

In particular we want to avoid losing track of our own intention to
have an entry masked. Physical unmasking now happens only when both
host and guest requested so.

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

# Commit 84d6add5593d865736831d150da7c38588f669f6
# Date 2015-07-10 12:36:24 +0200
# Author Jan Beulich <jbeulich@suse.com>
# Committer Jan Beulich <jbeulich@suse.com>
x86/MSI: fix guest unmasking when handling IRQ via event channel

Rather than assuming only PV guests need special treatment (and
dealing with that directly when an IRQ gets set up), keep all guest MSI
IRQs masked until either the (HVM) guest unmasks them via vMSI or the
(PV, PVHVM, or PVH) guest sets up an event channel for it.

To not further clutter the common evtchn_bind_pirq() with x86-specific
code, introduce an arch_evtchn_bind_pirq() hook instead.

Reported-by: Sander Eikelenboom <linux@eikelenboom.it>
Signed-off-by: Jan Beulich <jbeulich@suse.com>
Tested-by: Sander Eikelenboom <linux@eikelenboom.it>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>

Index: xen-4.5.2-testing/xen/arch/x86/hpet.c
===================================================================
--- xen-4.5.2-testing.orig/xen/arch/x86/hpet.c
+++ xen-4.5.2-testing/xen/arch/x86/hpet.c
@@ -240,7 +240,7 @@ static void hpet_msi_unmask(struct irq_d
     cfg = hpet_read32(HPET_Tn_CFG(ch->idx));
     cfg |= HPET_TN_ENABLE;
     hpet_write32(cfg, HPET_Tn_CFG(ch->idx));
-    ch->msi.msi_attrib.masked = 0;
+    ch->msi.msi_attrib.host_masked = 0;
 }
 
 static void hpet_msi_mask(struct irq_desc *desc)
@@ -251,7 +251,7 @@ static void hpet_msi_mask(struct irq_des
     cfg = hpet_read32(HPET_Tn_CFG(ch->idx));
     cfg &= ~HPET_TN_ENABLE;
     hpet_write32(cfg, HPET_Tn_CFG(ch->idx));
-    ch->msi.msi_attrib.masked = 1;
+    ch->msi.msi_attrib.host_masked = 1;
 }
 
 static int hpet_msi_write(struct hpet_event_channel *ch, struct msi_msg *msg)
Index: xen-4.5.2-testing/xen/arch/x86/hvm/vmsi.c
===================================================================
--- xen-4.5.2-testing.orig/xen/arch/x86/hvm/vmsi.c
+++ xen-4.5.2-testing/xen/arch/x86/hvm/vmsi.c
@@ -219,7 +219,6 @@ static int msixtbl_read(
 {
     unsigned long offset;
     struct msixtbl_entry *entry;
-    void *virt;
     unsigned int nr_entry, index;
     int r = X86EMUL_UNHANDLEABLE;
 
@@ -253,13 +252,20 @@ static int msixtbl_read(
     }
     if ( offset == PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET )
     {
-        virt = msixtbl_addr_to_virt(entry, address);
+        const struct msi_desc *msi_desc;
+        void *virt = msixtbl_addr_to_virt(entry, address);
+
         if ( !virt )
             goto out;
+        msi_desc = virt_to_msi_desc(entry->pdev, virt);
+        if ( !msi_desc )
+            goto out;
         if ( len == 4 )
-            *pval = readl(virt);
+            *pval = MASK_INSR(msi_desc->msi_attrib.guest_masked,
+                              PCI_MSIX_VECTOR_BITMASK);
         else
-            *pval |= (u64)readl(virt) << 32;
+            *pval |= (u64)MASK_INSR(msi_desc->msi_attrib.guest_masked,
+                                    PCI_MSIX_VECTOR_BITMASK) << 32;
     }
     
     r = X86EMUL_OKAY;
@@ -277,7 +283,7 @@ static int msixtbl_write(struct vcpu *v,
     void *virt;
     unsigned int nr_entry, index;
     int r = X86EMUL_UNHANDLEABLE;
-    unsigned long flags, orig;
+    unsigned long flags;
     struct irq_desc *desc;
 
     if ( (len != 4 && len != 8) || (address & (len - 1)) )
@@ -337,37 +343,7 @@ static int msixtbl_write(struct vcpu *v,
 
     ASSERT(msi_desc == desc->msi_desc);
    
-    orig = readl(virt);
-
-    /*
-     * Do not allow guest to modify MSI-X control bit if it is masked 
-     * by Xen. We'll only handle the case where Xen thinks that
-     * bit is unmasked, but hardware has silently masked the bit
-     * (in case of SR-IOV VF reset, etc). On the other hand, if Xen 
-     * thinks that the bit is masked, but it's really not, 
-     * we log a warning.
-     */
-    if ( msi_desc->msi_attrib.masked )
-    {
-        if ( !(orig & PCI_MSIX_VECTOR_BITMASK) )
-            printk(XENLOG_WARNING "MSI-X control bit is unmasked when"
-                   " it is expected to be masked [%04x:%02x:%02x.%u]\n", 
-                   entry->pdev->seg, entry->pdev->bus,
-                   PCI_SLOT(entry->pdev->devfn), 
-                   PCI_FUNC(entry->pdev->devfn));
-
-        goto unlock;
-    }
-
-    /*
-     * The mask bit is the only defined bit in the word. But we 
-     * ought to preserve the reserved bits. Clearing the reserved 
-     * bits can result in undefined behaviour (see PCI Local Bus
-     * Specification revision 2.3).
-     */
-    val &= PCI_MSIX_VECTOR_BITMASK;
-    val |= (orig & ~PCI_MSIX_VECTOR_BITMASK);
-    writel(val, virt);
+    guest_mask_msi_irq(desc, !!(val & PCI_MSIX_VECTOR_BITMASK));
 
 unlock:
     spin_unlock_irqrestore(&desc->lock, flags);
Index: xen-4.5.2-testing/xen/arch/x86/irq.c
===================================================================
--- xen-4.5.2-testing.orig/xen/arch/x86/irq.c
+++ xen-4.5.2-testing/xen/arch/x86/irq.c
@@ -2503,6 +2503,25 @@ int unmap_domain_pirq_emuirq(struct doma
     return ret;
 }
 
+void arch_evtchn_bind_pirq(struct domain *d, int pirq)
+{
+    int irq = domain_pirq_to_irq(d, pirq);
+    struct irq_desc *desc;
+    unsigned long flags;
+
+    if ( irq <= 0 )
+        return;
+
+    if ( is_hvm_domain(d) )
+        map_domain_emuirq_pirq(d, pirq, IRQ_PT);
+
+    desc = irq_to_desc(irq);
+    spin_lock_irqsave(&desc->lock, flags);
+    if ( desc->msi_desc )
+        guest_mask_msi_irq(desc, 0);
+    spin_unlock_irqrestore(&desc->lock, flags);
+}
+
 bool_t hvm_domain_use_pirq(const struct domain *d, const struct pirq *pirq)
 {
     return is_hvm_domain(d) && pirq &&
Index: xen-4.5.2-testing/xen/arch/x86/msi.c
===================================================================
--- xen-4.5.2-testing.orig/xen/arch/x86/msi.c
+++ xen-4.5.2-testing/xen/arch/x86/msi.c
@@ -349,9 +349,10 @@ int msi_maskable_irq(const struct msi_de
            || entry->msi_attrib.maskbit;
 }
 
-static void msi_set_mask_bit(struct irq_desc *desc, int flag)
+static void msi_set_mask_bit(struct irq_desc *desc, bool_t host, bool_t guest)
 {
     struct msi_desc *entry = desc->msi_desc;
+    bool_t flag = host || guest;
 
     ASSERT(spin_is_locked(&desc->lock));
     BUG_ON(!entry || !entry->dev);
@@ -383,7 +384,8 @@ static void msi_set_mask_bit(struct irq_
         BUG();
         break;
     }
-    entry->msi_attrib.masked = !!flag;
+    entry->msi_attrib.host_masked = host;
+    entry->msi_attrib.guest_masked = guest;
 }
 
 static int msi_get_mask_bit(const struct msi_desc *entry)
@@ -405,20 +407,30 @@ static int msi_get_mask_bit(const struct
 
 void mask_msi_irq(struct irq_desc *desc)
 {
-    msi_set_mask_bit(desc, 1);
+    msi_set_mask_bit(desc, 1, desc->msi_desc->msi_attrib.guest_masked);
 }
 
 void unmask_msi_irq(struct irq_desc *desc)
 {
-    msi_set_mask_bit(desc, 0);
+    msi_set_mask_bit(desc, 0, desc->msi_desc->msi_attrib.guest_masked);
+}
+
+void guest_mask_msi_irq(struct irq_desc *desc, bool_t mask)
+{
+    msi_set_mask_bit(desc, desc->msi_desc->msi_attrib.host_masked, mask);
 }
 
 static unsigned int startup_msi_irq(struct irq_desc *desc)
 {
-    unmask_msi_irq(desc);
+    msi_set_mask_bit(desc, 0, !!(desc->status & IRQ_GUEST));
     return 0;
 }
 
+static void shutdown_msi_irq(struct irq_desc *desc)
+{
+    msi_set_mask_bit(desc, 1, 1);
+}
+
 void ack_nonmaskable_msi_irq(struct irq_desc *desc)
 {
     irq_complete_move(desc);
@@ -443,7 +455,7 @@ void end_nonmaskable_msi_irq(struct irq_
 static hw_irq_controller pci_msi_maskable = {
     .typename     = "PCI-MSI/-X",
     .startup      = startup_msi_irq,
-    .shutdown     = mask_msi_irq,
+    .shutdown     = shutdown_msi_irq,
     .enable       = unmask_msi_irq,
     .disable      = mask_msi_irq,
     .ack          = ack_maskable_msi_irq,
@@ -593,7 +605,8 @@ static int msi_capability_init(struct pc
         entry[i].msi_attrib.is_64 = is_64bit_address(control);
         entry[i].msi_attrib.entry_nr = i;
         entry[i].msi_attrib.maskbit = is_mask_bit_support(control);
-        entry[i].msi_attrib.masked = 1;
+        entry[i].msi_attrib.host_masked = 1;
+        entry[i].msi_attrib.guest_masked = 0;
         entry[i].msi_attrib.pos = pos;
         if ( entry[i].msi_attrib.maskbit )
             entry[i].msi.mpos = mpos;
@@ -819,7 +832,8 @@ static int msix_capability_init(struct p
         entry->msi_attrib.is_64 = 1;
         entry->msi_attrib.entry_nr = msi->entry_nr;
         entry->msi_attrib.maskbit = 1;
-        entry->msi_attrib.masked = 1;
+        entry->msi_attrib.host_masked = 1;
+        entry->msi_attrib.guest_masked = 1;
         entry->msi_attrib.pos = pos;
         entry->irq = msi->irq;
         entry->dev = dev;
@@ -1154,7 +1168,8 @@ int pci_restore_msi_state(struct pci_dev
 
         for ( i = 0; ; )
         {
-            msi_set_mask_bit(desc, entry[i].msi_attrib.masked);
+            msi_set_mask_bit(desc, entry[i].msi_attrib.host_masked,
+                             entry[i].msi_attrib.guest_masked);
 
             if ( !--nr )
                 break;
@@ -1306,7 +1321,7 @@ static void dump_msi(unsigned char key)
         else
             mask = '?';
         printk(" %-6s%4u vec=%02x%7s%6s%3sassert%5s%7s"
-               " dest=%08x mask=%d/%d/%c\n",
+               " dest=%08x mask=%d/%c%c/%c\n",
                type, irq,
                (data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT,
                data & MSI_DATA_DELIVERY_LOWPRI ? "lowest" : "fixed",
@@ -1314,7 +1329,10 @@ static void dump_msi(unsigned char key)
                data & MSI_DATA_LEVEL_ASSERT ? "" : "de",
                addr & MSI_ADDR_DESTMODE_LOGIC ? "log" : "phys",
                addr & MSI_ADDR_REDIRECTION_LOWPRI ? "lowest" : "cpu",
-               dest32, attr.maskbit, attr.masked, mask);
+               dest32, attr.maskbit,
+               attr.host_masked ? 'H' : ' ',
+               attr.guest_masked ? 'G' : ' ',
+               mask);
     }
 }
 
Index: xen-4.5.2-testing/xen/common/event_channel.c
===================================================================
--- xen-4.5.2-testing.orig/xen/common/event_channel.c
+++ xen-4.5.2-testing/xen/common/event_channel.c
@@ -445,10 +445,7 @@ static long evtchn_bind_pirq(evtchn_bind
 
     bind->port = port;
 
-#ifdef CONFIG_X86
-    if ( is_hvm_domain(d) && domain_pirq_to_irq(d, pirq) > 0 )
-        map_domain_emuirq_pirq(d, pirq, IRQ_PT);
-#endif
+    arch_evtchn_bind_pirq(d, pirq);
 
  out:
     spin_unlock(&d->event_lock);
Index: xen-4.5.2-testing/xen/drivers/passthrough/amd/iommu_init.c
===================================================================
--- xen-4.5.2-testing.orig/xen/drivers/passthrough/amd/iommu_init.c
+++ xen-4.5.2-testing/xen/drivers/passthrough/amd/iommu_init.c
@@ -451,7 +451,7 @@ static void iommu_msi_unmask(struct irq_
     spin_lock_irqsave(&iommu->lock, flags);
     amd_iommu_msi_enable(iommu, IOMMU_CONTROL_ENABLED);
     spin_unlock_irqrestore(&iommu->lock, flags);
-    iommu->msi.msi_attrib.masked = 0;
+    iommu->msi.msi_attrib.host_masked = 0;
 }
 
 static void iommu_msi_mask(struct irq_desc *desc)
@@ -464,7 +464,7 @@ static void iommu_msi_mask(struct irq_de
     spin_lock_irqsave(&iommu->lock, flags);
     amd_iommu_msi_enable(iommu, IOMMU_CONTROL_DISABLED);
     spin_unlock_irqrestore(&iommu->lock, flags);
-    iommu->msi.msi_attrib.masked = 1;
+    iommu->msi.msi_attrib.host_masked = 1;
 }
 
 static unsigned int iommu_msi_startup(struct irq_desc *desc)
Index: xen-4.5.2-testing/xen/drivers/passthrough/vtd/iommu.c
===================================================================
--- xen-4.5.2-testing.orig/xen/drivers/passthrough/vtd/iommu.c
+++ xen-4.5.2-testing/xen/drivers/passthrough/vtd/iommu.c
@@ -999,7 +999,7 @@ static void dma_msi_unmask(struct irq_de
     sts &= ~DMA_FECTL_IM;
     dmar_writel(iommu->reg, DMAR_FECTL_REG, sts);
     spin_unlock_irqrestore(&iommu->register_lock, flags);
-    iommu->msi.msi_attrib.masked = 0;
+    iommu->msi.msi_attrib.host_masked = 0;
 }
 
 static void dma_msi_mask(struct irq_desc *desc)
@@ -1014,7 +1014,7 @@ static void dma_msi_mask(struct irq_desc
     sts |= DMA_FECTL_IM;
     dmar_writel(iommu->reg, DMAR_FECTL_REG, sts);
     spin_unlock_irqrestore(&iommu->register_lock, flags);
-    iommu->msi.msi_attrib.masked = 1;
+    iommu->msi.msi_attrib.host_masked = 1;
 }
 
 static unsigned int dma_msi_startup(struct irq_desc *desc)
Index: xen-4.5.2-testing/xen/include/asm-arm/irq.h
===================================================================
--- xen-4.5.2-testing.orig/xen/include/asm-arm/irq.h
+++ xen-4.5.2-testing/xen/include/asm-arm/irq.h
@@ -44,6 +44,8 @@ int route_irq_to_guest(struct domain *d,
                        const char *devname);
 void arch_move_irqs(struct vcpu *v);
 
+#define arch_evtchn_bind_pirq(d, pirq) ((void)((d) + (pirq)))
+
 /* Set IRQ type for an SPI */
 int irq_set_spi_type(unsigned int spi, unsigned int type);
 
Index: xen-4.5.2-testing/xen/include/asm-x86/msi.h
===================================================================
--- xen-4.5.2-testing.orig/xen/include/asm-x86/msi.h
+++ xen-4.5.2-testing/xen/include/asm-x86/msi.h
@@ -90,12 +90,13 @@ extern unsigned int pci_msix_get_table_l
 
 struct msi_desc {
 	struct msi_attrib {
-		__u8	type	: 5; 	/* {0: unused, 5h:MSI, 11h:MSI-X} */
-		__u8	maskbit	: 1; 	/* mask-pending bit supported ?   */
-		__u8	masked	: 1;
+		__u8	type;		/* {0: unused, 5h:MSI, 11h:MSI-X} */
+		__u8	pos;		/* Location of the MSI capability */
+		__u8	maskbit	: 1;	/* mask/pending bit supported ?   */
 		__u8	is_64	: 1;	/* Address size: 0=32bit 1=64bit  */
-		__u8	pos;	 	/* Location of the msi capability */
-		__u16	entry_nr;    	/* specific enabled entry 	  */
+		__u8	host_masked : 1;
+		__u8	guest_masked : 1;
+		__u16	entry_nr;	/* specific enabled entry 	  */
 	} msi_attrib;
 
 	struct list_head list;
@@ -236,6 +237,7 @@ void msi_compose_msg(unsigned vector, co
 void __msi_set_enable(u16 seg, u8 bus, u8 slot, u8 func, int pos, int enable);
 void mask_msi_irq(struct irq_desc *);
 void unmask_msi_irq(struct irq_desc *);
+void guest_mask_msi_irq(struct irq_desc *, bool_t mask);
 void ack_nonmaskable_msi_irq(struct irq_desc *);
 void end_nonmaskable_msi_irq(struct irq_desc *, u8 vector);
 void set_msi_affinity(struct irq_desc *, const cpumask_t *);
Index: xen-4.5.2-testing/xen/include/xen/irq.h
===================================================================
--- xen-4.5.2-testing.orig/xen/include/xen/irq.h
+++ xen-4.5.2-testing/xen/include/xen/irq.h
@@ -172,4 +172,8 @@ unsigned int set_desc_affinity(struct ir
 unsigned int arch_hwdom_irqs(domid_t);
 #endif
 
+#ifndef arch_evtchn_bind_pirq
+void arch_evtchn_bind_pirq(struct domain *, int pirq);
+#endif
+
 #endif /* __XEN_IRQ_H__ */