References: bsc#907514 bsc#910258 bsc#918984 bsc#923967 xen/MSI-X: drive maskall and enable bits through hypercalls Particularly the maskall bit has to be under exclusive hypervisor control (and since they live in the same config space field, the enable bit has to follow suit). Use the replacement hypercall interfaces. Signed-off-by: Jan Beulich Index: xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pass-through.c =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-traditional-dir-remote/hw/pass-through.c +++ xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pass-through.c @@ -814,9 +814,12 @@ static struct pt_reg_info_tbl pt_emu_reg .offset = PCI_MSI_FLAGS, // 2 .size = 2, .init_val = 0x0000, - .res_mask = 0x3800, - .ro_mask = 0x07FF, - .emu_mask = 0x0000, + /* This must not be split into res_mask (0x3800) and ro_mask (0x07FF) + * because even in permissive mode there must not be any write back + * to this register. + */ + .ro_mask = 0x3FFF, + .emu_mask = 0xC000, .init = pt_msixctrl_reg_init, .u.w.read = pt_word_reg_read, .u.w.write = pt_msixctrl_reg_write, @@ -4135,30 +4138,52 @@ static int pt_msixctrl_reg_write(struct uint16_t *value, uint16_t dev_value, uint16_t valid_mask) { struct pt_reg_info_tbl *reg = cfg_entry->reg; - uint16_t writable_mask = 0; + uint16_t writable_mask, val; uint16_t throughable_mask = get_throughable_mask(ptdev, reg, valid_mask); uint16_t old_ctrl = cfg_entry->data; /* modify emulate register */ writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask; - cfg_entry->data = PT_MERGE_VALUE(*value, cfg_entry->data, writable_mask); + val = PT_MERGE_VALUE(*value, cfg_entry->data, writable_mask); + cfg_entry->data = val; /* create value for writing to I/O device register */ *value = PT_MERGE_VALUE(*value, dev_value, throughable_mask); /* update MSI-X */ - if ((*value & PCI_MSIX_ENABLE) && !(*value & PCI_MSIX_MASK)) + if ((val & PCI_MSIX_ENABLE) && !(val & PCI_MSIX_MASK)) { if (ptdev->msi_trans_en) { PT_LOG("guest enabling MSI-X, disable MSI-INTx translation\n"); pt_disable_msi_translate(ptdev); } + if (!ptdev->msix->enabled) { + if (!ptdev->msix->maskall) + pt_msix_maskall(ptdev, 1); + pt_msix_enable(ptdev); + } pt_msix_update(ptdev); - } else if (!(*value & PCI_MSIX_ENABLE) && ptdev->msix->enabled) { - pt_msix_disable(ptdev); + ptdev->msix->enabled = 1; + ptdev->msix->maskall = 0; + pt_msix_maskall(ptdev, 0); + } else if (ptdev->msix->enabled) { + if (!(val & PCI_MSIX_ENABLE)) { + pt_msix_disable(ptdev); + ptdev->msix->enabled = 0; + } else if (!ptdev->msix->maskall) { + ptdev->msix->maskall = 1; + pt_msix_maskall(ptdev, 1); + } } - ptdev->msix->enabled = !!(*value & PCI_MSIX_ENABLE); + dev_value = pci_read_word(ptdev->pci_dev, ptdev->msix->ctrl_offset); + + if (ptdev->msix->enabled && !(dev_value & PCI_MSIX_ENABLE)) + PT_ERR("MSI-X unexpectedly disabled\n"); + else if ((dev_value & PCI_MSIX_ENABLE) && + ptdev->msix->maskall && + !(dev_value & PCI_MSIX_MASK)) + PT_ERR("MSI-X unexpectedly unmasked\n"); return 0; } Index: xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pass-through.h =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-traditional-dir-remote/hw/pass-through.h +++ xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pass-through.h @@ -206,6 +206,7 @@ struct msix_entry_info { struct pt_msix_info { uint32_t ctrl_offset; int enabled; + int maskall; int total_entries; int bar_index; uint64_t table_base; Index: xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pt-msi.c =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-traditional-dir-remote/hw/pt-msi.c +++ xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pt-msi.c @@ -41,20 +41,12 @@ void msi_set_enable(struct pt_dev *dev, static void msix_set_enable(struct pt_dev *dev, int en) { - uint16_t val = 0; - uint32_t address = 0; if (!dev->msix) return; - address = dev->msix->ctrl_offset; - if (!address) - return; - - val = pci_read_word(dev->pci_dev, address); - val &= ~PCI_MSIX_ENABLE; - if (en) - val |= PCI_MSIX_ENABLE; - pci_write_word(dev->pci_dev, address, val); + xc_physdev_msix_enable(xc_handle, dev->pci_dev->domain, dev->pci_dev->bus, + PCI_DEVFN(dev->pci_dev->dev, dev->pci_dev->func), + en); } /* MSI virtuailization functions */ @@ -349,6 +341,11 @@ int pt_msix_update(struct pt_dev *dev) return 0; } +void pt_msix_enable(struct pt_dev *dev) +{ + msix_set_enable(dev, 1); +} + void pt_msix_disable(struct pt_dev *dev) { PCIDevice *d = &dev->dev; @@ -394,6 +391,15 @@ void pt_msix_disable(struct pt_dev *dev) } } +int pt_msix_maskall(struct pt_dev *dev, int mask) +{ + return xc_physdev_msix_mask_all(xc_handle, dev->pci_dev->domain, + dev->pci_dev->bus, + PCI_DEVFN(dev->pci_dev->dev, + dev->pci_dev->func), + mask); +} + int pt_msix_update_remap(struct pt_dev *dev, int bar_index) { struct msix_entry_info *entry; Index: xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pt-msi.h =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-traditional-dir-remote/hw/pt-msi.h +++ xen-4.5.1-testing/tools/qemu-xen-traditional-dir-remote/hw/pt-msi.h @@ -106,9 +106,15 @@ int pt_msix_update(struct pt_dev *dev); void +pt_msix_enable(struct pt_dev *dev); + +void pt_msix_disable(struct pt_dev *dev); int +pt_msix_maskall(struct pt_dev *dev, int mask); + +int has_msix_mapping(struct pt_dev *dev, int bar_index); int Index: xen-4.5.1-testing/tools/qemu-xen-dir-remote/hw/xen/xen_pt.h =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-dir-remote/hw/xen/xen_pt.h +++ xen-4.5.1-testing/tools/qemu-xen-dir-remote/hw/xen/xen_pt.h @@ -181,6 +181,7 @@ typedef struct XenPTMSIXEntry { typedef struct XenPTMSIX { uint32_t ctrl_offset; bool enabled; + bool maskall; int total_entries; int bar_index; uint64_t table_base; @@ -293,7 +294,9 @@ int xen_pt_msix_init(XenPCIPassthroughSt void xen_pt_msix_delete(XenPCIPassthroughState *s); int xen_pt_msix_update(XenPCIPassthroughState *s); int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index); +void xen_pt_msix_enable(XenPCIPassthroughState *s); void xen_pt_msix_disable(XenPCIPassthroughState *s); +int xen_pt_msix_maskall(XenPCIPassthroughState *s, bool mask); static inline bool xen_pt_has_msix_mapping(XenPCIPassthroughState *s, int bar) { Index: xen-4.5.1-testing/tools/qemu-xen-dir-remote/hw/xen/xen_pt_config_init.c =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-dir-remote/hw/xen/xen_pt_config_init.c +++ xen-4.5.1-testing/tools/qemu-xen-dir-remote/hw/xen/xen_pt_config_init.c @@ -1436,32 +1436,58 @@ static int xen_pt_msixctrl_reg_write(Xen uint16_t dev_value, uint16_t valid_mask) { XenPTRegInfo *reg = cfg_entry->reg; - uint16_t writable_mask = 0; + uint16_t writable_mask, value; uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask); int debug_msix_enabled_old; /* modify emulate register */ writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask; - cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask); + value = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask); + cfg_entry->data = value; /* create value for writing to I/O device register */ *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask); + debug_msix_enabled_old = s->msix->enabled; + /* update MSI-X */ - if ((*val & PCI_MSIX_FLAGS_ENABLE) - && !(*val & PCI_MSIX_FLAGS_MASKALL)) { + if ((value & PCI_MSIX_FLAGS_ENABLE) + && !(value & PCI_MSIX_FLAGS_MASKALL)) { + if (!s->msix->enabled) { + if (!s->msix->maskall) { + xen_pt_msix_maskall(s, true); + } + xen_pt_msix_enable(s); + } xen_pt_msix_update(s); - } else if (!(*val & PCI_MSIX_FLAGS_ENABLE) && s->msix->enabled) { - xen_pt_msix_disable(s); + s->msix->enabled = true; + s->msix->maskall = false; + xen_pt_msix_maskall(s, false); + } else if (s->msix->enabled) { + if (!(value & PCI_MSIX_FLAGS_ENABLE)) { + xen_pt_msix_disable(s); + s->msix->enabled = false; + } else if (!s->msix->maskall) { + s->msix->maskall = true; + xen_pt_msix_maskall(s, true); + } } - debug_msix_enabled_old = s->msix->enabled; - s->msix->enabled = !!(*val & PCI_MSIX_FLAGS_ENABLE); if (s->msix->enabled != debug_msix_enabled_old) { XEN_PT_LOG(&s->dev, "%s MSI-X\n", s->msix->enabled ? "enable" : "disable"); } + xen_host_pci_get_word(&s->real_device, s->msix->ctrl_offset, &dev_value); + + if (s->msix->enabled && !(dev_value & PCI_MSIX_FLAGS_ENABLE)) { + XEN_PT_ERR(&s->dev, "MSI-X unexpectedly disabled\n"); + } else if ((dev_value & PCI_MSIX_FLAGS_ENABLE) && + s->msix->maskall && + !(dev_value & PCI_MSIX_FLAGS_MASKALL)) { + XEN_PT_ERR(&s->dev, "MSI-X unexpectedly unmasked\n"); + } + return 0; } @@ -1483,9 +1509,12 @@ static XenPTRegInfo xen_pt_emu_reg_msix[ .offset = PCI_MSI_FLAGS, .size = 2, .init_val = 0x0000, - .res_mask = 0x3800, - .ro_mask = 0x07FF, - .emu_mask = 0x0000, + /* This must not be split into res_mask (0x3800) and ro_mask (0x07FF) + * because even in permissive mode there must not be any write back + * to this register. + */ + .ro_mask = 0x3FFF, + .emu_mask = 0xC000, .init = xen_pt_msixctrl_reg_init, .u.w.read = xen_pt_word_reg_read, .u.w.write = xen_pt_msixctrl_reg_write, Index: xen-4.5.1-testing/tools/qemu-xen-dir-remote/hw/xen/xen_pt_msi.c =================================================================== --- xen-4.5.1-testing.orig/tools/qemu-xen-dir-remote/hw/xen/xen_pt_msi.c +++ xen-4.5.1-testing/tools/qemu-xen-dir-remote/hw/xen/xen_pt_msi.c @@ -301,8 +301,11 @@ static int msix_set_enable(XenPCIPassthr return -1; } - return msi_msix_enable(s, s->msix->ctrl_offset, PCI_MSIX_FLAGS_ENABLE, - enabled); + return xc_physdev_msix_enable(xen_xc, s->real_device.domain, + s->real_device.bus, + PCI_DEVFN(s->real_device.dev, + s->real_device.func), + enabled); } static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr) @@ -361,6 +364,11 @@ int xen_pt_msix_update(XenPCIPassthrough return 0; } +void xen_pt_msix_enable(XenPCIPassthroughState *s) +{ + msix_set_enable(s, true); +} + void xen_pt_msix_disable(XenPCIPassthroughState *s) { int i = 0; @@ -378,6 +386,15 @@ void xen_pt_msix_disable(XenPCIPassthrou } } +int xen_pt_msix_maskall(XenPCIPassthroughState *s, bool mask) +{ + return xc_physdev_msix_mask_all(xen_xc, s->real_device.domain, + s->real_device.bus, + PCI_DEVFN(s->real_device.dev, + s->real_device.func), + mask); +} + int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index) { XenPTMSIXEntry *entry;