xen/xend-managed-pci-device.patch

pci passthrough: handle managed pci devices

Handle managed pci devices for libvirt usage. If a pci device is set
"managed=1", it will be made assignable (unbound from original driver and bind
to pcistub driver) before vm start and reattach to original driver after vm
shut off.

FATE#313570

Note: This patch was rejected upstream since xend is deprecated.  See the
following thread for details

http://lists.xen.org/archives/html/xen-devel/2013-01/msg01145.html

Signed-off-by: Chunyan Liu <cyliu@suse.com>

Index: xen-4.4.0-testing/tools/python/xen/util/pci.py
===================================================================
--- xen-4.4.0-testing.orig/tools/python/xen/util/pci.py
+++ xen-4.4.0-testing/tools/python/xen/util/pci.py
@@ -20,6 +20,8 @@ from xen.xend import sxp
 from xen.xend.XendConstants import AUTO_PHP_SLOT
 from xen.xend.XendSXPDev import dev_dict_to_sxp
 from xen.xend.XendLogging import log
+from xen.xend.xenstore.xstransact import xstransact
+from xen.xend.XendError import XendError
 
 # for 2.3 compatibility
 try:
@@ -27,9 +29,11 @@ try:
 except NameError:
     from sets import Set as set
 
+XS_PCIBACK_PATH = '/xm/pciback'
 PROC_PCI_PATH = '/proc/bus/pci/devices'
 PROC_PCI_NUM_RESOURCES = 7
 
+SYSFS_PCI_DRVS_PATH = 'bus/pci/drivers'
 SYSFS_PCI_DEVS_PATH = '/bus/pci/devices'
 SYSFS_PCI_DEV_RESOURCE_PATH = '/resource'
 SYSFS_PCI_DEV_CONFIG_PATH = '/config'
@@ -161,7 +165,7 @@ def PCI_BDF(domain, bus, slot, func):
 
 def check_pci_opts(opts):
     def f((k, v)):
-        if k not in ['msitranslate', 'power_mgmt'] or \
+        if k not in ['msitranslate', 'power_mgmt', 'managed'] or \
            not v.lower() in ['0', '1', 'yes', 'no']:
             raise PciDeviceParseError('Invalid pci option %s=%s: ' % (k, v))
 
@@ -427,6 +431,9 @@ def __pci_dict_to_fmt_str(fmt, dev):
 def pci_dict_to_bdf_str(dev):
     return __pci_dict_to_fmt_str('%04x:%02x:%02x.%01x', dev)
 
+def pci_dict_to_xs_bdf_str(dev):
+    return __pci_dict_to_fmt_str('%04x-%02x-%02x-%01x', dev)
+
 def pci_dict_to_xc_str(dev):
     return __pci_dict_to_fmt_str('0x%x, 0x%x, 0x%x, 0x%x', dev)
 
@@ -561,6 +568,115 @@ def find_all_assignable_devices():
         dev_list = dev_list + [dev]
     return dev_list
 
+def pci_assignable_add(dev):
+    '''detach pci device from driver that we need to unbind from and rebind
+       to pciback driver, then it can be assigned to guest.
+    '''
+    sysfs_mnt = find_sysfs_mnt()
+    pcidev_path = sysfs_mnt + SYSFS_PCI_DEVS_PATH
+    pciback_path = sysfs_mnt + SYSFS_PCIBACK_PATH
+
+    # See if the device exists
+    pci_bdf = pci_dict_to_bdf_str(dev)
+    path = pcidev_path + '/' + pci_bdf
+    if not os.path.exists(path):
+        log.debug("Pci device %s doesn't exist" % pci_bdf)
+        return -1
+
+    # Check to see if it's already assigned to pciback
+    path = pciback_path + '/' + pci_bdf
+    if os.path.exists(path):
+        log.debug("Pci device %s is already assigned to pciback" % pci_bdf)
+        return 0
+
+    # Check to see if there's already a driver that we need to unbind from
+    path = pcidev_path + '/' + pci_bdf + '/driver'
+    drv_path = None
+    if os.path.exists(path):
+        drv_path = os.path.realpath(path).replace(" ", "\ ")
+        cmd = 'echo %s > %s/unbind' % (pci_bdf, drv_path)
+        if os.system(cmd):
+            log.debug("Couldn't unbind device")
+            return -1;
+
+    # Store driver_path for rebinding to dom0
+    if drv_path is not None:
+        xs_pci_bdf = pci_dict_to_xs_bdf_str(dev)
+        path = XS_PCIBACK_PATH + '/' + xs_pci_bdf
+        xstransact.Mkdir(path)
+        xstransact.Write(path, 'driver_path', drv_path)
+    else:
+        log.debug("Not bound to a driver, will not be rebound")
+
+    # Bind to pciback
+    try:
+        # Scan through /sys/.../pciback/slots looking for pcidev's BDF
+        slots = os.popen('cat %s/slots' % pciback_path).read()
+        if re.search(pci_bdf, slots) is None:
+            # write bdf to new_slot
+            cmd = 'echo %s > %s/new_slot' % (pci_bdf, pciback_path)
+            if os.system(cmd):
+                raise XendError("Couldn't add device to pciback new_slot")
+
+        # Bind to pciback
+        cmd = 'echo %s > %s/bind' % (pci_bdf, pciback_path)
+        if os.system(cmd):
+            raise XendError("Couldn't bind device to pciback")
+    except XendError:
+        # rebind to original driver
+        if drv_path is not None:
+            log.debug("Rebind to original driver")
+            cmd = 'echo %s > %s/bind' % (pci_bdf, drv_path)
+            if os.system(cmd):
+                log.debug("Failed to rebind")
+        return -1
+
+    return 0
+
+def pci_assignable_remove(dev):
+    '''unbind pci device from pciback, and rebind to host pci driver where it
+       was detached from in pci-assignable-add.
+    '''
+    sysfs_mnt = find_sysfs_mnt()
+    pcidrv_path = sysfs_mnt + SYSFS_PCI_DRVS_PATH
+    pciback_path = sysfs_mnt + SYSFS_PCIBACK_PATH
+    pci_bdf = pci_dict_to_bdf_str(dev)
+
+    # Unbind from pciback
+    path = pciback_path + '/' + pci_bdf
+    if os.path.exists(path):
+        # unbind
+        cmd = 'echo %s > %s/unbind' % (pci_bdf, pciback_path)
+        if os.system(cmd):
+            log.debug("Couldn't unbind device to pciback")
+            return -1
+
+        # remove slots if necessary
+        slots = os.popen('cat %s/slots' % pciback_path).read()
+        if re.search(pci_bdf, slots):
+            # write bdf to remove_slot
+            cmd = 'echo %s > %s/remove_slot' % (pci_bdf, pciback_path)
+            if os.system(cmd):
+                log.debug("Couldn't remove pciback slot")
+                return -1
+    else:
+        log.debug("Not bound to pciback")
+
+    # Rebind if necessary
+    xs_pci_bdf = pci_dict_to_xs_bdf_str(dev)
+    path = XS_PCIBACK_PATH + '/' + xs_pci_bdf
+    drv_path = xstransact.Read(path, 'driver_path')
+    if drv_path:
+        cmd = 'echo %s > %s/bind' % (pci_bdf, drv_path)
+        if os.system(cmd):
+           log.debug("Couldn't rebind to driver %s" % drv_path)
+           return -1
+        xstransact.Remove(path)
+    else:
+        log.debug("Counldn't find path for original driver. Not rebinding")
+
+    return 0
+
 def transform_list(target, src):
     ''' src: its element is pci string (Format: xxxx:xx:xx.x).
         target: its element is pci string, or a list of pci string.
Index: xen-4.4.0-testing/tools/python/xen/xend/XendDomainInfo.py
===================================================================
--- xen-4.4.0-testing.orig/tools/python/xen/xend/XendDomainInfo.py
+++ xen-4.4.0-testing/tools/python/xen/xend/XendDomainInfo.py
@@ -305,7 +305,8 @@ def dom_get(dom):
     return None
 
 from xen.xend.server.pciif import parse_pci_name, PciDevice,\
-    get_assigned_pci_devices, get_all_assigned_pci_devices
+    get_assigned_pci_devices, get_all_assigned_pci_devices,\
+    prepare_host_pci_devices, reattach_host_pci_devices
 
 
 def do_FLR(domid, is_hvm):
@@ -319,6 +320,20 @@ def do_FLR(domid, is_hvm):
                     "parse it's resources - "+str(e))
         dev.do_FLR(is_hvm, xoptions.get_pci_dev_assign_strict_check())
 
+def prepare_domain_pci_devices(domconfig):
+    ordered_refs = domconfig.ordered_device_refs()
+    for dev_uuid in ordered_refs:
+        devclass, devconfig = domconfig['devices'][dev_uuid]
+        if devclass == 'pci':
+            prepare_host_pci_devices(devconfig)
+
+def reattach_domain_pci_devices(domconfig):
+    ordered_refs = domconfig.ordered_device_refs()
+    for dev_uuid in ordered_refs:
+        devclass, devconfig = domconfig['devices'][dev_uuid]
+        if devclass == 'pci':
+            reattach_host_pci_devices(devconfig)
+
 class XendDomainInfo:
     """An object represents a domain.
 
@@ -472,6 +487,7 @@ class XendDomainInfo:
 
         if self._stateGet() in (XEN_API_VM_POWER_STATE_HALTED, XEN_API_VM_POWER_STATE_SUSPENDED, XEN_API_VM_POWER_STATE_CRASHED):
             try:
+                prepare_domain_pci_devices(self.info);
                 XendTask.log_progress(0, 30, self._constructDomain)
                 XendTask.log_progress(31, 60, self._initDomain)
                 
@@ -498,6 +514,7 @@ class XendDomainInfo:
         state = self._stateGet()
         if state in (DOM_STATE_SUSPENDED, DOM_STATE_HALTED):
             try:
+                prepare_domain_pci_devices(self.info)
                 self._constructDomain()
 
                 try:
@@ -714,6 +731,8 @@ class XendDomainInfo:
         the device.
         """
 
+        if self.domid is None:
+            return
         self.iommu_check_pod_mode()
 
         # Test whether the devices can be assigned
@@ -853,6 +872,9 @@ class XendDomainInfo:
 
         if self.domid is not None:
             try:
+                if dev_type == 'pci':
+                    prepare_host_pci_devices(dev_config_dict)
+
                 dev_config_dict['devid'] = devid = \
                     self._createDevice(dev_type, dev_config_dict)
                 if dev_type == 'tap2':
@@ -866,6 +888,7 @@ class XendDomainInfo:
                 if dev_type == 'pci':
                     for dev in dev_config_dict['devs']:
                         XendAPIStore.deregister(dev['uuid'], 'DPCI')
+                    reattach_host_pci_devices(dev_config_dict)
                 elif dev_type == 'vscsi':
                     for dev in dev_config_dict['devs']:
                         XendAPIStore.deregister(dev['uuid'], 'DSCSI')
@@ -910,6 +933,10 @@ class XendDomainInfo:
         dev_config = pci_convert_sxp_to_dict(dev_sxp)
         dev = dev_config['devs'][0]
 
+        # For attach only. For boot, prepare work has been done already in earlier stage.
+        if self.domid is not None and pci_state == 'Initialising' and pci_sub_state != 'Booting':
+            prepare_host_pci_devices(dev_config)
+
         stubdomid = self.getStubdomDomid()
         # Do HVM specific processing
         if self.info.is_hvm():
@@ -986,6 +1013,9 @@ class XendDomainInfo:
                 new_dev_sxp = dev_control.configuration(devid)
                 self.info.device_update(dev_uuid, new_dev_sxp)
 
+            if pci_state == 'Closing':
+                reattach_host_pci_devices(dev_config)
+
             # If there is no device left, destroy pci and remove config.
             if num_devs == 0:
                 if self.info.is_hvm():
@@ -3175,6 +3205,7 @@ class XendDomainInfo:
                 log.debug("%s KiB need to add to Memory pool" %self.alloc_mem)
                 MemoryPool.instance().increase_memory(self.alloc_mem)
 
+        reattach_domain_pci_devices(self.info)
         self._cleanup_phantom_devs(paths)
         self._cleanupVm()
 
Index: xen-4.4.0-testing/tools/python/xen/xend/server/pciif.py
===================================================================
--- xen-4.4.0-testing.orig/tools/python/xen/xend/server/pciif.py
+++ xen-4.4.0-testing/tools/python/xen/xend/server/pciif.py
@@ -86,6 +86,48 @@ def get_all_assigned_pci_devices(domid =
             pci_str_list = pci_str_list + get_assigned_pci_devices(int(d))
     return pci_str_list
 
+def reattach_host_pci_devices(devconfig):
+    pci_dev_list = devconfig.get('devs', [])
+    for pci_dev in pci_dev_list:
+        managed = 0
+        pci_opts_config = pci_dev.get('opts', [])
+        for opt in pci_opts_config:
+            if opt[0] == 'managed':
+                managed = opt[1]
+        if managed:
+            if pci_assignable_remove(pci_dev) != 0:
+               raise VmError('pci_assignable_remove failed')
+
+def detach_host_pci_devices(devconfig):
+    pci_dev_list = devconfig.get('devs', [])
+    reattach = 0
+    for pci_dev in pci_dev_list:
+        managed = 0
+        pci_opts_config = pci_dev.get('opts', [])
+        for opt in pci_opts_config:
+            if opt[0] == 'managed':
+                managed = opt[1]
+        if managed:
+            if pci_assignable_add(pci_dev) != 0:
+                log.debug('pci_assignable_add failed')
+                reattach = 1
+                break
+
+    if reattach:
+        reattach_host_pci_devices(devconfig)
+        raise VmError('detach_host_pci_devices failed')
+
+def prepare_host_pci_devices(devconfig):
+    # Test whether the device used by other domain
+    pci_dev_list = devconfig.get('devs', [])
+    for pci_dev in pci_dev_list:
+        pci_name = pci_dict_to_bdf_str(pci_dev)
+        if pci_name in get_all_assigned_pci_devices():
+            raise VmError("failed to assign device %s that has"
+                          " already been assigned to other domain." % pci_name)
+    # Detach 'managed' devices
+    detach_host_pci_devices(devconfig)
+
 class PciController(DevController):
 
     def __init__(self, vm):