1
#============================================================================
2
# This library is free software; you can redistribute it and/or
3
# modify it under the terms of version 2.1 of the GNU Lesser General Public
4
# License as published by the Free Software Foundation.
6
# This library is distributed in the hope that it will be useful,
7
# but WITHOUT ANY WARRANTY; without even the implied warranty of
8
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
9
# Lesser General Public License for more details.
11
# You should have received a copy of the GNU Lesser General Public
12
# License along with this library; if not, write to the Free Software
13
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
14
#============================================================================
15
# Copyright (C) 2004, 2005 Mike Wray <mike.wray@hp.com>
16
# Copyright (C) 2005 XenSource Ltd
17
#============================================================================
23
from xen.xend import sxp
24
from xen.xend import XendOptions
25
xoptions = XendOptions.instance()
27
from xen.xend import arch
28
from xen.xend.XendError import VmError
29
from xen.xend.XendLogging import log
30
from xen.xend.XendConstants import *
32
from xen.xend.server.DevController import DevController
33
from xen.xend.server.DevConstants import xenbusState
35
import xen.lowlevel.xc
37
from xen.util.pci import *
41
from xen.xend.server.pciquirk import *
42
from xen.xend.xenstore.xstransact import xstransact
43
from xen.xend.xenstore.xswatch import xswatch
45
xc = xen.lowlevel.xc.xc()
47
#Calculate PAGE_SHIFT: number of bits to shift an address to get the page number
48
PAGE_SIZE = resource.getpagesize()
57
if isinstance(val, types.StringTypes):
64
def get_assigned_pci_devices(domid):
66
path = '/local/domain/0/backend/pci/%u/0/' % domid
67
num_devs = xstransact.Read(path + 'num_devs');
68
if num_devs is None or num_devs == "":
70
num_devs = int(num_devs)
71
for i in range(num_devs):
72
dev_str = xstransact.Read(path + 'dev-%i' % i)
73
dev_str_list = dev_str_list + [dev_str]
76
def get_all_assigned_pci_devices(domid = 0):
77
dom_list = xstransact.List('/local/domain')
80
ts = xstransact.Read('/local/domain/' + str(domid) + '/target')
84
target = xstransact.Read('/local/domain/' + d + '/target')
85
if int(d) is not ti and target is None :
86
pci_str_list = pci_str_list + get_assigned_pci_devices(int(d))
89
class PciController(DevController):
91
def __init__(self, vm):
92
self.aerStateWatch = None
93
DevController.__init__(self, vm)
96
def getDeviceDetails(self, config):
97
"""@see DevController.getDeviceDetails"""
102
if 'pci_msitranslate' in self.vm.info['platform']:
103
pci_defopts.append(['msitranslate',
104
str(self.vm.info['platform']['pci_msitranslate'])])
105
if 'pci_power_mgmt' in self.vm.info['platform']:
106
pci_defopts.append(['power_mgmt',
107
str(self.vm.info['platform']['pci_power_mgmt'])])
109
for pci_config in config.get('devs', []):
110
domain = parse_hex(pci_config.get('domain', 0))
111
bus = parse_hex(pci_config.get('bus', 0))
112
slot = parse_hex(pci_config.get('slot', 0))
113
func = parse_hex(pci_config.get('func', 0))
114
vdevfn = parse_hex(pci_config.get('vdevfn', \
115
'0x%02x' % AUTO_PHP_SLOT))
118
if pci_config.has_key('opts'):
119
optslist += pci_config['opts']
120
if optslist or pci_defopts:
121
opts = serialise_pci_opts(
122
append_default_pci_opts(optslist, pci_defopts))
123
back['opts-%i' % pcidevid] = opts
125
back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%01x" % \
126
(domain, bus, slot, func)
127
back['uuid-%i' % pcidevid] = pci_config.get('uuid', '')
128
back['key-%i' % pcidevid] = pci_config.get('key', '')
129
back['vdevfn-%i' % pcidevid] = "%02x" % vdevfn
132
back['num_devs']=str(pcidevid)
133
back['uuid'] = config.get('uuid','')
137
def reconfigureDevice_find(self, devid, nsearch_dev, match_dev):
138
for j in range(nsearch_dev):
139
if match_dev == self.readBackend(devid, 'dev-%i' % j):
143
def reconfigureDevice(self, _, config):
144
"""@see DevController.reconfigureDevice"""
145
(devid, back, front) = self.getDeviceDetails(config)
146
num_devs = int(back['num_devs'])
147
states = config.get('states', [])
148
num_olddevs = int(self.readBackend(devid, 'num_devs'))
150
for i in range(num_devs):
152
dev = back['dev-%i' % i]
154
uuid = back['uuid-%i' %i]
155
key = back['key-%i' %i]
157
if 'opts-%i' % i in back:
158
opts = back['opts-%i' % i]
160
raise XendError('Error reading config')
162
if state == 'Initialising':
163
devno = self.reconfigureDevice_find(devid, num_olddevs, dev)
165
devno = num_olddevs + i
166
log.debug('Attaching PCI device %s.' % dev)
169
log.debug('Reconfiguring PCI device %s.' % dev)
172
self.setupOneDevice(parse_pci_name(dev))
174
self.writeBackend(devid, 'dev-%i' % devno, dev)
175
self.writeBackend(devid, 'state-%i' % devno,
176
str(xenbusState['Initialising']))
177
self.writeBackend(devid, 'uuid-%i' % devno, uuid)
178
self.writeBackend(devid, 'key-%i' % devno, key)
180
self.writeBackend(devid, 'opts-%i' % devno, opts)
181
if back.has_key('vdevfn-%i' % i):
182
self.writeBackend(devid, 'vdevfn-%i' % devno,
183
back['vdevfn-%i' % i])
185
# If a device is being attached then num_devs will grow
187
self.writeBackend(devid, 'num_devs', str(devno + 1))
189
elif state == 'Closing':
190
# PCI device detachment
191
devno = self.reconfigureDevice_find(devid, num_olddevs, dev)
193
raise XendError('Device %s is not connected' % dev)
194
log.debug('Detaching device %s' % dev)
195
self.writeBackend(devid, 'state-%i' % devno,
196
str(xenbusState['Closing']))
199
raise XendError('Error configuring device %s: invalid state %s'
202
self.writeBackend(devid, 'state', str(xenbusState['Reconfiguring']))
204
return self.readBackend(devid, 'uuid')
207
def getDeviceConfiguration(self, devid, transaction = None):
208
result = DevController.getDeviceConfiguration(self, devid, transaction)
209
num_devs = self.readBackend(devid, 'num_devs')
212
for i in range(int(num_devs)):
213
pci_dev = parse_pci_name(self.readBackend(devid, 'dev-%d' % i))
215
# Per device uuid info
216
pci_dev['uuid'] = self.readBackend(devid, 'uuid-%d' % i)
217
pci_dev['key'] = self.readBackend(devid, 'key-%d' % i)
218
pci_dev['vdevfn'] = '0x%s' % self.readBackend(devid,
222
opts = self.readBackend(devid, 'opts-%d' % i)
224
pci_dev['opts'] = opts
226
pci_devs.append(pci_dev)
228
result['devs'] = pci_devs
229
result['uuid'] = self.readBackend(devid, 'uuid')
232
def configuration(self, devid, transaction = None):
233
"""Returns SXPR for devices on domain.
235
@note: we treat this dict especially to convert to
236
SXP because it is not a straight dict of strings."""
238
configDict = self.getDeviceConfiguration(devid, transaction)
239
sxpr = [self.deviceClass]
242
devs = configDict.pop('devs', [])
246
for dev_key, dev_val in dev.items():
247
if dev_key == 'opts':
248
opts_sxpr = pci_opts_list_to_sxp(split_pci_opts(dev_val))
249
dev_sxpr = sxp.merge(dev_sxpr, opts_sxpr)
251
dev_sxpr.append([dev_key, dev_val])
252
sxpr.append(dev_sxpr)
254
for key, val in configDict.items():
255
if type(val) == type(list()):
257
sxpr.append([key, v])
259
sxpr.append([key, val])
263
def CheckSiblingDevices(self, domid, dev):
264
""" Check if all sibling devices of dev are owned by pciback or pci-stub
266
if not self.vm.info.is_hvm():
269
group_str = xc.get_device_group(domid, dev.domain, dev.bus, dev.slot, dev.func)
273
#group string format xx:xx.x,xx:xx.x,
274
for i in group_str.split(','):
277
pci_dev = parse_pci_name(i)
278
pci_dev['domain'] = '%04x' % dev.domain
280
sdev = PciDevice(pci_dev)
282
#no dom0 drivers bound to sdev
285
if sdev.driver!='pciback' and sdev.driver!='pci-stub':
286
raise VmError(("pci: PCI Backend and pci-stub don't "+ \
287
"own sibling device %s of device %s"\
288
)%(sdev.name, dev.name))
291
def setupOneDevice(self, pci_dev):
292
""" Attach I/O resources for device to frontend domain
294
fe_domid = self.getDomid()
297
dev = PciDevice(pci_dev)
299
raise VmError("pci: failed to locate device and "+
300
"parse its resources - "+str(e))
302
if dev.driver!='pciback' and dev.driver!='pci-stub':
303
raise VmError(("pci: PCI Backend and pci-stub don't own "+ \
304
"device %s") %(dev.name))
306
self.CheckSiblingDevices(fe_domid, dev)
308
if dev.driver == 'pciback':
311
if not self.vm.info.is_hvm() and not self.vm.info.is_stubdom() :
312
# Setup IOMMU device assignment
313
bdf = xc.assign_device(fe_domid, pci_dict_to_xc_str(pci_dev))
314
pci_str = pci_dict_to_bdf_str(pci_dev)
316
raise VmError("Failed to assign device to IOMMU (%s)" % pci_str)
317
log.debug("pci: assign device %s" % pci_str)
319
for (start, size) in dev.ioports:
320
log.debug('pci: enabling ioport 0x%x/0x%x'%(start,size))
321
rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start,
322
nr_ports = size, allow_access = True)
324
raise VmError(('pci: failed to configure I/O ports on device '+
325
'%s - errno=%d')%(dev.name,rc))
327
for (start, size) in dev.iomem:
328
# Convert start/size from bytes to page frame sizes
329
start_pfn = start>>PAGE_SHIFT
330
# Round number of pages up to nearest page boundary (if not on one)
331
nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
333
log.debug('pci: enabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \
334
(start,size,start_pfn,nr_pfns))
335
rc = xc.domain_iomem_permission(domid = fe_domid,
336
first_pfn = start_pfn,
340
raise VmError(('pci: failed to configure I/O memory on device '+
341
'%s - errno=%d')%(dev.name,rc))
343
if not self.vm.info.is_hvm() and dev.irq:
344
rc = xc.physdev_map_pirq(domid = fe_domid,
348
raise VmError(('pci: failed to map irq on device '+
349
'%s - errno=%d')%(dev.name,rc))
351
log.debug('pci: enabling irq %d'%dev.irq)
352
rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq,
355
raise VmError(('pci: failed to configure irq on device '+
356
'%s - errno=%d')%(dev.name,rc))
358
def dev_check_assignability_and_do_FLR(self, config):
359
pci_dev_list = config.get('devs', [])
360
pci_str_list = map(pci_dict_to_bdf_str, pci_dev_list)
362
if len(pci_str_list) != len(set(pci_str_list)):
363
raise VmError('pci: duplicate devices specified in guest config?')
365
strict_check = xoptions.get_pci_dev_assign_strict_check()
367
for pci_dev in pci_dev_list:
369
dev = PciDevice(pci_dev)
371
raise VmError("pci: failed to locate device and "+
372
"parse its resources - "+str(e))
373
if dev.driver!='pciback' and dev.driver!='pci-stub':
374
raise VmError(("pci: PCI Backend and pci-stub don't own device"\
379
if dev.has_non_page_aligned_bar and strict_check:
380
raise VmError("pci: %s: non-page-aligned MMIO BAR found." % dev.name)
382
# Check if there is intermediate PCIe switch bewteen the device and
384
if self.vm.info.is_hvm() and dev.is_behind_switch_lacking_acs() \
386
err_msg = 'pci: to avoid potential security issue, %s is not'+\
387
' allowed to be assigned to guest since it is behind'+\
388
' PCIe switch that does not support or enable ACS.'
389
raise VmError(err_msg % dev.name)
391
if (dev.dev_type == DEV_TYPE_PCIe_ENDPOINT) and not dev.pcie_flr:
393
# We cope with this case by using the Dstate transition
394
# method or some vendor specific methods for now.
395
err_msg = 'pci: %s: it is on bus 0, but has no PCIe' +\
396
' FLR Capability. Will try the Dstate transition'+\
397
' method or some vendor specific methods if available.'
398
log.warn(err_msg % dev.name)
400
if not self.vm.info.is_hvm():
405
funcs = dev.find_all_the_multi_functions()
406
dev.devs_check_driver(funcs)
408
if not f in pci_str_list:
409
# f has been assigned to other guest?
410
if f in get_all_assigned_pci_devices():
411
err_msg = 'pci: %s must be co-assigned to' + \
412
' the same guest with %s'
413
raise VmError(err_msg % (f, dev.name))
414
elif dev.dev_type == DEV_TYPE_PCI:
415
if dev.bus == 0 or arch.type == "ia64":
416
if not dev.pci_af_flr:
417
# We cope with this case by using the Dstate transition
418
# method or some vendor specific methods for now.
419
err_msg = 'pci: %s: it is on bus 0, but has no PCI' +\
420
' Advanced Capabilities for FLR. Will try the'+\
421
' Dstate transition method or some vendor' +\
422
' specific methods if available.'
423
log.warn(err_msg % dev.name)
425
if not self.vm.info.is_hvm():
430
# All devices behind the uppermost PCI/PCI-X bridge must be\
431
# co-assigned to the same guest.
432
devs_str = dev.find_coassigned_pci_devices(True)
433
# Remove the element 0 which is a bridge
436
dev.devs_check_driver(devs_str)
438
if not s in pci_str_list:
439
# s has been assigned to other guest?
440
if s in get_all_assigned_pci_devices():
441
err_msg = 'pci: %s must be co-assigned to the'+\
442
' same guest with %s'
443
raise VmError(err_msg % (s, dev.name))
446
dev.do_FLR(self.vm.info.is_hvm(), strict_check)
448
def setupDevice(self, config):
449
"""Setup devices from config
451
pci_dev_list = config.get('devs', [])
452
for d in pci_dev_list:
453
self.setupOneDevice(d)
454
wPath = '/local/domain/0/backend/pci/%u/0/aerState' % (self.getDomid())
455
self.aerStateWatch = xswatch(wPath, self._handleAerStateWatch)
456
log.debug('pci: register aer watch %s', wPath)
459
def _handleAerStateWatch(self, _):
460
log.debug('XendDomainInfo.handleAerStateWatch')
461
if self.getDomid() == 0:
462
raise XendError('Domain 0 cannot be shutdown')
463
readPath = '/local/domain/0/backend/pci/%u/0/aerState' % (self.getDomid())
464
action = xstransact.Read(readPath)
465
if action and action=='aerfail':
466
log.debug('shutdown domain because of aer handle error')
467
self.vm.shutdown('poweroff')
471
def cleanupOneDevice(self, pci_dev):
472
""" Detach I/O resources for device from frontend domain
474
fe_domid = self.getDomid()
477
dev = PciDevice(pci_dev)
479
raise VmError("pci: failed to locate device and "+
480
"parse its resources - "+str(e))
482
if dev.driver!='pciback' and dev.driver!='pci-stub':
483
raise VmError(("pci: PCI Backend and pci-stub don't own device "+ \
486
# Need to do FLR here before deassign device in order to terminate
487
# DMA transaction, etc
488
dev.do_FLR(self.vm.info.is_hvm(),
489
xoptions.get_pci_dev_assign_strict_check())
491
if not self.vm.info.is_stubdom() :
492
bdf = xc.deassign_device(fe_domid, pci_dict_to_xc_str(pci_dev))
493
pci_str = pci_dict_to_bdf_str(pci_dev)
495
raise VmError("Failed to deassign device from IOMMU (%s)" % pci_str)
496
log.debug("pci: Deassign device %s" % pci_str)
498
for (start, size) in dev.ioports:
499
log.debug('pci: disabling ioport 0x%x/0x%x'%(start,size))
500
rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start,
501
nr_ports = size, allow_access = False)
503
raise VmError(('pci: failed to configure I/O ports on device '+
504
'%s - errno=%d')%(dev.name,rc))
506
for (start, size) in dev.iomem:
507
# Convert start/size from bytes to page frame sizes
508
start_pfn = start>>PAGE_SHIFT
509
# Round number of pages up to nearest page boundary (if not on one)
510
nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
512
log.debug('pci: disabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \
513
(start,size,start_pfn,nr_pfns))
514
rc = xc.domain_iomem_permission(domid = fe_domid,
515
first_pfn = start_pfn,
517
allow_access = False)
519
raise VmError(('pci: failed to configure I/O memory on device '+
520
'%s - errno=%d')%(dev.name,rc))
523
log.debug('pci: disabling irq %d'%dev.irq)
524
rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq,
525
allow_access = False)
527
raise VmError(('pci: failed to configure irq on device '+
528
'%s - errno=%d')%(dev.name,rc))
530
def cleanupDevice(self, devid):
531
""" Detach I/O resources for device and cleanup xenstore nodes
534
@param devid: The device ID
536
@return: Return the number of devices connected
539
num_devs = int(self.readBackend(devid, 'num_devs'))
541
for i in range(num_devs):
543
state = int(self.readBackend(devid, 'state-%i' % i))
545
state = xenbusState['Unknown']
547
if state == xenbusState['Closing']:
548
# Detach I/O resources.
549
pci_dev = parse_pci_name(self.readBackend(devid, 'dev-%i' % i))
550
# In HVM case, I/O resources are disabled in ioemu.
551
self.cleanupOneDevice(pci_dev)
552
# Remove xenstore nodes.
553
list = ['dev', 'vdev', 'state', 'uuid', 'vdevfn', 'key']
554
if self.readBackend(devid, 'opts-%i' % i) is not None:
557
self.removeBackend(devid, '%s-%i' % (key, i))
559
new_num_devs = new_num_devs + 1
560
if new_num_devs == i + 1:
563
list = ['dev', 'vdev', 'state', 'uuid', 'opts', 'vdevfn', 'key']
565
tmp = self.readBackend(devid, '%s-%i' % (key, i))
568
self.removeBackend(devid, '%s-%i' % (key, i))
569
self.writeBackend(devid,
570
'%s-%i' % (key, new_num_devs - 1), tmp)
572
self.writeBackend(devid, 'num_devs', str(new_num_devs))
576
def destroyDevice(self, devid, force):
577
DevController.destroyDevice(self, devid, True)
578
log.debug('pci: unregister aer watch')
579
self.unwatchAerState()
581
def unwatchAerState(self):
582
"""Remove the watch on the domain's aerState node, if any."""
585
if self.aerStateWatch:
586
self.aerStateWatch.unwatch()
588
self.aerStateWatch = None
590
log.exception("Unwatching aerState failed.")
592
def waitForBackend(self,devid):
593
return (0, "ok - no hotplug")
595
def migrate(self, config, network, dst, step, domName):
596
raise XendError('Migration not permitted with assigned PCI device.')