2
* Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
6
* Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
7
* Copyright (C) 2006 Olof Johansson <olof@lixom.net>
9
* Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
12
* This program is free software; you can redistribute it and/or modify
13
* it under the terms of the GNU General Public License as published by
14
* the Free Software Foundation; either version 2 of the License, or
15
* (at your option) any later version.
17
* This program is distributed in the hope that it will be useful,
18
* but WITHOUT ANY WARRANTY; without even the implied warranty of
19
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20
* GNU General Public License for more details.
22
* You should have received a copy of the GNU General Public License
23
* along with this program; if not, write to the Free Software
24
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
27
#include <linux/init.h>
28
#include <linux/types.h>
29
#include <linux/slab.h>
31
#include <linux/spinlock.h>
32
#include <linux/sched.h> /* for show_stack */
33
#include <linux/string.h>
34
#include <linux/pci.h>
35
#include <linux/dma-mapping.h>
36
#include <linux/crash_dump.h>
37
#include <linux/memory.h>
41
#include <asm/iommu.h>
42
#include <asm/pci-bridge.h>
43
#include <asm/machdep.h>
44
#include <asm/abs_addr.h>
45
#include <asm/pSeries_reconfig.h>
46
#include <asm/firmware.h>
48
#include <asm/ppc-pci.h>
50
#include <asm/mmzone.h>
52
#include "plpar_wrappers.h"
55
static int tce_build_pSeries(struct iommu_table *tbl, long index,
56
long npages, unsigned long uaddr,
57
enum dma_data_direction direction,
58
struct dma_attrs *attrs)
64
proto_tce = TCE_PCI_READ; // Read allowed
66
if (direction != DMA_TO_DEVICE)
67
proto_tce |= TCE_PCI_WRITE;
69
tcep = ((u64 *)tbl->it_base) + index;
72
/* can't move this out since we might cross MEMBLOCK boundary */
73
rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
74
*tcep = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
76
uaddr += TCE_PAGE_SIZE;
83
static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
87
tcep = ((u64 *)tbl->it_base) + index;
93
static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
97
tcep = ((u64 *)tbl->it_base) + index;
102
static void tce_free_pSeriesLP(struct iommu_table*, long, long);
103
static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
105
static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
106
long npages, unsigned long uaddr,
107
enum dma_data_direction direction,
108
struct dma_attrs *attrs)
114
long tcenum_start = tcenum, npages_start = npages;
116
rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
117
proto_tce = TCE_PCI_READ;
118
if (direction != DMA_TO_DEVICE)
119
proto_tce |= TCE_PCI_WRITE;
122
tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
123
rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce);
125
if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
127
tce_free_pSeriesLP(tbl, tcenum_start,
128
(npages_start - (npages + 1)));
132
if (rc && printk_ratelimit()) {
133
printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
134
printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
135
printk("\ttcenum = 0x%llx\n", (u64)tcenum);
136
printk("\ttce val = 0x%llx\n", tce );
137
show_stack(current, (unsigned long *)__get_SP());
146
static DEFINE_PER_CPU(u64 *, tce_page);
148
static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
149
long npages, unsigned long uaddr,
150
enum dma_data_direction direction,
151
struct dma_attrs *attrs)
158
long tcenum_start = tcenum, npages_start = npages;
162
return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
166
tcep = __get_cpu_var(tce_page);
168
/* This is safe to do since interrupts are off when we're called
169
* from iommu_alloc{,_sg}()
172
tcep = (u64 *)__get_free_page(GFP_ATOMIC);
173
/* If allocation fails, fall back to the loop implementation */
175
return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
178
__get_cpu_var(tce_page) = tcep;
181
rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
182
proto_tce = TCE_PCI_READ;
183
if (direction != DMA_TO_DEVICE)
184
proto_tce |= TCE_PCI_WRITE;
186
/* We can map max one pageful of TCEs at a time */
189
* Set up the page with TCE data, looping through and setting
192
limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
194
for (l = 0; l < limit; l++) {
195
tcep[l] = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
199
rc = plpar_tce_put_indirect((u64)tbl->it_index,
201
(u64)virt_to_abs(tcep),
206
} while (npages > 0 && !rc);
208
if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
210
tce_freemulti_pSeriesLP(tbl, tcenum_start,
211
(npages_start - (npages + limit)));
215
if (rc && printk_ratelimit()) {
216
printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
217
printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
218
printk("\tnpages = 0x%llx\n", (u64)npages);
219
printk("\ttce[0] val = 0x%llx\n", tcep[0]);
220
show_stack(current, (unsigned long *)__get_SP());
225
static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
230
rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);
232
if (rc && printk_ratelimit()) {
233
printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
234
printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
235
printk("\ttcenum = 0x%llx\n", (u64)tcenum);
236
show_stack(current, (unsigned long *)__get_SP());
244
static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
248
rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
250
if (rc && printk_ratelimit()) {
251
printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
252
printk("\trc = %lld\n", rc);
253
printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
254
printk("\tnpages = 0x%llx\n", (u64)npages);
255
show_stack(current, (unsigned long *)__get_SP());
259
static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
262
unsigned long tce_ret;
264
rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);
266
if (rc && printk_ratelimit()) {
267
printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
268
printk("\tindex = 0x%llx\n", (u64)tbl->it_index);
269
printk("\ttcenum = 0x%llx\n", (u64)tcenum);
270
show_stack(current, (unsigned long *)__get_SP());
276
/* this is compatible with cells for the device tree property */
277
struct dynamic_dma_window_prop {
278
__be32 liobn; /* tce table number */
279
__be64 dma_base; /* address hi,lo */
280
__be32 tce_shift; /* ilog2(tce_page_size) */
281
__be32 window_shift; /* ilog2(tce_window_size) */
284
struct direct_window {
285
struct device_node *device;
286
const struct dynamic_dma_window_prop *prop;
287
struct list_head list;
290
/* Dynamic DMA Window support */
291
struct ddw_query_response {
292
u32 windows_available;
293
u32 largest_available_block;
295
u32 migration_capable;
298
struct ddw_create_response {
304
static LIST_HEAD(direct_window_list);
305
/* prevents races between memory on/offline and window creation */
306
static DEFINE_SPINLOCK(direct_window_list_lock);
307
/* protects initializing window twice for same device */
308
static DEFINE_MUTEX(direct_window_init_mutex);
309
#define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
311
static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
312
unsigned long num_pfn, const void *arg)
314
const struct dynamic_dma_window_prop *maprange = arg;
316
u64 tce_size, num_tce, dma_offset, next;
320
tce_shift = be32_to_cpu(maprange->tce_shift);
321
tce_size = 1ULL << tce_shift;
322
next = start_pfn << PAGE_SHIFT;
323
num_tce = num_pfn << PAGE_SHIFT;
325
/* round back to the beginning of the tce page size */
326
num_tce += next & (tce_size - 1);
327
next &= ~(tce_size - 1);
329
/* covert to number of tces */
330
num_tce |= tce_size - 1;
331
num_tce >>= tce_shift;
335
* Set up the page with TCE data, looping through and setting
338
limit = min_t(long, num_tce, 512);
339
dma_offset = next + be64_to_cpu(maprange->dma_base);
341
rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
345
} while (num_tce > 0 && !rc);
350
static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
351
unsigned long num_pfn, const void *arg)
353
const struct dynamic_dma_window_prop *maprange = arg;
354
u64 *tcep, tce_size, num_tce, dma_offset, next, proto_tce, liobn;
359
local_irq_disable(); /* to protect tcep and the page behind it */
360
tcep = __get_cpu_var(tce_page);
363
tcep = (u64 *)__get_free_page(GFP_ATOMIC);
368
__get_cpu_var(tce_page) = tcep;
371
proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
373
liobn = (u64)be32_to_cpu(maprange->liobn);
374
tce_shift = be32_to_cpu(maprange->tce_shift);
375
tce_size = 1ULL << tce_shift;
376
next = start_pfn << PAGE_SHIFT;
377
num_tce = num_pfn << PAGE_SHIFT;
379
/* round back to the beginning of the tce page size */
380
num_tce += next & (tce_size - 1);
381
next &= ~(tce_size - 1);
383
/* covert to number of tces */
384
num_tce |= tce_size - 1;
385
num_tce >>= tce_shift;
387
/* We can map max one pageful of TCEs at a time */
390
* Set up the page with TCE data, looping through and setting
393
limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
394
dma_offset = next + be64_to_cpu(maprange->dma_base);
396
for (l = 0; l < limit; l++) {
397
tcep[l] = proto_tce | next;
401
rc = plpar_tce_put_indirect(liobn,
403
(u64)virt_to_abs(tcep),
407
} while (num_tce > 0 && !rc);
409
/* error cleanup: caller will clear whole range */
415
static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
416
unsigned long num_pfn, void *arg)
418
return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
423
static void iommu_table_setparms(struct pci_controller *phb,
424
struct device_node *dn,
425
struct iommu_table *tbl)
427
struct device_node *node;
428
const unsigned long *basep;
433
basep = of_get_property(node, "linux,tce-base", NULL);
434
sizep = of_get_property(node, "linux,tce-size", NULL);
435
if (basep == NULL || sizep == NULL) {
436
printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
437
"missing tce entries !\n", dn->full_name);
441
tbl->it_base = (unsigned long)__va(*basep);
443
if (!is_kdump_kernel())
444
memset((void *)tbl->it_base, 0, *sizep);
446
tbl->it_busno = phb->bus->number;
448
/* Units of tce entries */
449
tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;
451
/* Test if we are going over 2GB of DMA space */
452
if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
453
udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
454
panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
457
phb->dma_window_base_cur += phb->dma_window_size;
459
/* Set the tce table size - measured in entries */
460
tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;
463
tbl->it_blocksize = 16;
464
tbl->it_type = TCE_PCI;
468
* iommu_table_setparms_lpar
470
* Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
472
static void iommu_table_setparms_lpar(struct pci_controller *phb,
473
struct device_node *dn,
474
struct iommu_table *tbl,
475
const void *dma_window)
477
unsigned long offset, size;
479
of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size);
481
tbl->it_busno = phb->bus->number;
483
tbl->it_blocksize = 16;
484
tbl->it_type = TCE_PCI;
485
tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
486
tbl->it_size = size >> IOMMU_PAGE_SHIFT;
489
static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
491
struct device_node *dn;
492
struct iommu_table *tbl;
493
struct device_node *isa_dn, *isa_dn_orig;
494
struct device_node *tmp;
498
dn = pci_bus_to_OF_node(bus);
500
pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name);
503
/* This is not a root bus, any setup will be done for the
504
* device-side of the bridge in iommu_dev_setup_pSeries().
510
/* Check if the ISA bus on the system is under
513
isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
515
while (isa_dn && isa_dn != dn)
516
isa_dn = isa_dn->parent;
519
of_node_put(isa_dn_orig);
521
/* Count number of direct PCI children of the PHB. */
522
for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
525
pr_debug("Children: %d\n", children);
527
/* Calculate amount of DMA window per slot. Each window must be
528
* a power of two (due to pci_alloc_consistent requirements).
530
* Keep 256MB aside for PHBs with ISA.
534
/* No ISA/IDE - just set window size and return */
535
pci->phb->dma_window_size = 0x80000000ul; /* To be divided */
537
while (pci->phb->dma_window_size * children > 0x80000000ul)
538
pci->phb->dma_window_size >>= 1;
539
pr_debug("No ISA/IDE, window size is 0x%llx\n",
540
pci->phb->dma_window_size);
541
pci->phb->dma_window_base_cur = 0;
546
/* If we have ISA, then we probably have an IDE
547
* controller too. Allocate a 128MB table but
548
* skip the first 128MB to avoid stepping on ISA
551
pci->phb->dma_window_size = 0x8000000ul;
552
pci->phb->dma_window_base_cur = 0x8000000ul;
554
tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
557
iommu_table_setparms(pci->phb, dn, tbl);
558
pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
560
/* Divide the rest (1.75GB) among the children */
561
pci->phb->dma_window_size = 0x80000000ul;
562
while (pci->phb->dma_window_size * children > 0x70000000ul)
563
pci->phb->dma_window_size >>= 1;
565
pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
569
static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
571
struct iommu_table *tbl;
572
struct device_node *dn, *pdn;
574
const void *dma_window = NULL;
576
dn = pci_bus_to_OF_node(bus);
578
pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n",
581
/* Find nearest ibm,dma-window, walking up the device tree */
582
for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
583
dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
584
if (dma_window != NULL)
588
if (dma_window == NULL) {
589
pr_debug(" no ibm,dma-window property !\n");
595
pr_debug(" parent is %s, iommu_table: 0x%p\n",
596
pdn->full_name, ppci->iommu_table);
598
if (!ppci->iommu_table) {
599
tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
601
iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window);
602
ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node);
603
pr_debug(" created table: %p\n", ppci->iommu_table);
608
static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
610
struct device_node *dn;
611
struct iommu_table *tbl;
613
pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
615
dn = dev->dev.of_node;
617
/* If we're the direct child of a root bus, then we need to allocate
618
* an iommu table ourselves. The bus setup code should have setup
619
* the window sizes already.
621
if (!dev->bus->self) {
622
struct pci_controller *phb = PCI_DN(dn)->phb;
624
pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
625
tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
627
iommu_table_setparms(phb, dn, tbl);
628
PCI_DN(dn)->iommu_table = iommu_init_table(tbl, phb->node);
629
set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
633
/* If this device is further down the bus tree, search upwards until
634
* an already allocated iommu table is found and use that.
637
while (dn && PCI_DN(dn) && PCI_DN(dn)->iommu_table == NULL)
640
if (dn && PCI_DN(dn))
641
set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
643
printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
647
static int __read_mostly disable_ddw;
649
static int __init disable_ddw_setup(char *str)
652
printk(KERN_INFO "ppc iommu: disabling ddw.\n");
657
early_param("disable_ddw", disable_ddw_setup);
659
static void remove_ddw(struct device_node *np)
661
struct dynamic_dma_window_prop *dwp;
662
struct property *win64;
663
const u32 *ddw_avail;
667
ddw_avail = of_get_property(np, "ibm,ddw-applicable", &len);
668
win64 = of_find_property(np, DIRECT64_PROPNAME, NULL);
672
if (!ddw_avail || len < 3 * sizeof(u32) || win64->length < sizeof(*dwp))
676
liobn = (u64)be32_to_cpu(dwp->liobn);
678
/* clear the whole window, note the arg is in kernel pages */
679
ret = tce_clearrange_multi_pSeriesLP(0,
680
1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
682
pr_warning("%s failed to clear tces in window.\n",
685
pr_debug("%s successfully cleared tces in window.\n",
688
ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn);
690
pr_warning("%s: failed to remove direct window: rtas returned "
691
"%d to ibm,remove-pe-dma-window(%x) %llx\n",
692
np->full_name, ret, ddw_avail[2], liobn);
694
pr_debug("%s: successfully removed direct window: rtas returned "
695
"%d to ibm,remove-pe-dma-window(%x) %llx\n",
696
np->full_name, ret, ddw_avail[2], liobn);
699
ret = prom_remove_property(np, win64);
701
pr_warning("%s: failed to remove direct window property: %d\n",
705
static u64 find_existing_ddw(struct device_node *pdn)
707
struct direct_window *window;
708
const struct dynamic_dma_window_prop *direct64;
711
spin_lock(&direct_window_list_lock);
712
/* check if we already created a window and dupe that config if so */
713
list_for_each_entry(window, &direct_window_list, list) {
714
if (window->device == pdn) {
715
direct64 = window->prop;
716
dma_addr = direct64->dma_base;
720
spin_unlock(&direct_window_list_lock);
725
static int find_existing_ddw_windows(void)
728
struct device_node *pdn;
729
struct direct_window *window;
730
const struct dynamic_dma_window_prop *direct64;
732
if (!firmware_has_feature(FW_FEATURE_LPAR))
735
for_each_node_with_property(pdn, DIRECT64_PROPNAME) {
736
direct64 = of_get_property(pdn, DIRECT64_PROPNAME, &len);
740
window = kzalloc(sizeof(*window), GFP_KERNEL);
741
if (!window || len < sizeof(struct dynamic_dma_window_prop)) {
747
window->device = pdn;
748
window->prop = direct64;
749
spin_lock(&direct_window_list_lock);
750
list_add(&window->list, &direct_window_list);
751
spin_unlock(&direct_window_list_lock);
756
machine_arch_initcall(pseries, find_existing_ddw_windows);
758
static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
759
struct ddw_query_response *query)
761
struct device_node *dn;
762
struct pci_dn *pcidn;
768
* Get the config address and phb buid of the PE window.
769
* Rely on eeh to retrieve this for us.
770
* Retrieve them from the pci device, not the node with the
771
* dma-window property
773
dn = pci_device_to_OF_node(dev);
775
cfg_addr = pcidn->eeh_config_addr;
776
if (pcidn->eeh_pe_config_addr)
777
cfg_addr = pcidn->eeh_pe_config_addr;
778
buid = pcidn->phb->buid;
779
ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query,
780
cfg_addr, BUID_HI(buid), BUID_LO(buid));
781
dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x"
782
" returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid),
787
static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
788
struct ddw_create_response *create, int page_shift,
791
struct device_node *dn;
792
struct pci_dn *pcidn;
798
* Get the config address and phb buid of the PE window.
799
* Rely on eeh to retrieve this for us.
800
* Retrieve them from the pci device, not the node with the
801
* dma-window property
803
dn = pci_device_to_OF_node(dev);
805
cfg_addr = pcidn->eeh_config_addr;
806
if (pcidn->eeh_pe_config_addr)
807
cfg_addr = pcidn->eeh_pe_config_addr;
808
buid = pcidn->phb->buid;
811
/* extra outputs are LIOBN and dma-addr (hi, lo) */
812
ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create, cfg_addr,
813
BUID_HI(buid), BUID_LO(buid), page_shift, window_shift);
814
} while (rtas_busy_delay(ret));
816
"ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
817
"(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1],
818
cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift,
819
window_shift, ret, create->liobn, create->addr_hi, create->addr_lo);
825
* If the PE supports dynamic dma windows, and there is space for a table
826
* that can map all pages in a linear offset, then setup such a table,
827
* and record the dma-offset in the struct device.
829
* dev: the pci device we are checking
830
* pdn: the parent pe node with the ibm,dma_window property
831
* Future: also check if we can remap the base window for our base page size
833
* returns the dma offset for use by dma_set_mask
835
static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
838
struct ddw_query_response query;
839
struct ddw_create_response create;
841
u64 dma_addr, max_addr;
842
struct device_node *dn;
843
const u32 *uninitialized_var(ddw_avail);
844
struct direct_window *window;
845
struct property *win64;
846
struct dynamic_dma_window_prop *ddwprop;
848
mutex_lock(&direct_window_init_mutex);
850
dma_addr = find_existing_ddw(pdn);
855
* the ibm,ddw-applicable property holds the tokens for:
856
* ibm,query-pe-dma-window
857
* ibm,create-pe-dma-window
858
* ibm,remove-pe-dma-window
859
* for the given node in that order.
860
* the property is actually in the parent, not the PE
862
ddw_avail = of_get_property(pdn, "ibm,ddw-applicable", &len);
863
if (!ddw_avail || len < 3 * sizeof(u32))
867
* Query if there is a second window of size to map the
868
* whole partition. Query returns number of windows, largest
869
* block assigned to PE (partition endpoint), and two bitmasks
870
* of page sizes: supported and supported for migrate-dma.
872
dn = pci_device_to_OF_node(dev);
873
ret = query_ddw(dev, ddw_avail, &query);
877
if (query.windows_available == 0) {
879
* no additional windows are available for this device.
880
* We might be able to reallocate the existing window,
881
* trading in for a larger page size.
883
dev_dbg(&dev->dev, "no free dynamic windows");
886
if (query.page_size & 4) {
887
page_shift = 24; /* 16MB */
888
} else if (query.page_size & 2) {
889
page_shift = 16; /* 64kB */
890
} else if (query.page_size & 1) {
891
page_shift = 12; /* 4kB */
893
dev_dbg(&dev->dev, "no supported direct page size in mask %x",
897
/* verify the window * number of ptes will map the partition */
898
/* check largest block * page size > max memory hotplug addr */
899
max_addr = memory_hotplug_max();
900
if (query.largest_available_block < (max_addr >> page_shift)) {
901
dev_dbg(&dev->dev, "can't map partiton max 0x%llx with %u "
902
"%llu-sized pages\n", max_addr, query.largest_available_block,
906
len = order_base_2(max_addr);
907
win64 = kzalloc(sizeof(struct property), GFP_KERNEL);
910
"couldn't allocate property for 64bit dma window\n");
913
win64->name = kstrdup(DIRECT64_PROPNAME, GFP_KERNEL);
914
win64->value = ddwprop = kmalloc(sizeof(*ddwprop), GFP_KERNEL);
915
win64->length = sizeof(*ddwprop);
916
if (!win64->name || !win64->value) {
918
"couldn't allocate property name and value\n");
922
ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
926
ddwprop->liobn = cpu_to_be32(create.liobn);
927
ddwprop->dma_base = cpu_to_be64(of_read_number(&create.addr_hi, 2));
928
ddwprop->tce_shift = cpu_to_be32(page_shift);
929
ddwprop->window_shift = cpu_to_be32(len);
931
dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %s\n",
932
create.liobn, dn->full_name);
934
window = kzalloc(sizeof(*window), GFP_KERNEL);
936
goto out_clear_window;
938
ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
939
win64->value, tce_setrange_multi_pSeriesLP_walk);
941
dev_info(&dev->dev, "failed to map direct window for %s: %d\n",
943
goto out_free_window;
946
ret = prom_add_property(pdn, win64);
948
dev_err(&dev->dev, "unable to add dma window property for %s: %d",
949
pdn->full_name, ret);
950
goto out_free_window;
953
window->device = pdn;
954
window->prop = ddwprop;
955
spin_lock(&direct_window_list_lock);
956
list_add(&window->list, &direct_window_list);
957
spin_unlock(&direct_window_list_lock);
959
dma_addr = of_read_number(&create.addr_hi, 2);
974
mutex_unlock(&direct_window_init_mutex);
978
static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
980
struct device_node *pdn, *dn;
981
struct iommu_table *tbl;
982
const void *dma_window = NULL;
985
pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
987
/* dev setup for LPAR is a little tricky, since the device tree might
988
* contain the dma-window properties per-device and not necessarily
989
* for the bus. So we need to search upwards in the tree until we
990
* either hit a dma-window property, OR find a parent with a table
993
dn = pci_device_to_OF_node(dev);
994
pr_debug(" node is %s\n", dn->full_name);
996
for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
998
dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1003
if (!pdn || !PCI_DN(pdn)) {
1004
printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1005
"no DMA window found for pci dev=%s dn=%s\n",
1006
pci_name(dev), dn? dn->full_name : "<null>");
1009
pr_debug(" parent is %s\n", pdn->full_name);
1012
if (!pci->iommu_table) {
1013
tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
1015
iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window);
1016
pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
1017
pr_debug(" created table: %p\n", pci->iommu_table);
1019
pr_debug(" found DMA window, table: %p\n", pci->iommu_table);
1022
set_iommu_table_base(&dev->dev, pci->iommu_table);
1025
static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask)
1027
bool ddw_enabled = false;
1028
struct device_node *pdn, *dn;
1029
struct pci_dev *pdev;
1030
const void *dma_window = NULL;
1036
if (!dev_is_pci(dev))
1039
pdev = to_pci_dev(dev);
1041
/* only attempt to use a new window if 64-bit DMA is requested */
1042
if (!disable_ddw && dma_mask == DMA_BIT_MASK(64)) {
1043
dn = pci_device_to_OF_node(pdev);
1044
dev_dbg(dev, "node is %s\n", dn->full_name);
1047
* the device tree might contain the dma-window properties
1048
* per-device and not necessarily for the bus. So we need to
1049
* search upwards in the tree until we either hit a dma-window
1050
* property, OR find a parent with a table already allocated.
1052
for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
1053
pdn = pdn->parent) {
1054
dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1058
if (pdn && PCI_DN(pdn)) {
1059
dma_offset = enable_ddw(pdev, pdn);
1060
if (dma_offset != 0) {
1061
dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset);
1062
set_dma_offset(dev, dma_offset);
1063
set_dma_ops(dev, &dma_direct_ops);
1069
/* fall back on iommu ops, restore table pointer with ops */
1070
if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) {
1071
dev_info(dev, "Restoring 32-bit DMA via iommu\n");
1072
set_dma_ops(dev, &dma_iommu_ops);
1073
pci_dma_dev_setup_pSeriesLP(pdev);
1077
if (!dma_supported(dev, dma_mask))
1080
*dev->dma_mask = dma_mask;
1084
static u64 dma_get_required_mask_pSeriesLP(struct device *dev)
1089
if (!disable_ddw && dev_is_pci(dev)) {
1090
struct pci_dev *pdev = to_pci_dev(dev);
1091
struct device_node *dn;
1093
dn = pci_device_to_OF_node(pdev);
1095
/* search upwards for ibm,dma-window */
1096
for (; dn && PCI_DN(dn) && !PCI_DN(dn)->iommu_table;
1098
if (of_get_property(dn, "ibm,dma-window", NULL))
1100
/* if there is a ibm,ddw-applicable property require 64 bits */
1101
if (dn && PCI_DN(dn) &&
1102
of_get_property(dn, "ibm,ddw-applicable", NULL))
1103
return DMA_BIT_MASK(64);
1106
return dma_iommu_ops.get_required_mask(dev);
1109
#else /* CONFIG_PCI */
1110
#define pci_dma_bus_setup_pSeries NULL
1111
#define pci_dma_dev_setup_pSeries NULL
1112
#define pci_dma_bus_setup_pSeriesLP NULL
1113
#define pci_dma_dev_setup_pSeriesLP NULL
1114
#define dma_set_mask_pSeriesLP NULL
1115
#define dma_get_required_mask_pSeriesLP NULL
1116
#endif /* !CONFIG_PCI */
1118
static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1121
struct direct_window *window;
1122
struct memory_notify *arg = data;
1126
case MEM_GOING_ONLINE:
1127
spin_lock(&direct_window_list_lock);
1128
list_for_each_entry(window, &direct_window_list, list) {
1129
ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1130
arg->nr_pages, window->prop);
1133
spin_unlock(&direct_window_list_lock);
1135
case MEM_CANCEL_ONLINE:
1137
spin_lock(&direct_window_list_lock);
1138
list_for_each_entry(window, &direct_window_list, list) {
1139
ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1140
arg->nr_pages, window->prop);
1143
spin_unlock(&direct_window_list_lock);
1148
if (ret && action != MEM_CANCEL_ONLINE)
1154
static struct notifier_block iommu_mem_nb = {
1155
.notifier_call = iommu_mem_notifier,
1158
static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
1160
int err = NOTIFY_OK;
1161
struct device_node *np = node;
1162
struct pci_dn *pci = PCI_DN(np);
1163
struct direct_window *window;
1166
case PSERIES_RECONFIG_REMOVE:
1167
if (pci && pci->iommu_table)
1168
iommu_free_table(pci->iommu_table, np->full_name);
1170
spin_lock(&direct_window_list_lock);
1171
list_for_each_entry(window, &direct_window_list, list) {
1172
if (window->device == np) {
1173
list_del(&window->list);
1178
spin_unlock(&direct_window_list_lock);
1181
* Because the notifier runs after isolation of the
1182
* slot, we are guaranteed any DMA window has already
1183
* been revoked and the TCEs have been marked invalid,
1184
* so we don't need a call to remove_ddw(np). However,
1185
* if an additional notifier action is added before the
1186
* isolate call, we should update this code for
1187
* completeness with such a call.
1197
static struct notifier_block iommu_reconfig_nb = {
1198
.notifier_call = iommu_reconfig_notifier,
1201
/* These are called very early. */
1202
void iommu_init_early_pSeries(void)
1204
if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1207
if (firmware_has_feature(FW_FEATURE_LPAR)) {
1208
if (firmware_has_feature(FW_FEATURE_MULTITCE)) {
1209
ppc_md.tce_build = tce_buildmulti_pSeriesLP;
1210
ppc_md.tce_free = tce_freemulti_pSeriesLP;
1212
ppc_md.tce_build = tce_build_pSeriesLP;
1213
ppc_md.tce_free = tce_free_pSeriesLP;
1215
ppc_md.tce_get = tce_get_pSeriesLP;
1216
ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1217
ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1218
ppc_md.dma_set_mask = dma_set_mask_pSeriesLP;
1219
ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP;
1221
ppc_md.tce_build = tce_build_pSeries;
1222
ppc_md.tce_free = tce_free_pSeries;
1223
ppc_md.tce_get = tce_get_pseries;
1224
ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeries;
1225
ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeries;
1229
pSeries_reconfig_notifier_register(&iommu_reconfig_nb);
1230
register_memory_notifier(&iommu_mem_nb);
1232
set_pci_dma_ops(&dma_iommu_ops);
1235
static int __init disable_multitce(char *str)
1237
if (strcmp(str, "off") == 0 &&
1238
firmware_has_feature(FW_FEATURE_LPAR) &&
1239
firmware_has_feature(FW_FEATURE_MULTITCE)) {
1240
printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1241
ppc_md.tce_build = tce_build_pSeriesLP;
1242
ppc_md.tce_free = tce_free_pSeriesLP;
1243
powerpc_firmware_features &= ~FW_FEATURE_MULTITCE;
1248
__setup("multitce=", disable_multitce);