git.itanic.dy.fi Git - linux-stable/blob - arch/powerpc/platforms/pseries/iommu.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
   4  *
   5  * Rewrite, cleanup:
   6  *
   7  * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
   8  * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
   9  *
  10  * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
  11  */
  12
  13 #include <linux/init.h>
  14 #include <linux/types.h>
  15 #include <linux/slab.h>
  16 #include <linux/mm.h>
  17 #include <linux/memblock.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/string.h>
  20 #include <linux/pci.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/crash_dump.h>
  23 #include <linux/memory.h>
  24 #include <linux/of.h>
  25 #include <linux/iommu.h>
  26 #include <linux/rculist.h>
  27 #include <asm/io.h>
  28 #include <asm/prom.h>
  29 #include <asm/rtas.h>
  30 #include <asm/iommu.h>
  31 #include <asm/pci-bridge.h>
  32 #include <asm/machdep.h>
  33 #include <asm/firmware.h>
  34 #include <asm/tce.h>
  35 #include <asm/ppc-pci.h>
  36 #include <asm/udbg.h>
  37 #include <asm/mmzone.h>
  38 #include <asm/plpar_wrappers.h>
  39
  40 #include "pseries.h"
  41
  42 enum {
  43         DDW_QUERY_PE_DMA_WIN  = 0,
  44         DDW_CREATE_PE_DMA_WIN = 1,
  45         DDW_REMOVE_PE_DMA_WIN = 2,
  46
  47         DDW_APPLICABLE_SIZE
  48 };
  49
  50 enum {
  51         DDW_EXT_SIZE = 0,
  52         DDW_EXT_RESET_DMA_WIN = 1,
  53         DDW_EXT_QUERY_OUT_SIZE = 2
  54 };
  55
  56 static struct iommu_table *iommu_pseries_alloc_table(int node)
  57 {
  58         struct iommu_table *tbl;
  59
  60         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node);
  61         if (!tbl)
  62                 return NULL;
  63
  64         INIT_LIST_HEAD_RCU(&tbl->it_group_list);
  65         kref_init(&tbl->it_kref);
  66         return tbl;
  67 }
  68
  69 static struct iommu_table_group *iommu_pseries_alloc_group(int node)
  70 {
  71         struct iommu_table_group *table_group;
  72
  73         table_group = kzalloc_node(sizeof(*table_group), GFP_KERNEL, node);
  74         if (!table_group)
  75                 return NULL;
  76
  77         table_group->tables[0] = iommu_pseries_alloc_table(node);
  78         if (table_group->tables[0])
  79                 return table_group;
  80
  81         kfree(table_group);
  82         return NULL;
  83 }
  84
  85 static void iommu_pseries_free_group(struct iommu_table_group *table_group,
  86                 const char *node_name)
  87 {
  88         if (!table_group)
  89                 return;
  90
  91 #ifdef CONFIG_IOMMU_API
  92         if (table_group->group) {
  93                 iommu_group_put(table_group->group);
  94                 BUG_ON(table_group->group);
  95         }
  96 #endif
  97
  98         /* Default DMA window table is at index 0, while DDW at 1. SR-IOV
  99          * adapters only have table on index 1.
 100          */
 101         if (table_group->tables[0])
 102                 iommu_tce_table_put(table_group->tables[0]);
 103
 104         if (table_group->tables[1])
 105                 iommu_tce_table_put(table_group->tables[1]);
 106
 107         kfree(table_group);
 108 }
 109
 110 static int tce_build_pSeries(struct iommu_table *tbl, long index,
 111                               long npages, unsigned long uaddr,
 112                               enum dma_data_direction direction,
 113                               unsigned long attrs)
 114 {
 115         u64 proto_tce;
 116         __be64 *tcep;
 117         u64 rpn;
 118         const unsigned long tceshift = tbl->it_page_shift;
 119         const unsigned long pagesize = IOMMU_PAGE_SIZE(tbl);
 120
 121         proto_tce = TCE_PCI_READ; // Read allowed
 122
 123         if (direction != DMA_TO_DEVICE)
 124                 proto_tce |= TCE_PCI_WRITE;
 125
 126         tcep = ((__be64 *)tbl->it_base) + index;
 127
 128         while (npages--) {
 129                 /* can't move this out since we might cross MEMBLOCK boundary */
 130                 rpn = __pa(uaddr) >> tceshift;
 131                 *tcep = cpu_to_be64(proto_tce | rpn << tceshift);
 132
 133                 uaddr += pagesize;
 134                 tcep++;
 135         }
 136         return 0;
 137 }
 138
 139
 140 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
 141 {
 142         __be64 *tcep;
 143
 144         tcep = ((__be64 *)tbl->it_base) + index;
 145
 146         while (npages--)
 147                 *(tcep++) = 0;
 148 }
 149
 150 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
 151 {
 152         __be64 *tcep;
 153
 154         tcep = ((__be64 *)tbl->it_base) + index;
 155
 156         return be64_to_cpu(*tcep);
 157 }
 158
 159 static void tce_free_pSeriesLP(unsigned long liobn, long, long, long);
 160 static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
 161
 162 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
 163                                 long npages, unsigned long uaddr,
 164                                 enum dma_data_direction direction,
 165                                 unsigned long attrs)
 166 {
 167         u64 rc = 0;
 168         u64 proto_tce, tce;
 169         u64 rpn;
 170         int ret = 0;
 171         long tcenum_start = tcenum, npages_start = npages;
 172
 173         rpn = __pa(uaddr) >> tceshift;
 174         proto_tce = TCE_PCI_READ;
 175         if (direction != DMA_TO_DEVICE)
 176                 proto_tce |= TCE_PCI_WRITE;
 177
 178         while (npages--) {
 179                 tce = proto_tce | rpn << tceshift;
 180                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, tce);
 181
 182                 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
 183                         ret = (int)rc;
 184                         tce_free_pSeriesLP(liobn, tcenum_start, tceshift,
 185                                            (npages_start - (npages + 1)));
 186                         break;
 187                 }
 188
 189                 if (rc && printk_ratelimit()) {
 190                         printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 191                         printk("\tindex   = 0x%llx\n", (u64)liobn);
 192                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 193                         printk("\ttce val = 0x%llx\n", tce );
 194                         dump_stack();
 195                 }
 196
 197                 tcenum++;
 198                 rpn++;
 199         }
 200         return ret;
 201 }
 202
 203 static DEFINE_PER_CPU(__be64 *, tce_page);
 204
 205 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
 206                                      long npages, unsigned long uaddr,
 207                                      enum dma_data_direction direction,
 208                                      unsigned long attrs)
 209 {
 210         u64 rc = 0;
 211         u64 proto_tce;
 212         __be64 *tcep;
 213         u64 rpn;
 214         long l, limit;
 215         long tcenum_start = tcenum, npages_start = npages;
 216         int ret = 0;
 217         unsigned long flags;
 218         const unsigned long tceshift = tbl->it_page_shift;
 219
 220         if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
 221                 return tce_build_pSeriesLP(tbl->it_index, tcenum,
 222                                            tceshift, npages, uaddr,
 223                                            direction, attrs);
 224         }
 225
 226         local_irq_save(flags);  /* to protect tcep and the page behind it */
 227
 228         tcep = __this_cpu_read(tce_page);
 229
 230         /* This is safe to do since interrupts are off when we're called
 231          * from iommu_alloc{,_sg}()
 232          */
 233         if (!tcep) {
 234                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
 235                 /* If allocation fails, fall back to the loop implementation */
 236                 if (!tcep) {
 237                         local_irq_restore(flags);
 238                         return tce_build_pSeriesLP(tbl->it_index, tcenum,
 239                                         tceshift,
 240                                         npages, uaddr, direction, attrs);
 241                 }
 242                 __this_cpu_write(tce_page, tcep);
 243         }
 244
 245         rpn = __pa(uaddr) >> tceshift;
 246         proto_tce = TCE_PCI_READ;
 247         if (direction != DMA_TO_DEVICE)
 248                 proto_tce |= TCE_PCI_WRITE;
 249
 250         /* We can map max one pageful of TCEs at a time */
 251         do {
 252                 /*
 253                  * Set up the page with TCE data, looping through and setting
 254                  * the values.
 255                  */
 256                 limit = min_t(long, npages, 4096 / TCE_ENTRY_SIZE);
 257
 258                 for (l = 0; l < limit; l++) {
 259                         tcep[l] = cpu_to_be64(proto_tce | rpn << tceshift);
 260                         rpn++;
 261                 }
 262
 263                 rc = plpar_tce_put_indirect((u64)tbl->it_index,
 264                                             (u64)tcenum << tceshift,
 265                                             (u64)__pa(tcep),
 266                                             limit);
 267
 268                 npages -= limit;
 269                 tcenum += limit;
 270         } while (npages > 0 && !rc);
 271
 272         local_irq_restore(flags);
 273
 274         if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
 275                 ret = (int)rc;
 276                 tce_freemulti_pSeriesLP(tbl, tcenum_start,
 277                                         (npages_start - (npages + limit)));
 278                 return ret;
 279         }
 280
 281         if (rc && printk_ratelimit()) {
 282                 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 283                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 284                 printk("\tnpages  = 0x%llx\n", (u64)npages);
 285                 printk("\ttce[0] val = 0x%llx\n", tcep[0]);
 286                 dump_stack();
 287         }
 288         return ret;
 289 }
 290
 291 static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
 292                                long npages)
 293 {
 294         u64 rc;
 295
 296         while (npages--) {
 297                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, 0);
 298
 299                 if (rc && printk_ratelimit()) {
 300                         printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 301                         printk("\tindex   = 0x%llx\n", (u64)liobn);
 302                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 303                         dump_stack();
 304                 }
 305
 306                 tcenum++;
 307         }
 308 }
 309
 310
 311 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
 312 {
 313         u64 rc;
 314
 315         if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
 316                 return tce_free_pSeriesLP(tbl->it_index, tcenum,
 317                                           tbl->it_page_shift, npages);
 318
 319         rc = plpar_tce_stuff((u64)tbl->it_index,
 320                              (u64)tcenum << tbl->it_page_shift, 0, npages);
 321
 322         if (rc && printk_ratelimit()) {
 323                 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
 324                 printk("\trc      = %lld\n", rc);
 325                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 326                 printk("\tnpages  = 0x%llx\n", (u64)npages);
 327                 dump_stack();
 328         }
 329 }
 330
 331 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
 332 {
 333         u64 rc;
 334         unsigned long tce_ret;
 335
 336         rc = plpar_tce_get((u64)tbl->it_index,
 337                            (u64)tcenum << tbl->it_page_shift, &tce_ret);
 338
 339         if (rc && printk_ratelimit()) {
 340                 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
 341                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 342                 printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 343                 dump_stack();
 344         }
 345
 346         return tce_ret;
 347 }
 348
 349 /* this is compatible with cells for the device tree property */
 350 struct dynamic_dma_window_prop {
 351         __be32  liobn;          /* tce table number */
 352         __be64  dma_base;       /* address hi,lo */
 353         __be32  tce_shift;      /* ilog2(tce_page_size) */
 354         __be32  window_shift;   /* ilog2(tce_window_size) */
 355 };
 356
 357 struct dma_win {
 358         struct device_node *device;
 359         const struct dynamic_dma_window_prop *prop;
 360         struct list_head list;
 361 };
 362
 363 /* Dynamic DMA Window support */
 364 struct ddw_query_response {
 365         u32 windows_available;
 366         u64 largest_available_block;
 367         u32 page_size;
 368         u32 migration_capable;
 369 };
 370
 371 struct ddw_create_response {
 372         u32 liobn;
 373         u32 addr_hi;
 374         u32 addr_lo;
 375 };
 376
 377 static LIST_HEAD(dma_win_list);
 378 /* prevents races between memory on/offline and window creation */
 379 static DEFINE_SPINLOCK(dma_win_list_lock);
 380 /* protects initializing window twice for same device */
 381 static DEFINE_MUTEX(dma_win_init_mutex);
 382 #define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
 383 #define DMA64_PROPNAME "linux,dma64-ddr-window-info"
 384
 385 static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
 386                                         unsigned long num_pfn, const void *arg)
 387 {
 388         const struct dynamic_dma_window_prop *maprange = arg;
 389         int rc;
 390         u64 tce_size, num_tce, dma_offset, next;
 391         u32 tce_shift;
 392         long limit;
 393
 394         tce_shift = be32_to_cpu(maprange->tce_shift);
 395         tce_size = 1ULL << tce_shift;
 396         next = start_pfn << PAGE_SHIFT;
 397         num_tce = num_pfn << PAGE_SHIFT;
 398
 399         /* round back to the beginning of the tce page size */
 400         num_tce += next & (tce_size - 1);
 401         next &= ~(tce_size - 1);
 402
 403         /* covert to number of tces */
 404         num_tce |= tce_size - 1;
 405         num_tce >>= tce_shift;
 406
 407         do {
 408                 /*
 409                  * Set up the page with TCE data, looping through and setting
 410                  * the values.
 411                  */
 412                 limit = min_t(long, num_tce, 512);
 413                 dma_offset = next + be64_to_cpu(maprange->dma_base);
 414
 415                 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
 416                                              dma_offset,
 417                                              0, limit);
 418                 next += limit * tce_size;
 419                 num_tce -= limit;
 420         } while (num_tce > 0 && !rc);
 421
 422         return rc;
 423 }
 424
 425 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
 426                                         unsigned long num_pfn, const void *arg)
 427 {
 428         const struct dynamic_dma_window_prop *maprange = arg;
 429         u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn;
 430         __be64 *tcep;
 431         u32 tce_shift;
 432         u64 rc = 0;
 433         long l, limit;
 434
 435         if (!firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
 436                 unsigned long tceshift = be32_to_cpu(maprange->tce_shift);
 437                 unsigned long dmastart = (start_pfn << PAGE_SHIFT) +
 438                                 be64_to_cpu(maprange->dma_base);
 439                 unsigned long tcenum = dmastart >> tceshift;
 440                 unsigned long npages = num_pfn << PAGE_SHIFT >> tceshift;
 441                 void *uaddr = __va(start_pfn << PAGE_SHIFT);
 442
 443                 return tce_build_pSeriesLP(be32_to_cpu(maprange->liobn),
 444                                 tcenum, tceshift, npages, (unsigned long) uaddr,
 445                                 DMA_BIDIRECTIONAL, 0);
 446         }
 447
 448         local_irq_disable();    /* to protect tcep and the page behind it */
 449         tcep = __this_cpu_read(tce_page);
 450
 451         if (!tcep) {
 452                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
 453                 if (!tcep) {
 454                         local_irq_enable();
 455                         return -ENOMEM;
 456                 }
 457                 __this_cpu_write(tce_page, tcep);
 458         }
 459
 460         proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
 461
 462         liobn = (u64)be32_to_cpu(maprange->liobn);
 463         tce_shift = be32_to_cpu(maprange->tce_shift);
 464         tce_size = 1ULL << tce_shift;
 465         next = start_pfn << PAGE_SHIFT;
 466         num_tce = num_pfn << PAGE_SHIFT;
 467
 468         /* round back to the beginning of the tce page size */
 469         num_tce += next & (tce_size - 1);
 470         next &= ~(tce_size - 1);
 471
 472         /* covert to number of tces */
 473         num_tce |= tce_size - 1;
 474         num_tce >>= tce_shift;
 475
 476         /* We can map max one pageful of TCEs at a time */
 477         do {
 478                 /*
 479                  * Set up the page with TCE data, looping through and setting
 480                  * the values.
 481                  */
 482                 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
 483                 dma_offset = next + be64_to_cpu(maprange->dma_base);
 484
 485                 for (l = 0; l < limit; l++) {
 486                         tcep[l] = cpu_to_be64(proto_tce | next);
 487                         next += tce_size;
 488                 }
 489
 490                 rc = plpar_tce_put_indirect(liobn,
 491                                             dma_offset,
 492                                             (u64)__pa(tcep),
 493                                             limit);
 494
 495                 num_tce -= limit;
 496         } while (num_tce > 0 && !rc);
 497
 498         /* error cleanup: caller will clear whole range */
 499
 500         local_irq_enable();
 501         return rc;
 502 }
 503
 504 static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
 505                 unsigned long num_pfn, void *arg)
 506 {
 507         return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
 508 }
 509
 510 static void iommu_table_setparms_common(struct iommu_table *tbl, unsigned long busno,
 511                                         unsigned long liobn, unsigned long win_addr,
 512                                         unsigned long window_size, unsigned long page_shift,
 513                                         void *base, struct iommu_table_ops *table_ops)
 514 {
 515         tbl->it_busno = busno;
 516         tbl->it_index = liobn;
 517         tbl->it_offset = win_addr >> page_shift;
 518         tbl->it_size = window_size >> page_shift;
 519         tbl->it_page_shift = page_shift;
 520         tbl->it_base = (unsigned long)base;
 521         tbl->it_blocksize = 16;
 522         tbl->it_type = TCE_PCI;
 523         tbl->it_ops = table_ops;
 524 }
 525
 526 struct iommu_table_ops iommu_table_pseries_ops;
 527
 528 static void iommu_table_setparms(struct pci_controller *phb,
 529                                  struct device_node *dn,
 530                                  struct iommu_table *tbl)
 531 {
 532         struct device_node *node;
 533         const unsigned long *basep;
 534         const u32 *sizep;
 535
 536         /* Test if we are going over 2GB of DMA space */
 537         if (phb->dma_window_base_cur + phb->dma_window_size > SZ_2G) {
 538                 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
 539                 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
 540         }
 541
 542         node = phb->dn;
 543         basep = of_get_property(node, "linux,tce-base", NULL);
 544         sizep = of_get_property(node, "linux,tce-size", NULL);
 545         if (basep == NULL || sizep == NULL) {
 546                 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %pOF has "
 547                                 "missing tce entries !\n", dn);
 548                 return;
 549         }
 550
 551         iommu_table_setparms_common(tbl, phb->bus->number, 0, phb->dma_window_base_cur,
 552                                     phb->dma_window_size, IOMMU_PAGE_SHIFT_4K,
 553                                     __va(*basep), &iommu_table_pseries_ops);
 554
 555         if (!is_kdump_kernel())
 556                 memset((void *)tbl->it_base, 0, *sizep);
 557
 558         phb->dma_window_base_cur += phb->dma_window_size;
 559 }
 560
 561 struct iommu_table_ops iommu_table_lpar_multi_ops;
 562
 563 /*
 564  * iommu_table_setparms_lpar
 565  *
 566  * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
 567  */
 568 static void iommu_table_setparms_lpar(struct pci_controller *phb,
 569                                       struct device_node *dn,
 570                                       struct iommu_table *tbl,
 571                                       struct iommu_table_group *table_group,
 572                                       const __be32 *dma_window)
 573 {
 574         unsigned long offset, size, liobn;
 575
 576         of_parse_dma_window(dn, dma_window, &liobn, &offset, &size);
 577
 578         iommu_table_setparms_common(tbl, phb->bus->number, liobn, offset, size, IOMMU_PAGE_SHIFT_4K, NULL,
 579                                     &iommu_table_lpar_multi_ops);
 580
 581
 582         table_group->tce32_start = offset;
 583         table_group->tce32_size = size;
 584 }
 585
 586 struct iommu_table_ops iommu_table_pseries_ops = {
 587         .set = tce_build_pSeries,
 588         .clear = tce_free_pSeries,
 589         .get = tce_get_pseries
 590 };
 591
 592 static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
 593 {
 594         struct device_node *dn;
 595         struct iommu_table *tbl;
 596         struct device_node *isa_dn, *isa_dn_orig;
 597         struct device_node *tmp;
 598         struct pci_dn *pci;
 599         int children;
 600
 601         dn = pci_bus_to_OF_node(bus);
 602
 603         pr_debug("pci_dma_bus_setup_pSeries: setting up bus %pOF\n", dn);
 604
 605         if (bus->self) {
 606                 /* This is not a root bus, any setup will be done for the
 607                  * device-side of the bridge in iommu_dev_setup_pSeries().
 608                  */
 609                 return;
 610         }
 611         pci = PCI_DN(dn);
 612
 613         /* Check if the ISA bus on the system is under
 614          * this PHB.
 615          */
 616         isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
 617
 618         while (isa_dn && isa_dn != dn)
 619                 isa_dn = isa_dn->parent;
 620
 621         of_node_put(isa_dn_orig);
 622
 623         /* Count number of direct PCI children of the PHB. */
 624         for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
 625                 children++;
 626
 627         pr_debug("Children: %d\n", children);
 628
 629         /* Calculate amount of DMA window per slot. Each window must be
 630          * a power of two (due to pci_alloc_consistent requirements).
 631          *
 632          * Keep 256MB aside for PHBs with ISA.
 633          */
 634
 635         if (!isa_dn) {
 636                 /* No ISA/IDE - just set window size and return */
 637                 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */
 638
 639                 while (pci->phb->dma_window_size * children > 0x80000000ul)
 640                         pci->phb->dma_window_size >>= 1;
 641                 pr_debug("No ISA/IDE, window size is 0x%llx\n",
 642                          pci->phb->dma_window_size);
 643                 pci->phb->dma_window_base_cur = 0;
 644
 645                 return;
 646         }
 647
 648         /* If we have ISA, then we probably have an IDE
 649          * controller too. Allocate a 128MB table but
 650          * skip the first 128MB to avoid stepping on ISA
 651          * space.
 652          */
 653         pci->phb->dma_window_size = 0x8000000ul;
 654         pci->phb->dma_window_base_cur = 0x8000000ul;
 655
 656         pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
 657         tbl = pci->table_group->tables[0];
 658
 659         iommu_table_setparms(pci->phb, dn, tbl);
 660
 661         if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
 662                 panic("Failed to initialize iommu table");
 663
 664         /* Divide the rest (1.75GB) among the children */
 665         pci->phb->dma_window_size = 0x80000000ul;
 666         while (pci->phb->dma_window_size * children > 0x70000000ul)
 667                 pci->phb->dma_window_size >>= 1;
 668
 669         pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
 670 }
 671
 672 #ifdef CONFIG_IOMMU_API
 673 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned
 674                                 long *tce, enum dma_data_direction *direction)
 675 {
 676         long rc;
 677         unsigned long ioba = (unsigned long) index << tbl->it_page_shift;
 678         unsigned long flags, oldtce = 0;
 679         u64 proto_tce = iommu_direction_to_tce_perm(*direction);
 680         unsigned long newtce = *tce | proto_tce;
 681
 682         spin_lock_irqsave(&tbl->large_pool.lock, flags);
 683
 684         rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce);
 685         if (!rc)
 686                 rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce);
 687
 688         if (!rc) {
 689                 *direction = iommu_tce_direction(oldtce);
 690                 *tce = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
 691         }
 692
 693         spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
 694
 695         return rc;
 696 }
 697 #endif
 698
 699 struct iommu_table_ops iommu_table_lpar_multi_ops = {
 700         .set = tce_buildmulti_pSeriesLP,
 701 #ifdef CONFIG_IOMMU_API
 702         .xchg_no_kill = tce_exchange_pseries,
 703 #endif
 704         .clear = tce_freemulti_pSeriesLP,
 705         .get = tce_get_pSeriesLP
 706 };
 707
 708 /*
 709  * Find nearest ibm,dma-window (default DMA window) or direct DMA window or
 710  * dynamic 64bit DMA window, walking up the device tree.
 711  */
 712 static struct device_node *pci_dma_find(struct device_node *dn,
 713                                         const __be32 **dma_window)
 714 {
 715         const __be32 *dw = NULL;
 716
 717         for ( ; dn && PCI_DN(dn); dn = dn->parent) {
 718                 dw = of_get_property(dn, "ibm,dma-window", NULL);
 719                 if (dw) {
 720                         if (dma_window)
 721                                 *dma_window = dw;
 722                         return dn;
 723                 }
 724                 dw = of_get_property(dn, DIRECT64_PROPNAME, NULL);
 725                 if (dw)
 726                         return dn;
 727                 dw = of_get_property(dn, DMA64_PROPNAME, NULL);
 728                 if (dw)
 729                         return dn;
 730         }
 731
 732         return NULL;
 733 }
 734
 735 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
 736 {
 737         struct iommu_table *tbl;
 738         struct device_node *dn, *pdn;
 739         struct pci_dn *ppci;
 740         const __be32 *dma_window = NULL;
 741
 742         dn = pci_bus_to_OF_node(bus);
 743
 744         pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %pOF\n",
 745                  dn);
 746
 747         pdn = pci_dma_find(dn, &dma_window);
 748
 749         if (dma_window == NULL)
 750                 pr_debug("  no ibm,dma-window property !\n");
 751
 752         ppci = PCI_DN(pdn);
 753
 754         pr_debug("  parent is %pOF, iommu_table: 0x%p\n",
 755                  pdn, ppci->table_group);
 756
 757         if (!ppci->table_group) {
 758                 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node);
 759                 tbl = ppci->table_group->tables[0];
 760                 if (dma_window) {
 761                         iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
 762                                                   ppci->table_group, dma_window);
 763
 764                         if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
 765                                 panic("Failed to initialize iommu table");
 766                 }
 767                 iommu_register_group(ppci->table_group,
 768                                 pci_domain_nr(bus), 0);
 769                 pr_debug("  created table: %p\n", ppci->table_group);
 770         }
 771 }
 772
 773
 774 static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
 775 {
 776         struct device_node *dn;
 777         struct iommu_table *tbl;
 778
 779         pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
 780
 781         dn = dev->dev.of_node;
 782
 783         /* If we're the direct child of a root bus, then we need to allocate
 784          * an iommu table ourselves. The bus setup code should have setup
 785          * the window sizes already.
 786          */
 787         if (!dev->bus->self) {
 788                 struct pci_controller *phb = PCI_DN(dn)->phb;
 789
 790                 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
 791                 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node);
 792                 tbl = PCI_DN(dn)->table_group->tables[0];
 793                 iommu_table_setparms(phb, dn, tbl);
 794
 795                 if (!iommu_init_table(tbl, phb->node, 0, 0))
 796                         panic("Failed to initialize iommu table");
 797
 798                 set_iommu_table_base(&dev->dev, tbl);
 799                 return;
 800         }
 801
 802         /* If this device is further down the bus tree, search upwards until
 803          * an already allocated iommu table is found and use that.
 804          */
 805
 806         while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL)
 807                 dn = dn->parent;
 808
 809         if (dn && PCI_DN(dn))
 810                 set_iommu_table_base(&dev->dev,
 811                                 PCI_DN(dn)->table_group->tables[0]);
 812         else
 813                 printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
 814                        pci_name(dev));
 815 }
 816
 817 static int __read_mostly disable_ddw;
 818
 819 static int __init disable_ddw_setup(char *str)
 820 {
 821         disable_ddw = 1;
 822         printk(KERN_INFO "ppc iommu: disabling ddw.\n");
 823
 824         return 0;
 825 }
 826
 827 early_param("disable_ddw", disable_ddw_setup);
 828
 829 static void clean_dma_window(struct device_node *np, struct dynamic_dma_window_prop *dwp)
 830 {
 831         int ret;
 832
 833         ret = tce_clearrange_multi_pSeriesLP(0,
 834                 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
 835         if (ret)
 836                 pr_warn("%pOF failed to clear tces in window.\n",
 837                         np);
 838         else
 839                 pr_debug("%pOF successfully cleared tces in window.\n",
 840                          np);
 841 }
 842
 843 /*
 844  * Call only if DMA window is clean.
 845  */
 846 static void __remove_dma_window(struct device_node *np, u32 *ddw_avail, u64 liobn)
 847 {
 848         int ret;
 849
 850         ret = rtas_call(ddw_avail[DDW_REMOVE_PE_DMA_WIN], 1, 1, NULL, liobn);
 851         if (ret)
 852                 pr_warn("%pOF: failed to remove DMA window: rtas returned "
 853                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
 854                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
 855         else
 856                 pr_debug("%pOF: successfully removed DMA window: rtas returned "
 857                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
 858                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
 859 }
 860
 861 static void remove_dma_window(struct device_node *np, u32 *ddw_avail,
 862                               struct property *win)
 863 {
 864         struct dynamic_dma_window_prop *dwp;
 865         u64 liobn;
 866
 867         dwp = win->value;
 868         liobn = (u64)be32_to_cpu(dwp->liobn);
 869
 870         clean_dma_window(np, dwp);
 871         __remove_dma_window(np, ddw_avail, liobn);
 872 }
 873
 874 static int remove_ddw(struct device_node *np, bool remove_prop, const char *win_name)
 875 {
 876         struct property *win;
 877         u32 ddw_avail[DDW_APPLICABLE_SIZE];
 878         int ret = 0;
 879
 880         win = of_find_property(np, win_name, NULL);
 881         if (!win)
 882                 return -EINVAL;
 883
 884         ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
 885                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
 886         if (ret)
 887                 return 0;
 888
 889
 890         if (win->length >= sizeof(struct dynamic_dma_window_prop))
 891                 remove_dma_window(np, ddw_avail, win);
 892
 893         if (!remove_prop)
 894                 return 0;
 895
 896         ret = of_remove_property(np, win);
 897         if (ret)
 898                 pr_warn("%pOF: failed to remove DMA window property: %d\n",
 899                         np, ret);
 900         return 0;
 901 }
 902
 903 static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift)
 904 {
 905         struct dma_win *window;
 906         const struct dynamic_dma_window_prop *dma64;
 907         bool found = false;
 908
 909         spin_lock(&dma_win_list_lock);
 910         /* check if we already created a window and dupe that config if so */
 911         list_for_each_entry(window, &dma_win_list, list) {
 912                 if (window->device == pdn) {
 913                         dma64 = window->prop;
 914                         *dma_addr = be64_to_cpu(dma64->dma_base);
 915                         *window_shift = be32_to_cpu(dma64->window_shift);
 916                         found = true;
 917                         break;
 918                 }
 919         }
 920         spin_unlock(&dma_win_list_lock);
 921
 922         return found;
 923 }
 924
 925 static struct dma_win *ddw_list_new_entry(struct device_node *pdn,
 926                                           const struct dynamic_dma_window_prop *dma64)
 927 {
 928         struct dma_win *window;
 929
 930         window = kzalloc(sizeof(*window), GFP_KERNEL);
 931         if (!window)
 932                 return NULL;
 933
 934         window->device = pdn;
 935         window->prop = dma64;
 936
 937         return window;
 938 }
 939
 940 static void find_existing_ddw_windows_named(const char *name)
 941 {
 942         int len;
 943         struct device_node *pdn;
 944         struct dma_win *window;
 945         const struct dynamic_dma_window_prop *dma64;
 946
 947         for_each_node_with_property(pdn, name) {
 948                 dma64 = of_get_property(pdn, name, &len);
 949                 if (!dma64 || len < sizeof(*dma64)) {
 950                         remove_ddw(pdn, true, name);
 951                         continue;
 952                 }
 953
 954                 window = ddw_list_new_entry(pdn, dma64);
 955                 if (!window) {
 956                         of_node_put(pdn);
 957                         break;
 958                 }
 959
 960                 spin_lock(&dma_win_list_lock);
 961                 list_add(&window->list, &dma_win_list);
 962                 spin_unlock(&dma_win_list_lock);
 963         }
 964 }
 965
 966 static int find_existing_ddw_windows(void)
 967 {
 968         if (!firmware_has_feature(FW_FEATURE_LPAR))
 969                 return 0;
 970
 971         find_existing_ddw_windows_named(DIRECT64_PROPNAME);
 972         find_existing_ddw_windows_named(DMA64_PROPNAME);
 973
 974         return 0;
 975 }
 976 machine_arch_initcall(pseries, find_existing_ddw_windows);
 977
 978 /**
 979  * ddw_read_ext - Get the value of an DDW extension
 980  * @np:         device node from which the extension value is to be read.
 981  * @extnum:     index number of the extension.
 982  * @value:      pointer to return value, modified when extension is available.
 983  *
 984  * Checks if "ibm,ddw-extensions" exists for this node, and get the value
 985  * on index 'extnum'.
 986  * It can be used only to check if a property exists, passing value == NULL.
 987  *
 988  * Returns:
 989  *      0 if extension successfully read
 990  *      -EINVAL if the "ibm,ddw-extensions" does not exist,
 991  *      -ENODATA if "ibm,ddw-extensions" does not have a value, and
 992  *      -EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
 993  */
 994 static inline int ddw_read_ext(const struct device_node *np, int extnum,
 995                                u32 *value)
 996 {
 997         static const char propname[] = "ibm,ddw-extensions";
 998         u32 count;
 999         int ret;
1000
1001         ret = of_property_read_u32_index(np, propname, DDW_EXT_SIZE, &count);
1002         if (ret)
1003                 return ret;
1004
1005         if (count < extnum)
1006                 return -EOVERFLOW;
1007
1008         if (!value)
1009                 value = &count;
1010
1011         return of_property_read_u32_index(np, propname, extnum, value);
1012 }
1013
1014 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1015                      struct ddw_query_response *query,
1016                      struct device_node *parent)
1017 {
1018         struct device_node *dn;
1019         struct pci_dn *pdn;
1020         u32 cfg_addr, ext_query, query_out[5];
1021         u64 buid;
1022         int ret, out_sz;
1023
1024         /*
1025          * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many
1026          * output parameters ibm,query-pe-dma-windows will have, ranging from
1027          * 5 to 6.
1028          */
1029         ret = ddw_read_ext(parent, DDW_EXT_QUERY_OUT_SIZE, &ext_query);
1030         if (!ret && ext_query == 1)
1031                 out_sz = 6;
1032         else
1033                 out_sz = 5;
1034
1035         /*
1036          * Get the config address and phb buid of the PE window.
1037          * Rely on eeh to retrieve this for us.
1038          * Retrieve them from the pci device, not the node with the
1039          * dma-window property
1040          */
1041         dn = pci_device_to_OF_node(dev);
1042         pdn = PCI_DN(dn);
1043         buid = pdn->phb->buid;
1044         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1045
1046         ret = rtas_call(ddw_avail[DDW_QUERY_PE_DMA_WIN], 3, out_sz, query_out,
1047                         cfg_addr, BUID_HI(buid), BUID_LO(buid));
1048
1049         switch (out_sz) {
1050         case 5:
1051                 query->windows_available = query_out[0];
1052                 query->largest_available_block = query_out[1];
1053                 query->page_size = query_out[2];
1054                 query->migration_capable = query_out[3];
1055                 break;
1056         case 6:
1057                 query->windows_available = query_out[0];
1058                 query->largest_available_block = ((u64)query_out[1] << 32) |
1059                                                  query_out[2];
1060                 query->page_size = query_out[3];
1061                 query->migration_capable = query_out[4];
1062                 break;
1063         }
1064
1065         dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d, lb=%llx ps=%x wn=%d\n",
1066                  ddw_avail[DDW_QUERY_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1067                  BUID_LO(buid), ret, query->largest_available_block,
1068                  query->page_size, query->windows_available);
1069
1070         return ret;
1071 }
1072
1073 static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1074                         struct ddw_create_response *create, int page_shift,
1075                         int window_shift)
1076 {
1077         struct device_node *dn;
1078         struct pci_dn *pdn;
1079         u32 cfg_addr;
1080         u64 buid;
1081         int ret;
1082
1083         /*
1084          * Get the config address and phb buid of the PE window.
1085          * Rely on eeh to retrieve this for us.
1086          * Retrieve them from the pci device, not the node with the
1087          * dma-window property
1088          */
1089         dn = pci_device_to_OF_node(dev);
1090         pdn = PCI_DN(dn);
1091         buid = pdn->phb->buid;
1092         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1093
1094         do {
1095                 /* extra outputs are LIOBN and dma-addr (hi, lo) */
1096                 ret = rtas_call(ddw_avail[DDW_CREATE_PE_DMA_WIN], 5, 4,
1097                                 (u32 *)create, cfg_addr, BUID_HI(buid),
1098                                 BUID_LO(buid), page_shift, window_shift);
1099         } while (rtas_busy_delay(ret));
1100         dev_info(&dev->dev,
1101                 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
1102                 "(liobn = 0x%x starting addr = %x %x)\n",
1103                  ddw_avail[DDW_CREATE_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1104                  BUID_LO(buid), page_shift, window_shift, ret, create->liobn,
1105                  create->addr_hi, create->addr_lo);
1106
1107         return ret;
1108 }
1109
1110 struct failed_ddw_pdn {
1111         struct device_node *pdn;
1112         struct list_head list;
1113 };
1114
1115 static LIST_HEAD(failed_ddw_pdn_list);
1116
1117 static phys_addr_t ddw_memory_hotplug_max(void)
1118 {
1119         phys_addr_t max_addr = memory_hotplug_max();
1120         struct device_node *memory;
1121
1122         for_each_node_by_type(memory, "memory") {
1123                 unsigned long start, size;
1124                 int n_mem_addr_cells, n_mem_size_cells, len;
1125                 const __be32 *memcell_buf;
1126
1127                 memcell_buf = of_get_property(memory, "reg", &len);
1128                 if (!memcell_buf || len <= 0)
1129                         continue;
1130
1131                 n_mem_addr_cells = of_n_addr_cells(memory);
1132                 n_mem_size_cells = of_n_size_cells(memory);
1133
1134                 start = of_read_number(memcell_buf, n_mem_addr_cells);
1135                 memcell_buf += n_mem_addr_cells;
1136                 size = of_read_number(memcell_buf, n_mem_size_cells);
1137                 memcell_buf += n_mem_size_cells;
1138
1139                 max_addr = max_t(phys_addr_t, max_addr, start + size);
1140         }
1141
1142         return max_addr;
1143 }
1144
1145 /*
1146  * Platforms supporting the DDW option starting with LoPAR level 2.7 implement
1147  * ibm,ddw-extensions, which carries the rtas token for
1148  * ibm,reset-pe-dma-windows.
1149  * That rtas-call can be used to restore the default DMA window for the device.
1150  */
1151 static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
1152 {
1153         int ret;
1154         u32 cfg_addr, reset_dma_win;
1155         u64 buid;
1156         struct device_node *dn;
1157         struct pci_dn *pdn;
1158
1159         ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win);
1160         if (ret)
1161                 return;
1162
1163         dn = pci_device_to_OF_node(dev);
1164         pdn = PCI_DN(dn);
1165         buid = pdn->phb->buid;
1166         cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8);
1167
1168         ret = rtas_call(reset_dma_win, 3, 1, NULL, cfg_addr, BUID_HI(buid),
1169                         BUID_LO(buid));
1170         if (ret)
1171                 dev_info(&dev->dev,
1172                          "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ",
1173                          reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid),
1174                          ret);
1175 }
1176
1177 /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
1178 static int iommu_get_page_shift(u32 query_page_size)
1179 {
1180         /* Supported IO page-sizes according to LoPAR, note that 2M is out of order */
1181         const int shift[] = {
1182                 __builtin_ctzll(SZ_4K),   __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
1183                 __builtin_ctzll(SZ_32M),  __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
1184                 __builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G), __builtin_ctzll(SZ_2M)
1185         };
1186
1187         int i = ARRAY_SIZE(shift) - 1;
1188         int ret = 0;
1189
1190         /*
1191          * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
1192          * - bit 31 means 4k pages are supported,
1193          * - bit 30 means 64k pages are supported, and so on.
1194          * Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
1195          */
1196         for (; i >= 0 ; i--) {
1197                 if (query_page_size & (1 << i))
1198                         ret = max(ret, shift[i]);
1199         }
1200
1201         return ret;
1202 }
1203
1204 static struct property *ddw_property_create(const char *propname, u32 liobn, u64 dma_addr,
1205                                             u32 page_shift, u32 window_shift)
1206 {
1207         struct dynamic_dma_window_prop *ddwprop;
1208         struct property *win64;
1209
1210         win64 = kzalloc(sizeof(*win64), GFP_KERNEL);
1211         if (!win64)
1212                 return NULL;
1213
1214         win64->name = kstrdup(propname, GFP_KERNEL);
1215         ddwprop = kzalloc(sizeof(*ddwprop), GFP_KERNEL);
1216         win64->value = ddwprop;
1217         win64->length = sizeof(*ddwprop);
1218         if (!win64->name || !win64->value) {
1219                 kfree(win64->name);
1220                 kfree(win64->value);
1221                 kfree(win64);
1222                 return NULL;
1223         }
1224
1225         ddwprop->liobn = cpu_to_be32(liobn);
1226         ddwprop->dma_base = cpu_to_be64(dma_addr);
1227         ddwprop->tce_shift = cpu_to_be32(page_shift);
1228         ddwprop->window_shift = cpu_to_be32(window_shift);
1229
1230         return win64;
1231 }
1232
1233 /*
1234  * If the PE supports dynamic dma windows, and there is space for a table
1235  * that can map all pages in a linear offset, then setup such a table,
1236  * and record the dma-offset in the struct device.
1237  *
1238  * dev: the pci device we are checking
1239  * pdn: the parent pe node with the ibm,dma_window property
1240  * Future: also check if we can remap the base window for our base page size
1241  *
1242  * returns true if can map all pages (direct mapping), false otherwise..
1243  */
1244 static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn)
1245 {
1246         int len = 0, ret;
1247         int max_ram_len = order_base_2(ddw_memory_hotplug_max());
1248         struct ddw_query_response query;
1249         struct ddw_create_response create;
1250         int page_shift;
1251         u64 win_addr;
1252         const char *win_name;
1253         struct device_node *dn;
1254         u32 ddw_avail[DDW_APPLICABLE_SIZE];
1255         struct dma_win *window;
1256         struct property *win64;
1257         struct failed_ddw_pdn *fpdn;
1258         bool default_win_removed = false, direct_mapping = false;
1259         bool pmem_present;
1260         struct pci_dn *pci = PCI_DN(pdn);
1261         struct property *default_win = NULL;
1262
1263         dn = of_find_node_by_type(NULL, "ibm,pmemory");
1264         pmem_present = dn != NULL;
1265         of_node_put(dn);
1266
1267         mutex_lock(&dma_win_init_mutex);
1268
1269         if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len)) {
1270                 direct_mapping = (len >= max_ram_len);
1271                 goto out_unlock;
1272         }
1273
1274         /*
1275          * If we already went through this for a previous function of
1276          * the same device and failed, we don't want to muck with the
1277          * DMA window again, as it will race with in-flight operations
1278          * and can lead to EEHs. The above mutex protects access to the
1279          * list.
1280          */
1281         list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) {
1282                 if (fpdn->pdn == pdn)
1283                         goto out_unlock;
1284         }
1285
1286         /*
1287          * the ibm,ddw-applicable property holds the tokens for:
1288          * ibm,query-pe-dma-window
1289          * ibm,create-pe-dma-window
1290          * ibm,remove-pe-dma-window
1291          * for the given node in that order.
1292          * the property is actually in the parent, not the PE
1293          */
1294         ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable",
1295                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
1296         if (ret)
1297                 goto out_failed;
1298
1299        /*
1300          * Query if there is a second window of size to map the
1301          * whole partition.  Query returns number of windows, largest
1302          * block assigned to PE (partition endpoint), and two bitmasks
1303          * of page sizes: supported and supported for migrate-dma.
1304          */
1305         dn = pci_device_to_OF_node(dev);
1306         ret = query_ddw(dev, ddw_avail, &query, pdn);
1307         if (ret != 0)
1308                 goto out_failed;
1309
1310         /*
1311          * If there is no window available, remove the default DMA window,
1312          * if it's present. This will make all the resources available to the
1313          * new DDW window.
1314          * If anything fails after this, we need to restore it, so also check
1315          * for extensions presence.
1316          */
1317         if (query.windows_available == 0) {
1318                 int reset_win_ext;
1319
1320                 /* DDW + IOMMU on single window may fail if there is any allocation */
1321                 if (iommu_table_in_use(pci->table_group->tables[0])) {
1322                         dev_warn(&dev->dev, "current IOMMU table in use, can't be replaced.\n");
1323                         goto out_failed;
1324                 }
1325
1326                 default_win = of_find_property(pdn, "ibm,dma-window", NULL);
1327                 if (!default_win)
1328                         goto out_failed;
1329
1330                 reset_win_ext = ddw_read_ext(pdn, DDW_EXT_RESET_DMA_WIN, NULL);
1331                 if (reset_win_ext)
1332                         goto out_failed;
1333
1334                 remove_dma_window(pdn, ddw_avail, default_win);
1335                 default_win_removed = true;
1336
1337                 /* Query again, to check if the window is available */
1338                 ret = query_ddw(dev, ddw_avail, &query, pdn);
1339                 if (ret != 0)
1340                         goto out_failed;
1341
1342                 if (query.windows_available == 0) {
1343                         /* no windows are available for this device. */
1344                         dev_dbg(&dev->dev, "no free dynamic windows");
1345                         goto out_failed;
1346                 }
1347         }
1348
1349         page_shift = iommu_get_page_shift(query.page_size);
1350         if (!page_shift) {
1351                 dev_dbg(&dev->dev, "no supported page size in mask %x",
1352                         query.page_size);
1353                 goto out_failed;
1354         }
1355
1356
1357         /*
1358          * The "ibm,pmemory" can appear anywhere in the address space.
1359          * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS
1360          * for the upper limit and fallback to max RAM otherwise but this
1361          * disables device::dma_ops_bypass.
1362          */
1363         len = max_ram_len;
1364         if (pmem_present) {
1365                 if (query.largest_available_block >=
1366                     (1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1367                         len = MAX_PHYSMEM_BITS;
1368                 else
1369                         dev_info(&dev->dev, "Skipping ibm,pmemory");
1370         }
1371
1372         /* check if the available block * number of ptes will map everything */
1373         if (query.largest_available_block < (1ULL << (len - page_shift))) {
1374                 dev_dbg(&dev->dev,
1375                         "can't map partition max 0x%llx with %llu %llu-sized pages\n",
1376                         1ULL << len,
1377                         query.largest_available_block,
1378                         1ULL << page_shift);
1379
1380                 len = order_base_2(query.largest_available_block << page_shift);
1381                 win_name = DMA64_PROPNAME;
1382         } else {
1383                 direct_mapping = !default_win_removed ||
1384                         (len == MAX_PHYSMEM_BITS) ||
1385                         (!pmem_present && (len == max_ram_len));
1386                 win_name = direct_mapping ? DIRECT64_PROPNAME : DMA64_PROPNAME;
1387         }
1388
1389         ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
1390         if (ret != 0)
1391                 goto out_failed;
1392
1393         dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n",
1394                   create.liobn, dn);
1395
1396         win_addr = ((u64)create.addr_hi << 32) | create.addr_lo;
1397         win64 = ddw_property_create(win_name, create.liobn, win_addr, page_shift, len);
1398
1399         if (!win64) {
1400                 dev_info(&dev->dev,
1401                          "couldn't allocate property, property name, or value\n");
1402                 goto out_remove_win;
1403         }
1404
1405         ret = of_add_property(pdn, win64);
1406         if (ret) {
1407                 dev_err(&dev->dev, "unable to add DMA window property for %pOF: %d",
1408                         pdn, ret);
1409                 goto out_free_prop;
1410         }
1411
1412         window = ddw_list_new_entry(pdn, win64->value);
1413         if (!window)
1414                 goto out_del_prop;
1415
1416         if (direct_mapping) {
1417                 /* DDW maps the whole partition, so enable direct DMA mapping */
1418                 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
1419                                             win64->value, tce_setrange_multi_pSeriesLP_walk);
1420                 if (ret) {
1421                         dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n",
1422                                  dn, ret);
1423
1424                         /* Make sure to clean DDW if any TCE was set*/
1425                         clean_dma_window(pdn, win64->value);
1426                         goto out_del_list;
1427                 }
1428         } else {
1429                 struct iommu_table *newtbl;
1430                 int i;
1431                 unsigned long start = 0, end = 0;
1432
1433                 for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) {
1434                         const unsigned long mask = IORESOURCE_MEM_64 | IORESOURCE_MEM;
1435
1436                         /* Look for MMIO32 */
1437                         if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) {
1438                                 start = pci->phb->mem_resources[i].start;
1439                                 end = pci->phb->mem_resources[i].end;
1440                                 break;
1441                         }
1442                 }
1443
1444                 /* New table for using DDW instead of the default DMA window */
1445                 newtbl = iommu_pseries_alloc_table(pci->phb->node);
1446                 if (!newtbl) {
1447                         dev_dbg(&dev->dev, "couldn't create new IOMMU table\n");
1448                         goto out_del_list;
1449                 }
1450
1451                 iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, win_addr,
1452                                             1UL << len, page_shift, NULL, &iommu_table_lpar_multi_ops);
1453                 iommu_init_table(newtbl, pci->phb->node, start, end);
1454
1455                 pci->table_group->tables[1] = newtbl;
1456
1457                 set_iommu_table_base(&dev->dev, newtbl);
1458         }
1459
1460         if (default_win_removed) {
1461                 iommu_tce_table_put(pci->table_group->tables[0]);
1462                 pci->table_group->tables[0] = NULL;
1463
1464                 /* default_win is valid here because default_win_removed == true */
1465                 of_remove_property(pdn, default_win);
1466                 dev_info(&dev->dev, "Removed default DMA window for %pOF\n", pdn);
1467         }
1468
1469         spin_lock(&dma_win_list_lock);
1470         list_add(&window->list, &dma_win_list);
1471         spin_unlock(&dma_win_list_lock);
1472
1473         dev->dev.archdata.dma_offset = win_addr;
1474         goto out_unlock;
1475
1476 out_del_list:
1477         kfree(window);
1478
1479 out_del_prop:
1480         of_remove_property(pdn, win64);
1481
1482 out_free_prop:
1483         kfree(win64->name);
1484         kfree(win64->value);
1485         kfree(win64);
1486
1487 out_remove_win:
1488         /* DDW is clean, so it's ok to call this directly. */
1489         __remove_dma_window(pdn, ddw_avail, create.liobn);
1490
1491 out_failed:
1492         if (default_win_removed)
1493                 reset_dma_window(dev, pdn);
1494
1495         fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
1496         if (!fpdn)
1497                 goto out_unlock;
1498         fpdn->pdn = pdn;
1499         list_add(&fpdn->list, &failed_ddw_pdn_list);
1500
1501 out_unlock:
1502         mutex_unlock(&dma_win_init_mutex);
1503
1504         /*
1505          * If we have persistent memory and the window size is only as big
1506          * as RAM, then we failed to create a window to cover persistent
1507          * memory and need to set the DMA limit.
1508          */
1509         if (pmem_present && direct_mapping && len == max_ram_len)
1510                 dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << len);
1511
1512         return direct_mapping;
1513 }
1514
1515 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
1516 {
1517         struct device_node *pdn, *dn;
1518         struct iommu_table *tbl;
1519         const __be32 *dma_window = NULL;
1520         struct pci_dn *pci;
1521
1522         pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1523
1524         /* dev setup for LPAR is a little tricky, since the device tree might
1525          * contain the dma-window properties per-device and not necessarily
1526          * for the bus. So we need to search upwards in the tree until we
1527          * either hit a dma-window property, OR find a parent with a table
1528          * already allocated.
1529          */
1530         dn = pci_device_to_OF_node(dev);
1531         pr_debug("  node is %pOF\n", dn);
1532
1533         pdn = pci_dma_find(dn, &dma_window);
1534         if (!pdn || !PCI_DN(pdn)) {
1535                 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1536                        "no DMA window found for pci dev=%s dn=%pOF\n",
1537                                  pci_name(dev), dn);
1538                 return;
1539         }
1540         pr_debug("  parent is %pOF\n", pdn);
1541
1542         pci = PCI_DN(pdn);
1543         if (!pci->table_group) {
1544                 pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
1545                 tbl = pci->table_group->tables[0];
1546                 iommu_table_setparms_lpar(pci->phb, pdn, tbl,
1547                                 pci->table_group, dma_window);
1548
1549                 iommu_init_table(tbl, pci->phb->node, 0, 0);
1550                 iommu_register_group(pci->table_group,
1551                                 pci_domain_nr(pci->phb->bus), 0);
1552                 pr_debug("  created table: %p\n", pci->table_group);
1553         } else {
1554                 pr_debug("  found DMA window, table: %p\n", pci->table_group);
1555         }
1556
1557         set_iommu_table_base(&dev->dev, pci->table_group->tables[0]);
1558         iommu_add_device(pci->table_group, &dev->dev);
1559 }
1560
1561 static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask)
1562 {
1563         struct device_node *dn = pci_device_to_OF_node(pdev), *pdn;
1564
1565         /* only attempt to use a new window if 64-bit DMA is requested */
1566         if (dma_mask < DMA_BIT_MASK(64))
1567                 return false;
1568
1569         dev_dbg(&pdev->dev, "node is %pOF\n", dn);
1570
1571         /*
1572          * the device tree might contain the dma-window properties
1573          * per-device and not necessarily for the bus. So we need to
1574          * search upwards in the tree until we either hit a dma-window
1575          * property, OR find a parent with a table already allocated.
1576          */
1577         pdn = pci_dma_find(dn, NULL);
1578         if (pdn && PCI_DN(pdn))
1579                 return enable_ddw(pdev, pdn);
1580
1581         return false;
1582 }
1583
1584 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1585                 void *data)
1586 {
1587         struct dma_win *window;
1588         struct memory_notify *arg = data;
1589         int ret = 0;
1590
1591         switch (action) {
1592         case MEM_GOING_ONLINE:
1593                 spin_lock(&dma_win_list_lock);
1594                 list_for_each_entry(window, &dma_win_list, list) {
1595                         ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1596                                         arg->nr_pages, window->prop);
1597                         /* XXX log error */
1598                 }
1599                 spin_unlock(&dma_win_list_lock);
1600                 break;
1601         case MEM_CANCEL_ONLINE:
1602         case MEM_OFFLINE:
1603                 spin_lock(&dma_win_list_lock);
1604                 list_for_each_entry(window, &dma_win_list, list) {
1605                         ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1606                                         arg->nr_pages, window->prop);
1607                         /* XXX log error */
1608                 }
1609                 spin_unlock(&dma_win_list_lock);
1610                 break;
1611         default:
1612                 break;
1613         }
1614         if (ret && action != MEM_CANCEL_ONLINE)
1615                 return NOTIFY_BAD;
1616
1617         return NOTIFY_OK;
1618 }
1619
1620 static struct notifier_block iommu_mem_nb = {
1621         .notifier_call = iommu_mem_notifier,
1622 };
1623
1624 static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data)
1625 {
1626         int err = NOTIFY_OK;
1627         struct of_reconfig_data *rd = data;
1628         struct device_node *np = rd->dn;
1629         struct pci_dn *pci = PCI_DN(np);
1630         struct dma_win *window;
1631
1632         switch (action) {
1633         case OF_RECONFIG_DETACH_NODE:
1634                 /*
1635                  * Removing the property will invoke the reconfig
1636                  * notifier again, which causes dead-lock on the
1637                  * read-write semaphore of the notifier chain. So
1638                  * we have to remove the property when releasing
1639                  * the device node.
1640                  */
1641                 if (remove_ddw(np, false, DIRECT64_PROPNAME))
1642                         remove_ddw(np, false, DMA64_PROPNAME);
1643
1644                 if (pci && pci->table_group)
1645                         iommu_pseries_free_group(pci->table_group,
1646                                         np->full_name);
1647
1648                 spin_lock(&dma_win_list_lock);
1649                 list_for_each_entry(window, &dma_win_list, list) {
1650                         if (window->device == np) {
1651                                 list_del(&window->list);
1652                                 kfree(window);
1653                                 break;
1654                         }
1655                 }
1656                 spin_unlock(&dma_win_list_lock);
1657                 break;
1658         default:
1659                 err = NOTIFY_DONE;
1660                 break;
1661         }
1662         return err;
1663 }
1664
1665 static struct notifier_block iommu_reconfig_nb = {
1666         .notifier_call = iommu_reconfig_notifier,
1667 };
1668
1669 /* These are called very early. */
1670 void __init iommu_init_early_pSeries(void)
1671 {
1672         if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1673                 return;
1674
1675         if (firmware_has_feature(FW_FEATURE_LPAR)) {
1676                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1677                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1678                 if (!disable_ddw)
1679                         pseries_pci_controller_ops.iommu_bypass_supported =
1680                                 iommu_bypass_supported_pSeriesLP;
1681         } else {
1682                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries;
1683                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries;
1684         }
1685
1686
1687         of_reconfig_notifier_register(&iommu_reconfig_nb);
1688         register_memory_notifier(&iommu_mem_nb);
1689
1690         set_pci_dma_ops(&dma_iommu_ops);
1691 }
1692
1693 static int __init disable_multitce(char *str)
1694 {
1695         if (strcmp(str, "off") == 0 &&
1696             firmware_has_feature(FW_FEATURE_LPAR) &&
1697             (firmware_has_feature(FW_FEATURE_PUT_TCE_IND) ||
1698              firmware_has_feature(FW_FEATURE_STUFF_TCE))) {
1699                 printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1700                 powerpc_firmware_features &=
1701                         ~(FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE);
1702         }
1703         return 1;
1704 }
1705
1706 __setup("multitce=", disable_multitce);
1707
1708 static int tce_iommu_bus_notifier(struct notifier_block *nb,
1709                 unsigned long action, void *data)
1710 {
1711         struct device *dev = data;
1712
1713         switch (action) {
1714         case BUS_NOTIFY_DEL_DEVICE:
1715                 iommu_del_device(dev);
1716                 return 0;
1717         default:
1718                 return 0;
1719         }
1720 }
1721
1722 static struct notifier_block tce_iommu_bus_nb = {
1723         .notifier_call = tce_iommu_bus_notifier,
1724 };
1725
1726 static int __init tce_iommu_bus_notifier_init(void)
1727 {
1728         bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1729         return 0;
1730 }
1731 machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init);