drivers/iommu/fsl_pamu_domain.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *
   4  * Copyright (C) 2013 Freescale Semiconductor, Inc.
   5  * Author: Varun Sethi <varun.sethi@freescale.com>
   6  */
   7
   8 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
   9
  10 #include "fsl_pamu_domain.h"
  11
  12 #include <sysdev/fsl_pci.h>
  13
  14 /*
  15  * Global spinlock that needs to be held while
  16  * configuring PAMU.
  17  */
  18 static DEFINE_SPINLOCK(iommu_lock);
  19
  20 static struct kmem_cache *fsl_pamu_domain_cache;
  21 static struct kmem_cache *iommu_devinfo_cache;
  22 static DEFINE_SPINLOCK(device_domain_lock);
  23
  24 struct iommu_device pamu_iommu; /* IOMMU core code handle */
  25
  26 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
  27 {
  28         return container_of(dom, struct fsl_dma_domain, iommu_domain);
  29 }
  30
  31 static int __init iommu_init_mempool(void)
  32 {
  33         fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  34                                                   sizeof(struct fsl_dma_domain),
  35                                                   0,
  36                                                   SLAB_HWCACHE_ALIGN,
  37                                                   NULL);
  38         if (!fsl_pamu_domain_cache) {
  39                 pr_debug("Couldn't create fsl iommu_domain cache\n");
  40                 return -ENOMEM;
  41         }
  42
  43         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  44                                                 sizeof(struct device_domain_info),
  45                                                 0,
  46                                                 SLAB_HWCACHE_ALIGN,
  47                                                 NULL);
  48         if (!iommu_devinfo_cache) {
  49                 pr_debug("Couldn't create devinfo cache\n");
  50                 kmem_cache_destroy(fsl_pamu_domain_cache);
  51                 return -ENOMEM;
  52         }
  53
  54         return 0;
  55 }
  56
  57 /* Map the DMA window corresponding to the LIODN */
  58 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
  59 {
  60         int ret;
  61         struct iommu_domain_geometry *geom = &dma_domain->iommu_domain.geometry;
  62         unsigned long flags;
  63
  64         spin_lock_irqsave(&iommu_lock, flags);
  65         ret = pamu_config_ppaace(liodn, geom->aperture_start,
  66                                  geom->aperture_end + 1, ~(u32)0,
  67                                  0, dma_domain->snoop_id, dma_domain->stash_id,
  68                                  PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE);
  69         spin_unlock_irqrestore(&iommu_lock, flags);
  70         if (ret)
  71                 pr_debug("PAACE configuration failed for liodn %d\n", liodn);
  72
  73         return ret;
  74 }
  75
  76 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
  77                               u32 val)
  78 {
  79         int ret = 0, i;
  80         unsigned long flags;
  81
  82         spin_lock_irqsave(&iommu_lock, flags);
  83         ret = pamu_update_paace_stash(liodn, val);
  84         if (ret) {
  85                 pr_debug("Failed to update SPAACE %d field for liodn %d\n ",
  86                          i, liodn);
  87                 spin_unlock_irqrestore(&iommu_lock, flags);
  88                 return ret;
  89         }
  90
  91         spin_unlock_irqrestore(&iommu_lock, flags);
  92
  93         return ret;
  94 }
  95
  96 /* Set the geometry parameters for a LIODN */
  97 static int pamu_set_liodn(int liodn, struct device *dev,
  98                           struct fsl_dma_domain *dma_domain,
  99                           struct iommu_domain_geometry *geom_attr)
 100 {
 101         phys_addr_t window_addr, window_size;
 102         u32 omi_index = ~(u32)0;
 103         unsigned long flags;
 104         int ret;
 105
 106         /*
 107          * Configure the omi_index at the geometry setup time.
 108          * This is a static value which depends on the type of
 109          * device and would not change thereafter.
 110          */
 111         get_ome_index(&omi_index, dev);
 112
 113         window_addr = geom_attr->aperture_start;
 114         window_size = geom_attr->aperture_end + 1;
 115
 116         spin_lock_irqsave(&iommu_lock, flags);
 117         ret = pamu_disable_liodn(liodn);
 118         if (!ret)
 119                 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
 120                                          0, dma_domain->snoop_id,
 121                                          dma_domain->stash_id, 0);
 122         spin_unlock_irqrestore(&iommu_lock, flags);
 123         if (ret) {
 124                 pr_debug("PAACE configuration failed for liodn %d\n",
 125                          liodn);
 126                 return ret;
 127         }
 128
 129         return ret;
 130 }
 131
 132 static void remove_device_ref(struct device_domain_info *info)
 133 {
 134         unsigned long flags;
 135
 136         list_del(&info->link);
 137         spin_lock_irqsave(&iommu_lock, flags);
 138         pamu_disable_liodn(info->liodn);
 139         spin_unlock_irqrestore(&iommu_lock, flags);
 140         spin_lock_irqsave(&device_domain_lock, flags);
 141         dev_iommu_priv_set(info->dev, NULL);
 142         kmem_cache_free(iommu_devinfo_cache, info);
 143         spin_unlock_irqrestore(&device_domain_lock, flags);
 144 }
 145
 146 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
 147 {
 148         struct device_domain_info *info, *tmp;
 149         unsigned long flags;
 150
 151         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 152         /* Remove the device from the domain device list */
 153         list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
 154                 if (!dev || (info->dev == dev))
 155                         remove_device_ref(info);
 156         }
 157         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 158 }
 159
 160 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
 161 {
 162         struct device_domain_info *info, *old_domain_info;
 163         unsigned long flags;
 164
 165         spin_lock_irqsave(&device_domain_lock, flags);
 166         /*
 167          * Check here if the device is already attached to domain or not.
 168          * If the device is already attached to a domain detach it.
 169          */
 170         old_domain_info = dev_iommu_priv_get(dev);
 171         if (old_domain_info && old_domain_info->domain != dma_domain) {
 172                 spin_unlock_irqrestore(&device_domain_lock, flags);
 173                 detach_device(dev, old_domain_info->domain);
 174                 spin_lock_irqsave(&device_domain_lock, flags);
 175         }
 176
 177         info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
 178
 179         info->dev = dev;
 180         info->liodn = liodn;
 181         info->domain = dma_domain;
 182
 183         list_add(&info->link, &dma_domain->devices);
 184         /*
 185          * In case of devices with multiple LIODNs just store
 186          * the info for the first LIODN as all
 187          * LIODNs share the same domain
 188          */
 189         if (!dev_iommu_priv_get(dev))
 190                 dev_iommu_priv_set(dev, info);
 191         spin_unlock_irqrestore(&device_domain_lock, flags);
 192 }
 193
 194 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
 195                                          dma_addr_t iova)
 196 {
 197         if (iova < domain->geometry.aperture_start ||
 198             iova > domain->geometry.aperture_end)
 199                 return 0;
 200         return iova;
 201 }
 202
 203 static bool fsl_pamu_capable(enum iommu_cap cap)
 204 {
 205         return cap == IOMMU_CAP_CACHE_COHERENCY;
 206 }
 207
 208 static void fsl_pamu_domain_free(struct iommu_domain *domain)
 209 {
 210         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 211
 212         /* remove all the devices from the device list */
 213         detach_device(NULL, dma_domain);
 214
 215         dma_domain->enabled = 0;
 216
 217         kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 218 }
 219
 220 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
 221 {
 222         struct fsl_dma_domain *dma_domain;
 223
 224         if (type != IOMMU_DOMAIN_UNMANAGED)
 225                 return NULL;
 226
 227         dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
 228         if (!dma_domain)
 229                 return NULL;
 230
 231         dma_domain->stash_id = ~(u32)0;
 232         dma_domain->snoop_id = ~(u32)0;
 233         INIT_LIST_HEAD(&dma_domain->devices);
 234         spin_lock_init(&dma_domain->domain_lock);
 235
 236         /* default geometry 64 GB i.e. maximum system address */
 237         dma_domain->iommu_domain. geometry.aperture_start = 0;
 238         dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
 239         dma_domain->iommu_domain.geometry.force_aperture = true;
 240
 241         return &dma_domain->iommu_domain;
 242 }
 243
 244 /* Update stash destination for all LIODNs associated with the domain */
 245 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
 246 {
 247         struct device_domain_info *info;
 248         int ret = 0;
 249
 250         list_for_each_entry(info, &dma_domain->devices, link) {
 251                 ret = update_liodn_stash(info->liodn, dma_domain, val);
 252                 if (ret)
 253                         break;
 254         }
 255
 256         return ret;
 257 }
 258
 259 /*
 260  * Attach the LIODN to the DMA domain and configure the geometry
 261  * and window mappings.
 262  */
 263 static int handle_attach_device(struct fsl_dma_domain *dma_domain,
 264                                 struct device *dev, const u32 *liodn,
 265                                 int num)
 266 {
 267         unsigned long flags;
 268         struct iommu_domain *domain = &dma_domain->iommu_domain;
 269         int ret = 0;
 270         int i;
 271
 272         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 273         for (i = 0; i < num; i++) {
 274                 /* Ensure that LIODN value is valid */
 275                 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
 276                         pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
 277                                  liodn[i], dev->of_node);
 278                         ret = -EINVAL;
 279                         break;
 280                 }
 281
 282                 attach_device(dma_domain, liodn[i], dev);
 283                 /*
 284                  * Check if geometry has already been configured
 285                  * for the domain. If yes, set the geometry for
 286                  * the LIODN.
 287                  */
 288                 ret = pamu_set_liodn(liodn[i], dev, dma_domain,
 289                                      &domain->geometry);
 290                 if (ret)
 291                         break;
 292
 293                 /*
 294                  * Create window/subwindow mapping for
 295                  * the LIODN.
 296                  */
 297                 ret = map_liodn(liodn[i], dma_domain);
 298                 if (ret)
 299                         break;
 300         }
 301         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 302
 303         return ret;
 304 }
 305
 306 static int fsl_pamu_attach_device(struct iommu_domain *domain,
 307                                   struct device *dev)
 308 {
 309         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 310         const u32 *liodn;
 311         u32 liodn_cnt;
 312         int len, ret = 0;
 313         struct pci_dev *pdev = NULL;
 314         struct pci_controller *pci_ctl;
 315
 316         /*
 317          * Use LIODN of the PCI controller while attaching a
 318          * PCI device.
 319          */
 320         if (dev_is_pci(dev)) {
 321                 pdev = to_pci_dev(dev);
 322                 pci_ctl = pci_bus_to_host(pdev->bus);
 323                 /*
 324                  * make dev point to pci controller device
 325                  * so we can get the LIODN programmed by
 326                  * u-boot.
 327                  */
 328                 dev = pci_ctl->parent;
 329         }
 330
 331         liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
 332         if (liodn) {
 333                 liodn_cnt = len / sizeof(u32);
 334                 ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt);
 335         } else {
 336                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 337                 ret = -EINVAL;
 338         }
 339
 340         return ret;
 341 }
 342
 343 static void fsl_pamu_detach_device(struct iommu_domain *domain,
 344                                    struct device *dev)
 345 {
 346         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 347         const u32 *prop;
 348         int len;
 349         struct pci_dev *pdev = NULL;
 350         struct pci_controller *pci_ctl;
 351
 352         /*
 353          * Use LIODN of the PCI controller while detaching a
 354          * PCI device.
 355          */
 356         if (dev_is_pci(dev)) {
 357                 pdev = to_pci_dev(dev);
 358                 pci_ctl = pci_bus_to_host(pdev->bus);
 359                 /*
 360                  * make dev point to pci controller device
 361                  * so we can get the LIODN programmed by
 362                  * u-boot.
 363                  */
 364                 dev = pci_ctl->parent;
 365         }
 366
 367         prop = of_get_property(dev->of_node, "fsl,liodn", &len);
 368         if (prop)
 369                 detach_device(dev, dma_domain);
 370         else
 371                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 372 }
 373
 374 /* Set the domain stash attribute */
 375 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
 376 {
 377         struct pamu_stash_attribute *stash_attr = data;
 378         unsigned long flags;
 379         int ret;
 380
 381         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 382
 383         memcpy(&dma_domain->dma_stash, stash_attr,
 384                sizeof(struct pamu_stash_attribute));
 385
 386         dma_domain->stash_id = get_stash_id(stash_attr->cache,
 387                                             stash_attr->cpu);
 388         if (dma_domain->stash_id == ~(u32)0) {
 389                 pr_debug("Invalid stash attributes\n");
 390                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 391                 return -EINVAL;
 392         }
 393
 394         ret = update_domain_stash(dma_domain, dma_domain->stash_id);
 395
 396         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 397
 398         return ret;
 399 }
 400
 401 /* Configure domain dma state i.e. enable/disable DMA */
 402 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
 403 {
 404         struct device_domain_info *info;
 405         unsigned long flags;
 406         int ret;
 407
 408         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 409         dma_domain->enabled = enable;
 410         list_for_each_entry(info, &dma_domain->devices, link) {
 411                 ret = (enable) ? pamu_enable_liodn(info->liodn) :
 412                         pamu_disable_liodn(info->liodn);
 413                 if (ret)
 414                         pr_debug("Unable to set dma state for liodn %d",
 415                                  info->liodn);
 416         }
 417         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 418
 419         return 0;
 420 }
 421
 422 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
 423                                     enum iommu_attr attr_type, void *data)
 424 {
 425         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 426         int ret = 0;
 427
 428         switch (attr_type) {
 429         case DOMAIN_ATTR_FSL_PAMU_STASH:
 430                 ret = configure_domain_stash(dma_domain, data);
 431                 break;
 432         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 433                 ret = configure_domain_dma_state(dma_domain, *(int *)data);
 434                 break;
 435         default:
 436                 pr_debug("Unsupported attribute type\n");
 437                 ret = -EINVAL;
 438                 break;
 439         }
 440
 441         return ret;
 442 }
 443
 444 static struct iommu_group *get_device_iommu_group(struct device *dev)
 445 {
 446         struct iommu_group *group;
 447
 448         group = iommu_group_get(dev);
 449         if (!group)
 450                 group = iommu_group_alloc();
 451
 452         return group;
 453 }
 454
 455 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
 456 {
 457         u32 version;
 458
 459         /* Check the PCI controller version number by readding BRR1 register */
 460         version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
 461         version &= PCI_FSL_BRR1_VER;
 462         /* If PCI controller version is >= 0x204 we can partition endpoints */
 463         return version >= 0x204;
 464 }
 465
 466 /* Get iommu group information from peer devices or devices on the parent bus */
 467 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
 468 {
 469         struct pci_dev *tmp;
 470         struct iommu_group *group;
 471         struct pci_bus *bus = pdev->bus;
 472
 473         /*
 474          * Traverese the pci bus device list to get
 475          * the shared iommu group.
 476          */
 477         while (bus) {
 478                 list_for_each_entry(tmp, &bus->devices, bus_list) {
 479                         if (tmp == pdev)
 480                                 continue;
 481                         group = iommu_group_get(&tmp->dev);
 482                         if (group)
 483                                 return group;
 484                 }
 485
 486                 bus = bus->parent;
 487         }
 488
 489         return NULL;
 490 }
 491
 492 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
 493 {
 494         struct pci_controller *pci_ctl;
 495         bool pci_endpt_partitioning;
 496         struct iommu_group *group = NULL;
 497
 498         pci_ctl = pci_bus_to_host(pdev->bus);
 499         pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl);
 500         /* We can partition PCIe devices so assign device group to the device */
 501         if (pci_endpt_partitioning) {
 502                 group = pci_device_group(&pdev->dev);
 503
 504                 /*
 505                  * PCIe controller is not a paritionable entity
 506                  * free the controller device iommu_group.
 507                  */
 508                 if (pci_ctl->parent->iommu_group)
 509                         iommu_group_remove_device(pci_ctl->parent);
 510         } else {
 511                 /*
 512                  * All devices connected to the controller will share the
 513                  * PCI controllers device group. If this is the first
 514                  * device to be probed for the pci controller, copy the
 515                  * device group information from the PCI controller device
 516                  * node and remove the PCI controller iommu group.
 517                  * For subsequent devices, the iommu group information can
 518                  * be obtained from sibling devices (i.e. from the bus_devices
 519                  * link list).
 520                  */
 521                 if (pci_ctl->parent->iommu_group) {
 522                         group = get_device_iommu_group(pci_ctl->parent);
 523                         iommu_group_remove_device(pci_ctl->parent);
 524                 } else {
 525                         group = get_shared_pci_device_group(pdev);
 526                 }
 527         }
 528
 529         if (!group)
 530                 group = ERR_PTR(-ENODEV);
 531
 532         return group;
 533 }
 534
 535 static struct iommu_group *fsl_pamu_device_group(struct device *dev)
 536 {
 537         struct iommu_group *group = ERR_PTR(-ENODEV);
 538         int len;
 539
 540         /*
 541          * For platform devices we allocate a separate group for
 542          * each of the devices.
 543          */
 544         if (dev_is_pci(dev))
 545                 group = get_pci_device_group(to_pci_dev(dev));
 546         else if (of_get_property(dev->of_node, "fsl,liodn", &len))
 547                 group = get_device_iommu_group(dev);
 548
 549         return group;
 550 }
 551
 552 static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
 553 {
 554         return &pamu_iommu;
 555 }
 556
 557 static void fsl_pamu_release_device(struct device *dev)
 558 {
 559 }
 560
 561 static const struct iommu_ops fsl_pamu_ops = {
 562         .capable        = fsl_pamu_capable,
 563         .domain_alloc   = fsl_pamu_domain_alloc,
 564         .domain_free    = fsl_pamu_domain_free,
 565         .attach_dev     = fsl_pamu_attach_device,
 566         .detach_dev     = fsl_pamu_detach_device,
 567         .iova_to_phys   = fsl_pamu_iova_to_phys,
 568         .domain_set_attr = fsl_pamu_set_domain_attr,
 569         .probe_device   = fsl_pamu_probe_device,
 570         .release_device = fsl_pamu_release_device,
 571         .device_group   = fsl_pamu_device_group,
 572 };
 573
 574 int __init pamu_domain_init(void)
 575 {
 576         int ret = 0;
 577
 578         ret = iommu_init_mempool();
 579         if (ret)
 580                 return ret;
 581
 582         ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
 583         if (ret)
 584                 return ret;
 585
 586         iommu_device_set_ops(&pamu_iommu, &fsl_pamu_ops);
 587
 588         ret = iommu_device_register(&pamu_iommu);
 589         if (ret) {
 590                 iommu_device_sysfs_remove(&pamu_iommu);
 591                 pr_err("Can't register iommu device\n");
 592                 return ret;
 593         }
 594
 595         bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
 596         bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
 597
 598         return ret;
 599 }