drivers/iommu/apple-dart.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Apple DART (Device Address Resolution Table) IOMMU driver
   4  *
   5  * Copyright (C) 2021 The Asahi Linux Contributors
   6  *
   7  * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
   8  *  Copyright (C) 2013 ARM Limited
   9  *  Copyright (C) 2015 ARM Limited
  10  * and on exynos-iommu.c
  11  *  Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
  12  */
  13
  14 #include <linux/atomic.h>
  15 #include <linux/bitfield.h>
  16 #include <linux/clk.h>
  17 #include <linux/dev_printk.h>
  18 #include <linux/dma-iommu.h>
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/io-pgtable.h>
  23 #include <linux/iommu.h>
  24 #include <linux/iopoll.h>
  25 #include <linux/module.h>
  26 #include <linux/of.h>
  27 #include <linux/of_address.h>
  28 #include <linux/of_iommu.h>
  29 #include <linux/of_platform.h>
  30 #include <linux/pci.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/slab.h>
  33 #include <linux/swab.h>
  34 #include <linux/types.h>
  35
  36 #define DART_MAX_STREAMS 16
  37 #define DART_MAX_TTBR 4
  38 #define MAX_DARTS_PER_DEVICE 2
  39
  40 #define DART_STREAM_ALL 0xffff
  41
  42 #define DART_PARAMS1 0x00
  43 #define DART_PARAMS_PAGE_SHIFT GENMASK(27, 24)
  44
  45 #define DART_PARAMS2 0x04
  46 #define DART_PARAMS_BYPASS_SUPPORT BIT(0)
  47
  48 #define DART_STREAM_COMMAND 0x20
  49 #define DART_STREAM_COMMAND_BUSY BIT(2)
  50 #define DART_STREAM_COMMAND_INVALIDATE BIT(20)
  51
  52 #define DART_STREAM_SELECT 0x34
  53
  54 #define DART_ERROR 0x40
  55 #define DART_ERROR_STREAM GENMASK(27, 24)
  56 #define DART_ERROR_CODE GENMASK(11, 0)
  57 #define DART_ERROR_FLAG BIT(31)
  58
  59 #define DART_ERROR_READ_FAULT BIT(4)
  60 #define DART_ERROR_WRITE_FAULT BIT(3)
  61 #define DART_ERROR_NO_PTE BIT(2)
  62 #define DART_ERROR_NO_PMD BIT(1)
  63 #define DART_ERROR_NO_TTBR BIT(0)
  64
  65 #define DART_CONFIG 0x60
  66 #define DART_CONFIG_LOCK BIT(15)
  67
  68 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
  69
  70 #define DART_ERROR_ADDR_HI 0x54
  71 #define DART_ERROR_ADDR_LO 0x50
  72
  73 #define DART_TCR(sid) (0x100 + 4 * (sid))
  74 #define DART_TCR_TRANSLATE_ENABLE BIT(7)
  75 #define DART_TCR_BYPASS0_ENABLE BIT(8)
  76 #define DART_TCR_BYPASS1_ENABLE BIT(12)
  77
  78 #define DART_TTBR(sid, idx) (0x200 + 16 * (sid) + 4 * (idx))
  79 #define DART_TTBR_VALID BIT(31)
  80 #define DART_TTBR_SHIFT 12
  81
  82 /*
  83  * Private structure associated with each DART device.
  84  *
  85  * @dev: device struct
  86  * @regs: mapped MMIO region
  87  * @irq: interrupt number, can be shared with other DARTs
  88  * @clks: clocks associated with this DART
  89  * @num_clks: number of @clks
  90  * @lock: lock for hardware operations involving this dart
  91  * @pgsize: pagesize supported by this DART
  92  * @supports_bypass: indicates if this DART supports bypass mode
  93  * @force_bypass: force bypass mode due to pagesize mismatch?
  94  * @sid2group: maps stream ids to iommu_groups
  95  * @iommu: iommu core device
  96  */
  97 struct apple_dart {
  98         struct device *dev;
  99
 100         void __iomem *regs;
 101
 102         int irq;
 103         struct clk_bulk_data *clks;
 104         int num_clks;
 105
 106         spinlock_t lock;
 107
 108         u32 pgsize;
 109         u32 supports_bypass : 1;
 110         u32 force_bypass : 1;
 111
 112         struct iommu_group *sid2group[DART_MAX_STREAMS];
 113         struct iommu_device iommu;
 114 };
 115
 116 /*
 117  * Convenience struct to identify streams.
 118  *
 119  * The normal variant is used inside apple_dart_master_cfg which isn't written
 120  * to concurrently.
 121  * The atomic variant is used inside apple_dart_domain where we have to guard
 122  * against races from potential parallel calls to attach/detach_device.
 123  * Note that even inside the atomic variant the apple_dart pointer is not
 124  * protected: This pointer is initialized once under the domain init mutex
 125  * and never changed again afterwards. Devices with different dart pointers
 126  * cannot be attached to the same domain.
 127  *
 128  * @dart dart pointer
 129  * @sid stream id bitmap
 130  */
 131 struct apple_dart_stream_map {
 132         struct apple_dart *dart;
 133         unsigned long sidmap;
 134 };
 135 struct apple_dart_atomic_stream_map {
 136         struct apple_dart *dart;
 137         atomic64_t sidmap;
 138 };
 139
 140 /*
 141  * This structure is attached to each iommu domain handled by a DART.
 142  *
 143  * @pgtbl_ops: pagetable ops allocated by io-pgtable
 144  * @finalized: true if the domain has been completely initialized
 145  * @init_lock: protects domain initialization
 146  * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
 147  * @domain: core iommu domain pointer
 148  */
 149 struct apple_dart_domain {
 150         struct io_pgtable_ops *pgtbl_ops;
 151
 152         bool finalized;
 153         struct mutex init_lock;
 154         struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 155
 156         struct iommu_domain domain;
 157 };
 158
 159 /*
 160  * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
 161  * and contains a list of streams bound to this device.
 162  * So far the worst case seen is a single device with two streams
 163  * from different darts, such that this simple static array is enough.
 164  *
 165  * @streams: streams for this device
 166  */
 167 struct apple_dart_master_cfg {
 168         struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 169 };
 170
 171 /*
 172  * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
 173  * apple_dart_domain.stream_maps
 174  *
 175  * @i int used as loop variable
 176  * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
 177  * @stream pointer to the apple_dart_streams struct for each loop iteration
 178  */
 179 #define for_each_stream_map(i, base, stream_map)                               \
 180         for (i = 0, stream_map = &(base)->stream_maps[0];                      \
 181              i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
 182              stream_map = &(base)->stream_maps[++i])
 183
 184 static struct platform_driver apple_dart_driver;
 185 static const struct iommu_ops apple_dart_iommu_ops;
 186 static const struct iommu_flush_ops apple_dart_tlb_ops;
 187
 188 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
 189 {
 190         return container_of(dom, struct apple_dart_domain, domain);
 191 }
 192
 193 static void
 194 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)
 195 {
 196         int sid;
 197
 198         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 199                 writel(DART_TCR_TRANSLATE_ENABLE,
 200                        stream_map->dart->regs + DART_TCR(sid));
 201 }
 202
 203 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
 204 {
 205         int sid;
 206
 207         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 208                 writel(0, stream_map->dart->regs + DART_TCR(sid));
 209 }
 210
 211 static void
 212 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
 213 {
 214         int sid;
 215
 216         WARN_ON(!stream_map->dart->supports_bypass);
 217         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 218                 writel(DART_TCR_BYPASS0_ENABLE | DART_TCR_BYPASS1_ENABLE,
 219                        stream_map->dart->regs + DART_TCR(sid));
 220 }
 221
 222 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
 223                                    u8 idx, phys_addr_t paddr)
 224 {
 225         int sid;
 226
 227         WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1));
 228         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 229                 writel(DART_TTBR_VALID | (paddr >> DART_TTBR_SHIFT),
 230                        stream_map->dart->regs + DART_TTBR(sid, idx));
 231 }
 232
 233 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
 234                                      u8 idx)
 235 {
 236         int sid;
 237
 238         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 239                 writel(0, stream_map->dart->regs + DART_TTBR(sid, idx));
 240 }
 241
 242 static void
 243 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
 244 {
 245         int i;
 246
 247         for (i = 0; i < DART_MAX_TTBR; ++i)
 248                 apple_dart_hw_clear_ttbr(stream_map, i);
 249 }
 250
 251 static int
 252 apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map,
 253                              u32 command)
 254 {
 255         unsigned long flags;
 256         int ret;
 257         u32 command_reg;
 258
 259         spin_lock_irqsave(&stream_map->dart->lock, flags);
 260
 261         writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT);
 262         writel(command, stream_map->dart->regs + DART_STREAM_COMMAND);
 263
 264         ret = readl_poll_timeout_atomic(
 265                 stream_map->dart->regs + DART_STREAM_COMMAND, command_reg,
 266                 !(command_reg & DART_STREAM_COMMAND_BUSY), 1,
 267                 DART_STREAM_COMMAND_BUSY_TIMEOUT);
 268
 269         spin_unlock_irqrestore(&stream_map->dart->lock, flags);
 270
 271         if (ret) {
 272                 dev_err(stream_map->dart->dev,
 273                         "busy bit did not clear after command %x for streams %lx\n",
 274                         command, stream_map->sidmap);
 275                 return ret;
 276         }
 277
 278         return 0;
 279 }
 280
 281 static int
 282 apple_dart_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
 283 {
 284         return apple_dart_hw_stream_command(stream_map,
 285                                             DART_STREAM_COMMAND_INVALIDATE);
 286 }
 287
 288 static int apple_dart_hw_reset(struct apple_dart *dart)
 289 {
 290         u32 config;
 291         struct apple_dart_stream_map stream_map;
 292
 293         config = readl(dart->regs + DART_CONFIG);
 294         if (config & DART_CONFIG_LOCK) {
 295                 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
 296                         config);
 297                 return -EINVAL;
 298         }
 299
 300         stream_map.dart = dart;
 301         stream_map.sidmap = DART_STREAM_ALL;
 302         apple_dart_hw_disable_dma(&stream_map);
 303         apple_dart_hw_clear_all_ttbrs(&stream_map);
 304
 305         /* clear any pending errors before the interrupt is unmasked */
 306         writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR);
 307
 308         return apple_dart_hw_invalidate_tlb(&stream_map);
 309 }
 310
 311 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
 312 {
 313         int i;
 314         struct apple_dart_atomic_stream_map *domain_stream_map;
 315         struct apple_dart_stream_map stream_map;
 316
 317         for_each_stream_map(i, domain, domain_stream_map) {
 318                 stream_map.dart = domain_stream_map->dart;
 319                 stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap);
 320                 apple_dart_hw_invalidate_tlb(&stream_map);
 321         }
 322 }
 323
 324 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
 325 {
 326         apple_dart_domain_flush_tlb(to_dart_domain(domain));
 327 }
 328
 329 static void apple_dart_iotlb_sync(struct iommu_domain *domain,
 330                                   struct iommu_iotlb_gather *gather)
 331 {
 332         apple_dart_domain_flush_tlb(to_dart_domain(domain));
 333 }
 334
 335 static void apple_dart_iotlb_sync_map(struct iommu_domain *domain,
 336                                       unsigned long iova, size_t size)
 337 {
 338         apple_dart_domain_flush_tlb(to_dart_domain(domain));
 339 }
 340
 341 static void apple_dart_tlb_flush_all(void *cookie)
 342 {
 343         apple_dart_domain_flush_tlb(cookie);
 344 }
 345
 346 static void apple_dart_tlb_flush_walk(unsigned long iova, size_t size,
 347                                       size_t granule, void *cookie)
 348 {
 349         apple_dart_domain_flush_tlb(cookie);
 350 }
 351
 352 static const struct iommu_flush_ops apple_dart_tlb_ops = {
 353         .tlb_flush_all = apple_dart_tlb_flush_all,
 354         .tlb_flush_walk = apple_dart_tlb_flush_walk,
 355 };
 356
 357 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
 358                                            dma_addr_t iova)
 359 {
 360         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 361         struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 362
 363         if (!ops)
 364                 return 0;
 365
 366         return ops->iova_to_phys(ops, iova);
 367 }
 368
 369 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
 370                                 phys_addr_t paddr, size_t pgsize,
 371                                 size_t pgcount, int prot, gfp_t gfp,
 372                                 size_t *mapped)
 373 {
 374         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 375         struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 376
 377         if (!ops)
 378                 return -ENODEV;
 379
 380         return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
 381                               mapped);
 382 }
 383
 384 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
 385                                      unsigned long iova, size_t pgsize,
 386                                      size_t pgcount,
 387                                      struct iommu_iotlb_gather *gather)
 388 {
 389         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 390         struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 391
 392         return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
 393 }
 394
 395 static void
 396 apple_dart_setup_translation(struct apple_dart_domain *domain,
 397                              struct apple_dart_stream_map *stream_map)
 398 {
 399         int i;
 400         struct io_pgtable_cfg *pgtbl_cfg =
 401                 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
 402
 403         for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
 404                 apple_dart_hw_set_ttbr(stream_map, i,
 405                                        pgtbl_cfg->apple_dart_cfg.ttbr[i]);
 406         for (; i < DART_MAX_TTBR; ++i)
 407                 apple_dart_hw_clear_ttbr(stream_map, i);
 408
 409         apple_dart_hw_enable_translation(stream_map);
 410         apple_dart_hw_invalidate_tlb(stream_map);
 411 }
 412
 413 static int apple_dart_finalize_domain(struct iommu_domain *domain,
 414                                       struct apple_dart_master_cfg *cfg)
 415 {
 416         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 417         struct apple_dart *dart = cfg->stream_maps[0].dart;
 418         struct io_pgtable_cfg pgtbl_cfg;
 419         int ret = 0;
 420         int i;
 421
 422         mutex_lock(&dart_domain->init_lock);
 423
 424         if (dart_domain->finalized)
 425                 goto done;
 426
 427         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 428                 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
 429                 atomic64_set(&dart_domain->stream_maps[i].sidmap,
 430                              cfg->stream_maps[i].sidmap);
 431         }
 432
 433         pgtbl_cfg = (struct io_pgtable_cfg){
 434                 .pgsize_bitmap = dart->pgsize,
 435                 .ias = 32,
 436                 .oas = 36,
 437                 .coherent_walk = 1,
 438                 .tlb = &apple_dart_tlb_ops,
 439                 .iommu_dev = dart->dev,
 440         };
 441
 442         dart_domain->pgtbl_ops =
 443                 alloc_io_pgtable_ops(APPLE_DART, &pgtbl_cfg, domain);
 444         if (!dart_domain->pgtbl_ops) {
 445                 ret = -ENOMEM;
 446                 goto done;
 447         }
 448
 449         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 450         domain->geometry.aperture_start = 0;
 451         domain->geometry.aperture_end = DMA_BIT_MASK(32);
 452         domain->geometry.force_aperture = true;
 453
 454         dart_domain->finalized = true;
 455
 456 done:
 457         mutex_unlock(&dart_domain->init_lock);
 458         return ret;
 459 }
 460
 461 static int
 462 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
 463                        struct apple_dart_stream_map *master_maps,
 464                        bool add_streams)
 465 {
 466         int i;
 467
 468         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 469                 if (domain_maps[i].dart != master_maps[i].dart)
 470                         return -EINVAL;
 471         }
 472
 473         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 474                 if (!domain_maps[i].dart)
 475                         break;
 476                 if (add_streams)
 477                         atomic64_or(master_maps[i].sidmap,
 478                                     &domain_maps[i].sidmap);
 479                 else
 480                         atomic64_and(~master_maps[i].sidmap,
 481                                      &domain_maps[i].sidmap);
 482         }
 483
 484         return 0;
 485 }
 486
 487 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
 488                                          struct apple_dart_master_cfg *cfg)
 489 {
 490         return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
 491                                       true);
 492 }
 493
 494 static int apple_dart_domain_remove_streams(struct apple_dart_domain *domain,
 495                                             struct apple_dart_master_cfg *cfg)
 496 {
 497         return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
 498                                       false);
 499 }
 500
 501 static int apple_dart_attach_dev(struct iommu_domain *domain,
 502                                  struct device *dev)
 503 {
 504         int ret, i;
 505         struct apple_dart_stream_map *stream_map;
 506         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 507         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 508
 509         if (cfg->stream_maps[0].dart->force_bypass &&
 510             domain->type != IOMMU_DOMAIN_IDENTITY)
 511                 return -EINVAL;
 512         if (!cfg->stream_maps[0].dart->supports_bypass &&
 513             domain->type == IOMMU_DOMAIN_IDENTITY)
 514                 return -EINVAL;
 515
 516         ret = apple_dart_finalize_domain(domain, cfg);
 517         if (ret)
 518                 return ret;
 519
 520         switch (domain->type) {
 521         case IOMMU_DOMAIN_DMA:
 522         case IOMMU_DOMAIN_UNMANAGED:
 523                 ret = apple_dart_domain_add_streams(dart_domain, cfg);
 524                 if (ret)
 525                         return ret;
 526
 527                 for_each_stream_map(i, cfg, stream_map)
 528                         apple_dart_setup_translation(dart_domain, stream_map);
 529                 break;
 530         case IOMMU_DOMAIN_BLOCKED:
 531                 for_each_stream_map(i, cfg, stream_map)
 532                         apple_dart_hw_disable_dma(stream_map);
 533                 break;
 534         case IOMMU_DOMAIN_IDENTITY:
 535                 for_each_stream_map(i, cfg, stream_map)
 536                         apple_dart_hw_enable_bypass(stream_map);
 537                 break;
 538         }
 539
 540         return ret;
 541 }
 542
 543 static void apple_dart_detach_dev(struct iommu_domain *domain,
 544                                   struct device *dev)
 545 {
 546         int i;
 547         struct apple_dart_stream_map *stream_map;
 548         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 549         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 550
 551         for_each_stream_map(i, cfg, stream_map)
 552                 apple_dart_hw_disable_dma(stream_map);
 553
 554         if (domain->type == IOMMU_DOMAIN_DMA ||
 555             domain->type == IOMMU_DOMAIN_UNMANAGED)
 556                 apple_dart_domain_remove_streams(dart_domain, cfg);
 557 }
 558
 559 static struct iommu_device *apple_dart_probe_device(struct device *dev)
 560 {
 561         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 562         struct apple_dart_stream_map *stream_map;
 563         int i;
 564
 565         if (!cfg)
 566                 return ERR_PTR(-ENODEV);
 567
 568         for_each_stream_map(i, cfg, stream_map)
 569                 device_link_add(
 570                         dev, stream_map->dart->dev,
 571                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
 572
 573         return &cfg->stream_maps[0].dart->iommu;
 574 }
 575
 576 static void apple_dart_release_device(struct device *dev)
 577 {
 578         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 579
 580         if (!cfg)
 581                 return;
 582
 583         dev_iommu_priv_set(dev, NULL);
 584         kfree(cfg);
 585 }
 586
 587 static struct iommu_domain *apple_dart_domain_alloc(unsigned int type)
 588 {
 589         struct apple_dart_domain *dart_domain;
 590
 591         if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED &&
 592             type != IOMMU_DOMAIN_IDENTITY && type != IOMMU_DOMAIN_BLOCKED)
 593                 return NULL;
 594
 595         dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
 596         if (!dart_domain)
 597                 return NULL;
 598
 599         iommu_get_dma_cookie(&dart_domain->domain);
 600         mutex_init(&dart_domain->init_lock);
 601
 602         /* no need to allocate pgtbl_ops or do any other finalization steps */
 603         if (type == IOMMU_DOMAIN_IDENTITY || type == IOMMU_DOMAIN_BLOCKED)
 604                 dart_domain->finalized = true;
 605
 606         return &dart_domain->domain;
 607 }
 608
 609 static void apple_dart_domain_free(struct iommu_domain *domain)
 610 {
 611         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 612
 613         if (dart_domain->pgtbl_ops)
 614                 free_io_pgtable_ops(dart_domain->pgtbl_ops);
 615
 616         kfree(dart_domain);
 617 }
 618
 619 static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args)
 620 {
 621         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 622         struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
 623         struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
 624         struct apple_dart *cfg_dart;
 625         int i, sid;
 626
 627         if (args->args_count != 1)
 628                 return -EINVAL;
 629         sid = args->args[0];
 630
 631         if (!cfg)
 632                 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
 633         if (!cfg)
 634                 return -ENOMEM;
 635         dev_iommu_priv_set(dev, cfg);
 636
 637         cfg_dart = cfg->stream_maps[0].dart;
 638         if (cfg_dart) {
 639                 if (cfg_dart->supports_bypass != dart->supports_bypass)
 640                         return -EINVAL;
 641                 if (cfg_dart->force_bypass != dart->force_bypass)
 642                         return -EINVAL;
 643                 if (cfg_dart->pgsize != dart->pgsize)
 644                         return -EINVAL;
 645         }
 646
 647         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 648                 if (cfg->stream_maps[i].dart == dart) {
 649                         cfg->stream_maps[i].sidmap |= 1 << sid;
 650                         return 0;
 651                 }
 652         }
 653         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 654                 if (!cfg->stream_maps[i].dart) {
 655                         cfg->stream_maps[i].dart = dart;
 656                         cfg->stream_maps[i].sidmap = 1 << sid;
 657                         return 0;
 658                 }
 659         }
 660
 661         return -EINVAL;
 662 }
 663
 664 static struct iommu_group *apple_dart_device_group(struct device *dev)
 665 {
 666         static DEFINE_MUTEX(lock);
 667         int i, sid;
 668         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 669         struct apple_dart_stream_map *stream_map;
 670         struct iommu_group *group = NULL;
 671         struct iommu_group *res = ERR_PTR(-EINVAL);
 672
 673         mutex_lock(&lock);
 674
 675         for_each_stream_map(i, cfg, stream_map) {
 676                 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) {
 677                         struct iommu_group *stream_group =
 678                                 stream_map->dart->sid2group[sid];
 679
 680                         if (group && group != stream_group) {
 681                                 res = ERR_PTR(-EINVAL);
 682                                 goto out;
 683                         }
 684
 685                         group = stream_group;
 686                 }
 687         }
 688
 689         if (group) {
 690                 res = iommu_group_ref_get(group);
 691                 goto out;
 692         }
 693
 694 #ifdef CONFIG_PCI
 695         if (dev_is_pci(dev))
 696                 group = pci_device_group(dev);
 697         else
 698 #endif
 699                 group = generic_device_group(dev);
 700
 701         for_each_stream_map(i, cfg, stream_map)
 702                 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 703                         stream_map->dart->sid2group[sid] = group;
 704
 705         res = group;
 706
 707 out:
 708         mutex_unlock(&lock);
 709         return res;
 710 }
 711
 712 static int apple_dart_def_domain_type(struct device *dev)
 713 {
 714         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 715
 716         if (cfg->stream_maps[0].dart->force_bypass)
 717                 return IOMMU_DOMAIN_IDENTITY;
 718         if (!cfg->stream_maps[0].dart->supports_bypass)
 719                 return IOMMU_DOMAIN_DMA;
 720
 721         return 0;
 722 }
 723
 724 static const struct iommu_ops apple_dart_iommu_ops = {
 725         .domain_alloc = apple_dart_domain_alloc,
 726         .domain_free = apple_dart_domain_free,
 727         .attach_dev = apple_dart_attach_dev,
 728         .detach_dev = apple_dart_detach_dev,
 729         .map_pages = apple_dart_map_pages,
 730         .unmap_pages = apple_dart_unmap_pages,
 731         .flush_iotlb_all = apple_dart_flush_iotlb_all,
 732         .iotlb_sync = apple_dart_iotlb_sync,
 733         .iotlb_sync_map = apple_dart_iotlb_sync_map,
 734         .iova_to_phys = apple_dart_iova_to_phys,
 735         .probe_device = apple_dart_probe_device,
 736         .release_device = apple_dart_release_device,
 737         .device_group = apple_dart_device_group,
 738         .of_xlate = apple_dart_of_xlate,
 739         .def_domain_type = apple_dart_def_domain_type,
 740         .pgsize_bitmap = -1UL, /* Restricted during dart probe */
 741 };
 742
 743 static irqreturn_t apple_dart_irq(int irq, void *dev)
 744 {
 745         struct apple_dart *dart = dev;
 746         const char *fault_name = NULL;
 747         u32 error = readl(dart->regs + DART_ERROR);
 748         u32 error_code = FIELD_GET(DART_ERROR_CODE, error);
 749         u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO);
 750         u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI);
 751         u64 addr = addr_lo | (((u64)addr_hi) << 32);
 752         u8 stream_idx = FIELD_GET(DART_ERROR_STREAM, error);
 753
 754         if (!(error & DART_ERROR_FLAG))
 755                 return IRQ_NONE;
 756
 757         /* there should only be a single bit set but let's use == to be sure */
 758         if (error_code == DART_ERROR_READ_FAULT)
 759                 fault_name = "READ FAULT";
 760         else if (error_code == DART_ERROR_WRITE_FAULT)
 761                 fault_name = "WRITE FAULT";
 762         else if (error_code == DART_ERROR_NO_PTE)
 763                 fault_name = "NO PTE FOR IOVA";
 764         else if (error_code == DART_ERROR_NO_PMD)
 765                 fault_name = "NO PMD FOR IOVA";
 766         else if (error_code == DART_ERROR_NO_TTBR)
 767                 fault_name = "NO TTBR FOR IOVA";
 768         else
 769                 fault_name = "unknown";
 770
 771         dev_err_ratelimited(
 772                 dart->dev,
 773                 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
 774                 error, stream_idx, error_code, fault_name, addr);
 775
 776         writel(error, dart->regs + DART_ERROR);
 777         return IRQ_HANDLED;
 778 }
 779
 780 static int apple_dart_set_bus_ops(const struct iommu_ops *ops)
 781 {
 782         int ret;
 783
 784         if (!iommu_present(&platform_bus_type)) {
 785                 ret = bus_set_iommu(&platform_bus_type, ops);
 786                 if (ret)
 787                         return ret;
 788         }
 789 #ifdef CONFIG_PCI
 790         if (!iommu_present(&pci_bus_type)) {
 791                 ret = bus_set_iommu(&pci_bus_type, ops);
 792                 if (ret) {
 793                         bus_set_iommu(&platform_bus_type, NULL);
 794                         return ret;
 795                 }
 796         }
 797 #endif
 798         return 0;
 799 }
 800
 801 static int apple_dart_probe(struct platform_device *pdev)
 802 {
 803         int ret;
 804         u32 dart_params[2];
 805         struct resource *res;
 806         struct apple_dart *dart;
 807         struct device *dev = &pdev->dev;
 808
 809         dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
 810         if (!dart)
 811                 return -ENOMEM;
 812
 813         dart->dev = dev;
 814         spin_lock_init(&dart->lock);
 815
 816         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 817         if (resource_size(res) < 0x4000) {
 818                 dev_err(dev, "MMIO region too small (%pr)\n", res);
 819                 return -EINVAL;
 820         }
 821
 822         dart->regs = devm_ioremap_resource(dev, res);
 823         if (IS_ERR(dart->regs))
 824                 return PTR_ERR(dart->regs);
 825
 826         dart->irq = platform_get_irq(pdev, 0);
 827         if (dart->irq < 0)
 828                 return -ENODEV;
 829
 830         ret = devm_clk_bulk_get_all(dev, &dart->clks);
 831         if (ret < 0)
 832                 return ret;
 833         dart->num_clks = ret;
 834
 835         ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
 836         if (ret)
 837                 return ret;
 838
 839         ret = apple_dart_hw_reset(dart);
 840         if (ret)
 841                 goto err_clk_disable;
 842
 843         dart_params[0] = readl(dart->regs + DART_PARAMS1);
 844         dart_params[1] = readl(dart->regs + DART_PARAMS2);
 845         dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]);
 846         dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT;
 847         dart->force_bypass = dart->pgsize > PAGE_SIZE;
 848
 849         ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED,
 850                           "apple-dart fault handler", dart);
 851         if (ret)
 852                 goto err_clk_disable;
 853
 854         platform_set_drvdata(pdev, dart);
 855
 856         ret = apple_dart_set_bus_ops(&apple_dart_iommu_ops);
 857         if (ret)
 858                 goto err_free_irq;
 859
 860         ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
 861                                      dev_name(&pdev->dev));
 862         if (ret)
 863                 goto err_remove_bus_ops;
 864
 865         ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
 866         if (ret)
 867                 goto err_sysfs_remove;
 868
 869         dev_info(
 870                 &pdev->dev,
 871                 "DART [pagesize %x, bypass support: %d, bypass forced: %d] initialized\n",
 872                 dart->pgsize, dart->supports_bypass, dart->force_bypass);
 873         return 0;
 874
 875 err_sysfs_remove:
 876         iommu_device_sysfs_remove(&dart->iommu);
 877 err_remove_bus_ops:
 878         apple_dart_set_bus_ops(NULL);
 879 err_free_irq:
 880         free_irq(dart->irq, dart);
 881 err_clk_disable:
 882         clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
 883
 884         return ret;
 885 }
 886
 887 static int apple_dart_remove(struct platform_device *pdev)
 888 {
 889         struct apple_dart *dart = platform_get_drvdata(pdev);
 890
 891         apple_dart_hw_reset(dart);
 892         free_irq(dart->irq, dart);
 893         apple_dart_set_bus_ops(NULL);
 894
 895         iommu_device_unregister(&dart->iommu);
 896         iommu_device_sysfs_remove(&dart->iommu);
 897
 898         clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
 899
 900         return 0;
 901 }
 902
 903 static const struct of_device_id apple_dart_of_match[] = {
 904         { .compatible = "apple,t8103-dart", .data = NULL },
 905         {},
 906 };
 907 MODULE_DEVICE_TABLE(of, apple_dart_of_match);
 908
 909 static struct platform_driver apple_dart_driver = {
 910         .driver = {
 911                 .name                   = "apple-dart",
 912                 .of_match_table         = apple_dart_of_match,
 913                 .suppress_bind_attrs    = true,
 914         },
 915         .probe  = apple_dart_probe,
 916         .remove = apple_dart_remove,
 917 };
 918
 919 module_platform_driver(apple_dart_driver);
 920
 921 MODULE_DESCRIPTION("IOMMU API for Apple's DART");
 922 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
 923 MODULE_LICENSE("GPL v2");