drivers/iommu/apple-dart.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Apple DART (Device Address Resolution Table) IOMMU driver
   4  *
   5  * Copyright (C) 2021 The Asahi Linux Contributors
   6  *
   7  * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
   8  *  Copyright (C) 2013 ARM Limited
   9  *  Copyright (C) 2015 ARM Limited
  10  * and on exynos-iommu.c
  11  *  Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
  12  */
  13
  14 #include <linux/atomic.h>
  15 #include <linux/bitfield.h>
  16 #include <linux/clk.h>
  17 #include <linux/dev_printk.h>
  18 #include <linux/dma-iommu.h>
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/io-pgtable.h>
  23 #include <linux/iommu.h>
  24 #include <linux/iopoll.h>
  25 #include <linux/module.h>
  26 #include <linux/of.h>
  27 #include <linux/of_address.h>
  28 #include <linux/of_iommu.h>
  29 #include <linux/of_platform.h>
  30 #include <linux/pci.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/slab.h>
  33 #include <linux/swab.h>
  34 #include <linux/types.h>
  35
  36 #define DART_MAX_STREAMS 16
  37 #define DART_MAX_TTBR 4
  38 #define MAX_DARTS_PER_DEVICE 2
  39
  40 #define DART_STREAM_ALL 0xffff
  41
  42 #define DART_PARAMS1 0x00
  43 #define DART_PARAMS_PAGE_SHIFT GENMASK(27, 24)
  44
  45 #define DART_PARAMS2 0x04
  46 #define DART_PARAMS_BYPASS_SUPPORT BIT(0)
  47
  48 #define DART_STREAM_COMMAND 0x20
  49 #define DART_STREAM_COMMAND_BUSY BIT(2)
  50 #define DART_STREAM_COMMAND_INVALIDATE BIT(20)
  51
  52 #define DART_STREAM_SELECT 0x34
  53
  54 #define DART_ERROR 0x40
  55 #define DART_ERROR_STREAM GENMASK(27, 24)
  56 #define DART_ERROR_CODE GENMASK(11, 0)
  57 #define DART_ERROR_FLAG BIT(31)
  58
  59 #define DART_ERROR_READ_FAULT BIT(4)
  60 #define DART_ERROR_WRITE_FAULT BIT(3)
  61 #define DART_ERROR_NO_PTE BIT(2)
  62 #define DART_ERROR_NO_PMD BIT(1)
  63 #define DART_ERROR_NO_TTBR BIT(0)
  64
  65 #define DART_CONFIG 0x60
  66 #define DART_CONFIG_LOCK BIT(15)
  67
  68 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
  69
  70 #define DART_ERROR_ADDR_HI 0x54
  71 #define DART_ERROR_ADDR_LO 0x50
  72
  73 #define DART_TCR(sid) (0x100 + 4 * (sid))
  74 #define DART_TCR_TRANSLATE_ENABLE BIT(7)
  75 #define DART_TCR_BYPASS0_ENABLE BIT(8)
  76 #define DART_TCR_BYPASS1_ENABLE BIT(12)
  77
  78 #define DART_TTBR(sid, idx) (0x200 + 16 * (sid) + 4 * (idx))
  79 #define DART_TTBR_VALID BIT(31)
  80 #define DART_TTBR_SHIFT 12
  81
  82 /*
  83  * Private structure associated with each DART device.
  84  *
  85  * @dev: device struct
  86  * @regs: mapped MMIO region
  87  * @irq: interrupt number, can be shared with other DARTs
  88  * @clks: clocks associated with this DART
  89  * @num_clks: number of @clks
  90  * @lock: lock for hardware operations involving this dart
  91  * @pgsize: pagesize supported by this DART
  92  * @supports_bypass: indicates if this DART supports bypass mode
  93  * @force_bypass: force bypass mode due to pagesize mismatch?
  94  * @sid2group: maps stream ids to iommu_groups
  95  * @iommu: iommu core device
  96  */
  97 struct apple_dart {
  98         struct device *dev;
  99
 100         void __iomem *regs;
 101
 102         int irq;
 103         struct clk_bulk_data *clks;
 104         int num_clks;
 105
 106         spinlock_t lock;
 107
 108         u32 pgsize;
 109         u32 supports_bypass : 1;
 110         u32 force_bypass : 1;
 111
 112         struct iommu_group *sid2group[DART_MAX_STREAMS];
 113         struct iommu_device iommu;
 114 };
 115
 116 /*
 117  * Convenience struct to identify streams.
 118  *
 119  * The normal variant is used inside apple_dart_master_cfg which isn't written
 120  * to concurrently.
 121  * The atomic variant is used inside apple_dart_domain where we have to guard
 122  * against races from potential parallel calls to attach/detach_device.
 123  * Note that even inside the atomic variant the apple_dart pointer is not
 124  * protected: This pointer is initialized once under the domain init mutex
 125  * and never changed again afterwards. Devices with different dart pointers
 126  * cannot be attached to the same domain.
 127  *
 128  * @dart dart pointer
 129  * @sid stream id bitmap
 130  */
 131 struct apple_dart_stream_map {
 132         struct apple_dart *dart;
 133         unsigned long sidmap;
 134 };
 135 struct apple_dart_atomic_stream_map {
 136         struct apple_dart *dart;
 137         atomic64_t sidmap;
 138 };
 139
 140 /*
 141  * This structure is attached to each iommu domain handled by a DART.
 142  *
 143  * @pgtbl_ops: pagetable ops allocated by io-pgtable
 144  * @finalized: true if the domain has been completely initialized
 145  * @init_lock: protects domain initialization
 146  * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
 147  * @domain: core iommu domain pointer
 148  */
 149 struct apple_dart_domain {
 150         struct io_pgtable_ops *pgtbl_ops;
 151
 152         bool finalized;
 153         struct mutex init_lock;
 154         struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 155
 156         struct iommu_domain domain;
 157 };
 158
 159 /*
 160  * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
 161  * and contains a list of streams bound to this device.
 162  * So far the worst case seen is a single device with two streams
 163  * from different darts, such that this simple static array is enough.
 164  *
 165  * @streams: streams for this device
 166  */
 167 struct apple_dart_master_cfg {
 168         struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
 169 };
 170
 171 /*
 172  * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
 173  * apple_dart_domain.stream_maps
 174  *
 175  * @i int used as loop variable
 176  * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
 177  * @stream pointer to the apple_dart_streams struct for each loop iteration
 178  */
 179 #define for_each_stream_map(i, base, stream_map)                               \
 180         for (i = 0, stream_map = &(base)->stream_maps[0];                      \
 181              i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
 182              stream_map = &(base)->stream_maps[++i])
 183
 184 static struct platform_driver apple_dart_driver;
 185 static const struct iommu_ops apple_dart_iommu_ops;
 186
 187 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
 188 {
 189         return container_of(dom, struct apple_dart_domain, domain);
 190 }
 191
 192 static void
 193 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)
 194 {
 195         int sid;
 196
 197         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 198                 writel(DART_TCR_TRANSLATE_ENABLE,
 199                        stream_map->dart->regs + DART_TCR(sid));
 200 }
 201
 202 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
 203 {
 204         int sid;
 205
 206         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 207                 writel(0, stream_map->dart->regs + DART_TCR(sid));
 208 }
 209
 210 static void
 211 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
 212 {
 213         int sid;
 214
 215         WARN_ON(!stream_map->dart->supports_bypass);
 216         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 217                 writel(DART_TCR_BYPASS0_ENABLE | DART_TCR_BYPASS1_ENABLE,
 218                        stream_map->dart->regs + DART_TCR(sid));
 219 }
 220
 221 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
 222                                    u8 idx, phys_addr_t paddr)
 223 {
 224         int sid;
 225
 226         WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1));
 227         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 228                 writel(DART_TTBR_VALID | (paddr >> DART_TTBR_SHIFT),
 229                        stream_map->dart->regs + DART_TTBR(sid, idx));
 230 }
 231
 232 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
 233                                      u8 idx)
 234 {
 235         int sid;
 236
 237         for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 238                 writel(0, stream_map->dart->regs + DART_TTBR(sid, idx));
 239 }
 240
 241 static void
 242 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
 243 {
 244         int i;
 245
 246         for (i = 0; i < DART_MAX_TTBR; ++i)
 247                 apple_dart_hw_clear_ttbr(stream_map, i);
 248 }
 249
 250 static int
 251 apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map,
 252                              u32 command)
 253 {
 254         unsigned long flags;
 255         int ret;
 256         u32 command_reg;
 257
 258         spin_lock_irqsave(&stream_map->dart->lock, flags);
 259
 260         writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT);
 261         writel(command, stream_map->dart->regs + DART_STREAM_COMMAND);
 262
 263         ret = readl_poll_timeout_atomic(
 264                 stream_map->dart->regs + DART_STREAM_COMMAND, command_reg,
 265                 !(command_reg & DART_STREAM_COMMAND_BUSY), 1,
 266                 DART_STREAM_COMMAND_BUSY_TIMEOUT);
 267
 268         spin_unlock_irqrestore(&stream_map->dart->lock, flags);
 269
 270         if (ret) {
 271                 dev_err(stream_map->dart->dev,
 272                         "busy bit did not clear after command %x for streams %lx\n",
 273                         command, stream_map->sidmap);
 274                 return ret;
 275         }
 276
 277         return 0;
 278 }
 279
 280 static int
 281 apple_dart_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
 282 {
 283         return apple_dart_hw_stream_command(stream_map,
 284                                             DART_STREAM_COMMAND_INVALIDATE);
 285 }
 286
 287 static int apple_dart_hw_reset(struct apple_dart *dart)
 288 {
 289         u32 config;
 290         struct apple_dart_stream_map stream_map;
 291
 292         config = readl(dart->regs + DART_CONFIG);
 293         if (config & DART_CONFIG_LOCK) {
 294                 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
 295                         config);
 296                 return -EINVAL;
 297         }
 298
 299         stream_map.dart = dart;
 300         stream_map.sidmap = DART_STREAM_ALL;
 301         apple_dart_hw_disable_dma(&stream_map);
 302         apple_dart_hw_clear_all_ttbrs(&stream_map);
 303
 304         /* clear any pending errors before the interrupt is unmasked */
 305         writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR);
 306
 307         return apple_dart_hw_invalidate_tlb(&stream_map);
 308 }
 309
 310 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
 311 {
 312         int i;
 313         struct apple_dart_atomic_stream_map *domain_stream_map;
 314         struct apple_dart_stream_map stream_map;
 315
 316         for_each_stream_map(i, domain, domain_stream_map) {
 317                 stream_map.dart = domain_stream_map->dart;
 318                 stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap);
 319                 apple_dart_hw_invalidate_tlb(&stream_map);
 320         }
 321 }
 322
 323 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
 324 {
 325         apple_dart_domain_flush_tlb(to_dart_domain(domain));
 326 }
 327
 328 static void apple_dart_iotlb_sync(struct iommu_domain *domain,
 329                                   struct iommu_iotlb_gather *gather)
 330 {
 331         apple_dart_domain_flush_tlb(to_dart_domain(domain));
 332 }
 333
 334 static void apple_dart_iotlb_sync_map(struct iommu_domain *domain,
 335                                       unsigned long iova, size_t size)
 336 {
 337         apple_dart_domain_flush_tlb(to_dart_domain(domain));
 338 }
 339
 340 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
 341                                            dma_addr_t iova)
 342 {
 343         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 344         struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 345
 346         if (!ops)
 347                 return 0;
 348
 349         return ops->iova_to_phys(ops, iova);
 350 }
 351
 352 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
 353                                 phys_addr_t paddr, size_t pgsize,
 354                                 size_t pgcount, int prot, gfp_t gfp,
 355                                 size_t *mapped)
 356 {
 357         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 358         struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 359
 360         if (!ops)
 361                 return -ENODEV;
 362
 363         return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
 364                               mapped);
 365 }
 366
 367 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
 368                                      unsigned long iova, size_t pgsize,
 369                                      size_t pgcount,
 370                                      struct iommu_iotlb_gather *gather)
 371 {
 372         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 373         struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
 374
 375         return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
 376 }
 377
 378 static void
 379 apple_dart_setup_translation(struct apple_dart_domain *domain,
 380                              struct apple_dart_stream_map *stream_map)
 381 {
 382         int i;
 383         struct io_pgtable_cfg *pgtbl_cfg =
 384                 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
 385
 386         for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
 387                 apple_dart_hw_set_ttbr(stream_map, i,
 388                                        pgtbl_cfg->apple_dart_cfg.ttbr[i]);
 389         for (; i < DART_MAX_TTBR; ++i)
 390                 apple_dart_hw_clear_ttbr(stream_map, i);
 391
 392         apple_dart_hw_enable_translation(stream_map);
 393         apple_dart_hw_invalidate_tlb(stream_map);
 394 }
 395
 396 static int apple_dart_finalize_domain(struct iommu_domain *domain,
 397                                       struct apple_dart_master_cfg *cfg)
 398 {
 399         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 400         struct apple_dart *dart = cfg->stream_maps[0].dart;
 401         struct io_pgtable_cfg pgtbl_cfg;
 402         int ret = 0;
 403         int i;
 404
 405         mutex_lock(&dart_domain->init_lock);
 406
 407         if (dart_domain->finalized)
 408                 goto done;
 409
 410         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 411                 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
 412                 atomic64_set(&dart_domain->stream_maps[i].sidmap,
 413                              cfg->stream_maps[i].sidmap);
 414         }
 415
 416         pgtbl_cfg = (struct io_pgtable_cfg){
 417                 .pgsize_bitmap = dart->pgsize,
 418                 .ias = 32,
 419                 .oas = 36,
 420                 .coherent_walk = 1,
 421                 .iommu_dev = dart->dev,
 422         };
 423
 424         dart_domain->pgtbl_ops =
 425                 alloc_io_pgtable_ops(APPLE_DART, &pgtbl_cfg, domain);
 426         if (!dart_domain->pgtbl_ops) {
 427                 ret = -ENOMEM;
 428                 goto done;
 429         }
 430
 431         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 432         domain->geometry.aperture_start = 0;
 433         domain->geometry.aperture_end = DMA_BIT_MASK(32);
 434         domain->geometry.force_aperture = true;
 435
 436         dart_domain->finalized = true;
 437
 438 done:
 439         mutex_unlock(&dart_domain->init_lock);
 440         return ret;
 441 }
 442
 443 static int
 444 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
 445                        struct apple_dart_stream_map *master_maps,
 446                        bool add_streams)
 447 {
 448         int i;
 449
 450         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 451                 if (domain_maps[i].dart != master_maps[i].dart)
 452                         return -EINVAL;
 453         }
 454
 455         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 456                 if (!domain_maps[i].dart)
 457                         break;
 458                 if (add_streams)
 459                         atomic64_or(master_maps[i].sidmap,
 460                                     &domain_maps[i].sidmap);
 461                 else
 462                         atomic64_and(~master_maps[i].sidmap,
 463                                      &domain_maps[i].sidmap);
 464         }
 465
 466         return 0;
 467 }
 468
 469 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
 470                                          struct apple_dart_master_cfg *cfg)
 471 {
 472         return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
 473                                       true);
 474 }
 475
 476 static int apple_dart_domain_remove_streams(struct apple_dart_domain *domain,
 477                                             struct apple_dart_master_cfg *cfg)
 478 {
 479         return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
 480                                       false);
 481 }
 482
 483 static int apple_dart_attach_dev(struct iommu_domain *domain,
 484                                  struct device *dev)
 485 {
 486         int ret, i;
 487         struct apple_dart_stream_map *stream_map;
 488         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 489         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 490
 491         if (cfg->stream_maps[0].dart->force_bypass &&
 492             domain->type != IOMMU_DOMAIN_IDENTITY)
 493                 return -EINVAL;
 494         if (!cfg->stream_maps[0].dart->supports_bypass &&
 495             domain->type == IOMMU_DOMAIN_IDENTITY)
 496                 return -EINVAL;
 497
 498         ret = apple_dart_finalize_domain(domain, cfg);
 499         if (ret)
 500                 return ret;
 501
 502         switch (domain->type) {
 503         case IOMMU_DOMAIN_DMA:
 504         case IOMMU_DOMAIN_UNMANAGED:
 505                 ret = apple_dart_domain_add_streams(dart_domain, cfg);
 506                 if (ret)
 507                         return ret;
 508
 509                 for_each_stream_map(i, cfg, stream_map)
 510                         apple_dart_setup_translation(dart_domain, stream_map);
 511                 break;
 512         case IOMMU_DOMAIN_BLOCKED:
 513                 for_each_stream_map(i, cfg, stream_map)
 514                         apple_dart_hw_disable_dma(stream_map);
 515                 break;
 516         case IOMMU_DOMAIN_IDENTITY:
 517                 for_each_stream_map(i, cfg, stream_map)
 518                         apple_dart_hw_enable_bypass(stream_map);
 519                 break;
 520         }
 521
 522         return ret;
 523 }
 524
 525 static void apple_dart_detach_dev(struct iommu_domain *domain,
 526                                   struct device *dev)
 527 {
 528         int i;
 529         struct apple_dart_stream_map *stream_map;
 530         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 531         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 532
 533         for_each_stream_map(i, cfg, stream_map)
 534                 apple_dart_hw_disable_dma(stream_map);
 535
 536         if (domain->type == IOMMU_DOMAIN_DMA ||
 537             domain->type == IOMMU_DOMAIN_UNMANAGED)
 538                 apple_dart_domain_remove_streams(dart_domain, cfg);
 539 }
 540
 541 static struct iommu_device *apple_dart_probe_device(struct device *dev)
 542 {
 543         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 544         struct apple_dart_stream_map *stream_map;
 545         int i;
 546
 547         if (!cfg)
 548                 return ERR_PTR(-ENODEV);
 549
 550         for_each_stream_map(i, cfg, stream_map)
 551                 device_link_add(
 552                         dev, stream_map->dart->dev,
 553                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
 554
 555         return &cfg->stream_maps[0].dart->iommu;
 556 }
 557
 558 static void apple_dart_release_device(struct device *dev)
 559 {
 560         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 561
 562         if (!cfg)
 563                 return;
 564
 565         dev_iommu_priv_set(dev, NULL);
 566         kfree(cfg);
 567 }
 568
 569 static struct iommu_domain *apple_dart_domain_alloc(unsigned int type)
 570 {
 571         struct apple_dart_domain *dart_domain;
 572
 573         if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED &&
 574             type != IOMMU_DOMAIN_IDENTITY && type != IOMMU_DOMAIN_BLOCKED)
 575                 return NULL;
 576
 577         dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
 578         if (!dart_domain)
 579                 return NULL;
 580
 581         iommu_get_dma_cookie(&dart_domain->domain);
 582         mutex_init(&dart_domain->init_lock);
 583
 584         /* no need to allocate pgtbl_ops or do any other finalization steps */
 585         if (type == IOMMU_DOMAIN_IDENTITY || type == IOMMU_DOMAIN_BLOCKED)
 586                 dart_domain->finalized = true;
 587
 588         return &dart_domain->domain;
 589 }
 590
 591 static void apple_dart_domain_free(struct iommu_domain *domain)
 592 {
 593         struct apple_dart_domain *dart_domain = to_dart_domain(domain);
 594
 595         if (dart_domain->pgtbl_ops)
 596                 free_io_pgtable_ops(dart_domain->pgtbl_ops);
 597
 598         kfree(dart_domain);
 599 }
 600
 601 static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args)
 602 {
 603         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 604         struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
 605         struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
 606         struct apple_dart *cfg_dart;
 607         int i, sid;
 608
 609         if (args->args_count != 1)
 610                 return -EINVAL;
 611         sid = args->args[0];
 612
 613         if (!cfg)
 614                 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
 615         if (!cfg)
 616                 return -ENOMEM;
 617         dev_iommu_priv_set(dev, cfg);
 618
 619         cfg_dart = cfg->stream_maps[0].dart;
 620         if (cfg_dart) {
 621                 if (cfg_dart->supports_bypass != dart->supports_bypass)
 622                         return -EINVAL;
 623                 if (cfg_dart->force_bypass != dart->force_bypass)
 624                         return -EINVAL;
 625                 if (cfg_dart->pgsize != dart->pgsize)
 626                         return -EINVAL;
 627         }
 628
 629         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 630                 if (cfg->stream_maps[i].dart == dart) {
 631                         cfg->stream_maps[i].sidmap |= 1 << sid;
 632                         return 0;
 633                 }
 634         }
 635         for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
 636                 if (!cfg->stream_maps[i].dart) {
 637                         cfg->stream_maps[i].dart = dart;
 638                         cfg->stream_maps[i].sidmap = 1 << sid;
 639                         return 0;
 640                 }
 641         }
 642
 643         return -EINVAL;
 644 }
 645
 646 static DEFINE_MUTEX(apple_dart_groups_lock);
 647
 648 static void apple_dart_release_group(void *iommu_data)
 649 {
 650         int i, sid;
 651         struct apple_dart_stream_map *stream_map;
 652         struct apple_dart_master_cfg *group_master_cfg = iommu_data;
 653
 654         mutex_lock(&apple_dart_groups_lock);
 655
 656         for_each_stream_map(i, group_master_cfg, stream_map)
 657                 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 658                         stream_map->dart->sid2group[sid] = NULL;
 659
 660         kfree(iommu_data);
 661         mutex_unlock(&apple_dart_groups_lock);
 662 }
 663
 664 static struct iommu_group *apple_dart_device_group(struct device *dev)
 665 {
 666         int i, sid;
 667         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 668         struct apple_dart_stream_map *stream_map;
 669         struct apple_dart_master_cfg *group_master_cfg;
 670         struct iommu_group *group = NULL;
 671         struct iommu_group *res = ERR_PTR(-EINVAL);
 672
 673         mutex_lock(&apple_dart_groups_lock);
 674
 675         for_each_stream_map(i, cfg, stream_map) {
 676                 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) {
 677                         struct iommu_group *stream_group =
 678                                 stream_map->dart->sid2group[sid];
 679
 680                         if (group && group != stream_group) {
 681                                 res = ERR_PTR(-EINVAL);
 682                                 goto out;
 683                         }
 684
 685                         group = stream_group;
 686                 }
 687         }
 688
 689         if (group) {
 690                 res = iommu_group_ref_get(group);
 691                 goto out;
 692         }
 693
 694 #ifdef CONFIG_PCI
 695         if (dev_is_pci(dev))
 696                 group = pci_device_group(dev);
 697         else
 698 #endif
 699                 group = generic_device_group(dev);
 700
 701         res = ERR_PTR(-ENOMEM);
 702         if (!group)
 703                 goto out;
 704
 705         group_master_cfg = kzalloc(sizeof(*group_master_cfg), GFP_KERNEL);
 706         if (!group_master_cfg) {
 707                 iommu_group_put(group);
 708                 goto out;
 709         }
 710
 711         memcpy(group_master_cfg, cfg, sizeof(*group_master_cfg));
 712         iommu_group_set_iommudata(group, group_master_cfg,
 713                 apple_dart_release_group);
 714
 715         for_each_stream_map(i, cfg, stream_map)
 716                 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
 717                         stream_map->dart->sid2group[sid] = group;
 718
 719         res = group;
 720
 721 out:
 722         mutex_unlock(&apple_dart_groups_lock);
 723         return res;
 724 }
 725
 726 static int apple_dart_def_domain_type(struct device *dev)
 727 {
 728         struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 729
 730         if (cfg->stream_maps[0].dart->force_bypass)
 731                 return IOMMU_DOMAIN_IDENTITY;
 732         if (!cfg->stream_maps[0].dart->supports_bypass)
 733                 return IOMMU_DOMAIN_DMA;
 734
 735         return 0;
 736 }
 737
 738 static const struct iommu_ops apple_dart_iommu_ops = {
 739         .domain_alloc = apple_dart_domain_alloc,
 740         .domain_free = apple_dart_domain_free,
 741         .attach_dev = apple_dart_attach_dev,
 742         .detach_dev = apple_dart_detach_dev,
 743         .map_pages = apple_dart_map_pages,
 744         .unmap_pages = apple_dart_unmap_pages,
 745         .flush_iotlb_all = apple_dart_flush_iotlb_all,
 746         .iotlb_sync = apple_dart_iotlb_sync,
 747         .iotlb_sync_map = apple_dart_iotlb_sync_map,
 748         .iova_to_phys = apple_dart_iova_to_phys,
 749         .probe_device = apple_dart_probe_device,
 750         .release_device = apple_dart_release_device,
 751         .device_group = apple_dart_device_group,
 752         .of_xlate = apple_dart_of_xlate,
 753         .def_domain_type = apple_dart_def_domain_type,
 754         .pgsize_bitmap = -1UL, /* Restricted during dart probe */
 755 };
 756
 757 static irqreturn_t apple_dart_irq(int irq, void *dev)
 758 {
 759         struct apple_dart *dart = dev;
 760         const char *fault_name = NULL;
 761         u32 error = readl(dart->regs + DART_ERROR);
 762         u32 error_code = FIELD_GET(DART_ERROR_CODE, error);
 763         u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO);
 764         u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI);
 765         u64 addr = addr_lo | (((u64)addr_hi) << 32);
 766         u8 stream_idx = FIELD_GET(DART_ERROR_STREAM, error);
 767
 768         if (!(error & DART_ERROR_FLAG))
 769                 return IRQ_NONE;
 770
 771         /* there should only be a single bit set but let's use == to be sure */
 772         if (error_code == DART_ERROR_READ_FAULT)
 773                 fault_name = "READ FAULT";
 774         else if (error_code == DART_ERROR_WRITE_FAULT)
 775                 fault_name = "WRITE FAULT";
 776         else if (error_code == DART_ERROR_NO_PTE)
 777                 fault_name = "NO PTE FOR IOVA";
 778         else if (error_code == DART_ERROR_NO_PMD)
 779                 fault_name = "NO PMD FOR IOVA";
 780         else if (error_code == DART_ERROR_NO_TTBR)
 781                 fault_name = "NO TTBR FOR IOVA";
 782         else
 783                 fault_name = "unknown";
 784
 785         dev_err_ratelimited(
 786                 dart->dev,
 787                 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
 788                 error, stream_idx, error_code, fault_name, addr);
 789
 790         writel(error, dart->regs + DART_ERROR);
 791         return IRQ_HANDLED;
 792 }
 793
 794 static int apple_dart_set_bus_ops(const struct iommu_ops *ops)
 795 {
 796         int ret;
 797
 798         if (!iommu_present(&platform_bus_type)) {
 799                 ret = bus_set_iommu(&platform_bus_type, ops);
 800                 if (ret)
 801                         return ret;
 802         }
 803 #ifdef CONFIG_PCI
 804         if (!iommu_present(&pci_bus_type)) {
 805                 ret = bus_set_iommu(&pci_bus_type, ops);
 806                 if (ret) {
 807                         bus_set_iommu(&platform_bus_type, NULL);
 808                         return ret;
 809                 }
 810         }
 811 #endif
 812         return 0;
 813 }
 814
 815 static int apple_dart_probe(struct platform_device *pdev)
 816 {
 817         int ret;
 818         u32 dart_params[2];
 819         struct resource *res;
 820         struct apple_dart *dart;
 821         struct device *dev = &pdev->dev;
 822
 823         dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
 824         if (!dart)
 825                 return -ENOMEM;
 826
 827         dart->dev = dev;
 828         spin_lock_init(&dart->lock);
 829
 830         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 831         if (resource_size(res) < 0x4000) {
 832                 dev_err(dev, "MMIO region too small (%pr)\n", res);
 833                 return -EINVAL;
 834         }
 835
 836         dart->regs = devm_ioremap_resource(dev, res);
 837         if (IS_ERR(dart->regs))
 838                 return PTR_ERR(dart->regs);
 839
 840         dart->irq = platform_get_irq(pdev, 0);
 841         if (dart->irq < 0)
 842                 return -ENODEV;
 843
 844         ret = devm_clk_bulk_get_all(dev, &dart->clks);
 845         if (ret < 0)
 846                 return ret;
 847         dart->num_clks = ret;
 848
 849         ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
 850         if (ret)
 851                 return ret;
 852
 853         ret = apple_dart_hw_reset(dart);
 854         if (ret)
 855                 goto err_clk_disable;
 856
 857         dart_params[0] = readl(dart->regs + DART_PARAMS1);
 858         dart_params[1] = readl(dart->regs + DART_PARAMS2);
 859         dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]);
 860         dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT;
 861         dart->force_bypass = dart->pgsize > PAGE_SIZE;
 862
 863         ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED,
 864                           "apple-dart fault handler", dart);
 865         if (ret)
 866                 goto err_clk_disable;
 867
 868         platform_set_drvdata(pdev, dart);
 869
 870         ret = apple_dart_set_bus_ops(&apple_dart_iommu_ops);
 871         if (ret)
 872                 goto err_free_irq;
 873
 874         ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
 875                                      dev_name(&pdev->dev));
 876         if (ret)
 877                 goto err_remove_bus_ops;
 878
 879         ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
 880         if (ret)
 881                 goto err_sysfs_remove;
 882
 883         dev_info(
 884                 &pdev->dev,
 885                 "DART [pagesize %x, bypass support: %d, bypass forced: %d] initialized\n",
 886                 dart->pgsize, dart->supports_bypass, dart->force_bypass);
 887         return 0;
 888
 889 err_sysfs_remove:
 890         iommu_device_sysfs_remove(&dart->iommu);
 891 err_remove_bus_ops:
 892         apple_dart_set_bus_ops(NULL);
 893 err_free_irq:
 894         free_irq(dart->irq, dart);
 895 err_clk_disable:
 896         clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
 897
 898         return ret;
 899 }
 900
 901 static int apple_dart_remove(struct platform_device *pdev)
 902 {
 903         struct apple_dart *dart = platform_get_drvdata(pdev);
 904
 905         apple_dart_hw_reset(dart);
 906         free_irq(dart->irq, dart);
 907         apple_dart_set_bus_ops(NULL);
 908
 909         iommu_device_unregister(&dart->iommu);
 910         iommu_device_sysfs_remove(&dart->iommu);
 911
 912         clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
 913
 914         return 0;
 915 }
 916
 917 static const struct of_device_id apple_dart_of_match[] = {
 918         { .compatible = "apple,t8103-dart", .data = NULL },
 919         {},
 920 };
 921 MODULE_DEVICE_TABLE(of, apple_dart_of_match);
 922
 923 static struct platform_driver apple_dart_driver = {
 924         .driver = {
 925                 .name                   = "apple-dart",
 926                 .of_match_table         = apple_dart_of_match,
 927                 .suppress_bind_attrs    = true,
 928         },
 929         .probe  = apple_dart_probe,
 930         .remove = apple_dart_remove,
 931 };
 932
 933 module_platform_driver(apple_dart_driver);
 934
 935 MODULE_DESCRIPTION("IOMMU API for Apple's DART");
 936 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
 937 MODULE_LICENSE("GPL v2");