drivers/vdpa/vdpa_user/iova_domain.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * MMU-based software IOTLB.
   4  *
   5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
   6  *
   7  * Author: Xie Yongji <xieyongji@bytedance.com>
   8  *
   9  */
  10
  11 #include <linux/slab.h>
  12 #include <linux/file.h>
  13 #include <linux/anon_inodes.h>
  14 #include <linux/highmem.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/vdpa.h>
  17
  18 #include "iova_domain.h"
  19
  20 static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,
  21                                  u64 start, u64 last,
  22                                  u64 addr, unsigned int perm,
  23                                  struct file *file, u64 offset)
  24 {
  25         struct vdpa_map_file *map_file;
  26         int ret;
  27
  28         map_file = kmalloc(sizeof(*map_file), GFP_ATOMIC);
  29         if (!map_file)
  30                 return -ENOMEM;
  31
  32         map_file->file = get_file(file);
  33         map_file->offset = offset;
  34
  35         ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,
  36                                         addr, perm, map_file);
  37         if (ret) {
  38                 fput(map_file->file);
  39                 kfree(map_file);
  40                 return ret;
  41         }
  42         return 0;
  43 }
  44
  45 static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,
  46                                   u64 start, u64 last)
  47 {
  48         struct vdpa_map_file *map_file;
  49         struct vhost_iotlb_map *map;
  50
  51         while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {
  52                 map_file = (struct vdpa_map_file *)map->opaque;
  53                 fput(map_file->file);
  54                 kfree(map_file);
  55                 vhost_iotlb_map_free(domain->iotlb, map);
  56         }
  57 }
  58
  59 int vduse_domain_set_map(struct vduse_iova_domain *domain,
  60                          struct vhost_iotlb *iotlb)
  61 {
  62         struct vdpa_map_file *map_file;
  63         struct vhost_iotlb_map *map;
  64         u64 start = 0ULL, last = ULLONG_MAX;
  65         int ret;
  66
  67         spin_lock(&domain->iotlb_lock);
  68         vduse_iotlb_del_range(domain, start, last);
  69
  70         for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
  71              map = vhost_iotlb_itree_next(map, start, last)) {
  72                 map_file = (struct vdpa_map_file *)map->opaque;
  73                 ret = vduse_iotlb_add_range(domain, map->start, map->last,
  74                                             map->addr, map->perm,
  75                                             map_file->file,
  76                                             map_file->offset);
  77                 if (ret)
  78                         goto err;
  79         }
  80         spin_unlock(&domain->iotlb_lock);
  81
  82         return 0;
  83 err:
  84         vduse_iotlb_del_range(domain, start, last);
  85         spin_unlock(&domain->iotlb_lock);
  86         return ret;
  87 }
  88
  89 void vduse_domain_clear_map(struct vduse_iova_domain *domain,
  90                             struct vhost_iotlb *iotlb)
  91 {
  92         struct vhost_iotlb_map *map;
  93         u64 start = 0ULL, last = ULLONG_MAX;
  94
  95         spin_lock(&domain->iotlb_lock);
  96         for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
  97              map = vhost_iotlb_itree_next(map, start, last)) {
  98                 vduse_iotlb_del_range(domain, map->start, map->last);
  99         }
 100         spin_unlock(&domain->iotlb_lock);
 101 }
 102
 103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,
 104                                          u64 iova, u64 size, u64 paddr)
 105 {
 106         struct vduse_bounce_map *map;
 107         u64 last = iova + size - 1;
 108
 109         while (iova <= last) {
 110                 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
 111                 if (!map->bounce_page) {
 112                         map->bounce_page = alloc_page(GFP_ATOMIC);
 113                         if (!map->bounce_page)
 114                                 return -ENOMEM;
 115                 }
 116                 map->orig_phys = paddr;
 117                 paddr += PAGE_SIZE;
 118                 iova += PAGE_SIZE;
 119         }
 120         return 0;
 121 }
 122
 123 static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,
 124                                            u64 iova, u64 size)
 125 {
 126         struct vduse_bounce_map *map;
 127         u64 last = iova + size - 1;
 128
 129         while (iova <= last) {
 130                 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
 131                 map->orig_phys = INVALID_PHYS_ADDR;
 132                 iova += PAGE_SIZE;
 133         }
 134 }
 135
 136 static void do_bounce(phys_addr_t orig, void *addr, size_t size,
 137                       enum dma_data_direction dir)
 138 {
 139         unsigned long pfn = PFN_DOWN(orig);
 140         unsigned int offset = offset_in_page(orig);
 141         char *buffer;
 142         unsigned int sz = 0;
 143
 144         while (size) {
 145                 sz = min_t(size_t, PAGE_SIZE - offset, size);
 146
 147                 buffer = kmap_atomic(pfn_to_page(pfn));
 148                 if (dir == DMA_TO_DEVICE)
 149                         memcpy(addr, buffer + offset, sz);
 150                 else
 151                         memcpy(buffer + offset, addr, sz);
 152                 kunmap_atomic(buffer);
 153
 154                 size -= sz;
 155                 pfn++;
 156                 addr += sz;
 157                 offset = 0;
 158         }
 159 }
 160
 161 static void vduse_domain_bounce(struct vduse_iova_domain *domain,
 162                                 dma_addr_t iova, size_t size,
 163                                 enum dma_data_direction dir)
 164 {
 165         struct vduse_bounce_map *map;
 166         unsigned int offset;
 167         void *addr;
 168         size_t sz;
 169
 170         if (iova >= domain->bounce_size)
 171                 return;
 172
 173         while (size) {
 174                 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
 175                 offset = offset_in_page(iova);
 176                 sz = min_t(size_t, PAGE_SIZE - offset, size);
 177
 178                 if (WARN_ON(!map->bounce_page ||
 179                             map->orig_phys == INVALID_PHYS_ADDR))
 180                         return;
 181
 182                 addr = page_address(map->bounce_page) + offset;
 183                 do_bounce(map->orig_phys + offset, addr, sz, dir);
 184                 size -= sz;
 185                 iova += sz;
 186         }
 187 }
 188
 189 static struct page *
 190 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)
 191 {
 192         u64 start = iova & PAGE_MASK;
 193         u64 last = start + PAGE_SIZE - 1;
 194         struct vhost_iotlb_map *map;
 195         struct page *page = NULL;
 196
 197         spin_lock(&domain->iotlb_lock);
 198         map = vhost_iotlb_itree_first(domain->iotlb, start, last);
 199         if (!map)
 200                 goto out;
 201
 202         page = pfn_to_page((map->addr + iova - map->start) >> PAGE_SHIFT);
 203         get_page(page);
 204 out:
 205         spin_unlock(&domain->iotlb_lock);
 206
 207         return page;
 208 }
 209
 210 static struct page *
 211 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)
 212 {
 213         struct vduse_bounce_map *map;
 214         struct page *page = NULL;
 215
 216         spin_lock(&domain->iotlb_lock);
 217         map = &domain->bounce_maps[iova >> PAGE_SHIFT];
 218         if (!map->bounce_page)
 219                 goto out;
 220
 221         page = map->bounce_page;
 222         get_page(page);
 223 out:
 224         spin_unlock(&domain->iotlb_lock);
 225
 226         return page;
 227 }
 228
 229 static void
 230 vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain)
 231 {
 232         struct vduse_bounce_map *map;
 233         unsigned long pfn, bounce_pfns;
 234
 235         bounce_pfns = domain->bounce_size >> PAGE_SHIFT;
 236
 237         for (pfn = 0; pfn < bounce_pfns; pfn++) {
 238                 map = &domain->bounce_maps[pfn];
 239                 if (WARN_ON(map->orig_phys != INVALID_PHYS_ADDR))
 240                         continue;
 241
 242                 if (!map->bounce_page)
 243                         continue;
 244
 245                 __free_page(map->bounce_page);
 246                 map->bounce_page = NULL;
 247         }
 248 }
 249
 250 void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)
 251 {
 252         if (!domain->bounce_map)
 253                 return;
 254
 255         spin_lock(&domain->iotlb_lock);
 256         if (!domain->bounce_map)
 257                 goto unlock;
 258
 259         vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);
 260         domain->bounce_map = 0;
 261 unlock:
 262         spin_unlock(&domain->iotlb_lock);
 263 }
 264
 265 static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)
 266 {
 267         int ret = 0;
 268
 269         if (domain->bounce_map)
 270                 return 0;
 271
 272         spin_lock(&domain->iotlb_lock);
 273         if (domain->bounce_map)
 274                 goto unlock;
 275
 276         ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,
 277                                     0, VHOST_MAP_RW, domain->file, 0);
 278         if (ret)
 279                 goto unlock;
 280
 281         domain->bounce_map = 1;
 282 unlock:
 283         spin_unlock(&domain->iotlb_lock);
 284         return ret;
 285 }
 286
 287 static dma_addr_t
 288 vduse_domain_alloc_iova(struct iova_domain *iovad,
 289                         unsigned long size, unsigned long limit)
 290 {
 291         unsigned long shift = iova_shift(iovad);
 292         unsigned long iova_len = iova_align(iovad, size) >> shift;
 293         unsigned long iova_pfn;
 294
 295         /*
 296          * Freeing non-power-of-two-sized allocations back into the IOVA caches
 297          * will come back to bite us badly, so we have to waste a bit of space
 298          * rounding up anything cacheable to make sure that can't happen. The
 299          * order of the unadjusted size will still match upon freeing.
 300          */
 301         if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
 302                 iova_len = roundup_pow_of_two(iova_len);
 303         iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
 304
 305         return iova_pfn << shift;
 306 }
 307
 308 static void vduse_domain_free_iova(struct iova_domain *iovad,
 309                                    dma_addr_t iova, size_t size)
 310 {
 311         unsigned long shift = iova_shift(iovad);
 312         unsigned long iova_len = iova_align(iovad, size) >> shift;
 313
 314         free_iova_fast(iovad, iova >> shift, iova_len);
 315 }
 316
 317 dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
 318                                  struct page *page, unsigned long offset,
 319                                  size_t size, enum dma_data_direction dir,
 320                                  unsigned long attrs)
 321 {
 322         struct iova_domain *iovad = &domain->stream_iovad;
 323         unsigned long limit = domain->bounce_size - 1;
 324         phys_addr_t pa = page_to_phys(page) + offset;
 325         dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
 326
 327         if (!iova)
 328                 return DMA_MAPPING_ERROR;
 329
 330         if (vduse_domain_init_bounce_map(domain))
 331                 goto err;
 332
 333         if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
 334                 goto err;
 335
 336         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
 337                 vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
 338
 339         return iova;
 340 err:
 341         vduse_domain_free_iova(iovad, iova, size);
 342         return DMA_MAPPING_ERROR;
 343 }
 344
 345 void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
 346                              dma_addr_t dma_addr, size_t size,
 347                              enum dma_data_direction dir, unsigned long attrs)
 348 {
 349         struct iova_domain *iovad = &domain->stream_iovad;
 350
 351         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
 352                 vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
 353
 354         vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
 355         vduse_domain_free_iova(iovad, dma_addr, size);
 356 }
 357
 358 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
 359                                   size_t size, dma_addr_t *dma_addr,
 360                                   gfp_t flag, unsigned long attrs)
 361 {
 362         struct iova_domain *iovad = &domain->consistent_iovad;
 363         unsigned long limit = domain->iova_limit;
 364         dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
 365         void *orig = alloc_pages_exact(size, flag);
 366
 367         if (!iova || !orig)
 368                 goto err;
 369
 370         spin_lock(&domain->iotlb_lock);
 371         if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,
 372                                   virt_to_phys(orig), VHOST_MAP_RW,
 373                                   domain->file, (u64)iova)) {
 374                 spin_unlock(&domain->iotlb_lock);
 375                 goto err;
 376         }
 377         spin_unlock(&domain->iotlb_lock);
 378
 379         *dma_addr = iova;
 380
 381         return orig;
 382 err:
 383         *dma_addr = DMA_MAPPING_ERROR;
 384         if (orig)
 385                 free_pages_exact(orig, size);
 386         if (iova)
 387                 vduse_domain_free_iova(iovad, iova, size);
 388
 389         return NULL;
 390 }
 391
 392 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
 393                                 void *vaddr, dma_addr_t dma_addr,
 394                                 unsigned long attrs)
 395 {
 396         struct iova_domain *iovad = &domain->consistent_iovad;
 397         struct vhost_iotlb_map *map;
 398         struct vdpa_map_file *map_file;
 399         phys_addr_t pa;
 400
 401         spin_lock(&domain->iotlb_lock);
 402         map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,
 403                                       (u64)dma_addr + size - 1);
 404         if (WARN_ON(!map)) {
 405                 spin_unlock(&domain->iotlb_lock);
 406                 return;
 407         }
 408         map_file = (struct vdpa_map_file *)map->opaque;
 409         fput(map_file->file);
 410         kfree(map_file);
 411         pa = map->addr;
 412         vhost_iotlb_map_free(domain->iotlb, map);
 413         spin_unlock(&domain->iotlb_lock);
 414
 415         vduse_domain_free_iova(iovad, dma_addr, size);
 416         free_pages_exact(phys_to_virt(pa), size);
 417 }
 418
 419 static vm_fault_t vduse_domain_mmap_fault(struct vm_fault *vmf)
 420 {
 421         struct vduse_iova_domain *domain = vmf->vma->vm_private_data;
 422         unsigned long iova = vmf->pgoff << PAGE_SHIFT;
 423         struct page *page;
 424
 425         if (!domain)
 426                 return VM_FAULT_SIGBUS;
 427
 428         if (iova < domain->bounce_size)
 429                 page = vduse_domain_get_bounce_page(domain, iova);
 430         else
 431                 page = vduse_domain_get_coherent_page(domain, iova);
 432
 433         if (!page)
 434                 return VM_FAULT_SIGBUS;
 435
 436         vmf->page = page;
 437
 438         return 0;
 439 }
 440
 441 static const struct vm_operations_struct vduse_domain_mmap_ops = {
 442         .fault = vduse_domain_mmap_fault,
 443 };
 444
 445 static int vduse_domain_mmap(struct file *file, struct vm_area_struct *vma)
 446 {
 447         struct vduse_iova_domain *domain = file->private_data;
 448
 449         vma->vm_flags |= VM_DONTDUMP | VM_DONTEXPAND;
 450         vma->vm_private_data = domain;
 451         vma->vm_ops = &vduse_domain_mmap_ops;
 452
 453         return 0;
 454 }
 455
 456 static int vduse_domain_release(struct inode *inode, struct file *file)
 457 {
 458         struct vduse_iova_domain *domain = file->private_data;
 459
 460         spin_lock(&domain->iotlb_lock);
 461         vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
 462         vduse_domain_free_bounce_pages(domain);
 463         spin_unlock(&domain->iotlb_lock);
 464         put_iova_domain(&domain->stream_iovad);
 465         put_iova_domain(&domain->consistent_iovad);
 466         vhost_iotlb_free(domain->iotlb);
 467         vfree(domain->bounce_maps);
 468         kfree(domain);
 469
 470         return 0;
 471 }
 472
 473 static const struct file_operations vduse_domain_fops = {
 474         .owner = THIS_MODULE,
 475         .mmap = vduse_domain_mmap,
 476         .release = vduse_domain_release,
 477 };
 478
 479 void vduse_domain_destroy(struct vduse_iova_domain *domain)
 480 {
 481         fput(domain->file);
 482 }
 483
 484 struct vduse_iova_domain *
 485 vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
 486 {
 487         struct vduse_iova_domain *domain;
 488         struct file *file;
 489         struct vduse_bounce_map *map;
 490         unsigned long pfn, bounce_pfns;
 491
 492         bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
 493         if (iova_limit <= bounce_size)
 494                 return NULL;
 495
 496         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
 497         if (!domain)
 498                 return NULL;
 499
 500         domain->iotlb = vhost_iotlb_alloc(0, 0);
 501         if (!domain->iotlb)
 502                 goto err_iotlb;
 503
 504         domain->iova_limit = iova_limit;
 505         domain->bounce_size = PAGE_ALIGN(bounce_size);
 506         domain->bounce_maps = vzalloc(bounce_pfns *
 507                                 sizeof(struct vduse_bounce_map));
 508         if (!domain->bounce_maps)
 509                 goto err_map;
 510
 511         for (pfn = 0; pfn < bounce_pfns; pfn++) {
 512                 map = &domain->bounce_maps[pfn];
 513                 map->orig_phys = INVALID_PHYS_ADDR;
 514         }
 515         file = anon_inode_getfile("[vduse-domain]", &vduse_domain_fops,
 516                                 domain, O_RDWR);
 517         if (IS_ERR(file))
 518                 goto err_file;
 519
 520         domain->file = file;
 521         spin_lock_init(&domain->iotlb_lock);
 522         init_iova_domain(&domain->stream_iovad,
 523                         PAGE_SIZE, IOVA_START_PFN);
 524         init_iova_domain(&domain->consistent_iovad,
 525                         PAGE_SIZE, bounce_pfns);
 526
 527         return domain;
 528 err_file:
 529         vfree(domain->bounce_maps);
 530 err_map:
 531         vhost_iotlb_free(domain->iotlb);
 532 err_iotlb:
 533         kfree(domain);
 534         return NULL;
 535 }
 536
 537 int vduse_domain_init(void)
 538 {
 539         return iova_cache_get();
 540 }
 541
 542 void vduse_domain_exit(void)
 543 {
 544         iova_cache_put();
 545 }