drivers/gpu/drm/nouveau/nouveau_dmem.c

   1 /*
   2  * Copyright 2018 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "nouveau_dmem.h"
  23 #include "nouveau_drv.h"
  24 #include "nouveau_chan.h"
  25 #include "nouveau_dma.h"
  26 #include "nouveau_mem.h"
  27 #include "nouveau_bo.h"
  28
  29 #include <nvif/class.h>
  30 #include <nvif/object.h>
  31 #include <nvif/if500b.h>
  32 #include <nvif/if900b.h>
  33
  34 #include <linux/sched/mm.h>
  35 #include <linux/hmm.h>
  36
  37 /*
  38  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  39  * it in vram while in use. We likely want to overhaul memory management for
  40  * nouveau to be more page like (not necessarily with system page size but a
  41  * bigger page size) at lowest level and have some shim layer on top that would
  42  * provide the same functionality as TTM.
  43  */
  44 #define DMEM_CHUNK_SIZE (2UL << 20)
  45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  46
  47 struct nouveau_migrate;
  48
  49 enum nouveau_aper {
  50         NOUVEAU_APER_VIRT,
  51         NOUVEAU_APER_VRAM,
  52         NOUVEAU_APER_HOST,
  53 };
  54
  55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  56                                       enum nouveau_aper, u64 dst_addr,
  57                                       enum nouveau_aper, u64 src_addr);
  58
  59 struct nouveau_dmem_chunk {
  60         struct list_head list;
  61         struct nouveau_bo *bo;
  62         struct nouveau_drm *drm;
  63         unsigned long pfn_first;
  64         unsigned long callocated;
  65         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
  66         spinlock_t lock;
  67 };
  68
  69 struct nouveau_dmem_migrate {
  70         nouveau_migrate_copy_t copy_func;
  71         struct nouveau_channel *chan;
  72 };
  73
  74 struct nouveau_dmem {
  75         struct nouveau_drm *drm;
  76         struct dev_pagemap pagemap;
  77         struct nouveau_dmem_migrate migrate;
  78         struct list_head chunk_free;
  79         struct list_head chunk_full;
  80         struct list_head chunk_empty;
  81         struct mutex mutex;
  82 };
  83
  84 static inline struct nouveau_dmem *page_to_dmem(struct page *page)
  85 {
  86         return container_of(page->pgmap, struct nouveau_dmem, pagemap);
  87 }
  88
  89 struct nouveau_dmem_fault {
  90         struct nouveau_drm *drm;
  91         struct nouveau_fence *fence;
  92         dma_addr_t *dma;
  93         unsigned long npages;
  94 };
  95
  96 struct nouveau_migrate {
  97         struct vm_area_struct *vma;
  98         struct nouveau_drm *drm;
  99         struct nouveau_fence *fence;
 100         unsigned long npages;
 101         dma_addr_t *dma;
 102         unsigned long dma_nr;
 103 };
 104
 105 static void nouveau_dmem_page_free(struct page *page)
 106 {
 107         struct nouveau_dmem_chunk *chunk = page->zone_device_data;
 108         unsigned long idx = page_to_pfn(page) - chunk->pfn_first;
 109
 110         /*
 111          * FIXME:
 112          *
 113          * This is really a bad example, we need to overhaul nouveau memory
 114          * management to be more page focus and allow lighter locking scheme
 115          * to be use in the process.
 116          */
 117         spin_lock(&chunk->lock);
 118         clear_bit(idx, chunk->bitmap);
 119         WARN_ON(!chunk->callocated);
 120         chunk->callocated--;
 121         /*
 122          * FIXME when chunk->callocated reach 0 we should add the chunk to
 123          * a reclaim list so that it can be freed in case of memory pressure.
 124          */
 125         spin_unlock(&chunk->lock);
 126 }
 127
 128 static void
 129 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
 130                                   const unsigned long *src_pfns,
 131                                   unsigned long *dst_pfns,
 132                                   unsigned long start,
 133                                   unsigned long end,
 134                                   void *private)
 135 {
 136         struct nouveau_dmem_fault *fault = private;
 137         struct nouveau_drm *drm = fault->drm;
 138         struct device *dev = drm->dev->dev;
 139         unsigned long addr, i, npages = 0;
 140         nouveau_migrate_copy_t copy;
 141         int ret;
 142
 143
 144         /* First allocate new memory */
 145         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 146                 struct page *dpage, *spage;
 147
 148                 dst_pfns[i] = 0;
 149                 spage = migrate_pfn_to_page(src_pfns[i]);
 150                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 151                         continue;
 152
 153                 dpage = alloc_page_vma(GFP_HIGHUSER, vma, addr);
 154                 if (!dpage) {
 155                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 156                         continue;
 157                 }
 158                 lock_page(dpage);
 159
 160                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 161                               MIGRATE_PFN_LOCKED;
 162                 npages++;
 163         }
 164
 165         /* Allocate storage for DMA addresses, so we can unmap later. */
 166         fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);
 167         if (!fault->dma)
 168                 goto error;
 169
 170         /* Copy things over */
 171         copy = drm->dmem->migrate.copy_func;
 172         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 173                 struct nouveau_dmem_chunk *chunk;
 174                 struct page *spage, *dpage;
 175                 u64 src_addr, dst_addr;
 176
 177                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 178                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 179                         continue;
 180
 181                 spage = migrate_pfn_to_page(src_pfns[i]);
 182                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 183                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 184                         __free_page(dpage);
 185                         continue;
 186                 }
 187
 188                 fault->dma[fault->npages] =
 189                         dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,
 190                                            PCI_DMA_BIDIRECTIONAL,
 191                                            DMA_ATTR_SKIP_CPU_SYNC);
 192                 if (dma_mapping_error(dev, fault->dma[fault->npages])) {
 193                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 194                         __free_page(dpage);
 195                         continue;
 196                 }
 197
 198                 dst_addr = fault->dma[fault->npages++];
 199
 200                 chunk = spage->zone_device_data;
 201                 src_addr = page_to_pfn(spage) - chunk->pfn_first;
 202                 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
 203
 204                 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,
 205                                    NOUVEAU_APER_VRAM, src_addr);
 206                 if (ret) {
 207                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 208                         __free_page(dpage);
 209                         continue;
 210                 }
 211         }
 212
 213         nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
 214
 215         return;
 216
 217 error:
 218         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 219                 struct page *page;
 220
 221                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 222                         continue;
 223
 224                 page = migrate_pfn_to_page(dst_pfns[i]);
 225                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 226                 if (page == NULL)
 227                         continue;
 228
 229                 __free_page(page);
 230         }
 231 }
 232
 233 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
 234                                          const unsigned long *src_pfns,
 235                                          const unsigned long *dst_pfns,
 236                                          unsigned long start,
 237                                          unsigned long end,
 238                                          void *private)
 239 {
 240         struct nouveau_dmem_fault *fault = private;
 241         struct nouveau_drm *drm = fault->drm;
 242
 243         if (fault->fence) {
 244                 nouveau_fence_wait(fault->fence, true, false);
 245                 nouveau_fence_unref(&fault->fence);
 246         } else {
 247                 /*
 248                  * FIXME wait for channel to be IDLE before calling finalizing
 249                  * the hmem object below (nouveau_migrate_hmem_fini()).
 250                  */
 251         }
 252
 253         while (fault->npages--) {
 254                 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],
 255                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 256         }
 257         kfree(fault->dma);
 258 }
 259
 260 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
 261         .alloc_and_copy         = nouveau_dmem_fault_alloc_and_copy,
 262         .finalize_and_map       = nouveau_dmem_fault_finalize_and_map,
 263 };
 264
 265 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
 266 {
 267         struct nouveau_dmem *dmem = page_to_dmem(vmf->page);
 268         unsigned long src[1] = {0}, dst[1] = {0};
 269         struct nouveau_dmem_fault fault = { .drm = dmem->drm };
 270         int ret;
 271
 272         /*
 273          * FIXME what we really want is to find some heuristic to migrate more
 274          * than just one page on CPU fault. When such fault happens it is very
 275          * likely that more surrounding page will CPU fault too.
 276          */
 277         ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vmf->vma,
 278                         vmf->address, vmf->address + PAGE_SIZE,
 279                         src, dst, &fault);
 280         if (ret)
 281                 return VM_FAULT_SIGBUS;
 282
 283         if (dst[0] == MIGRATE_PFN_ERROR)
 284                 return VM_FAULT_SIGBUS;
 285
 286         return 0;
 287 }
 288
 289 static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
 290         .page_free              = nouveau_dmem_page_free,
 291         .migrate_to_ram         = nouveau_dmem_migrate_to_ram,
 292 };
 293
 294 static int
 295 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
 296 {
 297         struct nouveau_dmem_chunk *chunk;
 298         int ret;
 299
 300         if (drm->dmem == NULL)
 301                 return -EINVAL;
 302
 303         mutex_lock(&drm->dmem->mutex);
 304         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 305                                          struct nouveau_dmem_chunk,
 306                                          list);
 307         if (chunk == NULL) {
 308                 mutex_unlock(&drm->dmem->mutex);
 309                 return -ENOMEM;
 310         }
 311
 312         list_del(&chunk->list);
 313         mutex_unlock(&drm->dmem->mutex);
 314
 315         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
 316                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
 317                              &chunk->bo);
 318         if (ret)
 319                 goto out;
 320
 321         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 322         if (ret) {
 323                 nouveau_bo_ref(NULL, &chunk->bo);
 324                 goto out;
 325         }
 326
 327         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
 328         spin_lock_init(&chunk->lock);
 329
 330 out:
 331         mutex_lock(&drm->dmem->mutex);
 332         if (chunk->bo)
 333                 list_add(&chunk->list, &drm->dmem->chunk_empty);
 334         else
 335                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 336         mutex_unlock(&drm->dmem->mutex);
 337
 338         return ret;
 339 }
 340
 341 static struct nouveau_dmem_chunk *
 342 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
 343 {
 344         struct nouveau_dmem_chunk *chunk;
 345
 346         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
 347                                          struct nouveau_dmem_chunk,
 348                                          list);
 349         if (chunk)
 350                 return chunk;
 351
 352         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 353                                          struct nouveau_dmem_chunk,
 354                                          list);
 355         if (chunk->bo)
 356                 return chunk;
 357
 358         return NULL;
 359 }
 360
 361 static int
 362 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
 363                          unsigned long npages,
 364                          unsigned long *pages)
 365 {
 366         struct nouveau_dmem_chunk *chunk;
 367         unsigned long c;
 368         int ret;
 369
 370         memset(pages, 0xff, npages * sizeof(*pages));
 371
 372         mutex_lock(&drm->dmem->mutex);
 373         for (c = 0; c < npages;) {
 374                 unsigned long i;
 375
 376                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
 377                 if (chunk == NULL) {
 378                         mutex_unlock(&drm->dmem->mutex);
 379                         ret = nouveau_dmem_chunk_alloc(drm);
 380                         if (ret) {
 381                                 if (c)
 382                                         return 0;
 383                                 return ret;
 384                         }
 385                         mutex_lock(&drm->dmem->mutex);
 386                         continue;
 387                 }
 388
 389                 spin_lock(&chunk->lock);
 390                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
 391                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
 392                         pages[c] = chunk->pfn_first + i;
 393                         set_bit(i, chunk->bitmap);
 394                         chunk->callocated++;
 395                         c++;
 396
 397                         i = find_next_zero_bit(chunk->bitmap,
 398                                         DMEM_CHUNK_NPAGES, i);
 399                 }
 400                 spin_unlock(&chunk->lock);
 401         }
 402         mutex_unlock(&drm->dmem->mutex);
 403
 404         return 0;
 405 }
 406
 407 static struct page *
 408 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
 409 {
 410         unsigned long pfns[1];
 411         struct page *page;
 412         int ret;
 413
 414         /* FIXME stop all the miss-match API ... */
 415         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
 416         if (ret)
 417                 return NULL;
 418
 419         page = pfn_to_page(pfns[0]);
 420         get_page(page);
 421         lock_page(page);
 422         return page;
 423 }
 424
 425 static void
 426 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
 427 {
 428         unlock_page(page);
 429         put_page(page);
 430 }
 431
 432 void
 433 nouveau_dmem_resume(struct nouveau_drm *drm)
 434 {
 435         struct nouveau_dmem_chunk *chunk;
 436         int ret;
 437
 438         if (drm->dmem == NULL)
 439                 return;
 440
 441         mutex_lock(&drm->dmem->mutex);
 442         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 443                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 444                 /* FIXME handle pin failure */
 445                 WARN_ON(ret);
 446         }
 447         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 448                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 449                 /* FIXME handle pin failure */
 450                 WARN_ON(ret);
 451         }
 452         mutex_unlock(&drm->dmem->mutex);
 453 }
 454
 455 void
 456 nouveau_dmem_suspend(struct nouveau_drm *drm)
 457 {
 458         struct nouveau_dmem_chunk *chunk;
 459
 460         if (drm->dmem == NULL)
 461                 return;
 462
 463         mutex_lock(&drm->dmem->mutex);
 464         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 465                 nouveau_bo_unpin(chunk->bo);
 466         }
 467         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 468                 nouveau_bo_unpin(chunk->bo);
 469         }
 470         mutex_unlock(&drm->dmem->mutex);
 471 }
 472
 473 void
 474 nouveau_dmem_fini(struct nouveau_drm *drm)
 475 {
 476         struct nouveau_dmem_chunk *chunk, *tmp;
 477
 478         if (drm->dmem == NULL)
 479                 return;
 480
 481         mutex_lock(&drm->dmem->mutex);
 482
 483         WARN_ON(!list_empty(&drm->dmem->chunk_free));
 484         WARN_ON(!list_empty(&drm->dmem->chunk_full));
 485
 486         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
 487                 if (chunk->bo) {
 488                         nouveau_bo_unpin(chunk->bo);
 489                         nouveau_bo_ref(NULL, &chunk->bo);
 490                 }
 491                 list_del(&chunk->list);
 492                 kfree(chunk);
 493         }
 494
 495         mutex_unlock(&drm->dmem->mutex);
 496 }
 497
 498 static int
 499 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
 500                     enum nouveau_aper dst_aper, u64 dst_addr,
 501                     enum nouveau_aper src_aper, u64 src_addr)
 502 {
 503         struct nouveau_channel *chan = drm->dmem->migrate.chan;
 504         u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
 505                          (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
 506                          (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
 507                          (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
 508                          (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
 509         int ret;
 510
 511         ret = RING_SPACE(chan, 13);
 512         if (ret)
 513                 return ret;
 514
 515         if (src_aper != NOUVEAU_APER_VIRT) {
 516                 switch (src_aper) {
 517                 case NOUVEAU_APER_VRAM:
 518                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
 519                         break;
 520                 case NOUVEAU_APER_HOST:
 521                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
 522                         break;
 523                 default:
 524                         return -EINVAL;
 525                 }
 526                 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
 527         }
 528
 529         if (dst_aper != NOUVEAU_APER_VIRT) {
 530                 switch (dst_aper) {
 531                 case NOUVEAU_APER_VRAM:
 532                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
 533                         break;
 534                 case NOUVEAU_APER_HOST:
 535                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
 536                         break;
 537                 default:
 538                         return -EINVAL;
 539                 }
 540                 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
 541         }
 542
 543         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 544         OUT_RING  (chan, upper_32_bits(src_addr));
 545         OUT_RING  (chan, lower_32_bits(src_addr));
 546         OUT_RING  (chan, upper_32_bits(dst_addr));
 547         OUT_RING  (chan, lower_32_bits(dst_addr));
 548         OUT_RING  (chan, PAGE_SIZE);
 549         OUT_RING  (chan, PAGE_SIZE);
 550         OUT_RING  (chan, PAGE_SIZE);
 551         OUT_RING  (chan, npages);
 552         BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 553         OUT_RING  (chan, launch_dma);
 554         return 0;
 555 }
 556
 557 static int
 558 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
 559 {
 560         switch (drm->ttm.copy.oclass) {
 561         case PASCAL_DMA_COPY_A:
 562         case PASCAL_DMA_COPY_B:
 563         case  VOLTA_DMA_COPY_A:
 564         case TURING_DMA_COPY_A:
 565                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
 566                 drm->dmem->migrate.chan = drm->ttm.chan;
 567                 return 0;
 568         default:
 569                 break;
 570         }
 571         return -ENODEV;
 572 }
 573
 574 void
 575 nouveau_dmem_init(struct nouveau_drm *drm)
 576 {
 577         struct device *device = drm->dev->dev;
 578         struct resource *res;
 579         unsigned long i, size, pfn_first;
 580         int ret;
 581
 582         /* This only make sense on PASCAL or newer */
 583         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
 584                 return;
 585
 586         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
 587                 return;
 588
 589         drm->dmem->drm = drm;
 590         mutex_init(&drm->dmem->mutex);
 591         INIT_LIST_HEAD(&drm->dmem->chunk_free);
 592         INIT_LIST_HEAD(&drm->dmem->chunk_full);
 593         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
 594
 595         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
 596
 597         /* Initialize migration dma helpers before registering memory */
 598         ret = nouveau_dmem_migrate_init(drm);
 599         if (ret)
 600                 goto out_free;
 601
 602         /*
 603          * FIXME we need some kind of policy to decide how much VRAM we
 604          * want to register with HMM. For now just register everything
 605          * and latter if we want to do thing like over commit then we
 606          * could revisit this.
 607          */
 608         res = devm_request_free_mem_region(device, &iomem_resource, size);
 609         if (IS_ERR(res))
 610                 goto out_free;
 611         drm->dmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
 612         drm->dmem->pagemap.res = *res;
 613         drm->dmem->pagemap.ops = &nouveau_dmem_pagemap_ops;
 614         if (IS_ERR(devm_memremap_pages(device, &drm->dmem->pagemap)))
 615                 goto out_free;
 616
 617         pfn_first = res->start >> PAGE_SHIFT;
 618         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
 619                 struct nouveau_dmem_chunk *chunk;
 620                 struct page *page;
 621                 unsigned long j;
 622
 623                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 624                 if (chunk == NULL) {
 625                         nouveau_dmem_fini(drm);
 626                         return;
 627                 }
 628
 629                 chunk->drm = drm;
 630                 chunk->pfn_first = pfn_first + (i * DMEM_CHUNK_NPAGES);
 631                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 632
 633                 page = pfn_to_page(chunk->pfn_first);
 634                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page)
 635                         page->zone_device_data = chunk;
 636         }
 637
 638         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
 639         return;
 640 out_free:
 641         kfree(drm->dmem);
 642         drm->dmem = NULL;
 643 }
 644
 645 static void
 646 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
 647                                     const unsigned long *src_pfns,
 648                                     unsigned long *dst_pfns,
 649                                     unsigned long start,
 650                                     unsigned long end,
 651                                     void *private)
 652 {
 653         struct nouveau_migrate *migrate = private;
 654         struct nouveau_drm *drm = migrate->drm;
 655         struct device *dev = drm->dev->dev;
 656         unsigned long addr, i, npages = 0;
 657         nouveau_migrate_copy_t copy;
 658         int ret;
 659
 660         /* First allocate new memory */
 661         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 662                 struct page *dpage, *spage;
 663
 664                 dst_pfns[i] = 0;
 665                 spage = migrate_pfn_to_page(src_pfns[i]);
 666                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 667                         continue;
 668
 669                 dpage = nouveau_dmem_page_alloc_locked(drm);
 670                 if (!dpage)
 671                         continue;
 672
 673                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 674                               MIGRATE_PFN_LOCKED |
 675                               MIGRATE_PFN_DEVICE;
 676                 npages++;
 677         }
 678
 679         if (!npages)
 680                 return;
 681
 682         /* Allocate storage for DMA addresses, so we can unmap later. */
 683         migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);
 684         if (!migrate->dma)
 685                 goto error;
 686
 687         /* Copy things over */
 688         copy = drm->dmem->migrate.copy_func;
 689         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 690                 struct nouveau_dmem_chunk *chunk;
 691                 struct page *spage, *dpage;
 692                 u64 src_addr, dst_addr;
 693
 694                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 695                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 696                         continue;
 697
 698                 chunk = dpage->zone_device_data;
 699                 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
 700                 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
 701
 702                 spage = migrate_pfn_to_page(src_pfns[i]);
 703                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 704                         nouveau_dmem_page_free_locked(drm, dpage);
 705                         dst_pfns[i] = 0;
 706                         continue;
 707                 }
 708
 709                 migrate->dma[migrate->dma_nr] =
 710                         dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,
 711                                            PCI_DMA_BIDIRECTIONAL,
 712                                            DMA_ATTR_SKIP_CPU_SYNC);
 713                 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {
 714                         nouveau_dmem_page_free_locked(drm, dpage);
 715                         dst_pfns[i] = 0;
 716                         continue;
 717                 }
 718
 719                 src_addr = migrate->dma[migrate->dma_nr++];
 720
 721                 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,
 722                                    NOUVEAU_APER_HOST, src_addr);
 723                 if (ret) {
 724                         nouveau_dmem_page_free_locked(drm, dpage);
 725                         dst_pfns[i] = 0;
 726                         continue;
 727                 }
 728         }
 729
 730         nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
 731
 732         return;
 733
 734 error:
 735         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 736                 struct page *page;
 737
 738                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 739                         continue;
 740
 741                 page = migrate_pfn_to_page(dst_pfns[i]);
 742                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 743                 if (page == NULL)
 744                         continue;
 745
 746                 __free_page(page);
 747         }
 748 }
 749
 750 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
 751                                            const unsigned long *src_pfns,
 752                                            const unsigned long *dst_pfns,
 753                                            unsigned long start,
 754                                            unsigned long end,
 755                                            void *private)
 756 {
 757         struct nouveau_migrate *migrate = private;
 758         struct nouveau_drm *drm = migrate->drm;
 759
 760         if (migrate->fence) {
 761                 nouveau_fence_wait(migrate->fence, true, false);
 762                 nouveau_fence_unref(&migrate->fence);
 763         } else {
 764                 /*
 765                  * FIXME wait for channel to be IDLE before finalizing
 766                  * the hmem object below (nouveau_migrate_hmem_fini()) ?
 767                  */
 768         }
 769
 770         while (migrate->dma_nr--) {
 771                 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],
 772                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 773         }
 774         kfree(migrate->dma);
 775
 776         /*
 777          * FIXME optimization: update GPU page table to point to newly
 778          * migrated memory.
 779          */
 780 }
 781
 782 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
 783         .alloc_and_copy         = nouveau_dmem_migrate_alloc_and_copy,
 784         .finalize_and_map       = nouveau_dmem_migrate_finalize_and_map,
 785 };
 786
 787 int
 788 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
 789                          struct vm_area_struct *vma,
 790                          unsigned long start,
 791                          unsigned long end)
 792 {
 793         unsigned long *src_pfns, *dst_pfns, npages;
 794         struct nouveau_migrate migrate = {0};
 795         unsigned long i, c, max;
 796         int ret = 0;
 797
 798         npages = (end - start) >> PAGE_SHIFT;
 799         max = min(SG_MAX_SINGLE_ALLOC, npages);
 800         src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 801         if (src_pfns == NULL)
 802                 return -ENOMEM;
 803         dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 804         if (dst_pfns == NULL) {
 805                 kfree(src_pfns);
 806                 return -ENOMEM;
 807         }
 808
 809         migrate.drm = drm;
 810         migrate.vma = vma;
 811         migrate.npages = npages;
 812         for (i = 0; i < npages; i += c) {
 813                 unsigned long next;
 814
 815                 c = min(SG_MAX_SINGLE_ALLOC, npages);
 816                 next = start + (c << PAGE_SHIFT);
 817                 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
 818                                   next, src_pfns, dst_pfns, &migrate);
 819                 if (ret)
 820                         goto out;
 821                 start = next;
 822         }
 823
 824 out:
 825         kfree(dst_pfns);
 826         kfree(src_pfns);
 827         return ret;
 828 }
 829
 830 static inline bool
 831 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
 832 {
 833         return is_device_private_page(page) && drm->dmem == page_to_dmem(page);
 834 }
 835
 836 void
 837 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
 838                          struct hmm_range *range)
 839 {
 840         unsigned long i, npages;
 841
 842         npages = (range->end - range->start) >> PAGE_SHIFT;
 843         for (i = 0; i < npages; ++i) {
 844                 struct nouveau_dmem_chunk *chunk;
 845                 struct page *page;
 846                 uint64_t addr;
 847
 848                 page = hmm_pfn_to_page(range, range->pfns[i]);
 849                 if (page == NULL)
 850                         continue;
 851
 852                 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
 853                         continue;
 854                 }
 855
 856                 if (!nouveau_dmem_page(drm, page)) {
 857                         WARN(1, "Some unknown device memory !\n");
 858                         range->pfns[i] = 0;
 859                         continue;
 860                 }
 861
 862                 chunk = page->zone_device_data;
 863                 addr = page_to_pfn(page) - chunk->pfn_first;
 864                 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
 865
 866                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
 867                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
 868         }
 869 }