drivers/gpu/drm/ttm/ttm_page_alloc.c

   1 /*
   2  * Copyright (c) Red Hat Inc.
   3
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors: Dave Airlie <airlied@redhat.com>
  24  *          Jerome Glisse <jglisse@redhat.com>
  25  *          Pauli Nieminen <suokkos@gmail.com>
  26  */
  27
  28 /* simple list based uncached page pool
  29  * - Pool collects resently freed pages for reuse
  30  * - Use page->lru to keep a free list
  31  * - doesn't track currently in use pages
  32  */
  33
  34 #define pr_fmt(fmt) "[TTM] " fmt
  35
  36 #include <linux/list.h>
  37 #include <linux/spinlock.h>
  38 #include <linux/highmem.h>
  39 #include <linux/mm_types.h>
  40 #include <linux/module.h>
  41 #include <linux/mm.h>
  42 #include <linux/seq_file.h> /* for seq_printf */
  43 #include <linux/slab.h>
  44 #include <linux/dma-mapping.h>
  45
  46 #include <linux/atomic.h>
  47
  48 #include <drm/ttm/ttm_bo_driver.h>
  49 #include <drm/ttm/ttm_page_alloc.h>
  50
  51 #include "ttm_set_memory.h"
  52
  53 #define NUM_PAGES_TO_ALLOC              (PAGE_SIZE/sizeof(struct page *))
  54 #define SMALL_ALLOCATION                16
  55 #define FREE_ALL_PAGES                  (~0U)
  56 /* times are in msecs */
  57 #define PAGE_FREE_INTERVAL              1000
  58
  59 /**
  60  * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
  61  *
  62  * @lock: Protects the shared pool from concurrnet access. Must be used with
  63  * irqsave/irqrestore variants because pool allocator maybe called from
  64  * delayed work.
  65  * @fill_lock: Prevent concurrent calls to fill.
  66  * @list: Pool of free uc/wc pages for fast reuse.
  67  * @gfp_flags: Flags to pass for alloc_page.
  68  * @npages: Number of pages in pool.
  69  */
  70 struct ttm_page_pool {
  71         spinlock_t              lock;
  72         bool                    fill_lock;
  73         struct list_head        list;
  74         gfp_t                   gfp_flags;
  75         unsigned                npages;
  76         char                    *name;
  77         unsigned long           nfrees;
  78         unsigned long           nrefills;
  79         unsigned int            order;
  80 };
  81
  82 /**
  83  * Limits for the pool. They are handled without locks because only place where
  84  * they may change is in sysfs store. They won't have immediate effect anyway
  85  * so forcing serialization to access them is pointless.
  86  */
  87
  88 struct ttm_pool_opts {
  89         unsigned        alloc_size;
  90         unsigned        max_size;
  91         unsigned        small;
  92 };
  93
  94 #define NUM_POOLS 6
  95
  96 /**
  97  * struct ttm_pool_manager - Holds memory pools for fst allocation
  98  *
  99  * Manager is read only object for pool code so it doesn't need locking.
 100  *
 101  * @free_interval: minimum number of jiffies between freeing pages from pool.
 102  * @page_alloc_inited: reference counting for pool allocation.
 103  * @work: Work that is used to shrink the pool. Work is only run when there is
 104  * some pages to free.
 105  * @small_allocation: Limit in number of pages what is small allocation.
 106  *
 107  * @pools: All pool objects in use.
 108  **/
 109 struct ttm_pool_manager {
 110         struct kobject          kobj;
 111         struct shrinker         mm_shrink;
 112         struct ttm_pool_opts    options;
 113
 114         union {
 115                 struct ttm_page_pool    pools[NUM_POOLS];
 116                 struct {
 117                         struct ttm_page_pool    wc_pool;
 118                         struct ttm_page_pool    uc_pool;
 119                         struct ttm_page_pool    wc_pool_dma32;
 120                         struct ttm_page_pool    uc_pool_dma32;
 121                         struct ttm_page_pool    wc_pool_huge;
 122                         struct ttm_page_pool    uc_pool_huge;
 123                 } ;
 124         };
 125 };
 126
 127 static struct attribute ttm_page_pool_max = {
 128         .name = "pool_max_size",
 129         .mode = S_IRUGO | S_IWUSR
 130 };
 131 static struct attribute ttm_page_pool_small = {
 132         .name = "pool_small_allocation",
 133         .mode = S_IRUGO | S_IWUSR
 134 };
 135 static struct attribute ttm_page_pool_alloc_size = {
 136         .name = "pool_allocation_size",
 137         .mode = S_IRUGO | S_IWUSR
 138 };
 139
 140 static struct attribute *ttm_pool_attrs[] = {
 141         &ttm_page_pool_max,
 142         &ttm_page_pool_small,
 143         &ttm_page_pool_alloc_size,
 144         NULL
 145 };
 146
 147 static void ttm_pool_kobj_release(struct kobject *kobj)
 148 {
 149         struct ttm_pool_manager *m =
 150                 container_of(kobj, struct ttm_pool_manager, kobj);
 151         kfree(m);
 152 }
 153
 154 static ssize_t ttm_pool_store(struct kobject *kobj,
 155                 struct attribute *attr, const char *buffer, size_t size)
 156 {
 157         struct ttm_pool_manager *m =
 158                 container_of(kobj, struct ttm_pool_manager, kobj);
 159         int chars;
 160         unsigned val;
 161         chars = sscanf(buffer, "%u", &val);
 162         if (chars == 0)
 163                 return size;
 164
 165         /* Convert kb to number of pages */
 166         val = val / (PAGE_SIZE >> 10);
 167
 168         if (attr == &ttm_page_pool_max)
 169                 m->options.max_size = val;
 170         else if (attr == &ttm_page_pool_small)
 171                 m->options.small = val;
 172         else if (attr == &ttm_page_pool_alloc_size) {
 173                 if (val > NUM_PAGES_TO_ALLOC*8) {
 174                         pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
 175                                NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
 176                                NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 177                         return size;
 178                 } else if (val > NUM_PAGES_TO_ALLOC) {
 179                         pr_warn("Setting allocation size to larger than %lu is not recommended\n",
 180                                 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
 181                 }
 182                 m->options.alloc_size = val;
 183         }
 184
 185         return size;
 186 }
 187
 188 static ssize_t ttm_pool_show(struct kobject *kobj,
 189                 struct attribute *attr, char *buffer)
 190 {
 191         struct ttm_pool_manager *m =
 192                 container_of(kobj, struct ttm_pool_manager, kobj);
 193         unsigned val = 0;
 194
 195         if (attr == &ttm_page_pool_max)
 196                 val = m->options.max_size;
 197         else if (attr == &ttm_page_pool_small)
 198                 val = m->options.small;
 199         else if (attr == &ttm_page_pool_alloc_size)
 200                 val = m->options.alloc_size;
 201
 202         val = val * (PAGE_SIZE >> 10);
 203
 204         return snprintf(buffer, PAGE_SIZE, "%u\n", val);
 205 }
 206
 207 static const struct sysfs_ops ttm_pool_sysfs_ops = {
 208         .show = &ttm_pool_show,
 209         .store = &ttm_pool_store,
 210 };
 211
 212 static struct kobj_type ttm_pool_kobj_type = {
 213         .release = &ttm_pool_kobj_release,
 214         .sysfs_ops = &ttm_pool_sysfs_ops,
 215         .default_attrs = ttm_pool_attrs,
 216 };
 217
 218 static struct ttm_pool_manager *_manager;
 219
 220 /**
 221  * Select the right pool or requested caching state and ttm flags. */
 222 static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
 223                                           enum ttm_caching_state cstate)
 224 {
 225         int pool_index;
 226
 227         if (cstate == tt_cached)
 228                 return NULL;
 229
 230         if (cstate == tt_wc)
 231                 pool_index = 0x0;
 232         else
 233                 pool_index = 0x1;
 234
 235         if (flags & TTM_PAGE_FLAG_DMA32) {
 236                 if (huge)
 237                         return NULL;
 238                 pool_index |= 0x2;
 239
 240         } else if (huge) {
 241                 pool_index |= 0x4;
 242         }
 243
 244         return &_manager->pools[pool_index];
 245 }
 246
 247 /* set memory back to wb and free the pages. */
 248 static void ttm_pages_put(struct page *pages[], unsigned npages,
 249                 unsigned int order)
 250 {
 251         unsigned int i, pages_nr = (1 << order);
 252
 253         if (order == 0) {
 254                 if (ttm_set_pages_array_wb(pages, npages))
 255                         pr_err("Failed to set %d pages to wb!\n", npages);
 256         }
 257
 258         for (i = 0; i < npages; ++i) {
 259                 if (order > 0) {
 260                         if (ttm_set_pages_wb(pages[i], pages_nr))
 261                                 pr_err("Failed to set %d pages to wb!\n", pages_nr);
 262                 }
 263                 __free_pages(pages[i], order);
 264         }
 265 }
 266
 267 static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
 268                 unsigned freed_pages)
 269 {
 270         pool->npages -= freed_pages;
 271         pool->nfrees += freed_pages;
 272 }
 273
 274 /**
 275  * Free pages from pool.
 276  *
 277  * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
 278  * number of pages in one go.
 279  *
 280  * @pool: to free the pages from
 281  * @free_all: If set to true will free all pages in pool
 282  * @use_static: Safe to use static buffer
 283  **/
 284 static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,
 285                               bool use_static)
 286 {
 287         static struct page *static_buf[NUM_PAGES_TO_ALLOC];
 288         unsigned long irq_flags;
 289         struct page *p;
 290         struct page **pages_to_free;
 291         unsigned freed_pages = 0,
 292                  npages_to_free = nr_free;
 293
 294         if (NUM_PAGES_TO_ALLOC < nr_free)
 295                 npages_to_free = NUM_PAGES_TO_ALLOC;
 296
 297         if (use_static)
 298                 pages_to_free = static_buf;
 299         else
 300                 pages_to_free = kmalloc_array(npages_to_free,
 301                                               sizeof(struct page *),
 302                                               GFP_KERNEL);
 303         if (!pages_to_free) {
 304                 pr_debug("Failed to allocate memory for pool free operation\n");
 305                 return 0;
 306         }
 307
 308 restart:
 309         spin_lock_irqsave(&pool->lock, irq_flags);
 310
 311         list_for_each_entry_reverse(p, &pool->list, lru) {
 312                 if (freed_pages >= npages_to_free)
 313                         break;
 314
 315                 pages_to_free[freed_pages++] = p;
 316                 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
 317                 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
 318                         /* remove range of pages from the pool */
 319                         __list_del(p->lru.prev, &pool->list);
 320
 321                         ttm_pool_update_free_locked(pool, freed_pages);
 322                         /**
 323                          * Because changing page caching is costly
 324                          * we unlock the pool to prevent stalling.
 325                          */
 326                         spin_unlock_irqrestore(&pool->lock, irq_flags);
 327
 328                         ttm_pages_put(pages_to_free, freed_pages, pool->order);
 329                         if (likely(nr_free != FREE_ALL_PAGES))
 330                                 nr_free -= freed_pages;
 331
 332                         if (NUM_PAGES_TO_ALLOC >= nr_free)
 333                                 npages_to_free = nr_free;
 334                         else
 335                                 npages_to_free = NUM_PAGES_TO_ALLOC;
 336
 337                         freed_pages = 0;
 338
 339                         /* free all so restart the processing */
 340                         if (nr_free)
 341                                 goto restart;
 342
 343                         /* Not allowed to fall through or break because
 344                          * following context is inside spinlock while we are
 345                          * outside here.
 346                          */
 347                         goto out;
 348
 349                 }
 350         }
 351
 352         /* remove range of pages from the pool */
 353         if (freed_pages) {
 354                 __list_del(&p->lru, &pool->list);
 355
 356                 ttm_pool_update_free_locked(pool, freed_pages);
 357                 nr_free -= freed_pages;
 358         }
 359
 360         spin_unlock_irqrestore(&pool->lock, irq_flags);
 361
 362         if (freed_pages)
 363                 ttm_pages_put(pages_to_free, freed_pages, pool->order);
 364 out:
 365         if (pages_to_free != static_buf)
 366                 kfree(pages_to_free);
 367         return nr_free;
 368 }
 369
 370 /**
 371  * Callback for mm to request pool to reduce number of page held.
 372  *
 373  * XXX: (dchinner) Deadlock warning!
 374  *
 375  * This code is crying out for a shrinker per pool....
 376  */
 377 static unsigned long
 378 ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
 379 {
 380         static DEFINE_MUTEX(lock);
 381         static unsigned start_pool;
 382         unsigned i;
 383         unsigned pool_offset;
 384         struct ttm_page_pool *pool;
 385         int shrink_pages = sc->nr_to_scan;
 386         unsigned long freed = 0;
 387         unsigned int nr_free_pool;
 388
 389         if (!mutex_trylock(&lock))
 390                 return SHRINK_STOP;
 391         pool_offset = ++start_pool % NUM_POOLS;
 392         /* select start pool in round robin fashion */
 393         for (i = 0; i < NUM_POOLS; ++i) {
 394                 unsigned nr_free = shrink_pages;
 395                 unsigned page_nr;
 396
 397                 if (shrink_pages == 0)
 398                         break;
 399
 400                 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
 401                 page_nr = (1 << pool->order);
 402                 /* OK to use static buffer since global mutex is held. */
 403                 nr_free_pool = roundup(nr_free, page_nr) >> pool->order;
 404                 shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true);
 405                 freed += (nr_free_pool - shrink_pages) << pool->order;
 406                 if (freed >= sc->nr_to_scan)
 407                         break;
 408                 shrink_pages <<= pool->order;
 409         }
 410         mutex_unlock(&lock);
 411         return freed;
 412 }
 413
 414
 415 static unsigned long
 416 ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 417 {
 418         unsigned i;
 419         unsigned long count = 0;
 420         struct ttm_page_pool *pool;
 421
 422         for (i = 0; i < NUM_POOLS; ++i) {
 423                 pool = &_manager->pools[i];
 424                 count += (pool->npages << pool->order);
 425         }
 426
 427         return count;
 428 }
 429
 430 static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
 431 {
 432         manager->mm_shrink.count_objects = ttm_pool_shrink_count;
 433         manager->mm_shrink.scan_objects = ttm_pool_shrink_scan;
 434         manager->mm_shrink.seeks = 1;
 435         return register_shrinker(&manager->mm_shrink);
 436 }
 437
 438 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
 439 {
 440         unregister_shrinker(&manager->mm_shrink);
 441 }
 442
 443 static int ttm_set_pages_caching(struct page **pages,
 444                 enum ttm_caching_state cstate, unsigned cpages)
 445 {
 446         int r = 0;
 447         /* Set page caching */
 448         switch (cstate) {
 449         case tt_uncached:
 450                 r = ttm_set_pages_array_uc(pages, cpages);
 451                 if (r)
 452                         pr_err("Failed to set %d pages to uc!\n", cpages);
 453                 break;
 454         case tt_wc:
 455                 r = ttm_set_pages_array_wc(pages, cpages);
 456                 if (r)
 457                         pr_err("Failed to set %d pages to wc!\n", cpages);
 458                 break;
 459         default:
 460                 break;
 461         }
 462         return r;
 463 }
 464
 465 /**
 466  * Free pages the pages that failed to change the caching state. If there is
 467  * any pages that have changed their caching state already put them to the
 468  * pool.
 469  */
 470 static void ttm_handle_caching_failure(struct page **failed_pages,
 471                                        unsigned cpages)
 472 {
 473         unsigned i;
 474
 475         /* Failed pages have to be freed */
 476         for (i = 0; i < cpages; ++i) {
 477                 list_del(&failed_pages[i]->lru);
 478                 __free_page(failed_pages[i]);
 479         }
 480 }
 481
 482 /**
 483  * Allocate new pages with correct caching.
 484  *
 485  * This function is reentrant if caller updates count depending on number of
 486  * pages returned in pages array.
 487  */
 488 static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
 489                                int ttm_flags, enum ttm_caching_state cstate,
 490                                unsigned count, unsigned order)
 491 {
 492         struct page **caching_array;
 493         struct page *p;
 494         int r = 0;
 495         unsigned i, j, cpages;
 496         unsigned npages = 1 << order;
 497         unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC);
 498
 499         /* allocate array for page caching change */
 500         caching_array = kmalloc_array(max_cpages, sizeof(struct page *),
 501                                       GFP_KERNEL);
 502
 503         if (!caching_array) {
 504                 pr_debug("Unable to allocate table for new pages\n");
 505                 return -ENOMEM;
 506         }
 507
 508         for (i = 0, cpages = 0; i < count; ++i) {
 509                 p = alloc_pages(gfp_flags, order);
 510
 511                 if (!p) {
 512                         pr_debug("Unable to get page %u\n", i);
 513
 514                         /* store already allocated pages in the pool after
 515                          * setting the caching state */
 516                         if (cpages) {
 517                                 r = ttm_set_pages_caching(caching_array,
 518                                                           cstate, cpages);
 519                                 if (r)
 520                                         ttm_handle_caching_failure(caching_array,
 521                                                                    cpages);
 522                         }
 523                         r = -ENOMEM;
 524                         goto out;
 525                 }
 526
 527                 list_add(&p->lru, pages);
 528
 529 #ifdef CONFIG_HIGHMEM
 530                 /* gfp flags of highmem page should never be dma32 so we
 531                  * we should be fine in such case
 532                  */
 533                 if (PageHighMem(p))
 534                         continue;
 535
 536 #endif
 537                 for (j = 0; j < npages; ++j) {
 538                         caching_array[cpages++] = p++;
 539                         if (cpages == max_cpages) {
 540
 541                                 r = ttm_set_pages_caching(caching_array,
 542                                                 cstate, cpages);
 543                                 if (r) {
 544                                         ttm_handle_caching_failure(caching_array,
 545                                                                    cpages);
 546                                         goto out;
 547                                 }
 548                                 cpages = 0;
 549                         }
 550                 }
 551         }
 552
 553         if (cpages) {
 554                 r = ttm_set_pages_caching(caching_array, cstate, cpages);
 555                 if (r)
 556                         ttm_handle_caching_failure(caching_array, cpages);
 557         }
 558 out:
 559         kfree(caching_array);
 560
 561         return r;
 562 }
 563
 564 /**
 565  * Fill the given pool if there aren't enough pages and the requested number of
 566  * pages is small.
 567  */
 568 static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags,
 569                                       enum ttm_caching_state cstate,
 570                                       unsigned count, unsigned long *irq_flags)
 571 {
 572         struct page *p;
 573         int r;
 574         unsigned cpages = 0;
 575         /**
 576          * Only allow one pool fill operation at a time.
 577          * If pool doesn't have enough pages for the allocation new pages are
 578          * allocated from outside of pool.
 579          */
 580         if (pool->fill_lock)
 581                 return;
 582
 583         pool->fill_lock = true;
 584
 585         /* If allocation request is small and there are not enough
 586          * pages in a pool we fill the pool up first. */
 587         if (count < _manager->options.small
 588                 && count > pool->npages) {
 589                 struct list_head new_pages;
 590                 unsigned alloc_size = _manager->options.alloc_size;
 591
 592                 /**
 593                  * Can't change page caching if in irqsave context. We have to
 594                  * drop the pool->lock.
 595                  */
 596                 spin_unlock_irqrestore(&pool->lock, *irq_flags);
 597
 598                 INIT_LIST_HEAD(&new_pages);
 599                 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
 600                                         cstate, alloc_size, 0);
 601                 spin_lock_irqsave(&pool->lock, *irq_flags);
 602
 603                 if (!r) {
 604                         list_splice(&new_pages, &pool->list);
 605                         ++pool->nrefills;
 606                         pool->npages += alloc_size;
 607                 } else {
 608                         pr_debug("Failed to fill pool (%p)\n", pool);
 609                         /* If we have any pages left put them to the pool. */
 610                         list_for_each_entry(p, &new_pages, lru) {
 611                                 ++cpages;
 612                         }
 613                         list_splice(&new_pages, &pool->list);
 614                         pool->npages += cpages;
 615                 }
 616
 617         }
 618         pool->fill_lock = false;
 619 }
 620
 621 /**
 622  * Allocate pages from the pool and put them on the return list.
 623  *
 624  * @return zero for success or negative error code.
 625  */
 626 static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
 627                                    struct list_head *pages,
 628                                    int ttm_flags,
 629                                    enum ttm_caching_state cstate,
 630                                    unsigned count, unsigned order)
 631 {
 632         unsigned long irq_flags;
 633         struct list_head *p;
 634         unsigned i;
 635         int r = 0;
 636
 637         spin_lock_irqsave(&pool->lock, irq_flags);
 638         if (!order)
 639                 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count,
 640                                           &irq_flags);
 641
 642         if (count >= pool->npages) {
 643                 /* take all pages from the pool */
 644                 list_splice_init(&pool->list, pages);
 645                 count -= pool->npages;
 646                 pool->npages = 0;
 647                 goto out;
 648         }
 649         /* find the last pages to include for requested number of pages. Split
 650          * pool to begin and halve it to reduce search space. */
 651         if (count <= pool->npages/2) {
 652                 i = 0;
 653                 list_for_each(p, &pool->list) {
 654                         if (++i == count)
 655                                 break;
 656                 }
 657         } else {
 658                 i = pool->npages + 1;
 659                 list_for_each_prev(p, &pool->list) {
 660                         if (--i == count)
 661                                 break;
 662                 }
 663         }
 664         /* Cut 'count' number of pages from the pool */
 665         list_cut_position(pages, &pool->list, p);
 666         pool->npages -= count;
 667         count = 0;
 668 out:
 669         spin_unlock_irqrestore(&pool->lock, irq_flags);
 670
 671         /* clear the pages coming from the pool if requested */
 672         if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
 673                 struct page *page;
 674
 675                 list_for_each_entry(page, pages, lru) {
 676                         if (PageHighMem(page))
 677                                 clear_highpage(page);
 678                         else
 679                                 clear_page(page_address(page));
 680                 }
 681         }
 682
 683         /* If pool didn't have enough pages allocate new one. */
 684         if (count) {
 685                 gfp_t gfp_flags = pool->gfp_flags;
 686
 687                 /* set zero flag for page allocation if required */
 688                 if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
 689                         gfp_flags |= __GFP_ZERO;
 690
 691                 if (ttm_flags & TTM_PAGE_FLAG_NO_RETRY)
 692                         gfp_flags |= __GFP_RETRY_MAYFAIL;
 693
 694                 /* ttm_alloc_new_pages doesn't reference pool so we can run
 695                  * multiple requests in parallel.
 696                  **/
 697                 r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate,
 698                                         count, order);
 699         }
 700
 701         return r;
 702 }
 703
 704 /* Put all pages in pages list to correct pool to wait for reuse */
 705 static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
 706                           enum ttm_caching_state cstate)
 707 {
 708         struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
 709 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 710         struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
 711 #endif
 712         unsigned long irq_flags;
 713         unsigned i;
 714
 715         if (pool == NULL) {
 716                 /* No pool for this memory type so free the pages */
 717                 i = 0;
 718                 while (i < npages) {
 719 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 720                         struct page *p = pages[i];
 721 #endif
 722                         unsigned order = 0, j;
 723
 724                         if (!pages[i]) {
 725                                 ++i;
 726                                 continue;
 727                         }
 728
 729 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 730                         if (!(flags & TTM_PAGE_FLAG_DMA32) &&
 731                             (npages - i) >= HPAGE_PMD_NR) {
 732                                 for (j = 1; j < HPAGE_PMD_NR; ++j)
 733                                         if (++p != pages[i + j])
 734                                             break;
 735
 736                                 if (j == HPAGE_PMD_NR)
 737                                         order = HPAGE_PMD_ORDER;
 738                         }
 739 #endif
 740
 741                         if (page_count(pages[i]) != 1)
 742                                 pr_err("Erroneous page count. Leaking pages.\n");
 743                         __free_pages(pages[i], order);
 744
 745                         j = 1 << order;
 746                         while (j) {
 747                                 pages[i++] = NULL;
 748                                 --j;
 749                         }
 750                 }
 751                 return;
 752         }
 753
 754         i = 0;
 755 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 756         if (huge) {
 757                 unsigned max_size, n2free;
 758
 759                 spin_lock_irqsave(&huge->lock, irq_flags);
 760                 while ((npages - i) >= HPAGE_PMD_NR) {
 761                         struct page *p = pages[i];
 762                         unsigned j;
 763
 764                         if (!p)
 765                                 break;
 766
 767                         for (j = 1; j < HPAGE_PMD_NR; ++j)
 768                                 if (++p != pages[i + j])
 769                                     break;
 770
 771                         if (j != HPAGE_PMD_NR)
 772                                 break;
 773
 774                         list_add_tail(&pages[i]->lru, &huge->list);
 775
 776                         for (j = 0; j < HPAGE_PMD_NR; ++j)
 777                                 pages[i++] = NULL;
 778                         huge->npages++;
 779                 }
 780
 781                 /* Check that we don't go over the pool limit */
 782                 max_size = _manager->options.max_size;
 783                 max_size /= HPAGE_PMD_NR;
 784                 if (huge->npages > max_size)
 785                         n2free = huge->npages - max_size;
 786                 else
 787                         n2free = 0;
 788                 spin_unlock_irqrestore(&huge->lock, irq_flags);
 789                 if (n2free)
 790                         ttm_page_pool_free(huge, n2free, false);
 791         }
 792 #endif
 793
 794         spin_lock_irqsave(&pool->lock, irq_flags);
 795         while (i < npages) {
 796                 if (pages[i]) {
 797                         if (page_count(pages[i]) != 1)
 798                                 pr_err("Erroneous page count. Leaking pages.\n");
 799                         list_add_tail(&pages[i]->lru, &pool->list);
 800                         pages[i] = NULL;
 801                         pool->npages++;
 802                 }
 803                 ++i;
 804         }
 805         /* Check that we don't go over the pool limit */
 806         npages = 0;
 807         if (pool->npages > _manager->options.max_size) {
 808                 npages = pool->npages - _manager->options.max_size;
 809                 /* free at least NUM_PAGES_TO_ALLOC number of pages
 810                  * to reduce calls to set_memory_wb */
 811                 if (npages < NUM_PAGES_TO_ALLOC)
 812                         npages = NUM_PAGES_TO_ALLOC;
 813         }
 814         spin_unlock_irqrestore(&pool->lock, irq_flags);
 815         if (npages)
 816                 ttm_page_pool_free(pool, npages, false);
 817 }
 818
 819 /*
 820  * On success pages list will hold count number of correctly
 821  * cached pages.
 822  */
 823 static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
 824                          enum ttm_caching_state cstate)
 825 {
 826         struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
 827 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 828         struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
 829 #endif
 830         struct list_head plist;
 831         struct page *p = NULL;
 832         unsigned count, first;
 833         int r;
 834
 835         /* No pool for cached pages */
 836         if (pool == NULL) {
 837                 gfp_t gfp_flags = GFP_USER;
 838                 unsigned i;
 839 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 840                 unsigned j;
 841 #endif
 842
 843                 /* set zero flag for page allocation if required */
 844                 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
 845                         gfp_flags |= __GFP_ZERO;
 846
 847                 if (flags & TTM_PAGE_FLAG_NO_RETRY)
 848                         gfp_flags |= __GFP_RETRY_MAYFAIL;
 849
 850                 if (flags & TTM_PAGE_FLAG_DMA32)
 851                         gfp_flags |= GFP_DMA32;
 852                 else
 853                         gfp_flags |= GFP_HIGHUSER;
 854
 855                 i = 0;
 856 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 857                 if (!(gfp_flags & GFP_DMA32)) {
 858                         while (npages >= HPAGE_PMD_NR) {
 859                                 gfp_t huge_flags = gfp_flags;
 860
 861                                 huge_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY |
 862                                         __GFP_KSWAPD_RECLAIM;
 863                                 huge_flags &= ~__GFP_MOVABLE;
 864                                 huge_flags &= ~__GFP_COMP;
 865                                 p = alloc_pages(huge_flags, HPAGE_PMD_ORDER);
 866                                 if (!p)
 867                                         break;
 868
 869                                 for (j = 0; j < HPAGE_PMD_NR; ++j)
 870                                         pages[i++] = p++;
 871
 872                                 npages -= HPAGE_PMD_NR;
 873                         }
 874                 }
 875 #endif
 876
 877                 first = i;
 878                 while (npages) {
 879                         p = alloc_page(gfp_flags);
 880                         if (!p) {
 881                                 pr_debug("Unable to allocate page\n");
 882                                 return -ENOMEM;
 883                         }
 884
 885                         /* Swap the pages if we detect consecutive order */
 886                         if (i > first && pages[i - 1] == p - 1)
 887                                 swap(p, pages[i - 1]);
 888
 889                         pages[i++] = p;
 890                         --npages;
 891                 }
 892                 return 0;
 893         }
 894
 895         count = 0;
 896
 897 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 898         if (huge && npages >= HPAGE_PMD_NR) {
 899                 INIT_LIST_HEAD(&plist);
 900                 ttm_page_pool_get_pages(huge, &plist, flags, cstate,
 901                                         npages / HPAGE_PMD_NR,
 902                                         HPAGE_PMD_ORDER);
 903
 904                 list_for_each_entry(p, &plist, lru) {
 905                         unsigned j;
 906
 907                         for (j = 0; j < HPAGE_PMD_NR; ++j)
 908                                 pages[count++] = &p[j];
 909                 }
 910         }
 911 #endif
 912
 913         INIT_LIST_HEAD(&plist);
 914         r = ttm_page_pool_get_pages(pool, &plist, flags, cstate,
 915                                     npages - count, 0);
 916
 917         first = count;
 918         list_for_each_entry(p, &plist, lru) {
 919                 struct page *tmp = p;
 920
 921                 /* Swap the pages if we detect consecutive order */
 922                 if (count > first && pages[count - 1] == tmp - 1)
 923                         swap(tmp, pages[count - 1]);
 924                 pages[count++] = tmp;
 925         }
 926
 927         if (r) {
 928                 /* If there is any pages in the list put them back to
 929                  * the pool.
 930                  */
 931                 pr_debug("Failed to allocate extra pages for large request\n");
 932                 ttm_put_pages(pages, count, flags, cstate);
 933                 return r;
 934         }
 935
 936         return 0;
 937 }
 938
 939 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,
 940                 char *name, unsigned int order)
 941 {
 942         spin_lock_init(&pool->lock);
 943         pool->fill_lock = false;
 944         INIT_LIST_HEAD(&pool->list);
 945         pool->npages = pool->nfrees = 0;
 946         pool->gfp_flags = flags;
 947         pool->name = name;
 948         pool->order = order;
 949 }
 950
 951 int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 952 {
 953         int ret;
 954 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 955         unsigned order = HPAGE_PMD_ORDER;
 956 #else
 957         unsigned order = 0;
 958 #endif
 959
 960         WARN_ON(_manager);
 961
 962         pr_info("Initializing pool allocator\n");
 963
 964         _manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
 965         if (!_manager)
 966                 return -ENOMEM;
 967
 968         ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0);
 969
 970         ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0);
 971
 972         ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
 973                                   GFP_USER | GFP_DMA32, "wc dma", 0);
 974
 975         ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
 976                                   GFP_USER | GFP_DMA32, "uc dma", 0);
 977
 978         ttm_page_pool_init_locked(&_manager->wc_pool_huge,
 979                                   (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY |
 980                                    __GFP_KSWAPD_RECLAIM) &
 981                                   ~(__GFP_MOVABLE | __GFP_COMP),
 982                                   "wc huge", order);
 983
 984         ttm_page_pool_init_locked(&_manager->uc_pool_huge,
 985                                   (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY |
 986                                    __GFP_KSWAPD_RECLAIM) &
 987                                   ~(__GFP_MOVABLE | __GFP_COMP)
 988                                   , "uc huge", order);
 989
 990         _manager->options.max_size = max_pages;
 991         _manager->options.small = SMALL_ALLOCATION;
 992         _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
 993
 994         ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
 995                                    &glob->kobj, "pool");
 996         if (unlikely(ret != 0))
 997                 goto error;
 998
 999         ret = ttm_pool_mm_shrink_init(_manager);
1000         if (unlikely(ret != 0))
1001                 goto error;
1002         return 0;
1003
1004 error:
1005         kobject_put(&_manager->kobj);
1006         _manager = NULL;
1007         return ret;
1008 }
1009
1010 void ttm_page_alloc_fini(void)
1011 {
1012         int i;
1013
1014         pr_info("Finalizing pool allocator\n");
1015         ttm_pool_mm_shrink_fini(_manager);
1016
1017         /* OK to use static buffer since global mutex is no longer used. */
1018         for (i = 0; i < NUM_POOLS; ++i)
1019                 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true);
1020
1021         kobject_put(&_manager->kobj);
1022         _manager = NULL;
1023 }
1024
1025 static void
1026 ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update)
1027 {
1028         struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1029         unsigned i;
1030
1031         if (mem_count_update == 0)
1032                 goto put_pages;
1033
1034         for (i = 0; i < mem_count_update; ++i) {
1035                 if (!ttm->pages[i])
1036                         continue;
1037
1038                 ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE);
1039         }
1040
1041 put_pages:
1042         ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
1043                       ttm->caching_state);
1044         ttm_tt_set_unpopulated(ttm);
1045 }
1046
1047 int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
1048 {
1049         struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1050         unsigned i;
1051         int ret;
1052
1053         if (ttm_tt_is_populated(ttm))
1054                 return 0;
1055
1056         if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx))
1057                 return -ENOMEM;
1058
1059         ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
1060                             ttm->caching_state);
1061         if (unlikely(ret != 0)) {
1062                 ttm_pool_unpopulate_helper(ttm, 0);
1063                 return ret;
1064         }
1065
1066         for (i = 0; i < ttm->num_pages; ++i) {
1067                 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
1068                                                 PAGE_SIZE, ctx);
1069                 if (unlikely(ret != 0)) {
1070                         ttm_pool_unpopulate_helper(ttm, i);
1071                         return -ENOMEM;
1072                 }
1073         }
1074
1075         if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
1076                 ret = ttm_tt_swapin(ttm);
1077                 if (unlikely(ret != 0)) {
1078                         ttm_pool_unpopulate(ttm);
1079                         return ret;
1080                 }
1081         }
1082
1083         ttm_tt_set_populated(ttm);
1084         return 0;
1085 }
1086 EXPORT_SYMBOL(ttm_pool_populate);
1087
1088 void ttm_pool_unpopulate(struct ttm_tt *ttm)
1089 {
1090         ttm_pool_unpopulate_helper(ttm, ttm->num_pages);
1091 }
1092 EXPORT_SYMBOL(ttm_pool_unpopulate);
1093
1094 int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt,
1095                                         struct ttm_operation_ctx *ctx)
1096 {
1097         unsigned i, j;
1098         int r;
1099
1100         r = ttm_pool_populate(&tt->ttm, ctx);
1101         if (r)
1102                 return r;
1103
1104         for (i = 0; i < tt->ttm.num_pages; ++i) {
1105                 struct page *p = tt->ttm.pages[i];
1106                 size_t num_pages = 1;
1107
1108                 for (j = i + 1; j < tt->ttm.num_pages; ++j) {
1109                         if (++p != tt->ttm.pages[j])
1110                                 break;
1111
1112                         ++num_pages;
1113                 }
1114
1115                 tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i],
1116                                                   0, num_pages * PAGE_SIZE,
1117                                                   DMA_BIDIRECTIONAL);
1118                 if (dma_mapping_error(dev, tt->dma_address[i])) {
1119                         while (i--) {
1120                                 dma_unmap_page(dev, tt->dma_address[i],
1121                                                PAGE_SIZE, DMA_BIDIRECTIONAL);
1122                                 tt->dma_address[i] = 0;
1123                         }
1124                         ttm_pool_unpopulate(&tt->ttm);
1125                         return -EFAULT;
1126                 }
1127
1128                 for (j = 1; j < num_pages; ++j) {
1129                         tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE;
1130                         ++i;
1131                 }
1132         }
1133         return 0;
1134 }
1135 EXPORT_SYMBOL(ttm_populate_and_map_pages);
1136
1137 void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt)
1138 {
1139         unsigned i, j;
1140
1141         for (i = 0; i < tt->ttm.num_pages;) {
1142                 struct page *p = tt->ttm.pages[i];
1143                 size_t num_pages = 1;
1144
1145                 if (!tt->dma_address[i] || !tt->ttm.pages[i]) {
1146                         ++i;
1147                         continue;
1148                 }
1149
1150                 for (j = i + 1; j < tt->ttm.num_pages; ++j) {
1151                         if (++p != tt->ttm.pages[j])
1152                                 break;
1153
1154                         ++num_pages;
1155                 }
1156
1157                 dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE,
1158                                DMA_BIDIRECTIONAL);
1159
1160                 i += num_pages;
1161         }
1162         ttm_pool_unpopulate(&tt->ttm);
1163 }
1164 EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages);
1165
1166 int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
1167 {
1168         struct ttm_page_pool *p;
1169         unsigned i;
1170         char *h[] = {"pool", "refills", "pages freed", "size"};
1171         if (!_manager) {
1172                 seq_printf(m, "No pool allocator running.\n");
1173                 return 0;
1174         }
1175         seq_printf(m, "%7s %12s %13s %8s\n",
1176                         h[0], h[1], h[2], h[3]);
1177         for (i = 0; i < NUM_POOLS; ++i) {
1178                 p = &_manager->pools[i];
1179
1180                 seq_printf(m, "%7s %12ld %13ld %8d\n",
1181                                 p->name, p->nrefills,
1182                                 p->nfrees, p->npages);
1183         }
1184         return 0;
1185 }
1186 EXPORT_SYMBOL(ttm_page_alloc_debugfs);