drivers/gpu/drm/i915/selftests/huge_pages.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "../i915_selftest.h"
  26
  27 #include <linux/prime_numbers.h>
  28
  29 #include "igt_gem_utils.h"
  30 #include "mock_drm.h"
  31 #include "i915_random.h"
  32
  33 static const unsigned int page_sizes[] = {
  34         I915_GTT_PAGE_SIZE_2M,
  35         I915_GTT_PAGE_SIZE_64K,
  36         I915_GTT_PAGE_SIZE_4K,
  37 };
  38
  39 static unsigned int get_largest_page_size(struct drm_i915_private *i915,
  40                                           u64 rem)
  41 {
  42         int i;
  43
  44         for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
  45                 unsigned int page_size = page_sizes[i];
  46
  47                 if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size)
  48                         return page_size;
  49         }
  50
  51         return 0;
  52 }
  53
  54 static void huge_pages_free_pages(struct sg_table *st)
  55 {
  56         struct scatterlist *sg;
  57
  58         for (sg = st->sgl; sg; sg = __sg_next(sg)) {
  59                 if (sg_page(sg))
  60                         __free_pages(sg_page(sg), get_order(sg->length));
  61         }
  62
  63         sg_free_table(st);
  64         kfree(st);
  65 }
  66
  67 static int get_huge_pages(struct drm_i915_gem_object *obj)
  68 {
  69 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY)
  70         unsigned int page_mask = obj->mm.page_mask;
  71         struct sg_table *st;
  72         struct scatterlist *sg;
  73         unsigned int sg_page_sizes;
  74         u64 rem;
  75
  76         st = kmalloc(sizeof(*st), GFP);
  77         if (!st)
  78                 return -ENOMEM;
  79
  80         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
  81                 kfree(st);
  82                 return -ENOMEM;
  83         }
  84
  85         rem = obj->base.size;
  86         sg = st->sgl;
  87         st->nents = 0;
  88         sg_page_sizes = 0;
  89
  90         /*
  91          * Our goal here is simple, we want to greedily fill the object from
  92          * largest to smallest page-size, while ensuring that we use *every*
  93          * page-size as per the given page-mask.
  94          */
  95         do {
  96                 unsigned int bit = ilog2(page_mask);
  97                 unsigned int page_size = BIT(bit);
  98                 int order = get_order(page_size);
  99
 100                 do {
 101                         struct page *page;
 102
 103                         GEM_BUG_ON(order >= MAX_ORDER);
 104                         page = alloc_pages(GFP | __GFP_ZERO, order);
 105                         if (!page)
 106                                 goto err;
 107
 108                         sg_set_page(sg, page, page_size, 0);
 109                         sg_page_sizes |= page_size;
 110                         st->nents++;
 111
 112                         rem -= page_size;
 113                         if (!rem) {
 114                                 sg_mark_end(sg);
 115                                 break;
 116                         }
 117
 118                         sg = __sg_next(sg);
 119                 } while ((rem - ((page_size-1) & page_mask)) >= page_size);
 120
 121                 page_mask &= (page_size-1);
 122         } while (page_mask);
 123
 124         if (i915_gem_gtt_prepare_pages(obj, st))
 125                 goto err;
 126
 127         obj->mm.madv = I915_MADV_DONTNEED;
 128
 129         GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask);
 130         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 131
 132         return 0;
 133
 134 err:
 135         sg_set_page(sg, NULL, 0, 0);
 136         sg_mark_end(sg);
 137         huge_pages_free_pages(st);
 138
 139         return -ENOMEM;
 140 }
 141
 142 static void put_huge_pages(struct drm_i915_gem_object *obj,
 143                            struct sg_table *pages)
 144 {
 145         i915_gem_gtt_finish_pages(obj, pages);
 146         huge_pages_free_pages(pages);
 147
 148         obj->mm.dirty = false;
 149         obj->mm.madv = I915_MADV_WILLNEED;
 150 }
 151
 152 static const struct drm_i915_gem_object_ops huge_page_ops = {
 153         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
 154                  I915_GEM_OBJECT_IS_SHRINKABLE,
 155         .get_pages = get_huge_pages,
 156         .put_pages = put_huge_pages,
 157 };
 158
 159 static struct drm_i915_gem_object *
 160 huge_pages_object(struct drm_i915_private *i915,
 161                   u64 size,
 162                   unsigned int page_mask)
 163 {
 164         struct drm_i915_gem_object *obj;
 165
 166         GEM_BUG_ON(!size);
 167         GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask))));
 168
 169         if (size >> PAGE_SHIFT > INT_MAX)
 170                 return ERR_PTR(-E2BIG);
 171
 172         if (overflows_type(size, obj->base.size))
 173                 return ERR_PTR(-E2BIG);
 174
 175         obj = i915_gem_object_alloc();
 176         if (!obj)
 177                 return ERR_PTR(-ENOMEM);
 178
 179         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 180         i915_gem_object_init(obj, &huge_page_ops);
 181
 182         obj->write_domain = I915_GEM_DOMAIN_CPU;
 183         obj->read_domains = I915_GEM_DOMAIN_CPU;
 184         obj->cache_level = I915_CACHE_NONE;
 185
 186         obj->mm.page_mask = page_mask;
 187
 188         return obj;
 189 }
 190
 191 static int fake_get_huge_pages(struct drm_i915_gem_object *obj)
 192 {
 193         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 194         const u64 max_len = rounddown_pow_of_two(UINT_MAX);
 195         struct sg_table *st;
 196         struct scatterlist *sg;
 197         unsigned int sg_page_sizes;
 198         u64 rem;
 199
 200         st = kmalloc(sizeof(*st), GFP);
 201         if (!st)
 202                 return -ENOMEM;
 203
 204         if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) {
 205                 kfree(st);
 206                 return -ENOMEM;
 207         }
 208
 209         /* Use optimal page sized chunks to fill in the sg table */
 210         rem = obj->base.size;
 211         sg = st->sgl;
 212         st->nents = 0;
 213         sg_page_sizes = 0;
 214         do {
 215                 unsigned int page_size = get_largest_page_size(i915, rem);
 216                 unsigned int len = min(page_size * div_u64(rem, page_size),
 217                                        max_len);
 218
 219                 GEM_BUG_ON(!page_size);
 220
 221                 sg->offset = 0;
 222                 sg->length = len;
 223                 sg_dma_len(sg) = len;
 224                 sg_dma_address(sg) = page_size;
 225
 226                 sg_page_sizes |= len;
 227
 228                 st->nents++;
 229
 230                 rem -= len;
 231                 if (!rem) {
 232                         sg_mark_end(sg);
 233                         break;
 234                 }
 235
 236                 sg = sg_next(sg);
 237         } while (1);
 238
 239         i915_sg_trim(st);
 240
 241         obj->mm.madv = I915_MADV_DONTNEED;
 242
 243         __i915_gem_object_set_pages(obj, st, sg_page_sizes);
 244
 245         return 0;
 246 }
 247
 248 static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj)
 249 {
 250         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 251         struct sg_table *st;
 252         struct scatterlist *sg;
 253         unsigned int page_size;
 254
 255         st = kmalloc(sizeof(*st), GFP);
 256         if (!st)
 257                 return -ENOMEM;
 258
 259         if (sg_alloc_table(st, 1, GFP)) {
 260                 kfree(st);
 261                 return -ENOMEM;
 262         }
 263
 264         sg = st->sgl;
 265         st->nents = 1;
 266
 267         page_size = get_largest_page_size(i915, obj->base.size);
 268         GEM_BUG_ON(!page_size);
 269
 270         sg->offset = 0;
 271         sg->length = obj->base.size;
 272         sg_dma_len(sg) = obj->base.size;
 273         sg_dma_address(sg) = page_size;
 274
 275         obj->mm.madv = I915_MADV_DONTNEED;
 276
 277         __i915_gem_object_set_pages(obj, st, sg->length);
 278
 279         return 0;
 280 #undef GFP
 281 }
 282
 283 static void fake_free_huge_pages(struct drm_i915_gem_object *obj,
 284                                  struct sg_table *pages)
 285 {
 286         sg_free_table(pages);
 287         kfree(pages);
 288 }
 289
 290 static void fake_put_huge_pages(struct drm_i915_gem_object *obj,
 291                                 struct sg_table *pages)
 292 {
 293         fake_free_huge_pages(obj, pages);
 294         obj->mm.dirty = false;
 295         obj->mm.madv = I915_MADV_WILLNEED;
 296 }
 297
 298 static const struct drm_i915_gem_object_ops fake_ops = {
 299         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 300         .get_pages = fake_get_huge_pages,
 301         .put_pages = fake_put_huge_pages,
 302 };
 303
 304 static const struct drm_i915_gem_object_ops fake_ops_single = {
 305         .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
 306         .get_pages = fake_get_huge_pages_single,
 307         .put_pages = fake_put_huge_pages,
 308 };
 309
 310 static struct drm_i915_gem_object *
 311 fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 312 {
 313         struct drm_i915_gem_object *obj;
 314
 315         GEM_BUG_ON(!size);
 316         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 317
 318         if (size >> PAGE_SHIFT > UINT_MAX)
 319                 return ERR_PTR(-E2BIG);
 320
 321         if (overflows_type(size, obj->base.size))
 322                 return ERR_PTR(-E2BIG);
 323
 324         obj = i915_gem_object_alloc();
 325         if (!obj)
 326                 return ERR_PTR(-ENOMEM);
 327
 328         drm_gem_private_object_init(&i915->drm, &obj->base, size);
 329
 330         if (single)
 331                 i915_gem_object_init(obj, &fake_ops_single);
 332         else
 333                 i915_gem_object_init(obj, &fake_ops);
 334
 335         obj->write_domain = I915_GEM_DOMAIN_CPU;
 336         obj->read_domains = I915_GEM_DOMAIN_CPU;
 337         obj->cache_level = I915_CACHE_NONE;
 338
 339         return obj;
 340 }
 341
 342 static int igt_check_page_sizes(struct i915_vma *vma)
 343 {
 344         struct drm_i915_private *i915 = vma->vm->i915;
 345         unsigned int supported = INTEL_INFO(i915)->page_sizes;
 346         struct drm_i915_gem_object *obj = vma->obj;
 347         int err = 0;
 348
 349         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) {
 350                 pr_err("unsupported page_sizes.sg=%u, supported=%u\n",
 351                        vma->page_sizes.sg & ~supported, supported);
 352                 err = -EINVAL;
 353         }
 354
 355         if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) {
 356                 pr_err("unsupported page_sizes.gtt=%u, supported=%u\n",
 357                        vma->page_sizes.gtt & ~supported, supported);
 358                 err = -EINVAL;
 359         }
 360
 361         if (vma->page_sizes.phys != obj->mm.page_sizes.phys) {
 362                 pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n",
 363                        vma->page_sizes.phys, obj->mm.page_sizes.phys);
 364                 err = -EINVAL;
 365         }
 366
 367         if (vma->page_sizes.sg != obj->mm.page_sizes.sg) {
 368                 pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n",
 369                        vma->page_sizes.sg, obj->mm.page_sizes.sg);
 370                 err = -EINVAL;
 371         }
 372
 373         if (obj->mm.page_sizes.gtt) {
 374                 pr_err("obj->page_sizes.gtt(%u) should never be set\n",
 375                        obj->mm.page_sizes.gtt);
 376                 err = -EINVAL;
 377         }
 378
 379         return err;
 380 }
 381
 382 static int igt_mock_exhaust_device_supported_pages(void *arg)
 383 {
 384         struct i915_hw_ppgtt *ppgtt = arg;
 385         struct drm_i915_private *i915 = ppgtt->vm.i915;
 386         unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
 387         struct drm_i915_gem_object *obj;
 388         struct i915_vma *vma;
 389         int i, j, single;
 390         int err;
 391
 392         /*
 393          * Sanity check creating objects with every valid page support
 394          * combination for our mock device.
 395          */
 396
 397         for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) {
 398                 unsigned int combination = 0;
 399
 400                 for (j = 0; j < ARRAY_SIZE(page_sizes); j++) {
 401                         if (i & BIT(j))
 402                                 combination |= page_sizes[j];
 403                 }
 404
 405                 mkwrite_device_info(i915)->page_sizes = combination;
 406
 407                 for (single = 0; single <= 1; ++single) {
 408                         obj = fake_huge_pages_object(i915, combination, !!single);
 409                         if (IS_ERR(obj)) {
 410                                 err = PTR_ERR(obj);
 411                                 goto out_device;
 412                         }
 413
 414                         if (obj->base.size != combination) {
 415                                 pr_err("obj->base.size=%zu, expected=%u\n",
 416                                        obj->base.size, combination);
 417                                 err = -EINVAL;
 418                                 goto out_put;
 419                         }
 420
 421                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 422                         if (IS_ERR(vma)) {
 423                                 err = PTR_ERR(vma);
 424                                 goto out_put;
 425                         }
 426
 427                         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 428                         if (err)
 429                                 goto out_close;
 430
 431                         err = igt_check_page_sizes(vma);
 432
 433                         if (vma->page_sizes.sg != combination) {
 434                                 pr_err("page_sizes.sg=%u, expected=%u\n",
 435                                        vma->page_sizes.sg, combination);
 436                                 err = -EINVAL;
 437                         }
 438
 439                         i915_vma_unpin(vma);
 440                         i915_vma_close(vma);
 441
 442                         i915_gem_object_put(obj);
 443
 444                         if (err)
 445                                 goto out_device;
 446                 }
 447         }
 448
 449         goto out_device;
 450
 451 out_close:
 452         i915_vma_close(vma);
 453 out_put:
 454         i915_gem_object_put(obj);
 455 out_device:
 456         mkwrite_device_info(i915)->page_sizes = saved_mask;
 457
 458         return err;
 459 }
 460
 461 static int igt_mock_ppgtt_misaligned_dma(void *arg)
 462 {
 463         struct i915_hw_ppgtt *ppgtt = arg;
 464         struct drm_i915_private *i915 = ppgtt->vm.i915;
 465         unsigned long supported = INTEL_INFO(i915)->page_sizes;
 466         struct drm_i915_gem_object *obj;
 467         int bit;
 468         int err;
 469
 470         /*
 471          * Sanity check dma misalignment for huge pages -- the dma addresses we
 472          * insert into the paging structures need to always respect the page
 473          * size alignment.
 474          */
 475
 476         bit = ilog2(I915_GTT_PAGE_SIZE_64K);
 477
 478         for_each_set_bit_from(bit, &supported,
 479                               ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
 480                 IGT_TIMEOUT(end_time);
 481                 unsigned int page_size = BIT(bit);
 482                 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
 483                 unsigned int offset;
 484                 unsigned int size =
 485                         round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1;
 486                 struct i915_vma *vma;
 487
 488                 obj = fake_huge_pages_object(i915, size, true);
 489                 if (IS_ERR(obj))
 490                         return PTR_ERR(obj);
 491
 492                 if (obj->base.size != size) {
 493                         pr_err("obj->base.size=%zu, expected=%u\n",
 494                                obj->base.size, size);
 495                         err = -EINVAL;
 496                         goto out_put;
 497                 }
 498
 499                 err = i915_gem_object_pin_pages(obj);
 500                 if (err)
 501                         goto out_put;
 502
 503                 /* Force the page size for this object */
 504                 obj->mm.page_sizes.sg = page_size;
 505
 506                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 507                 if (IS_ERR(vma)) {
 508                         err = PTR_ERR(vma);
 509                         goto out_unpin;
 510                 }
 511
 512                 err = i915_vma_pin(vma, 0, 0, flags);
 513                 if (err) {
 514                         i915_vma_close(vma);
 515                         goto out_unpin;
 516                 }
 517
 518
 519                 err = igt_check_page_sizes(vma);
 520
 521                 if (vma->page_sizes.gtt != page_size) {
 522                         pr_err("page_sizes.gtt=%u, expected %u\n",
 523                                vma->page_sizes.gtt, page_size);
 524                         err = -EINVAL;
 525                 }
 526
 527                 i915_vma_unpin(vma);
 528
 529                 if (err) {
 530                         i915_vma_close(vma);
 531                         goto out_unpin;
 532                 }
 533
 534                 /*
 535                  * Try all the other valid offsets until the next
 536                  * boundary -- should always fall back to using 4K
 537                  * pages.
 538                  */
 539                 for (offset = 4096; offset < page_size; offset += 4096) {
 540                         err = i915_vma_unbind(vma);
 541                         if (err) {
 542                                 i915_vma_close(vma);
 543                                 goto out_unpin;
 544                         }
 545
 546                         err = i915_vma_pin(vma, 0, 0, flags | offset);
 547                         if (err) {
 548                                 i915_vma_close(vma);
 549                                 goto out_unpin;
 550                         }
 551
 552                         err = igt_check_page_sizes(vma);
 553
 554                         if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) {
 555                                 pr_err("page_sizes.gtt=%u, expected %llu\n",
 556                                        vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K);
 557                                 err = -EINVAL;
 558                         }
 559
 560                         i915_vma_unpin(vma);
 561
 562                         if (err) {
 563                                 i915_vma_close(vma);
 564                                 goto out_unpin;
 565                         }
 566
 567                         if (igt_timeout(end_time,
 568                                         "%s timed out at offset %x with page-size %x\n",
 569                                         __func__, offset, page_size))
 570                                 break;
 571                 }
 572
 573                 i915_vma_close(vma);
 574
 575                 i915_gem_object_unpin_pages(obj);
 576                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 577                 i915_gem_object_put(obj);
 578         }
 579
 580         return 0;
 581
 582 out_unpin:
 583         i915_gem_object_unpin_pages(obj);
 584 out_put:
 585         i915_gem_object_put(obj);
 586
 587         return err;
 588 }
 589
 590 static void close_object_list(struct list_head *objects,
 591                               struct i915_hw_ppgtt *ppgtt)
 592 {
 593         struct drm_i915_gem_object *obj, *on;
 594
 595         list_for_each_entry_safe(obj, on, objects, st_link) {
 596                 struct i915_vma *vma;
 597
 598                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 599                 if (!IS_ERR(vma))
 600                         i915_vma_close(vma);
 601
 602                 list_del(&obj->st_link);
 603                 i915_gem_object_unpin_pages(obj);
 604                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 605                 i915_gem_object_put(obj);
 606         }
 607 }
 608
 609 static int igt_mock_ppgtt_huge_fill(void *arg)
 610 {
 611         struct i915_hw_ppgtt *ppgtt = arg;
 612         struct drm_i915_private *i915 = ppgtt->vm.i915;
 613         unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT;
 614         unsigned long page_num;
 615         bool single = false;
 616         LIST_HEAD(objects);
 617         IGT_TIMEOUT(end_time);
 618         int err = -ENODEV;
 619
 620         for_each_prime_number_from(page_num, 1, max_pages) {
 621                 struct drm_i915_gem_object *obj;
 622                 u64 size = page_num << PAGE_SHIFT;
 623                 struct i915_vma *vma;
 624                 unsigned int expected_gtt = 0;
 625                 int i;
 626
 627                 obj = fake_huge_pages_object(i915, size, single);
 628                 if (IS_ERR(obj)) {
 629                         err = PTR_ERR(obj);
 630                         break;
 631                 }
 632
 633                 if (obj->base.size != size) {
 634                         pr_err("obj->base.size=%zd, expected=%llu\n",
 635                                obj->base.size, size);
 636                         i915_gem_object_put(obj);
 637                         err = -EINVAL;
 638                         break;
 639                 }
 640
 641                 err = i915_gem_object_pin_pages(obj);
 642                 if (err) {
 643                         i915_gem_object_put(obj);
 644                         break;
 645                 }
 646
 647                 list_add(&obj->st_link, &objects);
 648
 649                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 650                 if (IS_ERR(vma)) {
 651                         err = PTR_ERR(vma);
 652                         break;
 653                 }
 654
 655                 err = i915_vma_pin(vma, 0, 0, PIN_USER);
 656                 if (err)
 657                         break;
 658
 659                 err = igt_check_page_sizes(vma);
 660                 if (err) {
 661                         i915_vma_unpin(vma);
 662                         break;
 663                 }
 664
 665                 /*
 666                  * Figure out the expected gtt page size knowing that we go from
 667                  * largest to smallest page size sg chunks, and that we align to
 668                  * the largest page size.
 669                  */
 670                 for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
 671                         unsigned int page_size = page_sizes[i];
 672
 673                         if (HAS_PAGE_SIZES(i915, page_size) &&
 674                             size >= page_size) {
 675                                 expected_gtt |= page_size;
 676                                 size &= page_size-1;
 677                         }
 678                 }
 679
 680                 GEM_BUG_ON(!expected_gtt);
 681                 GEM_BUG_ON(size);
 682
 683                 if (expected_gtt & I915_GTT_PAGE_SIZE_4K)
 684                         expected_gtt &= ~I915_GTT_PAGE_SIZE_64K;
 685
 686                 i915_vma_unpin(vma);
 687
 688                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 689                         if (!IS_ALIGNED(vma->node.start,
 690                                         I915_GTT_PAGE_SIZE_2M)) {
 691                                 pr_err("node.start(%llx) not aligned to 2M\n",
 692                                        vma->node.start);
 693                                 err = -EINVAL;
 694                                 break;
 695                         }
 696
 697                         if (!IS_ALIGNED(vma->node.size,
 698                                         I915_GTT_PAGE_SIZE_2M)) {
 699                                 pr_err("node.size(%llx) not aligned to 2M\n",
 700                                        vma->node.size);
 701                                 err = -EINVAL;
 702                                 break;
 703                         }
 704                 }
 705
 706                 if (vma->page_sizes.gtt != expected_gtt) {
 707                         pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n",
 708                                vma->page_sizes.gtt, expected_gtt,
 709                                obj->base.size, yesno(!!single));
 710                         err = -EINVAL;
 711                         break;
 712                 }
 713
 714                 if (igt_timeout(end_time,
 715                                 "%s timed out at size %zd\n",
 716                                 __func__, obj->base.size))
 717                         break;
 718
 719                 single = !single;
 720         }
 721
 722         close_object_list(&objects, ppgtt);
 723
 724         if (err == -ENOMEM || err == -ENOSPC)
 725                 err = 0;
 726
 727         return err;
 728 }
 729
 730 static int igt_mock_ppgtt_64K(void *arg)
 731 {
 732         struct i915_hw_ppgtt *ppgtt = arg;
 733         struct drm_i915_private *i915 = ppgtt->vm.i915;
 734         struct drm_i915_gem_object *obj;
 735         const struct object_info {
 736                 unsigned int size;
 737                 unsigned int gtt;
 738                 unsigned int offset;
 739         } objects[] = {
 740                 /* Cases with forced padding/alignment */
 741                 {
 742                         .size = SZ_64K,
 743                         .gtt = I915_GTT_PAGE_SIZE_64K,
 744                         .offset = 0,
 745                 },
 746                 {
 747                         .size = SZ_64K + SZ_4K,
 748                         .gtt = I915_GTT_PAGE_SIZE_4K,
 749                         .offset = 0,
 750                 },
 751                 {
 752                         .size = SZ_64K - SZ_4K,
 753                         .gtt = I915_GTT_PAGE_SIZE_4K,
 754                         .offset = 0,
 755                 },
 756                 {
 757                         .size = SZ_2M,
 758                         .gtt = I915_GTT_PAGE_SIZE_64K,
 759                         .offset = 0,
 760                 },
 761                 {
 762                         .size = SZ_2M - SZ_4K,
 763                         .gtt = I915_GTT_PAGE_SIZE_4K,
 764                         .offset = 0,
 765                 },
 766                 {
 767                         .size = SZ_2M + SZ_4K,
 768                         .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K,
 769                         .offset = 0,
 770                 },
 771                 {
 772                         .size = SZ_2M + SZ_64K,
 773                         .gtt = I915_GTT_PAGE_SIZE_64K,
 774                         .offset = 0,
 775                 },
 776                 {
 777                         .size = SZ_2M - SZ_64K,
 778                         .gtt = I915_GTT_PAGE_SIZE_64K,
 779                         .offset = 0,
 780                 },
 781                 /* Try without any forced padding/alignment */
 782                 {
 783                         .size = SZ_64K,
 784                         .offset = SZ_2M,
 785                         .gtt = I915_GTT_PAGE_SIZE_4K,
 786                 },
 787                 {
 788                         .size = SZ_128K,
 789                         .offset = SZ_2M - SZ_64K,
 790                         .gtt = I915_GTT_PAGE_SIZE_4K,
 791                 },
 792         };
 793         struct i915_vma *vma;
 794         int i, single;
 795         int err;
 796
 797         /*
 798          * Sanity check some of the trickiness with 64K pages -- either we can
 799          * safely mark the whole page-table(2M block) as 64K, or we have to
 800          * always fallback to 4K.
 801          */
 802
 803         if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K))
 804                 return 0;
 805
 806         for (i = 0; i < ARRAY_SIZE(objects); ++i) {
 807                 unsigned int size = objects[i].size;
 808                 unsigned int expected_gtt = objects[i].gtt;
 809                 unsigned int offset = objects[i].offset;
 810                 unsigned int flags = PIN_USER;
 811
 812                 for (single = 0; single <= 1; single++) {
 813                         obj = fake_huge_pages_object(i915, size, !!single);
 814                         if (IS_ERR(obj))
 815                                 return PTR_ERR(obj);
 816
 817                         err = i915_gem_object_pin_pages(obj);
 818                         if (err)
 819                                 goto out_object_put;
 820
 821                         /*
 822                          * Disable 2M pages -- We only want to use 64K/4K pages
 823                          * for this test.
 824                          */
 825                         obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M;
 826
 827                         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
 828                         if (IS_ERR(vma)) {
 829                                 err = PTR_ERR(vma);
 830                                 goto out_object_unpin;
 831                         }
 832
 833                         if (offset)
 834                                 flags |= PIN_OFFSET_FIXED | offset;
 835
 836                         err = i915_vma_pin(vma, 0, 0, flags);
 837                         if (err)
 838                                 goto out_vma_close;
 839
 840                         err = igt_check_page_sizes(vma);
 841                         if (err)
 842                                 goto out_vma_unpin;
 843
 844                         if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) {
 845                                 if (!IS_ALIGNED(vma->node.start,
 846                                                 I915_GTT_PAGE_SIZE_2M)) {
 847                                         pr_err("node.start(%llx) not aligned to 2M\n",
 848                                                vma->node.start);
 849                                         err = -EINVAL;
 850                                         goto out_vma_unpin;
 851                                 }
 852
 853                                 if (!IS_ALIGNED(vma->node.size,
 854                                                 I915_GTT_PAGE_SIZE_2M)) {
 855                                         pr_err("node.size(%llx) not aligned to 2M\n",
 856                                                vma->node.size);
 857                                         err = -EINVAL;
 858                                         goto out_vma_unpin;
 859                                 }
 860                         }
 861
 862                         if (vma->page_sizes.gtt != expected_gtt) {
 863                                 pr_err("gtt=%u, expected=%u, i=%d, single=%s\n",
 864                                        vma->page_sizes.gtt, expected_gtt, i,
 865                                        yesno(!!single));
 866                                 err = -EINVAL;
 867                                 goto out_vma_unpin;
 868                         }
 869
 870                         i915_vma_unpin(vma);
 871                         i915_vma_close(vma);
 872
 873                         i915_gem_object_unpin_pages(obj);
 874                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
 875                         i915_gem_object_put(obj);
 876                 }
 877         }
 878
 879         return 0;
 880
 881 out_vma_unpin:
 882         i915_vma_unpin(vma);
 883 out_vma_close:
 884         i915_vma_close(vma);
 885 out_object_unpin:
 886         i915_gem_object_unpin_pages(obj);
 887 out_object_put:
 888         i915_gem_object_put(obj);
 889
 890         return err;
 891 }
 892
 893 static struct i915_vma *
 894 gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
 895 {
 896         struct drm_i915_private *i915 = vma->vm->i915;
 897         const int gen = INTEL_GEN(i915);
 898         unsigned int count = vma->size >> PAGE_SHIFT;
 899         struct drm_i915_gem_object *obj;
 900         struct i915_vma *batch;
 901         unsigned int size;
 902         u32 *cmd;
 903         int n;
 904         int err;
 905
 906         size = (1 + 4 * count) * sizeof(u32);
 907         size = round_up(size, PAGE_SIZE);
 908         obj = i915_gem_object_create_internal(i915, size);
 909         if (IS_ERR(obj))
 910                 return ERR_CAST(obj);
 911
 912         cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
 913         if (IS_ERR(cmd)) {
 914                 err = PTR_ERR(cmd);
 915                 goto err;
 916         }
 917
 918         offset += vma->node.start;
 919
 920         for (n = 0; n < count; n++) {
 921                 if (gen >= 8) {
 922                         *cmd++ = MI_STORE_DWORD_IMM_GEN4;
 923                         *cmd++ = lower_32_bits(offset);
 924                         *cmd++ = upper_32_bits(offset);
 925                         *cmd++ = val;
 926                 } else if (gen >= 4) {
 927                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
 928                                 (gen < 6 ? MI_USE_GGTT : 0);
 929                         *cmd++ = 0;
 930                         *cmd++ = offset;
 931                         *cmd++ = val;
 932                 } else {
 933                         *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
 934                         *cmd++ = offset;
 935                         *cmd++ = val;
 936                 }
 937
 938                 offset += PAGE_SIZE;
 939         }
 940
 941         *cmd = MI_BATCH_BUFFER_END;
 942         i915_gem_chipset_flush(i915);
 943
 944         i915_gem_object_unpin_map(obj);
 945
 946         batch = i915_vma_instance(obj, vma->vm, NULL);
 947         if (IS_ERR(batch)) {
 948                 err = PTR_ERR(batch);
 949                 goto err;
 950         }
 951
 952         err = i915_vma_pin(batch, 0, 0, PIN_USER);
 953         if (err)
 954                 goto err;
 955
 956         return batch;
 957
 958 err:
 959         i915_gem_object_put(obj);
 960
 961         return ERR_PTR(err);
 962 }
 963
 964 static int gpu_write(struct i915_vma *vma,
 965                      struct i915_gem_context *ctx,
 966                      struct intel_engine_cs *engine,
 967                      u32 dword,
 968                      u32 value)
 969 {
 970         struct i915_request *rq;
 971         struct i915_vma *batch;
 972         int err;
 973
 974         GEM_BUG_ON(!intel_engine_can_store_dword(engine));
 975
 976         err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 977         if (err)
 978                 return err;
 979
 980         batch = gpu_write_dw(vma, dword * sizeof(u32), value);
 981         if (IS_ERR(batch))
 982                 return PTR_ERR(batch);
 983
 984         rq = igt_request_alloc(ctx, engine);
 985         if (IS_ERR(rq)) {
 986                 err = PTR_ERR(rq);
 987                 goto err_batch;
 988         }
 989
 990         err = i915_vma_move_to_active(batch, rq, 0);
 991         if (err)
 992                 goto err_request;
 993
 994         i915_gem_object_set_active_reference(batch->obj);
 995
 996         err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
 997         if (err)
 998                 goto err_request;
 999
1000         err = engine->emit_bb_start(rq,
1001                                     batch->node.start, batch->node.size,
1002                                     0);
1003 err_request:
1004         if (err)
1005                 i915_request_skip(rq, err);
1006         i915_request_add(rq);
1007 err_batch:
1008         i915_vma_unpin(batch);
1009         i915_vma_close(batch);
1010
1011         return err;
1012 }
1013
1014 static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
1015 {
1016         unsigned int needs_flush;
1017         unsigned long n;
1018         int err;
1019
1020         err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
1021         if (err)
1022                 return err;
1023
1024         for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) {
1025                 u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n));
1026
1027                 if (needs_flush & CLFLUSH_BEFORE)
1028                         drm_clflush_virt_range(ptr, PAGE_SIZE);
1029
1030                 if (ptr[dword] != val) {
1031                         pr_err("n=%lu ptr[%u]=%u, val=%u\n",
1032                                n, dword, ptr[dword], val);
1033                         kunmap_atomic(ptr);
1034                         err = -EINVAL;
1035                         break;
1036                 }
1037
1038                 kunmap_atomic(ptr);
1039         }
1040
1041         i915_gem_obj_finish_shmem_access(obj);
1042
1043         return err;
1044 }
1045
1046 static int __igt_write_huge(struct i915_gem_context *ctx,
1047                             struct intel_engine_cs *engine,
1048                             struct drm_i915_gem_object *obj,
1049                             u64 size, u64 offset,
1050                             u32 dword, u32 val)
1051 {
1052         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1053         struct i915_address_space *vm =
1054                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1055         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1056         struct i915_vma *vma;
1057         int err;
1058
1059         vma = i915_vma_instance(obj, vm, NULL);
1060         if (IS_ERR(vma))
1061                 return PTR_ERR(vma);
1062
1063         err = i915_vma_unbind(vma);
1064         if (err)
1065                 goto out_vma_close;
1066
1067         err = i915_vma_pin(vma, size, 0, flags | offset);
1068         if (err) {
1069                 /*
1070                  * The ggtt may have some pages reserved so
1071                  * refrain from erroring out.
1072                  */
1073                 if (err == -ENOSPC && i915_is_ggtt(vm))
1074                         err = 0;
1075
1076                 goto out_vma_close;
1077         }
1078
1079         err = igt_check_page_sizes(vma);
1080         if (err)
1081                 goto out_vma_unpin;
1082
1083         err = gpu_write(vma, ctx, engine, dword, val);
1084         if (err) {
1085                 pr_err("gpu-write failed at offset=%llx\n", offset);
1086                 goto out_vma_unpin;
1087         }
1088
1089         err = cpu_check(obj, dword, val);
1090         if (err) {
1091                 pr_err("cpu-check failed at offset=%llx\n", offset);
1092                 goto out_vma_unpin;
1093         }
1094
1095 out_vma_unpin:
1096         i915_vma_unpin(vma);
1097 out_vma_close:
1098         i915_vma_destroy(vma);
1099
1100         return err;
1101 }
1102
1103 static int igt_write_huge(struct i915_gem_context *ctx,
1104                           struct drm_i915_gem_object *obj)
1105 {
1106         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1107         struct i915_address_space *vm =
1108                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1109         static struct intel_engine_cs *engines[I915_NUM_ENGINES];
1110         struct intel_engine_cs *engine;
1111         I915_RND_STATE(prng);
1112         IGT_TIMEOUT(end_time);
1113         unsigned int max_page_size;
1114         unsigned int id;
1115         u64 max;
1116         u64 num;
1117         u64 size;
1118         int *order;
1119         int i, n;
1120         int err = 0;
1121
1122         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1123
1124         size = obj->base.size;
1125         if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1126                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
1127
1128         max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
1129         max = div_u64((vm->total - size), max_page_size);
1130
1131         n = 0;
1132         for_each_engine(engine, i915, id) {
1133                 if (!intel_engine_can_store_dword(engine)) {
1134                         pr_info("store-dword-imm not supported on engine=%u\n",
1135                                 id);
1136                         continue;
1137                 }
1138                 engines[n++] = engine;
1139         }
1140
1141         if (!n)
1142                 return 0;
1143
1144         /*
1145          * To keep things interesting when alternating between engines in our
1146          * randomized order, lets also make feeding to the same engine a few
1147          * times in succession a possibility by enlarging the permutation array.
1148          */
1149         order = i915_random_order(n * I915_NUM_ENGINES, &prng);
1150         if (!order)
1151                 return -ENOMEM;
1152
1153         /*
1154          * Try various offsets in an ascending/descending fashion until we
1155          * timeout -- we want to avoid issues hidden by effectively always using
1156          * offset = 0.
1157          */
1158         i = 0;
1159         for_each_prime_number_from(num, 0, max) {
1160                 u64 offset_low = num * max_page_size;
1161                 u64 offset_high = (max - num) * max_page_size;
1162                 u32 dword = offset_in_page(num) / 4;
1163
1164                 engine = engines[order[i] % n];
1165                 i = (i + 1) % (n * I915_NUM_ENGINES);
1166
1167                 /*
1168                  * In order to utilize 64K pages we need to both pad the vma
1169                  * size and ensure the vma offset is at the start of the pt
1170                  * boundary, however to improve coverage we opt for testing both
1171                  * aligned and unaligned offsets.
1172                  */
1173                 if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
1174                         offset_low = round_down(offset_low,
1175                                                 I915_GTT_PAGE_SIZE_2M);
1176
1177                 err = __igt_write_huge(ctx, engine, obj, size, offset_low,
1178                                        dword, num + 1);
1179                 if (err)
1180                         break;
1181
1182                 err = __igt_write_huge(ctx, engine, obj, size, offset_high,
1183                                        dword, num + 1);
1184                 if (err)
1185                         break;
1186
1187                 if (igt_timeout(end_time,
1188                                 "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
1189                                 __func__, engine->id, offset_low, offset_high,
1190                                 max_page_size))
1191                         break;
1192         }
1193
1194         kfree(order);
1195
1196         return err;
1197 }
1198
1199 static int igt_ppgtt_exhaust_huge(void *arg)
1200 {
1201         struct i915_gem_context *ctx = arg;
1202         struct drm_i915_private *i915 = ctx->i915;
1203         unsigned long supported = INTEL_INFO(i915)->page_sizes;
1204         static unsigned int pages[ARRAY_SIZE(page_sizes)];
1205         struct drm_i915_gem_object *obj;
1206         unsigned int size_mask;
1207         unsigned int page_mask;
1208         int n, i;
1209         int err = -ENODEV;
1210
1211         if (supported == I915_GTT_PAGE_SIZE_4K)
1212                 return 0;
1213
1214         /*
1215          * Sanity check creating objects with a varying mix of page sizes --
1216          * ensuring that our writes lands in the right place.
1217          */
1218
1219         n = 0;
1220         for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1)
1221                 pages[n++] = BIT(i);
1222
1223         for (size_mask = 2; size_mask < BIT(n); size_mask++) {
1224                 unsigned int size = 0;
1225
1226                 for (i = 0; i < n; i++) {
1227                         if (size_mask & BIT(i))
1228                                 size |= pages[i];
1229                 }
1230
1231                 /*
1232                  * For our page mask we want to enumerate all the page-size
1233                  * combinations which will fit into our chosen object size.
1234                  */
1235                 for (page_mask = 2; page_mask <= size_mask; page_mask++) {
1236                         unsigned int page_sizes = 0;
1237
1238                         for (i = 0; i < n; i++) {
1239                                 if (page_mask & BIT(i))
1240                                         page_sizes |= pages[i];
1241                         }
1242
1243                         /*
1244                          * Ensure that we can actually fill the given object
1245                          * with our chosen page mask.
1246                          */
1247                         if (!IS_ALIGNED(size, BIT(__ffs(page_sizes))))
1248                                 continue;
1249
1250                         obj = huge_pages_object(i915, size, page_sizes);
1251                         if (IS_ERR(obj)) {
1252                                 err = PTR_ERR(obj);
1253                                 goto out_device;
1254                         }
1255
1256                         err = i915_gem_object_pin_pages(obj);
1257                         if (err) {
1258                                 i915_gem_object_put(obj);
1259
1260                                 if (err == -ENOMEM) {
1261                                         pr_info("unable to get pages, size=%u, pages=%u\n",
1262                                                 size, page_sizes);
1263                                         err = 0;
1264                                         break;
1265                                 }
1266
1267                                 pr_err("pin_pages failed, size=%u, pages=%u\n",
1268                                        size_mask, page_mask);
1269
1270                                 goto out_device;
1271                         }
1272
1273                         /* Force the page-size for the gtt insertion */
1274                         obj->mm.page_sizes.sg = page_sizes;
1275
1276                         err = igt_write_huge(ctx, obj);
1277                         if (err) {
1278                                 pr_err("exhaust write-huge failed with size=%u\n",
1279                                        size);
1280                                 goto out_unpin;
1281                         }
1282
1283                         i915_gem_object_unpin_pages(obj);
1284                         __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1285                         i915_gem_object_put(obj);
1286                 }
1287         }
1288
1289         goto out_device;
1290
1291 out_unpin:
1292         i915_gem_object_unpin_pages(obj);
1293         i915_gem_object_put(obj);
1294 out_device:
1295         mkwrite_device_info(i915)->page_sizes = supported;
1296
1297         return err;
1298 }
1299
1300 static int igt_ppgtt_internal_huge(void *arg)
1301 {
1302         struct i915_gem_context *ctx = arg;
1303         struct drm_i915_private *i915 = ctx->i915;
1304         struct drm_i915_gem_object *obj;
1305         static const unsigned int sizes[] = {
1306                 SZ_64K,
1307                 SZ_128K,
1308                 SZ_256K,
1309                 SZ_512K,
1310                 SZ_1M,
1311                 SZ_2M,
1312         };
1313         int i;
1314         int err;
1315
1316         /*
1317          * Sanity check that the HW uses huge pages correctly through internal
1318          * -- ensure that our writes land in the right place.
1319          */
1320
1321         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1322                 unsigned int size = sizes[i];
1323
1324                 obj = i915_gem_object_create_internal(i915, size);
1325                 if (IS_ERR(obj))
1326                         return PTR_ERR(obj);
1327
1328                 err = i915_gem_object_pin_pages(obj);
1329                 if (err)
1330                         goto out_put;
1331
1332                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) {
1333                         pr_info("internal unable to allocate huge-page(s) with size=%u\n",
1334                                 size);
1335                         goto out_unpin;
1336                 }
1337
1338                 err = igt_write_huge(ctx, obj);
1339                 if (err) {
1340                         pr_err("internal write-huge failed with size=%u\n",
1341                                size);
1342                         goto out_unpin;
1343                 }
1344
1345                 i915_gem_object_unpin_pages(obj);
1346                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1347                 i915_gem_object_put(obj);
1348         }
1349
1350         return 0;
1351
1352 out_unpin:
1353         i915_gem_object_unpin_pages(obj);
1354 out_put:
1355         i915_gem_object_put(obj);
1356
1357         return err;
1358 }
1359
1360 static inline bool igt_can_allocate_thp(struct drm_i915_private *i915)
1361 {
1362         return i915->mm.gemfs && has_transparent_hugepage();
1363 }
1364
1365 static int igt_ppgtt_gemfs_huge(void *arg)
1366 {
1367         struct i915_gem_context *ctx = arg;
1368         struct drm_i915_private *i915 = ctx->i915;
1369         struct drm_i915_gem_object *obj;
1370         static const unsigned int sizes[] = {
1371                 SZ_2M,
1372                 SZ_4M,
1373                 SZ_8M,
1374                 SZ_16M,
1375                 SZ_32M,
1376         };
1377         int i;
1378         int err;
1379
1380         /*
1381          * Sanity check that the HW uses huge pages correctly through gemfs --
1382          * ensure that our writes land in the right place.
1383          */
1384
1385         if (!igt_can_allocate_thp(i915)) {
1386                 pr_info("missing THP support, skipping\n");
1387                 return 0;
1388         }
1389
1390         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
1391                 unsigned int size = sizes[i];
1392
1393                 obj = i915_gem_object_create(i915, size);
1394                 if (IS_ERR(obj))
1395                         return PTR_ERR(obj);
1396
1397                 err = i915_gem_object_pin_pages(obj);
1398                 if (err)
1399                         goto out_put;
1400
1401                 if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1402                         pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n",
1403                                 size);
1404                         goto out_unpin;
1405                 }
1406
1407                 err = igt_write_huge(ctx, obj);
1408                 if (err) {
1409                         pr_err("gemfs write-huge failed with size=%u\n",
1410                                size);
1411                         goto out_unpin;
1412                 }
1413
1414                 i915_gem_object_unpin_pages(obj);
1415                 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
1416                 i915_gem_object_put(obj);
1417         }
1418
1419         return 0;
1420
1421 out_unpin:
1422         i915_gem_object_unpin_pages(obj);
1423 out_put:
1424         i915_gem_object_put(obj);
1425
1426         return err;
1427 }
1428
1429 static int igt_ppgtt_pin_update(void *arg)
1430 {
1431         struct i915_gem_context *ctx = arg;
1432         struct drm_i915_private *dev_priv = ctx->i915;
1433         unsigned long supported = INTEL_INFO(dev_priv)->page_sizes;
1434         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
1435         struct drm_i915_gem_object *obj;
1436         struct i915_vma *vma;
1437         unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
1438         int first, last;
1439         int err;
1440
1441         /*
1442          * Make sure there's no funny business when doing a PIN_UPDATE -- in the
1443          * past we had a subtle issue with being able to incorrectly do multiple
1444          * alloc va ranges on the same object when doing a PIN_UPDATE, which
1445          * resulted in some pretty nasty bugs, though only when using
1446          * huge-gtt-pages.
1447          */
1448
1449         if (!ppgtt || !i915_vm_is_4lvl(&ppgtt->vm)) {
1450                 pr_info("48b PPGTT not supported, skipping\n");
1451                 return 0;
1452         }
1453
1454         first = ilog2(I915_GTT_PAGE_SIZE_64K);
1455         last = ilog2(I915_GTT_PAGE_SIZE_2M);
1456
1457         for_each_set_bit_from(first, &supported, last + 1) {
1458                 unsigned int page_size = BIT(first);
1459
1460                 obj = i915_gem_object_create_internal(dev_priv, page_size);
1461                 if (IS_ERR(obj))
1462                         return PTR_ERR(obj);
1463
1464                 vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
1465                 if (IS_ERR(vma)) {
1466                         err = PTR_ERR(vma);
1467                         goto out_put;
1468                 }
1469
1470                 err = i915_vma_pin(vma, SZ_2M, 0, flags);
1471                 if (err)
1472                         goto out_close;
1473
1474                 if (vma->page_sizes.sg < page_size) {
1475                         pr_info("Unable to allocate page-size %x, finishing test early\n",
1476                                 page_size);
1477                         goto out_unpin;
1478                 }
1479
1480                 err = igt_check_page_sizes(vma);
1481                 if (err)
1482                         goto out_unpin;
1483
1484                 if (vma->page_sizes.gtt != page_size) {
1485                         dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0);
1486
1487                         /*
1488                          * The only valid reason for this to ever fail would be
1489                          * if the dma-mapper screwed us over when we did the
1490                          * dma_map_sg(), since it has the final say over the dma
1491                          * address.
1492                          */
1493                         if (IS_ALIGNED(addr, page_size)) {
1494                                 pr_err("page_sizes.gtt=%u, expected=%u\n",
1495                                        vma->page_sizes.gtt, page_size);
1496                                 err = -EINVAL;
1497                         } else {
1498                                 pr_info("dma address misaligned, finishing test early\n");
1499                         }
1500
1501                         goto out_unpin;
1502                 }
1503
1504                 err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE);
1505                 if (err)
1506                         goto out_unpin;
1507
1508                 i915_vma_unpin(vma);
1509                 i915_vma_close(vma);
1510
1511                 i915_gem_object_put(obj);
1512         }
1513
1514         obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE);
1515         if (IS_ERR(obj))
1516                 return PTR_ERR(obj);
1517
1518         vma = i915_vma_instance(obj, &ppgtt->vm, NULL);
1519         if (IS_ERR(vma)) {
1520                 err = PTR_ERR(vma);
1521                 goto out_put;
1522         }
1523
1524         err = i915_vma_pin(vma, 0, 0, flags);
1525         if (err)
1526                 goto out_close;
1527
1528         /*
1529          * Make sure we don't end up with something like where the pde is still
1530          * pointing to the 2M page, and the pt we just filled-in is dangling --
1531          * we can check this by writing to the first page where it would then
1532          * land in the now stale 2M page.
1533          */
1534
1535         err = gpu_write(vma, ctx, dev_priv->engine[RCS0], 0, 0xdeadbeaf);
1536         if (err)
1537                 goto out_unpin;
1538
1539         err = cpu_check(obj, 0, 0xdeadbeaf);
1540
1541 out_unpin:
1542         i915_vma_unpin(vma);
1543 out_close:
1544         i915_vma_close(vma);
1545 out_put:
1546         i915_gem_object_put(obj);
1547
1548         return err;
1549 }
1550
1551 static int igt_tmpfs_fallback(void *arg)
1552 {
1553         struct i915_gem_context *ctx = arg;
1554         struct drm_i915_private *i915 = ctx->i915;
1555         struct vfsmount *gemfs = i915->mm.gemfs;
1556         struct i915_address_space *vm =
1557                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1558         struct drm_i915_gem_object *obj;
1559         struct i915_vma *vma;
1560         u32 *vaddr;
1561         int err = 0;
1562
1563         /*
1564          * Make sure that we don't burst into a ball of flames upon falling back
1565          * to tmpfs, which we rely on if on the off-chance we encouter a failure
1566          * when setting up gemfs.
1567          */
1568
1569         i915->mm.gemfs = NULL;
1570
1571         obj = i915_gem_object_create(i915, PAGE_SIZE);
1572         if (IS_ERR(obj)) {
1573                 err = PTR_ERR(obj);
1574                 goto out_restore;
1575         }
1576
1577         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1578         if (IS_ERR(vaddr)) {
1579                 err = PTR_ERR(vaddr);
1580                 goto out_put;
1581         }
1582         *vaddr = 0xdeadbeaf;
1583
1584         __i915_gem_object_flush_map(obj, 0, 64);
1585         i915_gem_object_unpin_map(obj);
1586
1587         vma = i915_vma_instance(obj, vm, NULL);
1588         if (IS_ERR(vma)) {
1589                 err = PTR_ERR(vma);
1590                 goto out_put;
1591         }
1592
1593         err = i915_vma_pin(vma, 0, 0, PIN_USER);
1594         if (err)
1595                 goto out_close;
1596
1597         err = igt_check_page_sizes(vma);
1598
1599         i915_vma_unpin(vma);
1600 out_close:
1601         i915_vma_close(vma);
1602 out_put:
1603         i915_gem_object_put(obj);
1604 out_restore:
1605         i915->mm.gemfs = gemfs;
1606
1607         return err;
1608 }
1609
1610 static int igt_shrink_thp(void *arg)
1611 {
1612         struct i915_gem_context *ctx = arg;
1613         struct drm_i915_private *i915 = ctx->i915;
1614         struct i915_address_space *vm =
1615                 ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm;
1616         struct drm_i915_gem_object *obj;
1617         struct i915_vma *vma;
1618         unsigned int flags = PIN_USER;
1619         int err;
1620
1621         /*
1622          * Sanity check shrinking huge-paged object -- make sure nothing blows
1623          * up.
1624          */
1625
1626         if (!igt_can_allocate_thp(i915)) {
1627                 pr_info("missing THP support, skipping\n");
1628                 return 0;
1629         }
1630
1631         obj = i915_gem_object_create(i915, SZ_2M);
1632         if (IS_ERR(obj))
1633                 return PTR_ERR(obj);
1634
1635         vma = i915_vma_instance(obj, vm, NULL);
1636         if (IS_ERR(vma)) {
1637                 err = PTR_ERR(vma);
1638                 goto out_put;
1639         }
1640
1641         err = i915_vma_pin(vma, 0, 0, flags);
1642         if (err)
1643                 goto out_close;
1644
1645         if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) {
1646                 pr_info("failed to allocate THP, finishing test early\n");
1647                 goto out_unpin;
1648         }
1649
1650         err = igt_check_page_sizes(vma);
1651         if (err)
1652                 goto out_unpin;
1653
1654         err = gpu_write(vma, ctx, i915->engine[RCS0], 0, 0xdeadbeaf);
1655         if (err)
1656                 goto out_unpin;
1657
1658         i915_vma_unpin(vma);
1659
1660         /*
1661          * Now that the pages are *unpinned* shrink-all should invoke
1662          * shmem to truncate our pages.
1663          */
1664         i915_gem_shrink_all(i915);
1665         if (i915_gem_object_has_pages(obj)) {
1666                 pr_err("shrink-all didn't truncate the pages\n");
1667                 err = -EINVAL;
1668                 goto out_close;
1669         }
1670
1671         if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) {
1672                 pr_err("residual page-size bits left\n");
1673                 err = -EINVAL;
1674                 goto out_close;
1675         }
1676
1677         err = i915_vma_pin(vma, 0, 0, flags);
1678         if (err)
1679                 goto out_close;
1680
1681         err = cpu_check(obj, 0, 0xdeadbeaf);
1682
1683 out_unpin:
1684         i915_vma_unpin(vma);
1685 out_close:
1686         i915_vma_close(vma);
1687 out_put:
1688         i915_gem_object_put(obj);
1689
1690         return err;
1691 }
1692
1693 int i915_gem_huge_page_mock_selftests(void)
1694 {
1695         static const struct i915_subtest tests[] = {
1696                 SUBTEST(igt_mock_exhaust_device_supported_pages),
1697                 SUBTEST(igt_mock_ppgtt_misaligned_dma),
1698                 SUBTEST(igt_mock_ppgtt_huge_fill),
1699                 SUBTEST(igt_mock_ppgtt_64K),
1700         };
1701         struct drm_i915_private *dev_priv;
1702         struct i915_hw_ppgtt *ppgtt;
1703         int err;
1704
1705         dev_priv = mock_gem_device();
1706         if (!dev_priv)
1707                 return -ENOMEM;
1708
1709         /* Pretend to be a device which supports the 48b PPGTT */
1710         mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
1711         mkwrite_device_info(dev_priv)->ppgtt_size = 48;
1712
1713         mutex_lock(&dev_priv->drm.struct_mutex);
1714         ppgtt = i915_ppgtt_create(dev_priv);
1715         if (IS_ERR(ppgtt)) {
1716                 err = PTR_ERR(ppgtt);
1717                 goto out_unlock;
1718         }
1719
1720         if (!i915_vm_is_4lvl(&ppgtt->vm)) {
1721                 pr_err("failed to create 48b PPGTT\n");
1722                 err = -EINVAL;
1723                 goto out_close;
1724         }
1725
1726         /* If we were ever hit this then it's time to mock the 64K scratch */
1727         if (!i915_vm_has_scratch_64K(&ppgtt->vm)) {
1728                 pr_err("PPGTT missing 64K scratch page\n");
1729                 err = -EINVAL;
1730                 goto out_close;
1731         }
1732
1733         err = i915_subtests(tests, ppgtt);
1734
1735 out_close:
1736         i915_ppgtt_put(ppgtt);
1737
1738 out_unlock:
1739         mutex_unlock(&dev_priv->drm.struct_mutex);
1740         drm_dev_put(&dev_priv->drm);
1741
1742         return err;
1743 }
1744
1745 int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
1746 {
1747         static const struct i915_subtest tests[] = {
1748                 SUBTEST(igt_shrink_thp),
1749                 SUBTEST(igt_ppgtt_pin_update),
1750                 SUBTEST(igt_tmpfs_fallback),
1751                 SUBTEST(igt_ppgtt_exhaust_huge),
1752                 SUBTEST(igt_ppgtt_gemfs_huge),
1753                 SUBTEST(igt_ppgtt_internal_huge),
1754         };
1755         struct drm_file *file;
1756         struct i915_gem_context *ctx;
1757         intel_wakeref_t wakeref;
1758         int err;
1759
1760         if (!HAS_PPGTT(dev_priv)) {
1761                 pr_info("PPGTT not supported, skipping live-selftests\n");
1762                 return 0;
1763         }
1764
1765         if (i915_terminally_wedged(dev_priv))
1766                 return 0;
1767
1768         file = mock_file(dev_priv);
1769         if (IS_ERR(file))
1770                 return PTR_ERR(file);
1771
1772         mutex_lock(&dev_priv->drm.struct_mutex);
1773         wakeref = intel_runtime_pm_get(dev_priv);
1774
1775         ctx = live_context(dev_priv, file);
1776         if (IS_ERR(ctx)) {
1777                 err = PTR_ERR(ctx);
1778                 goto out_unlock;
1779         }
1780
1781         if (ctx->ppgtt)
1782                 ctx->ppgtt->vm.scrub_64K = true;
1783
1784         err = i915_subtests(tests, ctx);
1785
1786 out_unlock:
1787         intel_runtime_pm_put(dev_priv, wakeref);
1788         mutex_unlock(&dev_priv->drm.struct_mutex);
1789
1790         mock_file_free(dev_priv, file);
1791
1792         return err;
1793 }