drivers/gpu/drm/i915/gem/i915_gem_stolen.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2008-2012 Intel Corporation
   5  */
   6
   7 #include <linux/errno.h>
   8 #include <linux/mutex.h>
   9
  10 #include <drm/drm_mm.h>
  11 #include <drm/i915_drm.h>
  12
  13 #include "gem/i915_gem_lmem.h"
  14 #include "gem/i915_gem_region.h"
  15 #include "i915_drv.h"
  16 #include "i915_gem_stolen.h"
  17 #include "i915_vgpu.h"
  18
  19 /*
  20  * The BIOS typically reserves some of the system's memory for the exclusive
  21  * use of the integrated graphics. This memory is no longer available for
  22  * use by the OS and so the user finds that his system has less memory
  23  * available than he put in. We refer to this memory as stolen.
  24  *
  25  * The BIOS will allocate its framebuffer from the stolen memory. Our
  26  * goal is try to reuse that object for our own fbcon which must always
  27  * be available for panics. Anything else we can reuse the stolen memory
  28  * for is a boon.
  29  */
  30
  31 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *i915,
  32                                          struct drm_mm_node *node, u64 size,
  33                                          unsigned alignment, u64 start, u64 end)
  34 {
  35         int ret;
  36
  37         if (!drm_mm_initialized(&i915->mm.stolen))
  38                 return -ENODEV;
  39
  40         /* WaSkipStolenMemoryFirstPage:bdw+ */
  41         if (GRAPHICS_VER(i915) >= 8 && start < 4096)
  42                 start = 4096;
  43
  44         mutex_lock(&i915->mm.stolen_lock);
  45         ret = drm_mm_insert_node_in_range(&i915->mm.stolen, node,
  46                                           size, alignment, 0,
  47                                           start, end, DRM_MM_INSERT_BEST);
  48         mutex_unlock(&i915->mm.stolen_lock);
  49
  50         return ret;
  51 }
  52
  53 int i915_gem_stolen_insert_node(struct drm_i915_private *i915,
  54                                 struct drm_mm_node *node, u64 size,
  55                                 unsigned alignment)
  56 {
  57         return i915_gem_stolen_insert_node_in_range(i915, node,
  58                                                     size, alignment,
  59                                                     I915_GEM_STOLEN_BIAS,
  60                                                     U64_MAX);
  61 }
  62
  63 void i915_gem_stolen_remove_node(struct drm_i915_private *i915,
  64                                  struct drm_mm_node *node)
  65 {
  66         mutex_lock(&i915->mm.stolen_lock);
  67         drm_mm_remove_node(node);
  68         mutex_unlock(&i915->mm.stolen_lock);
  69 }
  70
  71 static int i915_adjust_stolen(struct drm_i915_private *i915,
  72                               struct resource *dsm)
  73 {
  74         struct i915_ggtt *ggtt = &i915->ggtt;
  75         struct intel_uncore *uncore = ggtt->vm.gt->uncore;
  76         struct resource *r;
  77
  78         if (dsm->start == 0 || dsm->end <= dsm->start)
  79                 return -EINVAL;
  80
  81         /*
  82          * TODO: We have yet too encounter the case where the GTT wasn't at the
  83          * end of stolen. With that assumption we could simplify this.
  84          */
  85
  86         /* Make sure we don't clobber the GTT if it's within stolen memory */
  87         if (GRAPHICS_VER(i915) <= 4 &&
  88             !IS_G33(i915) && !IS_PINEVIEW(i915) && !IS_G4X(i915)) {
  89                 struct resource stolen[2] = {*dsm, *dsm};
  90                 struct resource ggtt_res;
  91                 resource_size_t ggtt_start;
  92
  93                 ggtt_start = intel_uncore_read(uncore, PGTBL_CTL);
  94                 if (GRAPHICS_VER(i915) == 4)
  95                         ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
  96                                      (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
  97                 else
  98                         ggtt_start &= PGTBL_ADDRESS_LO_MASK;
  99
 100                 ggtt_res =
 101                         (struct resource) DEFINE_RES_MEM(ggtt_start,
 102                                                          ggtt_total_entries(ggtt) * 4);
 103
 104                 if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
 105                         stolen[0].end = ggtt_res.start;
 106                 if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
 107                         stolen[1].start = ggtt_res.end;
 108
 109                 /* Pick the larger of the two chunks */
 110                 if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
 111                         *dsm = stolen[0];
 112                 else
 113                         *dsm = stolen[1];
 114
 115                 if (stolen[0].start != stolen[1].start ||
 116                     stolen[0].end != stolen[1].end) {
 117                         drm_dbg(&i915->drm,
 118                                 "GTT within stolen memory at %pR\n",
 119                                 &ggtt_res);
 120                         drm_dbg(&i915->drm, "Stolen memory adjusted to %pR\n",
 121                                 dsm);
 122                 }
 123         }
 124
 125         /*
 126          * With stolen lmem, we don't need to check if the address range
 127          * overlaps with the non-stolen system memory range, since lmem is local
 128          * to the gpu.
 129          */
 130         if (HAS_LMEM(i915))
 131                 return 0;
 132
 133         /*
 134          * Verify that nothing else uses this physical address. Stolen
 135          * memory should be reserved by the BIOS and hidden from the
 136          * kernel. So if the region is already marked as busy, something
 137          * is seriously wrong.
 138          */
 139         r = devm_request_mem_region(i915->drm.dev, dsm->start,
 140                                     resource_size(dsm),
 141                                     "Graphics Stolen Memory");
 142         if (r == NULL) {
 143                 /*
 144                  * One more attempt but this time requesting region from
 145                  * start + 1, as we have seen that this resolves the region
 146                  * conflict with the PCI Bus.
 147                  * This is a BIOS w/a: Some BIOS wrap stolen in the root
 148                  * PCI bus, but have an off-by-one error. Hence retry the
 149                  * reservation starting from 1 instead of 0.
 150                  * There's also BIOS with off-by-one on the other end.
 151                  */
 152                 r = devm_request_mem_region(i915->drm.dev, dsm->start + 1,
 153                                             resource_size(dsm) - 2,
 154                                             "Graphics Stolen Memory");
 155                 /*
 156                  * GEN3 firmware likes to smash pci bridges into the stolen
 157                  * range. Apparently this works.
 158                  */
 159                 if (!r && GRAPHICS_VER(i915) != 3) {
 160                         drm_err(&i915->drm,
 161                                 "conflict detected with stolen region: %pR\n",
 162                                 dsm);
 163
 164                         return -EBUSY;
 165                 }
 166         }
 167
 168         return 0;
 169 }
 170
 171 static void i915_gem_cleanup_stolen(struct drm_i915_private *i915)
 172 {
 173         if (!drm_mm_initialized(&i915->mm.stolen))
 174                 return;
 175
 176         drm_mm_takedown(&i915->mm.stolen);
 177 }
 178
 179 static void g4x_get_stolen_reserved(struct drm_i915_private *i915,
 180                                     struct intel_uncore *uncore,
 181                                     resource_size_t *base,
 182                                     resource_size_t *size)
 183 {
 184         u32 reg_val = intel_uncore_read(uncore,
 185                                         IS_GM45(i915) ?
 186                                         CTG_STOLEN_RESERVED :
 187                                         ELK_STOLEN_RESERVED);
 188         resource_size_t stolen_top = i915->dsm.end + 1;
 189
 190         drm_dbg(&i915->drm, "%s_STOLEN_RESERVED = %08x\n",
 191                 IS_GM45(i915) ? "CTG" : "ELK", reg_val);
 192
 193         if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
 194                 return;
 195
 196         /*
 197          * Whether ILK really reuses the ELK register for this is unclear.
 198          * Let's see if we catch anyone with this supposedly enabled on ILK.
 199          */
 200         drm_WARN(&i915->drm, GRAPHICS_VER(i915) == 5,
 201                  "ILK stolen reserved found? 0x%08x\n",
 202                  reg_val);
 203
 204         if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
 205                 return;
 206
 207         *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 208         drm_WARN_ON(&i915->drm,
 209                     (reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 210
 211         *size = stolen_top - *base;
 212 }
 213
 214 static void gen6_get_stolen_reserved(struct drm_i915_private *i915,
 215                                      struct intel_uncore *uncore,
 216                                      resource_size_t *base,
 217                                      resource_size_t *size)
 218 {
 219         u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 220
 221         drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 222
 223         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 224                 return;
 225
 226         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 227
 228         switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
 229         case GEN6_STOLEN_RESERVED_1M:
 230                 *size = 1024 * 1024;
 231                 break;
 232         case GEN6_STOLEN_RESERVED_512K:
 233                 *size = 512 * 1024;
 234                 break;
 235         case GEN6_STOLEN_RESERVED_256K:
 236                 *size = 256 * 1024;
 237                 break;
 238         case GEN6_STOLEN_RESERVED_128K:
 239                 *size = 128 * 1024;
 240                 break;
 241         default:
 242                 *size = 1024 * 1024;
 243                 MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK);
 244         }
 245 }
 246
 247 static void vlv_get_stolen_reserved(struct drm_i915_private *i915,
 248                                     struct intel_uncore *uncore,
 249                                     resource_size_t *base,
 250                                     resource_size_t *size)
 251 {
 252         u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 253         resource_size_t stolen_top = i915->dsm.end + 1;
 254
 255         drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 256
 257         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 258                 return;
 259
 260         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 261         default:
 262                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 263                 fallthrough;
 264         case GEN7_STOLEN_RESERVED_1M:
 265                 *size = 1024 * 1024;
 266                 break;
 267         }
 268
 269         /*
 270          * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
 271          * reserved location as (top - size).
 272          */
 273         *base = stolen_top - *size;
 274 }
 275
 276 static void gen7_get_stolen_reserved(struct drm_i915_private *i915,
 277                                      struct intel_uncore *uncore,
 278                                      resource_size_t *base,
 279                                      resource_size_t *size)
 280 {
 281         u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 282
 283         drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 284
 285         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 286                 return;
 287
 288         *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 289
 290         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 291         case GEN7_STOLEN_RESERVED_1M:
 292                 *size = 1024 * 1024;
 293                 break;
 294         case GEN7_STOLEN_RESERVED_256K:
 295                 *size = 256 * 1024;
 296                 break;
 297         default:
 298                 *size = 1024 * 1024;
 299                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 300         }
 301 }
 302
 303 static void chv_get_stolen_reserved(struct drm_i915_private *i915,
 304                                     struct intel_uncore *uncore,
 305                                     resource_size_t *base,
 306                                     resource_size_t *size)
 307 {
 308         u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 309
 310         drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 311
 312         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 313                 return;
 314
 315         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 316
 317         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 318         case GEN8_STOLEN_RESERVED_1M:
 319                 *size = 1024 * 1024;
 320                 break;
 321         case GEN8_STOLEN_RESERVED_2M:
 322                 *size = 2 * 1024 * 1024;
 323                 break;
 324         case GEN8_STOLEN_RESERVED_4M:
 325                 *size = 4 * 1024 * 1024;
 326                 break;
 327         case GEN8_STOLEN_RESERVED_8M:
 328                 *size = 8 * 1024 * 1024;
 329                 break;
 330         default:
 331                 *size = 8 * 1024 * 1024;
 332                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 333         }
 334 }
 335
 336 static void bdw_get_stolen_reserved(struct drm_i915_private *i915,
 337                                     struct intel_uncore *uncore,
 338                                     resource_size_t *base,
 339                                     resource_size_t *size)
 340 {
 341         u32 reg_val = intel_uncore_read(uncore, GEN6_STOLEN_RESERVED);
 342         resource_size_t stolen_top = i915->dsm.end + 1;
 343
 344         drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 345
 346         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 347                 return;
 348
 349         if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
 350                 return;
 351
 352         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 353         *size = stolen_top - *base;
 354 }
 355
 356 static void icl_get_stolen_reserved(struct drm_i915_private *i915,
 357                                     struct intel_uncore *uncore,
 358                                     resource_size_t *base,
 359                                     resource_size_t *size)
 360 {
 361         u64 reg_val = intel_uncore_read64(uncore, GEN6_STOLEN_RESERVED);
 362
 363         drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);
 364
 365         *base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;
 366
 367         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 368         case GEN8_STOLEN_RESERVED_1M:
 369                 *size = 1024 * 1024;
 370                 break;
 371         case GEN8_STOLEN_RESERVED_2M:
 372                 *size = 2 * 1024 * 1024;
 373                 break;
 374         case GEN8_STOLEN_RESERVED_4M:
 375                 *size = 4 * 1024 * 1024;
 376                 break;
 377         case GEN8_STOLEN_RESERVED_8M:
 378                 *size = 8 * 1024 * 1024;
 379                 break;
 380         default:
 381                 *size = 8 * 1024 * 1024;
 382                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 383         }
 384 }
 385
 386 static int i915_gem_init_stolen(struct intel_memory_region *mem)
 387 {
 388         struct drm_i915_private *i915 = mem->i915;
 389         struct intel_uncore *uncore = &i915->uncore;
 390         resource_size_t reserved_base, stolen_top;
 391         resource_size_t reserved_total, reserved_size;
 392
 393         mutex_init(&i915->mm.stolen_lock);
 394
 395         if (intel_vgpu_active(i915)) {
 396                 drm_notice(&i915->drm,
 397                            "%s, disabling use of stolen memory\n",
 398                            "iGVT-g active");
 399                 return 0;
 400         }
 401
 402         if (intel_vtd_active() && GRAPHICS_VER(i915) < 8) {
 403                 drm_notice(&i915->drm,
 404                            "%s, disabling use of stolen memory\n",
 405                            "DMAR active");
 406                 return 0;
 407         }
 408
 409         if (resource_size(&mem->region) == 0)
 410                 return 0;
 411
 412         i915->dsm = mem->region;
 413
 414         if (i915_adjust_stolen(i915, &i915->dsm))
 415                 return 0;
 416
 417         GEM_BUG_ON(i915->dsm.start == 0);
 418         GEM_BUG_ON(i915->dsm.end <= i915->dsm.start);
 419
 420         stolen_top = i915->dsm.end + 1;
 421         reserved_base = stolen_top;
 422         reserved_size = 0;
 423
 424         switch (GRAPHICS_VER(i915)) {
 425         case 2:
 426         case 3:
 427                 break;
 428         case 4:
 429                 if (!IS_G4X(i915))
 430                         break;
 431                 fallthrough;
 432         case 5:
 433                 g4x_get_stolen_reserved(i915, uncore,
 434                                         &reserved_base, &reserved_size);
 435                 break;
 436         case 6:
 437                 gen6_get_stolen_reserved(i915, uncore,
 438                                          &reserved_base, &reserved_size);
 439                 break;
 440         case 7:
 441                 if (IS_VALLEYVIEW(i915))
 442                         vlv_get_stolen_reserved(i915, uncore,
 443                                                 &reserved_base, &reserved_size);
 444                 else
 445                         gen7_get_stolen_reserved(i915, uncore,
 446                                                  &reserved_base, &reserved_size);
 447                 break;
 448         case 8:
 449         case 9:
 450         case 10:
 451                 if (IS_LP(i915))
 452                         chv_get_stolen_reserved(i915, uncore,
 453                                                 &reserved_base, &reserved_size);
 454                 else
 455                         bdw_get_stolen_reserved(i915, uncore,
 456                                                 &reserved_base, &reserved_size);
 457                 break;
 458         default:
 459                 MISSING_CASE(GRAPHICS_VER(i915));
 460                 fallthrough;
 461         case 11:
 462         case 12:
 463                 icl_get_stolen_reserved(i915, uncore,
 464                                         &reserved_base,
 465                                         &reserved_size);
 466                 break;
 467         }
 468
 469         /*
 470          * Our expectation is that the reserved space is at the top of the
 471          * stolen region and *never* at the bottom. If we see !reserved_base,
 472          * it likely means we failed to read the registers correctly.
 473          */
 474         if (!reserved_base) {
 475                 drm_err(&i915->drm,
 476                         "inconsistent reservation %pa + %pa; ignoring\n",
 477                         &reserved_base, &reserved_size);
 478                 reserved_base = stolen_top;
 479                 reserved_size = 0;
 480         }
 481
 482         i915->dsm_reserved =
 483                 (struct resource)DEFINE_RES_MEM(reserved_base, reserved_size);
 484
 485         if (!resource_contains(&i915->dsm, &i915->dsm_reserved)) {
 486                 drm_err(&i915->drm,
 487                         "Stolen reserved area %pR outside stolen memory %pR\n",
 488                         &i915->dsm_reserved, &i915->dsm);
 489                 return 0;
 490         }
 491
 492         /* It is possible for the reserved area to end before the end of stolen
 493          * memory, so just consider the start. */
 494         reserved_total = stolen_top - reserved_base;
 495
 496         drm_dbg(&i915->drm,
 497                 "Memory reserved for graphics device: %lluK, usable: %lluK\n",
 498                 (u64)resource_size(&i915->dsm) >> 10,
 499                 ((u64)resource_size(&i915->dsm) - reserved_total) >> 10);
 500
 501         i915->stolen_usable_size =
 502                 resource_size(&i915->dsm) - reserved_total;
 503
 504         /* Basic memrange allocator for stolen space. */
 505         drm_mm_init(&i915->mm.stolen, 0, i915->stolen_usable_size);
 506
 507         return 0;
 508 }
 509
 510 static void dbg_poison(struct i915_ggtt *ggtt,
 511                        dma_addr_t addr, resource_size_t size,
 512                        u8 x)
 513 {
 514 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 515         if (!drm_mm_node_allocated(&ggtt->error_capture))
 516                 return;
 517
 518         if (ggtt->vm.bind_async_flags & I915_VMA_GLOBAL_BIND)
 519                 return; /* beware stop_machine() inversion */
 520
 521         GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
 522
 523         mutex_lock(&ggtt->error_mutex);
 524         while (size) {
 525                 void __iomem *s;
 526
 527                 ggtt->vm.insert_page(&ggtt->vm, addr,
 528                                      ggtt->error_capture.start,
 529                                      I915_CACHE_NONE, 0);
 530                 mb();
 531
 532                 s = io_mapping_map_wc(&ggtt->iomap,
 533                                       ggtt->error_capture.start,
 534                                       PAGE_SIZE);
 535                 memset_io(s, x, PAGE_SIZE);
 536                 io_mapping_unmap(s);
 537
 538                 addr += PAGE_SIZE;
 539                 size -= PAGE_SIZE;
 540         }
 541         mb();
 542         ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE);
 543         mutex_unlock(&ggtt->error_mutex);
 544 #endif
 545 }
 546
 547 static struct sg_table *
 548 i915_pages_create_for_stolen(struct drm_device *dev,
 549                              resource_size_t offset, resource_size_t size)
 550 {
 551         struct drm_i915_private *i915 = to_i915(dev);
 552         struct sg_table *st;
 553         struct scatterlist *sg;
 554
 555         GEM_BUG_ON(range_overflows(offset, size, resource_size(&i915->dsm)));
 556
 557         /* We hide that we have no struct page backing our stolen object
 558          * by wrapping the contiguous physical allocation with a fake
 559          * dma mapping in a single scatterlist.
 560          */
 561
 562         st = kmalloc(sizeof(*st), GFP_KERNEL);
 563         if (st == NULL)
 564                 return ERR_PTR(-ENOMEM);
 565
 566         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 567                 kfree(st);
 568                 return ERR_PTR(-ENOMEM);
 569         }
 570
 571         sg = st->sgl;
 572         sg->offset = 0;
 573         sg->length = size;
 574
 575         sg_dma_address(sg) = (dma_addr_t)i915->dsm.start + offset;
 576         sg_dma_len(sg) = size;
 577
 578         return st;
 579 }
 580
 581 static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 582 {
 583         struct sg_table *pages =
 584                 i915_pages_create_for_stolen(obj->base.dev,
 585                                              obj->stolen->start,
 586                                              obj->stolen->size);
 587         if (IS_ERR(pages))
 588                 return PTR_ERR(pages);
 589
 590         dbg_poison(&to_i915(obj->base.dev)->ggtt,
 591                    sg_dma_address(pages->sgl),
 592                    sg_dma_len(pages->sgl),
 593                    POISON_INUSE);
 594
 595         __i915_gem_object_set_pages(obj, pages, obj->stolen->size);
 596
 597         return 0;
 598 }
 599
 600 static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
 601                                              struct sg_table *pages)
 602 {
 603         /* Should only be called from i915_gem_object_release_stolen() */
 604
 605         dbg_poison(&to_i915(obj->base.dev)->ggtt,
 606                    sg_dma_address(pages->sgl),
 607                    sg_dma_len(pages->sgl),
 608                    POISON_FREE);
 609
 610         sg_free_table(pages);
 611         kfree(pages);
 612 }
 613
 614 static void
 615 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 616 {
 617         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 618         struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
 619
 620         GEM_BUG_ON(!stolen);
 621         i915_gem_stolen_remove_node(i915, stolen);
 622         kfree(stolen);
 623
 624         i915_gem_object_release_memory_region(obj);
 625 }
 626
 627 static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
 628         .name = "i915_gem_object_stolen",
 629         .get_pages = i915_gem_object_get_pages_stolen,
 630         .put_pages = i915_gem_object_put_pages_stolen,
 631         .release = i915_gem_object_release_stolen,
 632 };
 633
 634 static int __i915_gem_object_create_stolen(struct intel_memory_region *mem,
 635                                            struct drm_i915_gem_object *obj,
 636                                            struct drm_mm_node *stolen)
 637 {
 638         static struct lock_class_key lock_class;
 639         unsigned int cache_level;
 640         unsigned int flags;
 641         int err;
 642
 643         /*
 644          * Stolen objects are always physically contiguous since we just
 645          * allocate one big block underneath using the drm_mm range allocator.
 646          */
 647         flags = I915_BO_ALLOC_CONTIGUOUS;
 648
 649         drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size);
 650         i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, flags);
 651
 652         obj->stolen = stolen;
 653         obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 654         cache_level = HAS_LLC(mem->i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
 655         i915_gem_object_set_cache_coherency(obj, cache_level);
 656
 657         if (WARN_ON(!i915_gem_object_trylock(obj)))
 658                 return -EBUSY;
 659
 660         i915_gem_object_init_memory_region(obj, mem);
 661
 662         err = i915_gem_object_pin_pages(obj);
 663         if (err)
 664                 i915_gem_object_release_memory_region(obj);
 665         i915_gem_object_unlock(obj);
 666
 667         return err;
 668 }
 669
 670 static int _i915_gem_object_stolen_init(struct intel_memory_region *mem,
 671                                         struct drm_i915_gem_object *obj,
 672                                         resource_size_t size,
 673                                         resource_size_t page_size,
 674                                         unsigned int flags)
 675 {
 676         struct drm_i915_private *i915 = mem->i915;
 677         struct drm_mm_node *stolen;
 678         int ret;
 679
 680         if (!drm_mm_initialized(&i915->mm.stolen))
 681                 return -ENODEV;
 682
 683         if (size == 0)
 684                 return -EINVAL;
 685
 686         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 687         if (!stolen)
 688                 return -ENOMEM;
 689
 690         ret = i915_gem_stolen_insert_node(i915, stolen, size,
 691                                           mem->min_page_size);
 692         if (ret)
 693                 goto err_free;
 694
 695         ret = __i915_gem_object_create_stolen(mem, obj, stolen);
 696         if (ret)
 697                 goto err_remove;
 698
 699         return 0;
 700
 701 err_remove:
 702         i915_gem_stolen_remove_node(i915, stolen);
 703 err_free:
 704         kfree(stolen);
 705         return ret;
 706 }
 707
 708 struct drm_i915_gem_object *
 709 i915_gem_object_create_stolen(struct drm_i915_private *i915,
 710                               resource_size_t size)
 711 {
 712         return i915_gem_object_create_region(i915->mm.stolen_region, size, 0, 0);
 713 }
 714
 715 static int init_stolen_smem(struct intel_memory_region *mem)
 716 {
 717         /*
 718          * Initialise stolen early so that we may reserve preallocated
 719          * objects for the BIOS to KMS transition.
 720          */
 721         return i915_gem_init_stolen(mem);
 722 }
 723
 724 static void release_stolen_smem(struct intel_memory_region *mem)
 725 {
 726         i915_gem_cleanup_stolen(mem->i915);
 727 }
 728
 729 static const struct intel_memory_region_ops i915_region_stolen_smem_ops = {
 730         .init = init_stolen_smem,
 731         .release = release_stolen_smem,
 732         .init_object = _i915_gem_object_stolen_init,
 733 };
 734
 735 static int init_stolen_lmem(struct intel_memory_region *mem)
 736 {
 737         int err;
 738
 739         if (GEM_WARN_ON(resource_size(&mem->region) == 0))
 740                 return -ENODEV;
 741
 742         if (!io_mapping_init_wc(&mem->iomap,
 743                                 mem->io_start,
 744                                 resource_size(&mem->region)))
 745                 return -EIO;
 746
 747         /*
 748          * TODO: For stolen lmem we mostly just care about populating the dsm
 749          * related bits and setting up the drm_mm allocator for the range.
 750          * Perhaps split up i915_gem_init_stolen() for this.
 751          */
 752         err = i915_gem_init_stolen(mem);
 753         if (err)
 754                 goto err_fini;
 755
 756         return 0;
 757
 758 err_fini:
 759         io_mapping_fini(&mem->iomap);
 760         return err;
 761 }
 762
 763 static void release_stolen_lmem(struct intel_memory_region *mem)
 764 {
 765         io_mapping_fini(&mem->iomap);
 766         i915_gem_cleanup_stolen(mem->i915);
 767 }
 768
 769 static const struct intel_memory_region_ops i915_region_stolen_lmem_ops = {
 770         .init = init_stolen_lmem,
 771         .release = release_stolen_lmem,
 772         .init_object = _i915_gem_object_stolen_init,
 773 };
 774
 775 struct intel_memory_region *
 776 i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 777                            u16 instance)
 778 {
 779         struct intel_uncore *uncore = &i915->uncore;
 780         struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 781         struct intel_memory_region *mem;
 782         resource_size_t io_start;
 783         resource_size_t lmem_size;
 784         u64 lmem_base;
 785
 786         lmem_base = intel_uncore_read64(uncore, GEN12_DSMBASE);
 787         if (GEM_WARN_ON(lmem_base >= pci_resource_len(pdev, 2)))
 788                 return ERR_PTR(-ENODEV);
 789
 790         lmem_size = pci_resource_len(pdev, 2) - lmem_base;
 791         io_start = pci_resource_start(pdev, 2) + lmem_base;
 792
 793         mem = intel_memory_region_create(i915, lmem_base, lmem_size,
 794                                          I915_GTT_PAGE_SIZE_4K, io_start,
 795                                          type, instance,
 796                                          &i915_region_stolen_lmem_ops);
 797         if (IS_ERR(mem))
 798                 return mem;
 799
 800         /*
 801          * TODO: consider creating common helper to just print all the
 802          * interesting stuff from intel_memory_region, which we can use for all
 803          * our probed regions.
 804          */
 805
 806         drm_dbg(&i915->drm, "Stolen Local memory IO start: %pa\n",
 807                 &mem->io_start);
 808
 809         intel_memory_region_set_name(mem, "stolen-local");
 810
 811         mem->private = true;
 812
 813         return mem;
 814 }
 815
 816 struct intel_memory_region*
 817 i915_gem_stolen_smem_setup(struct drm_i915_private *i915, u16 type,
 818                            u16 instance)
 819 {
 820         struct intel_memory_region *mem;
 821
 822         mem = intel_memory_region_create(i915,
 823                                          intel_graphics_stolen_res.start,
 824                                          resource_size(&intel_graphics_stolen_res),
 825                                          PAGE_SIZE, 0, type, instance,
 826                                          &i915_region_stolen_smem_ops);
 827         if (IS_ERR(mem))
 828                 return mem;
 829
 830         intel_memory_region_set_name(mem, "stolen-system");
 831
 832         mem->private = true;
 833         return mem;
 834 }
 835
 836 struct drm_i915_gem_object *
 837 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *i915,
 838                                                resource_size_t stolen_offset,
 839                                                resource_size_t size)
 840 {
 841         struct intel_memory_region *mem = i915->mm.stolen_region;
 842         struct drm_i915_gem_object *obj;
 843         struct drm_mm_node *stolen;
 844         int ret;
 845
 846         if (!drm_mm_initialized(&i915->mm.stolen))
 847                 return ERR_PTR(-ENODEV);
 848
 849         drm_dbg(&i915->drm,
 850                 "creating preallocated stolen object: stolen_offset=%pa, size=%pa\n",
 851                 &stolen_offset, &size);
 852
 853         /* KISS and expect everything to be page-aligned */
 854         if (GEM_WARN_ON(size == 0) ||
 855             GEM_WARN_ON(!IS_ALIGNED(size, mem->min_page_size)) ||
 856             GEM_WARN_ON(!IS_ALIGNED(stolen_offset, mem->min_page_size)))
 857                 return ERR_PTR(-EINVAL);
 858
 859         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 860         if (!stolen)
 861                 return ERR_PTR(-ENOMEM);
 862
 863         stolen->start = stolen_offset;
 864         stolen->size = size;
 865         mutex_lock(&i915->mm.stolen_lock);
 866         ret = drm_mm_reserve_node(&i915->mm.stolen, stolen);
 867         mutex_unlock(&i915->mm.stolen_lock);
 868         if (ret)
 869                 goto err_free;
 870
 871         obj = i915_gem_object_alloc();
 872         if (!obj) {
 873                 ret = -ENOMEM;
 874                 goto err_stolen;
 875         }
 876
 877         ret = __i915_gem_object_create_stolen(mem, obj, stolen);
 878         if (ret)
 879                 goto err_object_free;
 880
 881         i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
 882         return obj;
 883
 884 err_object_free:
 885         i915_gem_object_free(obj);
 886 err_stolen:
 887         i915_gem_stolen_remove_node(i915, stolen);
 888 err_free:
 889         kfree(stolen);
 890         return ERR_PTR(ret);
 891 }
 892
 893 bool i915_gem_object_is_stolen(const struct drm_i915_gem_object *obj)
 894 {
 895         return obj->ops == &i915_gem_object_stolen_ops;
 896 }