drivers/gpu/drm/i915/i915_gem_stolen.c

   1 /*
   2  * Copyright © 2008-2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Chris Wilson <chris@chris-wilson.co.uk>
  26  *
  27  */
  28
  29 #include <drm/drmP.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32
  33 /*
  34  * The BIOS typically reserves some of the system's memory for the exclusive
  35  * use of the integrated graphics. This memory is no longer available for
  36  * use by the OS and so the user finds that his system has less memory
  37  * available than he put in. We refer to this memory as stolen.
  38  *
  39  * The BIOS will allocate its framebuffer from the stolen memory. Our
  40  * goal is try to reuse that object for our own fbcon which must always
  41  * be available for panics. Anything else we can reuse the stolen memory
  42  * for is a boon.
  43  */
  44
  45 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
  46                                          struct drm_mm_node *node, u64 size,
  47                                          unsigned alignment, u64 start, u64 end)
  48 {
  49         int ret;
  50
  51         if (!drm_mm_initialized(&dev_priv->mm.stolen))
  52                 return -ENODEV;
  53
  54         /* WaSkipStolenMemoryFirstPage:bdw+ */
  55         if (INTEL_GEN(dev_priv) >= 8 && start < 4096)
  56                 start = 4096;
  57
  58         mutex_lock(&dev_priv->mm.stolen_lock);
  59         ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node,
  60                                           size, alignment, 0,
  61                                           start, end, DRM_MM_INSERT_BEST);
  62         mutex_unlock(&dev_priv->mm.stolen_lock);
  63
  64         return ret;
  65 }
  66
  67 int i915_gem_stolen_insert_node(struct drm_i915_private *dev_priv,
  68                                 struct drm_mm_node *node, u64 size,
  69                                 unsigned alignment)
  70 {
  71         return i915_gem_stolen_insert_node_in_range(dev_priv, node, size,
  72                                                     alignment, 0, U64_MAX);
  73 }
  74
  75 void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
  76                                  struct drm_mm_node *node)
  77 {
  78         mutex_lock(&dev_priv->mm.stolen_lock);
  79         drm_mm_remove_node(node);
  80         mutex_unlock(&dev_priv->mm.stolen_lock);
  81 }
  82
  83 static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
  84                               struct resource *dsm)
  85 {
  86         struct i915_ggtt *ggtt = &dev_priv->ggtt;
  87         struct resource *r;
  88
  89         if (dsm->start == 0 || dsm->end <= dsm->start)
  90                 return -EINVAL;
  91
  92         /*
  93          * TODO: We have yet too encounter the case where the GTT wasn't at the
  94          * end of stolen. With that assumption we could simplify this.
  95          */
  96
  97         /* Make sure we don't clobber the GTT if it's within stolen memory */
  98         if (INTEL_GEN(dev_priv) <= 4 &&
  99             !IS_G33(dev_priv) && !IS_PINEVIEW(dev_priv) && !IS_G4X(dev_priv)) {
 100                 struct resource stolen[2] = {*dsm, *dsm};
 101                 struct resource ggtt_res;
 102                 resource_size_t ggtt_start;
 103
 104                 ggtt_start = I915_READ(PGTBL_CTL);
 105                 if (IS_GEN4(dev_priv))
 106                         ggtt_start = (ggtt_start & PGTBL_ADDRESS_LO_MASK) |
 107                                      (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
 108                 else
 109                         ggtt_start &= PGTBL_ADDRESS_LO_MASK;
 110
 111                 ggtt_res =
 112                         (struct resource) DEFINE_RES_MEM(ggtt_start,
 113                                                          ggtt_total_entries(ggtt) * 4);
 114
 115                 if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
 116                         stolen[0].end = ggtt_res.start;
 117                 if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
 118                         stolen[1].start = ggtt_res.end;
 119
 120                 /* Pick the larger of the two chunks */
 121                 if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
 122                         *dsm = stolen[0];
 123                 else
 124                         *dsm = stolen[1];
 125
 126                 if (stolen[0].start != stolen[1].start ||
 127                     stolen[0].end != stolen[1].end) {
 128                         DRM_DEBUG_DRIVER("GTT within stolen memory at %pR\n", &ggtt_res);
 129                         DRM_DEBUG_DRIVER("Stolen memory adjusted to %pR\n", dsm);
 130                 }
 131         }
 132
 133         /*
 134          * Verify that nothing else uses this physical address. Stolen
 135          * memory should be reserved by the BIOS and hidden from the
 136          * kernel. So if the region is already marked as busy, something
 137          * is seriously wrong.
 138          */
 139         r = devm_request_mem_region(dev_priv->drm.dev, dsm->start,
 140                                     resource_size(dsm),
 141                                     "Graphics Stolen Memory");
 142         if (r == NULL) {
 143                 /*
 144                  * One more attempt but this time requesting region from
 145                  * start + 1, as we have seen that this resolves the region
 146                  * conflict with the PCI Bus.
 147                  * This is a BIOS w/a: Some BIOS wrap stolen in the root
 148                  * PCI bus, but have an off-by-one error. Hence retry the
 149                  * reservation starting from 1 instead of 0.
 150                  * There's also BIOS with off-by-one on the other end.
 151                  */
 152                 r = devm_request_mem_region(dev_priv->drm.dev, dsm->start + 1,
 153                                             resource_size(dsm) - 2,
 154                                             "Graphics Stolen Memory");
 155                 /*
 156                  * GEN3 firmware likes to smash pci bridges into the stolen
 157                  * range. Apparently this works.
 158                  */
 159                 if (r == NULL && !IS_GEN3(dev_priv)) {
 160                         DRM_ERROR("conflict detected with stolen region: %pR\n",
 161                                   dsm);
 162
 163                         return -EBUSY;
 164                 }
 165         }
 166
 167         return 0;
 168 }
 169
 170 void i915_gem_cleanup_stolen(struct drm_i915_private *dev_priv)
 171 {
 172         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 173                 return;
 174
 175         drm_mm_takedown(&dev_priv->mm.stolen);
 176 }
 177
 178 static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
 179                                     resource_size_t *base,
 180                                     resource_size_t *size)
 181 {
 182         u32 reg_val = I915_READ(IS_GM45(dev_priv) ?
 183                                 CTG_STOLEN_RESERVED :
 184                                 ELK_STOLEN_RESERVED);
 185         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 186
 187         DRM_DEBUG_DRIVER("%s_STOLEN_RESERVED = %08x\n",
 188                          IS_GM45(dev_priv) ? "CTG" : "ELK", reg_val);
 189
 190         if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
 191                 return;
 192
 193         /*
 194          * Whether ILK really reuses the ELK register for this is unclear.
 195          * Let's see if we catch anyone with this supposedly enabled on ILK.
 196          */
 197         WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
 198
 199         if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
 200                 return;
 201
 202         *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 203         WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 204
 205         *size = stolen_top - *base;
 206 }
 207
 208 static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
 209                                      resource_size_t *base,
 210                                      resource_size_t *size)
 211 {
 212         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 213
 214         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 215
 216         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 217                 return;
 218
 219         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 220
 221         switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
 222         case GEN6_STOLEN_RESERVED_1M:
 223                 *size = 1024 * 1024;
 224                 break;
 225         case GEN6_STOLEN_RESERVED_512K:
 226                 *size = 512 * 1024;
 227                 break;
 228         case GEN6_STOLEN_RESERVED_256K:
 229                 *size = 256 * 1024;
 230                 break;
 231         case GEN6_STOLEN_RESERVED_128K:
 232                 *size = 128 * 1024;
 233                 break;
 234         default:
 235                 *size = 1024 * 1024;
 236                 MISSING_CASE(reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK);
 237         }
 238 }
 239
 240 static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 241                                     resource_size_t *base,
 242                                     resource_size_t *size)
 243 {
 244         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 245         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 246
 247         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 248
 249         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 250                 return;
 251
 252         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 253         default:
 254                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 255                 /* fall through */
 256         case GEN7_STOLEN_RESERVED_1M:
 257                 *size = 1024 * 1024;
 258                 break;
 259         }
 260
 261         /*
 262          * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
 263          * reserved location as (top - size).
 264          */
 265         *base = stolen_top - *size;
 266 }
 267
 268 static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 269                                      resource_size_t *base,
 270                                      resource_size_t *size)
 271 {
 272         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 273
 274         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 275
 276         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 277                 return;
 278
 279         *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 280
 281         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
 282         case GEN7_STOLEN_RESERVED_1M:
 283                 *size = 1024 * 1024;
 284                 break;
 285         case GEN7_STOLEN_RESERVED_256K:
 286                 *size = 256 * 1024;
 287                 break;
 288         default:
 289                 *size = 1024 * 1024;
 290                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
 291         }
 292 }
 293
 294 static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 295                                     resource_size_t *base,
 296                                     resource_size_t *size)
 297 {
 298         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 299
 300         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 301
 302         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 303                 return;
 304
 305         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 306
 307         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 308         case GEN8_STOLEN_RESERVED_1M:
 309                 *size = 1024 * 1024;
 310                 break;
 311         case GEN8_STOLEN_RESERVED_2M:
 312                 *size = 2 * 1024 * 1024;
 313                 break;
 314         case GEN8_STOLEN_RESERVED_4M:
 315                 *size = 4 * 1024 * 1024;
 316                 break;
 317         case GEN8_STOLEN_RESERVED_8M:
 318                 *size = 8 * 1024 * 1024;
 319                 break;
 320         default:
 321                 *size = 8 * 1024 * 1024;
 322                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 323         }
 324 }
 325
 326 static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
 327                                     resource_size_t *base,
 328                                     resource_size_t *size)
 329 {
 330         u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 331         resource_size_t stolen_top = dev_priv->dsm.end + 1;
 332
 333         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 334
 335         if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
 336                 return;
 337
 338         if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
 339                 return;
 340
 341         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 342         *size = stolen_top - *base;
 343 }
 344
 345 static void icl_get_stolen_reserved(struct drm_i915_private *dev_priv,
 346                                     resource_size_t *base,
 347                                     resource_size_t *size)
 348 {
 349         u64 reg_val = I915_READ64(GEN6_STOLEN_RESERVED);
 350
 351         DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);
 352
 353         *base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;
 354
 355         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
 356         case GEN8_STOLEN_RESERVED_1M:
 357                 *size = 1024 * 1024;
 358                 break;
 359         case GEN8_STOLEN_RESERVED_2M:
 360                 *size = 2 * 1024 * 1024;
 361                 break;
 362         case GEN8_STOLEN_RESERVED_4M:
 363                 *size = 4 * 1024 * 1024;
 364                 break;
 365         case GEN8_STOLEN_RESERVED_8M:
 366                 *size = 8 * 1024 * 1024;
 367                 break;
 368         default:
 369                 *size = 8 * 1024 * 1024;
 370                 MISSING_CASE(reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK);
 371         }
 372 }
 373
 374 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 375 {
 376         resource_size_t reserved_base, stolen_top;
 377         resource_size_t reserved_total, reserved_size;
 378
 379         mutex_init(&dev_priv->mm.stolen_lock);
 380
 381         if (intel_vgpu_active(dev_priv)) {
 382                 DRM_INFO("iGVT-g active, disabling use of stolen memory\n");
 383                 return 0;
 384         }
 385
 386         if (intel_vtd_active() && INTEL_GEN(dev_priv) < 8) {
 387                 DRM_INFO("DMAR active, disabling use of stolen memory\n");
 388                 return 0;
 389         }
 390
 391         if (resource_size(&intel_graphics_stolen_res) == 0)
 392                 return 0;
 393
 394         dev_priv->dsm = intel_graphics_stolen_res;
 395
 396         if (i915_adjust_stolen(dev_priv, &dev_priv->dsm))
 397                 return 0;
 398
 399         GEM_BUG_ON(dev_priv->dsm.start == 0);
 400         GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
 401
 402         stolen_top = dev_priv->dsm.end + 1;
 403         reserved_base = stolen_top;
 404         reserved_size = 0;
 405
 406         switch (INTEL_GEN(dev_priv)) {
 407         case 2:
 408         case 3:
 409                 break;
 410         case 4:
 411                 if (!IS_G4X(dev_priv))
 412                         break;
 413                 /* fall through */
 414         case 5:
 415                 g4x_get_stolen_reserved(dev_priv,
 416                                         &reserved_base, &reserved_size);
 417                 break;
 418         case 6:
 419                 gen6_get_stolen_reserved(dev_priv,
 420                                          &reserved_base, &reserved_size);
 421                 break;
 422         case 7:
 423                 if (IS_VALLEYVIEW(dev_priv))
 424                         vlv_get_stolen_reserved(dev_priv,
 425                                                 &reserved_base, &reserved_size);
 426                 else
 427                         gen7_get_stolen_reserved(dev_priv,
 428                                                  &reserved_base, &reserved_size);
 429                 break;
 430         case 8:
 431         case 9:
 432         case 10:
 433                 if (IS_LP(dev_priv))
 434                         chv_get_stolen_reserved(dev_priv,
 435                                                 &reserved_base, &reserved_size);
 436                 else
 437                         bdw_get_stolen_reserved(dev_priv,
 438                                                 &reserved_base, &reserved_size);
 439                 break;
 440         case 11:
 441         default:
 442                 icl_get_stolen_reserved(dev_priv, &reserved_base,
 443                                         &reserved_size);
 444                 break;
 445         }
 446
 447         /*
 448          * Our expectation is that the reserved space is at the top of the
 449          * stolen region and *never* at the bottom. If we see !reserved_base,
 450          * it likely means we failed to read the registers correctly.
 451          */
 452         if (!reserved_base) {
 453                 DRM_ERROR("inconsistent reservation %pa + %pa; ignoring\n",
 454                           &reserved_base, &reserved_size);
 455                 reserved_base = stolen_top;
 456                 reserved_size = 0;
 457         }
 458
 459         dev_priv->dsm_reserved =
 460                 (struct resource) DEFINE_RES_MEM(reserved_base, reserved_size);
 461
 462         if (!resource_contains(&dev_priv->dsm, &dev_priv->dsm_reserved)) {
 463                 DRM_ERROR("Stolen reserved area %pR outside stolen memory %pR\n",
 464                           &dev_priv->dsm_reserved, &dev_priv->dsm);
 465                 return 0;
 466         }
 467
 468         /* It is possible for the reserved area to end before the end of stolen
 469          * memory, so just consider the start. */
 470         reserved_total = stolen_top - reserved_base;
 471
 472         DRM_DEBUG_DRIVER("Memory reserved for graphics device: %lluK, usable: %lluK\n",
 473                          (u64)resource_size(&dev_priv->dsm) >> 10,
 474                          ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
 475
 476         dev_priv->stolen_usable_size =
 477                 resource_size(&dev_priv->dsm) - reserved_total;
 478
 479         /* Basic memrange allocator for stolen space. */
 480         drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->stolen_usable_size);
 481
 482         return 0;
 483 }
 484
 485 static struct sg_table *
 486 i915_pages_create_for_stolen(struct drm_device *dev,
 487                              resource_size_t offset, resource_size_t size)
 488 {
 489         struct drm_i915_private *dev_priv = to_i915(dev);
 490         struct sg_table *st;
 491         struct scatterlist *sg;
 492
 493         GEM_BUG_ON(range_overflows(offset, size, resource_size(&dev_priv->dsm)));
 494
 495         /* We hide that we have no struct page backing our stolen object
 496          * by wrapping the contiguous physical allocation with a fake
 497          * dma mapping in a single scatterlist.
 498          */
 499
 500         st = kmalloc(sizeof(*st), GFP_KERNEL);
 501         if (st == NULL)
 502                 return ERR_PTR(-ENOMEM);
 503
 504         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 505                 kfree(st);
 506                 return ERR_PTR(-ENOMEM);
 507         }
 508
 509         sg = st->sgl;
 510         sg->offset = 0;
 511         sg->length = size;
 512
 513         sg_dma_address(sg) = (dma_addr_t)dev_priv->dsm.start + offset;
 514         sg_dma_len(sg) = size;
 515
 516         return st;
 517 }
 518
 519 static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 520 {
 521         struct sg_table *pages =
 522                 i915_pages_create_for_stolen(obj->base.dev,
 523                                              obj->stolen->start,
 524                                              obj->stolen->size);
 525         if (IS_ERR(pages))
 526                 return PTR_ERR(pages);
 527
 528         __i915_gem_object_set_pages(obj, pages, obj->stolen->size);
 529
 530         return 0;
 531 }
 532
 533 static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
 534                                              struct sg_table *pages)
 535 {
 536         /* Should only be called from i915_gem_object_release_stolen() */
 537         sg_free_table(pages);
 538         kfree(pages);
 539 }
 540
 541 static void
 542 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 543 {
 544         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 545         struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);
 546
 547         GEM_BUG_ON(!stolen);
 548
 549         __i915_gem_object_unpin_pages(obj);
 550
 551         i915_gem_stolen_remove_node(dev_priv, stolen);
 552         kfree(stolen);
 553 }
 554
 555 static const struct drm_i915_gem_object_ops i915_gem_object_stolen_ops = {
 556         .get_pages = i915_gem_object_get_pages_stolen,
 557         .put_pages = i915_gem_object_put_pages_stolen,
 558         .release = i915_gem_object_release_stolen,
 559 };
 560
 561 static struct drm_i915_gem_object *
 562 _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 563                                struct drm_mm_node *stolen)
 564 {
 565         struct drm_i915_gem_object *obj;
 566         unsigned int cache_level;
 567
 568         obj = i915_gem_object_alloc(dev_priv);
 569         if (obj == NULL)
 570                 return NULL;
 571
 572         drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size);
 573         i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 574
 575         obj->stolen = stolen;
 576         obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 577         cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE;
 578         i915_gem_object_set_cache_coherency(obj, cache_level);
 579
 580         if (i915_gem_object_pin_pages(obj))
 581                 goto cleanup;
 582
 583         return obj;
 584
 585 cleanup:
 586         i915_gem_object_free(obj);
 587         return NULL;
 588 }
 589
 590 struct drm_i915_gem_object *
 591 i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 592                               resource_size_t size)
 593 {
 594         struct drm_i915_gem_object *obj;
 595         struct drm_mm_node *stolen;
 596         int ret;
 597
 598         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 599                 return NULL;
 600
 601         if (size == 0)
 602                 return NULL;
 603
 604         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 605         if (!stolen)
 606                 return NULL;
 607
 608         ret = i915_gem_stolen_insert_node(dev_priv, stolen, size, 4096);
 609         if (ret) {
 610                 kfree(stolen);
 611                 return NULL;
 612         }
 613
 614         obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 615         if (obj)
 616                 return obj;
 617
 618         i915_gem_stolen_remove_node(dev_priv, stolen);
 619         kfree(stolen);
 620         return NULL;
 621 }
 622
 623 struct drm_i915_gem_object *
 624 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
 625                                                resource_size_t stolen_offset,
 626                                                resource_size_t gtt_offset,
 627                                                resource_size_t size)
 628 {
 629         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 630         struct drm_i915_gem_object *obj;
 631         struct drm_mm_node *stolen;
 632         struct i915_vma *vma;
 633         int ret;
 634
 635         if (!drm_mm_initialized(&dev_priv->mm.stolen))
 636                 return NULL;
 637
 638         lockdep_assert_held(&dev_priv->drm.struct_mutex);
 639
 640         DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
 641                          &stolen_offset, &gtt_offset, &size);
 642
 643         /* KISS and expect everything to be page-aligned */
 644         if (WARN_ON(size == 0) ||
 645             WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) ||
 646             WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT)))
 647                 return NULL;
 648
 649         stolen = kzalloc(sizeof(*stolen), GFP_KERNEL);
 650         if (!stolen)
 651                 return NULL;
 652
 653         stolen->start = stolen_offset;
 654         stolen->size = size;
 655         mutex_lock(&dev_priv->mm.stolen_lock);
 656         ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen);
 657         mutex_unlock(&dev_priv->mm.stolen_lock);
 658         if (ret) {
 659                 DRM_DEBUG_DRIVER("failed to allocate stolen space\n");
 660                 kfree(stolen);
 661                 return NULL;
 662         }
 663
 664         obj = _i915_gem_object_create_stolen(dev_priv, stolen);
 665         if (obj == NULL) {
 666                 DRM_DEBUG_DRIVER("failed to allocate stolen object\n");
 667                 i915_gem_stolen_remove_node(dev_priv, stolen);
 668                 kfree(stolen);
 669                 return NULL;
 670         }
 671
 672         /* Some objects just need physical mem from stolen space */
 673         if (gtt_offset == I915_GTT_OFFSET_NONE)
 674                 return obj;
 675
 676         ret = i915_gem_object_pin_pages(obj);
 677         if (ret)
 678                 goto err;
 679
 680         vma = i915_vma_instance(obj, &ggtt->vm, NULL);
 681         if (IS_ERR(vma)) {
 682                 ret = PTR_ERR(vma);
 683                 goto err_pages;
 684         }
 685
 686         /* To simplify the initialisation sequence between KMS and GTT,
 687          * we allow construction of the stolen object prior to
 688          * setting up the GTT space. The actual reservation will occur
 689          * later.
 690          */
 691         ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
 692                                    size, gtt_offset, obj->cache_level,
 693                                    0);
 694         if (ret) {
 695                 DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n");
 696                 goto err_pages;
 697         }
 698
 699         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 700
 701         vma->pages = obj->mm.pages;
 702         vma->flags |= I915_VMA_GLOBAL_BIND;
 703         __i915_vma_set_map_and_fenceable(vma);
 704         list_move_tail(&vma->vm_link, &ggtt->vm.inactive_list);
 705
 706         spin_lock(&dev_priv->mm.obj_lock);
 707         list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
 708         obj->bind_count++;
 709         spin_unlock(&dev_priv->mm.obj_lock);
 710
 711         return obj;
 712
 713 err_pages:
 714         i915_gem_object_unpin_pages(obj);
 715 err:
 716         i915_gem_object_put(obj);
 717         return NULL;
 718 }