drivers/gpu/drm/i915/intel_memory_region.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "intel_memory_region.h"
   7 #include "i915_drv.h"
   8
   9 /* XXX: Hysterical raisins. BIT(inst) needs to just be (inst) at some point. */
  10 #define REGION_MAP(type, inst) \
  11         BIT((type) + INTEL_MEMORY_TYPE_SHIFT) | BIT(inst)
  12
  13 const u32 intel_region_map[] = {
  14         [INTEL_REGION_SMEM] = REGION_MAP(INTEL_MEMORY_SYSTEM, 0),
  15         [INTEL_REGION_LMEM] = REGION_MAP(INTEL_MEMORY_LOCAL, 0),
  16         [INTEL_REGION_STOLEN] = REGION_MAP(INTEL_MEMORY_STOLEN, 0),
  17 };
  18
  19 struct intel_memory_region *
  20 intel_memory_region_by_type(struct drm_i915_private *i915,
  21                             enum intel_memory_type mem_type)
  22 {
  23         struct intel_memory_region *mr;
  24         int id;
  25
  26         for_each_memory_region(mr, i915, id)
  27                 if (mr->type == mem_type)
  28                         return mr;
  29
  30         return NULL;
  31 }
  32
  33 static u64
  34 intel_memory_region_free_pages(struct intel_memory_region *mem,
  35                                struct list_head *blocks)
  36 {
  37         struct i915_buddy_block *block, *on;
  38         u64 size = 0;
  39
  40         list_for_each_entry_safe(block, on, blocks, link) {
  41                 size += i915_buddy_block_size(&mem->mm, block);
  42                 i915_buddy_free(&mem->mm, block);
  43         }
  44         INIT_LIST_HEAD(blocks);
  45
  46         return size;
  47 }
  48
  49 void
  50 __intel_memory_region_put_pages_buddy(struct intel_memory_region *mem,
  51                                       struct list_head *blocks)
  52 {
  53         mutex_lock(&mem->mm_lock);
  54         mem->avail += intel_memory_region_free_pages(mem, blocks);
  55         mutex_unlock(&mem->mm_lock);
  56 }
  57
  58 void
  59 __intel_memory_region_put_block_buddy(struct i915_buddy_block *block)
  60 {
  61         struct list_head blocks;
  62
  63         INIT_LIST_HEAD(&blocks);
  64         list_add(&block->link, &blocks);
  65         __intel_memory_region_put_pages_buddy(block->private, &blocks);
  66 }
  67
  68 int
  69 __intel_memory_region_get_pages_buddy(struct intel_memory_region *mem,
  70                                       resource_size_t size,
  71                                       unsigned int flags,
  72                                       struct list_head *blocks)
  73 {
  74         unsigned int min_order = 0;
  75         unsigned long n_pages;
  76
  77         GEM_BUG_ON(!IS_ALIGNED(size, mem->mm.chunk_size));
  78         GEM_BUG_ON(!list_empty(blocks));
  79
  80         if (flags & I915_ALLOC_MIN_PAGE_SIZE) {
  81                 min_order = ilog2(mem->min_page_size) -
  82                             ilog2(mem->mm.chunk_size);
  83         }
  84
  85         if (flags & I915_ALLOC_CONTIGUOUS) {
  86                 size = roundup_pow_of_two(size);
  87                 min_order = ilog2(size) - ilog2(mem->mm.chunk_size);
  88         }
  89
  90         if (size > mem->mm.size)
  91                 return -E2BIG;
  92
  93         n_pages = size >> ilog2(mem->mm.chunk_size);
  94
  95         mutex_lock(&mem->mm_lock);
  96
  97         do {
  98                 struct i915_buddy_block *block;
  99                 unsigned int order;
 100
 101                 order = fls(n_pages) - 1;
 102                 GEM_BUG_ON(order > mem->mm.max_order);
 103                 GEM_BUG_ON(order < min_order);
 104
 105                 do {
 106                         block = i915_buddy_alloc(&mem->mm, order);
 107                         if (!IS_ERR(block))
 108                                 break;
 109
 110                         if (order-- == min_order)
 111                                 goto err_free_blocks;
 112                 } while (1);
 113
 114                 n_pages -= BIT(order);
 115
 116                 block->private = mem;
 117                 list_add(&block->link, blocks);
 118
 119                 if (!n_pages)
 120                         break;
 121         } while (1);
 122
 123         mem->avail -= size;
 124         mutex_unlock(&mem->mm_lock);
 125         return 0;
 126
 127 err_free_blocks:
 128         intel_memory_region_free_pages(mem, blocks);
 129         mutex_unlock(&mem->mm_lock);
 130         return -ENXIO;
 131 }
 132
 133 struct i915_buddy_block *
 134 __intel_memory_region_get_block_buddy(struct intel_memory_region *mem,
 135                                       resource_size_t size,
 136                                       unsigned int flags)
 137 {
 138         struct i915_buddy_block *block;
 139         LIST_HEAD(blocks);
 140         int ret;
 141
 142         ret = __intel_memory_region_get_pages_buddy(mem, size, flags, &blocks);
 143         if (ret)
 144                 return ERR_PTR(ret);
 145
 146         block = list_first_entry(&blocks, typeof(*block), link);
 147         list_del_init(&block->link);
 148         return block;
 149 }
 150
 151 int intel_memory_region_init_buddy(struct intel_memory_region *mem)
 152 {
 153         return i915_buddy_init(&mem->mm, resource_size(&mem->region),
 154                                PAGE_SIZE);
 155 }
 156
 157 void intel_memory_region_release_buddy(struct intel_memory_region *mem)
 158 {
 159         i915_buddy_fini(&mem->mm);
 160 }
 161
 162 struct intel_memory_region *
 163 intel_memory_region_create(struct drm_i915_private *i915,
 164                            resource_size_t start,
 165                            resource_size_t size,
 166                            resource_size_t min_page_size,
 167                            resource_size_t io_start,
 168                            const struct intel_memory_region_ops *ops)
 169 {
 170         struct intel_memory_region *mem;
 171         int err;
 172
 173         mem = kzalloc(sizeof(*mem), GFP_KERNEL);
 174         if (!mem)
 175                 return ERR_PTR(-ENOMEM);
 176
 177         mem->i915 = i915;
 178         mem->region = (struct resource)DEFINE_RES_MEM(start, size);
 179         mem->io_start = io_start;
 180         mem->min_page_size = min_page_size;
 181         mem->ops = ops;
 182         mem->total = size;
 183         mem->avail = mem->total;
 184
 185         mutex_init(&mem->objects.lock);
 186         INIT_LIST_HEAD(&mem->objects.list);
 187         INIT_LIST_HEAD(&mem->objects.purgeable);
 188
 189         mutex_init(&mem->mm_lock);
 190
 191         if (ops->init) {
 192                 err = ops->init(mem);
 193                 if (err)
 194                         goto err_free;
 195         }
 196
 197         kref_init(&mem->kref);
 198         return mem;
 199
 200 err_free:
 201         kfree(mem);
 202         return ERR_PTR(err);
 203 }
 204
 205 void intel_memory_region_set_name(struct intel_memory_region *mem,
 206                                   const char *fmt, ...)
 207 {
 208         va_list ap;
 209
 210         va_start(ap, fmt);
 211         vsnprintf(mem->name, sizeof(mem->name), fmt, ap);
 212         va_end(ap);
 213 }
 214
 215 static void __intel_memory_region_destroy(struct kref *kref)
 216 {
 217         struct intel_memory_region *mem =
 218                 container_of(kref, typeof(*mem), kref);
 219
 220         if (mem->ops->release)
 221                 mem->ops->release(mem);
 222
 223         mutex_destroy(&mem->mm_lock);
 224         mutex_destroy(&mem->objects.lock);
 225         kfree(mem);
 226 }
 227
 228 struct intel_memory_region *
 229 intel_memory_region_get(struct intel_memory_region *mem)
 230 {
 231         kref_get(&mem->kref);
 232         return mem;
 233 }
 234
 235 void intel_memory_region_put(struct intel_memory_region *mem)
 236 {
 237         kref_put(&mem->kref, __intel_memory_region_destroy);
 238 }
 239
 240 /* Global memory region registration -- only slight layer inversions! */
 241
 242 int intel_memory_regions_hw_probe(struct drm_i915_private *i915)
 243 {
 244         int err, i;
 245
 246         for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) {
 247                 struct intel_memory_region *mem = ERR_PTR(-ENODEV);
 248                 u32 type;
 249
 250                 if (!HAS_REGION(i915, BIT(i)))
 251                         continue;
 252
 253                 type = MEMORY_TYPE_FROM_REGION(intel_region_map[i]);
 254                 switch (type) {
 255                 case INTEL_MEMORY_SYSTEM:
 256                         mem = i915_gem_shmem_setup(i915);
 257                         break;
 258                 case INTEL_MEMORY_STOLEN:
 259                         mem = i915_gem_stolen_setup(i915);
 260                         break;
 261                 case INTEL_MEMORY_LOCAL:
 262                         mem = intel_setup_fake_lmem(i915);
 263                         break;
 264                 }
 265
 266                 if (IS_ERR(mem)) {
 267                         err = PTR_ERR(mem);
 268                         drm_err(&i915->drm,
 269                                 "Failed to setup region(%d) type=%d\n",
 270                                 err, type);
 271                         goto out_cleanup;
 272                 }
 273
 274                 mem->id = intel_region_map[i];
 275                 mem->type = type;
 276                 mem->instance = MEMORY_INSTANCE_FROM_REGION(intel_region_map[i]);
 277
 278                 i915->mm.regions[i] = mem;
 279         }
 280
 281         return 0;
 282
 283 out_cleanup:
 284         intel_memory_regions_driver_release(i915);
 285         return err;
 286 }
 287
 288 void intel_memory_regions_driver_release(struct drm_i915_private *i915)
 289 {
 290         int i;
 291
 292         for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) {
 293                 struct intel_memory_region *region =
 294                         fetch_and_zero(&i915->mm.regions[i]);
 295
 296                 if (region)
 297                         intel_memory_region_put(region);
 298         }
 299 }
 300
 301 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 302 #include "selftests/intel_memory_region.c"
 303 #include "selftests/mock_region.c"
 304 #endif