drivers/gpu/drm/panfrost/panfrost_gem.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
   3
   4 #include <linux/err.h>
   5 #include <linux/slab.h>
   6 #include <linux/dma-buf.h>
   7 #include <linux/dma-mapping.h>
   8
   9 #include <drm/panfrost_drm.h>
  10 #include "panfrost_device.h"
  11 #include "panfrost_gem.h"
  12 #include "panfrost_mmu.h"
  13
  14 /* Called DRM core on the last userspace/kernel unreference of the
  15  * BO.
  16  */
  17 static void panfrost_gem_free_object(struct drm_gem_object *obj)
  18 {
  19         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
  20         struct panfrost_device *pfdev = obj->dev->dev_private;
  21
  22         /*
  23          * Make sure the BO is no longer inserted in the shrinker list before
  24          * taking care of the destruction itself. If we don't do that we have a
  25          * race condition between this function and what's done in
  26          * panfrost_gem_shrinker_scan().
  27          */
  28         mutex_lock(&pfdev->shrinker_lock);
  29         list_del_init(&bo->base.madv_list);
  30         mutex_unlock(&pfdev->shrinker_lock);
  31
  32         /*
  33          * If we still have mappings attached to the BO, there's a problem in
  34          * our refcounting.
  35          */
  36         WARN_ON_ONCE(!list_empty(&bo->mappings.list));
  37
  38         if (bo->sgts) {
  39                 int i;
  40                 int n_sgt = bo->base.base.size / SZ_2M;
  41
  42                 for (i = 0; i < n_sgt; i++) {
  43                         if (bo->sgts[i].sgl) {
  44                                 dma_unmap_sgtable(pfdev->dev, &bo->sgts[i],
  45                                                   DMA_BIDIRECTIONAL, 0);
  46                                 sg_free_table(&bo->sgts[i]);
  47                         }
  48                 }
  49                 kvfree(bo->sgts);
  50         }
  51
  52         drm_gem_shmem_free_object(obj);
  53 }
  54
  55 struct panfrost_gem_mapping *
  56 panfrost_gem_mapping_get(struct panfrost_gem_object *bo,
  57                          struct panfrost_file_priv *priv)
  58 {
  59         struct panfrost_gem_mapping *iter, *mapping = NULL;
  60
  61         mutex_lock(&bo->mappings.lock);
  62         list_for_each_entry(iter, &bo->mappings.list, node) {
  63                 if (iter->mmu == &priv->mmu) {
  64                         kref_get(&iter->refcount);
  65                         mapping = iter;
  66                         break;
  67                 }
  68         }
  69         mutex_unlock(&bo->mappings.lock);
  70
  71         return mapping;
  72 }
  73
  74 static void
  75 panfrost_gem_teardown_mapping(struct panfrost_gem_mapping *mapping)
  76 {
  77         struct panfrost_file_priv *priv;
  78
  79         if (mapping->active)
  80                 panfrost_mmu_unmap(mapping);
  81
  82         priv = container_of(mapping->mmu, struct panfrost_file_priv, mmu);
  83         spin_lock(&priv->mm_lock);
  84         if (drm_mm_node_allocated(&mapping->mmnode))
  85                 drm_mm_remove_node(&mapping->mmnode);
  86         spin_unlock(&priv->mm_lock);
  87 }
  88
  89 static void panfrost_gem_mapping_release(struct kref *kref)
  90 {
  91         struct panfrost_gem_mapping *mapping;
  92
  93         mapping = container_of(kref, struct panfrost_gem_mapping, refcount);
  94
  95         panfrost_gem_teardown_mapping(mapping);
  96         drm_gem_object_put(&mapping->obj->base.base);
  97         kfree(mapping);
  98 }
  99
 100 void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping)
 101 {
 102         if (!mapping)
 103                 return;
 104
 105         kref_put(&mapping->refcount, panfrost_gem_mapping_release);
 106 }
 107
 108 void panfrost_gem_teardown_mappings(struct panfrost_gem_object *bo)
 109 {
 110         struct panfrost_gem_mapping *mapping;
 111
 112         mutex_lock(&bo->mappings.lock);
 113         list_for_each_entry(mapping, &bo->mappings.list, node)
 114                 panfrost_gem_teardown_mapping(mapping);
 115         mutex_unlock(&bo->mappings.lock);
 116 }
 117
 118 int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_priv)
 119 {
 120         int ret;
 121         size_t size = obj->size;
 122         u64 align;
 123         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 124         unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0;
 125         struct panfrost_file_priv *priv = file_priv->driver_priv;
 126         struct panfrost_gem_mapping *mapping;
 127
 128         mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 129         if (!mapping)
 130                 return -ENOMEM;
 131
 132         INIT_LIST_HEAD(&mapping->node);
 133         kref_init(&mapping->refcount);
 134         drm_gem_object_get(obj);
 135         mapping->obj = bo;
 136
 137         /*
 138          * Executable buffers cannot cross a 16MB boundary as the program
 139          * counter is 24-bits. We assume executable buffers will be less than
 140          * 16MB and aligning executable buffers to their size will avoid
 141          * crossing a 16MB boundary.
 142          */
 143         if (!bo->noexec)
 144                 align = size >> PAGE_SHIFT;
 145         else
 146                 align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0;
 147
 148         mapping->mmu = &priv->mmu;
 149         spin_lock(&priv->mm_lock);
 150         ret = drm_mm_insert_node_generic(&priv->mm, &mapping->mmnode,
 151                                          size >> PAGE_SHIFT, align, color, 0);
 152         spin_unlock(&priv->mm_lock);
 153         if (ret)
 154                 goto err;
 155
 156         if (!bo->is_heap) {
 157                 ret = panfrost_mmu_map(mapping);
 158                 if (ret)
 159                         goto err;
 160         }
 161
 162         mutex_lock(&bo->mappings.lock);
 163         WARN_ON(bo->base.madv != PANFROST_MADV_WILLNEED);
 164         list_add_tail(&mapping->node, &bo->mappings.list);
 165         mutex_unlock(&bo->mappings.lock);
 166
 167 err:
 168         if (ret)
 169                 panfrost_gem_mapping_put(mapping);
 170         return ret;
 171 }
 172
 173 void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv)
 174 {
 175         struct panfrost_file_priv *priv = file_priv->driver_priv;
 176         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 177         struct panfrost_gem_mapping *mapping = NULL, *iter;
 178
 179         mutex_lock(&bo->mappings.lock);
 180         list_for_each_entry(iter, &bo->mappings.list, node) {
 181                 if (iter->mmu == &priv->mmu) {
 182                         mapping = iter;
 183                         list_del(&iter->node);
 184                         break;
 185                 }
 186         }
 187         mutex_unlock(&bo->mappings.lock);
 188
 189         panfrost_gem_mapping_put(mapping);
 190 }
 191
 192 static int panfrost_gem_pin(struct drm_gem_object *obj)
 193 {
 194         if (to_panfrost_bo(obj)->is_heap)
 195                 return -EINVAL;
 196
 197         return drm_gem_shmem_pin(obj);
 198 }
 199
 200 static const struct drm_gem_object_funcs panfrost_gem_funcs = {
 201         .free = panfrost_gem_free_object,
 202         .open = panfrost_gem_open,
 203         .close = panfrost_gem_close,
 204         .print_info = drm_gem_shmem_print_info,
 205         .pin = panfrost_gem_pin,
 206         .unpin = drm_gem_shmem_unpin,
 207         .get_sg_table = drm_gem_shmem_get_sg_table,
 208         .vmap = drm_gem_shmem_vmap,
 209         .vunmap = drm_gem_shmem_vunmap,
 210         .mmap = drm_gem_shmem_mmap,
 211 };
 212
 213 /**
 214  * panfrost_gem_create_object - Implementation of driver->gem_create_object.
 215  * @dev: DRM device
 216  * @size: Size in bytes of the memory the object will reference
 217  *
 218  * This lets the GEM helpers allocate object structs for us, and keep
 219  * our BO stats correct.
 220  */
 221 struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size)
 222 {
 223         struct panfrost_gem_object *obj;
 224
 225         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 226         if (!obj)
 227                 return NULL;
 228
 229         INIT_LIST_HEAD(&obj->mappings.list);
 230         mutex_init(&obj->mappings.lock);
 231         obj->base.base.funcs = &panfrost_gem_funcs;
 232
 233         return &obj->base.base;
 234 }
 235
 236 struct panfrost_gem_object *
 237 panfrost_gem_create_with_handle(struct drm_file *file_priv,
 238                                 struct drm_device *dev, size_t size,
 239                                 u32 flags,
 240                                 uint32_t *handle)
 241 {
 242         int ret;
 243         struct drm_gem_shmem_object *shmem;
 244         struct panfrost_gem_object *bo;
 245
 246         /* Round up heap allocations to 2MB to keep fault handling simple */
 247         if (flags & PANFROST_BO_HEAP)
 248                 size = roundup(size, SZ_2M);
 249
 250         shmem = drm_gem_shmem_create(dev, size);
 251         if (IS_ERR(shmem))
 252                 return ERR_CAST(shmem);
 253
 254         bo = to_panfrost_bo(&shmem->base);
 255         bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
 256         bo->is_heap = !!(flags & PANFROST_BO_HEAP);
 257
 258         /*
 259          * Allocate an id of idr table where the obj is registered
 260          * and handle has the id what user can see.
 261          */
 262         ret = drm_gem_handle_create(file_priv, &shmem->base, handle);
 263         /* drop reference from allocate - handle holds it now. */
 264         drm_gem_object_put(&shmem->base);
 265         if (ret)
 266                 return ERR_PTR(ret);
 267
 268         return bo;
 269 }
 270
 271 struct drm_gem_object *
 272 panfrost_gem_prime_import_sg_table(struct drm_device *dev,
 273                                    struct dma_buf_attachment *attach,
 274                                    struct sg_table *sgt)
 275 {
 276         struct drm_gem_object *obj;
 277         struct panfrost_gem_object *bo;
 278
 279         obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
 280         if (IS_ERR(obj))
 281                 return ERR_CAST(obj);
 282
 283         bo = to_panfrost_bo(obj);
 284         bo->noexec = true;
 285
 286         return obj;
 287 }