drivers/gpu/drm/panfrost/panfrost_gem.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
   3
   4 #include <linux/err.h>
   5 #include <linux/slab.h>
   6 #include <linux/dma-buf.h>
   7 #include <linux/dma-mapping.h>
   8
   9 #include <drm/panfrost_drm.h>
  10 #include "panfrost_device.h"
  11 #include "panfrost_gem.h"
  12 #include "panfrost_mmu.h"
  13
  14 /* Called DRM core on the last userspace/kernel unreference of the
  15  * BO.
  16  */
  17 static void panfrost_gem_free_object(struct drm_gem_object *obj)
  18 {
  19         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
  20         struct panfrost_device *pfdev = obj->dev->dev_private;
  21
  22         /*
  23          * Make sure the BO is no longer inserted in the shrinker list before
  24          * taking care of the destruction itself. If we don't do that we have a
  25          * race condition between this function and what's done in
  26          * panfrost_gem_shrinker_scan().
  27          */
  28         mutex_lock(&pfdev->shrinker_lock);
  29         list_del_init(&bo->base.madv_list);
  30         mutex_unlock(&pfdev->shrinker_lock);
  31
  32         /*
  33          * If we still have mappings attached to the BO, there's a problem in
  34          * our refcounting.
  35          */
  36         WARN_ON_ONCE(!list_empty(&bo->mappings.list));
  37
  38         if (bo->sgts) {
  39                 int i;
  40                 int n_sgt = bo->base.base.size / SZ_2M;
  41
  42                 for (i = 0; i < n_sgt; i++) {
  43                         if (bo->sgts[i].sgl) {
  44                                 dma_unmap_sgtable(pfdev->dev, &bo->sgts[i],
  45                                                   DMA_BIDIRECTIONAL, 0);
  46                                 sg_free_table(&bo->sgts[i]);
  47                         }
  48                 }
  49                 kvfree(bo->sgts);
  50         }
  51
  52         drm_gem_shmem_free_object(obj);
  53 }
  54
  55 struct panfrost_gem_mapping *
  56 panfrost_gem_mapping_get(struct panfrost_gem_object *bo,
  57                          struct panfrost_file_priv *priv)
  58 {
  59         struct panfrost_gem_mapping *iter, *mapping = NULL;
  60
  61         mutex_lock(&bo->mappings.lock);
  62         list_for_each_entry(iter, &bo->mappings.list, node) {
  63                 if (iter->mmu == priv->mmu) {
  64                         kref_get(&iter->refcount);
  65                         mapping = iter;
  66                         break;
  67                 }
  68         }
  69         mutex_unlock(&bo->mappings.lock);
  70
  71         return mapping;
  72 }
  73
  74 static void
  75 panfrost_gem_teardown_mapping(struct panfrost_gem_mapping *mapping)
  76 {
  77         if (mapping->active)
  78                 panfrost_mmu_unmap(mapping);
  79
  80         spin_lock(&mapping->mmu->mm_lock);
  81         if (drm_mm_node_allocated(&mapping->mmnode))
  82                 drm_mm_remove_node(&mapping->mmnode);
  83         spin_unlock(&mapping->mmu->mm_lock);
  84 }
  85
  86 static void panfrost_gem_mapping_release(struct kref *kref)
  87 {
  88         struct panfrost_gem_mapping *mapping;
  89
  90         mapping = container_of(kref, struct panfrost_gem_mapping, refcount);
  91
  92         panfrost_gem_teardown_mapping(mapping);
  93         drm_gem_object_put(&mapping->obj->base.base);
  94         panfrost_mmu_ctx_put(mapping->mmu);
  95         kfree(mapping);
  96 }
  97
  98 void panfrost_gem_mapping_put(struct panfrost_gem_mapping *mapping)
  99 {
 100         if (!mapping)
 101                 return;
 102
 103         kref_put(&mapping->refcount, panfrost_gem_mapping_release);
 104 }
 105
 106 void panfrost_gem_teardown_mappings_locked(struct panfrost_gem_object *bo)
 107 {
 108         struct panfrost_gem_mapping *mapping;
 109
 110         list_for_each_entry(mapping, &bo->mappings.list, node)
 111                 panfrost_gem_teardown_mapping(mapping);
 112 }
 113
 114 int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_priv)
 115 {
 116         int ret;
 117         size_t size = obj->size;
 118         u64 align;
 119         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 120         unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0;
 121         struct panfrost_file_priv *priv = file_priv->driver_priv;
 122         struct panfrost_gem_mapping *mapping;
 123
 124         mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 125         if (!mapping)
 126                 return -ENOMEM;
 127
 128         INIT_LIST_HEAD(&mapping->node);
 129         kref_init(&mapping->refcount);
 130         drm_gem_object_get(obj);
 131         mapping->obj = bo;
 132
 133         /*
 134          * Executable buffers cannot cross a 16MB boundary as the program
 135          * counter is 24-bits. We assume executable buffers will be less than
 136          * 16MB and aligning executable buffers to their size will avoid
 137          * crossing a 16MB boundary.
 138          */
 139         if (!bo->noexec)
 140                 align = size >> PAGE_SHIFT;
 141         else
 142                 align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0;
 143
 144         mapping->mmu = panfrost_mmu_ctx_get(priv->mmu);
 145         spin_lock(&mapping->mmu->mm_lock);
 146         ret = drm_mm_insert_node_generic(&mapping->mmu->mm, &mapping->mmnode,
 147                                          size >> PAGE_SHIFT, align, color, 0);
 148         spin_unlock(&mapping->mmu->mm_lock);
 149         if (ret)
 150                 goto err;
 151
 152         if (!bo->is_heap) {
 153                 ret = panfrost_mmu_map(mapping);
 154                 if (ret)
 155                         goto err;
 156         }
 157
 158         mutex_lock(&bo->mappings.lock);
 159         WARN_ON(bo->base.madv != PANFROST_MADV_WILLNEED);
 160         list_add_tail(&mapping->node, &bo->mappings.list);
 161         mutex_unlock(&bo->mappings.lock);
 162
 163 err:
 164         if (ret)
 165                 panfrost_gem_mapping_put(mapping);
 166         return ret;
 167 }
 168
 169 void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv)
 170 {
 171         struct panfrost_file_priv *priv = file_priv->driver_priv;
 172         struct panfrost_gem_object *bo = to_panfrost_bo(obj);
 173         struct panfrost_gem_mapping *mapping = NULL, *iter;
 174
 175         mutex_lock(&bo->mappings.lock);
 176         list_for_each_entry(iter, &bo->mappings.list, node) {
 177                 if (iter->mmu == priv->mmu) {
 178                         mapping = iter;
 179                         list_del(&iter->node);
 180                         break;
 181                 }
 182         }
 183         mutex_unlock(&bo->mappings.lock);
 184
 185         panfrost_gem_mapping_put(mapping);
 186 }
 187
 188 static int panfrost_gem_pin(struct drm_gem_object *obj)
 189 {
 190         if (to_panfrost_bo(obj)->is_heap)
 191                 return -EINVAL;
 192
 193         return drm_gem_shmem_pin(obj);
 194 }
 195
 196 static const struct drm_gem_object_funcs panfrost_gem_funcs = {
 197         .free = panfrost_gem_free_object,
 198         .open = panfrost_gem_open,
 199         .close = panfrost_gem_close,
 200         .print_info = drm_gem_shmem_print_info,
 201         .pin = panfrost_gem_pin,
 202         .unpin = drm_gem_shmem_unpin,
 203         .get_sg_table = drm_gem_shmem_get_sg_table,
 204         .vmap = drm_gem_shmem_vmap,
 205         .vunmap = drm_gem_shmem_vunmap,
 206         .mmap = drm_gem_shmem_mmap,
 207 };
 208
 209 /**
 210  * panfrost_gem_create_object - Implementation of driver->gem_create_object.
 211  * @dev: DRM device
 212  * @size: Size in bytes of the memory the object will reference
 213  *
 214  * This lets the GEM helpers allocate object structs for us, and keep
 215  * our BO stats correct.
 216  */
 217 struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size)
 218 {
 219         struct panfrost_device *pfdev = dev->dev_private;
 220         struct panfrost_gem_object *obj;
 221
 222         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 223         if (!obj)
 224                 return NULL;
 225
 226         INIT_LIST_HEAD(&obj->mappings.list);
 227         mutex_init(&obj->mappings.lock);
 228         obj->base.base.funcs = &panfrost_gem_funcs;
 229         obj->base.map_wc = !pfdev->coherent;
 230
 231         return &obj->base.base;
 232 }
 233
 234 struct panfrost_gem_object *
 235 panfrost_gem_create_with_handle(struct drm_file *file_priv,
 236                                 struct drm_device *dev, size_t size,
 237                                 u32 flags,
 238                                 uint32_t *handle)
 239 {
 240         int ret;
 241         struct drm_gem_shmem_object *shmem;
 242         struct panfrost_gem_object *bo;
 243
 244         /* Round up heap allocations to 2MB to keep fault handling simple */
 245         if (flags & PANFROST_BO_HEAP)
 246                 size = roundup(size, SZ_2M);
 247
 248         shmem = drm_gem_shmem_create(dev, size);
 249         if (IS_ERR(shmem))
 250                 return ERR_CAST(shmem);
 251
 252         bo = to_panfrost_bo(&shmem->base);
 253         bo->noexec = !!(flags & PANFROST_BO_NOEXEC);
 254         bo->is_heap = !!(flags & PANFROST_BO_HEAP);
 255
 256         /*
 257          * Allocate an id of idr table where the obj is registered
 258          * and handle has the id what user can see.
 259          */
 260         ret = drm_gem_handle_create(file_priv, &shmem->base, handle);
 261         /* drop reference from allocate - handle holds it now. */
 262         drm_gem_object_put(&shmem->base);
 263         if (ret)
 264                 return ERR_PTR(ret);
 265
 266         return bo;
 267 }
 268
 269 struct drm_gem_object *
 270 panfrost_gem_prime_import_sg_table(struct drm_device *dev,
 271                                    struct dma_buf_attachment *attach,
 272                                    struct sg_table *sgt)
 273 {
 274         struct drm_gem_object *obj;
 275         struct panfrost_gem_object *bo;
 276
 277         obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
 278         if (IS_ERR(obj))
 279                 return ERR_CAST(obj);
 280
 281         bo = to_panfrost_bo(obj);
 282         bo->noexec = true;
 283
 284         return obj;
 285 }