drivers/gpu/drm/ttm/ttm_resource.c

   1 /*
   2  * Copyright 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Christian König
  23  */
  24
  25 #include <linux/dma-buf-map.h>
  26 #include <linux/io-mapping.h>
  27 #include <linux/scatterlist.h>
  28
  29 #include <drm/ttm/ttm_resource.h>
  30 #include <drm/ttm/ttm_bo_driver.h>
  31
  32 void ttm_resource_init(struct ttm_buffer_object *bo,
  33                        const struct ttm_place *place,
  34                        struct ttm_resource *res)
  35 {
  36         res->start = 0;
  37         res->num_pages = PFN_UP(bo->base.size);
  38         res->mem_type = place->mem_type;
  39         res->placement = place->flags;
  40         res->bus.addr = NULL;
  41         res->bus.offset = 0;
  42         res->bus.is_iomem = false;
  43         res->bus.caching = ttm_cached;
  44 }
  45 EXPORT_SYMBOL(ttm_resource_init);
  46
  47 int ttm_resource_alloc(struct ttm_buffer_object *bo,
  48                        const struct ttm_place *place,
  49                        struct ttm_resource **res_ptr)
  50 {
  51         struct ttm_resource_manager *man =
  52                 ttm_manager_type(bo->bdev, place->mem_type);
  53
  54         return man->func->alloc(man, bo, place, res_ptr);
  55 }
  56
  57 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
  58 {
  59         struct ttm_resource_manager *man;
  60
  61         if (!*res)
  62                 return;
  63
  64         man = ttm_manager_type(bo->bdev, (*res)->mem_type);
  65         man->func->free(man, *res);
  66         *res = NULL;
  67 }
  68 EXPORT_SYMBOL(ttm_resource_free);
  69
  70 static bool ttm_resource_places_compat(struct ttm_resource *res,
  71                                        const struct ttm_place *places,
  72                                        unsigned num_placement)
  73 {
  74         unsigned i;
  75
  76         if (res->placement & TTM_PL_FLAG_TEMPORARY)
  77                 return false;
  78
  79         for (i = 0; i < num_placement; i++) {
  80                 const struct ttm_place *heap = &places[i];
  81
  82                 if (res->start < heap->fpfn || (heap->lpfn &&
  83                     (res->start + res->num_pages) > heap->lpfn))
  84                         continue;
  85
  86                 if ((res->mem_type == heap->mem_type) &&
  87                     (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
  88                      (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
  89                         return true;
  90         }
  91         return false;
  92 }
  93
  94 /**
  95  * ttm_resource_compat - check if resource is compatible with placement
  96  *
  97  * @res: the resource to check
  98  * @placement: the placement to check against
  99  *
 100  * Returns true if the placement is compatible.
 101  */
 102 bool ttm_resource_compat(struct ttm_resource *res,
 103                          struct ttm_placement *placement)
 104 {
 105         if (ttm_resource_places_compat(res, placement->placement,
 106                                        placement->num_placement))
 107                 return true;
 108
 109         if ((placement->busy_placement != placement->placement ||
 110              placement->num_busy_placement > placement->num_placement) &&
 111             ttm_resource_places_compat(res, placement->busy_placement,
 112                                        placement->num_busy_placement))
 113                 return true;
 114
 115         return false;
 116 }
 117 EXPORT_SYMBOL(ttm_resource_compat);
 118
 119 /**
 120  * ttm_resource_manager_init
 121  *
 122  * @man: memory manager object to init
 123  * @p_size: size managed area in pages.
 124  *
 125  * Initialise core parts of a manager object.
 126  */
 127 void ttm_resource_manager_init(struct ttm_resource_manager *man,
 128                                unsigned long p_size)
 129 {
 130         unsigned i;
 131
 132         spin_lock_init(&man->move_lock);
 133         man->size = p_size;
 134
 135         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 136                 INIT_LIST_HEAD(&man->lru[i]);
 137         man->move = NULL;
 138 }
 139 EXPORT_SYMBOL(ttm_resource_manager_init);
 140
 141 /*
 142  * ttm_resource_manager_evict_all
 143  *
 144  * @bdev - device to use
 145  * @man - manager to use
 146  *
 147  * Evict all the objects out of a memory manager until it is empty.
 148  * Part of memory manager cleanup sequence.
 149  */
 150 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
 151                                    struct ttm_resource_manager *man)
 152 {
 153         struct ttm_operation_ctx ctx = {
 154                 .interruptible = false,
 155                 .no_wait_gpu = false,
 156                 .force_alloc = true
 157         };
 158         struct dma_fence *fence;
 159         int ret;
 160         unsigned i;
 161
 162         /*
 163          * Can't use standard list traversal since we're unlocking.
 164          */
 165
 166         spin_lock(&bdev->lru_lock);
 167         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 168                 while (!list_empty(&man->lru[i])) {
 169                         spin_unlock(&bdev->lru_lock);
 170                         ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
 171                                                   NULL);
 172                         if (ret)
 173                                 return ret;
 174                         spin_lock(&bdev->lru_lock);
 175                 }
 176         }
 177         spin_unlock(&bdev->lru_lock);
 178
 179         spin_lock(&man->move_lock);
 180         fence = dma_fence_get(man->move);
 181         spin_unlock(&man->move_lock);
 182
 183         if (fence) {
 184                 ret = dma_fence_wait(fence, false);
 185                 dma_fence_put(fence);
 186                 if (ret)
 187                         return ret;
 188         }
 189
 190         return 0;
 191 }
 192 EXPORT_SYMBOL(ttm_resource_manager_evict_all);
 193
 194 /**
 195  * ttm_resource_manager_debug
 196  *
 197  * @man: manager type to dump.
 198  * @p: printer to use for debug.
 199  */
 200 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
 201                                 struct drm_printer *p)
 202 {
 203         drm_printf(p, "  use_type: %d\n", man->use_type);
 204         drm_printf(p, "  use_tt: %d\n", man->use_tt);
 205         drm_printf(p, "  size: %llu\n", man->size);
 206         if (man->func->debug)
 207                 man->func->debug(man, p);
 208 }
 209 EXPORT_SYMBOL(ttm_resource_manager_debug);
 210
 211 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
 212                                           struct dma_buf_map *dmap,
 213                                           pgoff_t i)
 214 {
 215         struct ttm_kmap_iter_iomap *iter_io =
 216                 container_of(iter, typeof(*iter_io), base);
 217         void __iomem *addr;
 218
 219 retry:
 220         while (i >= iter_io->cache.end) {
 221                 iter_io->cache.sg = iter_io->cache.sg ?
 222                         sg_next(iter_io->cache.sg) : iter_io->st->sgl;
 223                 iter_io->cache.i = iter_io->cache.end;
 224                 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
 225                         PAGE_SHIFT;
 226                 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
 227                         iter_io->start;
 228         }
 229
 230         if (i < iter_io->cache.i) {
 231                 iter_io->cache.end = 0;
 232                 iter_io->cache.sg = NULL;
 233                 goto retry;
 234         }
 235
 236         addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
 237                                        (((resource_size_t)i - iter_io->cache.i)
 238                                         << PAGE_SHIFT));
 239         dma_buf_map_set_vaddr_iomem(dmap, addr);
 240 }
 241
 242 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
 243                                             struct dma_buf_map *map)
 244 {
 245         io_mapping_unmap_local(map->vaddr_iomem);
 246 }
 247
 248 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
 249         .map_local =  ttm_kmap_iter_iomap_map_local,
 250         .unmap_local = ttm_kmap_iter_iomap_unmap_local,
 251         .maps_tt = false,
 252 };
 253
 254 /**
 255  * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
 256  * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
 257  * @iomap: The struct io_mapping representing the underlying linear io_memory.
 258  * @st: sg_table into @iomap, representing the memory of the struct
 259  * ttm_resource.
 260  * @start: Offset that needs to be subtracted from @st to make
 261  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 262  *
 263  * Return: Pointer to the embedded struct ttm_kmap_iter.
 264  */
 265 struct ttm_kmap_iter *
 266 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
 267                          struct io_mapping *iomap,
 268                          struct sg_table *st,
 269                          resource_size_t start)
 270 {
 271         iter_io->base.ops = &ttm_kmap_iter_io_ops;
 272         iter_io->iomap = iomap;
 273         iter_io->st = st;
 274         iter_io->start = start;
 275         memset(&iter_io->cache, 0, sizeof(iter_io->cache));
 276
 277         return &iter_io->base;
 278 }
 279 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
 280
 281 /**
 282  * DOC: Linear io iterator
 283  *
 284  * This code should die in the not too near future. Best would be if we could
 285  * make io-mapping use memremap for all io memory, and have memremap
 286  * implement a kmap_local functionality. We could then strip a huge amount of
 287  * code. These linear io iterators are implemented to mimic old functionality,
 288  * and they don't use kmap_local semantics at all internally. Rather ioremap or
 289  * friends, and at least on 32-bit they add global TLB flushes and points
 290  * of failure.
 291  */
 292
 293 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
 294                                               struct dma_buf_map *dmap,
 295                                               pgoff_t i)
 296 {
 297         struct ttm_kmap_iter_linear_io *iter_io =
 298                 container_of(iter, typeof(*iter_io), base);
 299
 300         *dmap = iter_io->dmap;
 301         dma_buf_map_incr(dmap, i * PAGE_SIZE);
 302 }
 303
 304 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
 305         .map_local =  ttm_kmap_iter_linear_io_map_local,
 306         .maps_tt = false,
 307 };
 308
 309 /**
 310  * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
 311  * @iter_io: The iterator to initialize
 312  * @bdev: The TTM device
 313  * @mem: The ttm resource representing the iomap.
 314  *
 315  * This function is for internal TTM use only. It sets up a memcpy kmap iterator
 316  * pointing at a linear chunk of io memory.
 317  *
 318  * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
 319  * failure.
 320  */
 321 struct ttm_kmap_iter *
 322 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
 323                              struct ttm_device *bdev,
 324                              struct ttm_resource *mem)
 325 {
 326         int ret;
 327
 328         ret = ttm_mem_io_reserve(bdev, mem);
 329         if (ret)
 330                 goto out_err;
 331         if (!mem->bus.is_iomem) {
 332                 ret = -EINVAL;
 333                 goto out_io_free;
 334         }
 335
 336         if (mem->bus.addr) {
 337                 dma_buf_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
 338                 iter_io->needs_unmap = false;
 339         } else {
 340                 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
 341
 342                 iter_io->needs_unmap = true;
 343                 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
 344                 if (mem->bus.caching == ttm_write_combined)
 345                         dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
 346                                                     ioremap_wc(mem->bus.offset,
 347                                                                bus_size));
 348                 else if (mem->bus.caching == ttm_cached)
 349                         dma_buf_map_set_vaddr(&iter_io->dmap,
 350                                               memremap(mem->bus.offset, bus_size,
 351                                                        MEMREMAP_WB |
 352                                                        MEMREMAP_WT |
 353                                                        MEMREMAP_WC));
 354
 355                 /* If uncached requested or if mapping cached or wc failed */
 356                 if (dma_buf_map_is_null(&iter_io->dmap))
 357                         dma_buf_map_set_vaddr_iomem(&iter_io->dmap,
 358                                                     ioremap(mem->bus.offset,
 359                                                             bus_size));
 360
 361                 if (dma_buf_map_is_null(&iter_io->dmap)) {
 362                         ret = -ENOMEM;
 363                         goto out_io_free;
 364                 }
 365         }
 366
 367         iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
 368         return &iter_io->base;
 369
 370 out_io_free:
 371         ttm_mem_io_free(bdev, mem);
 372 out_err:
 373         return ERR_PTR(ret);
 374 }
 375
 376 /**
 377  * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
 378  * @iter_io: The iterator to initialize
 379  * @bdev: The TTM device
 380  * @mem: The ttm resource representing the iomap.
 381  *
 382  * This function is for internal TTM use only. It cleans up a memcpy kmap
 383  * iterator initialized by ttm_kmap_iter_linear_io_init.
 384  */
 385 void
 386 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
 387                              struct ttm_device *bdev,
 388                              struct ttm_resource *mem)
 389 {
 390         if (iter_io->needs_unmap && dma_buf_map_is_set(&iter_io->dmap)) {
 391                 if (iter_io->dmap.is_iomem)
 392                         iounmap(iter_io->dmap.vaddr_iomem);
 393                 else
 394                         memunmap(iter_io->dmap.vaddr);
 395         }
 396
 397         ttm_mem_io_free(bdev, mem);
 398 }