drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/errno.h>
   7 #include <linux/interrupt.h>
   8 #include <linux/pagemap.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/io.h>
  11 #include <linux/platform_device.h>
  12 #include <linux/uaccess.h>
  13 #include <linux/mm.h>
  14 #include <linux/of.h>
  15 #include <linux/slab.h>
  16 #include <soc/bcm2835/raspberrypi-firmware.h>
  17
  18 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
  19
  20 #include "vchiq_arm.h"
  21 #include "vchiq_connected.h"
  22 #include "vchiq_pagelist.h"
  23
  24 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
  25
  26 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
  27 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX  1
  28
  29 #define BELL0   0x00
  30 #define BELL2   0x08
  31
  32 #define ARM_DS_ACTIVE   BIT(2)
  33
  34 struct vchiq_2835_state {
  35         int inited;
  36         struct vchiq_arm_state arm_state;
  37 };
  38
  39 struct vchiq_pagelist_info {
  40         struct pagelist *pagelist;
  41         size_t pagelist_buffer_size;
  42         dma_addr_t dma_addr;
  43         enum dma_data_direction dma_dir;
  44         unsigned int num_pages;
  45         unsigned int pages_need_release;
  46         struct page **pages;
  47         struct scatterlist *scatterlist;
  48         unsigned int scatterlist_mapped;
  49 };
  50
  51 static void __iomem *g_regs;
  52 /* This value is the size of the L2 cache lines as understood by the
  53  * VPU firmware, which determines the required alignment of the
  54  * offsets/sizes in pagelists.
  55  *
  56  * Modern VPU firmware looks for a DT "cache-line-size" property in
  57  * the VCHIQ node and will overwrite it with the actual L2 cache size,
  58  * which the kernel must then respect.  That property was rejected
  59  * upstream, so we have to use the VPU firmware's compatibility value
  60  * of 32.
  61  */
  62 static unsigned int g_cache_line_size = 32;
  63 static unsigned int g_fragments_size;
  64 static char *g_fragments_base;
  65 static char *g_free_fragments;
  66 static struct semaphore g_free_fragments_sema;
  67 static struct device *g_dev;
  68
  69 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
  70
  71 static irqreturn_t
  72 vchiq_doorbell_irq(int irq, void *dev_id);
  73
  74 static struct vchiq_pagelist_info *
  75 create_pagelist(char *buf, char __user *ubuf, size_t count, unsigned short type);
  76
  77 static void
  78 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
  79               int actual);
  80
  81 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
  82 {
  83         struct device *dev = &pdev->dev;
  84         struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
  85         struct rpi_firmware *fw = drvdata->fw;
  86         struct vchiq_slot_zero *vchiq_slot_zero;
  87         void *slot_mem;
  88         dma_addr_t slot_phys;
  89         u32 channelbase;
  90         int slot_mem_size, frag_mem_size;
  91         int err, irq, i;
  92
  93         /*
  94          * VCHI messages between the CPU and firmware use
  95          * 32-bit bus addresses.
  96          */
  97         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
  98
  99         if (err < 0)
 100                 return err;
 101
 102         g_cache_line_size = drvdata->cache_line_size;
 103         g_fragments_size = 2 * g_cache_line_size;
 104
 105         /* Allocate space for the channels in coherent memory */
 106         slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
 107         frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
 108
 109         slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
 110                                        &slot_phys, GFP_KERNEL);
 111         if (!slot_mem) {
 112                 dev_err(dev, "could not allocate DMA memory\n");
 113                 return -ENOMEM;
 114         }
 115
 116         WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
 117
 118         vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
 119         if (!vchiq_slot_zero)
 120                 return -EINVAL;
 121
 122         vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
 123                 (int)slot_phys + slot_mem_size;
 124         vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
 125                 MAX_FRAGMENTS;
 126
 127         g_fragments_base = (char *)slot_mem + slot_mem_size;
 128
 129         g_free_fragments = g_fragments_base;
 130         for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
 131                 *(char **)&g_fragments_base[i*g_fragments_size] =
 132                         &g_fragments_base[(i + 1)*g_fragments_size];
 133         }
 134         *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
 135         sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
 136
 137         err = vchiq_init_state(state, vchiq_slot_zero);
 138         if (err)
 139                 return err;
 140
 141         g_regs = devm_platform_ioremap_resource(pdev, 0);
 142         if (IS_ERR(g_regs))
 143                 return PTR_ERR(g_regs);
 144
 145         irq = platform_get_irq(pdev, 0);
 146         if (irq <= 0)
 147                 return irq;
 148
 149         err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
 150                                "VCHIQ doorbell", state);
 151         if (err) {
 152                 dev_err(dev, "failed to register irq=%d\n", irq);
 153                 return err;
 154         }
 155
 156         /* Send the base address of the slots to VideoCore */
 157         channelbase = slot_phys;
 158         err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
 159                                     &channelbase, sizeof(channelbase));
 160         if (err || channelbase) {
 161                 dev_err(dev, "failed to set channelbase\n");
 162                 return err ? : -ENXIO;
 163         }
 164
 165         g_dev = dev;
 166         vchiq_log_info(vchiq_arm_log_level,
 167                 "vchiq_init - done (slots %pK, phys %pad)",
 168                 vchiq_slot_zero, &slot_phys);
 169
 170         vchiq_call_connected_callbacks();
 171
 172         return 0;
 173 }
 174
 175 int
 176 vchiq_platform_init_state(struct vchiq_state *state)
 177 {
 178         struct vchiq_2835_state *platform_state;
 179
 180         state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
 181         if (!state->platform_state)
 182                 return -ENOMEM;
 183
 184         platform_state = (struct vchiq_2835_state *)state->platform_state;
 185
 186         platform_state->inited = 1;
 187         vchiq_arm_init_state(state, &platform_state->arm_state);
 188
 189         return 0;
 190 }
 191
 192 struct vchiq_arm_state*
 193 vchiq_platform_get_arm_state(struct vchiq_state *state)
 194 {
 195         struct vchiq_2835_state *platform_state;
 196
 197         platform_state   = (struct vchiq_2835_state *)state->platform_state;
 198
 199         WARN_ON_ONCE(!platform_state->inited);
 200
 201         return &platform_state->arm_state;
 202 }
 203
 204 void
 205 remote_event_signal(struct remote_event *event)
 206 {
 207         wmb();
 208
 209         event->fired = 1;
 210
 211         dsb(sy);         /* data barrier operation */
 212
 213         if (event->armed)
 214                 writel(0, g_regs + BELL2); /* trigger vc interrupt */
 215 }
 216
 217 int
 218 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset,
 219                         void __user *uoffset, int size, int dir)
 220 {
 221         struct vchiq_pagelist_info *pagelistinfo;
 222
 223         pagelistinfo = create_pagelist(offset, uoffset, size,
 224                                        (dir == VCHIQ_BULK_RECEIVE)
 225                                        ? PAGELIST_READ
 226                                        : PAGELIST_WRITE);
 227
 228         if (!pagelistinfo)
 229                 return -ENOMEM;
 230
 231         bulk->data = pagelistinfo->dma_addr;
 232
 233         /*
 234          * Store the pagelistinfo address in remote_data,
 235          * which isn't used by the slave.
 236          */
 237         bulk->remote_data = pagelistinfo;
 238
 239         return 0;
 240 }
 241
 242 void
 243 vchiq_complete_bulk(struct vchiq_bulk *bulk)
 244 {
 245         if (bulk && bulk->remote_data && bulk->actual)
 246                 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
 247                               bulk->actual);
 248 }
 249
 250 int vchiq_dump_platform_state(void *dump_context)
 251 {
 252         char buf[80];
 253         int len;
 254
 255         len = snprintf(buf, sizeof(buf),
 256                 "  Platform: 2835 (VC master)");
 257         return vchiq_dump(dump_context, buf, len + 1);
 258 }
 259
 260 /*
 261  * Local functions
 262  */
 263
 264 static irqreturn_t
 265 vchiq_doorbell_irq(int irq, void *dev_id)
 266 {
 267         struct vchiq_state *state = dev_id;
 268         irqreturn_t ret = IRQ_NONE;
 269         unsigned int status;
 270
 271         /* Read (and clear) the doorbell */
 272         status = readl(g_regs + BELL0);
 273
 274         if (status & ARM_DS_ACTIVE) {  /* Was the doorbell rung? */
 275                 remote_event_pollall(state);
 276                 ret = IRQ_HANDLED;
 277         }
 278
 279         return ret;
 280 }
 281
 282 static void
 283 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
 284 {
 285         if (pagelistinfo->scatterlist_mapped) {
 286                 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
 287                              pagelistinfo->num_pages, pagelistinfo->dma_dir);
 288         }
 289
 290         if (pagelistinfo->pages_need_release)
 291                 unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages);
 292
 293         dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
 294                           pagelistinfo->pagelist, pagelistinfo->dma_addr);
 295 }
 296
 297 /* There is a potential problem with partial cache lines (pages?)
 298  * at the ends of the block when reading. If the CPU accessed anything in
 299  * the same line (page?) then it may have pulled old data into the cache,
 300  * obscuring the new data underneath. We can solve this by transferring the
 301  * partial cache lines separately, and allowing the ARM to copy into the
 302  * cached area.
 303  */
 304
 305 static struct vchiq_pagelist_info *
 306 create_pagelist(char *buf, char __user *ubuf,
 307                 size_t count, unsigned short type)
 308 {
 309         struct pagelist *pagelist;
 310         struct vchiq_pagelist_info *pagelistinfo;
 311         struct page **pages;
 312         u32 *addrs;
 313         unsigned int num_pages, offset, i, k;
 314         int actual_pages;
 315         size_t pagelist_size;
 316         struct scatterlist *scatterlist, *sg;
 317         int dma_buffers;
 318         dma_addr_t dma_addr;
 319
 320         if (count >= INT_MAX - PAGE_SIZE)
 321                 return NULL;
 322
 323         if (buf)
 324                 offset = (uintptr_t)buf & (PAGE_SIZE - 1);
 325         else
 326                 offset = (uintptr_t)ubuf & (PAGE_SIZE - 1);
 327         num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
 328
 329         if (num_pages > (SIZE_MAX - sizeof(struct pagelist) -
 330                          sizeof(struct vchiq_pagelist_info)) /
 331                         (sizeof(u32) + sizeof(pages[0]) +
 332                          sizeof(struct scatterlist)))
 333                 return NULL;
 334
 335         pagelist_size = sizeof(struct pagelist) +
 336                         (num_pages * sizeof(u32)) +
 337                         (num_pages * sizeof(pages[0]) +
 338                         (num_pages * sizeof(struct scatterlist))) +
 339                         sizeof(struct vchiq_pagelist_info);
 340
 341         /* Allocate enough storage to hold the page pointers and the page
 342          * list
 343          */
 344         pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
 345                                       GFP_KERNEL);
 346
 347         vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
 348
 349         if (!pagelist)
 350                 return NULL;
 351
 352         addrs           = pagelist->addrs;
 353         pages           = (struct page **)(addrs + num_pages);
 354         scatterlist     = (struct scatterlist *)(pages + num_pages);
 355         pagelistinfo    = (struct vchiq_pagelist_info *)
 356                           (scatterlist + num_pages);
 357
 358         pagelist->length = count;
 359         pagelist->type = type;
 360         pagelist->offset = offset;
 361
 362         /* Populate the fields of the pagelistinfo structure */
 363         pagelistinfo->pagelist = pagelist;
 364         pagelistinfo->pagelist_buffer_size = pagelist_size;
 365         pagelistinfo->dma_addr = dma_addr;
 366         pagelistinfo->dma_dir =  (type == PAGELIST_WRITE) ?
 367                                   DMA_TO_DEVICE : DMA_FROM_DEVICE;
 368         pagelistinfo->num_pages = num_pages;
 369         pagelistinfo->pages_need_release = 0;
 370         pagelistinfo->pages = pages;
 371         pagelistinfo->scatterlist = scatterlist;
 372         pagelistinfo->scatterlist_mapped = 0;
 373
 374         if (buf) {
 375                 unsigned long length = count;
 376                 unsigned int off = offset;
 377
 378                 for (actual_pages = 0; actual_pages < num_pages;
 379                      actual_pages++) {
 380                         struct page *pg =
 381                                 vmalloc_to_page((buf +
 382                                                  (actual_pages * PAGE_SIZE)));
 383                         size_t bytes = PAGE_SIZE - off;
 384
 385                         if (!pg) {
 386                                 cleanup_pagelistinfo(pagelistinfo);
 387                                 return NULL;
 388                         }
 389
 390                         if (bytes > length)
 391                                 bytes = length;
 392                         pages[actual_pages] = pg;
 393                         length -= bytes;
 394                         off = 0;
 395                 }
 396                 /* do not try and release vmalloc pages */
 397         } else {
 398                 actual_pages = pin_user_pages_fast(
 399                                           (unsigned long)ubuf & PAGE_MASK,
 400                                           num_pages,
 401                                           type == PAGELIST_READ,
 402                                           pages);
 403
 404                 if (actual_pages != num_pages) {
 405                         vchiq_log_info(vchiq_arm_log_level,
 406                                        "%s - only %d/%d pages locked",
 407                                        __func__, actual_pages, num_pages);
 408
 409                         /* This is probably due to the process being killed */
 410                         if (actual_pages > 0)
 411                                 unpin_user_pages(pages, actual_pages);
 412                         cleanup_pagelistinfo(pagelistinfo);
 413                         return NULL;
 414                 }
 415                  /* release user pages */
 416                 pagelistinfo->pages_need_release = 1;
 417         }
 418
 419         /*
 420          * Initialize the scatterlist so that the magic cookie
 421          *  is filled if debugging is enabled
 422          */
 423         sg_init_table(scatterlist, num_pages);
 424         /* Now set the pages for each scatterlist */
 425         for (i = 0; i < num_pages; i++) {
 426                 unsigned int len = PAGE_SIZE - offset;
 427
 428                 if (len > count)
 429                         len = count;
 430                 sg_set_page(scatterlist + i, pages[i], len, offset);
 431                 offset = 0;
 432                 count -= len;
 433         }
 434
 435         dma_buffers = dma_map_sg(g_dev,
 436                                  scatterlist,
 437                                  num_pages,
 438                                  pagelistinfo->dma_dir);
 439
 440         if (dma_buffers == 0) {
 441                 cleanup_pagelistinfo(pagelistinfo);
 442                 return NULL;
 443         }
 444
 445         pagelistinfo->scatterlist_mapped = 1;
 446
 447         /* Combine adjacent blocks for performance */
 448         k = 0;
 449         for_each_sg(scatterlist, sg, dma_buffers, i) {
 450                 u32 len = sg_dma_len(sg);
 451                 u32 addr = sg_dma_address(sg);
 452
 453                 /* Note: addrs is the address + page_count - 1
 454                  * The firmware expects blocks after the first to be page-
 455                  * aligned and a multiple of the page size
 456                  */
 457                 WARN_ON(len == 0);
 458                 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
 459                 WARN_ON(i && (addr & ~PAGE_MASK));
 460                 if (k > 0 &&
 461                     ((addrs[k - 1] & PAGE_MASK) +
 462                      (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
 463                     == (addr & PAGE_MASK))
 464                         addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
 465                 else
 466                         addrs[k++] = (addr & PAGE_MASK) |
 467                                 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
 468         }
 469
 470         /* Partial cache lines (fragments) require special measures */
 471         if ((type == PAGELIST_READ) &&
 472                 ((pagelist->offset & (g_cache_line_size - 1)) ||
 473                 ((pagelist->offset + pagelist->length) &
 474                 (g_cache_line_size - 1)))) {
 475                 char *fragments;
 476
 477                 if (down_interruptible(&g_free_fragments_sema)) {
 478                         cleanup_pagelistinfo(pagelistinfo);
 479                         return NULL;
 480                 }
 481
 482                 WARN_ON(!g_free_fragments);
 483
 484                 down(&g_free_fragments_mutex);
 485                 fragments = g_free_fragments;
 486                 WARN_ON(!fragments);
 487                 g_free_fragments = *(char **) g_free_fragments;
 488                 up(&g_free_fragments_mutex);
 489                 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
 490                         (fragments - g_fragments_base) / g_fragments_size;
 491         }
 492
 493         return pagelistinfo;
 494 }
 495
 496 static void
 497 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
 498               int actual)
 499 {
 500         struct pagelist *pagelist = pagelistinfo->pagelist;
 501         struct page **pages = pagelistinfo->pages;
 502         unsigned int num_pages = pagelistinfo->num_pages;
 503
 504         vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
 505                         __func__, pagelistinfo->pagelist, actual);
 506
 507         /*
 508          * NOTE: dma_unmap_sg must be called before the
 509          * cpu can touch any of the data/pages.
 510          */
 511         dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
 512                      pagelistinfo->num_pages, pagelistinfo->dma_dir);
 513         pagelistinfo->scatterlist_mapped = 0;
 514
 515         /* Deal with any partial cache lines (fragments) */
 516         if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
 517                 char *fragments = g_fragments_base +
 518                         (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
 519                         g_fragments_size;
 520                 int head_bytes, tail_bytes;
 521
 522                 head_bytes = (g_cache_line_size - pagelist->offset) &
 523                         (g_cache_line_size - 1);
 524                 tail_bytes = (pagelist->offset + actual) &
 525                         (g_cache_line_size - 1);
 526
 527                 if ((actual >= 0) && (head_bytes != 0)) {
 528                         if (head_bytes > actual)
 529                                 head_bytes = actual;
 530
 531                         memcpy((char *)kmap(pages[0]) +
 532                                 pagelist->offset,
 533                                 fragments,
 534                                 head_bytes);
 535                         kunmap(pages[0]);
 536                 }
 537                 if ((actual >= 0) && (head_bytes < actual) &&
 538                         (tail_bytes != 0)) {
 539                         memcpy((char *)kmap(pages[num_pages - 1]) +
 540                                 ((pagelist->offset + actual) &
 541                                 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
 542                                 fragments + g_cache_line_size,
 543                                 tail_bytes);
 544                         kunmap(pages[num_pages - 1]);
 545                 }
 546
 547                 down(&g_free_fragments_mutex);
 548                 *(char **)fragments = g_free_fragments;
 549                 g_free_fragments = fragments;
 550                 up(&g_free_fragments_mutex);
 551                 up(&g_free_fragments_sema);
 552         }
 553
 554         /* Need to mark all the pages dirty. */
 555         if (pagelist->type != PAGELIST_WRITE &&
 556             pagelistinfo->pages_need_release) {
 557                 unsigned int i;
 558
 559                 for (i = 0; i < num_pages; i++)
 560                         set_page_dirty(pages[i]);
 561         }
 562
 563         cleanup_pagelistinfo(pagelistinfo);
 564 }