drivers/staging/vc04_services/interface/vchiq_arm/vchiq_2835_arm.c

   1 /**
   2  * Copyright (c) 2010-2012 Broadcom. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions, and the following disclaimer,
   9  *    without modification.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  * 3. The names of the above-listed copyright holders may not be used
  14  *    to endorse or promote products derived from this software without
  15  *    specific prior written permission.
  16  *
  17  * ALTERNATIVELY, this software may be distributed under the terms of the
  18  * GNU General Public License ("GPL") version 2, as published by the Free
  19  * Software Foundation.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  22  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  23  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  24  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  25  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  26  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  27  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  28  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  29  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  30  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  31  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <linux/kernel.h>
  35 #include <linux/types.h>
  36 #include <linux/errno.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/pagemap.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/io.h>
  41 #include <linux/platform_device.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/mm.h>
  44 #include <linux/of.h>
  45 #include <soc/bcm2835/raspberrypi-firmware.h>
  46
  47 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
  48
  49 #include "vchiq_arm.h"
  50 #include "vchiq_connected.h"
  51 #include "vchiq_pagelist.h"
  52
  53 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
  54
  55 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
  56 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX  1
  57
  58 #define BELL0   0x00
  59 #define BELL2   0x08
  60
  61 struct vchiq_2835_state {
  62         int inited;
  63         struct vchiq_arm_state arm_state;
  64 };
  65
  66 struct vchiq_pagelist_info {
  67         struct pagelist *pagelist;
  68         size_t pagelist_buffer_size;
  69         dma_addr_t dma_addr;
  70         enum dma_data_direction dma_dir;
  71         unsigned int num_pages;
  72         unsigned int pages_need_release;
  73         struct page **pages;
  74         struct scatterlist *scatterlist;
  75         unsigned int scatterlist_mapped;
  76 };
  77
  78 static void __iomem *g_regs;
  79 /* This value is the size of the L2 cache lines as understood by the
  80  * VPU firmware, which determines the required alignment of the
  81  * offsets/sizes in pagelists.
  82  *
  83  * Modern VPU firmware looks for a DT "cache-line-size" property in
  84  * the VCHIQ node and will overwrite it with the actual L2 cache size,
  85  * which the kernel must then respect.  That property was rejected
  86  * upstream, so we have to use the VPU firmware's compatibility value
  87  * of 32.
  88  */
  89 static unsigned int g_cache_line_size = 32;
  90 static unsigned int g_fragments_size;
  91 static char *g_fragments_base;
  92 static char *g_free_fragments;
  93 static struct semaphore g_free_fragments_sema;
  94 static struct device *g_dev;
  95
  96 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
  97
  98 static irqreturn_t
  99 vchiq_doorbell_irq(int irq, void *dev_id);
 100
 101 static struct vchiq_pagelist_info *
 102 create_pagelist(char __user *buf, size_t count, unsigned short type);
 103
 104 static void
 105 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
 106               int actual);
 107
 108 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
 109 {
 110         struct device *dev = &pdev->dev;
 111         struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
 112         struct rpi_firmware *fw = drvdata->fw;
 113         struct vchiq_slot_zero *vchiq_slot_zero;
 114         struct resource *res;
 115         void *slot_mem;
 116         dma_addr_t slot_phys;
 117         u32 channelbase;
 118         int slot_mem_size, frag_mem_size;
 119         int err, irq, i;
 120
 121         /*
 122          * VCHI messages between the CPU and firmware use
 123          * 32-bit bus addresses.
 124          */
 125         err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 126
 127         if (err < 0)
 128                 return err;
 129
 130         g_cache_line_size = drvdata->cache_line_size;
 131         g_fragments_size = 2 * g_cache_line_size;
 132
 133         /* Allocate space for the channels in coherent memory */
 134         slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
 135         frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
 136
 137         slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
 138                                        &slot_phys, GFP_KERNEL);
 139         if (!slot_mem) {
 140                 dev_err(dev, "could not allocate DMA memory\n");
 141                 return -ENOMEM;
 142         }
 143
 144         WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
 145
 146         vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
 147         if (!vchiq_slot_zero)
 148                 return -EINVAL;
 149
 150         vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
 151                 (int)slot_phys + slot_mem_size;
 152         vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
 153                 MAX_FRAGMENTS;
 154
 155         g_fragments_base = (char *)slot_mem + slot_mem_size;
 156
 157         g_free_fragments = g_fragments_base;
 158         for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
 159                 *(char **)&g_fragments_base[i*g_fragments_size] =
 160                         &g_fragments_base[(i + 1)*g_fragments_size];
 161         }
 162         *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
 163         sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
 164
 165         if (vchiq_init_state(state, vchiq_slot_zero) != VCHIQ_SUCCESS)
 166                 return -EINVAL;
 167
 168         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 169         g_regs = devm_ioremap_resource(&pdev->dev, res);
 170         if (IS_ERR(g_regs))
 171                 return PTR_ERR(g_regs);
 172
 173         irq = platform_get_irq(pdev, 0);
 174         if (irq <= 0) {
 175                 dev_err(dev, "failed to get IRQ\n");
 176                 return irq;
 177         }
 178
 179         err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
 180                                "VCHIQ doorbell", state);
 181         if (err) {
 182                 dev_err(dev, "failed to register irq=%d\n", irq);
 183                 return err;
 184         }
 185
 186         /* Send the base address of the slots to VideoCore */
 187         channelbase = slot_phys;
 188         err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
 189                                     &channelbase, sizeof(channelbase));
 190         if (err || channelbase) {
 191                 dev_err(dev, "failed to set channelbase\n");
 192                 return err ? : -ENXIO;
 193         }
 194
 195         g_dev = dev;
 196         vchiq_log_info(vchiq_arm_log_level,
 197                 "vchiq_init - done (slots %pK, phys %pad)",
 198                 vchiq_slot_zero, &slot_phys);
 199
 200         vchiq_call_connected_callbacks();
 201
 202         return 0;
 203 }
 204
 205 VCHIQ_STATUS_T
 206 vchiq_platform_init_state(struct vchiq_state *state)
 207 {
 208         VCHIQ_STATUS_T status = VCHIQ_SUCCESS;
 209         struct vchiq_2835_state *platform_state;
 210
 211         state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
 212         platform_state = (struct vchiq_2835_state *)state->platform_state;
 213
 214         platform_state->inited = 1;
 215         status = vchiq_arm_init_state(state, &platform_state->arm_state);
 216
 217         if (status != VCHIQ_SUCCESS)
 218                 platform_state->inited = 0;
 219
 220         return status;
 221 }
 222
 223 struct vchiq_arm_state*
 224 vchiq_platform_get_arm_state(struct vchiq_state *state)
 225 {
 226         struct vchiq_2835_state *platform_state;
 227
 228         platform_state   = (struct vchiq_2835_state *)state->platform_state;
 229
 230         WARN_ON_ONCE(!platform_state->inited);
 231
 232         return &platform_state->arm_state;
 233 }
 234
 235 void
 236 remote_event_signal(struct remote_event *event)
 237 {
 238         wmb();
 239
 240         event->fired = 1;
 241
 242         dsb(sy);         /* data barrier operation */
 243
 244         if (event->armed)
 245                 writel(0, g_regs + BELL2); /* trigger vc interrupt */
 246 }
 247
 248 VCHIQ_STATUS_T
 249 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset, int size,
 250                         int dir)
 251 {
 252         struct vchiq_pagelist_info *pagelistinfo;
 253
 254         pagelistinfo = create_pagelist((char __user *)offset, size,
 255                                        (dir == VCHIQ_BULK_RECEIVE)
 256                                        ? PAGELIST_READ
 257                                        : PAGELIST_WRITE);
 258
 259         if (!pagelistinfo)
 260                 return VCHIQ_ERROR;
 261
 262         bulk->data = (void *)(unsigned long)pagelistinfo->dma_addr;
 263
 264         /*
 265          * Store the pagelistinfo address in remote_data,
 266          * which isn't used by the slave.
 267          */
 268         bulk->remote_data = pagelistinfo;
 269
 270         return VCHIQ_SUCCESS;
 271 }
 272
 273 void
 274 vchiq_complete_bulk(struct vchiq_bulk *bulk)
 275 {
 276         if (bulk && bulk->remote_data && bulk->actual)
 277                 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
 278                               bulk->actual);
 279 }
 280
 281 void
 282 vchiq_dump_platform_state(void *dump_context)
 283 {
 284         char buf[80];
 285         int len;
 286
 287         len = snprintf(buf, sizeof(buf),
 288                 "  Platform: 2835 (VC master)");
 289         vchiq_dump(dump_context, buf, len + 1);
 290 }
 291
 292 VCHIQ_STATUS_T
 293 vchiq_platform_suspend(struct vchiq_state *state)
 294 {
 295         return VCHIQ_ERROR;
 296 }
 297
 298 VCHIQ_STATUS_T
 299 vchiq_platform_resume(struct vchiq_state *state)
 300 {
 301         return VCHIQ_SUCCESS;
 302 }
 303
 304 void
 305 vchiq_platform_paused(struct vchiq_state *state)
 306 {
 307 }
 308
 309 void
 310 vchiq_platform_resumed(struct vchiq_state *state)
 311 {
 312 }
 313
 314 int
 315 vchiq_platform_videocore_wanted(struct vchiq_state *state)
 316 {
 317         return 1; // autosuspend not supported - videocore always wanted
 318 }
 319
 320 int
 321 vchiq_platform_use_suspend_timer(void)
 322 {
 323         return 0;
 324 }
 325 void
 326 vchiq_dump_platform_use_state(struct vchiq_state *state)
 327 {
 328         vchiq_log_info(vchiq_arm_log_level, "Suspend timer not in use");
 329 }
 330 void
 331 vchiq_platform_handle_timeout(struct vchiq_state *state)
 332 {
 333         (void)state;
 334 }
 335 /*
 336  * Local functions
 337  */
 338
 339 static irqreturn_t
 340 vchiq_doorbell_irq(int irq, void *dev_id)
 341 {
 342         struct vchiq_state *state = dev_id;
 343         irqreturn_t ret = IRQ_NONE;
 344         unsigned int status;
 345
 346         /* Read (and clear) the doorbell */
 347         status = readl(g_regs + BELL0);
 348
 349         if (status & 0x4) {  /* Was the doorbell rung? */
 350                 remote_event_pollall(state);
 351                 ret = IRQ_HANDLED;
 352         }
 353
 354         return ret;
 355 }
 356
 357 static void
 358 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
 359 {
 360         if (pagelistinfo->scatterlist_mapped) {
 361                 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
 362                              pagelistinfo->num_pages, pagelistinfo->dma_dir);
 363         }
 364
 365         if (pagelistinfo->pages_need_release) {
 366                 unsigned int i;
 367
 368                 for (i = 0; i < pagelistinfo->num_pages; i++)
 369                         put_page(pagelistinfo->pages[i]);
 370         }
 371
 372         dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
 373                           pagelistinfo->pagelist, pagelistinfo->dma_addr);
 374 }
 375
 376 /* There is a potential problem with partial cache lines (pages?)
 377  * at the ends of the block when reading. If the CPU accessed anything in
 378  * the same line (page?) then it may have pulled old data into the cache,
 379  * obscuring the new data underneath. We can solve this by transferring the
 380  * partial cache lines separately, and allowing the ARM to copy into the
 381  * cached area.
 382  */
 383
 384 static struct vchiq_pagelist_info *
 385 create_pagelist(char __user *buf, size_t count, unsigned short type)
 386 {
 387         struct pagelist *pagelist;
 388         struct vchiq_pagelist_info *pagelistinfo;
 389         struct page **pages;
 390         u32 *addrs;
 391         unsigned int num_pages, offset, i, k;
 392         int actual_pages;
 393         size_t pagelist_size;
 394         struct scatterlist *scatterlist, *sg;
 395         int dma_buffers;
 396         dma_addr_t dma_addr;
 397
 398         offset = ((unsigned int)(unsigned long)buf & (PAGE_SIZE - 1));
 399         num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
 400
 401         pagelist_size = sizeof(struct pagelist) +
 402                         (num_pages * sizeof(u32)) +
 403                         (num_pages * sizeof(pages[0]) +
 404                         (num_pages * sizeof(struct scatterlist))) +
 405                         sizeof(struct vchiq_pagelist_info);
 406
 407         /* Allocate enough storage to hold the page pointers and the page
 408          * list
 409          */
 410         pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
 411                                       GFP_KERNEL);
 412
 413         vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
 414
 415         if (!pagelist)
 416                 return NULL;
 417
 418         addrs           = pagelist->addrs;
 419         pages           = (struct page **)(addrs + num_pages);
 420         scatterlist     = (struct scatterlist *)(pages + num_pages);
 421         pagelistinfo    = (struct vchiq_pagelist_info *)
 422                           (scatterlist + num_pages);
 423
 424         pagelist->length = count;
 425         pagelist->type = type;
 426         pagelist->offset = offset;
 427
 428         /* Populate the fields of the pagelistinfo structure */
 429         pagelistinfo->pagelist = pagelist;
 430         pagelistinfo->pagelist_buffer_size = pagelist_size;
 431         pagelistinfo->dma_addr = dma_addr;
 432         pagelistinfo->dma_dir =  (type == PAGELIST_WRITE) ?
 433                                   DMA_TO_DEVICE : DMA_FROM_DEVICE;
 434         pagelistinfo->num_pages = num_pages;
 435         pagelistinfo->pages_need_release = 0;
 436         pagelistinfo->pages = pages;
 437         pagelistinfo->scatterlist = scatterlist;
 438         pagelistinfo->scatterlist_mapped = 0;
 439
 440         if (is_vmalloc_addr(buf)) {
 441                 unsigned long length = count;
 442                 unsigned int off = offset;
 443
 444                 for (actual_pages = 0; actual_pages < num_pages;
 445                      actual_pages++) {
 446                         struct page *pg = vmalloc_to_page(buf + (actual_pages *
 447                                                                  PAGE_SIZE));
 448                         size_t bytes = PAGE_SIZE - off;
 449
 450                         if (!pg) {
 451                                 cleanup_pagelistinfo(pagelistinfo);
 452                                 return NULL;
 453                         }
 454
 455                         if (bytes > length)
 456                                 bytes = length;
 457                         pages[actual_pages] = pg;
 458                         length -= bytes;
 459                         off = 0;
 460                 }
 461                 /* do not try and release vmalloc pages */
 462         } else {
 463                 actual_pages = get_user_pages_fast(
 464                                           (unsigned long)buf & PAGE_MASK,
 465                                           num_pages,
 466                                           type == PAGELIST_READ,
 467                                           pages);
 468
 469                 if (actual_pages != num_pages) {
 470                         vchiq_log_info(vchiq_arm_log_level,
 471                                        "%s - only %d/%d pages locked",
 472                                        __func__, actual_pages, num_pages);
 473
 474                         /* This is probably due to the process being killed */
 475                         while (actual_pages > 0) {
 476                                 actual_pages--;
 477                                 put_page(pages[actual_pages]);
 478                         }
 479                         cleanup_pagelistinfo(pagelistinfo);
 480                         return NULL;
 481                 }
 482                  /* release user pages */
 483                 pagelistinfo->pages_need_release = 1;
 484         }
 485
 486         /*
 487          * Initialize the scatterlist so that the magic cookie
 488          *  is filled if debugging is enabled
 489          */
 490         sg_init_table(scatterlist, num_pages);
 491         /* Now set the pages for each scatterlist */
 492         for (i = 0; i < num_pages; i++) {
 493                 unsigned int len = PAGE_SIZE - offset;
 494
 495                 if (len > count)
 496                         len = count;
 497                 sg_set_page(scatterlist + i, pages[i], len, offset);
 498                 offset = 0;
 499                 count -= len;
 500         }
 501
 502         dma_buffers = dma_map_sg(g_dev,
 503                                  scatterlist,
 504                                  num_pages,
 505                                  pagelistinfo->dma_dir);
 506
 507         if (dma_buffers == 0) {
 508                 cleanup_pagelistinfo(pagelistinfo);
 509                 return NULL;
 510         }
 511
 512         pagelistinfo->scatterlist_mapped = 1;
 513
 514         /* Combine adjacent blocks for performance */
 515         k = 0;
 516         for_each_sg(scatterlist, sg, dma_buffers, i) {
 517                 u32 len = sg_dma_len(sg);
 518                 u32 addr = sg_dma_address(sg);
 519
 520                 /* Note: addrs is the address + page_count - 1
 521                  * The firmware expects blocks after the first to be page-
 522                  * aligned and a multiple of the page size
 523                  */
 524                 WARN_ON(len == 0);
 525                 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
 526                 WARN_ON(i && (addr & ~PAGE_MASK));
 527                 if (k > 0 &&
 528                     ((addrs[k - 1] & PAGE_MASK) +
 529                      (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
 530                     == (addr & PAGE_MASK))
 531                         addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
 532                 else
 533                         addrs[k++] = (addr & PAGE_MASK) |
 534                                 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
 535         }
 536
 537         /* Partial cache lines (fragments) require special measures */
 538         if ((type == PAGELIST_READ) &&
 539                 ((pagelist->offset & (g_cache_line_size - 1)) ||
 540                 ((pagelist->offset + pagelist->length) &
 541                 (g_cache_line_size - 1)))) {
 542                 char *fragments;
 543
 544                 if (down_killable(&g_free_fragments_sema)) {
 545                         cleanup_pagelistinfo(pagelistinfo);
 546                         return NULL;
 547                 }
 548
 549                 WARN_ON(g_free_fragments == NULL);
 550
 551                 down(&g_free_fragments_mutex);
 552                 fragments = g_free_fragments;
 553                 WARN_ON(fragments == NULL);
 554                 g_free_fragments = *(char **) g_free_fragments;
 555                 up(&g_free_fragments_mutex);
 556                 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
 557                         (fragments - g_fragments_base) / g_fragments_size;
 558         }
 559
 560         return pagelistinfo;
 561 }
 562
 563 static void
 564 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
 565               int actual)
 566 {
 567         struct pagelist *pagelist = pagelistinfo->pagelist;
 568         struct page **pages = pagelistinfo->pages;
 569         unsigned int num_pages = pagelistinfo->num_pages;
 570
 571         vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
 572                         __func__, pagelistinfo->pagelist, actual);
 573
 574         /*
 575          * NOTE: dma_unmap_sg must be called before the
 576          * cpu can touch any of the data/pages.
 577          */
 578         dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
 579                      pagelistinfo->num_pages, pagelistinfo->dma_dir);
 580         pagelistinfo->scatterlist_mapped = 0;
 581
 582         /* Deal with any partial cache lines (fragments) */
 583         if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
 584                 char *fragments = g_fragments_base +
 585                         (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
 586                         g_fragments_size;
 587                 int head_bytes, tail_bytes;
 588
 589                 head_bytes = (g_cache_line_size - pagelist->offset) &
 590                         (g_cache_line_size - 1);
 591                 tail_bytes = (pagelist->offset + actual) &
 592                         (g_cache_line_size - 1);
 593
 594                 if ((actual >= 0) && (head_bytes != 0)) {
 595                         if (head_bytes > actual)
 596                                 head_bytes = actual;
 597
 598                         memcpy((char *)kmap(pages[0]) +
 599                                 pagelist->offset,
 600                                 fragments,
 601                                 head_bytes);
 602                         kunmap(pages[0]);
 603                 }
 604                 if ((actual >= 0) && (head_bytes < actual) &&
 605                         (tail_bytes != 0)) {
 606                         memcpy((char *)kmap(pages[num_pages - 1]) +
 607                                 ((pagelist->offset + actual) &
 608                                 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
 609                                 fragments + g_cache_line_size,
 610                                 tail_bytes);
 611                         kunmap(pages[num_pages - 1]);
 612                 }
 613
 614                 down(&g_free_fragments_mutex);
 615                 *(char **)fragments = g_free_fragments;
 616                 g_free_fragments = fragments;
 617                 up(&g_free_fragments_mutex);
 618                 up(&g_free_fragments_sema);
 619         }
 620
 621         /* Need to mark all the pages dirty. */
 622         if (pagelist->type != PAGELIST_WRITE &&
 623             pagelistinfo->pages_need_release) {
 624                 unsigned int i;
 625
 626                 for (i = 0; i < num_pages; i++)
 627                         set_page_dirty(pages[i]);
 628         }
 629
 630         cleanup_pagelistinfo(pagelistinfo);
 631 }