drivers/dma/mv_xor.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * offload engine driver for the Marvell XOR engine
   4  * Copyright (C) 2007, 2008, Marvell International Ltd.
   5  */
   6
   7 #include <linux/init.h>
   8 #include <linux/slab.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/spinlock.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/of_device.h>
  14 #include <linux/platform_device.h>
  15 #include <linux/memory.h>
  16 #include <linux/clk.h>
  17 #include <linux/of.h>
  18 #include <linux/of_irq.h>
  19 #include <linux/irqdomain.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/platform_data/dma-mv_xor.h>
  22
  23 #include "dmaengine.h"
  24 #include "mv_xor.h"
  25
  26 enum mv_xor_type {
  27         XOR_ORION,
  28         XOR_ARMADA_38X,
  29         XOR_ARMADA_37XX,
  30 };
  31
  32 enum mv_xor_mode {
  33         XOR_MODE_IN_REG,
  34         XOR_MODE_IN_DESC,
  35 };
  36
  37 static void mv_xor_issue_pending(struct dma_chan *chan);
  38
  39 #define to_mv_xor_chan(chan)            \
  40         container_of(chan, struct mv_xor_chan, dmachan)
  41
  42 #define to_mv_xor_slot(tx)              \
  43         container_of(tx, struct mv_xor_desc_slot, async_tx)
  44
  45 #define mv_chan_to_devp(chan)           \
  46         ((chan)->dmadev.dev)
  47
  48 static void mv_desc_init(struct mv_xor_desc_slot *desc,
  49                          dma_addr_t addr, u32 byte_count,
  50                          enum dma_ctrl_flags flags)
  51 {
  52         struct mv_xor_desc *hw_desc = desc->hw_desc;
  53
  54         hw_desc->status = XOR_DESC_DMA_OWNED;
  55         hw_desc->phy_next_desc = 0;
  56         /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
  57         hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
  58                                 XOR_DESC_EOD_INT_EN : 0;
  59         hw_desc->phy_dest_addr = addr;
  60         hw_desc->byte_count = byte_count;
  61 }
  62
  63 static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
  64 {
  65         struct mv_xor_desc *hw_desc = desc->hw_desc;
  66
  67         switch (desc->type) {
  68         case DMA_XOR:
  69         case DMA_INTERRUPT:
  70                 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
  71                 break;
  72         case DMA_MEMCPY:
  73                 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
  74                 break;
  75         default:
  76                 BUG();
  77                 return;
  78         }
  79 }
  80
  81 static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
  82                                   u32 next_desc_addr)
  83 {
  84         struct mv_xor_desc *hw_desc = desc->hw_desc;
  85         BUG_ON(hw_desc->phy_next_desc);
  86         hw_desc->phy_next_desc = next_desc_addr;
  87 }
  88
  89 static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
  90                                  int index, dma_addr_t addr)
  91 {
  92         struct mv_xor_desc *hw_desc = desc->hw_desc;
  93         hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
  94         if (desc->type == DMA_XOR)
  95                 hw_desc->desc_command |= (1 << index);
  96 }
  97
  98 static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
  99 {
 100         return readl_relaxed(XOR_CURR_DESC(chan));
 101 }
 102
 103 static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
 104                                         u32 next_desc_addr)
 105 {
 106         writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
 107 }
 108
 109 static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
 110 {
 111         u32 val = readl_relaxed(XOR_INTR_MASK(chan));
 112         val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
 113         writel_relaxed(val, XOR_INTR_MASK(chan));
 114 }
 115
 116 static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
 117 {
 118         u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
 119         intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
 120         return intr_cause;
 121 }
 122
 123 static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
 124 {
 125         u32 val;
 126
 127         val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
 128         val = ~(val << (chan->idx * 16));
 129         dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
 130         writel_relaxed(val, XOR_INTR_CAUSE(chan));
 131 }
 132
 133 static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
 134 {
 135         u32 val = 0xFFFF0000 >> (chan->idx * 16);
 136         writel_relaxed(val, XOR_INTR_CAUSE(chan));
 137 }
 138
 139 static void mv_chan_set_mode(struct mv_xor_chan *chan,
 140                              u32 op_mode)
 141 {
 142         u32 config = readl_relaxed(XOR_CONFIG(chan));
 143
 144         config &= ~0x7;
 145         config |= op_mode;
 146
 147 #if defined(__BIG_ENDIAN)
 148         config |= XOR_DESCRIPTOR_SWAP;
 149 #else
 150         config &= ~XOR_DESCRIPTOR_SWAP;
 151 #endif
 152
 153         writel_relaxed(config, XOR_CONFIG(chan));
 154 }
 155
 156 static void mv_chan_activate(struct mv_xor_chan *chan)
 157 {
 158         dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
 159
 160         /* writel ensures all descriptors are flushed before activation */
 161         writel(BIT(0), XOR_ACTIVATION(chan));
 162 }
 163
 164 static char mv_chan_is_busy(struct mv_xor_chan *chan)
 165 {
 166         u32 state = readl_relaxed(XOR_ACTIVATION(chan));
 167
 168         state = (state >> 4) & 0x3;
 169
 170         return (state == 1) ? 1 : 0;
 171 }
 172
 173 /*
 174  * mv_chan_start_new_chain - program the engine to operate on new
 175  * chain headed by sw_desc
 176  * Caller must hold &mv_chan->lock while calling this function
 177  */
 178 static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
 179                                     struct mv_xor_desc_slot *sw_desc)
 180 {
 181         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
 182                 __func__, __LINE__, sw_desc);
 183
 184         /* set the hardware chain */
 185         mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
 186
 187         mv_chan->pending++;
 188         mv_xor_issue_pending(&mv_chan->dmachan);
 189 }
 190
 191 static dma_cookie_t
 192 mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
 193                                 struct mv_xor_chan *mv_chan,
 194                                 dma_cookie_t cookie)
 195 {
 196         BUG_ON(desc->async_tx.cookie < 0);
 197
 198         if (desc->async_tx.cookie > 0) {
 199                 cookie = desc->async_tx.cookie;
 200
 201                 dma_descriptor_unmap(&desc->async_tx);
 202                 /* call the callback (must not sleep or submit new
 203                  * operations to this channel)
 204                  */
 205                 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
 206         }
 207
 208         /* run dependent operations */
 209         dma_run_dependencies(&desc->async_tx);
 210
 211         return cookie;
 212 }
 213
 214 static int
 215 mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
 216 {
 217         struct mv_xor_desc_slot *iter, *_iter;
 218
 219         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
 220         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
 221                                  node) {
 222
 223                 if (async_tx_test_ack(&iter->async_tx)) {
 224                         list_move_tail(&iter->node, &mv_chan->free_slots);
 225                         if (!list_empty(&iter->sg_tx_list)) {
 226                                 list_splice_tail_init(&iter->sg_tx_list,
 227                                                         &mv_chan->free_slots);
 228                         }
 229                 }
 230         }
 231         return 0;
 232 }
 233
 234 static int
 235 mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
 236                    struct mv_xor_chan *mv_chan)
 237 {
 238         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
 239                 __func__, __LINE__, desc, desc->async_tx.flags);
 240
 241         /* the client is allowed to attach dependent operations
 242          * until 'ack' is set
 243          */
 244         if (!async_tx_test_ack(&desc->async_tx)) {
 245                 /* move this slot to the completed_slots */
 246                 list_move_tail(&desc->node, &mv_chan->completed_slots);
 247                 if (!list_empty(&desc->sg_tx_list)) {
 248                         list_splice_tail_init(&desc->sg_tx_list,
 249                                               &mv_chan->completed_slots);
 250                 }
 251         } else {
 252                 list_move_tail(&desc->node, &mv_chan->free_slots);
 253                 if (!list_empty(&desc->sg_tx_list)) {
 254                         list_splice_tail_init(&desc->sg_tx_list,
 255                                               &mv_chan->free_slots);
 256                 }
 257         }
 258
 259         return 0;
 260 }
 261
 262 /* This function must be called with the mv_xor_chan spinlock held */
 263 static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
 264 {
 265         struct mv_xor_desc_slot *iter, *_iter;
 266         dma_cookie_t cookie = 0;
 267         int busy = mv_chan_is_busy(mv_chan);
 268         u32 current_desc = mv_chan_get_current_desc(mv_chan);
 269         int current_cleaned = 0;
 270         struct mv_xor_desc *hw_desc;
 271
 272         dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
 273         dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
 274         mv_chan_clean_completed_slots(mv_chan);
 275
 276         /* free completed slots from the chain starting with
 277          * the oldest descriptor
 278          */
 279
 280         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
 281                                  node) {
 282
 283                 /* clean finished descriptors */
 284                 hw_desc = iter->hw_desc;
 285                 if (hw_desc->status & XOR_DESC_SUCCESS) {
 286                         cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
 287                                                                  cookie);
 288
 289                         /* done processing desc, clean slot */
 290                         mv_desc_clean_slot(iter, mv_chan);
 291
 292                         /* break if we did cleaned the current */
 293                         if (iter->async_tx.phys == current_desc) {
 294                                 current_cleaned = 1;
 295                                 break;
 296                         }
 297                 } else {
 298                         if (iter->async_tx.phys == current_desc) {
 299                                 current_cleaned = 0;
 300                                 break;
 301                         }
 302                 }
 303         }
 304
 305         if ((busy == 0) && !list_empty(&mv_chan->chain)) {
 306                 if (current_cleaned) {
 307                         /*
 308                          * current descriptor cleaned and removed, run
 309                          * from list head
 310                          */
 311                         iter = list_entry(mv_chan->chain.next,
 312                                           struct mv_xor_desc_slot,
 313                                           node);
 314                         mv_chan_start_new_chain(mv_chan, iter);
 315                 } else {
 316                         if (!list_is_last(&iter->node, &mv_chan->chain)) {
 317                                 /*
 318                                  * descriptors are still waiting after
 319                                  * current, trigger them
 320                                  */
 321                                 iter = list_entry(iter->node.next,
 322                                                   struct mv_xor_desc_slot,
 323                                                   node);
 324                                 mv_chan_start_new_chain(mv_chan, iter);
 325                         } else {
 326                                 /*
 327                                  * some descriptors are still waiting
 328                                  * to be cleaned
 329                                  */
 330                                 tasklet_schedule(&mv_chan->irq_tasklet);
 331                         }
 332                 }
 333         }
 334
 335         if (cookie > 0)
 336                 mv_chan->dmachan.completed_cookie = cookie;
 337 }
 338
 339 static void mv_xor_tasklet(struct tasklet_struct *t)
 340 {
 341         struct mv_xor_chan *chan = from_tasklet(chan, t, irq_tasklet);
 342
 343         spin_lock(&chan->lock);
 344         mv_chan_slot_cleanup(chan);
 345         spin_unlock(&chan->lock);
 346 }
 347
 348 static struct mv_xor_desc_slot *
 349 mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
 350 {
 351         struct mv_xor_desc_slot *iter;
 352
 353         spin_lock_bh(&mv_chan->lock);
 354
 355         if (!list_empty(&mv_chan->free_slots)) {
 356                 iter = list_first_entry(&mv_chan->free_slots,
 357                                         struct mv_xor_desc_slot,
 358                                         node);
 359
 360                 list_move_tail(&iter->node, &mv_chan->allocated_slots);
 361
 362                 spin_unlock_bh(&mv_chan->lock);
 363
 364                 /* pre-ack descriptor */
 365                 async_tx_ack(&iter->async_tx);
 366                 iter->async_tx.cookie = -EBUSY;
 367
 368                 return iter;
 369
 370         }
 371
 372         spin_unlock_bh(&mv_chan->lock);
 373
 374         /* try to free some slots if the allocation fails */
 375         tasklet_schedule(&mv_chan->irq_tasklet);
 376
 377         return NULL;
 378 }
 379
 380 /************************ DMA engine API functions ****************************/
 381 static dma_cookie_t
 382 mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
 383 {
 384         struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
 385         struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
 386         struct mv_xor_desc_slot *old_chain_tail;
 387         dma_cookie_t cookie;
 388         int new_hw_chain = 1;
 389
 390         dev_dbg(mv_chan_to_devp(mv_chan),
 391                 "%s sw_desc %p: async_tx %p\n",
 392                 __func__, sw_desc, &sw_desc->async_tx);
 393
 394         spin_lock_bh(&mv_chan->lock);
 395         cookie = dma_cookie_assign(tx);
 396
 397         if (list_empty(&mv_chan->chain))
 398                 list_move_tail(&sw_desc->node, &mv_chan->chain);
 399         else {
 400                 new_hw_chain = 0;
 401
 402                 old_chain_tail = list_entry(mv_chan->chain.prev,
 403                                             struct mv_xor_desc_slot,
 404                                             node);
 405                 list_move_tail(&sw_desc->node, &mv_chan->chain);
 406
 407                 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
 408                         &old_chain_tail->async_tx.phys);
 409
 410                 /* fix up the hardware chain */
 411                 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
 412
 413                 /* if the channel is not busy */
 414                 if (!mv_chan_is_busy(mv_chan)) {
 415                         u32 current_desc = mv_chan_get_current_desc(mv_chan);
 416                         /*
 417                          * and the curren desc is the end of the chain before
 418                          * the append, then we need to start the channel
 419                          */
 420                         if (current_desc == old_chain_tail->async_tx.phys)
 421                                 new_hw_chain = 1;
 422                 }
 423         }
 424
 425         if (new_hw_chain)
 426                 mv_chan_start_new_chain(mv_chan, sw_desc);
 427
 428         spin_unlock_bh(&mv_chan->lock);
 429
 430         return cookie;
 431 }
 432
 433 /* returns the number of allocated descriptors */
 434 static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
 435 {
 436         void *virt_desc;
 437         dma_addr_t dma_desc;
 438         int idx;
 439         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 440         struct mv_xor_desc_slot *slot = NULL;
 441         int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
 442
 443         /* Allocate descriptor slots */
 444         idx = mv_chan->slots_allocated;
 445         while (idx < num_descs_in_pool) {
 446                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 447                 if (!slot) {
 448                         dev_info(mv_chan_to_devp(mv_chan),
 449                                  "channel only initialized %d descriptor slots",
 450                                  idx);
 451                         break;
 452                 }
 453                 virt_desc = mv_chan->dma_desc_pool_virt;
 454                 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
 455
 456                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 457                 slot->async_tx.tx_submit = mv_xor_tx_submit;
 458                 INIT_LIST_HEAD(&slot->node);
 459                 INIT_LIST_HEAD(&slot->sg_tx_list);
 460                 dma_desc = mv_chan->dma_desc_pool;
 461                 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
 462                 slot->idx = idx++;
 463
 464                 spin_lock_bh(&mv_chan->lock);
 465                 mv_chan->slots_allocated = idx;
 466                 list_add_tail(&slot->node, &mv_chan->free_slots);
 467                 spin_unlock_bh(&mv_chan->lock);
 468         }
 469
 470         dev_dbg(mv_chan_to_devp(mv_chan),
 471                 "allocated %d descriptor slots\n",
 472                 mv_chan->slots_allocated);
 473
 474         return mv_chan->slots_allocated ? : -ENOMEM;
 475 }
 476
 477 /*
 478  * Check if source or destination is an PCIe/IO address (non-SDRAM) and add
 479  * a new MBus window if necessary. Use a cache for these check so that
 480  * the MMIO mapped registers don't have to be accessed for this check
 481  * to speed up this process.
 482  */
 483 static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr)
 484 {
 485         struct mv_xor_device *xordev = mv_chan->xordev;
 486         void __iomem *base = mv_chan->mmr_high_base;
 487         u32 win_enable;
 488         u32 size;
 489         u8 target, attr;
 490         int ret;
 491         int i;
 492
 493         /* Nothing needs to get done for the Armada 3700 */
 494         if (xordev->xor_type == XOR_ARMADA_37XX)
 495                 return 0;
 496
 497         /*
 498          * Loop over the cached windows to check, if the requested area
 499          * is already mapped. If this the case, nothing needs to be done
 500          * and we can return.
 501          */
 502         for (i = 0; i < WINDOW_COUNT; i++) {
 503                 if (addr >= xordev->win_start[i] &&
 504                     addr <= xordev->win_end[i]) {
 505                         /* Window is already mapped */
 506                         return 0;
 507                 }
 508         }
 509
 510         /*
 511          * The window is not mapped, so we need to create the new mapping
 512          */
 513
 514         /* If no IO window is found that addr has to be located in SDRAM */
 515         ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr);
 516         if (ret < 0)
 517                 return 0;
 518
 519         /*
 520          * Mask the base addr 'addr' according to 'size' read back from the
 521          * MBus window. Otherwise we might end up with an address located
 522          * somewhere in the middle of this area here.
 523          */
 524         size -= 1;
 525         addr &= ~size;
 526
 527         /*
 528          * Reading one of both enabled register is enough, as they are always
 529          * programmed to the identical values
 530          */
 531         win_enable = readl(base + WINDOW_BAR_ENABLE(0));
 532
 533         /* Set 'i' to the first free window to write the new values to */
 534         i = ffs(~win_enable) - 1;
 535         if (i >= WINDOW_COUNT)
 536                 return -ENOMEM;
 537
 538         writel((addr & 0xffff0000) | (attr << 8) | target,
 539                base + WINDOW_BASE(i));
 540         writel(size & 0xffff0000, base + WINDOW_SIZE(i));
 541
 542         /* Fill the caching variables for later use */
 543         xordev->win_start[i] = addr;
 544         xordev->win_end[i] = addr + size;
 545
 546         win_enable |= (1 << i);
 547         win_enable |= 3 << (16 + (2 * i));
 548         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
 549         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
 550
 551         return 0;
 552 }
 553
 554 static struct dma_async_tx_descriptor *
 555 mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 556                     unsigned int src_cnt, size_t len, unsigned long flags)
 557 {
 558         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 559         struct mv_xor_desc_slot *sw_desc;
 560         int ret;
 561
 562         if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
 563                 return NULL;
 564
 565         BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
 566
 567         dev_dbg(mv_chan_to_devp(mv_chan),
 568                 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
 569                 __func__, src_cnt, len, &dest, flags);
 570
 571         /* Check if a new window needs to get added for 'dest' */
 572         ret = mv_xor_add_io_win(mv_chan, dest);
 573         if (ret)
 574                 return NULL;
 575
 576         sw_desc = mv_chan_alloc_slot(mv_chan);
 577         if (sw_desc) {
 578                 sw_desc->type = DMA_XOR;
 579                 sw_desc->async_tx.flags = flags;
 580                 mv_desc_init(sw_desc, dest, len, flags);
 581                 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
 582                         mv_desc_set_mode(sw_desc);
 583                 while (src_cnt--) {
 584                         /* Check if a new window needs to get added for 'src' */
 585                         ret = mv_xor_add_io_win(mv_chan, src[src_cnt]);
 586                         if (ret)
 587                                 return NULL;
 588                         mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
 589                 }
 590         }
 591
 592         dev_dbg(mv_chan_to_devp(mv_chan),
 593                 "%s sw_desc %p async_tx %p \n",
 594                 __func__, sw_desc, &sw_desc->async_tx);
 595         return sw_desc ? &sw_desc->async_tx : NULL;
 596 }
 597
 598 static struct dma_async_tx_descriptor *
 599 mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 600                 size_t len, unsigned long flags)
 601 {
 602         /*
 603          * A MEMCPY operation is identical to an XOR operation with only
 604          * a single source address.
 605          */
 606         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
 607 }
 608
 609 static struct dma_async_tx_descriptor *
 610 mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 611 {
 612         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 613         dma_addr_t src, dest;
 614         size_t len;
 615
 616         src = mv_chan->dummy_src_addr;
 617         dest = mv_chan->dummy_dst_addr;
 618         len = MV_XOR_MIN_BYTE_COUNT;
 619
 620         /*
 621          * We implement the DMA_INTERRUPT operation as a minimum sized
 622          * XOR operation with a single dummy source address.
 623          */
 624         return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
 625 }
 626
 627 static void mv_xor_free_chan_resources(struct dma_chan *chan)
 628 {
 629         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 630         struct mv_xor_desc_slot *iter, *_iter;
 631         int in_use_descs = 0;
 632
 633         spin_lock_bh(&mv_chan->lock);
 634
 635         mv_chan_slot_cleanup(mv_chan);
 636
 637         list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
 638                                         node) {
 639                 in_use_descs++;
 640                 list_move_tail(&iter->node, &mv_chan->free_slots);
 641         }
 642         list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
 643                                  node) {
 644                 in_use_descs++;
 645                 list_move_tail(&iter->node, &mv_chan->free_slots);
 646         }
 647         list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
 648                                  node) {
 649                 in_use_descs++;
 650                 list_move_tail(&iter->node, &mv_chan->free_slots);
 651         }
 652         list_for_each_entry_safe_reverse(
 653                 iter, _iter, &mv_chan->free_slots, node) {
 654                 list_del(&iter->node);
 655                 kfree(iter);
 656                 mv_chan->slots_allocated--;
 657         }
 658
 659         dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
 660                 __func__, mv_chan->slots_allocated);
 661         spin_unlock_bh(&mv_chan->lock);
 662
 663         if (in_use_descs)
 664                 dev_err(mv_chan_to_devp(mv_chan),
 665                         "freeing %d in use descriptors!\n", in_use_descs);
 666 }
 667
 668 /**
 669  * mv_xor_status - poll the status of an XOR transaction
 670  * @chan: XOR channel handle
 671  * @cookie: XOR transaction identifier
 672  * @txstate: XOR transactions state holder (or NULL)
 673  */
 674 static enum dma_status mv_xor_status(struct dma_chan *chan,
 675                                           dma_cookie_t cookie,
 676                                           struct dma_tx_state *txstate)
 677 {
 678         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 679         enum dma_status ret;
 680
 681         ret = dma_cookie_status(chan, cookie, txstate);
 682         if (ret == DMA_COMPLETE)
 683                 return ret;
 684
 685         spin_lock_bh(&mv_chan->lock);
 686         mv_chan_slot_cleanup(mv_chan);
 687         spin_unlock_bh(&mv_chan->lock);
 688
 689         return dma_cookie_status(chan, cookie, txstate);
 690 }
 691
 692 static void mv_chan_dump_regs(struct mv_xor_chan *chan)
 693 {
 694         u32 val;
 695
 696         val = readl_relaxed(XOR_CONFIG(chan));
 697         dev_err(mv_chan_to_devp(chan), "config       0x%08x\n", val);
 698
 699         val = readl_relaxed(XOR_ACTIVATION(chan));
 700         dev_err(mv_chan_to_devp(chan), "activation   0x%08x\n", val);
 701
 702         val = readl_relaxed(XOR_INTR_CAUSE(chan));
 703         dev_err(mv_chan_to_devp(chan), "intr cause   0x%08x\n", val);
 704
 705         val = readl_relaxed(XOR_INTR_MASK(chan));
 706         dev_err(mv_chan_to_devp(chan), "intr mask    0x%08x\n", val);
 707
 708         val = readl_relaxed(XOR_ERROR_CAUSE(chan));
 709         dev_err(mv_chan_to_devp(chan), "error cause  0x%08x\n", val);
 710
 711         val = readl_relaxed(XOR_ERROR_ADDR(chan));
 712         dev_err(mv_chan_to_devp(chan), "error addr   0x%08x\n", val);
 713 }
 714
 715 static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
 716                                           u32 intr_cause)
 717 {
 718         if (intr_cause & XOR_INT_ERR_DECODE) {
 719                 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
 720                 return;
 721         }
 722
 723         dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
 724                 chan->idx, intr_cause);
 725
 726         mv_chan_dump_regs(chan);
 727         WARN_ON(1);
 728 }
 729
 730 static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
 731 {
 732         struct mv_xor_chan *chan = data;
 733         u32 intr_cause = mv_chan_get_intr_cause(chan);
 734
 735         dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
 736
 737         if (intr_cause & XOR_INTR_ERRORS)
 738                 mv_chan_err_interrupt_handler(chan, intr_cause);
 739
 740         tasklet_schedule(&chan->irq_tasklet);
 741
 742         mv_chan_clear_eoc_cause(chan);
 743
 744         return IRQ_HANDLED;
 745 }
 746
 747 static void mv_xor_issue_pending(struct dma_chan *chan)
 748 {
 749         struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
 750
 751         if (mv_chan->pending >= MV_XOR_THRESHOLD) {
 752                 mv_chan->pending = 0;
 753                 mv_chan_activate(mv_chan);
 754         }
 755 }
 756
 757 /*
 758  * Perform a transaction to verify the HW works.
 759  */
 760
 761 static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
 762 {
 763         int i, ret;
 764         void *src, *dest;
 765         dma_addr_t src_dma, dest_dma;
 766         struct dma_chan *dma_chan;
 767         dma_cookie_t cookie;
 768         struct dma_async_tx_descriptor *tx;
 769         struct dmaengine_unmap_data *unmap;
 770         int err = 0;
 771
 772         src = kmalloc(PAGE_SIZE, GFP_KERNEL);
 773         if (!src)
 774                 return -ENOMEM;
 775
 776         dest = kzalloc(PAGE_SIZE, GFP_KERNEL);
 777         if (!dest) {
 778                 kfree(src);
 779                 return -ENOMEM;
 780         }
 781
 782         /* Fill in src buffer */
 783         for (i = 0; i < PAGE_SIZE; i++)
 784                 ((u8 *) src)[i] = (u8)i;
 785
 786         dma_chan = &mv_chan->dmachan;
 787         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
 788                 err = -ENODEV;
 789                 goto out;
 790         }
 791
 792         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
 793         if (!unmap) {
 794                 err = -ENOMEM;
 795                 goto free_resources;
 796         }
 797
 798         src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src),
 799                                offset_in_page(src), PAGE_SIZE,
 800                                DMA_TO_DEVICE);
 801         unmap->addr[0] = src_dma;
 802
 803         ret = dma_mapping_error(dma_chan->device->dev, src_dma);
 804         if (ret) {
 805                 err = -ENOMEM;
 806                 goto free_resources;
 807         }
 808         unmap->to_cnt = 1;
 809
 810         dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest),
 811                                 offset_in_page(dest), PAGE_SIZE,
 812                                 DMA_FROM_DEVICE);
 813         unmap->addr[1] = dest_dma;
 814
 815         ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
 816         if (ret) {
 817                 err = -ENOMEM;
 818                 goto free_resources;
 819         }
 820         unmap->from_cnt = 1;
 821         unmap->len = PAGE_SIZE;
 822
 823         tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
 824                                     PAGE_SIZE, 0);
 825         if (!tx) {
 826                 dev_err(dma_chan->device->dev,
 827                         "Self-test cannot prepare operation, disabling\n");
 828                 err = -ENODEV;
 829                 goto free_resources;
 830         }
 831
 832         cookie = mv_xor_tx_submit(tx);
 833         if (dma_submit_error(cookie)) {
 834                 dev_err(dma_chan->device->dev,
 835                         "Self-test submit error, disabling\n");
 836                 err = -ENODEV;
 837                 goto free_resources;
 838         }
 839
 840         mv_xor_issue_pending(dma_chan);
 841         async_tx_ack(tx);
 842         msleep(1);
 843
 844         if (mv_xor_status(dma_chan, cookie, NULL) !=
 845             DMA_COMPLETE) {
 846                 dev_err(dma_chan->device->dev,
 847                         "Self-test copy timed out, disabling\n");
 848                 err = -ENODEV;
 849                 goto free_resources;
 850         }
 851
 852         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
 853                                 PAGE_SIZE, DMA_FROM_DEVICE);
 854         if (memcmp(src, dest, PAGE_SIZE)) {
 855                 dev_err(dma_chan->device->dev,
 856                         "Self-test copy failed compare, disabling\n");
 857                 err = -ENODEV;
 858                 goto free_resources;
 859         }
 860
 861 free_resources:
 862         dmaengine_unmap_put(unmap);
 863         mv_xor_free_chan_resources(dma_chan);
 864 out:
 865         kfree(src);
 866         kfree(dest);
 867         return err;
 868 }
 869
 870 #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
 871 static int
 872 mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
 873 {
 874         int i, src_idx, ret;
 875         struct page *dest;
 876         struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
 877         dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
 878         dma_addr_t dest_dma;
 879         struct dma_async_tx_descriptor *tx;
 880         struct dmaengine_unmap_data *unmap;
 881         struct dma_chan *dma_chan;
 882         dma_cookie_t cookie;
 883         u8 cmp_byte = 0;
 884         u32 cmp_word;
 885         int err = 0;
 886         int src_count = MV_XOR_NUM_SRC_TEST;
 887
 888         for (src_idx = 0; src_idx < src_count; src_idx++) {
 889                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
 890                 if (!xor_srcs[src_idx]) {
 891                         while (src_idx--)
 892                                 __free_page(xor_srcs[src_idx]);
 893                         return -ENOMEM;
 894                 }
 895         }
 896
 897         dest = alloc_page(GFP_KERNEL);
 898         if (!dest) {
 899                 while (src_idx--)
 900                         __free_page(xor_srcs[src_idx]);
 901                 return -ENOMEM;
 902         }
 903
 904         /* Fill in src buffers */
 905         for (src_idx = 0; src_idx < src_count; src_idx++) {
 906                 u8 *ptr = page_address(xor_srcs[src_idx]);
 907                 for (i = 0; i < PAGE_SIZE; i++)
 908                         ptr[i] = (1 << src_idx);
 909         }
 910
 911         for (src_idx = 0; src_idx < src_count; src_idx++)
 912                 cmp_byte ^= (u8) (1 << src_idx);
 913
 914         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
 915                 (cmp_byte << 8) | cmp_byte;
 916
 917         memset(page_address(dest), 0, PAGE_SIZE);
 918
 919         dma_chan = &mv_chan->dmachan;
 920         if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
 921                 err = -ENODEV;
 922                 goto out;
 923         }
 924
 925         unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
 926                                          GFP_KERNEL);
 927         if (!unmap) {
 928                 err = -ENOMEM;
 929                 goto free_resources;
 930         }
 931
 932         /* test xor */
 933         for (i = 0; i < src_count; i++) {
 934                 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
 935                                               0, PAGE_SIZE, DMA_TO_DEVICE);
 936                 dma_srcs[i] = unmap->addr[i];
 937                 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
 938                 if (ret) {
 939                         err = -ENOMEM;
 940                         goto free_resources;
 941                 }
 942                 unmap->to_cnt++;
 943         }
 944
 945         unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
 946                                       DMA_FROM_DEVICE);
 947         dest_dma = unmap->addr[src_count];
 948         ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
 949         if (ret) {
 950                 err = -ENOMEM;
 951                 goto free_resources;
 952         }
 953         unmap->from_cnt = 1;
 954         unmap->len = PAGE_SIZE;
 955
 956         tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
 957                                  src_count, PAGE_SIZE, 0);
 958         if (!tx) {
 959                 dev_err(dma_chan->device->dev,
 960                         "Self-test cannot prepare operation, disabling\n");
 961                 err = -ENODEV;
 962                 goto free_resources;
 963         }
 964
 965         cookie = mv_xor_tx_submit(tx);
 966         if (dma_submit_error(cookie)) {
 967                 dev_err(dma_chan->device->dev,
 968                         "Self-test submit error, disabling\n");
 969                 err = -ENODEV;
 970                 goto free_resources;
 971         }
 972
 973         mv_xor_issue_pending(dma_chan);
 974         async_tx_ack(tx);
 975         msleep(8);
 976
 977         if (mv_xor_status(dma_chan, cookie, NULL) !=
 978             DMA_COMPLETE) {
 979                 dev_err(dma_chan->device->dev,
 980                         "Self-test xor timed out, disabling\n");
 981                 err = -ENODEV;
 982                 goto free_resources;
 983         }
 984
 985         dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
 986                                 PAGE_SIZE, DMA_FROM_DEVICE);
 987         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
 988                 u32 *ptr = page_address(dest);
 989                 if (ptr[i] != cmp_word) {
 990                         dev_err(dma_chan->device->dev,
 991                                 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
 992                                 i, ptr[i], cmp_word);
 993                         err = -ENODEV;
 994                         goto free_resources;
 995                 }
 996         }
 997
 998 free_resources:
 999         dmaengine_unmap_put(unmap);
1000         mv_xor_free_chan_resources(dma_chan);
1001 out:
1002         src_idx = src_count;
1003         while (src_idx--)
1004                 __free_page(xor_srcs[src_idx]);
1005         __free_page(dest);
1006         return err;
1007 }
1008
1009 static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
1010 {
1011         struct dma_chan *chan, *_chan;
1012         struct device *dev = mv_chan->dmadev.dev;
1013
1014         dma_async_device_unregister(&mv_chan->dmadev);
1015
1016         dma_free_coherent(dev, MV_XOR_POOL_SIZE,
1017                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1018         dma_unmap_single(dev, mv_chan->dummy_src_addr,
1019                          MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1020         dma_unmap_single(dev, mv_chan->dummy_dst_addr,
1021                          MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1022
1023         list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
1024                                  device_node) {
1025                 list_del(&chan->device_node);
1026         }
1027
1028         free_irq(mv_chan->irq, mv_chan);
1029
1030         return 0;
1031 }
1032
1033 static struct mv_xor_chan *
1034 mv_xor_channel_add(struct mv_xor_device *xordev,
1035                    struct platform_device *pdev,
1036                    int idx, dma_cap_mask_t cap_mask, int irq)
1037 {
1038         int ret = 0;
1039         struct mv_xor_chan *mv_chan;
1040         struct dma_device *dma_dev;
1041
1042         mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
1043         if (!mv_chan)
1044                 return ERR_PTR(-ENOMEM);
1045
1046         mv_chan->idx = idx;
1047         mv_chan->irq = irq;
1048         if (xordev->xor_type == XOR_ORION)
1049                 mv_chan->op_in_desc = XOR_MODE_IN_REG;
1050         else
1051                 mv_chan->op_in_desc = XOR_MODE_IN_DESC;
1052
1053         dma_dev = &mv_chan->dmadev;
1054         dma_dev->dev = &pdev->dev;
1055         mv_chan->xordev = xordev;
1056
1057         /*
1058          * These source and destination dummy buffers are used to implement
1059          * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
1060          * Hence, we only need to map the buffers at initialization-time.
1061          */
1062         mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
1063                 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
1064         mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
1065                 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
1066
1067         /* allocate coherent memory for hardware descriptors
1068          * note: writecombine gives slightly better performance, but
1069          * requires that we explicitly flush the writes
1070          */
1071         mv_chan->dma_desc_pool_virt =
1072           dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
1073                        GFP_KERNEL);
1074         if (!mv_chan->dma_desc_pool_virt)
1075                 return ERR_PTR(-ENOMEM);
1076
1077         /* discover transaction capabilites from the platform data */
1078         dma_dev->cap_mask = cap_mask;
1079
1080         INIT_LIST_HEAD(&dma_dev->channels);
1081
1082         /* set base routines */
1083         dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
1084         dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
1085         dma_dev->device_tx_status = mv_xor_status;
1086         dma_dev->device_issue_pending = mv_xor_issue_pending;
1087
1088         /* set prep routines based on capability */
1089         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1090                 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
1091         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1092                 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
1093         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1094                 dma_dev->max_xor = 8;
1095                 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1096         }
1097
1098         mv_chan->mmr_base = xordev->xor_base;
1099         mv_chan->mmr_high_base = xordev->xor_high_base;
1100         tasklet_setup(&mv_chan->irq_tasklet, mv_xor_tasklet);
1101
1102         /* clear errors before enabling interrupts */
1103         mv_chan_clear_err_status(mv_chan);
1104
1105         ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1106                           0, dev_name(&pdev->dev), mv_chan);
1107         if (ret)
1108                 goto err_free_dma;
1109
1110         mv_chan_unmask_interrupts(mv_chan);
1111
1112         if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
1113                 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
1114         else
1115                 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
1116
1117         spin_lock_init(&mv_chan->lock);
1118         INIT_LIST_HEAD(&mv_chan->chain);
1119         INIT_LIST_HEAD(&mv_chan->completed_slots);
1120         INIT_LIST_HEAD(&mv_chan->free_slots);
1121         INIT_LIST_HEAD(&mv_chan->allocated_slots);
1122         mv_chan->dmachan.device = dma_dev;
1123         dma_cookie_init(&mv_chan->dmachan);
1124
1125         list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
1126
1127         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1128                 ret = mv_chan_memcpy_self_test(mv_chan);
1129                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1130                 if (ret)
1131                         goto err_free_irq;
1132         }
1133
1134         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1135                 ret = mv_chan_xor_self_test(mv_chan);
1136                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1137                 if (ret)
1138                         goto err_free_irq;
1139         }
1140
1141         dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1142                  mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
1143                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1144                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1145                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1146
1147         ret = dma_async_device_register(dma_dev);
1148         if (ret)
1149                 goto err_free_irq;
1150
1151         return mv_chan;
1152
1153 err_free_irq:
1154         free_irq(mv_chan->irq, mv_chan);
1155 err_free_dma:
1156         dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
1157                           mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
1158         return ERR_PTR(ret);
1159 }
1160
1161 static void
1162 mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1163                          const struct mbus_dram_target_info *dram)
1164 {
1165         void __iomem *base = xordev->xor_high_base;
1166         u32 win_enable = 0;
1167         int i;
1168
1169         for (i = 0; i < 8; i++) {
1170                 writel(0, base + WINDOW_BASE(i));
1171                 writel(0, base + WINDOW_SIZE(i));
1172                 if (i < 4)
1173                         writel(0, base + WINDOW_REMAP_HIGH(i));
1174         }
1175
1176         for (i = 0; i < dram->num_cs; i++) {
1177                 const struct mbus_dram_window *cs = dram->cs + i;
1178
1179                 writel((cs->base & 0xffff0000) |
1180                        (cs->mbus_attr << 8) |
1181                        dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1182                 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1183
1184                 /* Fill the caching variables for later use */
1185                 xordev->win_start[i] = cs->base;
1186                 xordev->win_end[i] = cs->base + cs->size - 1;
1187
1188                 win_enable |= (1 << i);
1189                 win_enable |= 3 << (16 + (2 * i));
1190         }
1191
1192         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1193         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1194         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1195         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1196 }
1197
1198 static void
1199 mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1200 {
1201         void __iomem *base = xordev->xor_high_base;
1202         u32 win_enable = 0;
1203         int i;
1204
1205         for (i = 0; i < 8; i++) {
1206                 writel(0, base + WINDOW_BASE(i));
1207                 writel(0, base + WINDOW_SIZE(i));
1208                 if (i < 4)
1209                         writel(0, base + WINDOW_REMAP_HIGH(i));
1210         }
1211         /*
1212          * For Armada3700 open default 4GB Mbus window. The dram
1213          * related configuration are done at AXIS level.
1214          */
1215         writel(0xffff0000, base + WINDOW_SIZE(0));
1216         win_enable |= 1;
1217         win_enable |= 3 << 16;
1218
1219         writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1220         writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1221         writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1222         writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1223 }
1224
1225 /*
1226  * Since this XOR driver is basically used only for RAID5, we don't
1227  * need to care about synchronizing ->suspend with DMA activity,
1228  * because the DMA engine will naturally be quiet due to the block
1229  * devices being suspended.
1230  */
1231 static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1232 {
1233         struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1234         int i;
1235
1236         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1237                 struct mv_xor_chan *mv_chan = xordev->channels[i];
1238
1239                 if (!mv_chan)
1240                         continue;
1241
1242                 mv_chan->saved_config_reg =
1243                         readl_relaxed(XOR_CONFIG(mv_chan));
1244                 mv_chan->saved_int_mask_reg =
1245                         readl_relaxed(XOR_INTR_MASK(mv_chan));
1246         }
1247
1248         return 0;
1249 }
1250
1251 static int mv_xor_resume(struct platform_device *dev)
1252 {
1253         struct mv_xor_device *xordev = platform_get_drvdata(dev);
1254         const struct mbus_dram_target_info *dram;
1255         int i;
1256
1257         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1258                 struct mv_xor_chan *mv_chan = xordev->channels[i];
1259
1260                 if (!mv_chan)
1261                         continue;
1262
1263                 writel_relaxed(mv_chan->saved_config_reg,
1264                                XOR_CONFIG(mv_chan));
1265                 writel_relaxed(mv_chan->saved_int_mask_reg,
1266                                XOR_INTR_MASK(mv_chan));
1267         }
1268
1269         if (xordev->xor_type == XOR_ARMADA_37XX) {
1270                 mv_xor_conf_mbus_windows_a3700(xordev);
1271                 return 0;
1272         }
1273
1274         dram = mv_mbus_dram_info();
1275         if (dram)
1276                 mv_xor_conf_mbus_windows(xordev, dram);
1277
1278         return 0;
1279 }
1280
1281 static const struct of_device_id mv_xor_dt_ids[] = {
1282         { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1283         { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
1284         { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
1285         {},
1286 };
1287
1288 static unsigned int mv_xor_engine_count;
1289
1290 static int mv_xor_probe(struct platform_device *pdev)
1291 {
1292         const struct mbus_dram_target_info *dram;
1293         struct mv_xor_device *xordev;
1294         struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1295         struct resource *res;
1296         unsigned int max_engines, max_channels;
1297         int i, ret;
1298
1299         dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
1300
1301         xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1302         if (!xordev)
1303                 return -ENOMEM;
1304
1305         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1306         if (!res)
1307                 return -ENODEV;
1308
1309         xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1310                                         resource_size(res));
1311         if (!xordev->xor_base)
1312                 return -EBUSY;
1313
1314         res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1315         if (!res)
1316                 return -ENODEV;
1317
1318         xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1319                                              resource_size(res));
1320         if (!xordev->xor_high_base)
1321                 return -EBUSY;
1322
1323         platform_set_drvdata(pdev, xordev);
1324
1325
1326         /*
1327          * We need to know which type of XOR device we use before
1328          * setting up. In non-dt case it can only be the legacy one.
1329          */
1330         xordev->xor_type = XOR_ORION;
1331         if (pdev->dev.of_node) {
1332                 const struct of_device_id *of_id =
1333                         of_match_device(mv_xor_dt_ids,
1334                                         &pdev->dev);
1335
1336                 xordev->xor_type = (uintptr_t)of_id->data;
1337         }
1338
1339         /*
1340          * (Re-)program MBUS remapping windows if we are asked to.
1341          */
1342         if (xordev->xor_type == XOR_ARMADA_37XX) {
1343                 mv_xor_conf_mbus_windows_a3700(xordev);
1344         } else {
1345                 dram = mv_mbus_dram_info();
1346                 if (dram)
1347                         mv_xor_conf_mbus_windows(xordev, dram);
1348         }
1349
1350         /* Not all platforms can gate the clock, so it is not
1351          * an error if the clock does not exists.
1352          */
1353         xordev->clk = clk_get(&pdev->dev, NULL);
1354         if (!IS_ERR(xordev->clk))
1355                 clk_prepare_enable(xordev->clk);
1356
1357         /*
1358          * We don't want to have more than one channel per CPU in
1359          * order for async_tx to perform well. So we limit the number
1360          * of engines and channels so that we take into account this
1361          * constraint. Note that we also want to use channels from
1362          * separate engines when possible.  For dual-CPU Armada 3700
1363          * SoC with single XOR engine allow using its both channels.
1364          */
1365         max_engines = num_present_cpus();
1366         if (xordev->xor_type == XOR_ARMADA_37XX)
1367                 max_channels =  num_present_cpus();
1368         else
1369                 max_channels = min_t(unsigned int,
1370                                      MV_XOR_MAX_CHANNELS,
1371                                      DIV_ROUND_UP(num_present_cpus(), 2));
1372
1373         if (mv_xor_engine_count >= max_engines)
1374                 return 0;
1375
1376         if (pdev->dev.of_node) {
1377                 struct device_node *np;
1378                 int i = 0;
1379
1380                 for_each_child_of_node(pdev->dev.of_node, np) {
1381                         struct mv_xor_chan *chan;
1382                         dma_cap_mask_t cap_mask;
1383                         int irq;
1384
1385                         if (i >= max_channels)
1386                                 continue;
1387
1388                         dma_cap_zero(cap_mask);
1389                         dma_cap_set(DMA_MEMCPY, cap_mask);
1390                         dma_cap_set(DMA_XOR, cap_mask);
1391                         dma_cap_set(DMA_INTERRUPT, cap_mask);
1392
1393                         irq = irq_of_parse_and_map(np, 0);
1394                         if (!irq) {
1395                                 ret = -ENODEV;
1396                                 goto err_channel_add;
1397                         }
1398
1399                         chan = mv_xor_channel_add(xordev, pdev, i,
1400                                                   cap_mask, irq);
1401                         if (IS_ERR(chan)) {
1402                                 ret = PTR_ERR(chan);
1403                                 irq_dispose_mapping(irq);
1404                                 goto err_channel_add;
1405                         }
1406
1407                         xordev->channels[i] = chan;
1408                         i++;
1409                 }
1410         } else if (pdata && pdata->channels) {
1411                 for (i = 0; i < max_channels; i++) {
1412                         struct mv_xor_channel_data *cd;
1413                         struct mv_xor_chan *chan;
1414                         int irq;
1415
1416                         cd = &pdata->channels[i];
1417                         irq = platform_get_irq(pdev, i);
1418                         if (irq < 0) {
1419                                 ret = irq;
1420                                 goto err_channel_add;
1421                         }
1422
1423                         chan = mv_xor_channel_add(xordev, pdev, i,
1424                                                   cd->cap_mask, irq);
1425                         if (IS_ERR(chan)) {
1426                                 ret = PTR_ERR(chan);
1427                                 goto err_channel_add;
1428                         }
1429
1430                         xordev->channels[i] = chan;
1431                 }
1432         }
1433
1434         return 0;
1435
1436 err_channel_add:
1437         for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
1438                 if (xordev->channels[i]) {
1439                         mv_xor_channel_remove(xordev->channels[i]);
1440                         if (pdev->dev.of_node)
1441                                 irq_dispose_mapping(xordev->channels[i]->irq);
1442                 }
1443
1444         if (!IS_ERR(xordev->clk)) {
1445                 clk_disable_unprepare(xordev->clk);
1446                 clk_put(xordev->clk);
1447         }
1448
1449         return ret;
1450 }
1451
1452 static struct platform_driver mv_xor_driver = {
1453         .probe          = mv_xor_probe,
1454         .suspend        = mv_xor_suspend,
1455         .resume         = mv_xor_resume,
1456         .driver         = {
1457                 .name           = MV_XOR_NAME,
1458                 .of_match_table = of_match_ptr(mv_xor_dt_ids),
1459         },
1460 };
1461
1462 builtin_platform_driver(mv_xor_driver);
1463
1464 /*
1465 MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1466 MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1467 MODULE_LICENSE("GPL");
1468 */