drivers/dma/bcm-sba-raid.c

   1 /*
   2  * Copyright (C) 2017 Broadcom
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 /*
  10  * Broadcom SBA RAID Driver
  11  *
  12  * The Broadcom stream buffer accelerator (SBA) provides offloading
  13  * capabilities for RAID operations. The SBA offload engine is accessible
  14  * via Broadcom SoC specific ring manager. Two or more offload engines
  15  * can share same Broadcom SoC specific ring manager due to this Broadcom
  16  * SoC specific ring manager driver is implemented as a mailbox controller
  17  * driver and offload engine drivers are implemented as mallbox clients.
  18  *
  19  * Typically, Broadcom SoC specific ring manager will implement larger
  20  * number of hardware rings over one or more SBA hardware devices. By
  21  * design, the internal buffer size of SBA hardware device is limited
  22  * but all offload operations supported by SBA can be broken down into
  23  * multiple small size requests and executed parallely on multiple SBA
  24  * hardware devices for achieving high through-put.
  25  *
  26  * The Broadcom SBA RAID driver does not require any register programming
  27  * except submitting request to SBA hardware device via mailbox channels.
  28  * This driver implements a DMA device with one DMA channel using a set
  29  * of mailbox channels provided by Broadcom SoC specific ring manager
  30  * driver. To exploit parallelism (as described above), all DMA request
  31  * coming to SBA RAID DMA channel are broken down to smaller requests
  32  * and submitted to multiple mailbox channels in round-robin fashion.
  33  * For having more SBA DMA channels, we can create more SBA device nodes
  34  * in Broadcom SoC specific DTS based on number of hardware rings supported
  35  * by Broadcom SoC ring manager.
  36  */
  37
  38 #include <linux/bitops.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/dmaengine.h>
  41 #include <linux/list.h>
  42 #include <linux/mailbox_client.h>
  43 #include <linux/mailbox/brcm-message.h>
  44 #include <linux/module.h>
  45 #include <linux/of_device.h>
  46 #include <linux/slab.h>
  47 #include <linux/raid/pq.h>
  48
  49 #include "dmaengine.h"
  50
  51 /* SBA command related defines */
  52 #define SBA_TYPE_SHIFT                                  48
  53 #define SBA_TYPE_MASK                                   GENMASK(1, 0)
  54 #define SBA_TYPE_A                                      0x0
  55 #define SBA_TYPE_B                                      0x2
  56 #define SBA_TYPE_C                                      0x3
  57 #define SBA_USER_DEF_SHIFT                              32
  58 #define SBA_USER_DEF_MASK                               GENMASK(15, 0)
  59 #define SBA_R_MDATA_SHIFT                               24
  60 #define SBA_R_MDATA_MASK                                GENMASK(7, 0)
  61 #define SBA_C_MDATA_MS_SHIFT                            18
  62 #define SBA_C_MDATA_MS_MASK                             GENMASK(1, 0)
  63 #define SBA_INT_SHIFT                                   17
  64 #define SBA_INT_MASK                                    BIT(0)
  65 #define SBA_RESP_SHIFT                                  16
  66 #define SBA_RESP_MASK                                   BIT(0)
  67 #define SBA_C_MDATA_SHIFT                               8
  68 #define SBA_C_MDATA_MASK                                GENMASK(7, 0)
  69 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum)                 (2 * (__bnum))
  70 #define SBA_C_MDATA_BNUMx_MASK                          GENMASK(1, 0)
  71 #define SBA_C_MDATA_DNUM_SHIFT                          5
  72 #define SBA_C_MDATA_DNUM_MASK                           GENMASK(4, 0)
  73 #define SBA_C_MDATA_LS(__v)                             ((__v) & 0xff)
  74 #define SBA_C_MDATA_MS(__v)                             (((__v) >> 8) & 0x3)
  75 #define SBA_CMD_SHIFT                                   0
  76 #define SBA_CMD_MASK                                    GENMASK(3, 0)
  77 #define SBA_CMD_ZERO_BUFFER                             0x4
  78 #define SBA_CMD_ZERO_ALL_BUFFERS                        0x8
  79 #define SBA_CMD_LOAD_BUFFER                             0x9
  80 #define SBA_CMD_XOR                                     0xa
  81 #define SBA_CMD_GALOIS_XOR                              0xb
  82 #define SBA_CMD_WRITE_BUFFER                            0xc
  83 #define SBA_CMD_GALOIS                                  0xe
  84
  85 /* Driver helper macros */
  86 #define to_sba_request(tx)              \
  87         container_of(tx, struct sba_request, tx)
  88 #define to_sba_device(dchan)            \
  89         container_of(dchan, struct sba_device, dma_chan)
  90
  91 enum sba_request_state {
  92         SBA_REQUEST_STATE_FREE = 1,
  93         SBA_REQUEST_STATE_ALLOCED = 2,
  94         SBA_REQUEST_STATE_PENDING = 3,
  95         SBA_REQUEST_STATE_ACTIVE = 4,
  96         SBA_REQUEST_STATE_RECEIVED = 5,
  97         SBA_REQUEST_STATE_COMPLETED = 6,
  98         SBA_REQUEST_STATE_ABORTED = 7,
  99 };
 100
 101 struct sba_request {
 102         /* Global state */
 103         struct list_head node;
 104         struct sba_device *sba;
 105         enum sba_request_state state;
 106         bool fence;
 107         /* Chained requests management */
 108         struct sba_request *first;
 109         struct list_head next;
 110         unsigned int next_count;
 111         atomic_t next_pending_count;
 112         /* BRCM message data */
 113         void *resp;
 114         dma_addr_t resp_dma;
 115         struct brcm_sba_command *cmds;
 116         struct brcm_message msg;
 117         struct dma_async_tx_descriptor tx;
 118 };
 119
 120 enum sba_version {
 121         SBA_VER_1 = 0,
 122         SBA_VER_2
 123 };
 124
 125 struct sba_device {
 126         /* Underlying device */
 127         struct device *dev;
 128         /* DT configuration parameters */
 129         enum sba_version ver;
 130         /* Derived configuration parameters */
 131         u32 max_req;
 132         u32 hw_buf_size;
 133         u32 hw_resp_size;
 134         u32 max_pq_coefs;
 135         u32 max_pq_srcs;
 136         u32 max_cmd_per_req;
 137         u32 max_xor_srcs;
 138         u32 max_resp_pool_size;
 139         u32 max_cmds_pool_size;
 140         /* Maibox client and Mailbox channels */
 141         struct mbox_client client;
 142         int mchans_count;
 143         atomic_t mchans_current;
 144         struct mbox_chan **mchans;
 145         struct device *mbox_dev;
 146         /* DMA device and DMA channel */
 147         struct dma_device dma_dev;
 148         struct dma_chan dma_chan;
 149         /* DMA channel resources */
 150         void *resp_base;
 151         dma_addr_t resp_dma_base;
 152         void *cmds_base;
 153         dma_addr_t cmds_dma_base;
 154         spinlock_t reqs_lock;
 155         struct sba_request *reqs;
 156         bool reqs_fence;
 157         struct list_head reqs_alloc_list;
 158         struct list_head reqs_pending_list;
 159         struct list_head reqs_active_list;
 160         struct list_head reqs_received_list;
 161         struct list_head reqs_completed_list;
 162         struct list_head reqs_aborted_list;
 163         struct list_head reqs_free_list;
 164         int reqs_free_count;
 165 };
 166
 167 /* ====== SBA command helper routines ===== */
 168
 169 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask)
 170 {
 171         cmd &= ~((u64)mask << shift);
 172         cmd |= ((u64)(val & mask) << shift);
 173         return cmd;
 174 }
 175
 176 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0)
 177 {
 178         return b0 & SBA_C_MDATA_BNUMx_MASK;
 179 }
 180
 181 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0)
 182 {
 183         return b0 & SBA_C_MDATA_BNUMx_MASK;
 184 }
 185
 186 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0)
 187 {
 188         return (b0 & SBA_C_MDATA_BNUMx_MASK) |
 189                ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1));
 190 }
 191
 192 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0)
 193 {
 194         return (b0 & SBA_C_MDATA_BNUMx_MASK) |
 195                ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) |
 196                ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT);
 197 }
 198
 199 /* ====== Channel resource management routines ===== */
 200
 201 static struct sba_request *sba_alloc_request(struct sba_device *sba)
 202 {
 203         unsigned long flags;
 204         struct sba_request *req = NULL;
 205
 206         spin_lock_irqsave(&sba->reqs_lock, flags);
 207
 208         req = list_first_entry_or_null(&sba->reqs_free_list,
 209                                        struct sba_request, node);
 210         if (req) {
 211                 list_move_tail(&req->node, &sba->reqs_alloc_list);
 212                 req->state = SBA_REQUEST_STATE_ALLOCED;
 213                 req->fence = false;
 214                 req->first = req;
 215                 INIT_LIST_HEAD(&req->next);
 216                 req->next_count = 1;
 217                 atomic_set(&req->next_pending_count, 1);
 218
 219                 sba->reqs_free_count--;
 220
 221                 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
 222         }
 223
 224         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 225
 226         return req;
 227 }
 228
 229 /* Note: Must be called with sba->reqs_lock held */
 230 static void _sba_pending_request(struct sba_device *sba,
 231                                  struct sba_request *req)
 232 {
 233         lockdep_assert_held(&sba->reqs_lock);
 234         req->state = SBA_REQUEST_STATE_PENDING;
 235         list_move_tail(&req->node, &sba->reqs_pending_list);
 236         if (list_empty(&sba->reqs_active_list))
 237                 sba->reqs_fence = false;
 238 }
 239
 240 /* Note: Must be called with sba->reqs_lock held */
 241 static bool _sba_active_request(struct sba_device *sba,
 242                                 struct sba_request *req)
 243 {
 244         lockdep_assert_held(&sba->reqs_lock);
 245         if (list_empty(&sba->reqs_active_list))
 246                 sba->reqs_fence = false;
 247         if (sba->reqs_fence)
 248                 return false;
 249         req->state = SBA_REQUEST_STATE_ACTIVE;
 250         list_move_tail(&req->node, &sba->reqs_active_list);
 251         if (req->fence)
 252                 sba->reqs_fence = true;
 253         return true;
 254 }
 255
 256 /* Note: Must be called with sba->reqs_lock held */
 257 static void _sba_abort_request(struct sba_device *sba,
 258                                struct sba_request *req)
 259 {
 260         lockdep_assert_held(&sba->reqs_lock);
 261         req->state = SBA_REQUEST_STATE_ABORTED;
 262         list_move_tail(&req->node, &sba->reqs_aborted_list);
 263         if (list_empty(&sba->reqs_active_list))
 264                 sba->reqs_fence = false;
 265 }
 266
 267 /* Note: Must be called with sba->reqs_lock held */
 268 static void _sba_free_request(struct sba_device *sba,
 269                               struct sba_request *req)
 270 {
 271         lockdep_assert_held(&sba->reqs_lock);
 272         req->state = SBA_REQUEST_STATE_FREE;
 273         list_move_tail(&req->node, &sba->reqs_free_list);
 274         if (list_empty(&sba->reqs_active_list))
 275                 sba->reqs_fence = false;
 276         sba->reqs_free_count++;
 277 }
 278
 279 static void sba_received_request(struct sba_request *req)
 280 {
 281         unsigned long flags;
 282         struct sba_device *sba = req->sba;
 283
 284         spin_lock_irqsave(&sba->reqs_lock, flags);
 285         req->state = SBA_REQUEST_STATE_RECEIVED;
 286         list_move_tail(&req->node, &sba->reqs_received_list);
 287         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 288 }
 289
 290 static void sba_complete_chained_requests(struct sba_request *req)
 291 {
 292         unsigned long flags;
 293         struct sba_request *nreq;
 294         struct sba_device *sba = req->sba;
 295
 296         spin_lock_irqsave(&sba->reqs_lock, flags);
 297
 298         req->state = SBA_REQUEST_STATE_COMPLETED;
 299         list_move_tail(&req->node, &sba->reqs_completed_list);
 300         list_for_each_entry(nreq, &req->next, next) {
 301                 nreq->state = SBA_REQUEST_STATE_COMPLETED;
 302                 list_move_tail(&nreq->node, &sba->reqs_completed_list);
 303         }
 304         if (list_empty(&sba->reqs_active_list))
 305                 sba->reqs_fence = false;
 306
 307         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 308 }
 309
 310 static void sba_free_chained_requests(struct sba_request *req)
 311 {
 312         unsigned long flags;
 313         struct sba_request *nreq;
 314         struct sba_device *sba = req->sba;
 315
 316         spin_lock_irqsave(&sba->reqs_lock, flags);
 317
 318         _sba_free_request(sba, req);
 319         list_for_each_entry(nreq, &req->next, next)
 320                 _sba_free_request(sba, nreq);
 321
 322         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 323 }
 324
 325 static void sba_chain_request(struct sba_request *first,
 326                               struct sba_request *req)
 327 {
 328         unsigned long flags;
 329         struct sba_device *sba = req->sba;
 330
 331         spin_lock_irqsave(&sba->reqs_lock, flags);
 332
 333         list_add_tail(&req->next, &first->next);
 334         req->first = first;
 335         first->next_count++;
 336         atomic_set(&first->next_pending_count, first->next_count);
 337
 338         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 339 }
 340
 341 static void sba_cleanup_nonpending_requests(struct sba_device *sba)
 342 {
 343         unsigned long flags;
 344         struct sba_request *req, *req1;
 345
 346         spin_lock_irqsave(&sba->reqs_lock, flags);
 347
 348         /* Freeup all alloced request */
 349         list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node)
 350                 _sba_free_request(sba, req);
 351
 352         /* Freeup all received request */
 353         list_for_each_entry_safe(req, req1, &sba->reqs_received_list, node)
 354                 _sba_free_request(sba, req);
 355
 356         /* Freeup all completed request */
 357         list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node)
 358                 _sba_free_request(sba, req);
 359
 360         /* Set all active requests as aborted */
 361         list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node)
 362                 _sba_abort_request(sba, req);
 363
 364         /*
 365          * Note: We expect that aborted request will be eventually
 366          * freed by sba_receive_message()
 367          */
 368
 369         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 370 }
 371
 372 static void sba_cleanup_pending_requests(struct sba_device *sba)
 373 {
 374         unsigned long flags;
 375         struct sba_request *req, *req1;
 376
 377         spin_lock_irqsave(&sba->reqs_lock, flags);
 378
 379         /* Freeup all pending request */
 380         list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node)
 381                 _sba_free_request(sba, req);
 382
 383         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 384 }
 385
 386 /* ====== DMAENGINE callbacks ===== */
 387
 388 static void sba_free_chan_resources(struct dma_chan *dchan)
 389 {
 390         /*
 391          * Channel resources are pre-alloced so we just free-up
 392          * whatever we can so that we can re-use pre-alloced
 393          * channel resources next time.
 394          */
 395         sba_cleanup_nonpending_requests(to_sba_device(dchan));
 396 }
 397
 398 static int sba_device_terminate_all(struct dma_chan *dchan)
 399 {
 400         /* Cleanup all pending requests */
 401         sba_cleanup_pending_requests(to_sba_device(dchan));
 402
 403         return 0;
 404 }
 405
 406 static int sba_send_mbox_request(struct sba_device *sba,
 407                                  struct sba_request *req)
 408 {
 409         int mchans_idx, ret = 0;
 410
 411         /* Select mailbox channel in round-robin fashion */
 412         mchans_idx = atomic_inc_return(&sba->mchans_current);
 413         mchans_idx = mchans_idx % sba->mchans_count;
 414
 415         /* Send message for the request */
 416         req->msg.error = 0;
 417         ret = mbox_send_message(sba->mchans[mchans_idx], &req->msg);
 418         if (ret < 0) {
 419                 dev_err(sba->dev, "send message failed with error %d", ret);
 420                 return ret;
 421         }
 422         ret = req->msg.error;
 423         if (ret < 0) {
 424                 dev_err(sba->dev, "message error %d", ret);
 425                 return ret;
 426         }
 427
 428         return 0;
 429 }
 430
 431 static void sba_issue_pending(struct dma_chan *dchan)
 432 {
 433         int ret;
 434         unsigned long flags;
 435         struct sba_request *req, *req1;
 436         struct sba_device *sba = to_sba_device(dchan);
 437
 438         spin_lock_irqsave(&sba->reqs_lock, flags);
 439
 440         /* Process all pending request */
 441         list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) {
 442                 /* Try to make request active */
 443                 if (!_sba_active_request(sba, req))
 444                         break;
 445
 446                 /* Send request to mailbox channel */
 447                 spin_unlock_irqrestore(&sba->reqs_lock, flags);
 448                 ret = sba_send_mbox_request(sba, req);
 449                 spin_lock_irqsave(&sba->reqs_lock, flags);
 450
 451                 /* If something went wrong then keep request pending */
 452                 if (ret < 0) {
 453                         _sba_pending_request(sba, req);
 454                         break;
 455                 }
 456         }
 457
 458         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 459 }
 460
 461 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
 462 {
 463         unsigned long flags;
 464         dma_cookie_t cookie;
 465         struct sba_device *sba;
 466         struct sba_request *req, *nreq;
 467
 468         if (unlikely(!tx))
 469                 return -EINVAL;
 470
 471         sba = to_sba_device(tx->chan);
 472         req = to_sba_request(tx);
 473
 474         /* Assign cookie and mark all chained requests pending */
 475         spin_lock_irqsave(&sba->reqs_lock, flags);
 476         cookie = dma_cookie_assign(tx);
 477         _sba_pending_request(sba, req);
 478         list_for_each_entry(nreq, &req->next, next)
 479                 _sba_pending_request(sba, nreq);
 480         spin_unlock_irqrestore(&sba->reqs_lock, flags);
 481
 482         return cookie;
 483 }
 484
 485 static enum dma_status sba_tx_status(struct dma_chan *dchan,
 486                                      dma_cookie_t cookie,
 487                                      struct dma_tx_state *txstate)
 488 {
 489         int mchan_idx;
 490         enum dma_status ret;
 491         struct sba_device *sba = to_sba_device(dchan);
 492
 493         for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
 494                 mbox_client_peek_data(sba->mchans[mchan_idx]);
 495
 496         ret = dma_cookie_status(dchan, cookie, txstate);
 497         if (ret == DMA_COMPLETE)
 498                 return ret;
 499
 500         return dma_cookie_status(dchan, cookie, txstate);
 501 }
 502
 503 static void sba_fillup_interrupt_msg(struct sba_request *req,
 504                                      struct brcm_sba_command *cmds,
 505                                      struct brcm_message *msg)
 506 {
 507         u64 cmd;
 508         u32 c_mdata;
 509         struct brcm_sba_command *cmdsp = cmds;
 510
 511         /* Type-B command to load dummy data into buf0 */
 512         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 513                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 514         cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
 515                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 516         c_mdata = sba_cmd_load_c_mdata(0);
 517         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 518                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 519         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 520                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 521         cmdsp->cmd = cmd;
 522         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 523         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 524         cmdsp->data = req->resp_dma;
 525         cmdsp->data_len = req->sba->hw_resp_size;
 526         cmdsp++;
 527
 528         /* Type-A command to write buf0 to dummy location */
 529         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 530                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 531         cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size,
 532                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 533         cmd = sba_cmd_enc(cmd, 0x1,
 534                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 535         c_mdata = sba_cmd_write_c_mdata(0);
 536         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 537                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 538         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 539                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 540         cmdsp->cmd = cmd;
 541         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 542         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 543         if (req->sba->hw_resp_size) {
 544                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 545                 cmdsp->resp = req->resp_dma;
 546                 cmdsp->resp_len = req->sba->hw_resp_size;
 547         }
 548         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 549         cmdsp->data = req->resp_dma;
 550         cmdsp->data_len = req->sba->hw_resp_size;
 551         cmdsp++;
 552
 553         /* Fillup brcm_message */
 554         msg->type = BRCM_MESSAGE_SBA;
 555         msg->sba.cmds = cmds;
 556         msg->sba.cmds_count = cmdsp - cmds;
 557         msg->ctx = req;
 558         msg->error = 0;
 559 }
 560
 561 static struct dma_async_tx_descriptor *
 562 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags)
 563 {
 564         struct sba_request *req = NULL;
 565         struct sba_device *sba = to_sba_device(dchan);
 566
 567         /* Alloc new request */
 568         req = sba_alloc_request(sba);
 569         if (!req)
 570                 return NULL;
 571
 572         /*
 573          * Force fence so that no requests are submitted
 574          * until DMA callback for this request is invoked.
 575          */
 576         req->fence = true;
 577
 578         /* Fillup request message */
 579         sba_fillup_interrupt_msg(req, req->cmds, &req->msg);
 580
 581         /* Init async_tx descriptor */
 582         req->tx.flags = flags;
 583         req->tx.cookie = -EBUSY;
 584
 585         return &req->tx;
 586 }
 587
 588 static void sba_fillup_memcpy_msg(struct sba_request *req,
 589                                   struct brcm_sba_command *cmds,
 590                                   struct brcm_message *msg,
 591                                   dma_addr_t msg_offset, size_t msg_len,
 592                                   dma_addr_t dst, dma_addr_t src)
 593 {
 594         u64 cmd;
 595         u32 c_mdata;
 596         struct brcm_sba_command *cmdsp = cmds;
 597
 598         /* Type-B command to load data into buf0 */
 599         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 600                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 601         cmd = sba_cmd_enc(cmd, msg_len,
 602                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 603         c_mdata = sba_cmd_load_c_mdata(0);
 604         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 605                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 606         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 607                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 608         cmdsp->cmd = cmd;
 609         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 610         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 611         cmdsp->data = src + msg_offset;
 612         cmdsp->data_len = msg_len;
 613         cmdsp++;
 614
 615         /* Type-A command to write buf0 */
 616         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 617                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 618         cmd = sba_cmd_enc(cmd, msg_len,
 619                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 620         cmd = sba_cmd_enc(cmd, 0x1,
 621                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 622         c_mdata = sba_cmd_write_c_mdata(0);
 623         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 624                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 625         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 626                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 627         cmdsp->cmd = cmd;
 628         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 629         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 630         if (req->sba->hw_resp_size) {
 631                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 632                 cmdsp->resp = req->resp_dma;
 633                 cmdsp->resp_len = req->sba->hw_resp_size;
 634         }
 635         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 636         cmdsp->data = dst + msg_offset;
 637         cmdsp->data_len = msg_len;
 638         cmdsp++;
 639
 640         /* Fillup brcm_message */
 641         msg->type = BRCM_MESSAGE_SBA;
 642         msg->sba.cmds = cmds;
 643         msg->sba.cmds_count = cmdsp - cmds;
 644         msg->ctx = req;
 645         msg->error = 0;
 646 }
 647
 648 static struct sba_request *
 649 sba_prep_dma_memcpy_req(struct sba_device *sba,
 650                         dma_addr_t off, dma_addr_t dst, dma_addr_t src,
 651                         size_t len, unsigned long flags)
 652 {
 653         struct sba_request *req = NULL;
 654
 655         /* Alloc new request */
 656         req = sba_alloc_request(sba);
 657         if (!req)
 658                 return NULL;
 659         req->fence = (flags & DMA_PREP_FENCE) ? true : false;
 660
 661         /* Fillup request message */
 662         sba_fillup_memcpy_msg(req, req->cmds, &req->msg,
 663                               off, len, dst, src);
 664
 665         /* Init async_tx descriptor */
 666         req->tx.flags = flags;
 667         req->tx.cookie = -EBUSY;
 668
 669         return req;
 670 }
 671
 672 static struct dma_async_tx_descriptor *
 673 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
 674                     size_t len, unsigned long flags)
 675 {
 676         size_t req_len;
 677         dma_addr_t off = 0;
 678         struct sba_device *sba = to_sba_device(dchan);
 679         struct sba_request *first = NULL, *req;
 680
 681         /* Create chained requests where each request is upto hw_buf_size */
 682         while (len) {
 683                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
 684
 685                 req = sba_prep_dma_memcpy_req(sba, off, dst, src,
 686                                               req_len, flags);
 687                 if (!req) {
 688                         if (first)
 689                                 sba_free_chained_requests(first);
 690                         return NULL;
 691                 }
 692
 693                 if (first)
 694                         sba_chain_request(first, req);
 695                 else
 696                         first = req;
 697
 698                 off += req_len;
 699                 len -= req_len;
 700         }
 701
 702         return (first) ? &first->tx : NULL;
 703 }
 704
 705 static void sba_fillup_xor_msg(struct sba_request *req,
 706                                 struct brcm_sba_command *cmds,
 707                                 struct brcm_message *msg,
 708                                 dma_addr_t msg_offset, size_t msg_len,
 709                                 dma_addr_t dst, dma_addr_t *src, u32 src_cnt)
 710 {
 711         u64 cmd;
 712         u32 c_mdata;
 713         unsigned int i;
 714         struct brcm_sba_command *cmdsp = cmds;
 715
 716         /* Type-B command to load data into buf0 */
 717         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 718                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 719         cmd = sba_cmd_enc(cmd, msg_len,
 720                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 721         c_mdata = sba_cmd_load_c_mdata(0);
 722         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 723                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 724         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 725                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 726         cmdsp->cmd = cmd;
 727         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 728         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 729         cmdsp->data = src[0] + msg_offset;
 730         cmdsp->data_len = msg_len;
 731         cmdsp++;
 732
 733         /* Type-B commands to xor data with buf0 and put it back in buf0 */
 734         for (i = 1; i < src_cnt; i++) {
 735                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 736                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 737                 cmd = sba_cmd_enc(cmd, msg_len,
 738                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 739                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
 740                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 741                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 742                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
 743                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 744                 cmdsp->cmd = cmd;
 745                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 746                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 747                 cmdsp->data = src[i] + msg_offset;
 748                 cmdsp->data_len = msg_len;
 749                 cmdsp++;
 750         }
 751
 752         /* Type-A command to write buf0 */
 753         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 754                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 755         cmd = sba_cmd_enc(cmd, msg_len,
 756                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 757         cmd = sba_cmd_enc(cmd, 0x1,
 758                           SBA_RESP_SHIFT, SBA_RESP_MASK);
 759         c_mdata = sba_cmd_write_c_mdata(0);
 760         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 761                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 762         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 763                           SBA_CMD_SHIFT, SBA_CMD_MASK);
 764         cmdsp->cmd = cmd;
 765         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 766         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 767         if (req->sba->hw_resp_size) {
 768                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 769                 cmdsp->resp = req->resp_dma;
 770                 cmdsp->resp_len = req->sba->hw_resp_size;
 771         }
 772         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 773         cmdsp->data = dst + msg_offset;
 774         cmdsp->data_len = msg_len;
 775         cmdsp++;
 776
 777         /* Fillup brcm_message */
 778         msg->type = BRCM_MESSAGE_SBA;
 779         msg->sba.cmds = cmds;
 780         msg->sba.cmds_count = cmdsp - cmds;
 781         msg->ctx = req;
 782         msg->error = 0;
 783 }
 784
 785 struct sba_request *
 786 sba_prep_dma_xor_req(struct sba_device *sba,
 787                      dma_addr_t off, dma_addr_t dst, dma_addr_t *src,
 788                      u32 src_cnt, size_t len, unsigned long flags)
 789 {
 790         struct sba_request *req = NULL;
 791
 792         /* Alloc new request */
 793         req = sba_alloc_request(sba);
 794         if (!req)
 795                 return NULL;
 796         req->fence = (flags & DMA_PREP_FENCE) ? true : false;
 797
 798         /* Fillup request message */
 799         sba_fillup_xor_msg(req, req->cmds, &req->msg,
 800                            off, len, dst, src, src_cnt);
 801
 802         /* Init async_tx descriptor */
 803         req->tx.flags = flags;
 804         req->tx.cookie = -EBUSY;
 805
 806         return req;
 807 }
 808
 809 static struct dma_async_tx_descriptor *
 810 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src,
 811                  u32 src_cnt, size_t len, unsigned long flags)
 812 {
 813         size_t req_len;
 814         dma_addr_t off = 0;
 815         struct sba_device *sba = to_sba_device(dchan);
 816         struct sba_request *first = NULL, *req;
 817
 818         /* Sanity checks */
 819         if (unlikely(src_cnt > sba->max_xor_srcs))
 820                 return NULL;
 821
 822         /* Create chained requests where each request is upto hw_buf_size */
 823         while (len) {
 824                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
 825
 826                 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt,
 827                                            req_len, flags);
 828                 if (!req) {
 829                         if (first)
 830                                 sba_free_chained_requests(first);
 831                         return NULL;
 832                 }
 833
 834                 if (first)
 835                         sba_chain_request(first, req);
 836                 else
 837                         first = req;
 838
 839                 off += req_len;
 840                 len -= req_len;
 841         }
 842
 843         return (first) ? &first->tx : NULL;
 844 }
 845
 846 static void sba_fillup_pq_msg(struct sba_request *req,
 847                                 bool pq_continue,
 848                                 struct brcm_sba_command *cmds,
 849                                 struct brcm_message *msg,
 850                                 dma_addr_t msg_offset, size_t msg_len,
 851                                 dma_addr_t *dst_p, dma_addr_t *dst_q,
 852                                 const u8 *scf, dma_addr_t *src, u32 src_cnt)
 853 {
 854         u64 cmd;
 855         u32 c_mdata;
 856         unsigned int i;
 857         struct brcm_sba_command *cmdsp = cmds;
 858
 859         if (pq_continue) {
 860                 /* Type-B command to load old P into buf0 */
 861                 if (dst_p) {
 862                         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 863                                 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 864                         cmd = sba_cmd_enc(cmd, msg_len,
 865                                 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 866                         c_mdata = sba_cmd_load_c_mdata(0);
 867                         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 868                                 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 869                         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 870                                 SBA_CMD_SHIFT, SBA_CMD_MASK);
 871                         cmdsp->cmd = cmd;
 872                         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 873                         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 874                         cmdsp->data = *dst_p + msg_offset;
 875                         cmdsp->data_len = msg_len;
 876                         cmdsp++;
 877                 }
 878
 879                 /* Type-B command to load old Q into buf1 */
 880                 if (dst_q) {
 881                         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 882                                 SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 883                         cmd = sba_cmd_enc(cmd, msg_len,
 884                                 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 885                         c_mdata = sba_cmd_load_c_mdata(1);
 886                         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 887                                 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 888                         cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
 889                                 SBA_CMD_SHIFT, SBA_CMD_MASK);
 890                         cmdsp->cmd = cmd;
 891                         *cmdsp->cmd_dma = cpu_to_le64(cmd);
 892                         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 893                         cmdsp->data = *dst_q + msg_offset;
 894                         cmdsp->data_len = msg_len;
 895                         cmdsp++;
 896                 }
 897         } else {
 898                 /* Type-A command to zero all buffers */
 899                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 900                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 901                 cmd = sba_cmd_enc(cmd, msg_len,
 902                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 903                 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
 904                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 905                 cmdsp->cmd = cmd;
 906                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 907                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 908                 cmdsp++;
 909         }
 910
 911         /* Type-B commands for generate P onto buf0 and Q onto buf1 */
 912         for (i = 0; i < src_cnt; i++) {
 913                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
 914                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 915                 cmd = sba_cmd_enc(cmd, msg_len,
 916                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 917                 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0);
 918                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 919                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 920                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
 921                                   SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
 922                 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR,
 923                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 924                 cmdsp->cmd = cmd;
 925                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 926                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
 927                 cmdsp->data = src[i] + msg_offset;
 928                 cmdsp->data_len = msg_len;
 929                 cmdsp++;
 930         }
 931
 932         /* Type-A command to write buf0 */
 933         if (dst_p) {
 934                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 935                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 936                 cmd = sba_cmd_enc(cmd, msg_len,
 937                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 938                 cmd = sba_cmd_enc(cmd, 0x1,
 939                                   SBA_RESP_SHIFT, SBA_RESP_MASK);
 940                 c_mdata = sba_cmd_write_c_mdata(0);
 941                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 942                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 943                 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 944                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 945                 cmdsp->cmd = cmd;
 946                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 947                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 948                 if (req->sba->hw_resp_size) {
 949                         cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 950                         cmdsp->resp = req->resp_dma;
 951                         cmdsp->resp_len = req->sba->hw_resp_size;
 952                 }
 953                 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 954                 cmdsp->data = *dst_p + msg_offset;
 955                 cmdsp->data_len = msg_len;
 956                 cmdsp++;
 957         }
 958
 959         /* Type-A command to write buf1 */
 960         if (dst_q) {
 961                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
 962                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
 963                 cmd = sba_cmd_enc(cmd, msg_len,
 964                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
 965                 cmd = sba_cmd_enc(cmd, 0x1,
 966                                   SBA_RESP_SHIFT, SBA_RESP_MASK);
 967                 c_mdata = sba_cmd_write_c_mdata(1);
 968                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
 969                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
 970                 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
 971                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
 972                 cmdsp->cmd = cmd;
 973                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
 974                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
 975                 if (req->sba->hw_resp_size) {
 976                         cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
 977                         cmdsp->resp = req->resp_dma;
 978                         cmdsp->resp_len = req->sba->hw_resp_size;
 979                 }
 980                 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
 981                 cmdsp->data = *dst_q + msg_offset;
 982                 cmdsp->data_len = msg_len;
 983                 cmdsp++;
 984         }
 985
 986         /* Fillup brcm_message */
 987         msg->type = BRCM_MESSAGE_SBA;
 988         msg->sba.cmds = cmds;
 989         msg->sba.cmds_count = cmdsp - cmds;
 990         msg->ctx = req;
 991         msg->error = 0;
 992 }
 993
 994 struct sba_request *
 995 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off,
 996                     dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src,
 997                     u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
 998 {
 999         struct sba_request *req = NULL;
1000
1001         /* Alloc new request */
1002         req = sba_alloc_request(sba);
1003         if (!req)
1004                 return NULL;
1005         req->fence = (flags & DMA_PREP_FENCE) ? true : false;
1006
1007         /* Fillup request messages */
1008         sba_fillup_pq_msg(req, dmaf_continue(flags),
1009                           req->cmds, &req->msg,
1010                           off, len, dst_p, dst_q, scf, src, src_cnt);
1011
1012         /* Init async_tx descriptor */
1013         req->tx.flags = flags;
1014         req->tx.cookie = -EBUSY;
1015
1016         return req;
1017 }
1018
1019 static void sba_fillup_pq_single_msg(struct sba_request *req,
1020                                 bool pq_continue,
1021                                 struct brcm_sba_command *cmds,
1022                                 struct brcm_message *msg,
1023                                 dma_addr_t msg_offset, size_t msg_len,
1024                                 dma_addr_t *dst_p, dma_addr_t *dst_q,
1025                                 dma_addr_t src, u8 scf)
1026 {
1027         u64 cmd;
1028         u32 c_mdata;
1029         u8 pos, dpos = raid6_gflog[scf];
1030         struct brcm_sba_command *cmdsp = cmds;
1031
1032         if (!dst_p)
1033                 goto skip_p;
1034
1035         if (pq_continue) {
1036                 /* Type-B command to load old P into buf0 */
1037                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1038                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1039                 cmd = sba_cmd_enc(cmd, msg_len,
1040                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1041                 c_mdata = sba_cmd_load_c_mdata(0);
1042                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1043                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1044                 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1045                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1046                 cmdsp->cmd = cmd;
1047                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1048                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1049                 cmdsp->data = *dst_p + msg_offset;
1050                 cmdsp->data_len = msg_len;
1051                 cmdsp++;
1052
1053                 /*
1054                  * Type-B commands to xor data with buf0 and put it
1055                  * back in buf0
1056                  */
1057                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1058                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1059                 cmd = sba_cmd_enc(cmd, msg_len,
1060                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1061                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1062                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1063                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1064                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1065                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1066                 cmdsp->cmd = cmd;
1067                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1068                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1069                 cmdsp->data = src + msg_offset;
1070                 cmdsp->data_len = msg_len;
1071                 cmdsp++;
1072         } else {
1073                 /* Type-B command to load old P into buf0 */
1074                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1075                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1076                 cmd = sba_cmd_enc(cmd, msg_len,
1077                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1078                 c_mdata = sba_cmd_load_c_mdata(0);
1079                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1080                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1081                 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER,
1082                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1083                 cmdsp->cmd = cmd;
1084                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1085                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1086                 cmdsp->data = src + msg_offset;
1087                 cmdsp->data_len = msg_len;
1088                 cmdsp++;
1089         }
1090
1091         /* Type-A command to write buf0 */
1092         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1093                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1094         cmd = sba_cmd_enc(cmd, msg_len,
1095                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1096         cmd = sba_cmd_enc(cmd, 0x1,
1097                           SBA_RESP_SHIFT, SBA_RESP_MASK);
1098         c_mdata = sba_cmd_write_c_mdata(0);
1099         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1100                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1101         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1102                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1103         cmdsp->cmd = cmd;
1104         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1105         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1106         if (req->sba->hw_resp_size) {
1107                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1108                 cmdsp->resp = req->resp_dma;
1109                 cmdsp->resp_len = req->sba->hw_resp_size;
1110         }
1111         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1112         cmdsp->data = *dst_p + msg_offset;
1113         cmdsp->data_len = msg_len;
1114         cmdsp++;
1115
1116 skip_p:
1117         if (!dst_q)
1118                 goto skip_q;
1119
1120         /* Type-A command to zero all buffers */
1121         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1122                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1123         cmd = sba_cmd_enc(cmd, msg_len,
1124                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1125         cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS,
1126                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1127         cmdsp->cmd = cmd;
1128         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1129         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1130         cmdsp++;
1131
1132         if (dpos == 255)
1133                 goto skip_q_computation;
1134         pos = (dpos < req->sba->max_pq_coefs) ?
1135                 dpos : (req->sba->max_pq_coefs - 1);
1136
1137         /*
1138          * Type-B command to generate initial Q from data
1139          * and store output into buf0
1140          */
1141         cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1142                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1143         cmd = sba_cmd_enc(cmd, msg_len,
1144                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1145         c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0);
1146         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1147                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1148         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1149                           SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1150         cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1151                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1152         cmdsp->cmd = cmd;
1153         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1154         cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1155         cmdsp->data = src + msg_offset;
1156         cmdsp->data_len = msg_len;
1157         cmdsp++;
1158
1159         dpos -= pos;
1160
1161         /* Multiple Type-A command to generate final Q */
1162         while (dpos) {
1163                 pos = (dpos < req->sba->max_pq_coefs) ?
1164                         dpos : (req->sba->max_pq_coefs - 1);
1165
1166                 /*
1167                  * Type-A command to generate Q with buf0 and
1168                  * buf1 store result in buf0
1169                  */
1170                 cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1171                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1172                 cmd = sba_cmd_enc(cmd, msg_len,
1173                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1174                 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1);
1175                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1176                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1177                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata),
1178                                   SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK);
1179                 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS,
1180                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1181                 cmdsp->cmd = cmd;
1182                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1183                 cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1184                 cmdsp++;
1185
1186                 dpos -= pos;
1187         }
1188
1189 skip_q_computation:
1190         if (pq_continue) {
1191                 /*
1192                  * Type-B command to XOR previous output with
1193                  * buf0 and write it into buf0
1194                  */
1195                 cmd = sba_cmd_enc(0x0, SBA_TYPE_B,
1196                                   SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1197                 cmd = sba_cmd_enc(cmd, msg_len,
1198                                   SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1199                 c_mdata = sba_cmd_xor_c_mdata(0, 0);
1200                 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1201                                   SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1202                 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR,
1203                                   SBA_CMD_SHIFT, SBA_CMD_MASK);
1204                 cmdsp->cmd = cmd;
1205                 *cmdsp->cmd_dma = cpu_to_le64(cmd);
1206                 cmdsp->flags = BRCM_SBA_CMD_TYPE_B;
1207                 cmdsp->data = *dst_q + msg_offset;
1208                 cmdsp->data_len = msg_len;
1209                 cmdsp++;
1210         }
1211
1212         /* Type-A command to write buf0 */
1213         cmd = sba_cmd_enc(0x0, SBA_TYPE_A,
1214                           SBA_TYPE_SHIFT, SBA_TYPE_MASK);
1215         cmd = sba_cmd_enc(cmd, msg_len,
1216                           SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK);
1217         cmd = sba_cmd_enc(cmd, 0x1,
1218                           SBA_RESP_SHIFT, SBA_RESP_MASK);
1219         c_mdata = sba_cmd_write_c_mdata(0);
1220         cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata),
1221                           SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK);
1222         cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER,
1223                           SBA_CMD_SHIFT, SBA_CMD_MASK);
1224         cmdsp->cmd = cmd;
1225         *cmdsp->cmd_dma = cpu_to_le64(cmd);
1226         cmdsp->flags = BRCM_SBA_CMD_TYPE_A;
1227         if (req->sba->hw_resp_size) {
1228                 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP;
1229                 cmdsp->resp = req->resp_dma;
1230                 cmdsp->resp_len = req->sba->hw_resp_size;
1231         }
1232         cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT;
1233         cmdsp->data = *dst_q + msg_offset;
1234         cmdsp->data_len = msg_len;
1235         cmdsp++;
1236
1237 skip_q:
1238         /* Fillup brcm_message */
1239         msg->type = BRCM_MESSAGE_SBA;
1240         msg->sba.cmds = cmds;
1241         msg->sba.cmds_count = cmdsp - cmds;
1242         msg->ctx = req;
1243         msg->error = 0;
1244 }
1245
1246 struct sba_request *
1247 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off,
1248                            dma_addr_t *dst_p, dma_addr_t *dst_q,
1249                            dma_addr_t src, u8 scf, size_t len,
1250                            unsigned long flags)
1251 {
1252         struct sba_request *req = NULL;
1253
1254         /* Alloc new request */
1255         req = sba_alloc_request(sba);
1256         if (!req)
1257                 return NULL;
1258         req->fence = (flags & DMA_PREP_FENCE) ? true : false;
1259
1260         /* Fillup request messages */
1261         sba_fillup_pq_single_msg(req,  dmaf_continue(flags),
1262                                  req->cmds, &req->msg, off, len,
1263                                  dst_p, dst_q, src, scf);
1264
1265         /* Init async_tx descriptor */
1266         req->tx.flags = flags;
1267         req->tx.cookie = -EBUSY;
1268
1269         return req;
1270 }
1271
1272 static struct dma_async_tx_descriptor *
1273 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src,
1274                 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags)
1275 {
1276         u32 i, dst_q_index;
1277         size_t req_len;
1278         bool slow = false;
1279         dma_addr_t off = 0;
1280         dma_addr_t *dst_p = NULL, *dst_q = NULL;
1281         struct sba_device *sba = to_sba_device(dchan);
1282         struct sba_request *first = NULL, *req;
1283
1284         /* Sanity checks */
1285         if (unlikely(src_cnt > sba->max_pq_srcs))
1286                 return NULL;
1287         for (i = 0; i < src_cnt; i++)
1288                 if (sba->max_pq_coefs <= raid6_gflog[scf[i]])
1289                         slow = true;
1290
1291         /* Figure-out P and Q destination addresses */
1292         if (!(flags & DMA_PREP_PQ_DISABLE_P))
1293                 dst_p = &dst[0];
1294         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
1295                 dst_q = &dst[1];
1296
1297         /* Create chained requests where each request is upto hw_buf_size */
1298         while (len) {
1299                 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size;
1300
1301                 if (slow) {
1302                         dst_q_index = src_cnt;
1303
1304                         if (dst_q) {
1305                                 for (i = 0; i < src_cnt; i++) {
1306                                         if (*dst_q == src[i]) {
1307                                                 dst_q_index = i;
1308                                                 break;
1309                                         }
1310                                 }
1311                         }
1312
1313                         if (dst_q_index < src_cnt) {
1314                                 i = dst_q_index;
1315                                 req = sba_prep_dma_pq_single_req(sba,
1316                                         off, dst_p, dst_q, src[i], scf[i],
1317                                         req_len, flags | DMA_PREP_FENCE);
1318                                 if (!req)
1319                                         goto fail;
1320
1321                                 if (first)
1322                                         sba_chain_request(first, req);
1323                                 else
1324                                         first = req;
1325
1326                                 flags |= DMA_PREP_CONTINUE;
1327                         }
1328
1329                         for (i = 0; i < src_cnt; i++) {
1330                                 if (dst_q_index == i)
1331                                         continue;
1332
1333                                 req = sba_prep_dma_pq_single_req(sba,
1334                                         off, dst_p, dst_q, src[i], scf[i],
1335                                         req_len, flags | DMA_PREP_FENCE);
1336                                 if (!req)
1337                                         goto fail;
1338
1339                                 if (first)
1340                                         sba_chain_request(first, req);
1341                                 else
1342                                         first = req;
1343
1344                                 flags |= DMA_PREP_CONTINUE;
1345                         }
1346                 } else {
1347                         req = sba_prep_dma_pq_req(sba, off,
1348                                                   dst_p, dst_q, src, src_cnt,
1349                                                   scf, req_len, flags);
1350                         if (!req)
1351                                 goto fail;
1352
1353                         if (first)
1354                                 sba_chain_request(first, req);
1355                         else
1356                                 first = req;
1357                 }
1358
1359                 off += req_len;
1360                 len -= req_len;
1361         }
1362
1363         return (first) ? &first->tx : NULL;
1364
1365 fail:
1366         if (first)
1367                 sba_free_chained_requests(first);
1368         return NULL;
1369 }
1370
1371 /* ====== Mailbox callbacks ===== */
1372
1373 static void sba_dma_tx_actions(struct sba_request *req)
1374 {
1375         struct dma_async_tx_descriptor *tx = &req->tx;
1376
1377         WARN_ON(tx->cookie < 0);
1378
1379         if (tx->cookie > 0) {
1380                 dma_cookie_complete(tx);
1381
1382                 /*
1383                  * Call the callback (must not sleep or submit new
1384                  * operations to this channel)
1385                  */
1386                 if (tx->callback)
1387                         tx->callback(tx->callback_param);
1388
1389                 dma_descriptor_unmap(tx);
1390         }
1391
1392         /* Run dependent operations */
1393         dma_run_dependencies(tx);
1394
1395         /* If waiting for 'ack' then move to completed list */
1396         if (!async_tx_test_ack(&req->tx))
1397                 sba_complete_chained_requests(req);
1398         else
1399                 sba_free_chained_requests(req);
1400 }
1401
1402 static void sba_receive_message(struct mbox_client *cl, void *msg)
1403 {
1404         unsigned long flags;
1405         struct brcm_message *m = msg;
1406         struct sba_request *req = m->ctx, *req1;
1407         struct sba_device *sba = req->sba;
1408
1409         /* Error count if message has error */
1410         if (m->error < 0)
1411                 dev_err(sba->dev, "%s got message with error %d",
1412                         dma_chan_name(&sba->dma_chan), m->error);
1413
1414         /* Mark request as received */
1415         sba_received_request(req);
1416
1417         /* Wait for all chained requests to be completed */
1418         if (atomic_dec_return(&req->first->next_pending_count))
1419                 goto done;
1420
1421         /* Point to first request */
1422         req = req->first;
1423
1424         /* Update request */
1425         if (req->state == SBA_REQUEST_STATE_RECEIVED)
1426                 sba_dma_tx_actions(req);
1427         else
1428                 sba_free_chained_requests(req);
1429
1430         spin_lock_irqsave(&sba->reqs_lock, flags);
1431
1432         /* Re-check all completed request waiting for 'ack' */
1433         list_for_each_entry_safe(req, req1, &sba->reqs_completed_list, node) {
1434                 spin_unlock_irqrestore(&sba->reqs_lock, flags);
1435                 sba_dma_tx_actions(req);
1436                 spin_lock_irqsave(&sba->reqs_lock, flags);
1437         }
1438
1439         spin_unlock_irqrestore(&sba->reqs_lock, flags);
1440
1441 done:
1442         /* Try to submit pending request */
1443         sba_issue_pending(&sba->dma_chan);
1444 }
1445
1446 /* ====== Platform driver routines ===== */
1447
1448 static int sba_prealloc_channel_resources(struct sba_device *sba)
1449 {
1450         int i, j, p, ret = 0;
1451         struct sba_request *req = NULL;
1452
1453         sba->resp_base = dma_alloc_coherent(sba->dma_dev.dev,
1454                                             sba->max_resp_pool_size,
1455                                             &sba->resp_dma_base, GFP_KERNEL);
1456         if (!sba->resp_base)
1457                 return -ENOMEM;
1458
1459         sba->cmds_base = dma_alloc_coherent(sba->dma_dev.dev,
1460                                             sba->max_cmds_pool_size,
1461                                             &sba->cmds_dma_base, GFP_KERNEL);
1462         if (!sba->cmds_base) {
1463                 ret = -ENOMEM;
1464                 goto fail_free_resp_pool;
1465         }
1466
1467         spin_lock_init(&sba->reqs_lock);
1468         sba->reqs_fence = false;
1469         INIT_LIST_HEAD(&sba->reqs_alloc_list);
1470         INIT_LIST_HEAD(&sba->reqs_pending_list);
1471         INIT_LIST_HEAD(&sba->reqs_active_list);
1472         INIT_LIST_HEAD(&sba->reqs_received_list);
1473         INIT_LIST_HEAD(&sba->reqs_completed_list);
1474         INIT_LIST_HEAD(&sba->reqs_aborted_list);
1475         INIT_LIST_HEAD(&sba->reqs_free_list);
1476
1477         sba->reqs = devm_kcalloc(sba->dev, sba->max_req,
1478                                  sizeof(*req), GFP_KERNEL);
1479         if (!sba->reqs) {
1480                 ret = -ENOMEM;
1481                 goto fail_free_cmds_pool;
1482         }
1483
1484         for (i = 0, p = 0; i < sba->max_req; i++) {
1485                 req = &sba->reqs[i];
1486                 INIT_LIST_HEAD(&req->node);
1487                 req->sba = sba;
1488                 req->state = SBA_REQUEST_STATE_FREE;
1489                 INIT_LIST_HEAD(&req->next);
1490                 req->next_count = 1;
1491                 atomic_set(&req->next_pending_count, 0);
1492                 req->fence = false;
1493                 req->resp = sba->resp_base + p;
1494                 req->resp_dma = sba->resp_dma_base + p;
1495                 p += sba->hw_resp_size;
1496                 req->cmds = devm_kcalloc(sba->dev, sba->max_cmd_per_req,
1497                                          sizeof(*req->cmds), GFP_KERNEL);
1498                 if (!req->cmds) {
1499                         ret = -ENOMEM;
1500                         goto fail_free_cmds_pool;
1501                 }
1502                 for (j = 0; j < sba->max_cmd_per_req; j++) {
1503                         req->cmds[j].cmd = 0;
1504                         req->cmds[j].cmd_dma = sba->cmds_base +
1505                                 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1506                         req->cmds[j].cmd_dma_addr = sba->cmds_dma_base +
1507                                 (i * sba->max_cmd_per_req + j) * sizeof(u64);
1508                         req->cmds[j].flags = 0;
1509                 }
1510                 memset(&req->msg, 0, sizeof(req->msg));
1511                 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan);
1512                 req->tx.tx_submit = sba_tx_submit;
1513                 req->tx.phys = req->resp_dma;
1514                 list_add_tail(&req->node, &sba->reqs_free_list);
1515         }
1516
1517         sba->reqs_free_count = sba->max_req;
1518
1519         return 0;
1520
1521 fail_free_cmds_pool:
1522         dma_free_coherent(sba->dma_dev.dev,
1523                           sba->max_cmds_pool_size,
1524                           sba->cmds_base, sba->cmds_dma_base);
1525 fail_free_resp_pool:
1526         dma_free_coherent(sba->dma_dev.dev,
1527                           sba->max_resp_pool_size,
1528                           sba->resp_base, sba->resp_dma_base);
1529         return ret;
1530 }
1531
1532 static void sba_freeup_channel_resources(struct sba_device *sba)
1533 {
1534         dmaengine_terminate_all(&sba->dma_chan);
1535         dma_free_coherent(sba->dma_dev.dev, sba->max_cmds_pool_size,
1536                           sba->cmds_base, sba->cmds_dma_base);
1537         dma_free_coherent(sba->dma_dev.dev, sba->max_resp_pool_size,
1538                           sba->resp_base, sba->resp_dma_base);
1539         sba->resp_base = NULL;
1540         sba->resp_dma_base = 0;
1541 }
1542
1543 static int sba_async_register(struct sba_device *sba)
1544 {
1545         int ret;
1546         struct dma_device *dma_dev = &sba->dma_dev;
1547
1548         /* Initialize DMA channel cookie */
1549         sba->dma_chan.device = dma_dev;
1550         dma_cookie_init(&sba->dma_chan);
1551
1552         /* Initialize DMA device capability mask */
1553         dma_cap_zero(dma_dev->cap_mask);
1554         dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
1555         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1556         dma_cap_set(DMA_XOR, dma_dev->cap_mask);
1557         dma_cap_set(DMA_PQ, dma_dev->cap_mask);
1558
1559         /*
1560          * Set mailbox channel device as the base device of
1561          * our dma_device because the actual memory accesses
1562          * will be done by mailbox controller
1563          */
1564         dma_dev->dev = sba->mbox_dev;
1565
1566         /* Set base prep routines */
1567         dma_dev->device_free_chan_resources = sba_free_chan_resources;
1568         dma_dev->device_terminate_all = sba_device_terminate_all;
1569         dma_dev->device_issue_pending = sba_issue_pending;
1570         dma_dev->device_tx_status = sba_tx_status;
1571
1572         /* Set interrupt routine */
1573         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1574                 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt;
1575
1576         /* Set memcpy routine */
1577         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1578                 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy;
1579
1580         /* Set xor routine and capability */
1581         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1582                 dma_dev->device_prep_dma_xor = sba_prep_dma_xor;
1583                 dma_dev->max_xor = sba->max_xor_srcs;
1584         }
1585
1586         /* Set pq routine and capability */
1587         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1588                 dma_dev->device_prep_dma_pq = sba_prep_dma_pq;
1589                 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0);
1590         }
1591
1592         /* Initialize DMA device channel list */
1593         INIT_LIST_HEAD(&dma_dev->channels);
1594         list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels);
1595
1596         /* Register with Linux async DMA framework*/
1597         ret = dma_async_device_register(dma_dev);
1598         if (ret) {
1599                 dev_err(sba->dev, "async device register error %d", ret);
1600                 return ret;
1601         }
1602
1603         dev_info(sba->dev, "%s capabilities: %s%s%s%s\n",
1604         dma_chan_name(&sba->dma_chan),
1605         dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "",
1606         dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "",
1607         dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1608         dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "");
1609
1610         return 0;
1611 }
1612
1613 static int sba_probe(struct platform_device *pdev)
1614 {
1615         int i, ret = 0, mchans_count;
1616         struct sba_device *sba;
1617         struct platform_device *mbox_pdev;
1618         struct of_phandle_args args;
1619
1620         /* Allocate main SBA struct */
1621         sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL);
1622         if (!sba)
1623                 return -ENOMEM;
1624
1625         sba->dev = &pdev->dev;
1626         platform_set_drvdata(pdev, sba);
1627
1628         /* Determine SBA version from DT compatible string */
1629         if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba"))
1630                 sba->ver = SBA_VER_1;
1631         else if (of_device_is_compatible(sba->dev->of_node,
1632                                          "brcm,iproc-sba-v2"))
1633                 sba->ver = SBA_VER_2;
1634         else
1635                 return -ENODEV;
1636
1637         /* Derived Configuration parameters */
1638         switch (sba->ver) {
1639         case SBA_VER_1:
1640                 sba->max_req = 1024;
1641                 sba->hw_buf_size = 4096;
1642                 sba->hw_resp_size = 8;
1643                 sba->max_pq_coefs = 6;
1644                 sba->max_pq_srcs = 6;
1645                 break;
1646         case SBA_VER_2:
1647                 sba->max_req = 1024;
1648                 sba->hw_buf_size = 4096;
1649                 sba->hw_resp_size = 8;
1650                 sba->max_pq_coefs = 30;
1651                 /*
1652                  * We can support max_pq_srcs == max_pq_coefs because
1653                  * we are limited by number of SBA commands that we can
1654                  * fit in one message for underlying ring manager HW.
1655                  */
1656                 sba->max_pq_srcs = 12;
1657                 break;
1658         default:
1659                 return -EINVAL;
1660         }
1661         sba->max_cmd_per_req = sba->max_pq_srcs + 3;
1662         sba->max_xor_srcs = sba->max_cmd_per_req - 1;
1663         sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size;
1664         sba->max_cmds_pool_size = sba->max_req *
1665                                   sba->max_cmd_per_req * sizeof(u64);
1666
1667         /* Setup mailbox client */
1668         sba->client.dev                 = &pdev->dev;
1669         sba->client.rx_callback         = sba_receive_message;
1670         sba->client.tx_block            = false;
1671         sba->client.knows_txdone        = false;
1672         sba->client.tx_tout             = 0;
1673
1674         /* Number of channels equals number of mailbox channels */
1675         ret = of_count_phandle_with_args(pdev->dev.of_node,
1676                                          "mboxes", "#mbox-cells");
1677         if (ret <= 0)
1678                 return -ENODEV;
1679         mchans_count = ret;
1680         sba->mchans_count = 0;
1681         atomic_set(&sba->mchans_current, 0);
1682
1683         /* Allocate mailbox channel array */
1684         sba->mchans = devm_kcalloc(&pdev->dev, sba->mchans_count,
1685                                    sizeof(*sba->mchans), GFP_KERNEL);
1686         if (!sba->mchans)
1687                 return -ENOMEM;
1688
1689         /* Request mailbox channels */
1690         for (i = 0; i < mchans_count; i++) {
1691                 sba->mchans[i] = mbox_request_channel(&sba->client, i);
1692                 if (IS_ERR(sba->mchans[i])) {
1693                         ret = PTR_ERR(sba->mchans[i]);
1694                         goto fail_free_mchans;
1695                 }
1696                 sba->mchans_count++;
1697         }
1698
1699         /* Find-out underlying mailbox device */
1700         ret = of_parse_phandle_with_args(pdev->dev.of_node,
1701                                          "mboxes", "#mbox-cells", 0, &args);
1702         if (ret)
1703                 goto fail_free_mchans;
1704         mbox_pdev = of_find_device_by_node(args.np);
1705         of_node_put(args.np);
1706         if (!mbox_pdev) {
1707                 ret = -ENODEV;
1708                 goto fail_free_mchans;
1709         }
1710         sba->mbox_dev = &mbox_pdev->dev;
1711
1712         /* All mailbox channels should be of same ring manager device */
1713         for (i = 1; i < mchans_count; i++) {
1714                 ret = of_parse_phandle_with_args(pdev->dev.of_node,
1715                                          "mboxes", "#mbox-cells", i, &args);
1716                 if (ret)
1717                         goto fail_free_mchans;
1718                 mbox_pdev = of_find_device_by_node(args.np);
1719                 of_node_put(args.np);
1720                 if (sba->mbox_dev != &mbox_pdev->dev) {
1721                         ret = -EINVAL;
1722                         goto fail_free_mchans;
1723                 }
1724         }
1725
1726         /* Register DMA device with linux async framework */
1727         ret = sba_async_register(sba);
1728         if (ret)
1729                 goto fail_free_mchans;
1730
1731         /* Prealloc channel resource */
1732         ret = sba_prealloc_channel_resources(sba);
1733         if (ret)
1734                 goto fail_async_dev_unreg;
1735
1736         /* Print device info */
1737         dev_info(sba->dev, "%s using SBAv%d and %d mailbox channels",
1738                  dma_chan_name(&sba->dma_chan), sba->ver+1,
1739                  sba->mchans_count);
1740
1741         return 0;
1742
1743 fail_async_dev_unreg:
1744         dma_async_device_unregister(&sba->dma_dev);
1745 fail_free_mchans:
1746         for (i = 0; i < sba->mchans_count; i++)
1747                 mbox_free_channel(sba->mchans[i]);
1748         return ret;
1749 }
1750
1751 static int sba_remove(struct platform_device *pdev)
1752 {
1753         int i;
1754         struct sba_device *sba = platform_get_drvdata(pdev);
1755
1756         sba_freeup_channel_resources(sba);
1757
1758         dma_async_device_unregister(&sba->dma_dev);
1759
1760         for (i = 0; i < sba->mchans_count; i++)
1761                 mbox_free_channel(sba->mchans[i]);
1762
1763         return 0;
1764 }
1765
1766 static const struct of_device_id sba_of_match[] = {
1767         { .compatible = "brcm,iproc-sba", },
1768         { .compatible = "brcm,iproc-sba-v2", },
1769         {},
1770 };
1771 MODULE_DEVICE_TABLE(of, sba_of_match);
1772
1773 static struct platform_driver sba_driver = {
1774         .probe = sba_probe,
1775         .remove = sba_remove,
1776         .driver = {
1777                 .name = "bcm-sba-raid",
1778                 .of_match_table = sba_of_match,
1779         },
1780 };
1781 module_platform_driver(sba_driver);
1782
1783 MODULE_DESCRIPTION("Broadcom SBA RAID driver");
1784 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1785 MODULE_LICENSE("GPL v2");