drivers/dma/idxd/dma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/io-64-nonatomic-lo-hi.h>
   9 #include <linux/dmaengine.h>
  10 #include <uapi/linux/idxd.h>
  11 #include "../dmaengine.h"
  12 #include "registers.h"
  13 #include "idxd.h"
  14
  15 static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
  16 {
  17         struct idxd_dma_chan *idxd_chan;
  18
  19         idxd_chan = container_of(c, struct idxd_dma_chan, chan);
  20         return idxd_chan->wq;
  21 }
  22
  23 void idxd_dma_complete_txd(struct idxd_desc *desc,
  24                            enum idxd_complete_type comp_type)
  25 {
  26         struct dma_async_tx_descriptor *tx;
  27         struct dmaengine_result res;
  28         int complete = 1;
  29
  30         if (desc->completion->status == DSA_COMP_SUCCESS)
  31                 res.result = DMA_TRANS_NOERROR;
  32         else if (desc->completion->status)
  33                 res.result = DMA_TRANS_WRITE_FAILED;
  34         else if (comp_type == IDXD_COMPLETE_ABORT)
  35                 res.result = DMA_TRANS_ABORTED;
  36         else
  37                 complete = 0;
  38
  39         tx = &desc->txd;
  40         if (complete && tx->cookie) {
  41                 dma_cookie_complete(tx);
  42                 dma_descriptor_unmap(tx);
  43                 dmaengine_desc_get_callback_invoke(tx, &res);
  44                 tx->callback = NULL;
  45                 tx->callback_result = NULL;
  46         }
  47 }
  48
  49 static void op_flag_setup(unsigned long flags, u32 *desc_flags)
  50 {
  51         *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
  52         if (flags & DMA_PREP_INTERRUPT)
  53                 *desc_flags |= IDXD_OP_FLAG_RCI;
  54 }
  55
  56 static inline void set_completion_address(struct idxd_desc *desc,
  57                                           u64 *compl_addr)
  58 {
  59                 *compl_addr = desc->compl_dma;
  60 }
  61
  62 static inline void idxd_prep_desc_common(struct idxd_wq *wq,
  63                                          struct dsa_hw_desc *hw, char opcode,
  64                                          u64 addr_f1, u64 addr_f2, u64 len,
  65                                          u64 compl, u32 flags)
  66 {
  67         hw->flags = flags;
  68         hw->opcode = opcode;
  69         hw->src_addr = addr_f1;
  70         hw->dst_addr = addr_f2;
  71         hw->xfer_size = len;
  72         hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
  73         hw->completion_addr = compl;
  74 }
  75
  76 static struct dma_async_tx_descriptor *
  77 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
  78                        dma_addr_t dma_src, size_t len, unsigned long flags)
  79 {
  80         struct idxd_wq *wq = to_idxd_wq(c);
  81         u32 desc_flags;
  82         struct idxd_device *idxd = wq->idxd;
  83         struct idxd_desc *desc;
  84
  85         if (wq->state != IDXD_WQ_ENABLED)
  86                 return NULL;
  87
  88         if (len > idxd->max_xfer_bytes)
  89                 return NULL;
  90
  91         op_flag_setup(flags, &desc_flags);
  92         desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
  93         if (IS_ERR(desc))
  94                 return NULL;
  95
  96         idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
  97                               dma_src, dma_dest, len, desc->compl_dma,
  98                               desc_flags);
  99
 100         desc->txd.flags = flags;
 101
 102         return &desc->txd;
 103 }
 104
 105 static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
 106 {
 107         struct idxd_wq *wq = to_idxd_wq(chan);
 108         struct device *dev = &wq->idxd->pdev->dev;
 109
 110         idxd_wq_get(wq);
 111         dev_dbg(dev, "%s: client_count: %d\n", __func__,
 112                 idxd_wq_refcount(wq));
 113         return 0;
 114 }
 115
 116 static void idxd_dma_free_chan_resources(struct dma_chan *chan)
 117 {
 118         struct idxd_wq *wq = to_idxd_wq(chan);
 119         struct device *dev = &wq->idxd->pdev->dev;
 120
 121         idxd_wq_put(wq);
 122         dev_dbg(dev, "%s: client_count: %d\n", __func__,
 123                 idxd_wq_refcount(wq));
 124 }
 125
 126 static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
 127                                           dma_cookie_t cookie,
 128                                           struct dma_tx_state *txstate)
 129 {
 130         return DMA_OUT_OF_ORDER;
 131 }
 132
 133 /*
 134  * issue_pending() does not need to do anything since tx_submit() does the job
 135  * already.
 136  */
 137 static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
 138 {
 139 }
 140
 141 static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 142 {
 143         struct dma_chan *c = tx->chan;
 144         struct idxd_wq *wq = to_idxd_wq(c);
 145         dma_cookie_t cookie;
 146         int rc;
 147         struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
 148
 149         cookie = dma_cookie_assign(tx);
 150
 151         rc = idxd_submit_desc(wq, desc);
 152         if (rc < 0)
 153                 return rc;
 154
 155         return cookie;
 156 }
 157
 158 static void idxd_dma_release(struct dma_device *device)
 159 {
 160         struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma);
 161
 162         kfree(idxd_dma);
 163 }
 164
 165 int idxd_register_dma_device(struct idxd_device *idxd)
 166 {
 167         struct idxd_dma_dev *idxd_dma;
 168         struct dma_device *dma;
 169         struct device *dev = &idxd->pdev->dev;
 170         int rc;
 171
 172         idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev));
 173         if (!idxd_dma)
 174                 return -ENOMEM;
 175
 176         dma = &idxd_dma->dma;
 177         INIT_LIST_HEAD(&dma->channels);
 178         dma->dev = dev;
 179
 180         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 181         dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
 182         dma->device_release = idxd_dma_release;
 183
 184         if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
 185                 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 186                 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
 187         }
 188
 189         dma->device_tx_status = idxd_dma_tx_status;
 190         dma->device_issue_pending = idxd_dma_issue_pending;
 191         dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
 192         dma->device_free_chan_resources = idxd_dma_free_chan_resources;
 193
 194         rc = dma_async_device_register(dma);
 195         if (rc < 0) {
 196                 kfree(idxd_dma);
 197                 return rc;
 198         }
 199
 200         idxd_dma->idxd = idxd;
 201         /*
 202          * This pointer is protected by the refs taken by the dma_chan. It will remain valid
 203          * as long as there are outstanding channels.
 204          */
 205         idxd->idxd_dma = idxd_dma;
 206         return 0;
 207 }
 208
 209 void idxd_unregister_dma_device(struct idxd_device *idxd)
 210 {
 211         dma_async_device_unregister(&idxd->idxd_dma->dma);
 212 }
 213
 214 int idxd_register_dma_channel(struct idxd_wq *wq)
 215 {
 216         struct idxd_device *idxd = wq->idxd;
 217         struct dma_device *dma = &idxd->idxd_dma->dma;
 218         struct device *dev = &idxd->pdev->dev;
 219         struct idxd_dma_chan *idxd_chan;
 220         struct dma_chan *chan;
 221         int rc, i;
 222
 223         idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev));
 224         if (!idxd_chan)
 225                 return -ENOMEM;
 226
 227         chan = &idxd_chan->chan;
 228         chan->device = dma;
 229         list_add_tail(&chan->device_node, &dma->channels);
 230
 231         for (i = 0; i < wq->num_descs; i++) {
 232                 struct idxd_desc *desc = wq->descs[i];
 233
 234                 dma_async_tx_descriptor_init(&desc->txd, chan);
 235                 desc->txd.tx_submit = idxd_dma_tx_submit;
 236         }
 237
 238         rc = dma_async_device_channel_register(dma, chan);
 239         if (rc < 0) {
 240                 kfree(idxd_chan);
 241                 return rc;
 242         }
 243
 244         wq->idxd_chan = idxd_chan;
 245         idxd_chan->wq = wq;
 246         get_device(wq_confdev(wq));
 247
 248         return 0;
 249 }
 250
 251 void idxd_unregister_dma_channel(struct idxd_wq *wq)
 252 {
 253         struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
 254         struct dma_chan *chan = &idxd_chan->chan;
 255         struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma;
 256
 257         dma_async_device_channel_unregister(&idxd_dma->dma, chan);
 258         list_del(&chan->device_node);
 259         kfree(wq->idxd_chan);
 260         wq->idxd_chan = NULL;
 261         put_device(wq_confdev(wq));
 262 }
 263
 264 static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
 265 {
 266         struct device *dev = &idxd_dev->conf_dev;
 267         struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
 268         struct idxd_device *idxd = wq->idxd;
 269         int rc;
 270
 271         if (idxd->state != IDXD_DEV_ENABLED)
 272                 return -ENXIO;
 273
 274         mutex_lock(&wq->wq_lock);
 275         wq->type = IDXD_WQT_KERNEL;
 276         rc = __drv_enable_wq(wq);
 277         if (rc < 0) {
 278                 dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
 279                 rc = -ENXIO;
 280                 goto err;
 281         }
 282
 283         rc = idxd_wq_alloc_resources(wq);
 284         if (rc < 0) {
 285                 idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
 286                 dev_dbg(dev, "WQ resource alloc failed\n");
 287                 goto err_res_alloc;
 288         }
 289
 290         rc = idxd_wq_init_percpu_ref(wq);
 291         if (rc < 0) {
 292                 idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
 293                 dev_dbg(dev, "percpu_ref setup failed\n");
 294                 goto err_ref;
 295         }
 296
 297         rc = idxd_register_dma_channel(wq);
 298         if (rc < 0) {
 299                 idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
 300                 dev_dbg(dev, "Failed to register dma channel\n");
 301                 goto err_dma;
 302         }
 303
 304         idxd->cmd_status = 0;
 305         mutex_unlock(&wq->wq_lock);
 306         return 0;
 307
 308 err_dma:
 309         idxd_wq_quiesce(wq);
 310 err_ref:
 311         idxd_wq_free_resources(wq);
 312 err_res_alloc:
 313         __drv_disable_wq(wq);
 314 err:
 315         wq->type = IDXD_WQT_NONE;
 316         mutex_unlock(&wq->wq_lock);
 317         return rc;
 318 }
 319
 320 static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
 321 {
 322         struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
 323
 324         mutex_lock(&wq->wq_lock);
 325         idxd_wq_quiesce(wq);
 326         idxd_unregister_dma_channel(wq);
 327         __drv_disable_wq(wq);
 328         idxd_wq_free_resources(wq);
 329         wq->type = IDXD_WQT_NONE;
 330         mutex_unlock(&wq->wq_lock);
 331 }
 332
 333 static enum idxd_dev_type dev_types[] = {
 334         IDXD_DEV_WQ,
 335         IDXD_DEV_NONE,
 336 };
 337
 338 struct idxd_device_driver idxd_dmaengine_drv = {
 339         .probe = idxd_dmaengine_drv_probe,
 340         .remove = idxd_dmaengine_drv_remove,
 341         .name = "dmaengine",
 342         .type = dev_types,
 343 };
 344 EXPORT_SYMBOL_GPL(idxd_dmaengine_drv);