drivers/dma/idxd/dma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/device.h>
   8 #include <linux/io-64-nonatomic-lo-hi.h>
   9 #include <linux/dmaengine.h>
  10 #include <uapi/linux/idxd.h>
  11 #include "../dmaengine.h"
  12 #include "registers.h"
  13 #include "idxd.h"
  14
  15 static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
  16 {
  17         struct idxd_dma_chan *idxd_chan;
  18
  19         idxd_chan = container_of(c, struct idxd_dma_chan, chan);
  20         return idxd_chan->wq;
  21 }
  22
  23 void idxd_dma_complete_txd(struct idxd_desc *desc,
  24                            enum idxd_complete_type comp_type,
  25                            bool free_desc)
  26 {
  27         struct idxd_device *idxd = desc->wq->idxd;
  28         struct dma_async_tx_descriptor *tx;
  29         struct dmaengine_result res;
  30         int complete = 1;
  31
  32         if (desc->completion->status == DSA_COMP_SUCCESS) {
  33                 res.result = DMA_TRANS_NOERROR;
  34         } else if (desc->completion->status) {
  35                 if (idxd->request_int_handles && comp_type != IDXD_COMPLETE_ABORT &&
  36                     desc->completion->status == DSA_COMP_INT_HANDLE_INVAL &&
  37                     idxd_queue_int_handle_resubmit(desc))
  38                         return;
  39                 res.result = DMA_TRANS_WRITE_FAILED;
  40         } else if (comp_type == IDXD_COMPLETE_ABORT) {
  41                 res.result = DMA_TRANS_ABORTED;
  42         } else {
  43                 complete = 0;
  44         }
  45
  46         tx = &desc->txd;
  47         if (complete && tx->cookie) {
  48                 dma_cookie_complete(tx);
  49                 dma_descriptor_unmap(tx);
  50                 dmaengine_desc_get_callback_invoke(tx, &res);
  51                 tx->callback = NULL;
  52                 tx->callback_result = NULL;
  53         }
  54
  55         if (free_desc)
  56                 idxd_free_desc(desc->wq, desc);
  57 }
  58
  59 static void op_flag_setup(unsigned long flags, u32 *desc_flags)
  60 {
  61         *desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
  62         if (flags & DMA_PREP_INTERRUPT)
  63                 *desc_flags |= IDXD_OP_FLAG_RCI;
  64 }
  65
  66 static inline void set_completion_address(struct idxd_desc *desc,
  67                                           u64 *compl_addr)
  68 {
  69                 *compl_addr = desc->compl_dma;
  70 }
  71
  72 static inline void idxd_prep_desc_common(struct idxd_wq *wq,
  73                                          struct dsa_hw_desc *hw, char opcode,
  74                                          u64 addr_f1, u64 addr_f2, u64 len,
  75                                          u64 compl, u32 flags)
  76 {
  77         hw->flags = flags;
  78         hw->opcode = opcode;
  79         hw->src_addr = addr_f1;
  80         hw->dst_addr = addr_f2;
  81         hw->xfer_size = len;
  82         /*
  83          * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv
  84          * field instead. This field should be set to 1 for kernel descriptors.
  85          */
  86         hw->priv = 1;
  87         hw->completion_addr = compl;
  88 }
  89
  90 static struct dma_async_tx_descriptor *
  91 idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags)
  92 {
  93         struct idxd_wq *wq = to_idxd_wq(c);
  94         u32 desc_flags;
  95         struct idxd_desc *desc;
  96
  97         if (wq->state != IDXD_WQ_ENABLED)
  98                 return NULL;
  99
 100         op_flag_setup(flags, &desc_flags);
 101         desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
 102         if (IS_ERR(desc))
 103                 return NULL;
 104
 105         idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_NOOP,
 106                               0, 0, 0, desc->compl_dma, desc_flags);
 107         desc->txd.flags = flags;
 108         return &desc->txd;
 109 }
 110
 111 static struct dma_async_tx_descriptor *
 112 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
 113                        dma_addr_t dma_src, size_t len, unsigned long flags)
 114 {
 115         struct idxd_wq *wq = to_idxd_wq(c);
 116         u32 desc_flags;
 117         struct idxd_device *idxd = wq->idxd;
 118         struct idxd_desc *desc;
 119
 120         if (wq->state != IDXD_WQ_ENABLED)
 121                 return NULL;
 122
 123         if (len > idxd->max_xfer_bytes)
 124                 return NULL;
 125
 126         op_flag_setup(flags, &desc_flags);
 127         desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
 128         if (IS_ERR(desc))
 129                 return NULL;
 130
 131         idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
 132                               dma_src, dma_dest, len, desc->compl_dma,
 133                               desc_flags);
 134
 135         desc->txd.flags = flags;
 136
 137         return &desc->txd;
 138 }
 139
 140 static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
 141 {
 142         struct idxd_wq *wq = to_idxd_wq(chan);
 143         struct device *dev = &wq->idxd->pdev->dev;
 144
 145         idxd_wq_get(wq);
 146         dev_dbg(dev, "%s: client_count: %d\n", __func__,
 147                 idxd_wq_refcount(wq));
 148         return 0;
 149 }
 150
 151 static void idxd_dma_free_chan_resources(struct dma_chan *chan)
 152 {
 153         struct idxd_wq *wq = to_idxd_wq(chan);
 154         struct device *dev = &wq->idxd->pdev->dev;
 155
 156         idxd_wq_put(wq);
 157         dev_dbg(dev, "%s: client_count: %d\n", __func__,
 158                 idxd_wq_refcount(wq));
 159 }
 160
 161 static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
 162                                           dma_cookie_t cookie,
 163                                           struct dma_tx_state *txstate)
 164 {
 165         return DMA_OUT_OF_ORDER;
 166 }
 167
 168 /*
 169  * issue_pending() does not need to do anything since tx_submit() does the job
 170  * already.
 171  */
 172 static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
 173 {
 174 }
 175
 176 static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 177 {
 178         struct dma_chan *c = tx->chan;
 179         struct idxd_wq *wq = to_idxd_wq(c);
 180         dma_cookie_t cookie;
 181         int rc;
 182         struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
 183
 184         cookie = dma_cookie_assign(tx);
 185
 186         rc = idxd_submit_desc(wq, desc);
 187         if (rc < 0) {
 188                 idxd_free_desc(wq, desc);
 189                 return rc;
 190         }
 191
 192         return cookie;
 193 }
 194
 195 static void idxd_dma_release(struct dma_device *device)
 196 {
 197         struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma);
 198
 199         kfree(idxd_dma);
 200 }
 201
 202 int idxd_register_dma_device(struct idxd_device *idxd)
 203 {
 204         struct idxd_dma_dev *idxd_dma;
 205         struct dma_device *dma;
 206         struct device *dev = &idxd->pdev->dev;
 207         int rc;
 208
 209         idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev));
 210         if (!idxd_dma)
 211                 return -ENOMEM;
 212
 213         dma = &idxd_dma->dma;
 214         INIT_LIST_HEAD(&dma->channels);
 215         dma->dev = dev;
 216
 217         dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
 218         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 219         dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
 220         dma->device_release = idxd_dma_release;
 221
 222         dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt;
 223         if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
 224                 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
 225                 dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
 226         }
 227
 228         dma->device_tx_status = idxd_dma_tx_status;
 229         dma->device_issue_pending = idxd_dma_issue_pending;
 230         dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
 231         dma->device_free_chan_resources = idxd_dma_free_chan_resources;
 232
 233         rc = dma_async_device_register(dma);
 234         if (rc < 0) {
 235                 kfree(idxd_dma);
 236                 return rc;
 237         }
 238
 239         idxd_dma->idxd = idxd;
 240         /*
 241          * This pointer is protected by the refs taken by the dma_chan. It will remain valid
 242          * as long as there are outstanding channels.
 243          */
 244         idxd->idxd_dma = idxd_dma;
 245         return 0;
 246 }
 247
 248 void idxd_unregister_dma_device(struct idxd_device *idxd)
 249 {
 250         dma_async_device_unregister(&idxd->idxd_dma->dma);
 251 }
 252
 253 static int idxd_register_dma_channel(struct idxd_wq *wq)
 254 {
 255         struct idxd_device *idxd = wq->idxd;
 256         struct dma_device *dma = &idxd->idxd_dma->dma;
 257         struct device *dev = &idxd->pdev->dev;
 258         struct idxd_dma_chan *idxd_chan;
 259         struct dma_chan *chan;
 260         int rc, i;
 261
 262         idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev));
 263         if (!idxd_chan)
 264                 return -ENOMEM;
 265
 266         chan = &idxd_chan->chan;
 267         chan->device = dma;
 268         list_add_tail(&chan->device_node, &dma->channels);
 269
 270         for (i = 0; i < wq->num_descs; i++) {
 271                 struct idxd_desc *desc = wq->descs[i];
 272
 273                 dma_async_tx_descriptor_init(&desc->txd, chan);
 274                 desc->txd.tx_submit = idxd_dma_tx_submit;
 275         }
 276
 277         rc = dma_async_device_channel_register(dma, chan);
 278         if (rc < 0) {
 279                 kfree(idxd_chan);
 280                 return rc;
 281         }
 282
 283         wq->idxd_chan = idxd_chan;
 284         idxd_chan->wq = wq;
 285         get_device(wq_confdev(wq));
 286
 287         return 0;
 288 }
 289
 290 static void idxd_unregister_dma_channel(struct idxd_wq *wq)
 291 {
 292         struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
 293         struct dma_chan *chan = &idxd_chan->chan;
 294         struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma;
 295
 296         dma_async_device_channel_unregister(&idxd_dma->dma, chan);
 297         list_del(&chan->device_node);
 298         kfree(wq->idxd_chan);
 299         wq->idxd_chan = NULL;
 300         put_device(wq_confdev(wq));
 301 }
 302
 303 static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
 304 {
 305         struct device *dev = &idxd_dev->conf_dev;
 306         struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
 307         struct idxd_device *idxd = wq->idxd;
 308         int rc;
 309
 310         if (idxd->state != IDXD_DEV_ENABLED)
 311                 return -ENXIO;
 312
 313         mutex_lock(&wq->wq_lock);
 314         wq->type = IDXD_WQT_KERNEL;
 315
 316         rc = drv_enable_wq(wq);
 317         if (rc < 0) {
 318                 dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
 319                 rc = -ENXIO;
 320                 goto err;
 321         }
 322
 323         rc = idxd_register_dma_channel(wq);
 324         if (rc < 0) {
 325                 idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
 326                 dev_dbg(dev, "Failed to register dma channel\n");
 327                 goto err_dma;
 328         }
 329
 330         idxd->cmd_status = 0;
 331         mutex_unlock(&wq->wq_lock);
 332         return 0;
 333
 334 err_dma:
 335         drv_disable_wq(wq);
 336 err:
 337         wq->type = IDXD_WQT_NONE;
 338         mutex_unlock(&wq->wq_lock);
 339         return rc;
 340 }
 341
 342 static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
 343 {
 344         struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
 345
 346         mutex_lock(&wq->wq_lock);
 347         __idxd_wq_quiesce(wq);
 348         idxd_unregister_dma_channel(wq);
 349         drv_disable_wq(wq);
 350         mutex_unlock(&wq->wq_lock);
 351 }
 352
 353 static enum idxd_dev_type dev_types[] = {
 354         IDXD_DEV_WQ,
 355         IDXD_DEV_NONE,
 356 };
 357
 358 struct idxd_device_driver idxd_dmaengine_drv = {
 359         .probe = idxd_dmaengine_drv_probe,
 360         .remove = idxd_dmaengine_drv_remove,
 361         .name = "dmaengine",
 362         .type = dev_types,
 363 };
 364 EXPORT_SYMBOL_GPL(idxd_dmaengine_drv);