drivers/dma/idxd/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #include <linux/init.h>
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/io-64-nonatomic-lo-hi.h>
   8 #include <linux/dmaengine.h>
   9 #include <linux/irq.h>
  10 #include <linux/msi.h>
  11 #include <uapi/linux/idxd.h>
  12 #include "../dmaengine.h"
  13 #include "idxd.h"
  14 #include "registers.h"
  15
  16 static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
  17                           u32 *status);
  18 static void idxd_device_wqs_clear_state(struct idxd_device *idxd);
  19 static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
  20
  21 /* Interrupt control bits */
  22 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
  23 {
  24         struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector);
  25
  26         pci_msi_mask_irq(data);
  27 }
  28
  29 void idxd_mask_msix_vectors(struct idxd_device *idxd)
  30 {
  31         struct pci_dev *pdev = idxd->pdev;
  32         int msixcnt = pci_msix_vec_count(pdev);
  33         int i;
  34
  35         for (i = 0; i < msixcnt; i++)
  36                 idxd_mask_msix_vector(idxd, i);
  37 }
  38
  39 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
  40 {
  41         struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector);
  42
  43         pci_msi_unmask_irq(data);
  44 }
  45
  46 void idxd_unmask_error_interrupts(struct idxd_device *idxd)
  47 {
  48         union genctrl_reg genctrl;
  49
  50         genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
  51         genctrl.softerr_int_en = 1;
  52         genctrl.halt_int_en = 1;
  53         iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
  54 }
  55
  56 void idxd_mask_error_interrupts(struct idxd_device *idxd)
  57 {
  58         union genctrl_reg genctrl;
  59
  60         genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET);
  61         genctrl.softerr_int_en = 0;
  62         genctrl.halt_int_en = 0;
  63         iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET);
  64 }
  65
  66 static void free_hw_descs(struct idxd_wq *wq)
  67 {
  68         int i;
  69
  70         for (i = 0; i < wq->num_descs; i++)
  71                 kfree(wq->hw_descs[i]);
  72
  73         kfree(wq->hw_descs);
  74 }
  75
  76 static int alloc_hw_descs(struct idxd_wq *wq, int num)
  77 {
  78         struct device *dev = &wq->idxd->pdev->dev;
  79         int i;
  80         int node = dev_to_node(dev);
  81
  82         wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
  83                                     GFP_KERNEL, node);
  84         if (!wq->hw_descs)
  85                 return -ENOMEM;
  86
  87         for (i = 0; i < num; i++) {
  88                 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
  89                                                GFP_KERNEL, node);
  90                 if (!wq->hw_descs[i]) {
  91                         free_hw_descs(wq);
  92                         return -ENOMEM;
  93                 }
  94         }
  95
  96         return 0;
  97 }
  98
  99 static void free_descs(struct idxd_wq *wq)
 100 {
 101         int i;
 102
 103         for (i = 0; i < wq->num_descs; i++)
 104                 kfree(wq->descs[i]);
 105
 106         kfree(wq->descs);
 107 }
 108
 109 static int alloc_descs(struct idxd_wq *wq, int num)
 110 {
 111         struct device *dev = &wq->idxd->pdev->dev;
 112         int i;
 113         int node = dev_to_node(dev);
 114
 115         wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
 116                                  GFP_KERNEL, node);
 117         if (!wq->descs)
 118                 return -ENOMEM;
 119
 120         for (i = 0; i < num; i++) {
 121                 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
 122                                             GFP_KERNEL, node);
 123                 if (!wq->descs[i]) {
 124                         free_descs(wq);
 125                         return -ENOMEM;
 126                 }
 127         }
 128
 129         return 0;
 130 }
 131
 132 /* WQ control bits */
 133 int idxd_wq_alloc_resources(struct idxd_wq *wq)
 134 {
 135         struct idxd_device *idxd = wq->idxd;
 136         struct device *dev = &idxd->pdev->dev;
 137         int rc, num_descs, i;
 138         int align;
 139         u64 tmp;
 140
 141         if (wq->type != IDXD_WQT_KERNEL)
 142                 return 0;
 143
 144         num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;
 145         wq->num_descs = num_descs;
 146
 147         rc = alloc_hw_descs(wq, num_descs);
 148         if (rc < 0)
 149                 return rc;
 150
 151         align = idxd->data->align;
 152         wq->compls_size = num_descs * idxd->data->compl_size + align;
 153         wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size,
 154                                             &wq->compls_addr_raw, GFP_KERNEL);
 155         if (!wq->compls_raw) {
 156                 rc = -ENOMEM;
 157                 goto fail_alloc_compls;
 158         }
 159
 160         /* Adjust alignment */
 161         wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1);
 162         tmp = (u64)wq->compls_raw;
 163         tmp = (tmp + (align - 1)) & ~(align - 1);
 164         wq->compls = (struct dsa_completion_record *)tmp;
 165
 166         rc = alloc_descs(wq, num_descs);
 167         if (rc < 0)
 168                 goto fail_alloc_descs;
 169
 170         rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
 171                                      dev_to_node(dev));
 172         if (rc < 0)
 173                 goto fail_sbitmap_init;
 174
 175         for (i = 0; i < num_descs; i++) {
 176                 struct idxd_desc *desc = wq->descs[i];
 177
 178                 desc->hw = wq->hw_descs[i];
 179                 if (idxd->data->type == IDXD_TYPE_DSA)
 180                         desc->completion = &wq->compls[i];
 181                 else if (idxd->data->type == IDXD_TYPE_IAX)
 182                         desc->iax_completion = &wq->iax_compls[i];
 183                 desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;
 184                 desc->id = i;
 185                 desc->wq = wq;
 186                 desc->cpu = -1;
 187         }
 188
 189         return 0;
 190
 191  fail_sbitmap_init:
 192         free_descs(wq);
 193  fail_alloc_descs:
 194         dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
 195                           wq->compls_addr_raw);
 196  fail_alloc_compls:
 197         free_hw_descs(wq);
 198         return rc;
 199 }
 200
 201 void idxd_wq_free_resources(struct idxd_wq *wq)
 202 {
 203         struct device *dev = &wq->idxd->pdev->dev;
 204
 205         if (wq->type != IDXD_WQT_KERNEL)
 206                 return;
 207
 208         free_hw_descs(wq);
 209         free_descs(wq);
 210         dma_free_coherent(dev, wq->compls_size, wq->compls_raw,
 211                           wq->compls_addr_raw);
 212         sbitmap_queue_free(&wq->sbq);
 213 }
 214
 215 int idxd_wq_enable(struct idxd_wq *wq)
 216 {
 217         struct idxd_device *idxd = wq->idxd;
 218         struct device *dev = &idxd->pdev->dev;
 219         u32 status;
 220
 221         if (wq->state == IDXD_WQ_ENABLED) {
 222                 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
 223                 return -ENXIO;
 224         }
 225
 226         idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
 227
 228         if (status != IDXD_CMDSTS_SUCCESS &&
 229             status != IDXD_CMDSTS_ERR_WQ_ENABLED) {
 230                 dev_dbg(dev, "WQ enable failed: %#x\n", status);
 231                 return -ENXIO;
 232         }
 233
 234         wq->state = IDXD_WQ_ENABLED;
 235         dev_dbg(dev, "WQ %d enabled\n", wq->id);
 236         return 0;
 237 }
 238
 239 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)
 240 {
 241         struct idxd_device *idxd = wq->idxd;
 242         struct device *dev = &idxd->pdev->dev;
 243         u32 status, operand;
 244
 245         dev_dbg(dev, "Disabling WQ %d\n", wq->id);
 246
 247         if (wq->state != IDXD_WQ_ENABLED) {
 248                 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
 249                 return 0;
 250         }
 251
 252         operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
 253         idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status);
 254
 255         if (status != IDXD_CMDSTS_SUCCESS) {
 256                 dev_dbg(dev, "WQ disable failed: %#x\n", status);
 257                 return -ENXIO;
 258         }
 259
 260         if (reset_config)
 261                 idxd_wq_disable_cleanup(wq);
 262         wq->state = IDXD_WQ_DISABLED;
 263         dev_dbg(dev, "WQ %d disabled\n", wq->id);
 264         return 0;
 265 }
 266
 267 void idxd_wq_drain(struct idxd_wq *wq)
 268 {
 269         struct idxd_device *idxd = wq->idxd;
 270         struct device *dev = &idxd->pdev->dev;
 271         u32 operand;
 272
 273         if (wq->state != IDXD_WQ_ENABLED) {
 274                 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
 275                 return;
 276         }
 277
 278         dev_dbg(dev, "Draining WQ %d\n", wq->id);
 279         operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
 280         idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_WQ, operand, NULL);
 281 }
 282
 283 void idxd_wq_reset(struct idxd_wq *wq)
 284 {
 285         struct idxd_device *idxd = wq->idxd;
 286         struct device *dev = &idxd->pdev->dev;
 287         u32 operand;
 288
 289         if (wq->state != IDXD_WQ_ENABLED) {
 290                 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
 291                 return;
 292         }
 293
 294         operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
 295         idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL);
 296         idxd_wq_disable_cleanup(wq);
 297         wq->state = IDXD_WQ_DISABLED;
 298 }
 299
 300 int idxd_wq_map_portal(struct idxd_wq *wq)
 301 {
 302         struct idxd_device *idxd = wq->idxd;
 303         struct pci_dev *pdev = idxd->pdev;
 304         struct device *dev = &pdev->dev;
 305         resource_size_t start;
 306
 307         start = pci_resource_start(pdev, IDXD_WQ_BAR);
 308         start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
 309
 310         wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
 311         if (!wq->portal)
 312                 return -ENOMEM;
 313
 314         return 0;
 315 }
 316
 317 void idxd_wq_unmap_portal(struct idxd_wq *wq)
 318 {
 319         struct device *dev = &wq->idxd->pdev->dev;
 320
 321         devm_iounmap(dev, wq->portal);
 322         wq->portal = NULL;
 323         wq->portal_offset = 0;
 324 }
 325
 326 void idxd_wqs_unmap_portal(struct idxd_device *idxd)
 327 {
 328         int i;
 329
 330         for (i = 0; i < idxd->max_wqs; i++) {
 331                 struct idxd_wq *wq = idxd->wqs[i];
 332
 333                 if (wq->portal)
 334                         idxd_wq_unmap_portal(wq);
 335         }
 336 }
 337
 338 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
 339 {
 340         struct idxd_device *idxd = wq->idxd;
 341         int rc;
 342         union wqcfg wqcfg;
 343         unsigned int offset;
 344         unsigned long flags;
 345
 346         rc = idxd_wq_disable(wq, false);
 347         if (rc < 0)
 348                 return rc;
 349
 350         offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
 351         spin_lock_irqsave(&idxd->dev_lock, flags);
 352         wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
 353         wqcfg.pasid_en = 1;
 354         wqcfg.pasid = pasid;
 355         iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
 356         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 357
 358         rc = idxd_wq_enable(wq);
 359         if (rc < 0)
 360                 return rc;
 361
 362         return 0;
 363 }
 364
 365 int idxd_wq_disable_pasid(struct idxd_wq *wq)
 366 {
 367         struct idxd_device *idxd = wq->idxd;
 368         int rc;
 369         union wqcfg wqcfg;
 370         unsigned int offset;
 371         unsigned long flags;
 372
 373         rc = idxd_wq_disable(wq, false);
 374         if (rc < 0)
 375                 return rc;
 376
 377         offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
 378         spin_lock_irqsave(&idxd->dev_lock, flags);
 379         wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
 380         wqcfg.pasid_en = 0;
 381         wqcfg.pasid = 0;
 382         iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
 383         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 384
 385         rc = idxd_wq_enable(wq);
 386         if (rc < 0)
 387                 return rc;
 388
 389         return 0;
 390 }
 391
 392 static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
 393 {
 394         struct idxd_device *idxd = wq->idxd;
 395
 396         lockdep_assert_held(&wq->wq_lock);
 397         memset(wq->wqcfg, 0, idxd->wqcfg_size);
 398         wq->type = IDXD_WQT_NONE;
 399         wq->size = 0;
 400         wq->group = NULL;
 401         wq->threshold = 0;
 402         wq->priority = 0;
 403         wq->ats_dis = 0;
 404         clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
 405         clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
 406         memset(wq->name, 0, WQ_NAME_SIZE);
 407 }
 408
 409 static void idxd_wq_ref_release(struct percpu_ref *ref)
 410 {
 411         struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);
 412
 413         complete(&wq->wq_dead);
 414 }
 415
 416 int idxd_wq_init_percpu_ref(struct idxd_wq *wq)
 417 {
 418         int rc;
 419
 420         memset(&wq->wq_active, 0, sizeof(wq->wq_active));
 421         rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, 0, GFP_KERNEL);
 422         if (rc < 0)
 423                 return rc;
 424         reinit_completion(&wq->wq_dead);
 425         return 0;
 426 }
 427
 428 void idxd_wq_quiesce(struct idxd_wq *wq)
 429 {
 430         percpu_ref_kill(&wq->wq_active);
 431         wait_for_completion(&wq->wq_dead);
 432         percpu_ref_exit(&wq->wq_active);
 433 }
 434
 435 /* Device control bits */
 436 static inline bool idxd_is_enabled(struct idxd_device *idxd)
 437 {
 438         union gensts_reg gensts;
 439
 440         gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
 441
 442         if (gensts.state == IDXD_DEVICE_STATE_ENABLED)
 443                 return true;
 444         return false;
 445 }
 446
 447 static inline bool idxd_device_is_halted(struct idxd_device *idxd)
 448 {
 449         union gensts_reg gensts;
 450
 451         gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
 452
 453         return (gensts.state == IDXD_DEVICE_STATE_HALT);
 454 }
 455
 456 /*
 457  * This is function is only used for reset during probe and will
 458  * poll for completion. Once the device is setup with interrupts,
 459  * all commands will be done via interrupt completion.
 460  */
 461 int idxd_device_init_reset(struct idxd_device *idxd)
 462 {
 463         struct device *dev = &idxd->pdev->dev;
 464         union idxd_command_reg cmd;
 465         unsigned long flags;
 466
 467         if (idxd_device_is_halted(idxd)) {
 468                 dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
 469                 return -ENXIO;
 470         }
 471
 472         memset(&cmd, 0, sizeof(cmd));
 473         cmd.cmd = IDXD_CMD_RESET_DEVICE;
 474         dev_dbg(dev, "%s: sending reset for init.\n", __func__);
 475         spin_lock_irqsave(&idxd->cmd_lock, flags);
 476         iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
 477
 478         while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
 479                IDXD_CMDSTS_ACTIVE)
 480                 cpu_relax();
 481         spin_unlock_irqrestore(&idxd->cmd_lock, flags);
 482         return 0;
 483 }
 484
 485 static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
 486                           u32 *status)
 487 {
 488         union idxd_command_reg cmd;
 489         DECLARE_COMPLETION_ONSTACK(done);
 490         unsigned long flags;
 491         u32 stat;
 492
 493         if (idxd_device_is_halted(idxd)) {
 494                 dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
 495                 if (status)
 496                         *status = IDXD_CMDSTS_HW_ERR;
 497                 return;
 498         }
 499
 500         memset(&cmd, 0, sizeof(cmd));
 501         cmd.cmd = cmd_code;
 502         cmd.operand = operand;
 503         cmd.int_req = 1;
 504
 505         spin_lock_irqsave(&idxd->cmd_lock, flags);
 506         wait_event_lock_irq(idxd->cmd_waitq,
 507                             !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
 508                             idxd->cmd_lock);
 509
 510         dev_dbg(&idxd->pdev->dev, "%s: sending cmd: %#x op: %#x\n",
 511                 __func__, cmd_code, operand);
 512
 513         idxd->cmd_status = 0;
 514         __set_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
 515         idxd->cmd_done = &done;
 516         iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
 517
 518         /*
 519          * After command submitted, release lock and go to sleep until
 520          * the command completes via interrupt.
 521          */
 522         spin_unlock_irqrestore(&idxd->cmd_lock, flags);
 523         wait_for_completion(&done);
 524         stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
 525         spin_lock_irqsave(&idxd->cmd_lock, flags);
 526         if (status)
 527                 *status = stat;
 528         idxd->cmd_status = stat & GENMASK(7, 0);
 529
 530         __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
 531         /* Wake up other pending commands */
 532         wake_up(&idxd->cmd_waitq);
 533         spin_unlock_irqrestore(&idxd->cmd_lock, flags);
 534 }
 535
 536 int idxd_device_enable(struct idxd_device *idxd)
 537 {
 538         struct device *dev = &idxd->pdev->dev;
 539         u32 status;
 540
 541         if (idxd_is_enabled(idxd)) {
 542                 dev_dbg(dev, "Device already enabled\n");
 543                 return -ENXIO;
 544         }
 545
 546         idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_DEVICE, 0, &status);
 547
 548         /* If the command is successful or if the device was enabled */
 549         if (status != IDXD_CMDSTS_SUCCESS &&
 550             status != IDXD_CMDSTS_ERR_DEV_ENABLED) {
 551                 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
 552                 return -ENXIO;
 553         }
 554
 555         idxd->state = IDXD_DEV_ENABLED;
 556         return 0;
 557 }
 558
 559 int idxd_device_disable(struct idxd_device *idxd)
 560 {
 561         struct device *dev = &idxd->pdev->dev;
 562         u32 status;
 563         unsigned long flags;
 564
 565         if (!idxd_is_enabled(idxd)) {
 566                 dev_dbg(dev, "Device is not enabled\n");
 567                 return 0;
 568         }
 569
 570         idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_DEVICE, 0, &status);
 571
 572         /* If the command is successful or if the device was disabled */
 573         if (status != IDXD_CMDSTS_SUCCESS &&
 574             !(status & IDXD_CMDSTS_ERR_DIS_DEV_EN)) {
 575                 dev_dbg(dev, "%s: err_code: %#x\n", __func__, status);
 576                 return -ENXIO;
 577         }
 578
 579         spin_lock_irqsave(&idxd->dev_lock, flags);
 580         idxd_device_clear_state(idxd);
 581         idxd->state = IDXD_DEV_DISABLED;
 582         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 583         return 0;
 584 }
 585
 586 void idxd_device_reset(struct idxd_device *idxd)
 587 {
 588         unsigned long flags;
 589
 590         idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
 591         spin_lock_irqsave(&idxd->dev_lock, flags);
 592         idxd_device_clear_state(idxd);
 593         idxd->state = IDXD_DEV_DISABLED;
 594         spin_unlock_irqrestore(&idxd->dev_lock, flags);
 595 }
 596
 597 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
 598 {
 599         struct device *dev = &idxd->pdev->dev;
 600         u32 operand;
 601
 602         operand = pasid;
 603         dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_DRAIN_PASID, operand);
 604         idxd_cmd_exec(idxd, IDXD_CMD_DRAIN_PASID, operand, NULL);
 605         dev_dbg(dev, "pasid %d drained\n", pasid);
 606 }
 607
 608 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
 609                                    enum idxd_interrupt_type irq_type)
 610 {
 611         struct device *dev = &idxd->pdev->dev;
 612         u32 operand, status;
 613
 614         if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)))
 615                 return -EOPNOTSUPP;
 616
 617         dev_dbg(dev, "get int handle, idx %d\n", idx);
 618
 619         operand = idx & GENMASK(15, 0);
 620         if (irq_type == IDXD_IRQ_IMS)
 621                 operand |= CMD_INT_HANDLE_IMS;
 622
 623         dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_REQUEST_INT_HANDLE, operand);
 624
 625         idxd_cmd_exec(idxd, IDXD_CMD_REQUEST_INT_HANDLE, operand, &status);
 626
 627         if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
 628                 dev_dbg(dev, "request int handle failed: %#x\n", status);
 629                 return -ENXIO;
 630         }
 631
 632         *handle = (status >> IDXD_CMDSTS_RES_SHIFT) & GENMASK(15, 0);
 633
 634         dev_dbg(dev, "int handle acquired: %u\n", *handle);
 635         return 0;
 636 }
 637
 638 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
 639                                    enum idxd_interrupt_type irq_type)
 640 {
 641         struct device *dev = &idxd->pdev->dev;
 642         u32 operand, status;
 643         union idxd_command_reg cmd;
 644         unsigned long flags;
 645
 646         if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)))
 647                 return -EOPNOTSUPP;
 648
 649         dev_dbg(dev, "release int handle, handle %d\n", handle);
 650
 651         memset(&cmd, 0, sizeof(cmd));
 652         operand = handle & GENMASK(15, 0);
 653
 654         if (irq_type == IDXD_IRQ_IMS)
 655                 operand |= CMD_INT_HANDLE_IMS;
 656
 657         cmd.cmd = IDXD_CMD_RELEASE_INT_HANDLE;
 658         cmd.operand = operand;
 659
 660         dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand);
 661
 662         spin_lock_irqsave(&idxd->cmd_lock, flags);
 663         iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
 664
 665         while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE)
 666                 cpu_relax();
 667         status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
 668         spin_unlock_irqrestore(&idxd->cmd_lock, flags);
 669
 670         if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
 671                 dev_dbg(dev, "release int handle failed: %#x\n", status);
 672                 return -ENXIO;
 673         }
 674
 675         dev_dbg(dev, "int handle released.\n");
 676         return 0;
 677 }
 678
 679 /* Device configuration bits */
 680 static void idxd_engines_clear_state(struct idxd_device *idxd)
 681 {
 682         struct idxd_engine *engine;
 683         int i;
 684
 685         lockdep_assert_held(&idxd->dev_lock);
 686         for (i = 0; i < idxd->max_engines; i++) {
 687                 engine = idxd->engines[i];
 688                 engine->group = NULL;
 689         }
 690 }
 691
 692 static void idxd_groups_clear_state(struct idxd_device *idxd)
 693 {
 694         struct idxd_group *group;
 695         int i;
 696
 697         lockdep_assert_held(&idxd->dev_lock);
 698         for (i = 0; i < idxd->max_groups; i++) {
 699                 group = idxd->groups[i];
 700                 memset(&group->grpcfg, 0, sizeof(group->grpcfg));
 701                 group->num_engines = 0;
 702                 group->num_wqs = 0;
 703                 group->use_token_limit = false;
 704                 group->tokens_allowed = 0;
 705                 group->tokens_reserved = 0;
 706                 group->tc_a = -1;
 707                 group->tc_b = -1;
 708         }
 709 }
 710
 711 static void idxd_device_wqs_clear_state(struct idxd_device *idxd)
 712 {
 713         int i;
 714
 715         lockdep_assert_held(&idxd->dev_lock);
 716         for (i = 0; i < idxd->max_wqs; i++) {
 717                 struct idxd_wq *wq = idxd->wqs[i];
 718
 719                 if (wq->state == IDXD_WQ_ENABLED) {
 720                         idxd_wq_disable_cleanup(wq);
 721                         wq->state = IDXD_WQ_DISABLED;
 722                 }
 723         }
 724 }
 725
 726 void idxd_device_clear_state(struct idxd_device *idxd)
 727 {
 728         idxd_groups_clear_state(idxd);
 729         idxd_engines_clear_state(idxd);
 730         idxd_device_wqs_clear_state(idxd);
 731 }
 732
 733 void idxd_msix_perm_setup(struct idxd_device *idxd)
 734 {
 735         union msix_perm mperm;
 736         int i, msixcnt;
 737
 738         msixcnt = pci_msix_vec_count(idxd->pdev);
 739         if (msixcnt < 0)
 740                 return;
 741
 742         mperm.bits = 0;
 743         mperm.pasid = idxd->pasid;
 744         mperm.pasid_en = device_pasid_enabled(idxd);
 745         for (i = 1; i < msixcnt; i++)
 746                 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8);
 747 }
 748
 749 void idxd_msix_perm_clear(struct idxd_device *idxd)
 750 {
 751         union msix_perm mperm;
 752         int i, msixcnt;
 753
 754         msixcnt = pci_msix_vec_count(idxd->pdev);
 755         if (msixcnt < 0)
 756                 return;
 757
 758         mperm.bits = 0;
 759         for (i = 1; i < msixcnt; i++)
 760                 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8);
 761 }
 762
 763 static void idxd_group_config_write(struct idxd_group *group)
 764 {
 765         struct idxd_device *idxd = group->idxd;
 766         struct device *dev = &idxd->pdev->dev;
 767         int i;
 768         u32 grpcfg_offset;
 769
 770         dev_dbg(dev, "Writing group %d cfg registers\n", group->id);
 771
 772         /* setup GRPWQCFG */
 773         for (i = 0; i < GRPWQCFG_STRIDES; i++) {
 774                 grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
 775                 iowrite64(group->grpcfg.wqs[i], idxd->reg_base + grpcfg_offset);
 776                 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
 777                         group->id, i, grpcfg_offset,
 778                         ioread64(idxd->reg_base + grpcfg_offset));
 779         }
 780
 781         /* setup GRPENGCFG */
 782         grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
 783         iowrite64(group->grpcfg.engines, idxd->reg_base + grpcfg_offset);
 784         dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
 785                 grpcfg_offset, ioread64(idxd->reg_base + grpcfg_offset));
 786
 787         /* setup GRPFLAGS */
 788         grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
 789         iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset);
 790         dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
 791                 group->id, grpcfg_offset,
 792                 ioread32(idxd->reg_base + grpcfg_offset));
 793 }
 794
 795 static int idxd_groups_config_write(struct idxd_device *idxd)
 796
 797 {
 798         union gencfg_reg reg;
 799         int i;
 800         struct device *dev = &idxd->pdev->dev;
 801
 802         /* Setup bandwidth token limit */
 803         if (idxd->token_limit) {
 804                 reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
 805                 reg.token_limit = idxd->token_limit;
 806                 iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
 807         }
 808
 809         dev_dbg(dev, "GENCFG(%#x): %#x\n", IDXD_GENCFG_OFFSET,
 810                 ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET));
 811
 812         for (i = 0; i < idxd->max_groups; i++) {
 813                 struct idxd_group *group = idxd->groups[i];
 814
 815                 idxd_group_config_write(group);
 816         }
 817
 818         return 0;
 819 }
 820
 821 static int idxd_wq_config_write(struct idxd_wq *wq)
 822 {
 823         struct idxd_device *idxd = wq->idxd;
 824         struct device *dev = &idxd->pdev->dev;
 825         u32 wq_offset;
 826         int i;
 827
 828         if (!wq->group)
 829                 return 0;
 830
 831         /*
 832          * Instead of memset the entire shadow copy of WQCFG, copy from the hardware after
 833          * wq reset. This will copy back the sticky values that are present on some devices.
 834          */
 835         for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
 836                 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
 837                 wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset);
 838         }
 839
 840         /* byte 0-3 */
 841         wq->wqcfg->wq_size = wq->size;
 842
 843         if (wq->size == 0) {
 844                 idxd->cmd_status = IDXD_SCMD_WQ_NO_SIZE;
 845                 dev_warn(dev, "Incorrect work queue size: 0\n");
 846                 return -EINVAL;
 847         }
 848
 849         /* bytes 4-7 */
 850         wq->wqcfg->wq_thresh = wq->threshold;
 851
 852         /* byte 8-11 */
 853         wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
 854         if (wq_dedicated(wq))
 855                 wq->wqcfg->mode = 1;
 856
 857         if (device_pasid_enabled(idxd)) {
 858                 wq->wqcfg->pasid_en = 1;
 859                 if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq))
 860                         wq->wqcfg->pasid = idxd->pasid;
 861         }
 862
 863         wq->wqcfg->priority = wq->priority;
 864
 865         if (idxd->hw.gen_cap.block_on_fault &&
 866             test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags))
 867                 wq->wqcfg->bof = 1;
 868
 869         if (idxd->hw.wq_cap.wq_ats_support)
 870                 wq->wqcfg->wq_ats_disable = wq->ats_dis;
 871
 872         /* bytes 12-15 */
 873         wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
 874         wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size);
 875
 876         dev_dbg(dev, "WQ %d CFGs\n", wq->id);
 877         for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
 878                 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
 879                 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);
 880                 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n",
 881                         wq->id, i, wq_offset,
 882                         ioread32(idxd->reg_base + wq_offset));
 883         }
 884
 885         return 0;
 886 }
 887
 888 static int idxd_wqs_config_write(struct idxd_device *idxd)
 889 {
 890         int i, rc;
 891
 892         for (i = 0; i < idxd->max_wqs; i++) {
 893                 struct idxd_wq *wq = idxd->wqs[i];
 894
 895                 rc = idxd_wq_config_write(wq);
 896                 if (rc < 0)
 897                         return rc;
 898         }
 899
 900         return 0;
 901 }
 902
 903 static void idxd_group_flags_setup(struct idxd_device *idxd)
 904 {
 905         int i;
 906
 907         /* TC-A 0 and TC-B 1 should be defaults */
 908         for (i = 0; i < idxd->max_groups; i++) {
 909                 struct idxd_group *group = idxd->groups[i];
 910
 911                 if (group->tc_a == -1)
 912                         group->tc_a = group->grpcfg.flags.tc_a = 0;
 913                 else
 914                         group->grpcfg.flags.tc_a = group->tc_a;
 915                 if (group->tc_b == -1)
 916                         group->tc_b = group->grpcfg.flags.tc_b = 1;
 917                 else
 918                         group->grpcfg.flags.tc_b = group->tc_b;
 919                 group->grpcfg.flags.use_token_limit = group->use_token_limit;
 920                 group->grpcfg.flags.tokens_reserved = group->tokens_reserved;
 921                 if (group->tokens_allowed)
 922                         group->grpcfg.flags.tokens_allowed =
 923                                 group->tokens_allowed;
 924                 else
 925                         group->grpcfg.flags.tokens_allowed = idxd->max_tokens;
 926         }
 927 }
 928
 929 static int idxd_engines_setup(struct idxd_device *idxd)
 930 {
 931         int i, engines = 0;
 932         struct idxd_engine *eng;
 933         struct idxd_group *group;
 934
 935         for (i = 0; i < idxd->max_groups; i++) {
 936                 group = idxd->groups[i];
 937                 group->grpcfg.engines = 0;
 938         }
 939
 940         for (i = 0; i < idxd->max_engines; i++) {
 941                 eng = idxd->engines[i];
 942                 group = eng->group;
 943
 944                 if (!group)
 945                         continue;
 946
 947                 group->grpcfg.engines |= BIT(eng->id);
 948                 engines++;
 949         }
 950
 951         if (!engines)
 952                 return -EINVAL;
 953
 954         return 0;
 955 }
 956
 957 static int idxd_wqs_setup(struct idxd_device *idxd)
 958 {
 959         struct idxd_wq *wq;
 960         struct idxd_group *group;
 961         int i, j, configured = 0;
 962         struct device *dev = &idxd->pdev->dev;
 963
 964         for (i = 0; i < idxd->max_groups; i++) {
 965                 group = idxd->groups[i];
 966                 for (j = 0; j < 4; j++)
 967                         group->grpcfg.wqs[j] = 0;
 968         }
 969
 970         for (i = 0; i < idxd->max_wqs; i++) {
 971                 wq = idxd->wqs[i];
 972                 group = wq->group;
 973
 974                 if (!wq->group)
 975                         continue;
 976                 if (!wq->size)
 977                         continue;
 978
 979                 if (wq_shared(wq) && !device_swq_supported(idxd)) {
 980                         idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT;
 981                         dev_warn(dev, "No shared wq support but configured.\n");
 982                         return -EINVAL;
 983                 }
 984
 985                 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
 986                 configured++;
 987         }
 988
 989         if (configured == 0) {
 990                 idxd->cmd_status = IDXD_SCMD_WQ_NONE_CONFIGURED;
 991                 return -EINVAL;
 992         }
 993
 994         return 0;
 995 }
 996
 997 int idxd_device_config(struct idxd_device *idxd)
 998 {
 999         int rc;
1000
1001         lockdep_assert_held(&idxd->dev_lock);
1002         rc = idxd_wqs_setup(idxd);
1003         if (rc < 0)
1004                 return rc;
1005
1006         rc = idxd_engines_setup(idxd);
1007         if (rc < 0)
1008                 return rc;
1009
1010         idxd_group_flags_setup(idxd);
1011
1012         rc = idxd_wqs_config_write(idxd);
1013         if (rc < 0)
1014                 return rc;
1015
1016         rc = idxd_groups_config_write(idxd);
1017         if (rc < 0)
1018                 return rc;
1019
1020         return 0;
1021 }
1022
1023 static int idxd_wq_load_config(struct idxd_wq *wq)
1024 {
1025         struct idxd_device *idxd = wq->idxd;
1026         struct device *dev = &idxd->pdev->dev;
1027         int wqcfg_offset;
1028         int i;
1029
1030         wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);
1031         memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);
1032
1033         wq->size = wq->wqcfg->wq_size;
1034         wq->threshold = wq->wqcfg->wq_thresh;
1035         if (wq->wqcfg->priv)
1036                 wq->type = IDXD_WQT_KERNEL;
1037
1038         /* The driver does not support shared WQ mode in read-only config yet */
1039         if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)
1040                 return -EOPNOTSUPP;
1041
1042         set_bit(WQ_FLAG_DEDICATED, &wq->flags);
1043
1044         wq->priority = wq->wqcfg->priority;
1045
1046         for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
1047                 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
1048                 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
1049         }
1050
1051         return 0;
1052 }
1053
1054 static void idxd_group_load_config(struct idxd_group *group)
1055 {
1056         struct idxd_device *idxd = group->idxd;
1057         struct device *dev = &idxd->pdev->dev;
1058         int i, j, grpcfg_offset;
1059
1060         /*
1061          * Load WQS bit fields
1062          * Iterate through all 256 bits 64 bits at a time
1063          */
1064         for (i = 0; i < GRPWQCFG_STRIDES; i++) {
1065                 struct idxd_wq *wq;
1066
1067                 grpcfg_offset = GRPWQCFG_OFFSET(idxd, group->id, i);
1068                 group->grpcfg.wqs[i] = ioread64(idxd->reg_base + grpcfg_offset);
1069                 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
1070                         group->id, i, grpcfg_offset, group->grpcfg.wqs[i]);
1071
1072                 if (i * 64 >= idxd->max_wqs)
1073                         break;
1074
1075                 /* Iterate through all 64 bits and check for wq set */
1076                 for (j = 0; j < 64; j++) {
1077                         int id = i * 64 + j;
1078
1079                         /* No need to check beyond max wqs */
1080                         if (id >= idxd->max_wqs)
1081                                 break;
1082
1083                         /* Set group assignment for wq if wq bit is set */
1084                         if (group->grpcfg.wqs[i] & BIT(j)) {
1085                                 wq = idxd->wqs[id];
1086                                 wq->group = group;
1087                         }
1088                 }
1089         }
1090
1091         grpcfg_offset = GRPENGCFG_OFFSET(idxd, group->id);
1092         group->grpcfg.engines = ioread64(idxd->reg_base + grpcfg_offset);
1093         dev_dbg(dev, "GRPCFG engs[%d: %#x]: %#llx\n", group->id,
1094                 grpcfg_offset, group->grpcfg.engines);
1095
1096         /* Iterate through all 64 bits to check engines set */
1097         for (i = 0; i < 64; i++) {
1098                 if (i >= idxd->max_engines)
1099                         break;
1100
1101                 if (group->grpcfg.engines & BIT(i)) {
1102                         struct idxd_engine *engine = idxd->engines[i];
1103
1104                         engine->group = group;
1105                 }
1106         }
1107
1108         grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id);
1109         group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset);
1110         dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n",
1111                 group->id, grpcfg_offset, group->grpcfg.flags.bits);
1112 }
1113
1114 int idxd_device_load_config(struct idxd_device *idxd)
1115 {
1116         union gencfg_reg reg;
1117         int i, rc;
1118
1119         reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
1120         idxd->token_limit = reg.token_limit;
1121
1122         for (i = 0; i < idxd->max_groups; i++) {
1123                 struct idxd_group *group = idxd->groups[i];
1124
1125                 idxd_group_load_config(group);
1126         }
1127
1128         for (i = 0; i < idxd->max_wqs; i++) {
1129                 struct idxd_wq *wq = idxd->wqs[i];
1130
1131                 rc = idxd_wq_load_config(wq);
1132                 if (rc < 0)
1133                         return rc;
1134         }
1135
1136         return 0;
1137 }
1138
1139 int __drv_enable_wq(struct idxd_wq *wq)
1140 {
1141         struct idxd_device *idxd = wq->idxd;
1142         struct device *dev = &idxd->pdev->dev;
1143         unsigned long flags;
1144         int rc = -ENXIO;
1145
1146         lockdep_assert_held(&wq->wq_lock);
1147
1148         if (idxd->state != IDXD_DEV_ENABLED) {
1149                 idxd->cmd_status = IDXD_SCMD_DEV_NOT_ENABLED;
1150                 goto err;
1151         }
1152
1153         if (wq->state != IDXD_WQ_DISABLED) {
1154                 dev_dbg(dev, "wq %d already enabled.\n", wq->id);
1155                 idxd->cmd_status = IDXD_SCMD_WQ_ENABLED;
1156                 rc = -EBUSY;
1157                 goto err;
1158         }
1159
1160         if (!wq->group) {
1161                 dev_dbg(dev, "wq %d not attached to group.\n", wq->id);
1162                 idxd->cmd_status = IDXD_SCMD_WQ_NO_GRP;
1163                 goto err;
1164         }
1165
1166         if (strlen(wq->name) == 0) {
1167                 idxd->cmd_status = IDXD_SCMD_WQ_NO_NAME;
1168                 dev_dbg(dev, "wq %d name not set.\n", wq->id);
1169                 goto err;
1170         }
1171
1172         /* Shared WQ checks */
1173         if (wq_shared(wq)) {
1174                 if (!device_swq_supported(idxd)) {
1175                         idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM;
1176                         dev_dbg(dev, "PASID not enabled and shared wq.\n");
1177                         goto err;
1178                 }
1179                 /*
1180                  * Shared wq with the threshold set to 0 means the user
1181                  * did not set the threshold or transitioned from a
1182                  * dedicated wq but did not set threshold. A value
1183                  * of 0 would effectively disable the shared wq. The
1184                  * driver does not allow a value of 0 to be set for
1185                  * threshold via sysfs.
1186                  */
1187                 if (wq->threshold == 0) {
1188                         idxd->cmd_status = IDXD_SCMD_WQ_NO_THRESH;
1189                         dev_dbg(dev, "Shared wq and threshold 0.\n");
1190                         goto err;
1191                 }
1192         }
1193
1194         rc = 0;
1195         spin_lock_irqsave(&idxd->dev_lock, flags);
1196         if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1197                 rc = idxd_device_config(idxd);
1198         spin_unlock_irqrestore(&idxd->dev_lock, flags);
1199         if (rc < 0) {
1200                 dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);
1201                 goto err;
1202         }
1203
1204         rc = idxd_wq_enable(wq);
1205         if (rc < 0) {
1206                 dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);
1207                 goto err;
1208         }
1209
1210         rc = idxd_wq_map_portal(wq);
1211         if (rc < 0) {
1212                 idxd->cmd_status = IDXD_SCMD_WQ_PORTAL_ERR;
1213                 dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);
1214                 goto err_map_portal;
1215         }
1216
1217         wq->client_count = 0;
1218         return 0;
1219
1220 err_map_portal:
1221         rc = idxd_wq_disable(wq, false);
1222         if (rc < 0)
1223                 dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));
1224 err:
1225         return rc;
1226 }
1227
1228 int drv_enable_wq(struct idxd_wq *wq)
1229 {
1230         int rc;
1231
1232         mutex_lock(&wq->wq_lock);
1233         rc = __drv_enable_wq(wq);
1234         mutex_unlock(&wq->wq_lock);
1235         return rc;
1236 }
1237
1238 void __drv_disable_wq(struct idxd_wq *wq)
1239 {
1240         struct idxd_device *idxd = wq->idxd;
1241         struct device *dev = &idxd->pdev->dev;
1242
1243         lockdep_assert_held(&wq->wq_lock);
1244
1245         if (idxd_wq_refcount(wq))
1246                 dev_warn(dev, "Clients has claim on wq %d: %d\n",
1247                          wq->id, idxd_wq_refcount(wq));
1248
1249         idxd_wq_unmap_portal(wq);
1250
1251         idxd_wq_drain(wq);
1252         idxd_wq_reset(wq);
1253
1254         wq->client_count = 0;
1255 }
1256
1257 void drv_disable_wq(struct idxd_wq *wq)
1258 {
1259         mutex_lock(&wq->wq_lock);
1260         __drv_disable_wq(wq);
1261         mutex_unlock(&wq->wq_lock);
1262 }
1263
1264 int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
1265 {
1266         struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
1267         unsigned long flags;
1268         int rc = 0;
1269
1270         /*
1271          * Device should be in disabled state for the idxd_drv to load. If it's in
1272          * enabled state, then the device was altered outside of driver's control.
1273          * If the state is in halted state, then we don't want to proceed.
1274          */
1275         if (idxd->state != IDXD_DEV_DISABLED) {
1276                 idxd->cmd_status = IDXD_SCMD_DEV_ENABLED;
1277                 return -ENXIO;
1278         }
1279
1280         /* Device configuration */
1281         spin_lock_irqsave(&idxd->dev_lock, flags);
1282         if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1283                 rc = idxd_device_config(idxd);
1284         spin_unlock_irqrestore(&idxd->dev_lock, flags);
1285         if (rc < 0)
1286                 return -ENXIO;
1287
1288         /* Start device */
1289         rc = idxd_device_enable(idxd);
1290         if (rc < 0)
1291                 return rc;
1292
1293         /* Setup DMA device without channels */
1294         rc = idxd_register_dma_device(idxd);
1295         if (rc < 0) {
1296                 idxd_device_disable(idxd);
1297                 idxd->cmd_status = IDXD_SCMD_DEV_DMA_ERR;
1298                 return rc;
1299         }
1300
1301         idxd->cmd_status = 0;
1302         return 0;
1303 }
1304
1305 void idxd_device_drv_remove(struct idxd_dev *idxd_dev)
1306 {
1307         struct device *dev = &idxd_dev->conf_dev;
1308         struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
1309         int i;
1310
1311         for (i = 0; i < idxd->max_wqs; i++) {
1312                 struct idxd_wq *wq = idxd->wqs[i];
1313                 struct device *wq_dev = wq_confdev(wq);
1314
1315                 if (wq->state == IDXD_WQ_DISABLED)
1316                         continue;
1317                 dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev));
1318                 device_release_driver(wq_dev);
1319         }
1320
1321         idxd_unregister_dma_device(idxd);
1322         idxd_device_disable(idxd);
1323         if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1324                 idxd_device_reset(idxd);
1325 }
1326
1327 static enum idxd_dev_type dev_types[] = {
1328         IDXD_DEV_DSA,
1329         IDXD_DEV_IAX,
1330         IDXD_DEV_NONE,
1331 };
1332
1333 struct idxd_device_driver idxd_drv = {
1334         .type = dev_types,
1335         .probe = idxd_device_drv_probe,
1336         .remove = idxd_device_drv_remove,
1337         .name = "idxd",
1338 };
1339 EXPORT_SYMBOL_GPL(idxd_drv);