drivers/pci/switch/switchtec.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Microsemi Switchtec(tm) PCIe Management Driver
   4  * Copyright (c) 2017, Microsemi Corporation
   5  */
   6
   7 #include <linux/switchtec.h>
   8 #include <linux/switchtec_ioctl.h>
   9
  10 #include <linux/interrupt.h>
  11 #include <linux/module.h>
  12 #include <linux/fs.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/poll.h>
  15 #include <linux/wait.h>
  16 #include <linux/io-64-nonatomic-lo-hi.h>
  17 #include <linux/nospec.h>
  18
  19 MODULE_DESCRIPTION("Microsemi Switchtec(tm) PCIe Management Driver");
  20 MODULE_VERSION("0.1");
  21 MODULE_LICENSE("GPL");
  22 MODULE_AUTHOR("Microsemi Corporation");
  23
  24 static int max_devices = 16;
  25 module_param(max_devices, int, 0644);
  26 MODULE_PARM_DESC(max_devices, "max number of switchtec device instances");
  27
  28 static bool use_dma_mrpc = 1;
  29 module_param(use_dma_mrpc, bool, 0644);
  30 MODULE_PARM_DESC(use_dma_mrpc,
  31                  "Enable the use of the DMA MRPC feature");
  32
  33 static int nirqs = 32;
  34 module_param(nirqs, int, 0644);
  35 MODULE_PARM_DESC(nirqs, "number of interrupts to allocate (more may be useful for NTB applications)");
  36
  37 static dev_t switchtec_devt;
  38 static DEFINE_IDA(switchtec_minor_ida);
  39
  40 struct class *switchtec_class;
  41 EXPORT_SYMBOL_GPL(switchtec_class);
  42
  43 enum mrpc_state {
  44         MRPC_IDLE = 0,
  45         MRPC_QUEUED,
  46         MRPC_RUNNING,
  47         MRPC_DONE,
  48 };
  49
  50 struct switchtec_user {
  51         struct switchtec_dev *stdev;
  52
  53         enum mrpc_state state;
  54
  55         struct completion comp;
  56         struct kref kref;
  57         struct list_head list;
  58
  59         u32 cmd;
  60         u32 status;
  61         u32 return_code;
  62         size_t data_len;
  63         size_t read_len;
  64         unsigned char data[SWITCHTEC_MRPC_PAYLOAD_SIZE];
  65         int event_cnt;
  66 };
  67
  68 static struct switchtec_user *stuser_create(struct switchtec_dev *stdev)
  69 {
  70         struct switchtec_user *stuser;
  71
  72         stuser = kzalloc(sizeof(*stuser), GFP_KERNEL);
  73         if (!stuser)
  74                 return ERR_PTR(-ENOMEM);
  75
  76         get_device(&stdev->dev);
  77         stuser->stdev = stdev;
  78         kref_init(&stuser->kref);
  79         INIT_LIST_HEAD(&stuser->list);
  80         init_completion(&stuser->comp);
  81         stuser->event_cnt = atomic_read(&stdev->event_cnt);
  82
  83         dev_dbg(&stdev->dev, "%s: %p\n", __func__, stuser);
  84
  85         return stuser;
  86 }
  87
  88 static void stuser_free(struct kref *kref)
  89 {
  90         struct switchtec_user *stuser;
  91
  92         stuser = container_of(kref, struct switchtec_user, kref);
  93
  94         dev_dbg(&stuser->stdev->dev, "%s: %p\n", __func__, stuser);
  95
  96         put_device(&stuser->stdev->dev);
  97         kfree(stuser);
  98 }
  99
 100 static void stuser_put(struct switchtec_user *stuser)
 101 {
 102         kref_put(&stuser->kref, stuser_free);
 103 }
 104
 105 static void stuser_set_state(struct switchtec_user *stuser,
 106                              enum mrpc_state state)
 107 {
 108         /* requires the mrpc_mutex to already be held when called */
 109
 110         const char * const state_names[] = {
 111                 [MRPC_IDLE] = "IDLE",
 112                 [MRPC_QUEUED] = "QUEUED",
 113                 [MRPC_RUNNING] = "RUNNING",
 114                 [MRPC_DONE] = "DONE",
 115         };
 116
 117         stuser->state = state;
 118
 119         dev_dbg(&stuser->stdev->dev, "stuser state %p -> %s",
 120                 stuser, state_names[state]);
 121 }
 122
 123 static void mrpc_complete_cmd(struct switchtec_dev *stdev);
 124
 125 static void flush_wc_buf(struct switchtec_dev *stdev)
 126 {
 127         struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
 128
 129         /*
 130          * odb (outbound doorbell) register is processed by low latency
 131          * hardware and w/o side effect
 132          */
 133         mmio_dbmsg = (void __iomem *)stdev->mmio_ntb +
 134                 SWITCHTEC_NTB_REG_DBMSG_OFFSET;
 135         ioread32(&mmio_dbmsg->odb);
 136 }
 137
 138 static void mrpc_cmd_submit(struct switchtec_dev *stdev)
 139 {
 140         /* requires the mrpc_mutex to already be held when called */
 141
 142         struct switchtec_user *stuser;
 143
 144         if (stdev->mrpc_busy)
 145                 return;
 146
 147         if (list_empty(&stdev->mrpc_queue))
 148                 return;
 149
 150         stuser = list_entry(stdev->mrpc_queue.next, struct switchtec_user,
 151                             list);
 152
 153         if (stdev->dma_mrpc) {
 154                 stdev->dma_mrpc->status = SWITCHTEC_MRPC_STATUS_INPROGRESS;
 155                 memset(stdev->dma_mrpc->data, 0xFF, SWITCHTEC_MRPC_PAYLOAD_SIZE);
 156         }
 157
 158         stuser_set_state(stuser, MRPC_RUNNING);
 159         stdev->mrpc_busy = 1;
 160         memcpy_toio(&stdev->mmio_mrpc->input_data,
 161                     stuser->data, stuser->data_len);
 162         flush_wc_buf(stdev);
 163         iowrite32(stuser->cmd, &stdev->mmio_mrpc->cmd);
 164
 165         schedule_delayed_work(&stdev->mrpc_timeout,
 166                               msecs_to_jiffies(500));
 167 }
 168
 169 static int mrpc_queue_cmd(struct switchtec_user *stuser)
 170 {
 171         /* requires the mrpc_mutex to already be held when called */
 172
 173         struct switchtec_dev *stdev = stuser->stdev;
 174
 175         kref_get(&stuser->kref);
 176         stuser->read_len = sizeof(stuser->data);
 177         stuser_set_state(stuser, MRPC_QUEUED);
 178         init_completion(&stuser->comp);
 179         list_add_tail(&stuser->list, &stdev->mrpc_queue);
 180
 181         mrpc_cmd_submit(stdev);
 182
 183         return 0;
 184 }
 185
 186 static void mrpc_complete_cmd(struct switchtec_dev *stdev)
 187 {
 188         /* requires the mrpc_mutex to already be held when called */
 189         struct switchtec_user *stuser;
 190
 191         if (list_empty(&stdev->mrpc_queue))
 192                 return;
 193
 194         stuser = list_entry(stdev->mrpc_queue.next, struct switchtec_user,
 195                             list);
 196
 197         if (stdev->dma_mrpc)
 198                 stuser->status = stdev->dma_mrpc->status;
 199         else
 200                 stuser->status = ioread32(&stdev->mmio_mrpc->status);
 201
 202         if (stuser->status == SWITCHTEC_MRPC_STATUS_INPROGRESS)
 203                 return;
 204
 205         stuser_set_state(stuser, MRPC_DONE);
 206         stuser->return_code = 0;
 207
 208         if (stuser->status != SWITCHTEC_MRPC_STATUS_DONE)
 209                 goto out;
 210
 211         if (stdev->dma_mrpc)
 212                 stuser->return_code = stdev->dma_mrpc->rtn_code;
 213         else
 214                 stuser->return_code = ioread32(&stdev->mmio_mrpc->ret_value);
 215         if (stuser->return_code != 0)
 216                 goto out;
 217
 218         if (stdev->dma_mrpc)
 219                 memcpy(stuser->data, &stdev->dma_mrpc->data,
 220                               stuser->read_len);
 221         else
 222                 memcpy_fromio(stuser->data, &stdev->mmio_mrpc->output_data,
 223                               stuser->read_len);
 224 out:
 225         complete_all(&stuser->comp);
 226         list_del_init(&stuser->list);
 227         stuser_put(stuser);
 228         stdev->mrpc_busy = 0;
 229
 230         mrpc_cmd_submit(stdev);
 231 }
 232
 233 static void mrpc_event_work(struct work_struct *work)
 234 {
 235         struct switchtec_dev *stdev;
 236
 237         stdev = container_of(work, struct switchtec_dev, mrpc_work);
 238
 239         dev_dbg(&stdev->dev, "%s\n", __func__);
 240
 241         mutex_lock(&stdev->mrpc_mutex);
 242         cancel_delayed_work(&stdev->mrpc_timeout);
 243         mrpc_complete_cmd(stdev);
 244         mutex_unlock(&stdev->mrpc_mutex);
 245 }
 246
 247 static void mrpc_timeout_work(struct work_struct *work)
 248 {
 249         struct switchtec_dev *stdev;
 250         u32 status;
 251
 252         stdev = container_of(work, struct switchtec_dev, mrpc_timeout.work);
 253
 254         dev_dbg(&stdev->dev, "%s\n", __func__);
 255
 256         mutex_lock(&stdev->mrpc_mutex);
 257
 258         if (stdev->dma_mrpc)
 259                 status = stdev->dma_mrpc->status;
 260         else
 261                 status = ioread32(&stdev->mmio_mrpc->status);
 262         if (status == SWITCHTEC_MRPC_STATUS_INPROGRESS) {
 263                 schedule_delayed_work(&stdev->mrpc_timeout,
 264                                       msecs_to_jiffies(500));
 265                 goto out;
 266         }
 267
 268         mrpc_complete_cmd(stdev);
 269 out:
 270         mutex_unlock(&stdev->mrpc_mutex);
 271 }
 272
 273 static ssize_t device_version_show(struct device *dev,
 274         struct device_attribute *attr, char *buf)
 275 {
 276         struct switchtec_dev *stdev = to_stdev(dev);
 277         u32 ver;
 278
 279         ver = ioread32(&stdev->mmio_sys_info->device_version);
 280
 281         return sprintf(buf, "%x\n", ver);
 282 }
 283 static DEVICE_ATTR_RO(device_version);
 284
 285 static ssize_t fw_version_show(struct device *dev,
 286         struct device_attribute *attr, char *buf)
 287 {
 288         struct switchtec_dev *stdev = to_stdev(dev);
 289         u32 ver;
 290
 291         ver = ioread32(&stdev->mmio_sys_info->firmware_version);
 292
 293         return sprintf(buf, "%08x\n", ver);
 294 }
 295 static DEVICE_ATTR_RO(fw_version);
 296
 297 static ssize_t io_string_show(char *buf, void __iomem *attr, size_t len)
 298 {
 299         int i;
 300
 301         memcpy_fromio(buf, attr, len);
 302         buf[len] = '\n';
 303         buf[len + 1] = 0;
 304
 305         for (i = len - 1; i > 0; i--) {
 306                 if (buf[i] != ' ')
 307                         break;
 308                 buf[i] = '\n';
 309                 buf[i + 1] = 0;
 310         }
 311
 312         return strlen(buf);
 313 }
 314
 315 #define DEVICE_ATTR_SYS_INFO_STR(field) \
 316 static ssize_t field ## _show(struct device *dev, \
 317         struct device_attribute *attr, char *buf) \
 318 { \
 319         struct switchtec_dev *stdev = to_stdev(dev); \
 320         return io_string_show(buf, &stdev->mmio_sys_info->field, \
 321                             sizeof(stdev->mmio_sys_info->field)); \
 322 } \
 323 \
 324 static DEVICE_ATTR_RO(field)
 325
 326 DEVICE_ATTR_SYS_INFO_STR(vendor_id);
 327 DEVICE_ATTR_SYS_INFO_STR(product_id);
 328 DEVICE_ATTR_SYS_INFO_STR(product_revision);
 329 DEVICE_ATTR_SYS_INFO_STR(component_vendor);
 330
 331 static ssize_t component_id_show(struct device *dev,
 332         struct device_attribute *attr, char *buf)
 333 {
 334         struct switchtec_dev *stdev = to_stdev(dev);
 335         int id = ioread16(&stdev->mmio_sys_info->component_id);
 336
 337         return sprintf(buf, "PM%04X\n", id);
 338 }
 339 static DEVICE_ATTR_RO(component_id);
 340
 341 static ssize_t component_revision_show(struct device *dev,
 342         struct device_attribute *attr, char *buf)
 343 {
 344         struct switchtec_dev *stdev = to_stdev(dev);
 345         int rev = ioread8(&stdev->mmio_sys_info->component_revision);
 346
 347         return sprintf(buf, "%d\n", rev);
 348 }
 349 static DEVICE_ATTR_RO(component_revision);
 350
 351 static ssize_t partition_show(struct device *dev,
 352         struct device_attribute *attr, char *buf)
 353 {
 354         struct switchtec_dev *stdev = to_stdev(dev);
 355
 356         return sprintf(buf, "%d\n", stdev->partition);
 357 }
 358 static DEVICE_ATTR_RO(partition);
 359
 360 static ssize_t partition_count_show(struct device *dev,
 361         struct device_attribute *attr, char *buf)
 362 {
 363         struct switchtec_dev *stdev = to_stdev(dev);
 364
 365         return sprintf(buf, "%d\n", stdev->partition_count);
 366 }
 367 static DEVICE_ATTR_RO(partition_count);
 368
 369 static struct attribute *switchtec_device_attrs[] = {
 370         &dev_attr_device_version.attr,
 371         &dev_attr_fw_version.attr,
 372         &dev_attr_vendor_id.attr,
 373         &dev_attr_product_id.attr,
 374         &dev_attr_product_revision.attr,
 375         &dev_attr_component_vendor.attr,
 376         &dev_attr_component_id.attr,
 377         &dev_attr_component_revision.attr,
 378         &dev_attr_partition.attr,
 379         &dev_attr_partition_count.attr,
 380         NULL,
 381 };
 382
 383 ATTRIBUTE_GROUPS(switchtec_device);
 384
 385 static int switchtec_dev_open(struct inode *inode, struct file *filp)
 386 {
 387         struct switchtec_dev *stdev;
 388         struct switchtec_user *stuser;
 389
 390         stdev = container_of(inode->i_cdev, struct switchtec_dev, cdev);
 391
 392         stuser = stuser_create(stdev);
 393         if (IS_ERR(stuser))
 394                 return PTR_ERR(stuser);
 395
 396         filp->private_data = stuser;
 397         stream_open(inode, filp);
 398
 399         dev_dbg(&stdev->dev, "%s: %p\n", __func__, stuser);
 400
 401         return 0;
 402 }
 403
 404 static int switchtec_dev_release(struct inode *inode, struct file *filp)
 405 {
 406         struct switchtec_user *stuser = filp->private_data;
 407
 408         stuser_put(stuser);
 409
 410         return 0;
 411 }
 412
 413 static int lock_mutex_and_test_alive(struct switchtec_dev *stdev)
 414 {
 415         if (mutex_lock_interruptible(&stdev->mrpc_mutex))
 416                 return -EINTR;
 417
 418         if (!stdev->alive) {
 419                 mutex_unlock(&stdev->mrpc_mutex);
 420                 return -ENODEV;
 421         }
 422
 423         return 0;
 424 }
 425
 426 static ssize_t switchtec_dev_write(struct file *filp, const char __user *data,
 427                                    size_t size, loff_t *off)
 428 {
 429         struct switchtec_user *stuser = filp->private_data;
 430         struct switchtec_dev *stdev = stuser->stdev;
 431         int rc;
 432
 433         if (size < sizeof(stuser->cmd) ||
 434             size > sizeof(stuser->cmd) + sizeof(stuser->data))
 435                 return -EINVAL;
 436
 437         stuser->data_len = size - sizeof(stuser->cmd);
 438
 439         rc = lock_mutex_and_test_alive(stdev);
 440         if (rc)
 441                 return rc;
 442
 443         if (stuser->state != MRPC_IDLE) {
 444                 rc = -EBADE;
 445                 goto out;
 446         }
 447
 448         rc = copy_from_user(&stuser->cmd, data, sizeof(stuser->cmd));
 449         if (rc) {
 450                 rc = -EFAULT;
 451                 goto out;
 452         }
 453
 454         data += sizeof(stuser->cmd);
 455         rc = copy_from_user(&stuser->data, data, size - sizeof(stuser->cmd));
 456         if (rc) {
 457                 rc = -EFAULT;
 458                 goto out;
 459         }
 460
 461         rc = mrpc_queue_cmd(stuser);
 462
 463 out:
 464         mutex_unlock(&stdev->mrpc_mutex);
 465
 466         if (rc)
 467                 return rc;
 468
 469         return size;
 470 }
 471
 472 static ssize_t switchtec_dev_read(struct file *filp, char __user *data,
 473                                   size_t size, loff_t *off)
 474 {
 475         struct switchtec_user *stuser = filp->private_data;
 476         struct switchtec_dev *stdev = stuser->stdev;
 477         int rc;
 478
 479         if (size < sizeof(stuser->cmd) ||
 480             size > sizeof(stuser->cmd) + sizeof(stuser->data))
 481                 return -EINVAL;
 482
 483         rc = lock_mutex_and_test_alive(stdev);
 484         if (rc)
 485                 return rc;
 486
 487         if (stuser->state == MRPC_IDLE) {
 488                 mutex_unlock(&stdev->mrpc_mutex);
 489                 return -EBADE;
 490         }
 491
 492         stuser->read_len = size - sizeof(stuser->return_code);
 493
 494         mutex_unlock(&stdev->mrpc_mutex);
 495
 496         if (filp->f_flags & O_NONBLOCK) {
 497                 if (!try_wait_for_completion(&stuser->comp))
 498                         return -EAGAIN;
 499         } else {
 500                 rc = wait_for_completion_interruptible(&stuser->comp);
 501                 if (rc < 0)
 502                         return rc;
 503         }
 504
 505         rc = lock_mutex_and_test_alive(stdev);
 506         if (rc)
 507                 return rc;
 508
 509         if (stuser->state != MRPC_DONE) {
 510                 mutex_unlock(&stdev->mrpc_mutex);
 511                 return -EBADE;
 512         }
 513
 514         rc = copy_to_user(data, &stuser->return_code,
 515                           sizeof(stuser->return_code));
 516         if (rc) {
 517                 rc = -EFAULT;
 518                 goto out;
 519         }
 520
 521         data += sizeof(stuser->return_code);
 522         rc = copy_to_user(data, &stuser->data,
 523                           size - sizeof(stuser->return_code));
 524         if (rc) {
 525                 rc = -EFAULT;
 526                 goto out;
 527         }
 528
 529         stuser_set_state(stuser, MRPC_IDLE);
 530
 531 out:
 532         mutex_unlock(&stdev->mrpc_mutex);
 533
 534         if (stuser->status == SWITCHTEC_MRPC_STATUS_DONE)
 535                 return size;
 536         else if (stuser->status == SWITCHTEC_MRPC_STATUS_INTERRUPTED)
 537                 return -ENXIO;
 538         else
 539                 return -EBADMSG;
 540 }
 541
 542 static __poll_t switchtec_dev_poll(struct file *filp, poll_table *wait)
 543 {
 544         struct switchtec_user *stuser = filp->private_data;
 545         struct switchtec_dev *stdev = stuser->stdev;
 546         __poll_t ret = 0;
 547
 548         poll_wait(filp, &stuser->comp.wait, wait);
 549         poll_wait(filp, &stdev->event_wq, wait);
 550
 551         if (lock_mutex_and_test_alive(stdev))
 552                 return EPOLLIN | EPOLLRDHUP | EPOLLOUT | EPOLLERR | EPOLLHUP;
 553
 554         mutex_unlock(&stdev->mrpc_mutex);
 555
 556         if (try_wait_for_completion(&stuser->comp))
 557                 ret |= EPOLLIN | EPOLLRDNORM;
 558
 559         if (stuser->event_cnt != atomic_read(&stdev->event_cnt))
 560                 ret |= EPOLLPRI | EPOLLRDBAND;
 561
 562         return ret;
 563 }
 564
 565 static int ioctl_flash_info(struct switchtec_dev *stdev,
 566                             struct switchtec_ioctl_flash_info __user *uinfo)
 567 {
 568         struct switchtec_ioctl_flash_info info = {0};
 569         struct flash_info_regs __iomem *fi = stdev->mmio_flash_info;
 570
 571         info.flash_length = ioread32(&fi->flash_length);
 572         info.num_partitions = SWITCHTEC_IOCTL_NUM_PARTITIONS;
 573
 574         if (copy_to_user(uinfo, &info, sizeof(info)))
 575                 return -EFAULT;
 576
 577         return 0;
 578 }
 579
 580 static void set_fw_info_part(struct switchtec_ioctl_flash_part_info *info,
 581                              struct partition_info __iomem *pi)
 582 {
 583         info->address = ioread32(&pi->address);
 584         info->length = ioread32(&pi->length);
 585 }
 586
 587 static int ioctl_flash_part_info(struct switchtec_dev *stdev,
 588         struct switchtec_ioctl_flash_part_info __user *uinfo)
 589 {
 590         struct switchtec_ioctl_flash_part_info info = {0};
 591         struct flash_info_regs __iomem *fi = stdev->mmio_flash_info;
 592         struct sys_info_regs __iomem *si = stdev->mmio_sys_info;
 593         u32 active_addr = -1;
 594
 595         if (copy_from_user(&info, uinfo, sizeof(info)))
 596                 return -EFAULT;
 597
 598         switch (info.flash_partition) {
 599         case SWITCHTEC_IOCTL_PART_CFG0:
 600                 active_addr = ioread32(&fi->active_cfg);
 601                 set_fw_info_part(&info, &fi->cfg0);
 602                 if (ioread16(&si->cfg_running) == SWITCHTEC_CFG0_RUNNING)
 603                         info.active |= SWITCHTEC_IOCTL_PART_RUNNING;
 604                 break;
 605         case SWITCHTEC_IOCTL_PART_CFG1:
 606                 active_addr = ioread32(&fi->active_cfg);
 607                 set_fw_info_part(&info, &fi->cfg1);
 608                 if (ioread16(&si->cfg_running) == SWITCHTEC_CFG1_RUNNING)
 609                         info.active |= SWITCHTEC_IOCTL_PART_RUNNING;
 610                 break;
 611         case SWITCHTEC_IOCTL_PART_IMG0:
 612                 active_addr = ioread32(&fi->active_img);
 613                 set_fw_info_part(&info, &fi->img0);
 614                 if (ioread16(&si->img_running) == SWITCHTEC_IMG0_RUNNING)
 615                         info.active |= SWITCHTEC_IOCTL_PART_RUNNING;
 616                 break;
 617         case SWITCHTEC_IOCTL_PART_IMG1:
 618                 active_addr = ioread32(&fi->active_img);
 619                 set_fw_info_part(&info, &fi->img1);
 620                 if (ioread16(&si->img_running) == SWITCHTEC_IMG1_RUNNING)
 621                         info.active |= SWITCHTEC_IOCTL_PART_RUNNING;
 622                 break;
 623         case SWITCHTEC_IOCTL_PART_NVLOG:
 624                 set_fw_info_part(&info, &fi->nvlog);
 625                 break;
 626         case SWITCHTEC_IOCTL_PART_VENDOR0:
 627                 set_fw_info_part(&info, &fi->vendor[0]);
 628                 break;
 629         case SWITCHTEC_IOCTL_PART_VENDOR1:
 630                 set_fw_info_part(&info, &fi->vendor[1]);
 631                 break;
 632         case SWITCHTEC_IOCTL_PART_VENDOR2:
 633                 set_fw_info_part(&info, &fi->vendor[2]);
 634                 break;
 635         case SWITCHTEC_IOCTL_PART_VENDOR3:
 636                 set_fw_info_part(&info, &fi->vendor[3]);
 637                 break;
 638         case SWITCHTEC_IOCTL_PART_VENDOR4:
 639                 set_fw_info_part(&info, &fi->vendor[4]);
 640                 break;
 641         case SWITCHTEC_IOCTL_PART_VENDOR5:
 642                 set_fw_info_part(&info, &fi->vendor[5]);
 643                 break;
 644         case SWITCHTEC_IOCTL_PART_VENDOR6:
 645                 set_fw_info_part(&info, &fi->vendor[6]);
 646                 break;
 647         case SWITCHTEC_IOCTL_PART_VENDOR7:
 648                 set_fw_info_part(&info, &fi->vendor[7]);
 649                 break;
 650         default:
 651                 return -EINVAL;
 652         }
 653
 654         if (info.address == active_addr)
 655                 info.active |= SWITCHTEC_IOCTL_PART_ACTIVE;
 656
 657         if (copy_to_user(uinfo, &info, sizeof(info)))
 658                 return -EFAULT;
 659
 660         return 0;
 661 }
 662
 663 static int ioctl_event_summary(struct switchtec_dev *stdev,
 664         struct switchtec_user *stuser,
 665         struct switchtec_ioctl_event_summary __user *usum,
 666         size_t size)
 667 {
 668         struct switchtec_ioctl_event_summary *s;
 669         int i;
 670         u32 reg;
 671         int ret = 0;
 672
 673         s = kzalloc(sizeof(*s), GFP_KERNEL);
 674         if (!s)
 675                 return -ENOMEM;
 676
 677         s->global = ioread32(&stdev->mmio_sw_event->global_summary);
 678         s->part_bitmap = ioread64(&stdev->mmio_sw_event->part_event_bitmap);
 679         s->local_part = ioread32(&stdev->mmio_part_cfg->part_event_summary);
 680
 681         for (i = 0; i < stdev->partition_count; i++) {
 682                 reg = ioread32(&stdev->mmio_part_cfg_all[i].part_event_summary);
 683                 s->part[i] = reg;
 684         }
 685
 686         for (i = 0; i < stdev->pff_csr_count; i++) {
 687                 reg = ioread32(&stdev->mmio_pff_csr[i].pff_event_summary);
 688                 s->pff[i] = reg;
 689         }
 690
 691         if (copy_to_user(usum, s, size)) {
 692                 ret = -EFAULT;
 693                 goto error_case;
 694         }
 695
 696         stuser->event_cnt = atomic_read(&stdev->event_cnt);
 697
 698 error_case:
 699         kfree(s);
 700         return ret;
 701 }
 702
 703 static u32 __iomem *global_ev_reg(struct switchtec_dev *stdev,
 704                                   size_t offset, int index)
 705 {
 706         return (void __iomem *)stdev->mmio_sw_event + offset;
 707 }
 708
 709 static u32 __iomem *part_ev_reg(struct switchtec_dev *stdev,
 710                                 size_t offset, int index)
 711 {
 712         return (void __iomem *)&stdev->mmio_part_cfg_all[index] + offset;
 713 }
 714
 715 static u32 __iomem *pff_ev_reg(struct switchtec_dev *stdev,
 716                                size_t offset, int index)
 717 {
 718         return (void __iomem *)&stdev->mmio_pff_csr[index] + offset;
 719 }
 720
 721 #define EV_GLB(i, r)[i] = {offsetof(struct sw_event_regs, r), global_ev_reg}
 722 #define EV_PAR(i, r)[i] = {offsetof(struct part_cfg_regs, r), part_ev_reg}
 723 #define EV_PFF(i, r)[i] = {offsetof(struct pff_csr_regs, r), pff_ev_reg}
 724
 725 static const struct event_reg {
 726         size_t offset;
 727         u32 __iomem *(*map_reg)(struct switchtec_dev *stdev,
 728                                 size_t offset, int index);
 729 } event_regs[] = {
 730         EV_GLB(SWITCHTEC_IOCTL_EVENT_STACK_ERROR, stack_error_event_hdr),
 731         EV_GLB(SWITCHTEC_IOCTL_EVENT_PPU_ERROR, ppu_error_event_hdr),
 732         EV_GLB(SWITCHTEC_IOCTL_EVENT_ISP_ERROR, isp_error_event_hdr),
 733         EV_GLB(SWITCHTEC_IOCTL_EVENT_SYS_RESET, sys_reset_event_hdr),
 734         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_EXC, fw_exception_hdr),
 735         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_NMI, fw_nmi_hdr),
 736         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_NON_FATAL, fw_non_fatal_hdr),
 737         EV_GLB(SWITCHTEC_IOCTL_EVENT_FW_FATAL, fw_fatal_hdr),
 738         EV_GLB(SWITCHTEC_IOCTL_EVENT_TWI_MRPC_COMP, twi_mrpc_comp_hdr),
 739         EV_GLB(SWITCHTEC_IOCTL_EVENT_TWI_MRPC_COMP_ASYNC,
 740                twi_mrpc_comp_async_hdr),
 741         EV_GLB(SWITCHTEC_IOCTL_EVENT_CLI_MRPC_COMP, cli_mrpc_comp_hdr),
 742         EV_GLB(SWITCHTEC_IOCTL_EVENT_CLI_MRPC_COMP_ASYNC,
 743                cli_mrpc_comp_async_hdr),
 744         EV_GLB(SWITCHTEC_IOCTL_EVENT_GPIO_INT, gpio_interrupt_hdr),
 745         EV_GLB(SWITCHTEC_IOCTL_EVENT_GFMS, gfms_event_hdr),
 746         EV_PAR(SWITCHTEC_IOCTL_EVENT_PART_RESET, part_reset_hdr),
 747         EV_PAR(SWITCHTEC_IOCTL_EVENT_MRPC_COMP, mrpc_comp_hdr),
 748         EV_PAR(SWITCHTEC_IOCTL_EVENT_MRPC_COMP_ASYNC, mrpc_comp_async_hdr),
 749         EV_PAR(SWITCHTEC_IOCTL_EVENT_DYN_PART_BIND_COMP, dyn_binding_hdr),
 750         EV_PAR(SWITCHTEC_IOCTL_EVENT_INTERCOMM_REQ_NOTIFY,
 751                intercomm_notify_hdr),
 752         EV_PFF(SWITCHTEC_IOCTL_EVENT_AER_IN_P2P, aer_in_p2p_hdr),
 753         EV_PFF(SWITCHTEC_IOCTL_EVENT_AER_IN_VEP, aer_in_vep_hdr),
 754         EV_PFF(SWITCHTEC_IOCTL_EVENT_DPC, dpc_hdr),
 755         EV_PFF(SWITCHTEC_IOCTL_EVENT_CTS, cts_hdr),
 756         EV_PFF(SWITCHTEC_IOCTL_EVENT_UEC, uec_hdr),
 757         EV_PFF(SWITCHTEC_IOCTL_EVENT_HOTPLUG, hotplug_hdr),
 758         EV_PFF(SWITCHTEC_IOCTL_EVENT_IER, ier_hdr),
 759         EV_PFF(SWITCHTEC_IOCTL_EVENT_THRESH, threshold_hdr),
 760         EV_PFF(SWITCHTEC_IOCTL_EVENT_POWER_MGMT, power_mgmt_hdr),
 761         EV_PFF(SWITCHTEC_IOCTL_EVENT_TLP_THROTTLING, tlp_throttling_hdr),
 762         EV_PFF(SWITCHTEC_IOCTL_EVENT_FORCE_SPEED, force_speed_hdr),
 763         EV_PFF(SWITCHTEC_IOCTL_EVENT_CREDIT_TIMEOUT, credit_timeout_hdr),
 764         EV_PFF(SWITCHTEC_IOCTL_EVENT_LINK_STATE, link_state_hdr),
 765 };
 766
 767 static u32 __iomem *event_hdr_addr(struct switchtec_dev *stdev,
 768                                    int event_id, int index)
 769 {
 770         size_t off;
 771
 772         if (event_id < 0 || event_id >= SWITCHTEC_IOCTL_MAX_EVENTS)
 773                 return ERR_PTR(-EINVAL);
 774
 775         off = event_regs[event_id].offset;
 776
 777         if (event_regs[event_id].map_reg == part_ev_reg) {
 778                 if (index == SWITCHTEC_IOCTL_EVENT_LOCAL_PART_IDX)
 779                         index = stdev->partition;
 780                 else if (index < 0 || index >= stdev->partition_count)
 781                         return ERR_PTR(-EINVAL);
 782         } else if (event_regs[event_id].map_reg == pff_ev_reg) {
 783                 if (index < 0 || index >= stdev->pff_csr_count)
 784                         return ERR_PTR(-EINVAL);
 785         }
 786
 787         return event_regs[event_id].map_reg(stdev, off, index);
 788 }
 789
 790 static int event_ctl(struct switchtec_dev *stdev,
 791                      struct switchtec_ioctl_event_ctl *ctl)
 792 {
 793         int i;
 794         u32 __iomem *reg;
 795         u32 hdr;
 796
 797         reg = event_hdr_addr(stdev, ctl->event_id, ctl->index);
 798         if (IS_ERR(reg))
 799                 return PTR_ERR(reg);
 800
 801         hdr = ioread32(reg);
 802         for (i = 0; i < ARRAY_SIZE(ctl->data); i++)
 803                 ctl->data[i] = ioread32(&reg[i + 1]);
 804
 805         ctl->occurred = hdr & SWITCHTEC_EVENT_OCCURRED;
 806         ctl->count = (hdr >> 5) & 0xFF;
 807
 808         if (!(ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_CLEAR))
 809                 hdr &= ~SWITCHTEC_EVENT_CLEAR;
 810         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_POLL)
 811                 hdr |= SWITCHTEC_EVENT_EN_IRQ;
 812         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_POLL)
 813                 hdr &= ~SWITCHTEC_EVENT_EN_IRQ;
 814         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_LOG)
 815                 hdr |= SWITCHTEC_EVENT_EN_LOG;
 816         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_LOG)
 817                 hdr &= ~SWITCHTEC_EVENT_EN_LOG;
 818         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_CLI)
 819                 hdr |= SWITCHTEC_EVENT_EN_CLI;
 820         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_CLI)
 821                 hdr &= ~SWITCHTEC_EVENT_EN_CLI;
 822         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_EN_FATAL)
 823                 hdr |= SWITCHTEC_EVENT_FATAL;
 824         if (ctl->flags & SWITCHTEC_IOCTL_EVENT_FLAG_DIS_FATAL)
 825                 hdr &= ~SWITCHTEC_EVENT_FATAL;
 826
 827         if (ctl->flags)
 828                 iowrite32(hdr, reg);
 829
 830         ctl->flags = 0;
 831         if (hdr & SWITCHTEC_EVENT_EN_IRQ)
 832                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_POLL;
 833         if (hdr & SWITCHTEC_EVENT_EN_LOG)
 834                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_LOG;
 835         if (hdr & SWITCHTEC_EVENT_EN_CLI)
 836                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_CLI;
 837         if (hdr & SWITCHTEC_EVENT_FATAL)
 838                 ctl->flags |= SWITCHTEC_IOCTL_EVENT_FLAG_EN_FATAL;
 839
 840         return 0;
 841 }
 842
 843 static int ioctl_event_ctl(struct switchtec_dev *stdev,
 844         struct switchtec_ioctl_event_ctl __user *uctl)
 845 {
 846         int ret;
 847         int nr_idxs;
 848         unsigned int event_flags;
 849         struct switchtec_ioctl_event_ctl ctl;
 850
 851         if (copy_from_user(&ctl, uctl, sizeof(ctl)))
 852                 return -EFAULT;
 853
 854         if (ctl.event_id >= SWITCHTEC_IOCTL_MAX_EVENTS)
 855                 return -EINVAL;
 856
 857         if (ctl.flags & SWITCHTEC_IOCTL_EVENT_FLAG_UNUSED)
 858                 return -EINVAL;
 859
 860         if (ctl.index == SWITCHTEC_IOCTL_EVENT_IDX_ALL) {
 861                 if (event_regs[ctl.event_id].map_reg == global_ev_reg)
 862                         nr_idxs = 1;
 863                 else if (event_regs[ctl.event_id].map_reg == part_ev_reg)
 864                         nr_idxs = stdev->partition_count;
 865                 else if (event_regs[ctl.event_id].map_reg == pff_ev_reg)
 866                         nr_idxs = stdev->pff_csr_count;
 867                 else
 868                         return -EINVAL;
 869
 870                 event_flags = ctl.flags;
 871                 for (ctl.index = 0; ctl.index < nr_idxs; ctl.index++) {
 872                         ctl.flags = event_flags;
 873                         ret = event_ctl(stdev, &ctl);
 874                         if (ret < 0)
 875                                 return ret;
 876                 }
 877         } else {
 878                 ret = event_ctl(stdev, &ctl);
 879                 if (ret < 0)
 880                         return ret;
 881         }
 882
 883         if (copy_to_user(uctl, &ctl, sizeof(ctl)))
 884                 return -EFAULT;
 885
 886         return 0;
 887 }
 888
 889 static int ioctl_pff_to_port(struct switchtec_dev *stdev,
 890                              struct switchtec_ioctl_pff_port *up)
 891 {
 892         int i, part;
 893         u32 reg;
 894         struct part_cfg_regs *pcfg;
 895         struct switchtec_ioctl_pff_port p;
 896
 897         if (copy_from_user(&p, up, sizeof(p)))
 898                 return -EFAULT;
 899
 900         p.port = -1;
 901         for (part = 0; part < stdev->partition_count; part++) {
 902                 pcfg = &stdev->mmio_part_cfg_all[part];
 903                 p.partition = part;
 904
 905                 reg = ioread32(&pcfg->usp_pff_inst_id);
 906                 if (reg == p.pff) {
 907                         p.port = 0;
 908                         break;
 909                 }
 910
 911                 reg = ioread32(&pcfg->vep_pff_inst_id);
 912                 if (reg == p.pff) {
 913                         p.port = SWITCHTEC_IOCTL_PFF_VEP;
 914                         break;
 915                 }
 916
 917                 for (i = 0; i < ARRAY_SIZE(pcfg->dsp_pff_inst_id); i++) {
 918                         reg = ioread32(&pcfg->dsp_pff_inst_id[i]);
 919                         if (reg != p.pff)
 920                                 continue;
 921
 922                         p.port = i + 1;
 923                         break;
 924                 }
 925
 926                 if (p.port != -1)
 927                         break;
 928         }
 929
 930         if (copy_to_user(up, &p, sizeof(p)))
 931                 return -EFAULT;
 932
 933         return 0;
 934 }
 935
 936 static int ioctl_port_to_pff(struct switchtec_dev *stdev,
 937                              struct switchtec_ioctl_pff_port *up)
 938 {
 939         struct switchtec_ioctl_pff_port p;
 940         struct part_cfg_regs *pcfg;
 941
 942         if (copy_from_user(&p, up, sizeof(p)))
 943                 return -EFAULT;
 944
 945         if (p.partition == SWITCHTEC_IOCTL_EVENT_LOCAL_PART_IDX)
 946                 pcfg = stdev->mmio_part_cfg;
 947         else if (p.partition < stdev->partition_count)
 948                 pcfg = &stdev->mmio_part_cfg_all[p.partition];
 949         else
 950                 return -EINVAL;
 951
 952         switch (p.port) {
 953         case 0:
 954                 p.pff = ioread32(&pcfg->usp_pff_inst_id);
 955                 break;
 956         case SWITCHTEC_IOCTL_PFF_VEP:
 957                 p.pff = ioread32(&pcfg->vep_pff_inst_id);
 958                 break;
 959         default:
 960                 if (p.port > ARRAY_SIZE(pcfg->dsp_pff_inst_id))
 961                         return -EINVAL;
 962                 p.port = array_index_nospec(p.port,
 963                                         ARRAY_SIZE(pcfg->dsp_pff_inst_id) + 1);
 964                 p.pff = ioread32(&pcfg->dsp_pff_inst_id[p.port - 1]);
 965                 break;
 966         }
 967
 968         if (copy_to_user(up, &p, sizeof(p)))
 969                 return -EFAULT;
 970
 971         return 0;
 972 }
 973
 974 static long switchtec_dev_ioctl(struct file *filp, unsigned int cmd,
 975                                 unsigned long arg)
 976 {
 977         struct switchtec_user *stuser = filp->private_data;
 978         struct switchtec_dev *stdev = stuser->stdev;
 979         int rc;
 980         void __user *argp = (void __user *)arg;
 981
 982         rc = lock_mutex_and_test_alive(stdev);
 983         if (rc)
 984                 return rc;
 985
 986         switch (cmd) {
 987         case SWITCHTEC_IOCTL_FLASH_INFO:
 988                 rc = ioctl_flash_info(stdev, argp);
 989                 break;
 990         case SWITCHTEC_IOCTL_FLASH_PART_INFO:
 991                 rc = ioctl_flash_part_info(stdev, argp);
 992                 break;
 993         case SWITCHTEC_IOCTL_EVENT_SUMMARY_LEGACY:
 994                 rc = ioctl_event_summary(stdev, stuser, argp,
 995                                          sizeof(struct switchtec_ioctl_event_summary_legacy));
 996                 break;
 997         case SWITCHTEC_IOCTL_EVENT_CTL:
 998                 rc = ioctl_event_ctl(stdev, argp);
 999                 break;
1000         case SWITCHTEC_IOCTL_PFF_TO_PORT:
1001                 rc = ioctl_pff_to_port(stdev, argp);
1002                 break;
1003         case SWITCHTEC_IOCTL_PORT_TO_PFF:
1004                 rc = ioctl_port_to_pff(stdev, argp);
1005                 break;
1006         case SWITCHTEC_IOCTL_EVENT_SUMMARY:
1007                 rc = ioctl_event_summary(stdev, stuser, argp,
1008                                          sizeof(struct switchtec_ioctl_event_summary));
1009                 break;
1010         default:
1011                 rc = -ENOTTY;
1012                 break;
1013         }
1014
1015         mutex_unlock(&stdev->mrpc_mutex);
1016         return rc;
1017 }
1018
1019 static const struct file_operations switchtec_fops = {
1020         .owner = THIS_MODULE,
1021         .open = switchtec_dev_open,
1022         .release = switchtec_dev_release,
1023         .write = switchtec_dev_write,
1024         .read = switchtec_dev_read,
1025         .poll = switchtec_dev_poll,
1026         .unlocked_ioctl = switchtec_dev_ioctl,
1027         .compat_ioctl = compat_ptr_ioctl,
1028 };
1029
1030 static void link_event_work(struct work_struct *work)
1031 {
1032         struct switchtec_dev *stdev;
1033
1034         stdev = container_of(work, struct switchtec_dev, link_event_work);
1035
1036         if (stdev->link_notifier)
1037                 stdev->link_notifier(stdev);
1038 }
1039
1040 static void check_link_state_events(struct switchtec_dev *stdev)
1041 {
1042         int idx;
1043         u32 reg;
1044         int count;
1045         int occurred = 0;
1046
1047         for (idx = 0; idx < stdev->pff_csr_count; idx++) {
1048                 reg = ioread32(&stdev->mmio_pff_csr[idx].link_state_hdr);
1049                 dev_dbg(&stdev->dev, "link_state: %d->%08x\n", idx, reg);
1050                 count = (reg >> 5) & 0xFF;
1051
1052                 if (count != stdev->link_event_count[idx]) {
1053                         occurred = 1;
1054                         stdev->link_event_count[idx] = count;
1055                 }
1056         }
1057
1058         if (occurred)
1059                 schedule_work(&stdev->link_event_work);
1060 }
1061
1062 static void enable_link_state_events(struct switchtec_dev *stdev)
1063 {
1064         int idx;
1065
1066         for (idx = 0; idx < stdev->pff_csr_count; idx++) {
1067                 iowrite32(SWITCHTEC_EVENT_CLEAR |
1068                           SWITCHTEC_EVENT_EN_IRQ,
1069                           &stdev->mmio_pff_csr[idx].link_state_hdr);
1070         }
1071 }
1072
1073 static void enable_dma_mrpc(struct switchtec_dev *stdev)
1074 {
1075         writeq(stdev->dma_mrpc_dma_addr, &stdev->mmio_mrpc->dma_addr);
1076         flush_wc_buf(stdev);
1077         iowrite32(SWITCHTEC_DMA_MRPC_EN, &stdev->mmio_mrpc->dma_en);
1078 }
1079
1080 static void stdev_release(struct device *dev)
1081 {
1082         struct switchtec_dev *stdev = to_stdev(dev);
1083
1084         if (stdev->dma_mrpc) {
1085                 iowrite32(0, &stdev->mmio_mrpc->dma_en);
1086                 flush_wc_buf(stdev);
1087                 writeq(0, &stdev->mmio_mrpc->dma_addr);
1088                 dma_free_coherent(&stdev->pdev->dev, sizeof(*stdev->dma_mrpc),
1089                                 stdev->dma_mrpc, stdev->dma_mrpc_dma_addr);
1090         }
1091         kfree(stdev);
1092 }
1093
1094 static void stdev_kill(struct switchtec_dev *stdev)
1095 {
1096         struct switchtec_user *stuser, *tmpuser;
1097
1098         pci_clear_master(stdev->pdev);
1099
1100         cancel_delayed_work_sync(&stdev->mrpc_timeout);
1101
1102         /* Mark the hardware as unavailable and complete all completions */
1103         mutex_lock(&stdev->mrpc_mutex);
1104         stdev->alive = false;
1105
1106         /* Wake up and kill any users waiting on an MRPC request */
1107         list_for_each_entry_safe(stuser, tmpuser, &stdev->mrpc_queue, list) {
1108                 complete_all(&stuser->comp);
1109                 list_del_init(&stuser->list);
1110                 stuser_put(stuser);
1111         }
1112
1113         mutex_unlock(&stdev->mrpc_mutex);
1114
1115         /* Wake up any users waiting on event_wq */
1116         wake_up_interruptible(&stdev->event_wq);
1117 }
1118
1119 static struct switchtec_dev *stdev_create(struct pci_dev *pdev)
1120 {
1121         struct switchtec_dev *stdev;
1122         int minor;
1123         struct device *dev;
1124         struct cdev *cdev;
1125         int rc;
1126
1127         stdev = kzalloc_node(sizeof(*stdev), GFP_KERNEL,
1128                              dev_to_node(&pdev->dev));
1129         if (!stdev)
1130                 return ERR_PTR(-ENOMEM);
1131
1132         stdev->alive = true;
1133         stdev->pdev = pdev;
1134         INIT_LIST_HEAD(&stdev->mrpc_queue);
1135         mutex_init(&stdev->mrpc_mutex);
1136         stdev->mrpc_busy = 0;
1137         INIT_WORK(&stdev->mrpc_work, mrpc_event_work);
1138         INIT_DELAYED_WORK(&stdev->mrpc_timeout, mrpc_timeout_work);
1139         INIT_WORK(&stdev->link_event_work, link_event_work);
1140         init_waitqueue_head(&stdev->event_wq);
1141         atomic_set(&stdev->event_cnt, 0);
1142
1143         dev = &stdev->dev;
1144         device_initialize(dev);
1145         dev->class = switchtec_class;
1146         dev->parent = &pdev->dev;
1147         dev->groups = switchtec_device_groups;
1148         dev->release = stdev_release;
1149
1150         minor = ida_simple_get(&switchtec_minor_ida, 0, 0,
1151                                GFP_KERNEL);
1152         if (minor < 0) {
1153                 rc = minor;
1154                 goto err_put;
1155         }
1156
1157         dev->devt = MKDEV(MAJOR(switchtec_devt), minor);
1158         dev_set_name(dev, "switchtec%d", minor);
1159
1160         cdev = &stdev->cdev;
1161         cdev_init(cdev, &switchtec_fops);
1162         cdev->owner = THIS_MODULE;
1163
1164         return stdev;
1165
1166 err_put:
1167         put_device(&stdev->dev);
1168         return ERR_PTR(rc);
1169 }
1170
1171 static int mask_event(struct switchtec_dev *stdev, int eid, int idx)
1172 {
1173         size_t off = event_regs[eid].offset;
1174         u32 __iomem *hdr_reg;
1175         u32 hdr;
1176
1177         hdr_reg = event_regs[eid].map_reg(stdev, off, idx);
1178         hdr = ioread32(hdr_reg);
1179
1180         if (!(hdr & SWITCHTEC_EVENT_OCCURRED && hdr & SWITCHTEC_EVENT_EN_IRQ))
1181                 return 0;
1182
1183         dev_dbg(&stdev->dev, "%s: %d %d %x\n", __func__, eid, idx, hdr);
1184         hdr &= ~(SWITCHTEC_EVENT_EN_IRQ | SWITCHTEC_EVENT_OCCURRED);
1185         iowrite32(hdr, hdr_reg);
1186
1187         return 1;
1188 }
1189
1190 static int mask_all_events(struct switchtec_dev *stdev, int eid)
1191 {
1192         int idx;
1193         int count = 0;
1194
1195         if (event_regs[eid].map_reg == part_ev_reg) {
1196                 for (idx = 0; idx < stdev->partition_count; idx++)
1197                         count += mask_event(stdev, eid, idx);
1198         } else if (event_regs[eid].map_reg == pff_ev_reg) {
1199                 for (idx = 0; idx < stdev->pff_csr_count; idx++) {
1200                         if (!stdev->pff_local[idx])
1201                                 continue;
1202
1203                         count += mask_event(stdev, eid, idx);
1204                 }
1205         } else {
1206                 count += mask_event(stdev, eid, 0);
1207         }
1208
1209         return count;
1210 }
1211
1212 static irqreturn_t switchtec_event_isr(int irq, void *dev)
1213 {
1214         struct switchtec_dev *stdev = dev;
1215         u32 reg;
1216         irqreturn_t ret = IRQ_NONE;
1217         int eid, event_count = 0;
1218
1219         reg = ioread32(&stdev->mmio_part_cfg->mrpc_comp_hdr);
1220         if (reg & SWITCHTEC_EVENT_OCCURRED) {
1221                 dev_dbg(&stdev->dev, "%s: mrpc comp\n", __func__);
1222                 ret = IRQ_HANDLED;
1223                 schedule_work(&stdev->mrpc_work);
1224                 iowrite32(reg, &stdev->mmio_part_cfg->mrpc_comp_hdr);
1225         }
1226
1227         check_link_state_events(stdev);
1228
1229         for (eid = 0; eid < SWITCHTEC_IOCTL_MAX_EVENTS; eid++) {
1230                 if (eid == SWITCHTEC_IOCTL_EVENT_LINK_STATE ||
1231                     eid == SWITCHTEC_IOCTL_EVENT_MRPC_COMP)
1232                         continue;
1233
1234                 event_count += mask_all_events(stdev, eid);
1235         }
1236
1237         if (event_count) {
1238                 atomic_inc(&stdev->event_cnt);
1239                 wake_up_interruptible(&stdev->event_wq);
1240                 dev_dbg(&stdev->dev, "%s: %d events\n", __func__,
1241                         event_count);
1242                 return IRQ_HANDLED;
1243         }
1244
1245         return ret;
1246 }
1247
1248
1249 static irqreturn_t switchtec_dma_mrpc_isr(int irq, void *dev)
1250 {
1251         struct switchtec_dev *stdev = dev;
1252         irqreturn_t ret = IRQ_NONE;
1253
1254         iowrite32(SWITCHTEC_EVENT_CLEAR |
1255                   SWITCHTEC_EVENT_EN_IRQ,
1256                   &stdev->mmio_part_cfg->mrpc_comp_hdr);
1257         schedule_work(&stdev->mrpc_work);
1258
1259         ret = IRQ_HANDLED;
1260         return ret;
1261 }
1262
1263 static int switchtec_init_isr(struct switchtec_dev *stdev)
1264 {
1265         int nvecs;
1266         int event_irq;
1267         int dma_mrpc_irq;
1268         int rc;
1269
1270         if (nirqs < 4)
1271                 nirqs = 4;
1272
1273         nvecs = pci_alloc_irq_vectors(stdev->pdev, 1, nirqs,
1274                                       PCI_IRQ_MSIX | PCI_IRQ_MSI |
1275                                       PCI_IRQ_VIRTUAL);
1276         if (nvecs < 0)
1277                 return nvecs;
1278
1279         event_irq = ioread16(&stdev->mmio_part_cfg->vep_vector_number);
1280         if (event_irq < 0 || event_irq >= nvecs)
1281                 return -EFAULT;
1282
1283         event_irq = pci_irq_vector(stdev->pdev, event_irq);
1284         if (event_irq < 0)
1285                 return event_irq;
1286
1287         rc = devm_request_irq(&stdev->pdev->dev, event_irq,
1288                                 switchtec_event_isr, 0,
1289                                 KBUILD_MODNAME, stdev);
1290
1291         if (rc)
1292                 return rc;
1293
1294         if (!stdev->dma_mrpc)
1295                 return rc;
1296
1297         dma_mrpc_irq = ioread32(&stdev->mmio_mrpc->dma_vector);
1298         if (dma_mrpc_irq < 0 || dma_mrpc_irq >= nvecs)
1299                 return -EFAULT;
1300
1301         dma_mrpc_irq  = pci_irq_vector(stdev->pdev, dma_mrpc_irq);
1302         if (dma_mrpc_irq < 0)
1303                 return dma_mrpc_irq;
1304
1305         rc = devm_request_irq(&stdev->pdev->dev, dma_mrpc_irq,
1306                                 switchtec_dma_mrpc_isr, 0,
1307                                 KBUILD_MODNAME, stdev);
1308
1309         return rc;
1310 }
1311
1312 static void init_pff(struct switchtec_dev *stdev)
1313 {
1314         int i;
1315         u32 reg;
1316         struct part_cfg_regs *pcfg = stdev->mmio_part_cfg;
1317
1318         for (i = 0; i < SWITCHTEC_MAX_PFF_CSR; i++) {
1319                 reg = ioread16(&stdev->mmio_pff_csr[i].vendor_id);
1320                 if (reg != PCI_VENDOR_ID_MICROSEMI)
1321                         break;
1322         }
1323
1324         stdev->pff_csr_count = i;
1325
1326         reg = ioread32(&pcfg->usp_pff_inst_id);
1327         if (reg < stdev->pff_csr_count)
1328                 stdev->pff_local[reg] = 1;
1329
1330         reg = ioread32(&pcfg->vep_pff_inst_id);
1331         if (reg < stdev->pff_csr_count)
1332                 stdev->pff_local[reg] = 1;
1333
1334         for (i = 0; i < ARRAY_SIZE(pcfg->dsp_pff_inst_id); i++) {
1335                 reg = ioread32(&pcfg->dsp_pff_inst_id[i]);
1336                 if (reg < stdev->pff_csr_count)
1337                         stdev->pff_local[reg] = 1;
1338         }
1339 }
1340
1341 static int switchtec_init_pci(struct switchtec_dev *stdev,
1342                               struct pci_dev *pdev)
1343 {
1344         int rc;
1345         void __iomem *map;
1346         unsigned long res_start, res_len;
1347
1348         rc = pcim_enable_device(pdev);
1349         if (rc)
1350                 return rc;
1351
1352         rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1353         if (rc)
1354                 return rc;
1355
1356         pci_set_master(pdev);
1357
1358         res_start = pci_resource_start(pdev, 0);
1359         res_len = pci_resource_len(pdev, 0);
1360
1361         if (!devm_request_mem_region(&pdev->dev, res_start,
1362                                      res_len, KBUILD_MODNAME))
1363                 return -EBUSY;
1364
1365         stdev->mmio_mrpc = devm_ioremap_wc(&pdev->dev, res_start,
1366                                            SWITCHTEC_GAS_TOP_CFG_OFFSET);
1367         if (!stdev->mmio_mrpc)
1368                 return -ENOMEM;
1369
1370         map = devm_ioremap(&pdev->dev,
1371                            res_start + SWITCHTEC_GAS_TOP_CFG_OFFSET,
1372                            res_len - SWITCHTEC_GAS_TOP_CFG_OFFSET);
1373         if (!map)
1374                 return -ENOMEM;
1375
1376         stdev->mmio = map - SWITCHTEC_GAS_TOP_CFG_OFFSET;
1377         stdev->mmio_sw_event = stdev->mmio + SWITCHTEC_GAS_SW_EVENT_OFFSET;
1378         stdev->mmio_sys_info = stdev->mmio + SWITCHTEC_GAS_SYS_INFO_OFFSET;
1379         stdev->mmio_flash_info = stdev->mmio + SWITCHTEC_GAS_FLASH_INFO_OFFSET;
1380         stdev->mmio_ntb = stdev->mmio + SWITCHTEC_GAS_NTB_OFFSET;
1381         stdev->partition = ioread8(&stdev->mmio_sys_info->partition_id);
1382         stdev->partition_count = ioread8(&stdev->mmio_ntb->partition_count);
1383         stdev->mmio_part_cfg_all = stdev->mmio + SWITCHTEC_GAS_PART_CFG_OFFSET;
1384         stdev->mmio_part_cfg = &stdev->mmio_part_cfg_all[stdev->partition];
1385         stdev->mmio_pff_csr = stdev->mmio + SWITCHTEC_GAS_PFF_CSR_OFFSET;
1386
1387         if (stdev->partition_count < 1)
1388                 stdev->partition_count = 1;
1389
1390         init_pff(stdev);
1391
1392         pci_set_drvdata(pdev, stdev);
1393
1394         if (!use_dma_mrpc)
1395                 return 0;
1396
1397         if (ioread32(&stdev->mmio_mrpc->dma_ver) == 0)
1398                 return 0;
1399
1400         stdev->dma_mrpc = dma_alloc_coherent(&stdev->pdev->dev,
1401                                              sizeof(*stdev->dma_mrpc),
1402                                              &stdev->dma_mrpc_dma_addr,
1403                                              GFP_KERNEL);
1404         if (stdev->dma_mrpc == NULL)
1405                 return -ENOMEM;
1406
1407         return 0;
1408 }
1409
1410 static int switchtec_pci_probe(struct pci_dev *pdev,
1411                                const struct pci_device_id *id)
1412 {
1413         struct switchtec_dev *stdev;
1414         int rc;
1415
1416         if (pdev->class == (PCI_CLASS_BRIDGE_OTHER << 8))
1417                 request_module_nowait("ntb_hw_switchtec");
1418
1419         stdev = stdev_create(pdev);
1420         if (IS_ERR(stdev))
1421                 return PTR_ERR(stdev);
1422
1423         rc = switchtec_init_pci(stdev, pdev);
1424         if (rc)
1425                 goto err_put;
1426
1427         rc = switchtec_init_isr(stdev);
1428         if (rc) {
1429                 dev_err(&stdev->dev, "failed to init isr.\n");
1430                 goto err_put;
1431         }
1432
1433         iowrite32(SWITCHTEC_EVENT_CLEAR |
1434                   SWITCHTEC_EVENT_EN_IRQ,
1435                   &stdev->mmio_part_cfg->mrpc_comp_hdr);
1436         enable_link_state_events(stdev);
1437
1438         if (stdev->dma_mrpc)
1439                 enable_dma_mrpc(stdev);
1440
1441         rc = cdev_device_add(&stdev->cdev, &stdev->dev);
1442         if (rc)
1443                 goto err_devadd;
1444
1445         dev_info(&stdev->dev, "Management device registered.\n");
1446
1447         return 0;
1448
1449 err_devadd:
1450         stdev_kill(stdev);
1451 err_put:
1452         ida_simple_remove(&switchtec_minor_ida, MINOR(stdev->dev.devt));
1453         put_device(&stdev->dev);
1454         return rc;
1455 }
1456
1457 static void switchtec_pci_remove(struct pci_dev *pdev)
1458 {
1459         struct switchtec_dev *stdev = pci_get_drvdata(pdev);
1460
1461         pci_set_drvdata(pdev, NULL);
1462
1463         cdev_device_del(&stdev->cdev, &stdev->dev);
1464         ida_simple_remove(&switchtec_minor_ida, MINOR(stdev->dev.devt));
1465         dev_info(&stdev->dev, "unregistered.\n");
1466         stdev_kill(stdev);
1467         put_device(&stdev->dev);
1468 }
1469
1470 #define SWITCHTEC_PCI_DEVICE(device_id) \
1471         { \
1472                 .vendor     = PCI_VENDOR_ID_MICROSEMI, \
1473                 .device     = device_id, \
1474                 .subvendor  = PCI_ANY_ID, \
1475                 .subdevice  = PCI_ANY_ID, \
1476                 .class      = (PCI_CLASS_MEMORY_OTHER << 8), \
1477                 .class_mask = 0xFFFFFFFF, \
1478         }, \
1479         { \
1480                 .vendor     = PCI_VENDOR_ID_MICROSEMI, \
1481                 .device     = device_id, \
1482                 .subvendor  = PCI_ANY_ID, \
1483                 .subdevice  = PCI_ANY_ID, \
1484                 .class      = (PCI_CLASS_BRIDGE_OTHER << 8), \
1485                 .class_mask = 0xFFFFFFFF, \
1486         }
1487
1488 static const struct pci_device_id switchtec_pci_tbl[] = {
1489         SWITCHTEC_PCI_DEVICE(0x8531),  //PFX 24xG3
1490         SWITCHTEC_PCI_DEVICE(0x8532),  //PFX 32xG3
1491         SWITCHTEC_PCI_DEVICE(0x8533),  //PFX 48xG3
1492         SWITCHTEC_PCI_DEVICE(0x8534),  //PFX 64xG3
1493         SWITCHTEC_PCI_DEVICE(0x8535),  //PFX 80xG3
1494         SWITCHTEC_PCI_DEVICE(0x8536),  //PFX 96xG3
1495         SWITCHTEC_PCI_DEVICE(0x8541),  //PSX 24xG3
1496         SWITCHTEC_PCI_DEVICE(0x8542),  //PSX 32xG3
1497         SWITCHTEC_PCI_DEVICE(0x8543),  //PSX 48xG3
1498         SWITCHTEC_PCI_DEVICE(0x8544),  //PSX 64xG3
1499         SWITCHTEC_PCI_DEVICE(0x8545),  //PSX 80xG3
1500         SWITCHTEC_PCI_DEVICE(0x8546),  //PSX 96xG3
1501         SWITCHTEC_PCI_DEVICE(0x8551),  //PAX 24XG3
1502         SWITCHTEC_PCI_DEVICE(0x8552),  //PAX 32XG3
1503         SWITCHTEC_PCI_DEVICE(0x8553),  //PAX 48XG3
1504         SWITCHTEC_PCI_DEVICE(0x8554),  //PAX 64XG3
1505         SWITCHTEC_PCI_DEVICE(0x8555),  //PAX 80XG3
1506         SWITCHTEC_PCI_DEVICE(0x8556),  //PAX 96XG3
1507         SWITCHTEC_PCI_DEVICE(0x8561),  //PFXL 24XG3
1508         SWITCHTEC_PCI_DEVICE(0x8562),  //PFXL 32XG3
1509         SWITCHTEC_PCI_DEVICE(0x8563),  //PFXL 48XG3
1510         SWITCHTEC_PCI_DEVICE(0x8564),  //PFXL 64XG3
1511         SWITCHTEC_PCI_DEVICE(0x8565),  //PFXL 80XG3
1512         SWITCHTEC_PCI_DEVICE(0x8566),  //PFXL 96XG3
1513         SWITCHTEC_PCI_DEVICE(0x8571),  //PFXI 24XG3
1514         SWITCHTEC_PCI_DEVICE(0x8572),  //PFXI 32XG3
1515         SWITCHTEC_PCI_DEVICE(0x8573),  //PFXI 48XG3
1516         SWITCHTEC_PCI_DEVICE(0x8574),  //PFXI 64XG3
1517         SWITCHTEC_PCI_DEVICE(0x8575),  //PFXI 80XG3
1518         SWITCHTEC_PCI_DEVICE(0x8576),  //PFXI 96XG3
1519         {0}
1520 };
1521 MODULE_DEVICE_TABLE(pci, switchtec_pci_tbl);
1522
1523 static struct pci_driver switchtec_pci_driver = {
1524         .name           = KBUILD_MODNAME,
1525         .id_table       = switchtec_pci_tbl,
1526         .probe          = switchtec_pci_probe,
1527         .remove         = switchtec_pci_remove,
1528 };
1529
1530 static int __init switchtec_init(void)
1531 {
1532         int rc;
1533
1534         rc = alloc_chrdev_region(&switchtec_devt, 0, max_devices,
1535                                  "switchtec");
1536         if (rc)
1537                 return rc;
1538
1539         switchtec_class = class_create(THIS_MODULE, "switchtec");
1540         if (IS_ERR(switchtec_class)) {
1541                 rc = PTR_ERR(switchtec_class);
1542                 goto err_create_class;
1543         }
1544
1545         rc = pci_register_driver(&switchtec_pci_driver);
1546         if (rc)
1547                 goto err_pci_register;
1548
1549         pr_info(KBUILD_MODNAME ": loaded.\n");
1550
1551         return 0;
1552
1553 err_pci_register:
1554         class_destroy(switchtec_class);
1555
1556 err_create_class:
1557         unregister_chrdev_region(switchtec_devt, max_devices);
1558
1559         return rc;
1560 }
1561 module_init(switchtec_init);
1562
1563 static void __exit switchtec_exit(void)
1564 {
1565         pci_unregister_driver(&switchtec_pci_driver);
1566         class_destroy(switchtec_class);
1567         unregister_chrdev_region(switchtec_devt, max_devices);
1568         ida_destroy(&switchtec_minor_ida);
1569
1570         pr_info(KBUILD_MODNAME ": unloaded.\n");
1571 }
1572 module_exit(switchtec_exit);