drivers/pci/endpoint/functions/pci-epf-ntb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Endpoint Function Driver to implement Non-Transparent Bridge functionality
   4  *
   5  * Copyright (C) 2020 Texas Instruments
   6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
   7  */
   8
   9 /*
  10  * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint
  11  * (EP) controller instances (see diagram below) in such a way that
  12  * transactions from one EP controller are routed to the other EP controller.
  13  * Once PCI NTB function driver configures the SoC with multiple EP instances,
  14  * HOST1 and HOST2 can communicate with each other using SoC as a bridge.
  15  *
  16  *    +-------------+                                   +-------------+
  17  *    |             |                                   |             |
  18  *    |    HOST1    |                                   |    HOST2    |
  19  *    |             |                                   |             |
  20  *    +------^------+                                   +------^------+
  21  *           |                                                 |
  22  *           |                                                 |
  23  * +---------|-------------------------------------------------|---------+
  24  * |  +------v------+                                   +------v------+  |
  25  * |  |             |                                   |             |  |
  26  * |  |     EP      |                                   |     EP      |  |
  27  * |  | CONTROLLER1 |                                   | CONTROLLER2 |  |
  28  * |  |             <----------------------------------->             |  |
  29  * |  |             |                                   |             |  |
  30  * |  |             |                                   |             |  |
  31  * |  |             |  SoC With Multiple EP Instances   |             |  |
  32  * |  |             |  (Configured using NTB Function)  |             |  |
  33  * |  +-------------+                                   +-------------+  |
  34  * +---------------------------------------------------------------------+
  35  */
  36
  37 #include <linux/delay.h>
  38 #include <linux/io.h>
  39 #include <linux/module.h>
  40 #include <linux/slab.h>
  41
  42 #include <linux/pci-epc.h>
  43 #include <linux/pci-epf.h>
  44
  45 static struct workqueue_struct *kpcintb_workqueue;
  46
  47 #define COMMAND_CONFIGURE_DOORBELL      1
  48 #define COMMAND_TEARDOWN_DOORBELL       2
  49 #define COMMAND_CONFIGURE_MW            3
  50 #define COMMAND_TEARDOWN_MW             4
  51 #define COMMAND_LINK_UP                 5
  52 #define COMMAND_LINK_DOWN               6
  53
  54 #define COMMAND_STATUS_OK               1
  55 #define COMMAND_STATUS_ERROR            2
  56
  57 #define LINK_STATUS_UP                  BIT(0)
  58
  59 #define SPAD_COUNT                      64
  60 #define DB_COUNT                        4
  61 #define NTB_MW_OFFSET                   2
  62 #define DB_COUNT_MASK                   GENMASK(15, 0)
  63 #define MSIX_ENABLE                     BIT(16)
  64 #define MAX_DB_COUNT                    32
  65 #define MAX_MW                          4
  66
  67 enum epf_ntb_bar {
  68         BAR_CONFIG,
  69         BAR_PEER_SPAD,
  70         BAR_DB_MW1,
  71         BAR_MW2,
  72         BAR_MW3,
  73         BAR_MW4,
  74 };
  75
  76 struct epf_ntb {
  77         u32 num_mws;
  78         u32 db_count;
  79         u32 spad_count;
  80         struct pci_epf *epf;
  81         u64 mws_size[MAX_MW];
  82         struct config_group group;
  83         struct epf_ntb_epc *epc[2];
  84 };
  85
  86 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
  87
  88 struct epf_ntb_epc {
  89         u8 func_no;
  90         u8 vfunc_no;
  91         bool linkup;
  92         bool is_msix;
  93         int msix_bar;
  94         u32 spad_size;
  95         struct pci_epc *epc;
  96         struct epf_ntb *epf_ntb;
  97         void __iomem *mw_addr[6];
  98         size_t msix_table_offset;
  99         struct epf_ntb_ctrl *reg;
 100         struct pci_epf_bar *epf_bar;
 101         enum pci_barno epf_ntb_bar[6];
 102         struct delayed_work cmd_handler;
 103         enum pci_epc_interface_type type;
 104         const struct pci_epc_features *epc_features;
 105 };
 106
 107 struct epf_ntb_ctrl {
 108         u32     command;
 109         u32     argument;
 110         u16     command_status;
 111         u16     link_status;
 112         u32     topology;
 113         u64     addr;
 114         u64     size;
 115         u32     num_mws;
 116         u32     mw1_offset;
 117         u32     spad_offset;
 118         u32     spad_count;
 119         u32     db_entry_size;
 120         u32     db_data[MAX_DB_COUNT];
 121         u32     db_offset[MAX_DB_COUNT];
 122 } __packed;
 123
 124 static struct pci_epf_header epf_ntb_header = {
 125         .vendorid       = PCI_ANY_ID,
 126         .deviceid       = PCI_ANY_ID,
 127         .baseclass_code = PCI_BASE_CLASS_MEMORY,
 128         .interrupt_pin  = PCI_INTERRUPT_INTA,
 129 };
 130
 131 /**
 132  * epf_ntb_link_up() - Raise link_up interrupt to both the hosts
 133  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 134  * @link_up: true or false indicating Link is UP or Down
 135  *
 136  * Once NTB function in HOST1 and the NTB function in HOST2 invoke
 137  * ntb_link_enable(), this NTB function driver will trigger a link event to
 138  * the NTB client in both the hosts.
 139  */
 140 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
 141 {
 142         enum pci_epc_interface_type type;
 143         enum pci_epc_irq_type irq_type;
 144         struct epf_ntb_epc *ntb_epc;
 145         struct epf_ntb_ctrl *ctrl;
 146         struct pci_epc *epc;
 147         u8 func_no, vfunc_no;
 148         bool is_msix;
 149         int ret;
 150
 151         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
 152                 ntb_epc = ntb->epc[type];
 153                 epc = ntb_epc->epc;
 154                 func_no = ntb_epc->func_no;
 155                 vfunc_no = ntb_epc->vfunc_no;
 156                 is_msix = ntb_epc->is_msix;
 157                 ctrl = ntb_epc->reg;
 158                 if (link_up)
 159                         ctrl->link_status |= LINK_STATUS_UP;
 160                 else
 161                         ctrl->link_status &= ~LINK_STATUS_UP;
 162                 irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI;
 163                 ret = pci_epc_raise_irq(epc, func_no, vfunc_no, irq_type, 1);
 164                 if (ret) {
 165                         dev_err(&epc->dev,
 166                                 "%s intf: Failed to raise Link Up IRQ\n",
 167                                 pci_epc_interface_string(type));
 168                         return ret;
 169                 }
 170         }
 171
 172         return 0;
 173 }
 174
 175 /**
 176  * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host
 177  *   to access the memory window of other host
 178  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 179  * @type: PRIMARY interface or SECONDARY interface
 180  * @mw: Index of the memory window (either 0, 1, 2 or 3)
 181  *
 182  * +-----------------+    +---->+----------------+-----------+-----------------+
 183  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
 184  * +-----------------+    |     +----------------+           +-----------------+
 185  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
 186  * +-----------------+----+     +----------------+         | |                 |
 187  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
 188  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
 189  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
 190  * +-----------------+    |     |----------------+     | |   |                 |
 191  * |       BAR4      |    |     |                |     | |   +-----------------+
 192  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
 193  * |       BAR5      |    |     |                |   | |     +-----------------+
 194  * +-----------------+    +---->-----------------+   | |     |                 |
 195  *   EP CONTROLLER 1            |                |   | |     +-----------------+
 196  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
 197  *                              +----------------+   |       +-----------------+
 198  *                      (A)      EP CONTROLLER 2     |       |                 |
 199  *                                 (OB SPACE)        |       |                 |
 200  *                                                   +------->      MW1        |
 201  *                                                           |                 |
 202  *                                                           |                 |
 203  *                                                   (B)     +-----------------+
 204  *                                                           |                 |
 205  *                                                           |                 |
 206  *                                                           |                 |
 207  *                                                           |                 |
 208  *                                                           |                 |
 209  *                                                           +-----------------+
 210  *                                                           PCI Address Space
 211  *                                                           (Managed by HOST2)
 212  *
 213  * This function performs stage (B) in the above diagram (see MW1) i.e., map OB
 214  * address space of memory window to PCI address space.
 215  *
 216  * This operation requires 3 parameters
 217  *  1) Address in the outbound address space
 218  *  2) Address in the PCI Address space
 219  *  3) Size of the address region to be mapped
 220  *
 221  * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is
 222  * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3
 223  * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This
 224  * is populated in epf_ntb_alloc_peer_mem() in this driver.
 225  *
 226  * The address and size of the PCI address region that has to be mapped would
 227  * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is
 228  * connected to HOST2.
 229  *
 230  * Please note Memory window1 (MW1) and Doorbell registers together will be
 231  * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's
 232  * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1],
 233  * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2].
 234  */
 235 static int epf_ntb_configure_mw(struct epf_ntb *ntb,
 236                                 enum pci_epc_interface_type type, u32 mw)
 237 {
 238         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 239         struct pci_epf_bar *peer_epf_bar;
 240         enum pci_barno peer_barno;
 241         struct epf_ntb_ctrl *ctrl;
 242         phys_addr_t phys_addr;
 243         u8 func_no, vfunc_no;
 244         struct pci_epc *epc;
 245         u64 addr, size;
 246         int ret = 0;
 247
 248         ntb_epc = ntb->epc[type];
 249         epc = ntb_epc->epc;
 250
 251         peer_ntb_epc = ntb->epc[!type];
 252         peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
 253         peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
 254
 255         phys_addr = peer_epf_bar->phys_addr;
 256         ctrl = ntb_epc->reg;
 257         addr = ctrl->addr;
 258         size = ctrl->size;
 259         if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
 260                 phys_addr += ctrl->mw1_offset;
 261
 262         if (size > ntb->mws_size[mw]) {
 263                 dev_err(&epc->dev,
 264                         "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n",
 265                         pci_epc_interface_string(type), mw, size,
 266                         ntb->mws_size[mw]);
 267                 ret = -EINVAL;
 268                 goto err_invalid_size;
 269         }
 270
 271         func_no = ntb_epc->func_no;
 272         vfunc_no = ntb_epc->vfunc_no;
 273
 274         ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size);
 275         if (ret)
 276                 dev_err(&epc->dev,
 277                         "%s intf: Failed to map memory window %d address\n",
 278                         pci_epc_interface_string(type), mw);
 279
 280 err_invalid_size:
 281
 282         return ret;
 283 }
 284
 285 /**
 286  * epf_ntb_teardown_mw() - Teardown the configured OB ATU
 287  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 288  * @type: PRIMARY interface or SECONDARY interface
 289  * @mw: Index of the memory window (either 0, 1, 2 or 3)
 290  *
 291  * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
 292  * pci_epc_unmap_addr()
 293  */
 294 static void epf_ntb_teardown_mw(struct epf_ntb *ntb,
 295                                 enum pci_epc_interface_type type, u32 mw)
 296 {
 297         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 298         struct pci_epf_bar *peer_epf_bar;
 299         enum pci_barno peer_barno;
 300         struct epf_ntb_ctrl *ctrl;
 301         phys_addr_t phys_addr;
 302         u8 func_no, vfunc_no;
 303         struct pci_epc *epc;
 304
 305         ntb_epc = ntb->epc[type];
 306         epc = ntb_epc->epc;
 307
 308         peer_ntb_epc = ntb->epc[!type];
 309         peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
 310         peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
 311
 312         phys_addr = peer_epf_bar->phys_addr;
 313         ctrl = ntb_epc->reg;
 314         if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
 315                 phys_addr += ctrl->mw1_offset;
 316         func_no = ntb_epc->func_no;
 317         vfunc_no = ntb_epc->vfunc_no;
 318
 319         pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);
 320 }
 321
 322 /**
 323  * epf_ntb_configure_msi() - Map OB address space to MSI address
 324  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 325  * @type: PRIMARY interface or SECONDARY interface
 326  * @db_count: Number of doorbell interrupts to map
 327  *
 328  *+-----------------+    +----->+----------------+-----------+-----------------+
 329  *|       BAR0      |    |      |   Doorbell 1   +---+------->   MSI ADDRESS   |
 330  *+-----------------+    |      +----------------+   |       +-----------------+
 331  *|       BAR1      |    |      |   Doorbell 2   +---+       |                 |
 332  *+-----------------+----+      +----------------+   |       |                 |
 333  *|       BAR2      |           |   Doorbell 3   +---+       |                 |
 334  *+-----------------+----+      +----------------+   |       |                 |
 335  *|       BAR3      |    |      |   Doorbell 4   +---+       |                 |
 336  *+-----------------+    |      |----------------+           |                 |
 337  *|       BAR4      |    |      |                |           |                 |
 338  *+-----------------+    |      |      MW1       |           |                 |
 339  *|       BAR5      |    |      |                |           |                 |
 340  *+-----------------+    +----->-----------------+           |                 |
 341  *  EP CONTROLLER 1             |                |           |                 |
 342  *                              |                |           |                 |
 343  *                              +----------------+           +-----------------+
 344  *                     (A)       EP CONTROLLER 2             |                 |
 345  *                                 (OB SPACE)                |                 |
 346  *                                                           |      MW1        |
 347  *                                                           |                 |
 348  *                                                           |                 |
 349  *                                                   (B)     +-----------------+
 350  *                                                           |                 |
 351  *                                                           |                 |
 352  *                                                           |                 |
 353  *                                                           |                 |
 354  *                                                           |                 |
 355  *                                                           +-----------------+
 356  *                                                           PCI Address Space
 357  *                                                           (Managed by HOST2)
 358  *
 359  *
 360  * This function performs stage (B) in the above diagram (see Doorbell 1,
 361  * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
 362  * doorbell to MSI address in PCI address space.
 363  *
 364  * This operation requires 3 parameters
 365  *  1) Address reserved for doorbell in the outbound address space
 366  *  2) MSI-X address in the PCIe Address space
 367  *  3) Number of MSI-X interrupts that has to be configured
 368  *
 369  * The address in the outbound address space (for the Doorbell) is stored in
 370  * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
 371  * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
 372  * with address for MW1.
 373  *
 374  * pci_epc_map_msi_irq() takes the MSI address from MSI capability register
 375  * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI
 376  * address.
 377  *
 378  * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt
 379  * in db_data of the peer's control region. This helps the peer to raise
 380  * doorbell of the other host by writing db_data to the BAR corresponding to
 381  * BAR_DB_MW1.
 382  */
 383 static int epf_ntb_configure_msi(struct epf_ntb *ntb,
 384                                  enum pci_epc_interface_type type, u16 db_count)
 385 {
 386         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 387         u32 db_entry_size, db_data, db_offset;
 388         struct pci_epf_bar *peer_epf_bar;
 389         struct epf_ntb_ctrl *peer_ctrl;
 390         enum pci_barno peer_barno;
 391         phys_addr_t phys_addr;
 392         u8 func_no, vfunc_no;
 393         struct pci_epc *epc;
 394         int ret, i;
 395
 396         ntb_epc = ntb->epc[type];
 397         epc = ntb_epc->epc;
 398
 399         peer_ntb_epc = ntb->epc[!type];
 400         peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
 401         peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
 402         peer_ctrl = peer_ntb_epc->reg;
 403         db_entry_size = peer_ctrl->db_entry_size;
 404
 405         phys_addr = peer_epf_bar->phys_addr;
 406         func_no = ntb_epc->func_no;
 407         vfunc_no = ntb_epc->vfunc_no;
 408
 409         ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count,
 410                                   db_entry_size, &db_data, &db_offset);
 411         if (ret) {
 412                 dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n",
 413                         pci_epc_interface_string(type));
 414                 return ret;
 415         }
 416
 417         for (i = 0; i < db_count; i++) {
 418                 peer_ctrl->db_data[i] = db_data | i;
 419                 peer_ctrl->db_offset[i] = db_offset;
 420         }
 421
 422         return 0;
 423 }
 424
 425 /**
 426  * epf_ntb_configure_msix() - Map OB address space to MSI-X address
 427  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 428  * @type: PRIMARY interface or SECONDARY interface
 429  * @db_count: Number of doorbell interrupts to map
 430  *
 431  *+-----------------+    +----->+----------------+-----------+-----------------+
 432  *|       BAR0      |    |      |   Doorbell 1   +-----------> MSI-X ADDRESS 1 |
 433  *+-----------------+    |      +----------------+           +-----------------+
 434  *|       BAR1      |    |      |   Doorbell 2   +---------+ |                 |
 435  *+-----------------+----+      +----------------+         | |                 |
 436  *|       BAR2      |           |   Doorbell 3   +-------+ | +-----------------+
 437  *+-----------------+----+      +----------------+       | +-> MSI-X ADDRESS 2 |
 438  *|       BAR3      |    |      |   Doorbell 4   +-----+ |   +-----------------+
 439  *+-----------------+    |      |----------------+     | |   |                 |
 440  *|       BAR4      |    |      |                |     | |   +-----------------+
 441  *+-----------------+    |      |      MW1       +     | +-->+ MSI-X ADDRESS 3||
 442  *|       BAR5      |    |      |                |     |     +-----------------+
 443  *+-----------------+    +----->-----------------+     |     |                 |
 444  *  EP CONTROLLER 1             |                |     |     +-----------------+
 445  *                              |                |     +---->+ MSI-X ADDRESS 4 |
 446  *                              +----------------+           +-----------------+
 447  *                     (A)       EP CONTROLLER 2             |                 |
 448  *                                 (OB SPACE)                |                 |
 449  *                                                           |      MW1        |
 450  *                                                           |                 |
 451  *                                                           |                 |
 452  *                                                   (B)     +-----------------+
 453  *                                                           |                 |
 454  *                                                           |                 |
 455  *                                                           |                 |
 456  *                                                           |                 |
 457  *                                                           |                 |
 458  *                                                           +-----------------+
 459  *                                                           PCI Address Space
 460  *                                                           (Managed by HOST2)
 461  *
 462  * This function performs stage (B) in the above diagram (see Doorbell 1,
 463  * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
 464  * doorbell to MSI-X address in PCI address space.
 465  *
 466  * This operation requires 3 parameters
 467  *  1) Address reserved for doorbell in the outbound address space
 468  *  2) MSI-X address in the PCIe Address space
 469  *  3) Number of MSI-X interrupts that has to be configured
 470  *
 471  * The address in the outbound address space (for the Doorbell) is stored in
 472  * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
 473  * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
 474  * with address for MW1.
 475  *
 476  * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and
 477  * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected
 478  * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the
 479  * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix()
 480  * gets the MSI-X address and data.
 481  *
 482  * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt
 483  * in db_data of the peer's control region. This helps the peer to raise
 484  * doorbell of the other host by writing db_data to the BAR corresponding to
 485  * BAR_DB_MW1.
 486  */
 487 static int epf_ntb_configure_msix(struct epf_ntb *ntb,
 488                                   enum pci_epc_interface_type type,
 489                                   u16 db_count)
 490 {
 491         const struct pci_epc_features *epc_features;
 492         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 493         struct pci_epf_bar *peer_epf_bar, *epf_bar;
 494         struct pci_epf_msix_tbl *msix_tbl;
 495         struct epf_ntb_ctrl *peer_ctrl;
 496         u32 db_entry_size, msg_data;
 497         enum pci_barno peer_barno;
 498         phys_addr_t phys_addr;
 499         u8 func_no, vfunc_no;
 500         struct pci_epc *epc;
 501         size_t align;
 502         u64 msg_addr;
 503         int ret, i;
 504
 505         ntb_epc = ntb->epc[type];
 506         epc = ntb_epc->epc;
 507
 508         epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar];
 509         msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset;
 510
 511         peer_ntb_epc = ntb->epc[!type];
 512         peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
 513         peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
 514         phys_addr = peer_epf_bar->phys_addr;
 515         peer_ctrl = peer_ntb_epc->reg;
 516         epc_features = ntb_epc->epc_features;
 517         align = epc_features->align;
 518
 519         func_no = ntb_epc->func_no;
 520         vfunc_no = ntb_epc->vfunc_no;
 521         db_entry_size = peer_ctrl->db_entry_size;
 522
 523         for (i = 0; i < db_count; i++) {
 524                 msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align);
 525                 msg_data = msix_tbl[i].msg_data;
 526                 ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, msg_addr,
 527                                        db_entry_size);
 528                 if (ret) {
 529                         dev_err(&epc->dev,
 530                                 "%s intf: Failed to configure MSI-X IRQ\n",
 531                                 pci_epc_interface_string(type));
 532                         return ret;
 533                 }
 534                 phys_addr = phys_addr + db_entry_size;
 535                 peer_ctrl->db_data[i] = msg_data;
 536                 peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1);
 537         }
 538         ntb_epc->is_msix = true;
 539
 540         return 0;
 541 }
 542
 543 /**
 544  * epf_ntb_configure_db() - Configure the Outbound Address Space for one host
 545  *   to ring the doorbell of other host
 546  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 547  * @type: PRIMARY interface or SECONDARY interface
 548  * @db_count: Count of the number of doorbells that has to be configured
 549  * @msix: Indicates whether MSI-X or MSI should be used
 550  *
 551  * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for
 552  * one HOST to ring the doorbell of other HOST.
 553  */
 554 static int epf_ntb_configure_db(struct epf_ntb *ntb,
 555                                 enum pci_epc_interface_type type,
 556                                 u16 db_count, bool msix)
 557 {
 558         struct epf_ntb_epc *ntb_epc;
 559         struct pci_epc *epc;
 560         int ret;
 561
 562         if (db_count > MAX_DB_COUNT)
 563                 return -EINVAL;
 564
 565         ntb_epc = ntb->epc[type];
 566         epc = ntb_epc->epc;
 567
 568         if (msix)
 569                 ret = epf_ntb_configure_msix(ntb, type, db_count);
 570         else
 571                 ret = epf_ntb_configure_msi(ntb, type, db_count);
 572
 573         if (ret)
 574                 dev_err(&epc->dev, "%s intf: Failed to configure DB\n",
 575                         pci_epc_interface_string(type));
 576
 577         return ret;
 578 }
 579
 580 /**
 581  * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X
 582  *   address
 583  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 584  * @type: PRIMARY interface or SECONDARY interface
 585  *
 586  * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address.
 587  */
 588 static void
 589 epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type)
 590 {
 591         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 592         struct pci_epf_bar *peer_epf_bar;
 593         enum pci_barno peer_barno;
 594         phys_addr_t phys_addr;
 595         u8 func_no, vfunc_no;
 596         struct pci_epc *epc;
 597
 598         ntb_epc = ntb->epc[type];
 599         epc = ntb_epc->epc;
 600
 601         peer_ntb_epc = ntb->epc[!type];
 602         peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
 603         peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
 604         phys_addr = peer_epf_bar->phys_addr;
 605         func_no = ntb_epc->func_no;
 606         vfunc_no = ntb_epc->vfunc_no;
 607
 608         pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);
 609 }
 610
 611 /**
 612  * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host
 613  * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY)
 614  *
 615  * Workqueue function that gets invoked for the two epf_ntb_epc
 616  * periodically (once every 5ms) to see if it has received any commands
 617  * from NTB host. The host can send commands to configure doorbell or
 618  * configure memory window or to update link status.
 619  */
 620 static void epf_ntb_cmd_handler(struct work_struct *work)
 621 {
 622         enum pci_epc_interface_type type;
 623         struct epf_ntb_epc *ntb_epc;
 624         struct epf_ntb_ctrl *ctrl;
 625         u32 command, argument;
 626         struct epf_ntb *ntb;
 627         struct device *dev;
 628         u16 db_count;
 629         bool is_msix;
 630         int ret;
 631
 632         ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work);
 633         ctrl = ntb_epc->reg;
 634         command = ctrl->command;
 635         if (!command)
 636                 goto reset_handler;
 637         argument = ctrl->argument;
 638
 639         ctrl->command = 0;
 640         ctrl->argument = 0;
 641
 642         ctrl = ntb_epc->reg;
 643         type = ntb_epc->type;
 644         ntb = ntb_epc->epf_ntb;
 645         dev = &ntb->epf->dev;
 646
 647         switch (command) {
 648         case COMMAND_CONFIGURE_DOORBELL:
 649                 db_count = argument & DB_COUNT_MASK;
 650                 is_msix = argument & MSIX_ENABLE;
 651                 ret = epf_ntb_configure_db(ntb, type, db_count, is_msix);
 652                 if (ret < 0)
 653                         ctrl->command_status = COMMAND_STATUS_ERROR;
 654                 else
 655                         ctrl->command_status = COMMAND_STATUS_OK;
 656                 break;
 657         case COMMAND_TEARDOWN_DOORBELL:
 658                 epf_ntb_teardown_db(ntb, type);
 659                 ctrl->command_status = COMMAND_STATUS_OK;
 660                 break;
 661         case COMMAND_CONFIGURE_MW:
 662                 ret = epf_ntb_configure_mw(ntb, type, argument);
 663                 if (ret < 0)
 664                         ctrl->command_status = COMMAND_STATUS_ERROR;
 665                 else
 666                         ctrl->command_status = COMMAND_STATUS_OK;
 667                 break;
 668         case COMMAND_TEARDOWN_MW:
 669                 epf_ntb_teardown_mw(ntb, type, argument);
 670                 ctrl->command_status = COMMAND_STATUS_OK;
 671                 break;
 672         case COMMAND_LINK_UP:
 673                 ntb_epc->linkup = true;
 674                 if (ntb->epc[PRIMARY_INTERFACE]->linkup &&
 675                     ntb->epc[SECONDARY_INTERFACE]->linkup) {
 676                         ret = epf_ntb_link_up(ntb, true);
 677                         if (ret < 0)
 678                                 ctrl->command_status = COMMAND_STATUS_ERROR;
 679                         else
 680                                 ctrl->command_status = COMMAND_STATUS_OK;
 681                         goto reset_handler;
 682                 }
 683                 ctrl->command_status = COMMAND_STATUS_OK;
 684                 break;
 685         case COMMAND_LINK_DOWN:
 686                 ntb_epc->linkup = false;
 687                 ret = epf_ntb_link_up(ntb, false);
 688                 if (ret < 0)
 689                         ctrl->command_status = COMMAND_STATUS_ERROR;
 690                 else
 691                         ctrl->command_status = COMMAND_STATUS_OK;
 692                 break;
 693         default:
 694                 dev_err(dev, "%s intf UNKNOWN command: %d\n",
 695                         pci_epc_interface_string(type), command);
 696                 break;
 697         }
 698
 699 reset_handler:
 700         queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler,
 701                            msecs_to_jiffies(5));
 702 }
 703
 704 /**
 705  * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR
 706  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
 707  *           address.
 708  *
 709  *+-----------------+------->+------------------+        +-----------------+
 710  *|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
 711  *+-----------------+----+   +------------------+<-------+-----------------+
 712  *|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
 713  *+-----------------+    +-->+------------------+<-------+-----------------+
 714  *|       BAR2      |            Local Memory            |       BAR2      |
 715  *+-----------------+                                    +-----------------+
 716  *|       BAR3      |                                    |       BAR3      |
 717  *+-----------------+                                    +-----------------+
 718  *|       BAR4      |                                    |       BAR4      |
 719  *+-----------------+                                    +-----------------+
 720  *|       BAR5      |                                    |       BAR5      |
 721  *+-----------------+                                    +-----------------+
 722  *  EP CONTROLLER 1                                        EP CONTROLLER 2
 723  *
 724  * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
 725  * region. While BAR1 is the default peer scratchpad BAR, an NTB could have
 726  * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
 727  * This function can get the exact BAR used for peer scratchpad from
 728  * epf_ntb_bar[BAR_PEER_SPAD].
 729  *
 730  * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
 731  * gets the address of peer scratchpad from
 732  * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
 733  */
 734 static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc)
 735 {
 736         struct pci_epf_bar *epf_bar;
 737         enum pci_barno barno;
 738         u8 func_no, vfunc_no;
 739         struct pci_epc *epc;
 740
 741         epc = ntb_epc->epc;
 742         func_no = ntb_epc->func_no;
 743         vfunc_no = ntb_epc->vfunc_no;
 744         barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
 745         epf_bar = &ntb_epc->epf_bar[barno];
 746         pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
 747 }
 748
 749 /**
 750  * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR
 751  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 752  * @type: PRIMARY interface or SECONDARY interface
 753  *
 754  *+-----------------+------->+------------------+        +-----------------+
 755  *|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
 756  *+-----------------+----+   +------------------+<-------+-----------------+
 757  *|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
 758  *+-----------------+    +-->+------------------+<-------+-----------------+
 759  *|       BAR2      |            Local Memory            |       BAR2      |
 760  *+-----------------+                                    +-----------------+
 761  *|       BAR3      |                                    |       BAR3      |
 762  *+-----------------+                                    +-----------------+
 763  *|       BAR4      |                                    |       BAR4      |
 764  *+-----------------+                                    +-----------------+
 765  *|       BAR5      |                                    |       BAR5      |
 766  *+-----------------+                                    +-----------------+
 767  *  EP CONTROLLER 1                                        EP CONTROLLER 2
 768  *
 769  * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
 770  * region. While BAR1 is the default peer scratchpad BAR, an NTB could have
 771  * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
 772  * This function can get the exact BAR used for peer scratchpad from
 773  * epf_ntb_bar[BAR_PEER_SPAD].
 774  *
 775  * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
 776  * gets the address of peer scratchpad from
 777  * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
 778  */
 779 static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb,
 780                                      enum pci_epc_interface_type type)
 781 {
 782         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 783         struct pci_epf_bar *peer_epf_bar, *epf_bar;
 784         enum pci_barno peer_barno, barno;
 785         u32 peer_spad_offset;
 786         u8 func_no, vfunc_no;
 787         struct pci_epc *epc;
 788         struct device *dev;
 789         int ret;
 790
 791         dev = &ntb->epf->dev;
 792
 793         peer_ntb_epc = ntb->epc[!type];
 794         peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG];
 795         peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
 796
 797         ntb_epc = ntb->epc[type];
 798         barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
 799         epf_bar = &ntb_epc->epf_bar[barno];
 800         func_no = ntb_epc->func_no;
 801         vfunc_no = ntb_epc->vfunc_no;
 802         epc = ntb_epc->epc;
 803
 804         peer_spad_offset = peer_ntb_epc->reg->spad_offset;
 805         epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset;
 806         epf_bar->size = peer_ntb_epc->spad_size;
 807         epf_bar->barno = barno;
 808         epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
 809
 810         ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
 811         if (ret) {
 812                 dev_err(dev, "%s intf: peer SPAD BAR set failed\n",
 813                         pci_epc_interface_string(type));
 814                 return ret;
 815         }
 816
 817         return 0;
 818 }
 819
 820 /**
 821  * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
 822  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
 823  *           address.
 824  *
 825  * +-----------------+------->+------------------+        +-----------------+
 826  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
 827  * +-----------------+----+   +------------------+<-------+-----------------+
 828  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
 829  * +-----------------+    +-->+------------------+<-------+-----------------+
 830  * |       BAR2      |            Local Memory            |       BAR2      |
 831  * +-----------------+                                    +-----------------+
 832  * |       BAR3      |                                    |       BAR3      |
 833  * +-----------------+                                    +-----------------+
 834  * |       BAR4      |                                    |       BAR4      |
 835  * +-----------------+                                    +-----------------+
 836  * |       BAR5      |                                    |       BAR5      |
 837  * +-----------------+                                    +-----------------+
 838  *   EP CONTROLLER 1                                        EP CONTROLLER 2
 839  *
 840  * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
 841  * self scratchpad region (removes inbound ATU configuration). While BAR0 is
 842  * the default self scratchpad BAR, an NTB could have other BARs for self
 843  * scratchpad (because of reserved BARs). This function can get the exact BAR
 844  * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
 845  *
 846  * Please note the self scratchpad region and config region is combined to
 847  * a single region and mapped using the same BAR. Also note HOST2's peer
 848  * scratchpad is HOST1's self scratchpad.
 849  */
 850 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc)
 851 {
 852         struct pci_epf_bar *epf_bar;
 853         enum pci_barno barno;
 854         u8 func_no, vfunc_no;
 855         struct pci_epc *epc;
 856
 857         epc = ntb_epc->epc;
 858         func_no = ntb_epc->func_no;
 859         vfunc_no = ntb_epc->vfunc_no;
 860         barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
 861         epf_bar = &ntb_epc->epf_bar[barno];
 862         pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
 863 }
 864
 865 /**
 866  * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
 867  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
 868  *           address.
 869  *
 870  * +-----------------+------->+------------------+        +-----------------+
 871  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
 872  * +-----------------+----+   +------------------+<-------+-----------------+
 873  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
 874  * +-----------------+    +-->+------------------+<-------+-----------------+
 875  * |       BAR2      |            Local Memory            |       BAR2      |
 876  * +-----------------+                                    +-----------------+
 877  * |       BAR3      |                                    |       BAR3      |
 878  * +-----------------+                                    +-----------------+
 879  * |       BAR4      |                                    |       BAR4      |
 880  * +-----------------+                                    +-----------------+
 881  * |       BAR5      |                                    |       BAR5      |
 882  * +-----------------+                                    +-----------------+
 883  *   EP CONTROLLER 1                                        EP CONTROLLER 2
 884  *
 885  * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
 886  * self scratchpad region. While BAR0 is the default self scratchpad BAR, an
 887  * NTB could have other BARs for self scratchpad (because of reserved BARs).
 888  * This function can get the exact BAR used for self scratchpad from
 889  * epf_ntb_bar[BAR_CONFIG].
 890  *
 891  * Please note the self scratchpad region and config region is combined to
 892  * a single region and mapped using the same BAR. Also note HOST2's peer
 893  * scratchpad is HOST1's self scratchpad.
 894  */
 895 static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc)
 896 {
 897         struct pci_epf_bar *epf_bar;
 898         enum pci_barno barno;
 899         u8 func_no, vfunc_no;
 900         struct epf_ntb *ntb;
 901         struct pci_epc *epc;
 902         struct device *dev;
 903         int ret;
 904
 905         ntb = ntb_epc->epf_ntb;
 906         dev = &ntb->epf->dev;
 907
 908         epc = ntb_epc->epc;
 909         func_no = ntb_epc->func_no;
 910         vfunc_no = ntb_epc->vfunc_no;
 911         barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
 912         epf_bar = &ntb_epc->epf_bar[barno];
 913
 914         ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
 915         if (ret) {
 916                 dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n",
 917                         pci_epc_interface_string(ntb_epc->type));
 918                 return ret;
 919         }
 920
 921         return 0;
 922 }
 923
 924 /**
 925  * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
 926  *   config + scratchpad region
 927  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 928  *
 929  * +-----------------+------->+------------------+        +-----------------+
 930  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
 931  * +-----------------+----+   +------------------+<-------+-----------------+
 932  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
 933  * +-----------------+    +-->+------------------+<-------+-----------------+
 934  * |       BAR2      |            Local Memory            |       BAR2      |
 935  * +-----------------+                                    +-----------------+
 936  * |       BAR3      |                                    |       BAR3      |
 937  * +-----------------+                                    +-----------------+
 938  * |       BAR4      |                                    |       BAR4      |
 939  * +-----------------+                                    +-----------------+
 940  * |       BAR5      |                                    |       BAR5      |
 941  * +-----------------+                                    +-----------------+
 942  *   EP CONTROLLER 1                                        EP CONTROLLER 2
 943  *
 944  * Free the Local Memory mentioned in the above diagram. After invoking this
 945  * function, any of config + self scratchpad region of HOST1 or peer scratchpad
 946  * region of HOST2 should not be accessed.
 947  */
 948 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
 949 {
 950         enum pci_epc_interface_type type;
 951         struct epf_ntb_epc *ntb_epc;
 952         enum pci_barno barno;
 953         struct pci_epf *epf;
 954
 955         epf = ntb->epf;
 956         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
 957                 ntb_epc = ntb->epc[type];
 958                 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
 959                 if (ntb_epc->reg)
 960                         pci_epf_free_space(epf, ntb_epc->reg, barno, type);
 961         }
 962 }
 963
 964 /**
 965  * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
 966  *   region
 967  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
 968  * @type: PRIMARY interface or SECONDARY interface
 969  *
 970  * +-----------------+------->+------------------+        +-----------------+
 971  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
 972  * +-----------------+----+   +------------------+<-------+-----------------+
 973  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
 974  * +-----------------+    +-->+------------------+<-------+-----------------+
 975  * |       BAR2      |            Local Memory            |       BAR2      |
 976  * +-----------------+                                    +-----------------+
 977  * |       BAR3      |                                    |       BAR3      |
 978  * +-----------------+                                    +-----------------+
 979  * |       BAR4      |                                    |       BAR4      |
 980  * +-----------------+                                    +-----------------+
 981  * |       BAR5      |                                    |       BAR5      |
 982  * +-----------------+                                    +-----------------+
 983  *   EP CONTROLLER 1                                        EP CONTROLLER 2
 984  *
 985  * Allocate the Local Memory mentioned in the above diagram. The size of
 986  * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
 987  * is obtained from "spad-count" configfs entry.
 988  *
 989  * The size of both config region and scratchpad region has to be aligned,
 990  * since the scratchpad region will also be mapped as PEER SCRATCHPAD of
 991  * other host using a separate BAR.
 992  */
 993 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb,
 994                                          enum pci_epc_interface_type type)
 995 {
 996         const struct pci_epc_features *peer_epc_features, *epc_features;
 997         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
 998         size_t msix_table_size, pba_size, align;
 999         enum pci_barno peer_barno, barno;
1000         struct epf_ntb_ctrl *ctrl;
1001         u32 spad_size, ctrl_size;
1002         u64 size, peer_size;
1003         struct pci_epf *epf;
1004         struct device *dev;
1005         bool msix_capable;
1006         u32 spad_count;
1007         void *base;
1008
1009         epf = ntb->epf;
1010         dev = &epf->dev;
1011         ntb_epc = ntb->epc[type];
1012
1013         epc_features = ntb_epc->epc_features;
1014         barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
1015         size = epc_features->bar_fixed_size[barno];
1016         align = epc_features->align;
1017
1018         peer_ntb_epc = ntb->epc[!type];
1019         peer_epc_features = peer_ntb_epc->epc_features;
1020         peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
1021         peer_size = peer_epc_features->bar_fixed_size[peer_barno];
1022
1023         /* Check if epc_features is populated incorrectly */
1024         if ((!IS_ALIGNED(size, align)))
1025                 return -EINVAL;
1026
1027         spad_count = ntb->spad_count;
1028
1029         ctrl_size = sizeof(struct epf_ntb_ctrl);
1030         spad_size = spad_count * 4;
1031
1032         msix_capable = epc_features->msix_capable;
1033         if (msix_capable) {
1034                 msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count;
1035                 ctrl_size = ALIGN(ctrl_size, 8);
1036                 ntb_epc->msix_table_offset = ctrl_size;
1037                 ntb_epc->msix_bar = barno;
1038                 /* Align to QWORD or 8 Bytes */
1039                 pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8);
1040                 ctrl_size = ctrl_size + msix_table_size + pba_size;
1041         }
1042
1043         if (!align) {
1044                 ctrl_size = roundup_pow_of_two(ctrl_size);
1045                 spad_size = roundup_pow_of_two(spad_size);
1046         } else {
1047                 ctrl_size = ALIGN(ctrl_size, align);
1048                 spad_size = ALIGN(spad_size, align);
1049         }
1050
1051         if (peer_size) {
1052                 if (peer_size < spad_size)
1053                         spad_count = peer_size / 4;
1054                 spad_size = peer_size;
1055         }
1056
1057         /*
1058          * In order to make sure SPAD offset is aligned to its size,
1059          * expand control region size to the size of SPAD if SPAD size
1060          * is greater than control region size.
1061          */
1062         if (spad_size > ctrl_size)
1063                 ctrl_size = spad_size;
1064
1065         if (!size)
1066                 size = ctrl_size + spad_size;
1067         else if (size < ctrl_size + spad_size)
1068                 return -EINVAL;
1069
1070         base = pci_epf_alloc_space(epf, size, barno, align, type);
1071         if (!base) {
1072                 dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n",
1073                         pci_epc_interface_string(type));
1074                 return -ENOMEM;
1075         }
1076
1077         ntb_epc->reg = base;
1078
1079         ctrl = ntb_epc->reg;
1080         ctrl->spad_offset = ctrl_size;
1081         ctrl->spad_count = spad_count;
1082         ctrl->num_mws = ntb->num_mws;
1083         ctrl->db_entry_size = align ? align : 4;
1084         ntb_epc->spad_size = spad_size;
1085
1086         return 0;
1087 }
1088
1089 /**
1090  * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config +
1091  *   scratchpad region for each of PRIMARY and SECONDARY interface
1092  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1093  *
1094  * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for
1095  * config + scratchpad region for a specific interface
1096  */
1097 static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb)
1098 {
1099         enum pci_epc_interface_type type;
1100         struct device *dev;
1101         int ret;
1102
1103         dev = &ntb->epf->dev;
1104
1105         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1106                 ret = epf_ntb_config_spad_bar_alloc(ntb, type);
1107                 if (ret) {
1108                         dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n",
1109                                 pci_epc_interface_string(type));
1110                         return ret;
1111                 }
1112         }
1113
1114         return 0;
1115 }
1116
1117 /**
1118  * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address
1119  *   space
1120  * @ntb_epc: EPC associated with one of the HOST which holds peers outbound
1121  *   address regions
1122  *
1123  * +-----------------+    +---->+----------------+-----------+-----------------+
1124  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1125  * +-----------------+    |     +----------------+           +-----------------+
1126  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1127  * +-----------------+----+     +----------------+         | |                 |
1128  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1129  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1130  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1131  * +-----------------+    |     |----------------+     | |   |                 |
1132  * |       BAR4      |    |     |                |     | |   +-----------------+
1133  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1134  * |       BAR5      |    |     |                |   | |     +-----------------+
1135  * +-----------------+    +---->-----------------+   | |     |                 |
1136  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1137  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1138  *                              +----------------+   |       +-----------------+
1139  *                      (A)      EP CONTROLLER 2     |       |                 |
1140  *                                 (OB SPACE)        |       |                 |
1141  *                                                   +------->      MW1        |
1142  *                                                           |                 |
1143  *                                                           |                 |
1144  *                                                   (B)     +-----------------+
1145  *                                                           |                 |
1146  *                                                           |                 |
1147  *                                                           |                 |
1148  *                                                           |                 |
1149  *                                                           |                 |
1150  *                                                           +-----------------+
1151  *                                                           PCI Address Space
1152  *                                                           (Managed by HOST2)
1153  *
1154  * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram.
1155  * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3,
1156  * MW4).
1157  */
1158 static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc)
1159 {
1160         struct pci_epf_bar *epf_bar;
1161         void __iomem *mw_addr;
1162         phys_addr_t phys_addr;
1163         enum epf_ntb_bar bar;
1164         enum pci_barno barno;
1165         struct pci_epc *epc;
1166         size_t size;
1167
1168         epc = ntb_epc->epc;
1169
1170         for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1171                 barno = ntb_epc->epf_ntb_bar[bar];
1172                 mw_addr = ntb_epc->mw_addr[barno];
1173                 epf_bar = &ntb_epc->epf_bar[barno];
1174                 phys_addr = epf_bar->phys_addr;
1175                 size = epf_bar->size;
1176                 if (mw_addr) {
1177                         pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size);
1178                         ntb_epc->mw_addr[barno] = NULL;
1179                 }
1180         }
1181 }
1182
1183 /**
1184  * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR
1185  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
1186  *   address
1187  *
1188  * +-----------------+    +---->+----------------+-----------+-----------------+
1189  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1190  * +-----------------+    |     +----------------+           +-----------------+
1191  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1192  * +-----------------+----+     +----------------+         | |                 |
1193  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1194  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1195  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1196  * +-----------------+    |     |----------------+     | |   |                 |
1197  * |       BAR4      |    |     |                |     | |   +-----------------+
1198  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1199  * |       BAR5      |    |     |                |   | |     +-----------------+
1200  * +-----------------+    +---->-----------------+   | |     |                 |
1201  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1202  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1203  *                              +----------------+   |       +-----------------+
1204  *                      (A)      EP CONTROLLER 2     |       |                 |
1205  *                                 (OB SPACE)        |       |                 |
1206  *                                                   +------->      MW1        |
1207  *                                                           |                 |
1208  *                                                           |                 |
1209  *                                                   (B)     +-----------------+
1210  *                                                           |                 |
1211  *                                                           |                 |
1212  *                                                           |                 |
1213  *                                                           |                 |
1214  *                                                           |                 |
1215  *                                                           +-----------------+
1216  *                                                           PCI Address Space
1217  *                                                           (Managed by HOST2)
1218  *
1219  * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above
1220  * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2
1221  * BAR, MW3 BAR and MW4 BAR).
1222  */
1223 static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc)
1224 {
1225         struct pci_epf_bar *epf_bar;
1226         enum epf_ntb_bar bar;
1227         enum pci_barno barno;
1228         u8 func_no, vfunc_no;
1229         struct pci_epc *epc;
1230
1231         epc = ntb_epc->epc;
1232
1233         func_no = ntb_epc->func_no;
1234         vfunc_no = ntb_epc->vfunc_no;
1235
1236         for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1237                 barno = ntb_epc->epf_ntb_bar[bar];
1238                 epf_bar = &ntb_epc->epf_bar[barno];
1239                 pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
1240         }
1241 }
1242
1243 /**
1244  * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory
1245  *   allocated in peers outbound address space
1246  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1247  * @type: PRIMARY interface or SECONDARY interface
1248  *
1249  * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and
1250  * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory.
1251  */
1252 static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb,
1253                                       enum pci_epc_interface_type type)
1254 {
1255         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1256
1257         ntb_epc = ntb->epc[type];
1258         peer_ntb_epc = ntb->epc[!type];
1259
1260         epf_ntb_db_mw_bar_clear(ntb_epc);
1261         epf_ntb_free_peer_mem(peer_ntb_epc);
1262 }
1263
1264 /**
1265  * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity
1266  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1267  * @type: PRIMARY interface or SECONDARY interface
1268  *
1269  * Configure MSI/MSI-X capability for each interface with number of
1270  * interrupts equal to "db_count" configfs entry.
1271  */
1272 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
1273                                        enum pci_epc_interface_type type)
1274 {
1275         const struct pci_epc_features *epc_features;
1276         bool msix_capable, msi_capable;
1277         struct epf_ntb_epc *ntb_epc;
1278         u8 func_no, vfunc_no;
1279         struct pci_epc *epc;
1280         struct device *dev;
1281         u32 db_count;
1282         int ret;
1283
1284         ntb_epc = ntb->epc[type];
1285         dev = &ntb->epf->dev;
1286
1287         epc_features = ntb_epc->epc_features;
1288         msix_capable = epc_features->msix_capable;
1289         msi_capable = epc_features->msi_capable;
1290
1291         if (!(msix_capable || msi_capable)) {
1292                 dev_err(dev, "MSI or MSI-X is required for doorbell\n");
1293                 return -EINVAL;
1294         }
1295
1296         func_no = ntb_epc->func_no;
1297         vfunc_no = ntb_epc->vfunc_no;
1298
1299         db_count = ntb->db_count;
1300         if (db_count > MAX_DB_COUNT) {
1301                 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
1302                 return -EINVAL;
1303         }
1304
1305         ntb->db_count = db_count;
1306         epc = ntb_epc->epc;
1307
1308         if (msi_capable) {
1309                 ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count);
1310                 if (ret) {
1311                         dev_err(dev, "%s intf: MSI configuration failed\n",
1312                                 pci_epc_interface_string(type));
1313                         return ret;
1314                 }
1315         }
1316
1317         if (msix_capable) {
1318                 ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count,
1319                                        ntb_epc->msix_bar,
1320                                        ntb_epc->msix_table_offset);
1321                 if (ret) {
1322                         dev_err(dev, "MSI configuration failed\n");
1323                         return ret;
1324                 }
1325         }
1326
1327         return 0;
1328 }
1329
1330 /**
1331  * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space
1332  * @dev: The PCI device.
1333  * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound
1334  *   address
1335  * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be
1336  *   BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4)
1337  * @peer_ntb_epc: EPC associated with HOST whose outbound address space is
1338  *   used by @ntb_epc
1339  * @size: Size of the address region that has to be allocated in peers OB SPACE
1340  *
1341  *
1342  * +-----------------+    +---->+----------------+-----------+-----------------+
1343  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1344  * +-----------------+    |     +----------------+           +-----------------+
1345  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1346  * +-----------------+----+     +----------------+         | |                 |
1347  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1348  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1349  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1350  * +-----------------+    |     |----------------+     | |   |                 |
1351  * |       BAR4      |    |     |                |     | |   +-----------------+
1352  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1353  * |       BAR5      |    |     |                |   | |     +-----------------+
1354  * +-----------------+    +---->-----------------+   | |     |                 |
1355  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1356  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1357  *                              +----------------+   |       +-----------------+
1358  *                      (A)      EP CONTROLLER 2     |       |                 |
1359  *                                 (OB SPACE)        |       |                 |
1360  *                                                   +------->      MW1        |
1361  *                                                           |                 |
1362  *                                                           |                 |
1363  *                                                   (B)     +-----------------+
1364  *                                                           |                 |
1365  *                                                           |                 |
1366  *                                                           |                 |
1367  *                                                           |                 |
1368  *                                                           |                 |
1369  *                                                           +-----------------+
1370  *                                                           PCI Address Space
1371  *                                                           (Managed by HOST2)
1372  *
1373  * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate
1374  * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4).
1375  */
1376 static int epf_ntb_alloc_peer_mem(struct device *dev,
1377                                   struct epf_ntb_epc *ntb_epc,
1378                                   enum epf_ntb_bar bar,
1379                                   struct epf_ntb_epc *peer_ntb_epc,
1380                                   size_t size)
1381 {
1382         const struct pci_epc_features *epc_features;
1383         struct pci_epf_bar *epf_bar;
1384         struct pci_epc *peer_epc;
1385         phys_addr_t phys_addr;
1386         void __iomem *mw_addr;
1387         enum pci_barno barno;
1388         size_t align;
1389
1390         epc_features = ntb_epc->epc_features;
1391         align = epc_features->align;
1392
1393         if (size < 128)
1394                 size = 128;
1395
1396         if (align)
1397                 size = ALIGN(size, align);
1398         else
1399                 size = roundup_pow_of_two(size);
1400
1401         peer_epc = peer_ntb_epc->epc;
1402         mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size);
1403         if (!mw_addr) {
1404                 dev_err(dev, "%s intf: Failed to allocate OB address\n",
1405                         pci_epc_interface_string(peer_ntb_epc->type));
1406                 return -ENOMEM;
1407         }
1408
1409         barno = ntb_epc->epf_ntb_bar[bar];
1410         epf_bar = &ntb_epc->epf_bar[barno];
1411         ntb_epc->mw_addr[barno] = mw_addr;
1412
1413         epf_bar->phys_addr = phys_addr;
1414         epf_bar->size = size;
1415         epf_bar->barno = barno;
1416         epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
1417
1418         return 0;
1419 }
1420
1421 /**
1422  * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs
1423  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1424  * @type: PRIMARY interface or SECONDARY interface
1425  *
1426  * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates
1427  * memory in OB address space of HOST2 and configures BAR of HOST1
1428  */
1429 static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,
1430                                   enum pci_epc_interface_type type)
1431 {
1432         const struct pci_epc_features *epc_features;
1433         struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1434         struct pci_epf_bar *epf_bar;
1435         struct epf_ntb_ctrl *ctrl;
1436         u32 num_mws, db_count;
1437         enum epf_ntb_bar bar;
1438         enum pci_barno barno;
1439         u8 func_no, vfunc_no;
1440         struct pci_epc *epc;
1441         struct device *dev;
1442         size_t align;
1443         int ret, i;
1444         u64 size;
1445
1446         ntb_epc = ntb->epc[type];
1447         peer_ntb_epc = ntb->epc[!type];
1448
1449         dev = &ntb->epf->dev;
1450         epc_features = ntb_epc->epc_features;
1451         align = epc_features->align;
1452         func_no = ntb_epc->func_no;
1453         vfunc_no = ntb_epc->vfunc_no;
1454         epc = ntb_epc->epc;
1455         num_mws = ntb->num_mws;
1456         db_count = ntb->db_count;
1457
1458         for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) {
1459                 if (bar == BAR_DB_MW1) {
1460                         align = align ? align : 4;
1461                         size = db_count * align;
1462                         size = ALIGN(size, ntb->mws_size[i]);
1463                         ctrl = ntb_epc->reg;
1464                         ctrl->mw1_offset = size;
1465                         size += ntb->mws_size[i];
1466                 } else {
1467                         size = ntb->mws_size[i];
1468                 }
1469
1470                 ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar,
1471                                              peer_ntb_epc, size);
1472                 if (ret) {
1473                         dev_err(dev, "%s intf: DoorBell mem alloc failed\n",
1474                                 pci_epc_interface_string(type));
1475                         goto err_alloc_peer_mem;
1476                 }
1477
1478                 barno = ntb_epc->epf_ntb_bar[bar];
1479                 epf_bar = &ntb_epc->epf_bar[barno];
1480
1481                 ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
1482                 if (ret) {
1483                         dev_err(dev, "%s intf: DoorBell BAR set failed\n",
1484                                 pci_epc_interface_string(type));
1485                         goto err_alloc_peer_mem;
1486                 }
1487         }
1488
1489         return 0;
1490
1491 err_alloc_peer_mem:
1492         epf_ntb_db_mw_bar_cleanup(ntb, type);
1493
1494         return ret;
1495 }
1496
1497 /**
1498  * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface
1499  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1500  * @type: PRIMARY interface or SECONDARY interface
1501  *
1502  * Unbind NTB function device from EPC and relinquish reference to pci_epc
1503  * for each of the interface.
1504  */
1505 static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb,
1506                                           enum pci_epc_interface_type type)
1507 {
1508         struct epf_ntb_epc *ntb_epc;
1509         struct pci_epc *epc;
1510         struct pci_epf *epf;
1511
1512         if (type < 0)
1513                 return;
1514
1515         epf = ntb->epf;
1516         ntb_epc = ntb->epc[type];
1517         if (!ntb_epc)
1518                 return;
1519         epc = ntb_epc->epc;
1520         pci_epc_remove_epf(epc, epf, type);
1521         pci_epc_put(epc);
1522 }
1523
1524 /**
1525  * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
1526  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1527  *
1528  * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
1529  */
1530 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
1531 {
1532         enum pci_epc_interface_type type;
1533
1534         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1535                 epf_ntb_epc_destroy_interface(ntb, type);
1536 }
1537
1538 /**
1539  * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface
1540  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1541  * @epc: struct pci_epc to which a particular NTB interface should be associated
1542  * @type: PRIMARY interface or SECONDARY interface
1543  *
1544  * Allocate memory for NTB EPC interface and initialize it.
1545  */
1546 static int epf_ntb_epc_create_interface(struct epf_ntb *ntb,
1547                                         struct pci_epc *epc,
1548                                         enum pci_epc_interface_type type)
1549 {
1550         const struct pci_epc_features *epc_features;
1551         struct pci_epf_bar *epf_bar;
1552         struct epf_ntb_epc *ntb_epc;
1553         u8 func_no, vfunc_no;
1554         struct pci_epf *epf;
1555         struct device *dev;
1556
1557         dev = &ntb->epf->dev;
1558
1559         ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL);
1560         if (!ntb_epc)
1561                 return -ENOMEM;
1562
1563         epf = ntb->epf;
1564         vfunc_no = epf->vfunc_no;
1565         if (type == PRIMARY_INTERFACE) {
1566                 func_no = epf->func_no;
1567                 epf_bar = epf->bar;
1568         } else {
1569                 func_no = epf->sec_epc_func_no;
1570                 epf_bar = epf->sec_epc_bar;
1571         }
1572
1573         ntb_epc->linkup = false;
1574         ntb_epc->epc = epc;
1575         ntb_epc->func_no = func_no;
1576         ntb_epc->vfunc_no = vfunc_no;
1577         ntb_epc->type = type;
1578         ntb_epc->epf_bar = epf_bar;
1579         ntb_epc->epf_ntb = ntb;
1580
1581         epc_features = pci_epc_get_features(epc, func_no, vfunc_no);
1582         if (!epc_features)
1583                 return -EINVAL;
1584         ntb_epc->epc_features = epc_features;
1585
1586         ntb->epc[type] = ntb_epc;
1587
1588         return 0;
1589 }
1590
1591 /**
1592  * epf_ntb_epc_create() - Create and initialize NTB EPC interface
1593  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1594  *
1595  * Get a reference to EPC device and bind NTB function device to that EPC
1596  * for each of the interface. It is also a wrapper to
1597  * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface
1598  * and initialize it
1599  */
1600 static int epf_ntb_epc_create(struct epf_ntb *ntb)
1601 {
1602         struct pci_epf *epf;
1603         struct device *dev;
1604         int ret;
1605
1606         epf = ntb->epf;
1607         dev = &epf->dev;
1608
1609         ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE);
1610         if (ret) {
1611                 dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n");
1612                 return ret;
1613         }
1614
1615         ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc,
1616                                            SECONDARY_INTERFACE);
1617         if (ret) {
1618                 dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n");
1619                 goto err_epc_create;
1620         }
1621
1622         return 0;
1623
1624 err_epc_create:
1625         epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE);
1626
1627         return ret;
1628 }
1629
1630 /**
1631  * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of
1632  *   the NTB constructs (scratchpad region, doorbell, memorywindow)
1633  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1634  * @type: PRIMARY interface or SECONDARY interface
1635  *
1636  * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD,
1637  * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4.
1638  */
1639 static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb,
1640                                           enum pci_epc_interface_type type)
1641 {
1642         const struct pci_epc_features *epc_features;
1643         struct epf_ntb_epc *ntb_epc;
1644         enum pci_barno barno;
1645         enum epf_ntb_bar bar;
1646         struct device *dev;
1647         u32 num_mws;
1648         int i;
1649
1650         barno = BAR_0;
1651         ntb_epc = ntb->epc[type];
1652         num_mws = ntb->num_mws;
1653         dev = &ntb->epf->dev;
1654         epc_features = ntb_epc->epc_features;
1655
1656         /* These are required BARs which are mandatory for NTB functionality */
1657         for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) {
1658                 barno = pci_epc_get_next_free_bar(epc_features, barno);
1659                 if (barno < 0) {
1660                         dev_err(dev, "%s intf: Fail to get NTB function BAR\n",
1661                                 pci_epc_interface_string(type));
1662                         return barno;
1663                 }
1664                 ntb_epc->epf_ntb_bar[bar] = barno;
1665         }
1666
1667         /* These are optional BARs which don't impact NTB functionality */
1668         for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) {
1669                 barno = pci_epc_get_next_free_bar(epc_features, barno);
1670                 if (barno < 0) {
1671                         ntb->num_mws = i;
1672                         dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
1673                 }
1674                 ntb_epc->epf_ntb_bar[bar] = barno;
1675         }
1676
1677         return 0;
1678 }
1679
1680 /**
1681  * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
1682  * constructs (scratchpad region, doorbell, memorywindow)
1683  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1684  *
1685  * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs
1686  * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2,
1687  * BAR_MW3 and BAR_MW4 for all the interfaces.
1688  */
1689 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
1690 {
1691         enum pci_epc_interface_type type;
1692         struct device *dev;
1693         int ret;
1694
1695         dev = &ntb->epf->dev;
1696         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1697                 ret = epf_ntb_init_epc_bar_interface(ntb, type);
1698                 if (ret) {
1699                         dev_err(dev, "Fail to init EPC bar for %s interface\n",
1700                                 pci_epc_interface_string(type));
1701                         return ret;
1702                 }
1703         }
1704
1705         return 0;
1706 }
1707
1708 /**
1709  * epf_ntb_epc_init_interface() - Initialize NTB interface
1710  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1711  * @type: PRIMARY interface or SECONDARY interface
1712  *
1713  * Wrapper to initialize a particular EPC interface and start the workqueue
1714  * to check for commands from host. This function will write to the
1715  * EP controller HW for configuring it.
1716  */
1717 static int epf_ntb_epc_init_interface(struct epf_ntb *ntb,
1718                                       enum pci_epc_interface_type type)
1719 {
1720         struct epf_ntb_epc *ntb_epc;
1721         u8 func_no, vfunc_no;
1722         struct pci_epc *epc;
1723         struct pci_epf *epf;
1724         struct device *dev;
1725         int ret;
1726
1727         ntb_epc = ntb->epc[type];
1728         epf = ntb->epf;
1729         dev = &epf->dev;
1730         epc = ntb_epc->epc;
1731         func_no = ntb_epc->func_no;
1732         vfunc_no = ntb_epc->vfunc_no;
1733
1734         ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]);
1735         if (ret) {
1736                 dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n",
1737                         pci_epc_interface_string(type));
1738                 return ret;
1739         }
1740
1741         ret = epf_ntb_peer_spad_bar_set(ntb, type);
1742         if (ret) {
1743                 dev_err(dev, "%s intf: Peer SPAD BAR init failed\n",
1744                         pci_epc_interface_string(type));
1745                 goto err_peer_spad_bar_init;
1746         }
1747
1748         ret = epf_ntb_configure_interrupt(ntb, type);
1749         if (ret) {
1750                 dev_err(dev, "%s intf: Interrupt configuration failed\n",
1751                         pci_epc_interface_string(type));
1752                 goto err_peer_spad_bar_init;
1753         }
1754
1755         ret = epf_ntb_db_mw_bar_init(ntb, type);
1756         if (ret) {
1757                 dev_err(dev, "%s intf: DB/MW BAR init failed\n",
1758                         pci_epc_interface_string(type));
1759                 goto err_db_mw_bar_init;
1760         }
1761
1762         if (vfunc_no <= 1) {
1763                 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
1764                 if (ret) {
1765                         dev_err(dev, "%s intf: Configuration header write failed\n",
1766                                 pci_epc_interface_string(type));
1767                         goto err_write_header;
1768                 }
1769         }
1770
1771         INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler);
1772         queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work);
1773
1774         return 0;
1775
1776 err_write_header:
1777         epf_ntb_db_mw_bar_cleanup(ntb, type);
1778
1779 err_db_mw_bar_init:
1780         epf_ntb_peer_spad_bar_clear(ntb->epc[type]);
1781
1782 err_peer_spad_bar_init:
1783         epf_ntb_config_sspad_bar_clear(ntb->epc[type]);
1784
1785         return ret;
1786 }
1787
1788 /**
1789  * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface
1790  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1791  * @type: PRIMARY interface or SECONDARY interface
1792  *
1793  * Wrapper to cleanup a particular NTB interface.
1794  */
1795 static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb,
1796                                           enum pci_epc_interface_type type)
1797 {
1798         struct epf_ntb_epc *ntb_epc;
1799
1800         if (type < 0)
1801                 return;
1802
1803         ntb_epc = ntb->epc[type];
1804         cancel_delayed_work(&ntb_epc->cmd_handler);
1805         epf_ntb_db_mw_bar_cleanup(ntb, type);
1806         epf_ntb_peer_spad_bar_clear(ntb_epc);
1807         epf_ntb_config_sspad_bar_clear(ntb_epc);
1808 }
1809
1810 /**
1811  * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
1812  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1813  *
1814  * Wrapper to cleanup all NTB interfaces.
1815  */
1816 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
1817 {
1818         enum pci_epc_interface_type type;
1819
1820         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1821                 epf_ntb_epc_cleanup_interface(ntb, type);
1822 }
1823
1824 /**
1825  * epf_ntb_epc_init() - Initialize all NTB interfaces
1826  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1827  *
1828  * Wrapper to initialize all NTB interface and start the workqueue
1829  * to check for commands from host.
1830  */
1831 static int epf_ntb_epc_init(struct epf_ntb *ntb)
1832 {
1833         enum pci_epc_interface_type type;
1834         struct device *dev;
1835         int ret;
1836
1837         dev = &ntb->epf->dev;
1838
1839         for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1840                 ret = epf_ntb_epc_init_interface(ntb, type);
1841                 if (ret) {
1842                         dev_err(dev, "%s intf: Failed to initialize\n",
1843                                 pci_epc_interface_string(type));
1844                         goto err_init_type;
1845                 }
1846         }
1847
1848         return 0;
1849
1850 err_init_type:
1851         epf_ntb_epc_cleanup_interface(ntb, type - 1);
1852
1853         return ret;
1854 }
1855
1856 /**
1857  * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1858  * @epf: NTB endpoint function device
1859  *
1860  * Initialize both the endpoint controllers associated with NTB function device.
1861  * Invoked when a primary interface or secondary interface is bound to EPC
1862  * device. This function will succeed only when EPC is bound to both the
1863  * interfaces.
1864  */
1865 static int epf_ntb_bind(struct pci_epf *epf)
1866 {
1867         struct epf_ntb *ntb = epf_get_drvdata(epf);
1868         struct device *dev = &epf->dev;
1869         int ret;
1870
1871         if (!epf->epc) {
1872                 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1873                 return 0;
1874         }
1875
1876         if (!epf->sec_epc) {
1877                 dev_dbg(dev, "SECONDARY EPC interface not yet bound\n");
1878                 return 0;
1879         }
1880
1881         ret = epf_ntb_epc_create(ntb);
1882         if (ret) {
1883                 dev_err(dev, "Failed to create NTB EPC\n");
1884                 return ret;
1885         }
1886
1887         ret = epf_ntb_init_epc_bar(ntb);
1888         if (ret) {
1889                 dev_err(dev, "Failed to create NTB EPC\n");
1890                 goto err_bar_init;
1891         }
1892
1893         ret = epf_ntb_config_spad_bar_alloc_interface(ntb);
1894         if (ret) {
1895                 dev_err(dev, "Failed to allocate BAR memory\n");
1896                 goto err_bar_alloc;
1897         }
1898
1899         ret = epf_ntb_epc_init(ntb);
1900         if (ret) {
1901                 dev_err(dev, "Failed to initialize EPC\n");
1902                 goto err_bar_alloc;
1903         }
1904
1905         epf_set_drvdata(epf, ntb);
1906
1907         return 0;
1908
1909 err_bar_alloc:
1910         epf_ntb_config_spad_bar_free(ntb);
1911
1912 err_bar_init:
1913         epf_ntb_epc_destroy(ntb);
1914
1915         return ret;
1916 }
1917
1918 /**
1919  * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1920  * @epf: NTB endpoint function device
1921  *
1922  * Cleanup the initialization from epf_ntb_bind()
1923  */
1924 static void epf_ntb_unbind(struct pci_epf *epf)
1925 {
1926         struct epf_ntb *ntb = epf_get_drvdata(epf);
1927
1928         epf_ntb_epc_cleanup(ntb);
1929         epf_ntb_config_spad_bar_free(ntb);
1930         epf_ntb_epc_destroy(ntb);
1931 }
1932
1933 #define EPF_NTB_R(_name)                                                \
1934 static ssize_t epf_ntb_##_name##_show(struct config_item *item,         \
1935                                       char *page)                       \
1936 {                                                                       \
1937         struct config_group *group = to_config_group(item);             \
1938         struct epf_ntb *ntb = to_epf_ntb(group);                        \
1939                                                                         \
1940         return sprintf(page, "%d\n", ntb->_name);                       \
1941 }
1942
1943 #define EPF_NTB_W(_name)                                                \
1944 static ssize_t epf_ntb_##_name##_store(struct config_item *item,        \
1945                                        const char *page, size_t len)    \
1946 {                                                                       \
1947         struct config_group *group = to_config_group(item);             \
1948         struct epf_ntb *ntb = to_epf_ntb(group);                        \
1949         u32 val;                                                        \
1950         int ret;                                                        \
1951                                                                         \
1952         ret = kstrtou32(page, 0, &val);                                 \
1953         if (ret)                                                        \
1954                 return ret;                                             \
1955                                                                         \
1956         ntb->_name = val;                                               \
1957                                                                         \
1958         return len;                                                     \
1959 }
1960
1961 #define EPF_NTB_MW_R(_name)                                             \
1962 static ssize_t epf_ntb_##_name##_show(struct config_item *item,         \
1963                                       char *page)                       \
1964 {                                                                       \
1965         struct config_group *group = to_config_group(item);             \
1966         struct epf_ntb *ntb = to_epf_ntb(group);                        \
1967         int win_no;                                                     \
1968                                                                         \
1969         sscanf(#_name, "mw%d", &win_no);                                \
1970                                                                         \
1971         return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]);      \
1972 }
1973
1974 #define EPF_NTB_MW_W(_name)                                             \
1975 static ssize_t epf_ntb_##_name##_store(struct config_item *item,        \
1976                                        const char *page, size_t len)    \
1977 {                                                                       \
1978         struct config_group *group = to_config_group(item);             \
1979         struct epf_ntb *ntb = to_epf_ntb(group);                        \
1980         struct device *dev = &ntb->epf->dev;                            \
1981         int win_no;                                                     \
1982         u64 val;                                                        \
1983         int ret;                                                        \
1984                                                                         \
1985         ret = kstrtou64(page, 0, &val);                                 \
1986         if (ret)                                                        \
1987                 return ret;                                             \
1988                                                                         \
1989         if (sscanf(#_name, "mw%d", &win_no) != 1)                       \
1990                 return -EINVAL;                                         \
1991                                                                         \
1992         if (ntb->num_mws < win_no) {                                    \
1993                 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
1994                 return -EINVAL;                                         \
1995         }                                                               \
1996                                                                         \
1997         ntb->mws_size[win_no - 1] = val;                                \
1998                                                                         \
1999         return len;                                                     \
2000 }
2001
2002 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
2003                                      const char *page, size_t len)
2004 {
2005         struct config_group *group = to_config_group(item);
2006         struct epf_ntb *ntb = to_epf_ntb(group);
2007         u32 val;
2008         int ret;
2009
2010         ret = kstrtou32(page, 0, &val);
2011         if (ret)
2012                 return ret;
2013
2014         if (val > MAX_MW)
2015                 return -EINVAL;
2016
2017         ntb->num_mws = val;
2018
2019         return len;
2020 }
2021
2022 EPF_NTB_R(spad_count)
2023 EPF_NTB_W(spad_count)
2024 EPF_NTB_R(db_count)
2025 EPF_NTB_W(db_count)
2026 EPF_NTB_R(num_mws)
2027 EPF_NTB_MW_R(mw1)
2028 EPF_NTB_MW_W(mw1)
2029 EPF_NTB_MW_R(mw2)
2030 EPF_NTB_MW_W(mw2)
2031 EPF_NTB_MW_R(mw3)
2032 EPF_NTB_MW_W(mw3)
2033 EPF_NTB_MW_R(mw4)
2034 EPF_NTB_MW_W(mw4)
2035
2036 CONFIGFS_ATTR(epf_ntb_, spad_count);
2037 CONFIGFS_ATTR(epf_ntb_, db_count);
2038 CONFIGFS_ATTR(epf_ntb_, num_mws);
2039 CONFIGFS_ATTR(epf_ntb_, mw1);
2040 CONFIGFS_ATTR(epf_ntb_, mw2);
2041 CONFIGFS_ATTR(epf_ntb_, mw3);
2042 CONFIGFS_ATTR(epf_ntb_, mw4);
2043
2044 static struct configfs_attribute *epf_ntb_attrs[] = {
2045         &epf_ntb_attr_spad_count,
2046         &epf_ntb_attr_db_count,
2047         &epf_ntb_attr_num_mws,
2048         &epf_ntb_attr_mw1,
2049         &epf_ntb_attr_mw2,
2050         &epf_ntb_attr_mw3,
2051         &epf_ntb_attr_mw4,
2052         NULL,
2053 };
2054
2055 static const struct config_item_type ntb_group_type = {
2056         .ct_attrs       = epf_ntb_attrs,
2057         .ct_owner       = THIS_MODULE,
2058 };
2059
2060 /**
2061  * epf_ntb_add_cfs() - Add configfs directory specific to NTB
2062  * @epf: NTB endpoint function device
2063  * @group: A pointer to the config_group structure referencing a group of
2064  *         config_items of a specific type that belong to a specific sub-system.
2065  *
2066  * Add configfs directory specific to NTB. This directory will hold
2067  * NTB specific properties like db_count, spad_count, num_mws etc.,
2068  */
2069 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
2070                                             struct config_group *group)
2071 {
2072         struct epf_ntb *ntb = epf_get_drvdata(epf);
2073         struct config_group *ntb_group = &ntb->group;
2074         struct device *dev = &epf->dev;
2075
2076         config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
2077
2078         return ntb_group;
2079 }
2080
2081 /**
2082  * epf_ntb_probe() - Probe NTB function driver
2083  * @epf: NTB endpoint function device
2084  *
2085  * Probe NTB function driver when endpoint function bus detects a NTB
2086  * endpoint function.
2087  */
2088 static int epf_ntb_probe(struct pci_epf *epf)
2089 {
2090         struct epf_ntb *ntb;
2091         struct device *dev;
2092
2093         dev = &epf->dev;
2094
2095         ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
2096         if (!ntb)
2097                 return -ENOMEM;
2098
2099         epf->header = &epf_ntb_header;
2100         ntb->epf = epf;
2101         epf_set_drvdata(epf, ntb);
2102
2103         return 0;
2104 }
2105
2106 static struct pci_epf_ops epf_ntb_ops = {
2107         .bind   = epf_ntb_bind,
2108         .unbind = epf_ntb_unbind,
2109         .add_cfs = epf_ntb_add_cfs,
2110 };
2111
2112 static const struct pci_epf_device_id epf_ntb_ids[] = {
2113         {
2114                 .name = "pci_epf_ntb",
2115         },
2116         {},
2117 };
2118
2119 static struct pci_epf_driver epf_ntb_driver = {
2120         .driver.name    = "pci_epf_ntb",
2121         .probe          = epf_ntb_probe,
2122         .id_table       = epf_ntb_ids,
2123         .ops            = &epf_ntb_ops,
2124         .owner          = THIS_MODULE,
2125 };
2126
2127 static int __init epf_ntb_init(void)
2128 {
2129         int ret;
2130
2131         kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
2132                                             WQ_HIGHPRI, 0);
2133         ret = pci_epf_register_driver(&epf_ntb_driver);
2134         if (ret) {
2135                 destroy_workqueue(kpcintb_workqueue);
2136                 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
2137                 return ret;
2138         }
2139
2140         return 0;
2141 }
2142 module_init(epf_ntb_init);
2143
2144 static void __exit epf_ntb_exit(void)
2145 {
2146         pci_epf_unregister_driver(&epf_ntb_driver);
2147         destroy_workqueue(kpcintb_workqueue);
2148 }
2149 module_exit(epf_ntb_exit);
2150
2151 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
2152 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
2153 MODULE_LICENSE("GPL v2");