drivers/ntb/hw/intel/ntb_hw_gen1.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   9  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  10  *
  11  *   This program is free software; you can redistribute it and/or modify
  12  *   it under the terms of version 2 of the GNU General Public License as
  13  *   published by the Free Software Foundation.
  14  *
  15  *   BSD LICENSE
  16  *
  17  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  18  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  19  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  20  *
  21  *   Redistribution and use in source and binary forms, with or without
  22  *   modification, are permitted provided that the following conditions
  23  *   are met:
  24  *
  25  *     * Redistributions of source code must retain the above copyright
  26  *       notice, this list of conditions and the following disclaimer.
  27  *     * Redistributions in binary form must reproduce the above copy
  28  *       notice, this list of conditions and the following disclaimer in
  29  *       the documentation and/or other materials provided with the
  30  *       distribution.
  31  *     * Neither the name of Intel Corporation nor the names of its
  32  *       contributors may be used to endorse or promote products derived
  33  *       from this software without specific prior written permission.
  34  *
  35  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46  *
  47  * Intel PCIe NTB Linux driver
  48  */
  49
  50 #include <linux/debugfs.h>
  51 #include <linux/delay.h>
  52 #include <linux/init.h>
  53 #include <linux/interrupt.h>
  54 #include <linux/module.h>
  55 #include <linux/pci.h>
  56 #include <linux/random.h>
  57 #include <linux/slab.h>
  58 #include <linux/ntb.h>
  59
  60 #include "ntb_hw_intel.h"
  61 #include "ntb_hw_gen1.h"
  62 #include "ntb_hw_gen3.h"
  63 #include "ntb_hw_gen4.h"
  64
  65 #define NTB_NAME        "ntb_hw_intel"
  66 #define NTB_DESC        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  67 #define NTB_VER         "2.0"
  68
  69 MODULE_DESCRIPTION(NTB_DESC);
  70 MODULE_VERSION(NTB_VER);
  71 MODULE_LICENSE("Dual BSD/GPL");
  72 MODULE_AUTHOR("Intel Corporation");
  73
  74 #define bar0_off(base, bar) ((base) + ((bar) << 2))
  75 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
  76
  77 static const struct intel_ntb_reg xeon_reg;
  78 static const struct intel_ntb_alt_reg xeon_pri_reg;
  79 static const struct intel_ntb_alt_reg xeon_sec_reg;
  80 static const struct intel_ntb_alt_reg xeon_b2b_reg;
  81 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
  82 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
  83 static const struct ntb_dev_ops intel_ntb_ops;
  84
  85 static const struct file_operations intel_ntb_debugfs_info;
  86 static struct dentry *debugfs_dir;
  87
  88 static int b2b_mw_idx = -1;
  89 module_param(b2b_mw_idx, int, 0644);
  90 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
  91                  "value of zero or positive starts from first mw idx, and a "
  92                  "negative value starts from last mw idx.  Both sides MUST "
  93                  "set the same value here!");
  94
  95 static unsigned int b2b_mw_share;
  96 module_param(b2b_mw_share, uint, 0644);
  97 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
  98                  "ntb so that the peer ntb only occupies the first half of "
  99                  "the mw, so the second half can still be used as a mw.  Both "
 100                  "sides MUST set the same value here!");
 101
 102 module_param_named(xeon_b2b_usd_bar2_addr64,
 103                    xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
 104 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 105                  "XEON B2B USD BAR 2 64-bit address");
 106
 107 module_param_named(xeon_b2b_usd_bar4_addr64,
 108                    xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
 109 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
 110                  "XEON B2B USD BAR 4 64-bit address");
 111
 112 module_param_named(xeon_b2b_usd_bar4_addr32,
 113                    xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
 114 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
 115                  "XEON B2B USD split-BAR 4 32-bit address");
 116
 117 module_param_named(xeon_b2b_usd_bar5_addr32,
 118                    xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
 119 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
 120                  "XEON B2B USD split-BAR 5 32-bit address");
 121
 122 module_param_named(xeon_b2b_dsd_bar2_addr64,
 123                    xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
 124 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 125                  "XEON B2B DSD BAR 2 64-bit address");
 126
 127 module_param_named(xeon_b2b_dsd_bar4_addr64,
 128                    xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
 129 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
 130                  "XEON B2B DSD BAR 4 64-bit address");
 131
 132 module_param_named(xeon_b2b_dsd_bar4_addr32,
 133                    xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
 134 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
 135                  "XEON B2B DSD split-BAR 4 32-bit address");
 136
 137 module_param_named(xeon_b2b_dsd_bar5_addr32,
 138                    xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
 139 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
 140                  "XEON B2B DSD split-BAR 5 32-bit address");
 141
 142
 143 static int xeon_init_isr(struct intel_ntb_dev *ndev);
 144
 145 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
 146 {
 147         ndev->unsafe_flags = 0;
 148         ndev->unsafe_flags_ignore = 0;
 149
 150         /* Only B2B has a workaround to avoid SDOORBELL */
 151         if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
 152                 if (!ntb_topo_is_b2b(ndev->ntb.topo))
 153                         ndev->unsafe_flags |= NTB_UNSAFE_DB;
 154
 155         /* No low level workaround to avoid SB01BASE */
 156         if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
 157                 ndev->unsafe_flags |= NTB_UNSAFE_DB;
 158                 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
 159         }
 160 }
 161
 162 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
 163                                  unsigned long flag)
 164 {
 165         return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
 166 }
 167
 168 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
 169                                      unsigned long flag)
 170 {
 171         flag &= ndev->unsafe_flags;
 172         ndev->unsafe_flags_ignore |= flag;
 173
 174         return !!flag;
 175 }
 176
 177 int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
 178 {
 179         if (idx < 0 || idx >= ndev->mw_count)
 180                 return -EINVAL;
 181         return ndev->reg->mw_bar[idx];
 182 }
 183
 184 void ndev_db_addr(struct intel_ntb_dev *ndev,
 185                                phys_addr_t *db_addr, resource_size_t *db_size,
 186                                phys_addr_t reg_addr, unsigned long reg)
 187 {
 188         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 189                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 190
 191         if (db_addr) {
 192                 *db_addr = reg_addr + reg;
 193                 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
 194         }
 195
 196         if (db_size) {
 197                 *db_size = ndev->reg->db_size;
 198                 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
 199         }
 200 }
 201
 202 u64 ndev_db_read(struct intel_ntb_dev *ndev,
 203                                void __iomem *mmio)
 204 {
 205         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 206                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 207
 208         return ndev->reg->db_ioread(mmio);
 209 }
 210
 211 int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
 212                                 void __iomem *mmio)
 213 {
 214         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 215                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 216
 217         if (db_bits & ~ndev->db_valid_mask)
 218                 return -EINVAL;
 219
 220         ndev->reg->db_iowrite(db_bits, mmio);
 221
 222         return 0;
 223 }
 224
 225 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 226                                    void __iomem *mmio)
 227 {
 228         unsigned long irqflags;
 229
 230         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 231                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 232
 233         if (db_bits & ~ndev->db_valid_mask)
 234                 return -EINVAL;
 235
 236         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 237         {
 238                 ndev->db_mask |= db_bits;
 239                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 240         }
 241         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 242
 243         return 0;
 244 }
 245
 246 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 247                                      void __iomem *mmio)
 248 {
 249         unsigned long irqflags;
 250
 251         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 252                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 253
 254         if (db_bits & ~ndev->db_valid_mask)
 255                 return -EINVAL;
 256
 257         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 258         {
 259                 ndev->db_mask &= ~db_bits;
 260                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 261         }
 262         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 263
 264         return 0;
 265 }
 266
 267 static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
 268 {
 269         u64 shift, mask;
 270
 271         shift = ndev->db_vec_shift;
 272         mask = BIT_ULL(shift) - 1;
 273
 274         return mask << (shift * db_vector);
 275 }
 276
 277 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
 278                                  phys_addr_t *spad_addr, phys_addr_t reg_addr,
 279                                  unsigned long reg)
 280 {
 281         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 282                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 283
 284         if (idx < 0 || idx >= ndev->spad_count)
 285                 return -EINVAL;
 286
 287         if (spad_addr) {
 288                 *spad_addr = reg_addr + reg + (idx << 2);
 289                 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
 290                         *spad_addr);
 291         }
 292
 293         return 0;
 294 }
 295
 296 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
 297                                  void __iomem *mmio)
 298 {
 299         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 300                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 301
 302         if (idx < 0 || idx >= ndev->spad_count)
 303                 return 0;
 304
 305         return ioread32(mmio + (idx << 2));
 306 }
 307
 308 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
 309                                   void __iomem *mmio)
 310 {
 311         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 312                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 313
 314         if (idx < 0 || idx >= ndev->spad_count)
 315                 return -EINVAL;
 316
 317         iowrite32(val, mmio + (idx << 2));
 318
 319         return 0;
 320 }
 321
 322 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
 323 {
 324         u64 vec_mask;
 325
 326         vec_mask = ndev_vec_mask(ndev, vec);
 327
 328         if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
 329                 vec_mask |= ndev->db_link_mask;
 330
 331         dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
 332
 333         ndev->last_ts = jiffies;
 334
 335         if (vec_mask & ndev->db_link_mask) {
 336                 if (ndev->reg->poll_link(ndev))
 337                         ntb_link_event(&ndev->ntb);
 338         }
 339
 340         if (vec_mask & ndev->db_valid_mask)
 341                 ntb_db_event(&ndev->ntb, vec);
 342
 343         return IRQ_HANDLED;
 344 }
 345
 346 static irqreturn_t ndev_vec_isr(int irq, void *dev)
 347 {
 348         struct intel_ntb_vec *nvec = dev;
 349
 350         dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d  nvec->num: %d\n",
 351                 irq, nvec->num);
 352
 353         return ndev_interrupt(nvec->ndev, nvec->num);
 354 }
 355
 356 static irqreturn_t ndev_irq_isr(int irq, void *dev)
 357 {
 358         struct intel_ntb_dev *ndev = dev;
 359
 360         return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
 361 }
 362
 363 int ndev_init_isr(struct intel_ntb_dev *ndev,
 364                          int msix_min, int msix_max,
 365                          int msix_shift, int total_shift)
 366 {
 367         struct pci_dev *pdev;
 368         int rc, i, msix_count, node;
 369
 370         pdev = ndev->ntb.pdev;
 371
 372         node = dev_to_node(&pdev->dev);
 373
 374         /* Mask all doorbell interrupts */
 375         ndev->db_mask = ndev->db_valid_mask;
 376         ndev->reg->db_iowrite(ndev->db_mask,
 377                               ndev->self_mmio +
 378                               ndev->self_reg->db_mask);
 379
 380         /* Try to set up msix irq */
 381
 382         ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
 383                                  GFP_KERNEL, node);
 384         if (!ndev->vec)
 385                 goto err_msix_vec_alloc;
 386
 387         ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
 388                                   GFP_KERNEL, node);
 389         if (!ndev->msix)
 390                 goto err_msix_alloc;
 391
 392         for (i = 0; i < msix_max; ++i)
 393                 ndev->msix[i].entry = i;
 394
 395         msix_count = pci_enable_msix_range(pdev, ndev->msix,
 396                                            msix_min, msix_max);
 397         if (msix_count < 0)
 398                 goto err_msix_enable;
 399
 400         for (i = 0; i < msix_count; ++i) {
 401                 ndev->vec[i].ndev = ndev;
 402                 ndev->vec[i].num = i;
 403                 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 404                                  "ndev_vec_isr", &ndev->vec[i]);
 405                 if (rc)
 406                         goto err_msix_request;
 407         }
 408
 409         dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
 410         ndev->db_vec_count = msix_count;
 411         ndev->db_vec_shift = msix_shift;
 412         return 0;
 413
 414 err_msix_request:
 415         while (i-- > 0)
 416                 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 417         pci_disable_msix(pdev);
 418 err_msix_enable:
 419         kfree(ndev->msix);
 420 err_msix_alloc:
 421         kfree(ndev->vec);
 422 err_msix_vec_alloc:
 423         ndev->msix = NULL;
 424         ndev->vec = NULL;
 425
 426         /* Try to set up msi irq */
 427
 428         rc = pci_enable_msi(pdev);
 429         if (rc)
 430                 goto err_msi_enable;
 431
 432         rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 433                          "ndev_irq_isr", ndev);
 434         if (rc)
 435                 goto err_msi_request;
 436
 437         dev_dbg(&pdev->dev, "Using msi interrupts\n");
 438         ndev->db_vec_count = 1;
 439         ndev->db_vec_shift = total_shift;
 440         return 0;
 441
 442 err_msi_request:
 443         pci_disable_msi(pdev);
 444 err_msi_enable:
 445
 446         /* Try to set up intx irq */
 447
 448         pci_intx(pdev, 1);
 449
 450         rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 451                          "ndev_irq_isr", ndev);
 452         if (rc)
 453                 goto err_intx_request;
 454
 455         dev_dbg(&pdev->dev, "Using intx interrupts\n");
 456         ndev->db_vec_count = 1;
 457         ndev->db_vec_shift = total_shift;
 458         return 0;
 459
 460 err_intx_request:
 461         return rc;
 462 }
 463
 464 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
 465 {
 466         struct pci_dev *pdev;
 467         int i;
 468
 469         pdev = ndev->ntb.pdev;
 470
 471         /* Mask all doorbell interrupts */
 472         ndev->db_mask = ndev->db_valid_mask;
 473         ndev->reg->db_iowrite(ndev->db_mask,
 474                               ndev->self_mmio +
 475                               ndev->self_reg->db_mask);
 476
 477         if (ndev->msix) {
 478                 i = ndev->db_vec_count;
 479                 while (i--)
 480                         free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 481                 pci_disable_msix(pdev);
 482                 kfree(ndev->msix);
 483                 kfree(ndev->vec);
 484         } else {
 485                 free_irq(pdev->irq, ndev);
 486                 if (pci_dev_msi_enabled(pdev))
 487                         pci_disable_msi(pdev);
 488         }
 489 }
 490
 491 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
 492                                      size_t count, loff_t *offp)
 493 {
 494         struct intel_ntb_dev *ndev;
 495         struct pci_dev *pdev;
 496         void __iomem *mmio;
 497         char *buf;
 498         size_t buf_size;
 499         ssize_t ret, off;
 500         union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
 501
 502         ndev = filp->private_data;
 503         pdev = ndev->ntb.pdev;
 504         mmio = ndev->self_mmio;
 505
 506         buf_size = min(count, 0x800ul);
 507
 508         buf = kmalloc(buf_size, GFP_KERNEL);
 509         if (!buf)
 510                 return -ENOMEM;
 511
 512         off = 0;
 513
 514         off += scnprintf(buf + off, buf_size - off,
 515                          "NTB Device Information:\n");
 516
 517         off += scnprintf(buf + off, buf_size - off,
 518                          "Connection Topology -\t%s\n",
 519                          ntb_topo_string(ndev->ntb.topo));
 520
 521         if (ndev->b2b_idx != UINT_MAX) {
 522                 off += scnprintf(buf + off, buf_size - off,
 523                                  "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
 524                 off += scnprintf(buf + off, buf_size - off,
 525                                  "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
 526         }
 527
 528         off += scnprintf(buf + off, buf_size - off,
 529                          "BAR4 Split -\t\t%s\n",
 530                          ndev->bar4_split ? "yes" : "no");
 531
 532         off += scnprintf(buf + off, buf_size - off,
 533                          "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
 534         off += scnprintf(buf + off, buf_size - off,
 535                          "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 536
 537         if (!ndev->reg->link_is_up(ndev)) {
 538                 off += scnprintf(buf + off, buf_size - off,
 539                                  "Link Status -\t\tDown\n");
 540         } else {
 541                 off += scnprintf(buf + off, buf_size - off,
 542                                  "Link Status -\t\tUp\n");
 543                 off += scnprintf(buf + off, buf_size - off,
 544                                  "Link Speed -\t\tPCI-E Gen %u\n",
 545                                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
 546                 off += scnprintf(buf + off, buf_size - off,
 547                                  "Link Width -\t\tx%u\n",
 548                                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 549         }
 550
 551         off += scnprintf(buf + off, buf_size - off,
 552                          "Memory Window Count -\t%u\n", ndev->mw_count);
 553         off += scnprintf(buf + off, buf_size - off,
 554                          "Scratchpad Count -\t%u\n", ndev->spad_count);
 555         off += scnprintf(buf + off, buf_size - off,
 556                          "Doorbell Count -\t%u\n", ndev->db_count);
 557         off += scnprintf(buf + off, buf_size - off,
 558                          "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
 559         off += scnprintf(buf + off, buf_size - off,
 560                          "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
 561
 562         off += scnprintf(buf + off, buf_size - off,
 563                          "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 564         off += scnprintf(buf + off, buf_size - off,
 565                          "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
 566         off += scnprintf(buf + off, buf_size - off,
 567                          "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
 568
 569         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
 570         off += scnprintf(buf + off, buf_size - off,
 571                          "Doorbell Mask -\t\t%#llx\n", u.v64);
 572
 573         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
 574         off += scnprintf(buf + off, buf_size - off,
 575                          "Doorbell Bell -\t\t%#llx\n", u.v64);
 576
 577         off += scnprintf(buf + off, buf_size - off,
 578                          "\nNTB Window Size:\n");
 579
 580         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
 581         off += scnprintf(buf + off, buf_size - off,
 582                          "PBAR23SZ %hhu\n", u.v8);
 583         if (!ndev->bar4_split) {
 584                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
 585                 off += scnprintf(buf + off, buf_size - off,
 586                                  "PBAR45SZ %hhu\n", u.v8);
 587         } else {
 588                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
 589                 off += scnprintf(buf + off, buf_size - off,
 590                                  "PBAR4SZ %hhu\n", u.v8);
 591                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
 592                 off += scnprintf(buf + off, buf_size - off,
 593                                  "PBAR5SZ %hhu\n", u.v8);
 594         }
 595
 596         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
 597         off += scnprintf(buf + off, buf_size - off,
 598                          "SBAR23SZ %hhu\n", u.v8);
 599         if (!ndev->bar4_split) {
 600                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
 601                 off += scnprintf(buf + off, buf_size - off,
 602                                  "SBAR45SZ %hhu\n", u.v8);
 603         } else {
 604                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
 605                 off += scnprintf(buf + off, buf_size - off,
 606                                  "SBAR4SZ %hhu\n", u.v8);
 607                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
 608                 off += scnprintf(buf + off, buf_size - off,
 609                                  "SBAR5SZ %hhu\n", u.v8);
 610         }
 611
 612         off += scnprintf(buf + off, buf_size - off,
 613                          "\nNTB Incoming XLAT:\n");
 614
 615         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
 616         off += scnprintf(buf + off, buf_size - off,
 617                          "XLAT23 -\t\t%#018llx\n", u.v64);
 618
 619         if (ndev->bar4_split) {
 620                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 621                 off += scnprintf(buf + off, buf_size - off,
 622                                  "XLAT4 -\t\t\t%#06x\n", u.v32);
 623
 624                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
 625                 off += scnprintf(buf + off, buf_size - off,
 626                                  "XLAT5 -\t\t\t%#06x\n", u.v32);
 627         } else {
 628                 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 629                 off += scnprintf(buf + off, buf_size - off,
 630                                  "XLAT45 -\t\t%#018llx\n", u.v64);
 631         }
 632
 633         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
 634         off += scnprintf(buf + off, buf_size - off,
 635                          "LMT23 -\t\t\t%#018llx\n", u.v64);
 636
 637         if (ndev->bar4_split) {
 638                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 639                 off += scnprintf(buf + off, buf_size - off,
 640                                  "LMT4 -\t\t\t%#06x\n", u.v32);
 641                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
 642                 off += scnprintf(buf + off, buf_size - off,
 643                                  "LMT5 -\t\t\t%#06x\n", u.v32);
 644         } else {
 645                 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 646                 off += scnprintf(buf + off, buf_size - off,
 647                                  "LMT45 -\t\t\t%#018llx\n", u.v64);
 648         }
 649
 650         if (pdev_is_gen1(pdev)) {
 651                 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
 652                         off += scnprintf(buf + off, buf_size - off,
 653                                          "\nNTB Outgoing B2B XLAT:\n");
 654
 655                         u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
 656                         off += scnprintf(buf + off, buf_size - off,
 657                                          "B2B XLAT23 -\t\t%#018llx\n", u.v64);
 658
 659                         if (ndev->bar4_split) {
 660                                 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
 661                                 off += scnprintf(buf + off, buf_size - off,
 662                                                  "B2B XLAT4 -\t\t%#06x\n",
 663                                                  u.v32);
 664                                 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
 665                                 off += scnprintf(buf + off, buf_size - off,
 666                                                  "B2B XLAT5 -\t\t%#06x\n",
 667                                                  u.v32);
 668                         } else {
 669                                 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
 670                                 off += scnprintf(buf + off, buf_size - off,
 671                                                  "B2B XLAT45 -\t\t%#018llx\n",
 672                                                  u.v64);
 673                         }
 674
 675                         u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
 676                         off += scnprintf(buf + off, buf_size - off,
 677                                          "B2B LMT23 -\t\t%#018llx\n", u.v64);
 678
 679                         if (ndev->bar4_split) {
 680                                 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
 681                                 off += scnprintf(buf + off, buf_size - off,
 682                                                  "B2B LMT4 -\t\t%#06x\n",
 683                                                  u.v32);
 684                                 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
 685                                 off += scnprintf(buf + off, buf_size - off,
 686                                                  "B2B LMT5 -\t\t%#06x\n",
 687                                                  u.v32);
 688                         } else {
 689                                 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
 690                                 off += scnprintf(buf + off, buf_size - off,
 691                                                  "B2B LMT45 -\t\t%#018llx\n",
 692                                                  u.v64);
 693                         }
 694
 695                         off += scnprintf(buf + off, buf_size - off,
 696                                          "\nNTB Secondary BAR:\n");
 697
 698                         u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
 699                         off += scnprintf(buf + off, buf_size - off,
 700                                          "SBAR01 -\t\t%#018llx\n", u.v64);
 701
 702                         u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
 703                         off += scnprintf(buf + off, buf_size - off,
 704                                          "SBAR23 -\t\t%#018llx\n", u.v64);
 705
 706                         if (ndev->bar4_split) {
 707                                 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
 708                                 off += scnprintf(buf + off, buf_size - off,
 709                                                  "SBAR4 -\t\t\t%#06x\n", u.v32);
 710                                 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
 711                                 off += scnprintf(buf + off, buf_size - off,
 712                                                  "SBAR5 -\t\t\t%#06x\n", u.v32);
 713                         } else {
 714                                 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
 715                                 off += scnprintf(buf + off, buf_size - off,
 716                                                  "SBAR45 -\t\t%#018llx\n",
 717                                                  u.v64);
 718                         }
 719                 }
 720
 721                 off += scnprintf(buf + off, buf_size - off,
 722                                  "\nXEON NTB Statistics:\n");
 723
 724                 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
 725                 off += scnprintf(buf + off, buf_size - off,
 726                                  "Upstream Memory Miss -\t%u\n", u.v16);
 727
 728                 off += scnprintf(buf + off, buf_size - off,
 729                                  "\nXEON NTB Hardware Errors:\n");
 730
 731                 if (!pci_read_config_word(pdev,
 732                                           XEON_DEVSTS_OFFSET, &u.v16))
 733                         off += scnprintf(buf + off, buf_size - off,
 734                                          "DEVSTS -\t\t%#06x\n", u.v16);
 735
 736                 if (!pci_read_config_word(pdev,
 737                                           XEON_LINK_STATUS_OFFSET, &u.v16))
 738                         off += scnprintf(buf + off, buf_size - off,
 739                                          "LNKSTS -\t\t%#06x\n", u.v16);
 740
 741                 if (!pci_read_config_dword(pdev,
 742                                            XEON_UNCERRSTS_OFFSET, &u.v32))
 743                         off += scnprintf(buf + off, buf_size - off,
 744                                          "UNCERRSTS -\t\t%#06x\n", u.v32);
 745
 746                 if (!pci_read_config_dword(pdev,
 747                                            XEON_CORERRSTS_OFFSET, &u.v32))
 748                         off += scnprintf(buf + off, buf_size - off,
 749                                          "CORERRSTS -\t\t%#06x\n", u.v32);
 750         }
 751
 752         ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 753         kfree(buf);
 754         return ret;
 755 }
 756
 757 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 758                                  size_t count, loff_t *offp)
 759 {
 760         struct intel_ntb_dev *ndev = filp->private_data;
 761
 762         if (pdev_is_gen1(ndev->ntb.pdev))
 763                 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
 764         else if (pdev_is_gen3(ndev->ntb.pdev))
 765                 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
 766         else if (pdev_is_gen4(ndev->ntb.pdev))
 767                 return ndev_ntb4_debugfs_read(filp, ubuf, count, offp);
 768
 769         return -ENXIO;
 770 }
 771
 772 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
 773 {
 774         if (!debugfs_dir) {
 775                 ndev->debugfs_dir = NULL;
 776                 ndev->debugfs_info = NULL;
 777         } else {
 778                 ndev->debugfs_dir =
 779                         debugfs_create_dir(pci_name(ndev->ntb.pdev),
 780                                            debugfs_dir);
 781                 if (!ndev->debugfs_dir)
 782                         ndev->debugfs_info = NULL;
 783                 else
 784                         ndev->debugfs_info =
 785                                 debugfs_create_file("info", S_IRUSR,
 786                                                     ndev->debugfs_dir, ndev,
 787                                                     &intel_ntb_debugfs_info);
 788         }
 789 }
 790
 791 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
 792 {
 793         debugfs_remove_recursive(ndev->debugfs_dir);
 794 }
 795
 796 int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 797 {
 798         if (pidx != NTB_DEF_PEER_IDX)
 799                 return -EINVAL;
 800
 801         return ntb_ndev(ntb)->mw_count;
 802 }
 803
 804 int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
 805                            resource_size_t *addr_align,
 806                            resource_size_t *size_align,
 807                            resource_size_t *size_max)
 808 {
 809         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 810         resource_size_t bar_size, mw_size;
 811         int bar;
 812
 813         if (pidx != NTB_DEF_PEER_IDX)
 814                 return -EINVAL;
 815
 816         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 817                 idx += 1;
 818
 819         bar = ndev_mw_to_bar(ndev, idx);
 820         if (bar < 0)
 821                 return bar;
 822
 823         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 824
 825         if (idx == ndev->b2b_idx)
 826                 mw_size = bar_size - ndev->b2b_off;
 827         else
 828                 mw_size = bar_size;
 829
 830         if (addr_align)
 831                 *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
 832
 833         if (size_align)
 834                 *size_align = 1;
 835
 836         if (size_max)
 837                 *size_max = mw_size;
 838
 839         return 0;
 840 }
 841
 842 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
 843                                   dma_addr_t addr, resource_size_t size)
 844 {
 845         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 846         unsigned long base_reg, xlat_reg, limit_reg;
 847         resource_size_t bar_size, mw_size;
 848         void __iomem *mmio;
 849         u64 base, limit, reg_val;
 850         int bar;
 851
 852         if (pidx != NTB_DEF_PEER_IDX)
 853                 return -EINVAL;
 854
 855         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
 856                 idx += 1;
 857
 858         bar = ndev_mw_to_bar(ndev, idx);
 859         if (bar < 0)
 860                 return bar;
 861
 862         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
 863
 864         if (idx == ndev->b2b_idx)
 865                 mw_size = bar_size - ndev->b2b_off;
 866         else
 867                 mw_size = bar_size;
 868
 869         /* hardware requires that addr is aligned to bar size */
 870         if (addr & (bar_size - 1))
 871                 return -EINVAL;
 872
 873         /* make sure the range fits in the usable mw size */
 874         if (size > mw_size)
 875                 return -EINVAL;
 876
 877         mmio = ndev->self_mmio;
 878         base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
 879         xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
 880         limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
 881
 882         if (bar < 4 || !ndev->bar4_split) {
 883                 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
 884
 885                 /* Set the limit if supported, if size is not mw_size */
 886                 if (limit_reg && size != mw_size)
 887                         limit = base + size;
 888                 else
 889                         limit = 0;
 890
 891                 /* set and verify setting the translation address */
 892                 iowrite64(addr, mmio + xlat_reg);
 893                 reg_val = ioread64(mmio + xlat_reg);
 894                 if (reg_val != addr) {
 895                         iowrite64(0, mmio + xlat_reg);
 896                         return -EIO;
 897                 }
 898
 899                 /* set and verify setting the limit */
 900                 iowrite64(limit, mmio + limit_reg);
 901                 reg_val = ioread64(mmio + limit_reg);
 902                 if (reg_val != limit) {
 903                         iowrite64(base, mmio + limit_reg);
 904                         iowrite64(0, mmio + xlat_reg);
 905                         return -EIO;
 906                 }
 907         } else {
 908                 /* split bar addr range must all be 32 bit */
 909                 if (addr & (~0ull << 32))
 910                         return -EINVAL;
 911                 if ((addr + size) & (~0ull << 32))
 912                         return -EINVAL;
 913
 914                 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
 915
 916                 /* Set the limit if supported, if size is not mw_size */
 917                 if (limit_reg && size != mw_size)
 918                         limit = base + size;
 919                 else
 920                         limit = 0;
 921
 922                 /* set and verify setting the translation address */
 923                 iowrite32(addr, mmio + xlat_reg);
 924                 reg_val = ioread32(mmio + xlat_reg);
 925                 if (reg_val != addr) {
 926                         iowrite32(0, mmio + xlat_reg);
 927                         return -EIO;
 928                 }
 929
 930                 /* set and verify setting the limit */
 931                 iowrite32(limit, mmio + limit_reg);
 932                 reg_val = ioread32(mmio + limit_reg);
 933                 if (reg_val != limit) {
 934                         iowrite32(base, mmio + limit_reg);
 935                         iowrite32(0, mmio + xlat_reg);
 936                         return -EIO;
 937                 }
 938         }
 939
 940         return 0;
 941 }
 942
 943 u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
 944                          enum ntb_width *width)
 945 {
 946         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
 947
 948         if (ndev->reg->link_is_up(ndev)) {
 949                 if (speed)
 950                         *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
 951                 if (width)
 952                         *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
 953                 return 1;
 954         } else {
 955                 /* TODO MAYBE: is it possible to observe the link speed and
 956                  * width while link is training? */
 957                 if (speed)
 958                         *speed = NTB_SPEED_NONE;
 959                 if (width)
 960                         *width = NTB_WIDTH_NONE;
 961                 return 0;
 962         }
 963 }
 964
 965 static int intel_ntb_link_enable(struct ntb_dev *ntb,
 966                                  enum ntb_speed max_speed,
 967                                  enum ntb_width max_width)
 968 {
 969         struct intel_ntb_dev *ndev;
 970         u32 ntb_ctl;
 971
 972         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
 973
 974         if (ndev->ntb.topo == NTB_TOPO_SEC)
 975                 return -EINVAL;
 976
 977         dev_dbg(&ntb->pdev->dev,
 978                 "Enabling link with max_speed %d max_width %d\n",
 979                 max_speed, max_width);
 980         if (max_speed != NTB_SPEED_AUTO)
 981                 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
 982         if (max_width != NTB_WIDTH_AUTO)
 983                 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
 984
 985         ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
 986         ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
 987         ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 988         ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 989         if (ndev->bar4_split)
 990                 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
 991         iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
 992
 993         return 0;
 994 }
 995
 996 int intel_ntb_link_disable(struct ntb_dev *ntb)
 997 {
 998         struct intel_ntb_dev *ndev;
 999         u32 ntb_cntl;
1000
1001         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1002
1003         if (ndev->ntb.topo == NTB_TOPO_SEC)
1004                 return -EINVAL;
1005
1006         dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1007
1008         /* Bring NTB link down */
1009         ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1010         ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1011         ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1012         if (ndev->bar4_split)
1013                 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1014         ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1015         iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1016
1017         return 0;
1018 }
1019
1020 int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1021 {
1022         /* Numbers of inbound and outbound memory windows match */
1023         return ntb_ndev(ntb)->mw_count;
1024 }
1025
1026 int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1027                                phys_addr_t *base, resource_size_t *size)
1028 {
1029         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1030         int bar;
1031
1032         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1033                 idx += 1;
1034
1035         bar = ndev_mw_to_bar(ndev, idx);
1036         if (bar < 0)
1037                 return bar;
1038
1039         if (base)
1040                 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1041                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1042
1043         if (size)
1044                 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1045                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1046
1047         return 0;
1048 }
1049
1050 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1051 {
1052         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1053 }
1054
1055 u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1056 {
1057         return ntb_ndev(ntb)->db_valid_mask;
1058 }
1059
1060 int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1061 {
1062         struct intel_ntb_dev *ndev;
1063
1064         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1065
1066         return ndev->db_vec_count;
1067 }
1068
1069 u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1070 {
1071         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1072
1073         if (db_vector < 0 || db_vector > ndev->db_vec_count)
1074                 return 0;
1075
1076         return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1077 }
1078
1079 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1080 {
1081         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1082
1083         return ndev_db_read(ndev,
1084                             ndev->self_mmio +
1085                             ndev->self_reg->db_bell);
1086 }
1087
1088 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1089 {
1090         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1091
1092         return ndev_db_write(ndev, db_bits,
1093                              ndev->self_mmio +
1094                              ndev->self_reg->db_bell);
1095 }
1096
1097 int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1098 {
1099         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1100
1101         return ndev_db_set_mask(ndev, db_bits,
1102                                 ndev->self_mmio +
1103                                 ndev->self_reg->db_mask);
1104 }
1105
1106 int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1107 {
1108         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1109
1110         return ndev_db_clear_mask(ndev, db_bits,
1111                                   ndev->self_mmio +
1112                                   ndev->self_reg->db_mask);
1113 }
1114
1115 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
1116                            resource_size_t *db_size, u64 *db_data, int db_bit)
1117 {
1118         u64 db_bits;
1119         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1120
1121         if (unlikely(db_bit >= BITS_PER_LONG_LONG))
1122                 return -EINVAL;
1123
1124         db_bits = BIT_ULL(db_bit);
1125
1126         if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask))
1127                 return -EINVAL;
1128
1129         ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1130                             ndev->peer_reg->db_bell);
1131
1132         if (db_data)
1133                 *db_data = db_bits;
1134
1135
1136         return 0;
1137 }
1138
1139 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1140 {
1141         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1142
1143         return ndev_db_write(ndev, db_bits,
1144                              ndev->peer_mmio +
1145                              ndev->peer_reg->db_bell);
1146 }
1147
1148 int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1149 {
1150         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1151 }
1152
1153 int intel_ntb_spad_count(struct ntb_dev *ntb)
1154 {
1155         struct intel_ntb_dev *ndev;
1156
1157         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1158
1159         return ndev->spad_count;
1160 }
1161
1162 u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1163 {
1164         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1165
1166         return ndev_spad_read(ndev, idx,
1167                               ndev->self_mmio +
1168                               ndev->self_reg->spad);
1169 }
1170
1171 int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
1172 {
1173         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1174
1175         return ndev_spad_write(ndev, idx, val,
1176                                ndev->self_mmio +
1177                                ndev->self_reg->spad);
1178 }
1179
1180 int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1181                              phys_addr_t *spad_addr)
1182 {
1183         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1184
1185         return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1186                               ndev->peer_reg->spad);
1187 }
1188
1189 u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1190 {
1191         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1192
1193         return ndev_spad_read(ndev, sidx,
1194                               ndev->peer_mmio +
1195                               ndev->peer_reg->spad);
1196 }
1197
1198 int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
1199                               u32 val)
1200 {
1201         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1202
1203         return ndev_spad_write(ndev, sidx, val,
1204                                ndev->peer_mmio +
1205                                ndev->peer_reg->spad);
1206 }
1207
1208 static u64 xeon_db_ioread(const void __iomem *mmio)
1209 {
1210         return (u64)ioread16(mmio);
1211 }
1212
1213 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1214 {
1215         iowrite16((u16)bits, mmio);
1216 }
1217
1218 static int xeon_poll_link(struct intel_ntb_dev *ndev)
1219 {
1220         u16 reg_val;
1221         int rc;
1222
1223         ndev->reg->db_iowrite(ndev->db_link_mask,
1224                               ndev->self_mmio +
1225                               ndev->self_reg->db_bell);
1226
1227         rc = pci_read_config_word(ndev->ntb.pdev,
1228                                   XEON_LINK_STATUS_OFFSET, &reg_val);
1229         if (rc)
1230                 return 0;
1231
1232         if (reg_val == ndev->lnk_sta)
1233                 return 0;
1234
1235         ndev->lnk_sta = reg_val;
1236
1237         return 1;
1238 }
1239
1240 int xeon_link_is_up(struct intel_ntb_dev *ndev)
1241 {
1242         if (ndev->ntb.topo == NTB_TOPO_SEC)
1243                 return 1;
1244
1245         return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1246 }
1247
1248 enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1249 {
1250         switch (ppd & XEON_PPD_TOPO_MASK) {
1251         case XEON_PPD_TOPO_B2B_USD:
1252                 return NTB_TOPO_B2B_USD;
1253
1254         case XEON_PPD_TOPO_B2B_DSD:
1255                 return NTB_TOPO_B2B_DSD;
1256
1257         case XEON_PPD_TOPO_PRI_USD:
1258         case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1259                 return NTB_TOPO_PRI;
1260
1261         case XEON_PPD_TOPO_SEC_USD:
1262         case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1263                 return NTB_TOPO_SEC;
1264         }
1265
1266         return NTB_TOPO_NONE;
1267 }
1268
1269 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1270 {
1271         if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1272                 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
1273                 return 1;
1274         }
1275         return 0;
1276 }
1277
1278 static int xeon_init_isr(struct intel_ntb_dev *ndev)
1279 {
1280         return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1281                              XEON_DB_MSIX_VECTOR_COUNT,
1282                              XEON_DB_MSIX_VECTOR_SHIFT,
1283                              XEON_DB_TOTAL_SHIFT);
1284 }
1285
1286 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1287 {
1288         ndev_deinit_isr(ndev);
1289 }
1290
1291 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1292                              const struct intel_b2b_addr *addr,
1293                              const struct intel_b2b_addr *peer_addr)
1294 {
1295         struct pci_dev *pdev;
1296         void __iomem *mmio;
1297         resource_size_t bar_size;
1298         phys_addr_t bar_addr;
1299         int b2b_bar;
1300         u8 bar_sz;
1301
1302         pdev = ndev->ntb.pdev;
1303         mmio = ndev->self_mmio;
1304
1305         if (ndev->b2b_idx == UINT_MAX) {
1306                 dev_dbg(&pdev->dev, "not using b2b mw\n");
1307                 b2b_bar = 0;
1308                 ndev->b2b_off = 0;
1309         } else {
1310                 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1311                 if (b2b_bar < 0)
1312                         return -EIO;
1313
1314                 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1315
1316                 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1317
1318                 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1319
1320                 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1321                         dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1322                         ndev->b2b_off = bar_size >> 1;
1323                 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1324                         dev_dbg(&pdev->dev, "b2b using whole bar\n");
1325                         ndev->b2b_off = 0;
1326                         --ndev->mw_count;
1327                 } else {
1328                         dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1329                         return -EIO;
1330                 }
1331         }
1332
1333         /* Reset the secondary bar sizes to match the primary bar sizes,
1334          * except disable or halve the size of the b2b secondary bar.
1335          *
1336          * Note: code for each specific bar size register, because the register
1337          * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1338          */
1339         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1340         dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
1341         if (b2b_bar == 2) {
1342                 if (ndev->b2b_off)
1343                         bar_sz -= 1;
1344                 else
1345                         bar_sz = 0;
1346         }
1347         pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1348         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1349         dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
1350
1351         if (!ndev->bar4_split) {
1352                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1353                 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
1354                 if (b2b_bar == 4) {
1355                         if (ndev->b2b_off)
1356                                 bar_sz -= 1;
1357                         else
1358                                 bar_sz = 0;
1359                 }
1360                 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1361                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1362                 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
1363         } else {
1364                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1365                 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
1366                 if (b2b_bar == 4) {
1367                         if (ndev->b2b_off)
1368                                 bar_sz -= 1;
1369                         else
1370                                 bar_sz = 0;
1371                 }
1372                 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1373                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1374                 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
1375
1376                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1377                 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
1378                 if (b2b_bar == 5) {
1379                         if (ndev->b2b_off)
1380                                 bar_sz -= 1;
1381                         else
1382                                 bar_sz = 0;
1383                 }
1384                 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1385                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1386                 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
1387         }
1388
1389         /* SBAR01 hit by first part of the b2b bar */
1390         if (b2b_bar == 0)
1391                 bar_addr = addr->bar0_addr;
1392         else if (b2b_bar == 2)
1393                 bar_addr = addr->bar2_addr64;
1394         else if (b2b_bar == 4 && !ndev->bar4_split)
1395                 bar_addr = addr->bar4_addr64;
1396         else if (b2b_bar == 4)
1397                 bar_addr = addr->bar4_addr32;
1398         else if (b2b_bar == 5)
1399                 bar_addr = addr->bar5_addr32;
1400         else
1401                 return -EIO;
1402
1403         dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
1404         iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1405
1406         /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1407          * The b2b bar is either disabled above, or configured half-size, and
1408          * it starts at the PBAR xlat + offset.
1409          */
1410
1411         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1412         iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1413         bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1414         dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
1415
1416         if (!ndev->bar4_split) {
1417                 bar_addr = addr->bar4_addr64 +
1418                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1419                 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1420                 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1421                 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
1422         } else {
1423                 bar_addr = addr->bar4_addr32 +
1424                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1425                 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1426                 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1427                 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
1428
1429                 bar_addr = addr->bar5_addr32 +
1430                         (b2b_bar == 5 ? ndev->b2b_off : 0);
1431                 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1432                 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1433                 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
1434         }
1435
1436         /* setup incoming bar limits == base addrs (zero length windows) */
1437
1438         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1439         iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1440         bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1441         dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
1442
1443         if (!ndev->bar4_split) {
1444                 bar_addr = addr->bar4_addr64 +
1445                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1446                 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1447                 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1448                 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
1449         } else {
1450                 bar_addr = addr->bar4_addr32 +
1451                         (b2b_bar == 4 ? ndev->b2b_off : 0);
1452                 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1453                 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1454                 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
1455
1456                 bar_addr = addr->bar5_addr32 +
1457                         (b2b_bar == 5 ? ndev->b2b_off : 0);
1458                 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1459                 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1460                 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
1461         }
1462
1463         /* zero incoming translation addrs */
1464         iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1465
1466         if (!ndev->bar4_split) {
1467                 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1468         } else {
1469                 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1470                 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1471         }
1472
1473         /* zero outgoing translation limits (whole bar size windows) */
1474         iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1475         if (!ndev->bar4_split) {
1476                 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1477         } else {
1478                 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1479                 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1480         }
1481
1482         /* set outgoing translation offsets */
1483         bar_addr = peer_addr->bar2_addr64;
1484         iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1485         bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1486         dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
1487
1488         if (!ndev->bar4_split) {
1489                 bar_addr = peer_addr->bar4_addr64;
1490                 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1491                 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1492                 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
1493         } else {
1494                 bar_addr = peer_addr->bar4_addr32;
1495                 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1496                 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1497                 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
1498
1499                 bar_addr = peer_addr->bar5_addr32;
1500                 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1501                 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1502                 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
1503         }
1504
1505         /* set the translation offset for b2b registers */
1506         if (b2b_bar == 0)
1507                 bar_addr = peer_addr->bar0_addr;
1508         else if (b2b_bar == 2)
1509                 bar_addr = peer_addr->bar2_addr64;
1510         else if (b2b_bar == 4 && !ndev->bar4_split)
1511                 bar_addr = peer_addr->bar4_addr64;
1512         else if (b2b_bar == 4)
1513                 bar_addr = peer_addr->bar4_addr32;
1514         else if (b2b_bar == 5)
1515                 bar_addr = peer_addr->bar5_addr32;
1516         else
1517                 return -EIO;
1518
1519         /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1520         dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
1521         iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1522         iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1523
1524         if (b2b_bar) {
1525                 /* map peer ntb mmio config space registers */
1526                 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1527                                             XEON_B2B_MIN_SIZE);
1528                 if (!ndev->peer_mmio)
1529                         return -EIO;
1530
1531                 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
1532         }
1533
1534         return 0;
1535 }
1536
1537 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1538 {
1539         struct device *dev = &ndev->ntb.pdev->dev;
1540         int rc;
1541         u32 ntb_ctl;
1542
1543         if (ndev->bar4_split)
1544                 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1545         else
1546                 ndev->mw_count = XEON_MW_COUNT;
1547
1548         ndev->spad_count = XEON_SPAD_COUNT;
1549         ndev->db_count = XEON_DB_COUNT;
1550         ndev->db_link_mask = XEON_DB_LINK_BIT;
1551
1552         switch (ndev->ntb.topo) {
1553         case NTB_TOPO_PRI:
1554                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1555                         dev_err(dev, "NTB Primary config disabled\n");
1556                         return -EINVAL;
1557                 }
1558
1559                 /* enable link to allow secondary side device to appear */
1560                 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1561                 ntb_ctl &= ~NTB_CTL_DISABLE;
1562                 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1563
1564                 /* use half the spads for the peer */
1565                 ndev->spad_count >>= 1;
1566                 ndev->self_reg = &xeon_pri_reg;
1567                 ndev->peer_reg = &xeon_sec_reg;
1568                 ndev->xlat_reg = &xeon_sec_xlat;
1569                 break;
1570
1571         case NTB_TOPO_SEC:
1572                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1573                         dev_err(dev, "NTB Secondary config disabled\n");
1574                         return -EINVAL;
1575                 }
1576                 /* use half the spads for the peer */
1577                 ndev->spad_count >>= 1;
1578                 ndev->self_reg = &xeon_sec_reg;
1579                 ndev->peer_reg = &xeon_pri_reg;
1580                 ndev->xlat_reg = &xeon_pri_xlat;
1581                 break;
1582
1583         case NTB_TOPO_B2B_USD:
1584         case NTB_TOPO_B2B_DSD:
1585                 ndev->self_reg = &xeon_pri_reg;
1586                 ndev->peer_reg = &xeon_b2b_reg;
1587                 ndev->xlat_reg = &xeon_sec_xlat;
1588
1589                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1590                         ndev->peer_reg = &xeon_pri_reg;
1591
1592                         if (b2b_mw_idx < 0)
1593                                 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1594                         else
1595                                 ndev->b2b_idx = b2b_mw_idx;
1596
1597                         if (ndev->b2b_idx >= ndev->mw_count) {
1598                                 dev_dbg(dev,
1599                                         "b2b_mw_idx %d invalid for mw_count %u\n",
1600                                         b2b_mw_idx, ndev->mw_count);
1601                                 return -EINVAL;
1602                         }
1603
1604                         dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
1605                                 b2b_mw_idx, ndev->b2b_idx);
1606
1607                 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1608                         dev_warn(dev, "Reduce doorbell count by 1\n");
1609                         ndev->db_count -= 1;
1610                 }
1611
1612                 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1613                         rc = xeon_setup_b2b_mw(ndev,
1614                                                &xeon_b2b_dsd_addr,
1615                                                &xeon_b2b_usd_addr);
1616                 } else {
1617                         rc = xeon_setup_b2b_mw(ndev,
1618                                                &xeon_b2b_usd_addr,
1619                                                &xeon_b2b_dsd_addr);
1620                 }
1621                 if (rc)
1622                         return rc;
1623
1624                 /* Enable Bus Master and Memory Space on the secondary side */
1625                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1626                           ndev->self_mmio + XEON_SPCICMD_OFFSET);
1627
1628                 break;
1629
1630         default:
1631                 return -EINVAL;
1632         }
1633
1634         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1635
1636         ndev->reg->db_iowrite(ndev->db_valid_mask,
1637                               ndev->self_mmio +
1638                               ndev->self_reg->db_mask);
1639
1640         return 0;
1641 }
1642
1643 static int xeon_init_dev(struct intel_ntb_dev *ndev)
1644 {
1645         struct pci_dev *pdev;
1646         u8 ppd;
1647         int rc, mem;
1648
1649         pdev = ndev->ntb.pdev;
1650
1651         switch (pdev->device) {
1652         /* There is a Xeon hardware errata related to writes to SDOORBELL or
1653          * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1654          * which may hang the system.  To workaround this use the second memory
1655          * window to access the interrupt and scratch pad registers on the
1656          * remote system.
1657          */
1658         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1659         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1660         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1661         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1662         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1663         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1664         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1665         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1666         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1667         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1668         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1669         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1670         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1671         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1672         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1673                 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1674                 break;
1675         }
1676
1677         switch (pdev->device) {
1678         /* There is a hardware errata related to accessing any register in
1679          * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1680          */
1681         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1682         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1683         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1684         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1685         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1686         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1687         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1688         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1689         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1690                 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1691                 break;
1692         }
1693
1694         switch (pdev->device) {
1695         /* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
1696          * mirrored to the remote system.  Shrink the number of bits by one,
1697          * since bit 14 is the last bit.
1698          */
1699         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1700         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1701         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1702         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1703         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1704         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1705         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1706         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1707         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1708         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1709         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1710         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1711         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1712         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1713         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1714                 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1715                 break;
1716         }
1717
1718         ndev->reg = &xeon_reg;
1719
1720         rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1721         if (rc)
1722                 return -EIO;
1723
1724         ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1725         dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1726                 ntb_topo_string(ndev->ntb.topo));
1727         if (ndev->ntb.topo == NTB_TOPO_NONE)
1728                 return -EINVAL;
1729
1730         if (ndev->ntb.topo != NTB_TOPO_SEC) {
1731                 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1732                 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
1733                         ppd, ndev->bar4_split);
1734         } else {
1735                 /* This is a way for transparent BAR to figure out if we are
1736                  * doing split BAR or not. There is no way for the hw on the
1737                  * transparent side to know and set the PPD.
1738                  */
1739                 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1740                 ndev->bar4_split = hweight32(mem) ==
1741                         HSX_SPLIT_BAR_MW_COUNT + 1;
1742                 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
1743                         mem, ndev->bar4_split);
1744         }
1745
1746         rc = xeon_init_ntb(ndev);
1747         if (rc)
1748                 return rc;
1749
1750         return xeon_init_isr(ndev);
1751 }
1752
1753 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1754 {
1755         xeon_deinit_isr(ndev);
1756 }
1757
1758 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1759 {
1760         int rc;
1761
1762         pci_set_drvdata(pdev, ndev);
1763
1764         rc = pci_enable_device(pdev);
1765         if (rc)
1766                 goto err_pci_enable;
1767
1768         rc = pci_request_regions(pdev, NTB_NAME);
1769         if (rc)
1770                 goto err_pci_regions;
1771
1772         pci_set_master(pdev);
1773
1774         rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1775         if (rc) {
1776                 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1777                 if (rc)
1778                         goto err_dma_mask;
1779                 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1780         }
1781
1782         ndev->self_mmio = pci_iomap(pdev, 0, 0);
1783         if (!ndev->self_mmio) {
1784                 rc = -EIO;
1785                 goto err_mmio;
1786         }
1787         ndev->peer_mmio = ndev->self_mmio;
1788         ndev->peer_addr = pci_resource_start(pdev, 0);
1789
1790         return 0;
1791
1792 err_mmio:
1793 err_dma_mask:
1794         pci_clear_master(pdev);
1795         pci_release_regions(pdev);
1796 err_pci_regions:
1797         pci_disable_device(pdev);
1798 err_pci_enable:
1799         pci_set_drvdata(pdev, NULL);
1800         return rc;
1801 }
1802
1803 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1804 {
1805         struct pci_dev *pdev = ndev->ntb.pdev;
1806
1807         if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1808                 pci_iounmap(pdev, ndev->peer_mmio);
1809         pci_iounmap(pdev, ndev->self_mmio);
1810
1811         pci_clear_master(pdev);
1812         pci_release_regions(pdev);
1813         pci_disable_device(pdev);
1814         pci_set_drvdata(pdev, NULL);
1815 }
1816
1817 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1818                                     struct pci_dev *pdev)
1819 {
1820         ndev->ntb.pdev = pdev;
1821         ndev->ntb.topo = NTB_TOPO_NONE;
1822         ndev->ntb.ops = &intel_ntb_ops;
1823
1824         ndev->b2b_off = 0;
1825         ndev->b2b_idx = UINT_MAX;
1826
1827         ndev->bar4_split = 0;
1828
1829         ndev->mw_count = 0;
1830         ndev->spad_count = 0;
1831         ndev->db_count = 0;
1832         ndev->db_vec_count = 0;
1833         ndev->db_vec_shift = 0;
1834
1835         ndev->ntb_ctl = 0;
1836         ndev->lnk_sta = 0;
1837
1838         ndev->db_valid_mask = 0;
1839         ndev->db_link_mask = 0;
1840         ndev->db_mask = 0;
1841
1842         spin_lock_init(&ndev->db_mask_lock);
1843 }
1844
1845 static int intel_ntb_pci_probe(struct pci_dev *pdev,
1846                                const struct pci_device_id *id)
1847 {
1848         struct intel_ntb_dev *ndev;
1849         int rc, node;
1850
1851         node = dev_to_node(&pdev->dev);
1852         ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1853         if (!ndev) {
1854                 rc = -ENOMEM;
1855                 goto err_ndev;
1856         }
1857
1858         ndev_init_struct(ndev, pdev);
1859
1860         if (pdev_is_gen1(pdev)) {
1861                 rc = intel_ntb_init_pci(ndev, pdev);
1862                 if (rc)
1863                         goto err_init_pci;
1864
1865                 rc = xeon_init_dev(ndev);
1866                 if (rc)
1867                         goto err_init_dev;
1868         } else if (pdev_is_gen3(pdev)) {
1869                 ndev->ntb.ops = &intel_ntb3_ops;
1870                 rc = intel_ntb_init_pci(ndev, pdev);
1871                 if (rc)
1872                         goto err_init_pci;
1873
1874                 rc = gen3_init_dev(ndev);
1875                 if (rc)
1876                         goto err_init_dev;
1877         } else if (pdev_is_gen4(pdev)) {
1878                 ndev->ntb.ops = &intel_ntb4_ops;
1879                 rc = intel_ntb_init_pci(ndev, pdev);
1880                 if (rc)
1881                         goto err_init_pci;
1882
1883                 rc = gen4_init_dev(ndev);
1884                 if (rc)
1885                         goto err_init_dev;
1886         } else {
1887                 rc = -EINVAL;
1888                 goto err_init_pci;
1889         }
1890
1891         ndev_reset_unsafe_flags(ndev);
1892
1893         ndev->reg->poll_link(ndev);
1894
1895         ndev_init_debugfs(ndev);
1896
1897         rc = ntb_register_device(&ndev->ntb);
1898         if (rc)
1899                 goto err_register;
1900
1901         dev_info(&pdev->dev, "NTB device registered.\n");
1902
1903         return 0;
1904
1905 err_register:
1906         ndev_deinit_debugfs(ndev);
1907         if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) || pdev_is_gen4(pdev))
1908                 xeon_deinit_dev(ndev);
1909 err_init_dev:
1910         intel_ntb_deinit_pci(ndev);
1911 err_init_pci:
1912         kfree(ndev);
1913 err_ndev:
1914         return rc;
1915 }
1916
1917 static void intel_ntb_pci_remove(struct pci_dev *pdev)
1918 {
1919         struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
1920
1921         ntb_unregister_device(&ndev->ntb);
1922         ndev_deinit_debugfs(ndev);
1923         if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) || pdev_is_gen4(pdev))
1924                 xeon_deinit_dev(ndev);
1925         intel_ntb_deinit_pci(ndev);
1926         kfree(ndev);
1927 }
1928
1929 static const struct intel_ntb_reg xeon_reg = {
1930         .poll_link              = xeon_poll_link,
1931         .link_is_up             = xeon_link_is_up,
1932         .db_ioread              = xeon_db_ioread,
1933         .db_iowrite             = xeon_db_iowrite,
1934         .db_size                = sizeof(u32),
1935         .ntb_ctl                = XEON_NTBCNTL_OFFSET,
1936         .mw_bar                 = {2, 4, 5},
1937 };
1938
1939 static const struct intel_ntb_alt_reg xeon_pri_reg = {
1940         .db_bell                = XEON_PDOORBELL_OFFSET,
1941         .db_mask                = XEON_PDBMSK_OFFSET,
1942         .spad                   = XEON_SPAD_OFFSET,
1943 };
1944
1945 static const struct intel_ntb_alt_reg xeon_sec_reg = {
1946         .db_bell                = XEON_SDOORBELL_OFFSET,
1947         .db_mask                = XEON_SDBMSK_OFFSET,
1948         /* second half of the scratchpads */
1949         .spad                   = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
1950 };
1951
1952 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
1953         .db_bell                = XEON_B2B_DOORBELL_OFFSET,
1954         .spad                   = XEON_B2B_SPAD_OFFSET,
1955 };
1956
1957 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
1958         /* Note: no primary .bar0_base visible to the secondary side.
1959          *
1960          * The secondary side cannot get the base address stored in primary
1961          * bars.  The base address is necessary to set the limit register to
1962          * any value other than zero, or unlimited.
1963          *
1964          * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
1965          * window by setting the limit equal to base, nor can it limit the size
1966          * of the memory window by setting the limit to base + size.
1967          */
1968         .bar2_limit             = XEON_PBAR23LMT_OFFSET,
1969         .bar2_xlat              = XEON_PBAR23XLAT_OFFSET,
1970 };
1971
1972 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
1973         .bar0_base              = XEON_SBAR0BASE_OFFSET,
1974         .bar2_limit             = XEON_SBAR23LMT_OFFSET,
1975         .bar2_xlat              = XEON_SBAR23XLAT_OFFSET,
1976 };
1977
1978 struct intel_b2b_addr xeon_b2b_usd_addr = {
1979         .bar2_addr64            = XEON_B2B_BAR2_ADDR64,
1980         .bar4_addr64            = XEON_B2B_BAR4_ADDR64,
1981         .bar4_addr32            = XEON_B2B_BAR4_ADDR32,
1982         .bar5_addr32            = XEON_B2B_BAR5_ADDR32,
1983 };
1984
1985 struct intel_b2b_addr xeon_b2b_dsd_addr = {
1986         .bar2_addr64            = XEON_B2B_BAR2_ADDR64,
1987         .bar4_addr64            = XEON_B2B_BAR4_ADDR64,
1988         .bar4_addr32            = XEON_B2B_BAR4_ADDR32,
1989         .bar5_addr32            = XEON_B2B_BAR5_ADDR32,
1990 };
1991
1992 /* operations for primary side of local ntb */
1993 static const struct ntb_dev_ops intel_ntb_ops = {
1994         .mw_count               = intel_ntb_mw_count,
1995         .mw_get_align           = intel_ntb_mw_get_align,
1996         .mw_set_trans           = intel_ntb_mw_set_trans,
1997         .peer_mw_count          = intel_ntb_peer_mw_count,
1998         .peer_mw_get_addr       = intel_ntb_peer_mw_get_addr,
1999         .link_is_up             = intel_ntb_link_is_up,
2000         .link_enable            = intel_ntb_link_enable,
2001         .link_disable           = intel_ntb_link_disable,
2002         .db_is_unsafe           = intel_ntb_db_is_unsafe,
2003         .db_valid_mask          = intel_ntb_db_valid_mask,
2004         .db_vector_count        = intel_ntb_db_vector_count,
2005         .db_vector_mask         = intel_ntb_db_vector_mask,
2006         .db_read                = intel_ntb_db_read,
2007         .db_clear               = intel_ntb_db_clear,
2008         .db_set_mask            = intel_ntb_db_set_mask,
2009         .db_clear_mask          = intel_ntb_db_clear_mask,
2010         .peer_db_addr           = intel_ntb_peer_db_addr,
2011         .peer_db_set            = intel_ntb_peer_db_set,
2012         .spad_is_unsafe         = intel_ntb_spad_is_unsafe,
2013         .spad_count             = intel_ntb_spad_count,
2014         .spad_read              = intel_ntb_spad_read,
2015         .spad_write             = intel_ntb_spad_write,
2016         .peer_spad_addr         = intel_ntb_peer_spad_addr,
2017         .peer_spad_read         = intel_ntb_peer_spad_read,
2018         .peer_spad_write        = intel_ntb_peer_spad_write,
2019 };
2020
2021 static const struct file_operations intel_ntb_debugfs_info = {
2022         .owner = THIS_MODULE,
2023         .open = simple_open,
2024         .read = ndev_debugfs_read,
2025 };
2026
2027 static const struct pci_device_id intel_ntb_pci_tbl[] = {
2028         /* GEN1 */
2029         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2030         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2031         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2032         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2033         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2034         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2035         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2036         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2037         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2038         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2039         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2040         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2041         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2042         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2043         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2044
2045         /* GEN3 */
2046         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
2047
2048         /* GEN4 */
2049         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)},
2050         {0}
2051 };
2052 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2053
2054 static struct pci_driver intel_ntb_pci_driver = {
2055         .name = KBUILD_MODNAME,
2056         .id_table = intel_ntb_pci_tbl,
2057         .probe = intel_ntb_pci_probe,
2058         .remove = intel_ntb_pci_remove,
2059 };
2060
2061 static int __init intel_ntb_pci_driver_init(void)
2062 {
2063         pr_info("%s %s\n", NTB_DESC, NTB_VER);
2064
2065         if (debugfs_initialized())
2066                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2067
2068         return pci_register_driver(&intel_ntb_pci_driver);
2069 }
2070 module_init(intel_ntb_pci_driver_init);
2071
2072 static void __exit intel_ntb_pci_driver_exit(void)
2073 {
2074         pci_unregister_driver(&intel_ntb_pci_driver);
2075
2076         debugfs_remove_recursive(debugfs_dir);
2077 }
2078 module_exit(intel_ntb_pci_driver_exit);