drivers/infiniband/hw/hfi1/chip.c

   1 /*
   2  * Copyright(c) 2015 - 2018 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 /*
  49  * This file contains all of the code that is specific to the HFI chip
  50  */
  51
  52 #include <linux/pci.h>
  53 #include <linux/delay.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/module.h>
  56
  57 #include "hfi.h"
  58 #include "trace.h"
  59 #include "mad.h"
  60 #include "pio.h"
  61 #include "sdma.h"
  62 #include "eprom.h"
  63 #include "efivar.h"
  64 #include "platform.h"
  65 #include "aspm.h"
  66 #include "affinity.h"
  67 #include "debugfs.h"
  68 #include "fault.h"
  69
  70 uint kdeth_qp;
  71 module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO);
  72 MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix");
  73
  74 uint num_vls = HFI1_MAX_VLS_SUPPORTED;
  75 module_param(num_vls, uint, S_IRUGO);
  76 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)");
  77
  78 /*
  79  * Default time to aggregate two 10K packets from the idle state
  80  * (timer not running). The timer starts at the end of the first packet,
  81  * so only the time for one 10K packet and header plus a bit extra is needed.
  82  * 10 * 1024 + 64 header byte = 10304 byte
  83  * 10304 byte / 12.5 GB/s = 824.32ns
  84  */
  85 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */
  86 module_param(rcv_intr_timeout, uint, S_IRUGO);
  87 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns");
  88
  89 uint rcv_intr_count = 16; /* same as qib */
  90 module_param(rcv_intr_count, uint, S_IRUGO);
  91 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count");
  92
  93 ushort link_crc_mask = SUPPORTED_CRCS;
  94 module_param(link_crc_mask, ushort, S_IRUGO);
  95 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link");
  96
  97 uint loopback;
  98 module_param_named(loopback, loopback, uint, S_IRUGO);
  99 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable");
 100
 101 /* Other driver tunables */
 102 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/
 103 static ushort crc_14b_sideband = 1;
 104 static uint use_flr = 1;
 105 uint quick_linkup; /* skip LNI */
 106
 107 struct flag_table {
 108         u64 flag;       /* the flag */
 109         char *str;      /* description string */
 110         u16 extra;      /* extra information */
 111         u16 unused0;
 112         u32 unused1;
 113 };
 114
 115 /* str must be a string constant */
 116 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra}
 117 #define FLAG_ENTRY0(str, flag) {flag, str, 0}
 118
 119 /* Send Error Consequences */
 120 #define SEC_WRITE_DROPPED       0x1
 121 #define SEC_PACKET_DROPPED      0x2
 122 #define SEC_SC_HALTED           0x4     /* per-context only */
 123 #define SEC_SPC_FREEZE          0x8     /* per-HFI only */
 124
 125 #define DEFAULT_KRCVQS            2
 126 #define MIN_KERNEL_KCTXTS         2
 127 #define FIRST_KERNEL_KCTXT        1
 128
 129 /*
 130  * RSM instance allocation
 131  *   0 - Verbs
 132  *   1 - User Fecn Handling
 133  *   2 - Vnic
 134  */
 135 #define RSM_INS_VERBS             0
 136 #define RSM_INS_FECN              1
 137 #define RSM_INS_VNIC              2
 138
 139 /* Bit offset into the GUID which carries HFI id information */
 140 #define GUID_HFI_INDEX_SHIFT     39
 141
 142 /* extract the emulation revision */
 143 #define emulator_rev(dd) ((dd)->irev >> 8)
 144 /* parallel and serial emulation versions are 3 and 4 respectively */
 145 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3)
 146 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4)
 147
 148 /* RSM fields for Verbs */
 149 /* packet type */
 150 #define IB_PACKET_TYPE         2ull
 151 #define QW_SHIFT               6ull
 152 /* QPN[7..1] */
 153 #define QPN_WIDTH              7ull
 154
 155 /* LRH.BTH: QW 0, OFFSET 48 - for match */
 156 #define LRH_BTH_QW             0ull
 157 #define LRH_BTH_BIT_OFFSET     48ull
 158 #define LRH_BTH_OFFSET(off)    ((LRH_BTH_QW << QW_SHIFT) | (off))
 159 #define LRH_BTH_MATCH_OFFSET   LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET)
 160 #define LRH_BTH_SELECT
 161 #define LRH_BTH_MASK           3ull
 162 #define LRH_BTH_VALUE          2ull
 163
 164 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */
 165 #define LRH_SC_QW              0ull
 166 #define LRH_SC_BIT_OFFSET      56ull
 167 #define LRH_SC_OFFSET(off)     ((LRH_SC_QW << QW_SHIFT) | (off))
 168 #define LRH_SC_MATCH_OFFSET    LRH_SC_OFFSET(LRH_SC_BIT_OFFSET)
 169 #define LRH_SC_MASK            128ull
 170 #define LRH_SC_VALUE           0ull
 171
 172 /* SC[n..0] QW 0, OFFSET 60 - for select */
 173 #define LRH_SC_SELECT_OFFSET  ((LRH_SC_QW << QW_SHIFT) | (60ull))
 174
 175 /* QPN[m+n:1] QW 1, OFFSET 1 */
 176 #define QPN_SELECT_OFFSET      ((1ull << QW_SHIFT) | (1ull))
 177
 178 /* RSM fields for Vnic */
 179 /* L2_TYPE: QW 0, OFFSET 61 - for match */
 180 #define L2_TYPE_QW             0ull
 181 #define L2_TYPE_BIT_OFFSET     61ull
 182 #define L2_TYPE_OFFSET(off)    ((L2_TYPE_QW << QW_SHIFT) | (off))
 183 #define L2_TYPE_MATCH_OFFSET   L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET)
 184 #define L2_TYPE_MASK           3ull
 185 #define L2_16B_VALUE           2ull
 186
 187 /* L4_TYPE QW 1, OFFSET 0 - for match */
 188 #define L4_TYPE_QW              1ull
 189 #define L4_TYPE_BIT_OFFSET      0ull
 190 #define L4_TYPE_OFFSET(off)     ((L4_TYPE_QW << QW_SHIFT) | (off))
 191 #define L4_TYPE_MATCH_OFFSET    L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET)
 192 #define L4_16B_TYPE_MASK        0xFFull
 193 #define L4_16B_ETH_VALUE        0x78ull
 194
 195 /* 16B VESWID - for select */
 196 #define L4_16B_HDR_VESWID_OFFSET  ((2 << QW_SHIFT) | (16ull))
 197 /* 16B ENTROPY - for select */
 198 #define L2_16B_ENTROPY_OFFSET     ((1 << QW_SHIFT) | (32ull))
 199
 200 /* defines to build power on SC2VL table */
 201 #define SC2VL_VAL( \
 202         num, \
 203         sc0, sc0val, \
 204         sc1, sc1val, \
 205         sc2, sc2val, \
 206         sc3, sc3val, \
 207         sc4, sc4val, \
 208         sc5, sc5val, \
 209         sc6, sc6val, \
 210         sc7, sc7val) \
 211 ( \
 212         ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \
 213         ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \
 214         ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \
 215         ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \
 216         ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \
 217         ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \
 218         ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \
 219         ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT)   \
 220 )
 221
 222 #define DC_SC_VL_VAL( \
 223         range, \
 224         e0, e0val, \
 225         e1, e1val, \
 226         e2, e2val, \
 227         e3, e3val, \
 228         e4, e4val, \
 229         e5, e5val, \
 230         e6, e6val, \
 231         e7, e7val, \
 232         e8, e8val, \
 233         e9, e9val, \
 234         e10, e10val, \
 235         e11, e11val, \
 236         e12, e12val, \
 237         e13, e13val, \
 238         e14, e14val, \
 239         e15, e15val) \
 240 ( \
 241         ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \
 242         ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \
 243         ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \
 244         ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \
 245         ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \
 246         ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \
 247         ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \
 248         ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \
 249         ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \
 250         ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \
 251         ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \
 252         ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \
 253         ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \
 254         ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \
 255         ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \
 256         ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \
 257 )
 258
 259 /* all CceStatus sub-block freeze bits */
 260 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \
 261                         | CCE_STATUS_RXE_FROZE_SMASK \
 262                         | CCE_STATUS_TXE_FROZE_SMASK \
 263                         | CCE_STATUS_TXE_PIO_FROZE_SMASK)
 264 /* all CceStatus sub-block TXE pause bits */
 265 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \
 266                         | CCE_STATUS_TXE_PAUSED_SMASK \
 267                         | CCE_STATUS_SDMA_PAUSED_SMASK)
 268 /* all CceStatus sub-block RXE pause bits */
 269 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK
 270
 271 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL
 272 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF
 273
 274 /*
 275  * CCE Error flags.
 276  */
 277 static struct flag_table cce_err_status_flags[] = {
 278 /* 0*/  FLAG_ENTRY0("CceCsrParityErr",
 279                 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK),
 280 /* 1*/  FLAG_ENTRY0("CceCsrReadBadAddrErr",
 281                 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK),
 282 /* 2*/  FLAG_ENTRY0("CceCsrWriteBadAddrErr",
 283                 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK),
 284 /* 3*/  FLAG_ENTRY0("CceTrgtAsyncFifoParityErr",
 285                 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK),
 286 /* 4*/  FLAG_ENTRY0("CceTrgtAccessErr",
 287                 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK),
 288 /* 5*/  FLAG_ENTRY0("CceRspdDataParityErr",
 289                 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK),
 290 /* 6*/  FLAG_ENTRY0("CceCli0AsyncFifoParityErr",
 291                 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK),
 292 /* 7*/  FLAG_ENTRY0("CceCsrCfgBusParityErr",
 293                 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK),
 294 /* 8*/  FLAG_ENTRY0("CceCli2AsyncFifoParityErr",
 295                 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK),
 296 /* 9*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
 297             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK),
 298 /*10*/  FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr",
 299             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK),
 300 /*11*/  FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError",
 301             CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK),
 302 /*12*/  FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError",
 303                 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK),
 304 /*13*/  FLAG_ENTRY0("PcicRetryMemCorErr",
 305                 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK),
 306 /*14*/  FLAG_ENTRY0("PcicRetryMemCorErr",
 307                 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK),
 308 /*15*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 309                 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK),
 310 /*16*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 311                 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK),
 312 /*17*/  FLAG_ENTRY0("PcicPostHdQCorErr",
 313                 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK),
 314 /*18*/  FLAG_ENTRY0("PcicCplDatQCorErr",
 315                 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK),
 316 /*19*/  FLAG_ENTRY0("PcicNPostHQParityErr",
 317                 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK),
 318 /*20*/  FLAG_ENTRY0("PcicNPostDatQParityErr",
 319                 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK),
 320 /*21*/  FLAG_ENTRY0("PcicRetryMemUncErr",
 321                 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK),
 322 /*22*/  FLAG_ENTRY0("PcicRetrySotMemUncErr",
 323                 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK),
 324 /*23*/  FLAG_ENTRY0("PcicPostHdQUncErr",
 325                 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK),
 326 /*24*/  FLAG_ENTRY0("PcicPostDatQUncErr",
 327                 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK),
 328 /*25*/  FLAG_ENTRY0("PcicCplHdQUncErr",
 329                 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK),
 330 /*26*/  FLAG_ENTRY0("PcicCplDatQUncErr",
 331                 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK),
 332 /*27*/  FLAG_ENTRY0("PcicTransmitFrontParityErr",
 333                 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK),
 334 /*28*/  FLAG_ENTRY0("PcicTransmitBackParityErr",
 335                 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK),
 336 /*29*/  FLAG_ENTRY0("PcicReceiveParityErr",
 337                 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK),
 338 /*30*/  FLAG_ENTRY0("CceTrgtCplTimeoutErr",
 339                 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK),
 340 /*31*/  FLAG_ENTRY0("LATriggered",
 341                 CCE_ERR_STATUS_LA_TRIGGERED_SMASK),
 342 /*32*/  FLAG_ENTRY0("CceSegReadBadAddrErr",
 343                 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK),
 344 /*33*/  FLAG_ENTRY0("CceSegWriteBadAddrErr",
 345                 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK),
 346 /*34*/  FLAG_ENTRY0("CceRcplAsyncFifoParityErr",
 347                 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK),
 348 /*35*/  FLAG_ENTRY0("CceRxdmaConvFifoParityErr",
 349                 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK),
 350 /*36*/  FLAG_ENTRY0("CceMsixTableCorErr",
 351                 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK),
 352 /*37*/  FLAG_ENTRY0("CceMsixTableUncErr",
 353                 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK),
 354 /*38*/  FLAG_ENTRY0("CceIntMapCorErr",
 355                 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK),
 356 /*39*/  FLAG_ENTRY0("CceIntMapUncErr",
 357                 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK),
 358 /*40*/  FLAG_ENTRY0("CceMsixCsrParityErr",
 359                 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK),
 360 /*41-63 reserved*/
 361 };
 362
 363 /*
 364  * Misc Error flags
 365  */
 366 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK
 367 static struct flag_table misc_err_status_flags[] = {
 368 /* 0*/  FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)),
 369 /* 1*/  FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)),
 370 /* 2*/  FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)),
 371 /* 3*/  FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)),
 372 /* 4*/  FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)),
 373 /* 5*/  FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)),
 374 /* 6*/  FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)),
 375 /* 7*/  FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)),
 376 /* 8*/  FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)),
 377 /* 9*/  FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)),
 378 /*10*/  FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)),
 379 /*11*/  FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)),
 380 /*12*/  FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL))
 381 };
 382
 383 /*
 384  * TXE PIO Error flags and consequences
 385  */
 386 static struct flag_table pio_err_status_flags[] = {
 387 /* 0*/  FLAG_ENTRY("PioWriteBadCtxt",
 388         SEC_WRITE_DROPPED,
 389         SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK),
 390 /* 1*/  FLAG_ENTRY("PioWriteAddrParity",
 391         SEC_SPC_FREEZE,
 392         SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK),
 393 /* 2*/  FLAG_ENTRY("PioCsrParity",
 394         SEC_SPC_FREEZE,
 395         SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK),
 396 /* 3*/  FLAG_ENTRY("PioSbMemFifo0",
 397         SEC_SPC_FREEZE,
 398         SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK),
 399 /* 4*/  FLAG_ENTRY("PioSbMemFifo1",
 400         SEC_SPC_FREEZE,
 401         SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK),
 402 /* 5*/  FLAG_ENTRY("PioPccFifoParity",
 403         SEC_SPC_FREEZE,
 404         SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK),
 405 /* 6*/  FLAG_ENTRY("PioPecFifoParity",
 406         SEC_SPC_FREEZE,
 407         SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK),
 408 /* 7*/  FLAG_ENTRY("PioSbrdctlCrrelParity",
 409         SEC_SPC_FREEZE,
 410         SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK),
 411 /* 8*/  FLAG_ENTRY("PioSbrdctrlCrrelFifoParity",
 412         SEC_SPC_FREEZE,
 413         SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK),
 414 /* 9*/  FLAG_ENTRY("PioPktEvictFifoParityErr",
 415         SEC_SPC_FREEZE,
 416         SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK),
 417 /*10*/  FLAG_ENTRY("PioSmPktResetParity",
 418         SEC_SPC_FREEZE,
 419         SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK),
 420 /*11*/  FLAG_ENTRY("PioVlLenMemBank0Unc",
 421         SEC_SPC_FREEZE,
 422         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK),
 423 /*12*/  FLAG_ENTRY("PioVlLenMemBank1Unc",
 424         SEC_SPC_FREEZE,
 425         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK),
 426 /*13*/  FLAG_ENTRY("PioVlLenMemBank0Cor",
 427         0,
 428         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK),
 429 /*14*/  FLAG_ENTRY("PioVlLenMemBank1Cor",
 430         0,
 431         SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK),
 432 /*15*/  FLAG_ENTRY("PioCreditRetFifoParity",
 433         SEC_SPC_FREEZE,
 434         SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK),
 435 /*16*/  FLAG_ENTRY("PioPpmcPblFifo",
 436         SEC_SPC_FREEZE,
 437         SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK),
 438 /*17*/  FLAG_ENTRY("PioInitSmIn",
 439         0,
 440         SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK),
 441 /*18*/  FLAG_ENTRY("PioPktEvictSmOrArbSm",
 442         SEC_SPC_FREEZE,
 443         SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK),
 444 /*19*/  FLAG_ENTRY("PioHostAddrMemUnc",
 445         SEC_SPC_FREEZE,
 446         SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK),
 447 /*20*/  FLAG_ENTRY("PioHostAddrMemCor",
 448         0,
 449         SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK),
 450 /*21*/  FLAG_ENTRY("PioWriteDataParity",
 451         SEC_SPC_FREEZE,
 452         SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK),
 453 /*22*/  FLAG_ENTRY("PioStateMachine",
 454         SEC_SPC_FREEZE,
 455         SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK),
 456 /*23*/  FLAG_ENTRY("PioWriteQwValidParity",
 457         SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
 458         SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK),
 459 /*24*/  FLAG_ENTRY("PioBlockQwCountParity",
 460         SEC_WRITE_DROPPED | SEC_SPC_FREEZE,
 461         SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK),
 462 /*25*/  FLAG_ENTRY("PioVlfVlLenParity",
 463         SEC_SPC_FREEZE,
 464         SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK),
 465 /*26*/  FLAG_ENTRY("PioVlfSopParity",
 466         SEC_SPC_FREEZE,
 467         SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK),
 468 /*27*/  FLAG_ENTRY("PioVlFifoParity",
 469         SEC_SPC_FREEZE,
 470         SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK),
 471 /*28*/  FLAG_ENTRY("PioPpmcBqcMemParity",
 472         SEC_SPC_FREEZE,
 473         SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK),
 474 /*29*/  FLAG_ENTRY("PioPpmcSopLen",
 475         SEC_SPC_FREEZE,
 476         SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK),
 477 /*30-31 reserved*/
 478 /*32*/  FLAG_ENTRY("PioCurrentFreeCntParity",
 479         SEC_SPC_FREEZE,
 480         SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK),
 481 /*33*/  FLAG_ENTRY("PioLastReturnedCntParity",
 482         SEC_SPC_FREEZE,
 483         SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK),
 484 /*34*/  FLAG_ENTRY("PioPccSopHeadParity",
 485         SEC_SPC_FREEZE,
 486         SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK),
 487 /*35*/  FLAG_ENTRY("PioPecSopHeadParityErr",
 488         SEC_SPC_FREEZE,
 489         SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK),
 490 /*36-63 reserved*/
 491 };
 492
 493 /* TXE PIO errors that cause an SPC freeze */
 494 #define ALL_PIO_FREEZE_ERR \
 495         (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \
 496         | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \
 497         | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \
 498         | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \
 499         | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \
 500         | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \
 501         | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \
 502         | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \
 503         | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \
 504         | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \
 505         | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \
 506         | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \
 507         | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \
 508         | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \
 509         | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \
 510         | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \
 511         | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \
 512         | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \
 513         | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \
 514         | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \
 515         | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \
 516         | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \
 517         | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \
 518         | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \
 519         | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \
 520         | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \
 521         | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \
 522         | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \
 523         | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK)
 524
 525 /*
 526  * TXE SDMA Error flags
 527  */
 528 static struct flag_table sdma_err_status_flags[] = {
 529 /* 0*/  FLAG_ENTRY0("SDmaRpyTagErr",
 530                 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK),
 531 /* 1*/  FLAG_ENTRY0("SDmaCsrParityErr",
 532                 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK),
 533 /* 2*/  FLAG_ENTRY0("SDmaPcieReqTrackingUncErr",
 534                 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK),
 535 /* 3*/  FLAG_ENTRY0("SDmaPcieReqTrackingCorErr",
 536                 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK),
 537 /*04-63 reserved*/
 538 };
 539
 540 /* TXE SDMA errors that cause an SPC freeze */
 541 #define ALL_SDMA_FREEZE_ERR  \
 542                 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \
 543                 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \
 544                 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK)
 545
 546 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */
 547 #define PORT_DISCARD_EGRESS_ERRS \
 548         (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \
 549         | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \
 550         | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK)
 551
 552 /*
 553  * TXE Egress Error flags
 554  */
 555 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK
 556 static struct flag_table egress_err_status_flags[] = {
 557 /* 0*/  FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)),
 558 /* 1*/  FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)),
 559 /* 2 reserved */
 560 /* 3*/  FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr",
 561                 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)),
 562 /* 4*/  FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)),
 563 /* 5*/  FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)),
 564 /* 6 reserved */
 565 /* 7*/  FLAG_ENTRY0("TxPioLaunchIntfParityErr",
 566                 SEES(TX_PIO_LAUNCH_INTF_PARITY)),
 567 /* 8*/  FLAG_ENTRY0("TxSdmaLaunchIntfParityErr",
 568                 SEES(TX_SDMA_LAUNCH_INTF_PARITY)),
 569 /* 9-10 reserved */
 570 /*11*/  FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr",
 571                 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)),
 572 /*12*/  FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)),
 573 /*13*/  FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)),
 574 /*14*/  FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)),
 575 /*15*/  FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)),
 576 /*16*/  FLAG_ENTRY0("TxSdma0DisallowedPacketErr",
 577                 SEES(TX_SDMA0_DISALLOWED_PACKET)),
 578 /*17*/  FLAG_ENTRY0("TxSdma1DisallowedPacketErr",
 579                 SEES(TX_SDMA1_DISALLOWED_PACKET)),
 580 /*18*/  FLAG_ENTRY0("TxSdma2DisallowedPacketErr",
 581                 SEES(TX_SDMA2_DISALLOWED_PACKET)),
 582 /*19*/  FLAG_ENTRY0("TxSdma3DisallowedPacketErr",
 583                 SEES(TX_SDMA3_DISALLOWED_PACKET)),
 584 /*20*/  FLAG_ENTRY0("TxSdma4DisallowedPacketErr",
 585                 SEES(TX_SDMA4_DISALLOWED_PACKET)),
 586 /*21*/  FLAG_ENTRY0("TxSdma5DisallowedPacketErr",
 587                 SEES(TX_SDMA5_DISALLOWED_PACKET)),
 588 /*22*/  FLAG_ENTRY0("TxSdma6DisallowedPacketErr",
 589                 SEES(TX_SDMA6_DISALLOWED_PACKET)),
 590 /*23*/  FLAG_ENTRY0("TxSdma7DisallowedPacketErr",
 591                 SEES(TX_SDMA7_DISALLOWED_PACKET)),
 592 /*24*/  FLAG_ENTRY0("TxSdma8DisallowedPacketErr",
 593                 SEES(TX_SDMA8_DISALLOWED_PACKET)),
 594 /*25*/  FLAG_ENTRY0("TxSdma9DisallowedPacketErr",
 595                 SEES(TX_SDMA9_DISALLOWED_PACKET)),
 596 /*26*/  FLAG_ENTRY0("TxSdma10DisallowedPacketErr",
 597                 SEES(TX_SDMA10_DISALLOWED_PACKET)),
 598 /*27*/  FLAG_ENTRY0("TxSdma11DisallowedPacketErr",
 599                 SEES(TX_SDMA11_DISALLOWED_PACKET)),
 600 /*28*/  FLAG_ENTRY0("TxSdma12DisallowedPacketErr",
 601                 SEES(TX_SDMA12_DISALLOWED_PACKET)),
 602 /*29*/  FLAG_ENTRY0("TxSdma13DisallowedPacketErr",
 603                 SEES(TX_SDMA13_DISALLOWED_PACKET)),
 604 /*30*/  FLAG_ENTRY0("TxSdma14DisallowedPacketErr",
 605                 SEES(TX_SDMA14_DISALLOWED_PACKET)),
 606 /*31*/  FLAG_ENTRY0("TxSdma15DisallowedPacketErr",
 607                 SEES(TX_SDMA15_DISALLOWED_PACKET)),
 608 /*32*/  FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr",
 609                 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)),
 610 /*33*/  FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr",
 611                 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)),
 612 /*34*/  FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr",
 613                 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)),
 614 /*35*/  FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr",
 615                 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)),
 616 /*36*/  FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr",
 617                 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)),
 618 /*37*/  FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr",
 619                 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)),
 620 /*38*/  FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr",
 621                 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)),
 622 /*39*/  FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr",
 623                 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)),
 624 /*40*/  FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr",
 625                 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)),
 626 /*41*/  FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)),
 627 /*42*/  FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)),
 628 /*43*/  FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)),
 629 /*44*/  FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)),
 630 /*45*/  FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)),
 631 /*46*/  FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)),
 632 /*47*/  FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)),
 633 /*48*/  FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)),
 634 /*49*/  FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)),
 635 /*50*/  FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)),
 636 /*51*/  FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)),
 637 /*52*/  FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)),
 638 /*53*/  FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)),
 639 /*54*/  FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)),
 640 /*55*/  FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)),
 641 /*56*/  FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)),
 642 /*57*/  FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)),
 643 /*58*/  FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)),
 644 /*59*/  FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)),
 645 /*60*/  FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)),
 646 /*61*/  FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)),
 647 /*62*/  FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr",
 648                 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)),
 649 /*63*/  FLAG_ENTRY0("TxReadPioMemoryCsrUncErr",
 650                 SEES(TX_READ_PIO_MEMORY_CSR_UNC)),
 651 };
 652
 653 /*
 654  * TXE Egress Error Info flags
 655  */
 656 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK
 657 static struct flag_table egress_err_info_flags[] = {
 658 /* 0*/  FLAG_ENTRY0("Reserved", 0ull),
 659 /* 1*/  FLAG_ENTRY0("VLErr", SEEI(VL)),
 660 /* 2*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
 661 /* 3*/  FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)),
 662 /* 4*/  FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)),
 663 /* 5*/  FLAG_ENTRY0("SLIDErr", SEEI(SLID)),
 664 /* 6*/  FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)),
 665 /* 7*/  FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)),
 666 /* 8*/  FLAG_ENTRY0("RawErr", SEEI(RAW)),
 667 /* 9*/  FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)),
 668 /*10*/  FLAG_ENTRY0("GRHErr", SEEI(GRH)),
 669 /*11*/  FLAG_ENTRY0("BypassErr", SEEI(BYPASS)),
 670 /*12*/  FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)),
 671 /*13*/  FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)),
 672 /*14*/  FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)),
 673 /*15*/  FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)),
 674 /*16*/  FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)),
 675 /*17*/  FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)),
 676 /*18*/  FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)),
 677 /*19*/  FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)),
 678 /*20*/  FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)),
 679 /*21*/  FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)),
 680 };
 681
 682 /* TXE Egress errors that cause an SPC freeze */
 683 #define ALL_TXE_EGRESS_FREEZE_ERR \
 684         (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \
 685         | SEES(TX_PIO_LAUNCH_INTF_PARITY) \
 686         | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \
 687         | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \
 688         | SEES(TX_LAUNCH_CSR_PARITY) \
 689         | SEES(TX_SBRD_CTL_CSR_PARITY) \
 690         | SEES(TX_CONFIG_PARITY) \
 691         | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \
 692         | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \
 693         | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \
 694         | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \
 695         | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \
 696         | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \
 697         | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \
 698         | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \
 699         | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \
 700         | SEES(TX_CREDIT_RETURN_PARITY))
 701
 702 /*
 703  * TXE Send error flags
 704  */
 705 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK
 706 static struct flag_table send_err_status_flags[] = {
 707 /* 0*/  FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)),
 708 /* 1*/  FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)),
 709 /* 2*/  FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR))
 710 };
 711
 712 /*
 713  * TXE Send Context Error flags and consequences
 714  */
 715 static struct flag_table sc_err_status_flags[] = {
 716 /* 0*/  FLAG_ENTRY("InconsistentSop",
 717                 SEC_PACKET_DROPPED | SEC_SC_HALTED,
 718                 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK),
 719 /* 1*/  FLAG_ENTRY("DisallowedPacket",
 720                 SEC_PACKET_DROPPED | SEC_SC_HALTED,
 721                 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK),
 722 /* 2*/  FLAG_ENTRY("WriteCrossesBoundary",
 723                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 724                 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK),
 725 /* 3*/  FLAG_ENTRY("WriteOverflow",
 726                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 727                 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK),
 728 /* 4*/  FLAG_ENTRY("WriteOutOfBounds",
 729                 SEC_WRITE_DROPPED | SEC_SC_HALTED,
 730                 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK),
 731 /* 5-63 reserved*/
 732 };
 733
 734 /*
 735  * RXE Receive Error flags
 736  */
 737 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK
 738 static struct flag_table rxe_err_status_flags[] = {
 739 /* 0*/  FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)),
 740 /* 1*/  FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)),
 741 /* 2*/  FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)),
 742 /* 3*/  FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)),
 743 /* 4*/  FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)),
 744 /* 5*/  FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)),
 745 /* 6*/  FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)),
 746 /* 7*/  FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)),
 747 /* 8*/  FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)),
 748 /* 9*/  FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)),
 749 /*10*/  FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)),
 750 /*11*/  FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)),
 751 /*12*/  FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)),
 752 /*13*/  FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)),
 753 /*14*/  FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)),
 754 /*15*/  FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)),
 755 /*16*/  FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr",
 756                 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)),
 757 /*17*/  FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)),
 758 /*18*/  FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)),
 759 /*19*/  FLAG_ENTRY0("RxRbufBlockListReadUncErr",
 760                 RXES(RBUF_BLOCK_LIST_READ_UNC)),
 761 /*20*/  FLAG_ENTRY0("RxRbufBlockListReadCorErr",
 762                 RXES(RBUF_BLOCK_LIST_READ_COR)),
 763 /*21*/  FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr",
 764                 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)),
 765 /*22*/  FLAG_ENTRY0("RxRbufCsrQEntCntParityErr",
 766                 RXES(RBUF_CSR_QENT_CNT_PARITY)),
 767 /*23*/  FLAG_ENTRY0("RxRbufCsrQNextBufParityErr",
 768                 RXES(RBUF_CSR_QNEXT_BUF_PARITY)),
 769 /*24*/  FLAG_ENTRY0("RxRbufCsrQVldBitParityErr",
 770                 RXES(RBUF_CSR_QVLD_BIT_PARITY)),
 771 /*25*/  FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)),
 772 /*26*/  FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)),
 773 /*27*/  FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr",
 774                 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)),
 775 /*28*/  FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)),
 776 /*29*/  FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)),
 777 /*30*/  FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)),
 778 /*31*/  FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)),
 779 /*32*/  FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)),
 780 /*33*/  FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)),
 781 /*34*/  FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)),
 782 /*35*/  FLAG_ENTRY0("RxRbufFlInitdoneParityErr",
 783                 RXES(RBUF_FL_INITDONE_PARITY)),
 784 /*36*/  FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr",
 785                 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)),
 786 /*37*/  FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)),
 787 /*38*/  FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)),
 788 /*39*/  FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)),
 789 /*40*/  FLAG_ENTRY0("RxLookupDesPart1UncCorErr",
 790                 RXES(LOOKUP_DES_PART1_UNC_COR)),
 791 /*41*/  FLAG_ENTRY0("RxLookupDesPart2ParityErr",
 792                 RXES(LOOKUP_DES_PART2_PARITY)),
 793 /*42*/  FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)),
 794 /*43*/  FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)),
 795 /*44*/  FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)),
 796 /*45*/  FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)),
 797 /*46*/  FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)),
 798 /*47*/  FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)),
 799 /*48*/  FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)),
 800 /*49*/  FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)),
 801 /*50*/  FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)),
 802 /*51*/  FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)),
 803 /*52*/  FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)),
 804 /*53*/  FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)),
 805 /*54*/  FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)),
 806 /*55*/  FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)),
 807 /*56*/  FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)),
 808 /*57*/  FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)),
 809 /*58*/  FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)),
 810 /*59*/  FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)),
 811 /*60*/  FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)),
 812 /*61*/  FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)),
 813 /*62*/  FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)),
 814 /*63*/  FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY))
 815 };
 816
 817 /* RXE errors that will trigger an SPC freeze */
 818 #define ALL_RXE_FREEZE_ERR  \
 819         (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \
 820         | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \
 821         | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \
 822         | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \
 823         | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \
 824         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \
 825         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \
 826         | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \
 827         | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \
 828         | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \
 829         | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \
 830         | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \
 831         | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \
 832         | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \
 833         | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \
 834         | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \
 835         | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \
 836         | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \
 837         | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \
 838         | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \
 839         | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \
 840         | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \
 841         | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \
 842         | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \
 843         | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \
 844         | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \
 845         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \
 846         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \
 847         | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \
 848         | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \
 849         | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \
 850         | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \
 851         | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \
 852         | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \
 853         | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \
 854         | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \
 855         | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \
 856         | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \
 857         | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \
 858         | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \
 859         | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \
 860         | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \
 861         | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \
 862         | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK)
 863
 864 #define RXE_FREEZE_ABORT_MASK \
 865         (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \
 866         RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \
 867         RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK)
 868
 869 /*
 870  * DCC Error Flags
 871  */
 872 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK
 873 static struct flag_table dcc_err_flags[] = {
 874         FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)),
 875         FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)),
 876         FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)),
 877         FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)),
 878         FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)),
 879         FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)),
 880         FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)),
 881         FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)),
 882         FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)),
 883         FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)),
 884         FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)),
 885         FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)),
 886         FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)),
 887         FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)),
 888         FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)),
 889         FLAG_ENTRY0("link_err", DCCE(LINK_ERR)),
 890         FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)),
 891         FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)),
 892         FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)),
 893         FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)),
 894         FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)),
 895         FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)),
 896         FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)),
 897         FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)),
 898         FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)),
 899         FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)),
 900         FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)),
 901         FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)),
 902         FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)),
 903         FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)),
 904         FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)),
 905         FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)),
 906         FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)),
 907         FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)),
 908         FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)),
 909         FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)),
 910         FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)),
 911         FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)),
 912         FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)),
 913         FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)),
 914         FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)),
 915         FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)),
 916         FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)),
 917         FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)),
 918         FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)),
 919         FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)),
 920 };
 921
 922 /*
 923  * LCB error flags
 924  */
 925 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK
 926 static struct flag_table lcb_err_flags[] = {
 927 /* 0*/  FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)),
 928 /* 1*/  FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)),
 929 /* 2*/  FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)),
 930 /* 3*/  FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST",
 931                 LCBE(ALL_LNS_FAILED_REINIT_TEST)),
 932 /* 4*/  FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)),
 933 /* 5*/  FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)),
 934 /* 6*/  FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)),
 935 /* 7*/  FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)),
 936 /* 8*/  FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)),
 937 /* 9*/  FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)),
 938 /*10*/  FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)),
 939 /*11*/  FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)),
 940 /*12*/  FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)),
 941 /*13*/  FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER",
 942                 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)),
 943 /*14*/  FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)),
 944 /*15*/  FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)),
 945 /*16*/  FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)),
 946 /*17*/  FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)),
 947 /*18*/  FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)),
 948 /*19*/  FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE",
 949                 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)),
 950 /*20*/  FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)),
 951 /*21*/  FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)),
 952 /*22*/  FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)),
 953 /*23*/  FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)),
 954 /*24*/  FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)),
 955 /*25*/  FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)),
 956 /*26*/  FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP",
 957                 LCBE(RST_FOR_INCOMPLT_RND_TRIP)),
 958 /*27*/  FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)),
 959 /*28*/  FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE",
 960                 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)),
 961 /*29*/  FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR",
 962                 LCBE(REDUNDANT_FLIT_PARITY_ERR))
 963 };
 964
 965 /*
 966  * DC8051 Error Flags
 967  */
 968 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK
 969 static struct flag_table dc8051_err_flags[] = {
 970         FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)),
 971         FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)),
 972         FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)),
 973         FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)),
 974         FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)),
 975         FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)),
 976         FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)),
 977         FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)),
 978         FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES",
 979                     D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)),
 980         FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)),
 981 };
 982
 983 /*
 984  * DC8051 Information Error flags
 985  *
 986  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field.
 987  */
 988 static struct flag_table dc8051_info_err_flags[] = {
 989         FLAG_ENTRY0("Spico ROM check failed",  SPICO_ROM_FAILED),
 990         FLAG_ENTRY0("Unknown frame received",  UNKNOWN_FRAME),
 991         FLAG_ENTRY0("Target BER not met",      TARGET_BER_NOT_MET),
 992         FLAG_ENTRY0("Serdes internal loopback failure",
 993                     FAILED_SERDES_INTERNAL_LOOPBACK),
 994         FLAG_ENTRY0("Failed SerDes init",      FAILED_SERDES_INIT),
 995         FLAG_ENTRY0("Failed LNI(Polling)",     FAILED_LNI_POLLING),
 996         FLAG_ENTRY0("Failed LNI(Debounce)",    FAILED_LNI_DEBOUNCE),
 997         FLAG_ENTRY0("Failed LNI(EstbComm)",    FAILED_LNI_ESTBCOMM),
 998         FLAG_ENTRY0("Failed LNI(OptEq)",       FAILED_LNI_OPTEQ),
 999         FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1),
1000         FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2),
1001         FLAG_ENTRY0("Failed LNI(ConfigLT)",    FAILED_LNI_CONFIGLT),
1002         FLAG_ENTRY0("Host Handshake Timeout",  HOST_HANDSHAKE_TIMEOUT),
1003         FLAG_ENTRY0("External Device Request Timeout",
1004                     EXTERNAL_DEVICE_REQ_TIMEOUT),
1005 };
1006
1007 /*
1008  * DC8051 Information Host Information flags
1009  *
1010  * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field.
1011  */
1012 static struct flag_table dc8051_info_host_msg_flags[] = {
1013         FLAG_ENTRY0("Host request done", 0x0001),
1014         FLAG_ENTRY0("BC PWR_MGM message", 0x0002),
1015         FLAG_ENTRY0("BC SMA message", 0x0004),
1016         FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008),
1017         FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010),
1018         FLAG_ENTRY0("External device config request", 0x0020),
1019         FLAG_ENTRY0("VerifyCap all frames received", 0x0040),
1020         FLAG_ENTRY0("LinkUp achieved", 0x0080),
1021         FLAG_ENTRY0("Link going down", 0x0100),
1022         FLAG_ENTRY0("Link width downgraded", 0x0200),
1023 };
1024
1025 static u32 encoded_size(u32 size);
1026 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate);
1027 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state);
1028 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
1029                                u8 *continuous);
1030 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
1031                                   u8 *vcu, u16 *vl15buf, u8 *crc_sizes);
1032 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
1033                                       u8 *remote_tx_rate, u16 *link_widths);
1034 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
1035                                     u8 *flag_bits, u16 *link_widths);
1036 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
1037                                   u8 *device_rev);
1038 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx);
1039 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx,
1040                             u8 *tx_polarity_inversion,
1041                             u8 *rx_polarity_inversion, u8 *max_rate);
1042 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
1043                                 unsigned int context, u64 err_status);
1044 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg);
1045 static void handle_dcc_err(struct hfi1_devdata *dd,
1046                            unsigned int context, u64 err_status);
1047 static void handle_lcb_err(struct hfi1_devdata *dd,
1048                            unsigned int context, u64 err_status);
1049 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg);
1050 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1051 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1052 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1053 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1054 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1055 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1056 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg);
1057 static void set_partition_keys(struct hfi1_pportdata *ppd);
1058 static const char *link_state_name(u32 state);
1059 static const char *link_state_reason_name(struct hfi1_pportdata *ppd,
1060                                           u32 state);
1061 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
1062                            u64 *out_data);
1063 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data);
1064 static int thermal_init(struct hfi1_devdata *dd);
1065
1066 static void update_statusp(struct hfi1_pportdata *ppd, u32 state);
1067 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
1068                                             int msecs);
1069 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1070                                   int msecs);
1071 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state);
1072 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state);
1073 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
1074                                    int msecs);
1075 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc);
1076 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr);
1077 static void handle_temp_err(struct hfi1_devdata *dd);
1078 static void dc_shutdown(struct hfi1_devdata *dd);
1079 static void dc_start(struct hfi1_devdata *dd);
1080 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
1081                            unsigned int *np);
1082 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
1083 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
1084 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
1085 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);
1086
1087 /*
1088  * Error interrupt table entry.  This is used as input to the interrupt
1089  * "clear down" routine used for all second tier error interrupt register.
1090  * Second tier interrupt registers have a single bit representing them
1091  * in the top-level CceIntStatus.
1092  */
1093 struct err_reg_info {
1094         u32 status;             /* status CSR offset */
1095         u32 clear;              /* clear CSR offset */
1096         u32 mask;               /* mask CSR offset */
1097         void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg);
1098         const char *desc;
1099 };
1100
1101 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START)
1102 #define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START)
1103 #define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START)
1104
1105 /*
1106  * Helpers for building HFI and DC error interrupt table entries.  Different
1107  * helpers are needed because of inconsistent register names.
1108  */
1109 #define EE(reg, handler, desc) \
1110         { reg##_STATUS, reg##_CLEAR, reg##_MASK, \
1111                 handler, desc }
1112 #define DC_EE1(reg, handler, desc) \
1113         { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc }
1114 #define DC_EE2(reg, handler, desc) \
1115         { reg##_FLG, reg##_CLR, reg##_EN, handler, desc }
1116
1117 /*
1118  * Table of the "misc" grouping of error interrupts.  Each entry refers to
1119  * another register containing more information.
1120  */
1121 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = {
1122 /* 0*/  EE(CCE_ERR,             handle_cce_err,    "CceErr"),
1123 /* 1*/  EE(RCV_ERR,             handle_rxe_err,    "RxeErr"),
1124 /* 2*/  EE(MISC_ERR,    handle_misc_err,   "MiscErr"),
1125 /* 3*/  { 0, 0, 0, NULL }, /* reserved */
1126 /* 4*/  EE(SEND_PIO_ERR,    handle_pio_err,    "PioErr"),
1127 /* 5*/  EE(SEND_DMA_ERR,    handle_sdma_err,   "SDmaErr"),
1128 /* 6*/  EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"),
1129 /* 7*/  EE(SEND_ERR,    handle_txe_err,    "TxeErr")
1130         /* the rest are reserved */
1131 };
1132
1133 /*
1134  * Index into the Various section of the interrupt sources
1135  * corresponding to the Critical Temperature interrupt.
1136  */
1137 #define TCRIT_INT_SOURCE 4
1138
1139 /*
1140  * SDMA error interrupt entry - refers to another register containing more
1141  * information.
1142  */
1143 static const struct err_reg_info sdma_eng_err =
1144         EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr");
1145
1146 static const struct err_reg_info various_err[NUM_VARIOUS] = {
1147 /* 0*/  { 0, 0, 0, NULL }, /* PbcInt */
1148 /* 1*/  { 0, 0, 0, NULL }, /* GpioAssertInt */
1149 /* 2*/  EE(ASIC_QSFP1,  handle_qsfp_int,        "QSFP1"),
1150 /* 3*/  EE(ASIC_QSFP2,  handle_qsfp_int,        "QSFP2"),
1151 /* 4*/  { 0, 0, 0, NULL }, /* TCritInt */
1152         /* rest are reserved */
1153 };
1154
1155 /*
1156  * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG
1157  * register can not be derived from the MTU value because 10K is not
1158  * a power of 2. Therefore, we need a constant. Everything else can
1159  * be calculated.
1160  */
1161 #define DCC_CFG_PORT_MTU_CAP_10240 7
1162
1163 /*
1164  * Table of the DC grouping of error interrupts.  Each entry refers to
1165  * another register containing more information.
1166  */
1167 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = {
1168 /* 0*/  DC_EE1(DCC_ERR,         handle_dcc_err,        "DCC Err"),
1169 /* 1*/  DC_EE2(DC_LCB_ERR,      handle_lcb_err,        "LCB Err"),
1170 /* 2*/  DC_EE2(DC_DC8051_ERR,   handle_8051_interrupt, "DC8051 Interrupt"),
1171 /* 3*/  /* dc_lbm_int - special, see is_dc_int() */
1172         /* the rest are reserved */
1173 };
1174
1175 struct cntr_entry {
1176         /*
1177          * counter name
1178          */
1179         char *name;
1180
1181         /*
1182          * csr to read for name (if applicable)
1183          */
1184         u64 csr;
1185
1186         /*
1187          * offset into dd or ppd to store the counter's value
1188          */
1189         int offset;
1190
1191         /*
1192          * flags
1193          */
1194         u8 flags;
1195
1196         /*
1197          * accessor for stat element, context either dd or ppd
1198          */
1199         u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl,
1200                        int mode, u64 data);
1201 };
1202
1203 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0
1204 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159
1205
1206 #define CNTR_ELEM(name, csr, offset, flags, accessor) \
1207 { \
1208         name, \
1209         csr, \
1210         offset, \
1211         flags, \
1212         accessor \
1213 }
1214
1215 /* 32bit RXE */
1216 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \
1217 CNTR_ELEM(#name, \
1218           (counter * 8 + RCV_COUNTER_ARRAY32), \
1219           0, flags | CNTR_32BIT, \
1220           port_access_u32_csr)
1221
1222 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \
1223 CNTR_ELEM(#name, \
1224           (counter * 8 + RCV_COUNTER_ARRAY32), \
1225           0, flags | CNTR_32BIT, \
1226           dev_access_u32_csr)
1227
1228 /* 64bit RXE */
1229 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \
1230 CNTR_ELEM(#name, \
1231           (counter * 8 + RCV_COUNTER_ARRAY64), \
1232           0, flags, \
1233           port_access_u64_csr)
1234
1235 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \
1236 CNTR_ELEM(#name, \
1237           (counter * 8 + RCV_COUNTER_ARRAY64), \
1238           0, flags, \
1239           dev_access_u64_csr)
1240
1241 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx
1242 #define OVR_ELM(ctx) \
1243 CNTR_ELEM("RcvHdrOvr" #ctx, \
1244           (RCV_HDR_OVFL_CNT + ctx * 0x100), \
1245           0, CNTR_NORMAL, port_access_u64_csr)
1246
1247 /* 32bit TXE */
1248 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \
1249 CNTR_ELEM(#name, \
1250           (counter * 8 + SEND_COUNTER_ARRAY32), \
1251           0, flags | CNTR_32BIT, \
1252           port_access_u32_csr)
1253
1254 /* 64bit TXE */
1255 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \
1256 CNTR_ELEM(#name, \
1257           (counter * 8 + SEND_COUNTER_ARRAY64), \
1258           0, flags, \
1259           port_access_u64_csr)
1260
1261 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \
1262 CNTR_ELEM(#name,\
1263           counter * 8 + SEND_COUNTER_ARRAY64, \
1264           0, \
1265           flags, \
1266           dev_access_u64_csr)
1267
1268 /* CCE */
1269 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \
1270 CNTR_ELEM(#name, \
1271           (counter * 8 + CCE_COUNTER_ARRAY32), \
1272           0, flags | CNTR_32BIT, \
1273           dev_access_u32_csr)
1274
1275 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \
1276 CNTR_ELEM(#name, \
1277           (counter * 8 + CCE_INT_COUNTER_ARRAY32), \
1278           0, flags | CNTR_32BIT, \
1279           dev_access_u32_csr)
1280
1281 /* DC */
1282 #define DC_PERF_CNTR(name, counter, flags) \
1283 CNTR_ELEM(#name, \
1284           counter, \
1285           0, \
1286           flags, \
1287           dev_access_u64_csr)
1288
1289 #define DC_PERF_CNTR_LCB(name, counter, flags) \
1290 CNTR_ELEM(#name, \
1291           counter, \
1292           0, \
1293           flags, \
1294           dc_access_lcb_cntr)
1295
1296 /* ibp counters */
1297 #define SW_IBP_CNTR(name, cntr) \
1298 CNTR_ELEM(#name, \
1299           0, \
1300           0, \
1301           CNTR_SYNTH, \
1302           access_ibp_##cntr)
1303
1304 /**
1305  * hfi_addr_from_offset - return addr for readq/writeq
1306  * @dd - the dd device
1307  * @offset - the offset of the CSR within bar0
1308  *
1309  * This routine selects the appropriate base address
1310  * based on the indicated offset.
1311  */
1312 static inline void __iomem *hfi1_addr_from_offset(
1313         const struct hfi1_devdata *dd,
1314         u32 offset)
1315 {
1316         if (offset >= dd->base2_start)
1317                 return dd->kregbase2 + (offset - dd->base2_start);
1318         return dd->kregbase1 + offset;
1319 }
1320
1321 /**
1322  * read_csr - read CSR at the indicated offset
1323  * @dd - the dd device
1324  * @offset - the offset of the CSR within bar0
1325  *
1326  * Return: the value read or all FF's if there
1327  * is no mapping
1328  */
1329 u64 read_csr(const struct hfi1_devdata *dd, u32 offset)
1330 {
1331         if (dd->flags & HFI1_PRESENT)
1332                 return readq(hfi1_addr_from_offset(dd, offset));
1333         return -1;
1334 }
1335
1336 /**
1337  * write_csr - write CSR at the indicated offset
1338  * @dd - the dd device
1339  * @offset - the offset of the CSR within bar0
1340  * @value - value to write
1341  */
1342 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value)
1343 {
1344         if (dd->flags & HFI1_PRESENT) {
1345                 void __iomem *base = hfi1_addr_from_offset(dd, offset);
1346
1347                 /* avoid write to RcvArray */
1348                 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start))
1349                         return;
1350                 writeq(value, base);
1351         }
1352 }
1353
1354 /**
1355  * get_csr_addr - return te iomem address for offset
1356  * @dd - the dd device
1357  * @offset - the offset of the CSR within bar0
1358  *
1359  * Return: The iomem address to use in subsequent
1360  * writeq/readq operations.
1361  */
1362 void __iomem *get_csr_addr(
1363         const struct hfi1_devdata *dd,
1364         u32 offset)
1365 {
1366         if (dd->flags & HFI1_PRESENT)
1367                 return hfi1_addr_from_offset(dd, offset);
1368         return NULL;
1369 }
1370
1371 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr,
1372                                  int mode, u64 value)
1373 {
1374         u64 ret;
1375
1376         if (mode == CNTR_MODE_R) {
1377                 ret = read_csr(dd, csr);
1378         } else if (mode == CNTR_MODE_W) {
1379                 write_csr(dd, csr, value);
1380                 ret = value;
1381         } else {
1382                 dd_dev_err(dd, "Invalid cntr register access mode");
1383                 return 0;
1384         }
1385
1386         hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode);
1387         return ret;
1388 }
1389
1390 /* Dev Access */
1391 static u64 dev_access_u32_csr(const struct cntr_entry *entry,
1392                               void *context, int vl, int mode, u64 data)
1393 {
1394         struct hfi1_devdata *dd = context;
1395         u64 csr = entry->csr;
1396
1397         if (entry->flags & CNTR_SDMA) {
1398                 if (vl == CNTR_INVALID_VL)
1399                         return 0;
1400                 csr += 0x100 * vl;
1401         } else {
1402                 if (vl != CNTR_INVALID_VL)
1403                         return 0;
1404         }
1405         return read_write_csr(dd, csr, mode, data);
1406 }
1407
1408 static u64 access_sde_err_cnt(const struct cntr_entry *entry,
1409                               void *context, int idx, int mode, u64 data)
1410 {
1411         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1412
1413         if (dd->per_sdma && idx < dd->num_sdma)
1414                 return dd->per_sdma[idx].err_cnt;
1415         return 0;
1416 }
1417
1418 static u64 access_sde_int_cnt(const struct cntr_entry *entry,
1419                               void *context, int idx, int mode, u64 data)
1420 {
1421         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1422
1423         if (dd->per_sdma && idx < dd->num_sdma)
1424                 return dd->per_sdma[idx].sdma_int_cnt;
1425         return 0;
1426 }
1427
1428 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry,
1429                                    void *context, int idx, int mode, u64 data)
1430 {
1431         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1432
1433         if (dd->per_sdma && idx < dd->num_sdma)
1434                 return dd->per_sdma[idx].idle_int_cnt;
1435         return 0;
1436 }
1437
1438 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry,
1439                                        void *context, int idx, int mode,
1440                                        u64 data)
1441 {
1442         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1443
1444         if (dd->per_sdma && idx < dd->num_sdma)
1445                 return dd->per_sdma[idx].progress_int_cnt;
1446         return 0;
1447 }
1448
1449 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context,
1450                               int vl, int mode, u64 data)
1451 {
1452         struct hfi1_devdata *dd = context;
1453
1454         u64 val = 0;
1455         u64 csr = entry->csr;
1456
1457         if (entry->flags & CNTR_VL) {
1458                 if (vl == CNTR_INVALID_VL)
1459                         return 0;
1460                 csr += 8 * vl;
1461         } else {
1462                 if (vl != CNTR_INVALID_VL)
1463                         return 0;
1464         }
1465
1466         val = read_write_csr(dd, csr, mode, data);
1467         return val;
1468 }
1469
1470 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context,
1471                               int vl, int mode, u64 data)
1472 {
1473         struct hfi1_devdata *dd = context;
1474         u32 csr = entry->csr;
1475         int ret = 0;
1476
1477         if (vl != CNTR_INVALID_VL)
1478                 return 0;
1479         if (mode == CNTR_MODE_R)
1480                 ret = read_lcb_csr(dd, csr, &data);
1481         else if (mode == CNTR_MODE_W)
1482                 ret = write_lcb_csr(dd, csr, data);
1483
1484         if (ret) {
1485                 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr);
1486                 return 0;
1487         }
1488
1489         hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode);
1490         return data;
1491 }
1492
1493 /* Port Access */
1494 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context,
1495                                int vl, int mode, u64 data)
1496 {
1497         struct hfi1_pportdata *ppd = context;
1498
1499         if (vl != CNTR_INVALID_VL)
1500                 return 0;
1501         return read_write_csr(ppd->dd, entry->csr, mode, data);
1502 }
1503
1504 static u64 port_access_u64_csr(const struct cntr_entry *entry,
1505                                void *context, int vl, int mode, u64 data)
1506 {
1507         struct hfi1_pportdata *ppd = context;
1508         u64 val;
1509         u64 csr = entry->csr;
1510
1511         if (entry->flags & CNTR_VL) {
1512                 if (vl == CNTR_INVALID_VL)
1513                         return 0;
1514                 csr += 8 * vl;
1515         } else {
1516                 if (vl != CNTR_INVALID_VL)
1517                         return 0;
1518         }
1519         val = read_write_csr(ppd->dd, csr, mode, data);
1520         return val;
1521 }
1522
1523 /* Software defined */
1524 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode,
1525                                 u64 data)
1526 {
1527         u64 ret;
1528
1529         if (mode == CNTR_MODE_R) {
1530                 ret = *cntr;
1531         } else if (mode == CNTR_MODE_W) {
1532                 *cntr = data;
1533                 ret = data;
1534         } else {
1535                 dd_dev_err(dd, "Invalid cntr sw access mode");
1536                 return 0;
1537         }
1538
1539         hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode);
1540
1541         return ret;
1542 }
1543
1544 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context,
1545                                  int vl, int mode, u64 data)
1546 {
1547         struct hfi1_pportdata *ppd = context;
1548
1549         if (vl != CNTR_INVALID_VL)
1550                 return 0;
1551         return read_write_sw(ppd->dd, &ppd->link_downed, mode, data);
1552 }
1553
1554 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context,
1555                                  int vl, int mode, u64 data)
1556 {
1557         struct hfi1_pportdata *ppd = context;
1558
1559         if (vl != CNTR_INVALID_VL)
1560                 return 0;
1561         return read_write_sw(ppd->dd, &ppd->link_up, mode, data);
1562 }
1563
1564 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry,
1565                                        void *context, int vl, int mode,
1566                                        u64 data)
1567 {
1568         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1569
1570         if (vl != CNTR_INVALID_VL)
1571                 return 0;
1572         return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data);
1573 }
1574
1575 static u64 access_sw_xmit_discards(const struct cntr_entry *entry,
1576                                    void *context, int vl, int mode, u64 data)
1577 {
1578         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;
1579         u64 zero = 0;
1580         u64 *counter;
1581
1582         if (vl == CNTR_INVALID_VL)
1583                 counter = &ppd->port_xmit_discards;
1584         else if (vl >= 0 && vl < C_VL_COUNT)
1585                 counter = &ppd->port_xmit_discards_vl[vl];
1586         else
1587                 counter = &zero;
1588
1589         return read_write_sw(ppd->dd, counter, mode, data);
1590 }
1591
1592 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry,
1593                                        void *context, int vl, int mode,
1594                                        u64 data)
1595 {
1596         struct hfi1_pportdata *ppd = context;
1597
1598         if (vl != CNTR_INVALID_VL)
1599                 return 0;
1600
1601         return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors,
1602                              mode, data);
1603 }
1604
1605 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry,
1606                                       void *context, int vl, int mode, u64 data)
1607 {
1608         struct hfi1_pportdata *ppd = context;
1609
1610         if (vl != CNTR_INVALID_VL)
1611                 return 0;
1612
1613         return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors,
1614                              mode, data);
1615 }
1616
1617 u64 get_all_cpu_total(u64 __percpu *cntr)
1618 {
1619         int cpu;
1620         u64 counter = 0;
1621
1622         for_each_possible_cpu(cpu)
1623                 counter += *per_cpu_ptr(cntr, cpu);
1624         return counter;
1625 }
1626
1627 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val,
1628                           u64 __percpu *cntr,
1629                           int vl, int mode, u64 data)
1630 {
1631         u64 ret = 0;
1632
1633         if (vl != CNTR_INVALID_VL)
1634                 return 0;
1635
1636         if (mode == CNTR_MODE_R) {
1637                 ret = get_all_cpu_total(cntr) - *z_val;
1638         } else if (mode == CNTR_MODE_W) {
1639                 /* A write can only zero the counter */
1640                 if (data == 0)
1641                         *z_val = get_all_cpu_total(cntr);
1642                 else
1643                         dd_dev_err(dd, "Per CPU cntrs can only be zeroed");
1644         } else {
1645                 dd_dev_err(dd, "Invalid cntr sw cpu access mode");
1646                 return 0;
1647         }
1648
1649         return ret;
1650 }
1651
1652 static u64 access_sw_cpu_intr(const struct cntr_entry *entry,
1653                               void *context, int vl, int mode, u64 data)
1654 {
1655         struct hfi1_devdata *dd = context;
1656
1657         return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl,
1658                               mode, data);
1659 }
1660
1661 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry,
1662                                    void *context, int vl, int mode, u64 data)
1663 {
1664         struct hfi1_devdata *dd = context;
1665
1666         return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl,
1667                               mode, data);
1668 }
1669
1670 static u64 access_sw_pio_wait(const struct cntr_entry *entry,
1671                               void *context, int vl, int mode, u64 data)
1672 {
1673         struct hfi1_devdata *dd = context;
1674
1675         return dd->verbs_dev.n_piowait;
1676 }
1677
1678 static u64 access_sw_pio_drain(const struct cntr_entry *entry,
1679                                void *context, int vl, int mode, u64 data)
1680 {
1681         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1682
1683         return dd->verbs_dev.n_piodrain;
1684 }
1685
1686 static u64 access_sw_vtx_wait(const struct cntr_entry *entry,
1687                               void *context, int vl, int mode, u64 data)
1688 {
1689         struct hfi1_devdata *dd = context;
1690
1691         return dd->verbs_dev.n_txwait;
1692 }
1693
1694 static u64 access_sw_kmem_wait(const struct cntr_entry *entry,
1695                                void *context, int vl, int mode, u64 data)
1696 {
1697         struct hfi1_devdata *dd = context;
1698
1699         return dd->verbs_dev.n_kmem_wait;
1700 }
1701
1702 static u64 access_sw_send_schedule(const struct cntr_entry *entry,
1703                                    void *context, int vl, int mode, u64 data)
1704 {
1705         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1706
1707         return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl,
1708                               mode, data);
1709 }
1710
1711 /* Software counters for the error status bits within MISC_ERR_STATUS */
1712 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry,
1713                                              void *context, int vl, int mode,
1714                                              u64 data)
1715 {
1716         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1717
1718         return dd->misc_err_status_cnt[12];
1719 }
1720
1721 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry,
1722                                           void *context, int vl, int mode,
1723                                           u64 data)
1724 {
1725         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1726
1727         return dd->misc_err_status_cnt[11];
1728 }
1729
1730 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry,
1731                                                void *context, int vl, int mode,
1732                                                u64 data)
1733 {
1734         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1735
1736         return dd->misc_err_status_cnt[10];
1737 }
1738
1739 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry,
1740                                                  void *context, int vl,
1741                                                  int mode, u64 data)
1742 {
1743         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1744
1745         return dd->misc_err_status_cnt[9];
1746 }
1747
1748 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry,
1749                                            void *context, int vl, int mode,
1750                                            u64 data)
1751 {
1752         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1753
1754         return dd->misc_err_status_cnt[8];
1755 }
1756
1757 static u64 access_misc_efuse_read_bad_addr_err_cnt(
1758                                 const struct cntr_entry *entry,
1759                                 void *context, int vl, int mode, u64 data)
1760 {
1761         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1762
1763         return dd->misc_err_status_cnt[7];
1764 }
1765
1766 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry,
1767                                                 void *context, int vl,
1768                                                 int mode, u64 data)
1769 {
1770         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1771
1772         return dd->misc_err_status_cnt[6];
1773 }
1774
1775 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry,
1776                                               void *context, int vl, int mode,
1777                                               u64 data)
1778 {
1779         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1780
1781         return dd->misc_err_status_cnt[5];
1782 }
1783
1784 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry,
1785                                             void *context, int vl, int mode,
1786                                             u64 data)
1787 {
1788         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1789
1790         return dd->misc_err_status_cnt[4];
1791 }
1792
1793 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry,
1794                                                  void *context, int vl,
1795                                                  int mode, u64 data)
1796 {
1797         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1798
1799         return dd->misc_err_status_cnt[3];
1800 }
1801
1802 static u64 access_misc_csr_write_bad_addr_err_cnt(
1803                                 const struct cntr_entry *entry,
1804                                 void *context, int vl, int mode, u64 data)
1805 {
1806         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1807
1808         return dd->misc_err_status_cnt[2];
1809 }
1810
1811 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1812                                                  void *context, int vl,
1813                                                  int mode, u64 data)
1814 {
1815         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1816
1817         return dd->misc_err_status_cnt[1];
1818 }
1819
1820 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry,
1821                                           void *context, int vl, int mode,
1822                                           u64 data)
1823 {
1824         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1825
1826         return dd->misc_err_status_cnt[0];
1827 }
1828
1829 /*
1830  * Software counter for the aggregate of
1831  * individual CceErrStatus counters
1832  */
1833 static u64 access_sw_cce_err_status_aggregated_cnt(
1834                                 const struct cntr_entry *entry,
1835                                 void *context, int vl, int mode, u64 data)
1836 {
1837         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1838
1839         return dd->sw_cce_err_status_aggregate;
1840 }
1841
1842 /*
1843  * Software counters corresponding to each of the
1844  * error status bits within CceErrStatus
1845  */
1846 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry,
1847                                               void *context, int vl, int mode,
1848                                               u64 data)
1849 {
1850         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1851
1852         return dd->cce_err_status_cnt[40];
1853 }
1854
1855 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry,
1856                                           void *context, int vl, int mode,
1857                                           u64 data)
1858 {
1859         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1860
1861         return dd->cce_err_status_cnt[39];
1862 }
1863
1864 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry,
1865                                           void *context, int vl, int mode,
1866                                           u64 data)
1867 {
1868         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1869
1870         return dd->cce_err_status_cnt[38];
1871 }
1872
1873 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry,
1874                                              void *context, int vl, int mode,
1875                                              u64 data)
1876 {
1877         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1878
1879         return dd->cce_err_status_cnt[37];
1880 }
1881
1882 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry,
1883                                              void *context, int vl, int mode,
1884                                              u64 data)
1885 {
1886         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1887
1888         return dd->cce_err_status_cnt[36];
1889 }
1890
1891 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt(
1892                                 const struct cntr_entry *entry,
1893                                 void *context, int vl, int mode, u64 data)
1894 {
1895         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1896
1897         return dd->cce_err_status_cnt[35];
1898 }
1899
1900 static u64 access_cce_rcpl_async_fifo_parity_err_cnt(
1901                                 const struct cntr_entry *entry,
1902                                 void *context, int vl, int mode, u64 data)
1903 {
1904         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1905
1906         return dd->cce_err_status_cnt[34];
1907 }
1908
1909 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry,
1910                                                  void *context, int vl,
1911                                                  int mode, u64 data)
1912 {
1913         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1914
1915         return dd->cce_err_status_cnt[33];
1916 }
1917
1918 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry,
1919                                                 void *context, int vl, int mode,
1920                                                 u64 data)
1921 {
1922         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1923
1924         return dd->cce_err_status_cnt[32];
1925 }
1926
1927 static u64 access_la_triggered_cnt(const struct cntr_entry *entry,
1928                                    void *context, int vl, int mode, u64 data)
1929 {
1930         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1931
1932         return dd->cce_err_status_cnt[31];
1933 }
1934
1935 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry,
1936                                                void *context, int vl, int mode,
1937                                                u64 data)
1938 {
1939         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1940
1941         return dd->cce_err_status_cnt[30];
1942 }
1943
1944 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry,
1945                                               void *context, int vl, int mode,
1946                                               u64 data)
1947 {
1948         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1949
1950         return dd->cce_err_status_cnt[29];
1951 }
1952
1953 static u64 access_pcic_transmit_back_parity_err_cnt(
1954                                 const struct cntr_entry *entry,
1955                                 void *context, int vl, int mode, u64 data)
1956 {
1957         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1958
1959         return dd->cce_err_status_cnt[28];
1960 }
1961
1962 static u64 access_pcic_transmit_front_parity_err_cnt(
1963                                 const struct cntr_entry *entry,
1964                                 void *context, int vl, int mode, u64 data)
1965 {
1966         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1967
1968         return dd->cce_err_status_cnt[27];
1969 }
1970
1971 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1972                                              void *context, int vl, int mode,
1973                                              u64 data)
1974 {
1975         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1976
1977         return dd->cce_err_status_cnt[26];
1978 }
1979
1980 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1981                                             void *context, int vl, int mode,
1982                                             u64 data)
1983 {
1984         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1985
1986         return dd->cce_err_status_cnt[25];
1987 }
1988
1989 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry,
1990                                               void *context, int vl, int mode,
1991                                               u64 data)
1992 {
1993         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
1994
1995         return dd->cce_err_status_cnt[24];
1996 }
1997
1998 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry,
1999                                              void *context, int vl, int mode,
2000                                              u64 data)
2001 {
2002         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2003
2004         return dd->cce_err_status_cnt[23];
2005 }
2006
2007 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry,
2008                                                  void *context, int vl,
2009                                                  int mode, u64 data)
2010 {
2011         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2012
2013         return dd->cce_err_status_cnt[22];
2014 }
2015
2016 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry,
2017                                          void *context, int vl, int mode,
2018                                          u64 data)
2019 {
2020         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2021
2022         return dd->cce_err_status_cnt[21];
2023 }
2024
2025 static u64 access_pcic_n_post_dat_q_parity_err_cnt(
2026                                 const struct cntr_entry *entry,
2027                                 void *context, int vl, int mode, u64 data)
2028 {
2029         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2030
2031         return dd->cce_err_status_cnt[20];
2032 }
2033
2034 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry,
2035                                                  void *context, int vl,
2036                                                  int mode, u64 data)
2037 {
2038         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2039
2040         return dd->cce_err_status_cnt[19];
2041 }
2042
2043 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2044                                              void *context, int vl, int mode,
2045                                              u64 data)
2046 {
2047         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2048
2049         return dd->cce_err_status_cnt[18];
2050 }
2051
2052 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2053                                             void *context, int vl, int mode,
2054                                             u64 data)
2055 {
2056         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2057
2058         return dd->cce_err_status_cnt[17];
2059 }
2060
2061 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry,
2062                                               void *context, int vl, int mode,
2063                                               u64 data)
2064 {
2065         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2066
2067         return dd->cce_err_status_cnt[16];
2068 }
2069
2070 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry,
2071                                              void *context, int vl, int mode,
2072                                              u64 data)
2073 {
2074         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2075
2076         return dd->cce_err_status_cnt[15];
2077 }
2078
2079 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry,
2080                                                  void *context, int vl,
2081                                                  int mode, u64 data)
2082 {
2083         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2084
2085         return dd->cce_err_status_cnt[14];
2086 }
2087
2088 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry,
2089                                              void *context, int vl, int mode,
2090                                              u64 data)
2091 {
2092         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2093
2094         return dd->cce_err_status_cnt[13];
2095 }
2096
2097 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt(
2098                                 const struct cntr_entry *entry,
2099                                 void *context, int vl, int mode, u64 data)
2100 {
2101         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2102
2103         return dd->cce_err_status_cnt[12];
2104 }
2105
2106 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt(
2107                                 const struct cntr_entry *entry,
2108                                 void *context, int vl, int mode, u64 data)
2109 {
2110         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2111
2112         return dd->cce_err_status_cnt[11];
2113 }
2114
2115 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt(
2116                                 const struct cntr_entry *entry,
2117                                 void *context, int vl, int mode, u64 data)
2118 {
2119         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2120
2121         return dd->cce_err_status_cnt[10];
2122 }
2123
2124 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt(
2125                                 const struct cntr_entry *entry,
2126                                 void *context, int vl, int mode, u64 data)
2127 {
2128         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2129
2130         return dd->cce_err_status_cnt[9];
2131 }
2132
2133 static u64 access_cce_cli2_async_fifo_parity_err_cnt(
2134                                 const struct cntr_entry *entry,
2135                                 void *context, int vl, int mode, u64 data)
2136 {
2137         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2138
2139         return dd->cce_err_status_cnt[8];
2140 }
2141
2142 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry,
2143                                                  void *context, int vl,
2144                                                  int mode, u64 data)
2145 {
2146         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2147
2148         return dd->cce_err_status_cnt[7];
2149 }
2150
2151 static u64 access_cce_cli0_async_fifo_parity_err_cnt(
2152                                 const struct cntr_entry *entry,
2153                                 void *context, int vl, int mode, u64 data)
2154 {
2155         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2156
2157         return dd->cce_err_status_cnt[6];
2158 }
2159
2160 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry,
2161                                                void *context, int vl, int mode,
2162                                                u64 data)
2163 {
2164         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2165
2166         return dd->cce_err_status_cnt[5];
2167 }
2168
2169 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry,
2170                                           void *context, int vl, int mode,
2171                                           u64 data)
2172 {
2173         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2174
2175         return dd->cce_err_status_cnt[4];
2176 }
2177
2178 static u64 access_cce_trgt_async_fifo_parity_err_cnt(
2179                                 const struct cntr_entry *entry,
2180                                 void *context, int vl, int mode, u64 data)
2181 {
2182         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2183
2184         return dd->cce_err_status_cnt[3];
2185 }
2186
2187 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2188                                                  void *context, int vl,
2189                                                  int mode, u64 data)
2190 {
2191         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2192
2193         return dd->cce_err_status_cnt[2];
2194 }
2195
2196 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2197                                                 void *context, int vl,
2198                                                 int mode, u64 data)
2199 {
2200         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2201
2202         return dd->cce_err_status_cnt[1];
2203 }
2204
2205 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry,
2206                                          void *context, int vl, int mode,
2207                                          u64 data)
2208 {
2209         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2210
2211         return dd->cce_err_status_cnt[0];
2212 }
2213
2214 /*
2215  * Software counters corresponding to each of the
2216  * error status bits within RcvErrStatus
2217  */
2218 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry,
2219                                         void *context, int vl, int mode,
2220                                         u64 data)
2221 {
2222         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2223
2224         return dd->rcv_err_status_cnt[63];
2225 }
2226
2227 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry,
2228                                                 void *context, int vl,
2229                                                 int mode, u64 data)
2230 {
2231         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2232
2233         return dd->rcv_err_status_cnt[62];
2234 }
2235
2236 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
2237                                                void *context, int vl, int mode,
2238                                                u64 data)
2239 {
2240         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2241
2242         return dd->rcv_err_status_cnt[61];
2243 }
2244
2245 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry,
2246                                          void *context, int vl, int mode,
2247                                          u64 data)
2248 {
2249         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2250
2251         return dd->rcv_err_status_cnt[60];
2252 }
2253
2254 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2255                                                  void *context, int vl,
2256                                                  int mode, u64 data)
2257 {
2258         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2259
2260         return dd->rcv_err_status_cnt[59];
2261 }
2262
2263 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2264                                                  void *context, int vl,
2265                                                  int mode, u64 data)
2266 {
2267         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2268
2269         return dd->rcv_err_status_cnt[58];
2270 }
2271
2272 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry,
2273                                             void *context, int vl, int mode,
2274                                             u64 data)
2275 {
2276         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2277
2278         return dd->rcv_err_status_cnt[57];
2279 }
2280
2281 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry,
2282                                            void *context, int vl, int mode,
2283                                            u64 data)
2284 {
2285         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2286
2287         return dd->rcv_err_status_cnt[56];
2288 }
2289
2290 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry,
2291                                            void *context, int vl, int mode,
2292                                            u64 data)
2293 {
2294         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2295
2296         return dd->rcv_err_status_cnt[55];
2297 }
2298
2299 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt(
2300                                 const struct cntr_entry *entry,
2301                                 void *context, int vl, int mode, u64 data)
2302 {
2303         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2304
2305         return dd->rcv_err_status_cnt[54];
2306 }
2307
2308 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt(
2309                                 const struct cntr_entry *entry,
2310                                 void *context, int vl, int mode, u64 data)
2311 {
2312         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2313
2314         return dd->rcv_err_status_cnt[53];
2315 }
2316
2317 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry,
2318                                                  void *context, int vl,
2319                                                  int mode, u64 data)
2320 {
2321         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2322
2323         return dd->rcv_err_status_cnt[52];
2324 }
2325
2326 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry,
2327                                                  void *context, int vl,
2328                                                  int mode, u64 data)
2329 {
2330         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2331
2332         return dd->rcv_err_status_cnt[51];
2333 }
2334
2335 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry,
2336                                                  void *context, int vl,
2337                                                  int mode, u64 data)
2338 {
2339         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2340
2341         return dd->rcv_err_status_cnt[50];
2342 }
2343
2344 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry,
2345                                                  void *context, int vl,
2346                                                  int mode, u64 data)
2347 {
2348         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2349
2350         return dd->rcv_err_status_cnt[49];
2351 }
2352
2353 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry,
2354                                                  void *context, int vl,
2355                                                  int mode, u64 data)
2356 {
2357         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2358
2359         return dd->rcv_err_status_cnt[48];
2360 }
2361
2362 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry,
2363                                                  void *context, int vl,
2364                                                  int mode, u64 data)
2365 {
2366         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2367
2368         return dd->rcv_err_status_cnt[47];
2369 }
2370
2371 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry,
2372                                          void *context, int vl, int mode,
2373                                          u64 data)
2374 {
2375         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2376
2377         return dd->rcv_err_status_cnt[46];
2378 }
2379
2380 static u64 access_rx_hq_intr_csr_parity_err_cnt(
2381                                 const struct cntr_entry *entry,
2382                                 void *context, int vl, int mode, u64 data)
2383 {
2384         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2385
2386         return dd->rcv_err_status_cnt[45];
2387 }
2388
2389 static u64 access_rx_lookup_csr_parity_err_cnt(
2390                                 const struct cntr_entry *entry,
2391                                 void *context, int vl, int mode, u64 data)
2392 {
2393         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2394
2395         return dd->rcv_err_status_cnt[44];
2396 }
2397
2398 static u64 access_rx_lookup_rcv_array_cor_err_cnt(
2399                                 const struct cntr_entry *entry,
2400                                 void *context, int vl, int mode, u64 data)
2401 {
2402         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2403
2404         return dd->rcv_err_status_cnt[43];
2405 }
2406
2407 static u64 access_rx_lookup_rcv_array_unc_err_cnt(
2408                                 const struct cntr_entry *entry,
2409                                 void *context, int vl, int mode, u64 data)
2410 {
2411         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2412
2413         return dd->rcv_err_status_cnt[42];
2414 }
2415
2416 static u64 access_rx_lookup_des_part2_parity_err_cnt(
2417                                 const struct cntr_entry *entry,
2418                                 void *context, int vl, int mode, u64 data)
2419 {
2420         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2421
2422         return dd->rcv_err_status_cnt[41];
2423 }
2424
2425 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt(
2426                                 const struct cntr_entry *entry,
2427                                 void *context, int vl, int mode, u64 data)
2428 {
2429         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2430
2431         return dd->rcv_err_status_cnt[40];
2432 }
2433
2434 static u64 access_rx_lookup_des_part1_unc_err_cnt(
2435                                 const struct cntr_entry *entry,
2436                                 void *context, int vl, int mode, u64 data)
2437 {
2438         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2439
2440         return dd->rcv_err_status_cnt[39];
2441 }
2442
2443 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt(
2444                                 const struct cntr_entry *entry,
2445                                 void *context, int vl, int mode, u64 data)
2446 {
2447         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2448
2449         return dd->rcv_err_status_cnt[38];
2450 }
2451
2452 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt(
2453                                 const struct cntr_entry *entry,
2454                                 void *context, int vl, int mode, u64 data)
2455 {
2456         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2457
2458         return dd->rcv_err_status_cnt[37];
2459 }
2460
2461 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt(
2462                                 const struct cntr_entry *entry,
2463                                 void *context, int vl, int mode, u64 data)
2464 {
2465         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2466
2467         return dd->rcv_err_status_cnt[36];
2468 }
2469
2470 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt(
2471                                 const struct cntr_entry *entry,
2472                                 void *context, int vl, int mode, u64 data)
2473 {
2474         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2475
2476         return dd->rcv_err_status_cnt[35];
2477 }
2478
2479 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt(
2480                                 const struct cntr_entry *entry,
2481                                 void *context, int vl, int mode, u64 data)
2482 {
2483         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2484
2485         return dd->rcv_err_status_cnt[34];
2486 }
2487
2488 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt(
2489                                 const struct cntr_entry *entry,
2490                                 void *context, int vl, int mode, u64 data)
2491 {
2492         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2493
2494         return dd->rcv_err_status_cnt[33];
2495 }
2496
2497 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry,
2498                                         void *context, int vl, int mode,
2499                                         u64 data)
2500 {
2501         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2502
2503         return dd->rcv_err_status_cnt[32];
2504 }
2505
2506 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry,
2507                                        void *context, int vl, int mode,
2508                                        u64 data)
2509 {
2510         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2511
2512         return dd->rcv_err_status_cnt[31];
2513 }
2514
2515 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry,
2516                                           void *context, int vl, int mode,
2517                                           u64 data)
2518 {
2519         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2520
2521         return dd->rcv_err_status_cnt[30];
2522 }
2523
2524 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry,
2525                                              void *context, int vl, int mode,
2526                                              u64 data)
2527 {
2528         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2529
2530         return dd->rcv_err_status_cnt[29];
2531 }
2532
2533 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry,
2534                                                  void *context, int vl,
2535                                                  int mode, u64 data)
2536 {
2537         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2538
2539         return dd->rcv_err_status_cnt[28];
2540 }
2541
2542 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt(
2543                                 const struct cntr_entry *entry,
2544                                 void *context, int vl, int mode, u64 data)
2545 {
2546         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2547
2548         return dd->rcv_err_status_cnt[27];
2549 }
2550
2551 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt(
2552                                 const struct cntr_entry *entry,
2553                                 void *context, int vl, int mode, u64 data)
2554 {
2555         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2556
2557         return dd->rcv_err_status_cnt[26];
2558 }
2559
2560 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt(
2561                                 const struct cntr_entry *entry,
2562                                 void *context, int vl, int mode, u64 data)
2563 {
2564         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2565
2566         return dd->rcv_err_status_cnt[25];
2567 }
2568
2569 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt(
2570                                 const struct cntr_entry *entry,
2571                                 void *context, int vl, int mode, u64 data)
2572 {
2573         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2574
2575         return dd->rcv_err_status_cnt[24];
2576 }
2577
2578 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt(
2579                                 const struct cntr_entry *entry,
2580                                 void *context, int vl, int mode, u64 data)
2581 {
2582         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2583
2584         return dd->rcv_err_status_cnt[23];
2585 }
2586
2587 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt(
2588                                 const struct cntr_entry *entry,
2589                                 void *context, int vl, int mode, u64 data)
2590 {
2591         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2592
2593         return dd->rcv_err_status_cnt[22];
2594 }
2595
2596 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt(
2597                                 const struct cntr_entry *entry,
2598                                 void *context, int vl, int mode, u64 data)
2599 {
2600         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2601
2602         return dd->rcv_err_status_cnt[21];
2603 }
2604
2605 static u64 access_rx_rbuf_block_list_read_cor_err_cnt(
2606                                 const struct cntr_entry *entry,
2607                                 void *context, int vl, int mode, u64 data)
2608 {
2609         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2610
2611         return dd->rcv_err_status_cnt[20];
2612 }
2613
2614 static u64 access_rx_rbuf_block_list_read_unc_err_cnt(
2615                                 const struct cntr_entry *entry,
2616                                 void *context, int vl, int mode, u64 data)
2617 {
2618         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2619
2620         return dd->rcv_err_status_cnt[19];
2621 }
2622
2623 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry,
2624                                                  void *context, int vl,
2625                                                  int mode, u64 data)
2626 {
2627         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2628
2629         return dd->rcv_err_status_cnt[18];
2630 }
2631
2632 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry,
2633                                                  void *context, int vl,
2634                                                  int mode, u64 data)
2635 {
2636         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2637
2638         return dd->rcv_err_status_cnt[17];
2639 }
2640
2641 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt(
2642                                 const struct cntr_entry *entry,
2643                                 void *context, int vl, int mode, u64 data)
2644 {
2645         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2646
2647         return dd->rcv_err_status_cnt[16];
2648 }
2649
2650 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt(
2651                                 const struct cntr_entry *entry,
2652                                 void *context, int vl, int mode, u64 data)
2653 {
2654         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2655
2656         return dd->rcv_err_status_cnt[15];
2657 }
2658
2659 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry,
2660                                                 void *context, int vl,
2661                                                 int mode, u64 data)
2662 {
2663         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2664
2665         return dd->rcv_err_status_cnt[14];
2666 }
2667
2668 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry,
2669                                                 void *context, int vl,
2670                                                 int mode, u64 data)
2671 {
2672         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2673
2674         return dd->rcv_err_status_cnt[13];
2675 }
2676
2677 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry,
2678                                               void *context, int vl, int mode,
2679                                               u64 data)
2680 {
2681         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2682
2683         return dd->rcv_err_status_cnt[12];
2684 }
2685
2686 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry,
2687                                           void *context, int vl, int mode,
2688                                           u64 data)
2689 {
2690         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2691
2692         return dd->rcv_err_status_cnt[11];
2693 }
2694
2695 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry,
2696                                           void *context, int vl, int mode,
2697                                           u64 data)
2698 {
2699         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2700
2701         return dd->rcv_err_status_cnt[10];
2702 }
2703
2704 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry,
2705                                                void *context, int vl, int mode,
2706                                                u64 data)
2707 {
2708         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2709
2710         return dd->rcv_err_status_cnt[9];
2711 }
2712
2713 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry,
2714                                             void *context, int vl, int mode,
2715                                             u64 data)
2716 {
2717         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2718
2719         return dd->rcv_err_status_cnt[8];
2720 }
2721
2722 static u64 access_rx_rcv_qp_map_table_cor_err_cnt(
2723                                 const struct cntr_entry *entry,
2724                                 void *context, int vl, int mode, u64 data)
2725 {
2726         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2727
2728         return dd->rcv_err_status_cnt[7];
2729 }
2730
2731 static u64 access_rx_rcv_qp_map_table_unc_err_cnt(
2732                                 const struct cntr_entry *entry,
2733                                 void *context, int vl, int mode, u64 data)
2734 {
2735         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2736
2737         return dd->rcv_err_status_cnt[6];
2738 }
2739
2740 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry,
2741                                           void *context, int vl, int mode,
2742                                           u64 data)
2743 {
2744         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2745
2746         return dd->rcv_err_status_cnt[5];
2747 }
2748
2749 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry,
2750                                           void *context, int vl, int mode,
2751                                           u64 data)
2752 {
2753         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2754
2755         return dd->rcv_err_status_cnt[4];
2756 }
2757
2758 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry,
2759                                          void *context, int vl, int mode,
2760                                          u64 data)
2761 {
2762         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2763
2764         return dd->rcv_err_status_cnt[3];
2765 }
2766
2767 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry,
2768                                          void *context, int vl, int mode,
2769                                          u64 data)
2770 {
2771         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2772
2773         return dd->rcv_err_status_cnt[2];
2774 }
2775
2776 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry,
2777                                             void *context, int vl, int mode,
2778                                             u64 data)
2779 {
2780         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2781
2782         return dd->rcv_err_status_cnt[1];
2783 }
2784
2785 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry,
2786                                          void *context, int vl, int mode,
2787                                          u64 data)
2788 {
2789         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2790
2791         return dd->rcv_err_status_cnt[0];
2792 }
2793
2794 /*
2795  * Software counters corresponding to each of the
2796  * error status bits within SendPioErrStatus
2797  */
2798 static u64 access_pio_pec_sop_head_parity_err_cnt(
2799                                 const struct cntr_entry *entry,
2800                                 void *context, int vl, int mode, u64 data)
2801 {
2802         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2803
2804         return dd->send_pio_err_status_cnt[35];
2805 }
2806
2807 static u64 access_pio_pcc_sop_head_parity_err_cnt(
2808                                 const struct cntr_entry *entry,
2809                                 void *context, int vl, int mode, u64 data)
2810 {
2811         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2812
2813         return dd->send_pio_err_status_cnt[34];
2814 }
2815
2816 static u64 access_pio_last_returned_cnt_parity_err_cnt(
2817                                 const struct cntr_entry *entry,
2818                                 void *context, int vl, int mode, u64 data)
2819 {
2820         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2821
2822         return dd->send_pio_err_status_cnt[33];
2823 }
2824
2825 static u64 access_pio_current_free_cnt_parity_err_cnt(
2826                                 const struct cntr_entry *entry,
2827                                 void *context, int vl, int mode, u64 data)
2828 {
2829         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2830
2831         return dd->send_pio_err_status_cnt[32];
2832 }
2833
2834 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry,
2835                                           void *context, int vl, int mode,
2836                                           u64 data)
2837 {
2838         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2839
2840         return dd->send_pio_err_status_cnt[31];
2841 }
2842
2843 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry,
2844                                           void *context, int vl, int mode,
2845                                           u64 data)
2846 {
2847         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2848
2849         return dd->send_pio_err_status_cnt[30];
2850 }
2851
2852 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry,
2853                                            void *context, int vl, int mode,
2854                                            u64 data)
2855 {
2856         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2857
2858         return dd->send_pio_err_status_cnt[29];
2859 }
2860
2861 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt(
2862                                 const struct cntr_entry *entry,
2863                                 void *context, int vl, int mode, u64 data)
2864 {
2865         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2866
2867         return dd->send_pio_err_status_cnt[28];
2868 }
2869
2870 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry,
2871                                              void *context, int vl, int mode,
2872                                              u64 data)
2873 {
2874         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2875
2876         return dd->send_pio_err_status_cnt[27];
2877 }
2878
2879 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry,
2880                                              void *context, int vl, int mode,
2881                                              u64 data)
2882 {
2883         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2884
2885         return dd->send_pio_err_status_cnt[26];
2886 }
2887
2888 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry,
2889                                                 void *context, int vl,
2890                                                 int mode, u64 data)
2891 {
2892         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2893
2894         return dd->send_pio_err_status_cnt[25];
2895 }
2896
2897 static u64 access_pio_block_qw_count_parity_err_cnt(
2898                                 const struct cntr_entry *entry,
2899                                 void *context, int vl, int mode, u64 data)
2900 {
2901         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2902
2903         return dd->send_pio_err_status_cnt[24];
2904 }
2905
2906 static u64 access_pio_write_qw_valid_parity_err_cnt(
2907                                 const struct cntr_entry *entry,
2908                                 void *context, int vl, int mode, u64 data)
2909 {
2910         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2911
2912         return dd->send_pio_err_status_cnt[23];
2913 }
2914
2915 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry,
2916                                             void *context, int vl, int mode,
2917                                             u64 data)
2918 {
2919         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2920
2921         return dd->send_pio_err_status_cnt[22];
2922 }
2923
2924 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry,
2925                                                 void *context, int vl,
2926                                                 int mode, u64 data)
2927 {
2928         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2929
2930         return dd->send_pio_err_status_cnt[21];
2931 }
2932
2933 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry,
2934                                                 void *context, int vl,
2935                                                 int mode, u64 data)
2936 {
2937         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2938
2939         return dd->send_pio_err_status_cnt[20];
2940 }
2941
2942 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry,
2943                                                 void *context, int vl,
2944                                                 int mode, u64 data)
2945 {
2946         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2947
2948         return dd->send_pio_err_status_cnt[19];
2949 }
2950
2951 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt(
2952                                 const struct cntr_entry *entry,
2953                                 void *context, int vl, int mode, u64 data)
2954 {
2955         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2956
2957         return dd->send_pio_err_status_cnt[18];
2958 }
2959
2960 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry,
2961                                          void *context, int vl, int mode,
2962                                          u64 data)
2963 {
2964         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2965
2966         return dd->send_pio_err_status_cnt[17];
2967 }
2968
2969 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry,
2970                                             void *context, int vl, int mode,
2971                                             u64 data)
2972 {
2973         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2974
2975         return dd->send_pio_err_status_cnt[16];
2976 }
2977
2978 static u64 access_pio_credit_ret_fifo_parity_err_cnt(
2979                                 const struct cntr_entry *entry,
2980                                 void *context, int vl, int mode, u64 data)
2981 {
2982         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2983
2984         return dd->send_pio_err_status_cnt[15];
2985 }
2986
2987 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt(
2988                                 const struct cntr_entry *entry,
2989                                 void *context, int vl, int mode, u64 data)
2990 {
2991         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
2992
2993         return dd->send_pio_err_status_cnt[14];
2994 }
2995
2996 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt(
2997                                 const struct cntr_entry *entry,
2998                                 void *context, int vl, int mode, u64 data)
2999 {
3000         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3001
3002         return dd->send_pio_err_status_cnt[13];
3003 }
3004
3005 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt(
3006                                 const struct cntr_entry *entry,
3007                                 void *context, int vl, int mode, u64 data)
3008 {
3009         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3010
3011         return dd->send_pio_err_status_cnt[12];
3012 }
3013
3014 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt(
3015                                 const struct cntr_entry *entry,
3016                                 void *context, int vl, int mode, u64 data)
3017 {
3018         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3019
3020         return dd->send_pio_err_status_cnt[11];
3021 }
3022
3023 static u64 access_pio_sm_pkt_reset_parity_err_cnt(
3024                                 const struct cntr_entry *entry,
3025                                 void *context, int vl, int mode, u64 data)
3026 {
3027         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3028
3029         return dd->send_pio_err_status_cnt[10];
3030 }
3031
3032 static u64 access_pio_pkt_evict_fifo_parity_err_cnt(
3033                                 const struct cntr_entry *entry,
3034                                 void *context, int vl, int mode, u64 data)
3035 {
3036         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3037
3038         return dd->send_pio_err_status_cnt[9];
3039 }
3040
3041 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt(
3042                                 const struct cntr_entry *entry,
3043                                 void *context, int vl, int mode, u64 data)
3044 {
3045         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3046
3047         return dd->send_pio_err_status_cnt[8];
3048 }
3049
3050 static u64 access_pio_sbrdctl_crrel_parity_err_cnt(
3051                                 const struct cntr_entry *entry,
3052                                 void *context, int vl, int mode, u64 data)
3053 {
3054         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3055
3056         return dd->send_pio_err_status_cnt[7];
3057 }
3058
3059 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry,
3060                                               void *context, int vl, int mode,
3061                                               u64 data)
3062 {
3063         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3064
3065         return dd->send_pio_err_status_cnt[6];
3066 }
3067
3068 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry,
3069                                               void *context, int vl, int mode,
3070                                               u64 data)
3071 {
3072         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3073
3074         return dd->send_pio_err_status_cnt[5];
3075 }
3076
3077 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry,
3078                                            void *context, int vl, int mode,
3079                                            u64 data)
3080 {
3081         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3082
3083         return dd->send_pio_err_status_cnt[4];
3084 }
3085
3086 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry,
3087                                            void *context, int vl, int mode,
3088                                            u64 data)
3089 {
3090         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3091
3092         return dd->send_pio_err_status_cnt[3];
3093 }
3094
3095 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry,
3096                                          void *context, int vl, int mode,
3097                                          u64 data)
3098 {
3099         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3100
3101         return dd->send_pio_err_status_cnt[2];
3102 }
3103
3104 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry,
3105                                                 void *context, int vl,
3106                                                 int mode, u64 data)
3107 {
3108         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3109
3110         return dd->send_pio_err_status_cnt[1];
3111 }
3112
3113 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry,
3114                                              void *context, int vl, int mode,
3115                                              u64 data)
3116 {
3117         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3118
3119         return dd->send_pio_err_status_cnt[0];
3120 }
3121
3122 /*
3123  * Software counters corresponding to each of the
3124  * error status bits within SendDmaErrStatus
3125  */
3126 static u64 access_sdma_pcie_req_tracking_cor_err_cnt(
3127                                 const struct cntr_entry *entry,
3128                                 void *context, int vl, int mode, u64 data)
3129 {
3130         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3131
3132         return dd->send_dma_err_status_cnt[3];
3133 }
3134
3135 static u64 access_sdma_pcie_req_tracking_unc_err_cnt(
3136                                 const struct cntr_entry *entry,
3137                                 void *context, int vl, int mode, u64 data)
3138 {
3139         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3140
3141         return dd->send_dma_err_status_cnt[2];
3142 }
3143
3144 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry,
3145                                           void *context, int vl, int mode,
3146                                           u64 data)
3147 {
3148         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3149
3150         return dd->send_dma_err_status_cnt[1];
3151 }
3152
3153 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry,
3154                                        void *context, int vl, int mode,
3155                                        u64 data)
3156 {
3157         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3158
3159         return dd->send_dma_err_status_cnt[0];
3160 }
3161
3162 /*
3163  * Software counters corresponding to each of the
3164  * error status bits within SendEgressErrStatus
3165  */
3166 static u64 access_tx_read_pio_memory_csr_unc_err_cnt(
3167                                 const struct cntr_entry *entry,
3168                                 void *context, int vl, int mode, u64 data)
3169 {
3170         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3171
3172         return dd->send_egress_err_status_cnt[63];
3173 }
3174
3175 static u64 access_tx_read_sdma_memory_csr_err_cnt(
3176                                 const struct cntr_entry *entry,
3177                                 void *context, int vl, int mode, u64 data)
3178 {
3179         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3180
3181         return dd->send_egress_err_status_cnt[62];
3182 }
3183
3184 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry,
3185                                              void *context, int vl, int mode,
3186                                              u64 data)
3187 {
3188         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3189
3190         return dd->send_egress_err_status_cnt[61];
3191 }
3192
3193 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry,
3194                                                  void *context, int vl,
3195                                                  int mode, u64 data)
3196 {
3197         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3198
3199         return dd->send_egress_err_status_cnt[60];
3200 }
3201
3202 static u64 access_tx_read_sdma_memory_cor_err_cnt(
3203                                 const struct cntr_entry *entry,
3204                                 void *context, int vl, int mode, u64 data)
3205 {
3206         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3207
3208         return dd->send_egress_err_status_cnt[59];
3209 }
3210
3211 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry,
3212                                         void *context, int vl, int mode,
3213                                         u64 data)
3214 {
3215         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3216
3217         return dd->send_egress_err_status_cnt[58];
3218 }
3219
3220 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry,
3221                                             void *context, int vl, int mode,
3222                                             u64 data)
3223 {
3224         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3225
3226         return dd->send_egress_err_status_cnt[57];
3227 }
3228
3229 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry,
3230                                               void *context, int vl, int mode,
3231                                               u64 data)
3232 {
3233         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3234
3235         return dd->send_egress_err_status_cnt[56];
3236 }
3237
3238 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry,
3239                                               void *context, int vl, int mode,
3240                                               u64 data)
3241 {
3242         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3243
3244         return dd->send_egress_err_status_cnt[55];
3245 }
3246
3247 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry,
3248                                               void *context, int vl, int mode,
3249                                               u64 data)
3250 {
3251         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3252
3253         return dd->send_egress_err_status_cnt[54];
3254 }
3255
3256 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry,
3257                                               void *context, int vl, int mode,
3258                                               u64 data)
3259 {
3260         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3261
3262         return dd->send_egress_err_status_cnt[53];
3263 }
3264
3265 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry,
3266                                               void *context, int vl, int mode,
3267                                               u64 data)
3268 {
3269         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3270
3271         return dd->send_egress_err_status_cnt[52];
3272 }
3273
3274 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry,
3275                                               void *context, int vl, int mode,
3276                                               u64 data)
3277 {
3278         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3279
3280         return dd->send_egress_err_status_cnt[51];
3281 }
3282
3283 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry,
3284                                               void *context, int vl, int mode,
3285                                               u64 data)
3286 {
3287         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3288
3289         return dd->send_egress_err_status_cnt[50];
3290 }
3291
3292 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry,
3293                                               void *context, int vl, int mode,
3294                                               u64 data)
3295 {
3296         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3297
3298         return dd->send_egress_err_status_cnt[49];
3299 }
3300
3301 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry,
3302                                               void *context, int vl, int mode,
3303                                               u64 data)
3304 {
3305         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3306
3307         return dd->send_egress_err_status_cnt[48];
3308 }
3309
3310 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry,
3311                                               void *context, int vl, int mode,
3312                                               u64 data)
3313 {
3314         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3315
3316         return dd->send_egress_err_status_cnt[47];
3317 }
3318
3319 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry,
3320                                             void *context, int vl, int mode,
3321                                             u64 data)
3322 {
3323         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3324
3325         return dd->send_egress_err_status_cnt[46];
3326 }
3327
3328 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry,
3329                                              void *context, int vl, int mode,
3330                                              u64 data)
3331 {
3332         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3333
3334         return dd->send_egress_err_status_cnt[45];
3335 }
3336
3337 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry,
3338                                                  void *context, int vl,
3339                                                  int mode, u64 data)
3340 {
3341         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3342
3343         return dd->send_egress_err_status_cnt[44];
3344 }
3345
3346 static u64 access_tx_read_sdma_memory_unc_err_cnt(
3347                                 const struct cntr_entry *entry,
3348                                 void *context, int vl, int mode, u64 data)
3349 {
3350         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3351
3352         return dd->send_egress_err_status_cnt[43];
3353 }
3354
3355 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry,
3356                                         void *context, int vl, int mode,
3357                                         u64 data)
3358 {
3359         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3360
3361         return dd->send_egress_err_status_cnt[42];
3362 }
3363
3364 static u64 access_tx_credit_return_partiy_err_cnt(
3365                                 const struct cntr_entry *entry,
3366                                 void *context, int vl, int mode, u64 data)
3367 {
3368         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3369
3370         return dd->send_egress_err_status_cnt[41];
3371 }
3372
3373 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt(
3374                                 const struct cntr_entry *entry,
3375                                 void *context, int vl, int mode, u64 data)
3376 {
3377         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3378
3379         return dd->send_egress_err_status_cnt[40];
3380 }
3381
3382 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt(
3383                                 const struct cntr_entry *entry,
3384                                 void *context, int vl, int mode, u64 data)
3385 {
3386         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3387
3388         return dd->send_egress_err_status_cnt[39];
3389 }
3390
3391 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt(
3392                                 const struct cntr_entry *entry,
3393                                 void *context, int vl, int mode, u64 data)
3394 {
3395         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3396
3397         return dd->send_egress_err_status_cnt[38];
3398 }
3399
3400 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt(
3401                                 const struct cntr_entry *entry,
3402                                 void *context, int vl, int mode, u64 data)
3403 {
3404         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3405
3406         return dd->send_egress_err_status_cnt[37];
3407 }
3408
3409 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt(
3410                                 const struct cntr_entry *entry,
3411                                 void *context, int vl, int mode, u64 data)
3412 {
3413         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3414
3415         return dd->send_egress_err_status_cnt[36];
3416 }
3417
3418 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt(
3419                                 const struct cntr_entry *entry,
3420                                 void *context, int vl, int mode, u64 data)
3421 {
3422         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3423
3424         return dd->send_egress_err_status_cnt[35];
3425 }
3426
3427 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt(
3428                                 const struct cntr_entry *entry,
3429                                 void *context, int vl, int mode, u64 data)
3430 {
3431         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3432
3433         return dd->send_egress_err_status_cnt[34];
3434 }
3435
3436 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt(
3437                                 const struct cntr_entry *entry,
3438                                 void *context, int vl, int mode, u64 data)
3439 {
3440         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3441
3442         return dd->send_egress_err_status_cnt[33];
3443 }
3444
3445 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt(
3446                                 const struct cntr_entry *entry,
3447                                 void *context, int vl, int mode, u64 data)
3448 {
3449         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3450
3451         return dd->send_egress_err_status_cnt[32];
3452 }
3453
3454 static u64 access_tx_sdma15_disallowed_packet_err_cnt(
3455                                 const struct cntr_entry *entry,
3456                                 void *context, int vl, int mode, u64 data)
3457 {
3458         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3459
3460         return dd->send_egress_err_status_cnt[31];
3461 }
3462
3463 static u64 access_tx_sdma14_disallowed_packet_err_cnt(
3464                                 const struct cntr_entry *entry,
3465                                 void *context, int vl, int mode, u64 data)
3466 {
3467         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3468
3469         return dd->send_egress_err_status_cnt[30];
3470 }
3471
3472 static u64 access_tx_sdma13_disallowed_packet_err_cnt(
3473                                 const struct cntr_entry *entry,
3474                                 void *context, int vl, int mode, u64 data)
3475 {
3476         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3477
3478         return dd->send_egress_err_status_cnt[29];
3479 }
3480
3481 static u64 access_tx_sdma12_disallowed_packet_err_cnt(
3482                                 const struct cntr_entry *entry,
3483                                 void *context, int vl, int mode, u64 data)
3484 {
3485         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3486
3487         return dd->send_egress_err_status_cnt[28];
3488 }
3489
3490 static u64 access_tx_sdma11_disallowed_packet_err_cnt(
3491                                 const struct cntr_entry *entry,
3492                                 void *context, int vl, int mode, u64 data)
3493 {
3494         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3495
3496         return dd->send_egress_err_status_cnt[27];
3497 }
3498
3499 static u64 access_tx_sdma10_disallowed_packet_err_cnt(
3500                                 const struct cntr_entry *entry,
3501                                 void *context, int vl, int mode, u64 data)
3502 {
3503         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3504
3505         return dd->send_egress_err_status_cnt[26];
3506 }
3507
3508 static u64 access_tx_sdma9_disallowed_packet_err_cnt(
3509                                 const struct cntr_entry *entry,
3510                                 void *context, int vl, int mode, u64 data)
3511 {
3512         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3513
3514         return dd->send_egress_err_status_cnt[25];
3515 }
3516
3517 static u64 access_tx_sdma8_disallowed_packet_err_cnt(
3518                                 const struct cntr_entry *entry,
3519                                 void *context, int vl, int mode, u64 data)
3520 {
3521         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3522
3523         return dd->send_egress_err_status_cnt[24];
3524 }
3525
3526 static u64 access_tx_sdma7_disallowed_packet_err_cnt(
3527                                 const struct cntr_entry *entry,
3528                                 void *context, int vl, int mode, u64 data)
3529 {
3530         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3531
3532         return dd->send_egress_err_status_cnt[23];
3533 }
3534
3535 static u64 access_tx_sdma6_disallowed_packet_err_cnt(
3536                                 const struct cntr_entry *entry,
3537                                 void *context, int vl, int mode, u64 data)
3538 {
3539         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3540
3541         return dd->send_egress_err_status_cnt[22];
3542 }
3543
3544 static u64 access_tx_sdma5_disallowed_packet_err_cnt(
3545                                 const struct cntr_entry *entry,
3546                                 void *context, int vl, int mode, u64 data)
3547 {
3548         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3549
3550         return dd->send_egress_err_status_cnt[21];
3551 }
3552
3553 static u64 access_tx_sdma4_disallowed_packet_err_cnt(
3554                                 const struct cntr_entry *entry,
3555                                 void *context, int vl, int mode, u64 data)
3556 {
3557         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3558
3559         return dd->send_egress_err_status_cnt[20];
3560 }
3561
3562 static u64 access_tx_sdma3_disallowed_packet_err_cnt(
3563                                 const struct cntr_entry *entry,
3564                                 void *context, int vl, int mode, u64 data)
3565 {
3566         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3567
3568         return dd->send_egress_err_status_cnt[19];
3569 }
3570
3571 static u64 access_tx_sdma2_disallowed_packet_err_cnt(
3572                                 const struct cntr_entry *entry,
3573                                 void *context, int vl, int mode, u64 data)
3574 {
3575         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3576
3577         return dd->send_egress_err_status_cnt[18];
3578 }
3579
3580 static u64 access_tx_sdma1_disallowed_packet_err_cnt(
3581                                 const struct cntr_entry *entry,
3582                                 void *context, int vl, int mode, u64 data)
3583 {
3584         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3585
3586         return dd->send_egress_err_status_cnt[17];
3587 }
3588
3589 static u64 access_tx_sdma0_disallowed_packet_err_cnt(
3590                                 const struct cntr_entry *entry,
3591                                 void *context, int vl, int mode, u64 data)
3592 {
3593         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3594
3595         return dd->send_egress_err_status_cnt[16];
3596 }
3597
3598 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry,
3599                                            void *context, int vl, int mode,
3600                                            u64 data)
3601 {
3602         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3603
3604         return dd->send_egress_err_status_cnt[15];
3605 }
3606
3607 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry,
3608                                                  void *context, int vl,
3609                                                  int mode, u64 data)
3610 {
3611         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3612
3613         return dd->send_egress_err_status_cnt[14];
3614 }
3615
3616 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry,
3617                                                void *context, int vl, int mode,
3618                                                u64 data)
3619 {
3620         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3621
3622         return dd->send_egress_err_status_cnt[13];
3623 }
3624
3625 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry,
3626                                         void *context, int vl, int mode,
3627                                         u64 data)
3628 {
3629         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3630
3631         return dd->send_egress_err_status_cnt[12];
3632 }
3633
3634 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt(
3635                                 const struct cntr_entry *entry,
3636                                 void *context, int vl, int mode, u64 data)
3637 {
3638         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3639
3640         return dd->send_egress_err_status_cnt[11];
3641 }
3642
3643 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry,
3644                                              void *context, int vl, int mode,
3645                                              u64 data)
3646 {
3647         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3648
3649         return dd->send_egress_err_status_cnt[10];
3650 }
3651
3652 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry,
3653                                             void *context, int vl, int mode,
3654                                             u64 data)
3655 {
3656         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3657
3658         return dd->send_egress_err_status_cnt[9];
3659 }
3660
3661 static u64 access_tx_sdma_launch_intf_parity_err_cnt(
3662                                 const struct cntr_entry *entry,
3663                                 void *context, int vl, int mode, u64 data)
3664 {
3665         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3666
3667         return dd->send_egress_err_status_cnt[8];
3668 }
3669
3670 static u64 access_tx_pio_launch_intf_parity_err_cnt(
3671                                 const struct cntr_entry *entry,
3672                                 void *context, int vl, int mode, u64 data)
3673 {
3674         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3675
3676         return dd->send_egress_err_status_cnt[7];
3677 }
3678
3679 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry,
3680                                             void *context, int vl, int mode,
3681                                             u64 data)
3682 {
3683         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3684
3685         return dd->send_egress_err_status_cnt[6];
3686 }
3687
3688 static u64 access_tx_incorrect_link_state_err_cnt(
3689                                 const struct cntr_entry *entry,
3690                                 void *context, int vl, int mode, u64 data)
3691 {
3692         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3693
3694         return dd->send_egress_err_status_cnt[5];
3695 }
3696
3697 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry,
3698                                       void *context, int vl, int mode,
3699                                       u64 data)
3700 {
3701         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3702
3703         return dd->send_egress_err_status_cnt[4];
3704 }
3705
3706 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt(
3707                                 const struct cntr_entry *entry,
3708                                 void *context, int vl, int mode, u64 data)
3709 {
3710         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3711
3712         return dd->send_egress_err_status_cnt[3];
3713 }
3714
3715 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry,
3716                                             void *context, int vl, int mode,
3717                                             u64 data)
3718 {
3719         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3720
3721         return dd->send_egress_err_status_cnt[2];
3722 }
3723
3724 static u64 access_tx_pkt_integrity_mem_unc_err_cnt(
3725                                 const struct cntr_entry *entry,
3726                                 void *context, int vl, int mode, u64 data)
3727 {
3728         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3729
3730         return dd->send_egress_err_status_cnt[1];
3731 }
3732
3733 static u64 access_tx_pkt_integrity_mem_cor_err_cnt(
3734                                 const struct cntr_entry *entry,
3735                                 void *context, int vl, int mode, u64 data)
3736 {
3737         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3738
3739         return dd->send_egress_err_status_cnt[0];
3740 }
3741
3742 /*
3743  * Software counters corresponding to each of the
3744  * error status bits within SendErrStatus
3745  */
3746 static u64 access_send_csr_write_bad_addr_err_cnt(
3747                                 const struct cntr_entry *entry,
3748                                 void *context, int vl, int mode, u64 data)
3749 {
3750         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3751
3752         return dd->send_err_status_cnt[2];
3753 }
3754
3755 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry,
3756                                                  void *context, int vl,
3757                                                  int mode, u64 data)
3758 {
3759         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3760
3761         return dd->send_err_status_cnt[1];
3762 }
3763
3764 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry,
3765                                       void *context, int vl, int mode,
3766                                       u64 data)
3767 {
3768         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3769
3770         return dd->send_err_status_cnt[0];
3771 }
3772
3773 /*
3774  * Software counters corresponding to each of the
3775  * error status bits within SendCtxtErrStatus
3776  */
3777 static u64 access_pio_write_out_of_bounds_err_cnt(
3778                                 const struct cntr_entry *entry,
3779                                 void *context, int vl, int mode, u64 data)
3780 {
3781         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3782
3783         return dd->sw_ctxt_err_status_cnt[4];
3784 }
3785
3786 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry,
3787                                              void *context, int vl, int mode,
3788                                              u64 data)
3789 {
3790         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3791
3792         return dd->sw_ctxt_err_status_cnt[3];
3793 }
3794
3795 static u64 access_pio_write_crosses_boundary_err_cnt(
3796                                 const struct cntr_entry *entry,
3797                                 void *context, int vl, int mode, u64 data)
3798 {
3799         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3800
3801         return dd->sw_ctxt_err_status_cnt[2];
3802 }
3803
3804 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry,
3805                                                 void *context, int vl,
3806                                                 int mode, u64 data)
3807 {
3808         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3809
3810         return dd->sw_ctxt_err_status_cnt[1];
3811 }
3812
3813 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry,
3814                                                void *context, int vl, int mode,
3815                                                u64 data)
3816 {
3817         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3818
3819         return dd->sw_ctxt_err_status_cnt[0];
3820 }
3821
3822 /*
3823  * Software counters corresponding to each of the
3824  * error status bits within SendDmaEngErrStatus
3825  */
3826 static u64 access_sdma_header_request_fifo_cor_err_cnt(
3827                                 const struct cntr_entry *entry,
3828                                 void *context, int vl, int mode, u64 data)
3829 {
3830         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3831
3832         return dd->sw_send_dma_eng_err_status_cnt[23];
3833 }
3834
3835 static u64 access_sdma_header_storage_cor_err_cnt(
3836                                 const struct cntr_entry *entry,
3837                                 void *context, int vl, int mode, u64 data)
3838 {
3839         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3840
3841         return dd->sw_send_dma_eng_err_status_cnt[22];
3842 }
3843
3844 static u64 access_sdma_packet_tracking_cor_err_cnt(
3845                                 const struct cntr_entry *entry,
3846                                 void *context, int vl, int mode, u64 data)
3847 {
3848         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3849
3850         return dd->sw_send_dma_eng_err_status_cnt[21];
3851 }
3852
3853 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry,
3854                                             void *context, int vl, int mode,
3855                                             u64 data)
3856 {
3857         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3858
3859         return dd->sw_send_dma_eng_err_status_cnt[20];
3860 }
3861
3862 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry,
3863                                               void *context, int vl, int mode,
3864                                               u64 data)
3865 {
3866         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3867
3868         return dd->sw_send_dma_eng_err_status_cnt[19];
3869 }
3870
3871 static u64 access_sdma_header_request_fifo_unc_err_cnt(
3872                                 const struct cntr_entry *entry,
3873                                 void *context, int vl, int mode, u64 data)
3874 {
3875         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3876
3877         return dd->sw_send_dma_eng_err_status_cnt[18];
3878 }
3879
3880 static u64 access_sdma_header_storage_unc_err_cnt(
3881                                 const struct cntr_entry *entry,
3882                                 void *context, int vl, int mode, u64 data)
3883 {
3884         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3885
3886         return dd->sw_send_dma_eng_err_status_cnt[17];
3887 }
3888
3889 static u64 access_sdma_packet_tracking_unc_err_cnt(
3890                                 const struct cntr_entry *entry,
3891                                 void *context, int vl, int mode, u64 data)
3892 {
3893         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3894
3895         return dd->sw_send_dma_eng_err_status_cnt[16];
3896 }
3897
3898 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry,
3899                                             void *context, int vl, int mode,
3900                                             u64 data)
3901 {
3902         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3903
3904         return dd->sw_send_dma_eng_err_status_cnt[15];
3905 }
3906
3907 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry,
3908                                               void *context, int vl, int mode,
3909                                               u64 data)
3910 {
3911         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3912
3913         return dd->sw_send_dma_eng_err_status_cnt[14];
3914 }
3915
3916 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry,
3917                                        void *context, int vl, int mode,
3918                                        u64 data)
3919 {
3920         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3921
3922         return dd->sw_send_dma_eng_err_status_cnt[13];
3923 }
3924
3925 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry,
3926                                              void *context, int vl, int mode,
3927                                              u64 data)
3928 {
3929         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3930
3931         return dd->sw_send_dma_eng_err_status_cnt[12];
3932 }
3933
3934 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry,
3935                                               void *context, int vl, int mode,
3936                                               u64 data)
3937 {
3938         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3939
3940         return dd->sw_send_dma_eng_err_status_cnt[11];
3941 }
3942
3943 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry,
3944                                              void *context, int vl, int mode,
3945                                              u64 data)
3946 {
3947         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3948
3949         return dd->sw_send_dma_eng_err_status_cnt[10];
3950 }
3951
3952 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry,
3953                                           void *context, int vl, int mode,
3954                                           u64 data)
3955 {
3956         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3957
3958         return dd->sw_send_dma_eng_err_status_cnt[9];
3959 }
3960
3961 static u64 access_sdma_packet_desc_overflow_err_cnt(
3962                                 const struct cntr_entry *entry,
3963                                 void *context, int vl, int mode, u64 data)
3964 {
3965         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3966
3967         return dd->sw_send_dma_eng_err_status_cnt[8];
3968 }
3969
3970 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry,
3971                                                void *context, int vl,
3972                                                int mode, u64 data)
3973 {
3974         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3975
3976         return dd->sw_send_dma_eng_err_status_cnt[7];
3977 }
3978
3979 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry,
3980                                     void *context, int vl, int mode, u64 data)
3981 {
3982         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3983
3984         return dd->sw_send_dma_eng_err_status_cnt[6];
3985 }
3986
3987 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry,
3988                                         void *context, int vl, int mode,
3989                                         u64 data)
3990 {
3991         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
3992
3993         return dd->sw_send_dma_eng_err_status_cnt[5];
3994 }
3995
3996 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry,
3997                                           void *context, int vl, int mode,
3998                                           u64 data)
3999 {
4000         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4001
4002         return dd->sw_send_dma_eng_err_status_cnt[4];
4003 }
4004
4005 static u64 access_sdma_tail_out_of_bounds_err_cnt(
4006                                 const struct cntr_entry *entry,
4007                                 void *context, int vl, int mode, u64 data)
4008 {
4009         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4010
4011         return dd->sw_send_dma_eng_err_status_cnt[3];
4012 }
4013
4014 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry,
4015                                         void *context, int vl, int mode,
4016                                         u64 data)
4017 {
4018         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4019
4020         return dd->sw_send_dma_eng_err_status_cnt[2];
4021 }
4022
4023 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry,
4024                                             void *context, int vl, int mode,
4025                                             u64 data)
4026 {
4027         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4028
4029         return dd->sw_send_dma_eng_err_status_cnt[1];
4030 }
4031
4032 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry,
4033                                         void *context, int vl, int mode,
4034                                         u64 data)
4035 {
4036         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4037
4038         return dd->sw_send_dma_eng_err_status_cnt[0];
4039 }
4040
4041 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry,
4042                                  void *context, int vl, int mode,
4043                                  u64 data)
4044 {
4045         struct hfi1_devdata *dd = (struct hfi1_devdata *)context;
4046
4047         u64 val = 0;
4048         u64 csr = entry->csr;
4049
4050         val = read_write_csr(dd, csr, mode, data);
4051         if (mode == CNTR_MODE_R) {
4052                 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ?
4053                         CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors;
4054         } else if (mode == CNTR_MODE_W) {
4055                 dd->sw_rcv_bypass_packet_errors = 0;
4056         } else {
4057                 dd_dev_err(dd, "Invalid cntr register access mode");
4058                 return 0;
4059         }
4060         return val;
4061 }
4062
4063 #define def_access_sw_cpu(cntr) \
4064 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry,               \
4065                               void *context, int vl, int mode, u64 data)      \
4066 {                                                                             \
4067         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;        \
4068         return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr,       \
4069                               ppd->ibport_data.rvp.cntr, vl,                  \
4070                               mode, data);                                    \
4071 }
4072
4073 def_access_sw_cpu(rc_acks);
4074 def_access_sw_cpu(rc_qacks);
4075 def_access_sw_cpu(rc_delayed_comp);
4076
4077 #define def_access_ibp_counter(cntr) \
4078 static u64 access_ibp_##cntr(const struct cntr_entry *entry,                  \
4079                                 void *context, int vl, int mode, u64 data)    \
4080 {                                                                             \
4081         struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context;        \
4082                                                                               \
4083         if (vl != CNTR_INVALID_VL)                                            \
4084                 return 0;                                                     \
4085                                                                               \
4086         return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr,        \
4087                              mode, data);                                     \
4088 }
4089
4090 def_access_ibp_counter(loop_pkts);
4091 def_access_ibp_counter(rc_resends);
4092 def_access_ibp_counter(rnr_naks);
4093 def_access_ibp_counter(other_naks);
4094 def_access_ibp_counter(rc_timeouts);
4095 def_access_ibp_counter(pkt_drops);
4096 def_access_ibp_counter(dmawait);
4097 def_access_ibp_counter(rc_seqnak);
4098 def_access_ibp_counter(rc_dupreq);
4099 def_access_ibp_counter(rdma_seq);
4100 def_access_ibp_counter(unaligned);
4101 def_access_ibp_counter(seq_naks);
4102
4103 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
4104 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
4105 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
4106                         CNTR_NORMAL),
4107 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
4108                         CNTR_NORMAL),
4109 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs,
4110                         RCV_TID_FLOW_GEN_MISMATCH_CNT,
4111                         CNTR_NORMAL),
4112 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL,
4113                         CNTR_NORMAL),
4114 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs,
4115                         RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL),
4116 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt,
4117                         CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL),
4118 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT,
4119                         CNTR_NORMAL),
4120 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT,
4121                         CNTR_NORMAL),
4122 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT,
4123                         CNTR_NORMAL),
4124 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT,
4125                         CNTR_NORMAL),
4126 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT,
4127                         CNTR_NORMAL),
4128 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT,
4129                         CNTR_NORMAL),
4130 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt,
4131                         CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL),
4132 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt,
4133                         CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL),
4134 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT,
4135                               CNTR_SYNTH),
4136 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH,
4137                             access_dc_rcv_err_cnt),
4138 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT,
4139                                  CNTR_SYNTH),
4140 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT,
4141                                   CNTR_SYNTH),
4142 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT,
4143                                   CNTR_SYNTH),
4144 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts,
4145                                    DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH),
4146 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts,
4147                                   DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT,
4148                                   CNTR_SYNTH),
4149 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr,
4150                                 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH),
4151 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT,
4152                                CNTR_SYNTH),
4153 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT,
4154                               CNTR_SYNTH),
4155 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT,
4156                                CNTR_SYNTH),
4157 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT,
4158                                  CNTR_SYNTH),
4159 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT,
4160                                 CNTR_SYNTH),
4161 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT,
4162                                 CNTR_SYNTH),
4163 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT,
4164                                CNTR_SYNTH),
4165 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT,
4166                                  CNTR_SYNTH | CNTR_VL),
4167 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT,
4168                                 CNTR_SYNTH | CNTR_VL),
4169 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH),
4170 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT,
4171                                  CNTR_SYNTH | CNTR_VL),
4172 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH),
4173 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT,
4174                                  CNTR_SYNTH | CNTR_VL),
4175 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT,
4176                               CNTR_SYNTH),
4177 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT,
4178                                  CNTR_SYNTH | CNTR_VL),
4179 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT,
4180                                 CNTR_SYNTH),
4181 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT,
4182                                    CNTR_SYNTH | CNTR_VL),
4183 [C_DC_TOTAL_CRC] =
4184         DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR,
4185                          CNTR_SYNTH),
4186 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0,
4187                                   CNTR_SYNTH),
4188 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1,
4189                                   CNTR_SYNTH),
4190 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2,
4191                                   CNTR_SYNTH),
4192 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3,
4193                                   CNTR_SYNTH),
4194 [C_DC_CRC_MULT_LN] =
4195         DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN,
4196                          CNTR_SYNTH),
4197 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT,
4198                                     CNTR_SYNTH),
4199 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT,
4200                                     CNTR_SYNTH),
4201 [C_DC_SEQ_CRC_CNT] =
4202         DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT,
4203                          CNTR_SYNTH),
4204 [C_DC_ESC0_ONLY_CNT] =
4205         DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT,
4206                          CNTR_SYNTH),
4207 [C_DC_ESC0_PLUS1_CNT] =
4208         DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT,
4209                          CNTR_SYNTH),
4210 [C_DC_ESC0_PLUS2_CNT] =
4211         DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT,
4212                          CNTR_SYNTH),
4213 [C_DC_REINIT_FROM_PEER_CNT] =
4214         DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT,
4215                          CNTR_SYNTH),
4216 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT,
4217                                   CNTR_SYNTH),
4218 [C_DC_MISC_FLG_CNT] =
4219         DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT,
4220                          CNTR_SYNTH),
4221 [C_DC_PRF_GOOD_LTP_CNT] =
4222         DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH),
4223 [C_DC_PRF_ACCEPTED_LTP_CNT] =
4224         DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT,
4225                          CNTR_SYNTH),
4226 [C_DC_PRF_RX_FLIT_CNT] =
4227         DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH),
4228 [C_DC_PRF_TX_FLIT_CNT] =
4229         DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH),
4230 [C_DC_PRF_CLK_CNTR] =
4231         DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH),
4232 [C_DC_PG_DBG_FLIT_CRDTS_CNT] =
4233         DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH),
4234 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] =
4235         DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT,
4236                          CNTR_SYNTH),
4237 [C_DC_PG_STS_TX_SBE_CNT] =
4238         DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH),
4239 [C_DC_PG_STS_TX_MBE_CNT] =
4240         DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT,
4241                          CNTR_SYNTH),
4242 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL,
4243                             access_sw_cpu_intr),
4244 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL,
4245                             access_sw_cpu_rcv_limit),
4246 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL,
4247                             access_sw_vtx_wait),
4248 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL,
4249                             access_sw_pio_wait),
4250 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL,
4251                             access_sw_pio_drain),
4252 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL,
4253                             access_sw_kmem_wait),
4254 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL,
4255                             access_sw_send_schedule),
4256 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn",
4257                                       SEND_DMA_DESC_FETCHED_CNT, 0,
4258                                       CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4259                                       dev_access_u32_csr),
4260 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0,
4261                              CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4262                              access_sde_int_cnt),
4263 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0,
4264                              CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4265                              access_sde_err_cnt),
4266 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0,
4267                                   CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4268                                   access_sde_idle_int_cnt),
4269 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0,
4270                                       CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA,
4271                                       access_sde_progress_int_cnt),
4272 /* MISC_ERR_STATUS */
4273 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0,
4274                                 CNTR_NORMAL,
4275                                 access_misc_pll_lock_fail_err_cnt),
4276 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0,
4277                                 CNTR_NORMAL,
4278                                 access_misc_mbist_fail_err_cnt),
4279 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0,
4280                                 CNTR_NORMAL,
4281                                 access_misc_invalid_eep_cmd_err_cnt),
4282 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0,
4283                                 CNTR_NORMAL,
4284                                 access_misc_efuse_done_parity_err_cnt),
4285 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0,
4286                                 CNTR_NORMAL,
4287                                 access_misc_efuse_write_err_cnt),
4288 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0,
4289                                 0, CNTR_NORMAL,
4290                                 access_misc_efuse_read_bad_addr_err_cnt),
4291 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0,
4292                                 CNTR_NORMAL,
4293                                 access_misc_efuse_csr_parity_err_cnt),
4294 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0,
4295                                 CNTR_NORMAL,
4296                                 access_misc_fw_auth_failed_err_cnt),
4297 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0,
4298                                 CNTR_NORMAL,
4299                                 access_misc_key_mismatch_err_cnt),
4300 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0,
4301                                 CNTR_NORMAL,
4302                                 access_misc_sbus_write_failed_err_cnt),
4303 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0,
4304                                 CNTR_NORMAL,
4305                                 access_misc_csr_write_bad_addr_err_cnt),
4306 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0,
4307                                 CNTR_NORMAL,
4308                                 access_misc_csr_read_bad_addr_err_cnt),
4309 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0,
4310                                 CNTR_NORMAL,
4311                                 access_misc_csr_parity_err_cnt),
4312 /* CceErrStatus */
4313 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0,
4314                                 CNTR_NORMAL,
4315                                 access_sw_cce_err_status_aggregated_cnt),
4316 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0,
4317                                 CNTR_NORMAL,
4318                                 access_cce_msix_csr_parity_err_cnt),
4319 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0,
4320                                 CNTR_NORMAL,
4321                                 access_cce_int_map_unc_err_cnt),
4322 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0,
4323                                 CNTR_NORMAL,
4324                                 access_cce_int_map_cor_err_cnt),
4325 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0,
4326                                 CNTR_NORMAL,
4327                                 access_cce_msix_table_unc_err_cnt),
4328 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0,
4329                                 CNTR_NORMAL,
4330                                 access_cce_msix_table_cor_err_cnt),
4331 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0,
4332                                 0, CNTR_NORMAL,
4333                                 access_cce_rxdma_conv_fifo_parity_err_cnt),
4334 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0,
4335                                 0, CNTR_NORMAL,
4336                                 access_cce_rcpl_async_fifo_parity_err_cnt),
4337 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0,
4338                                 CNTR_NORMAL,
4339                                 access_cce_seg_write_bad_addr_err_cnt),
4340 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0,
4341                                 CNTR_NORMAL,
4342                                 access_cce_seg_read_bad_addr_err_cnt),
4343 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0,
4344                                 CNTR_NORMAL,
4345                                 access_la_triggered_cnt),
4346 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0,
4347                                 CNTR_NORMAL,
4348                                 access_cce_trgt_cpl_timeout_err_cnt),
4349 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0,
4350                                 CNTR_NORMAL,
4351                                 access_pcic_receive_parity_err_cnt),
4352 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0,
4353                                 CNTR_NORMAL,
4354                                 access_pcic_transmit_back_parity_err_cnt),
4355 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0,
4356                                 0, CNTR_NORMAL,
4357                                 access_pcic_transmit_front_parity_err_cnt),
4358 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0,
4359                                 CNTR_NORMAL,
4360                                 access_pcic_cpl_dat_q_unc_err_cnt),
4361 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0,
4362                                 CNTR_NORMAL,
4363                                 access_pcic_cpl_hd_q_unc_err_cnt),
4364 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0,
4365                                 CNTR_NORMAL,
4366                                 access_pcic_post_dat_q_unc_err_cnt),
4367 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0,
4368                                 CNTR_NORMAL,
4369                                 access_pcic_post_hd_q_unc_err_cnt),
4370 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0,
4371                                 CNTR_NORMAL,
4372                                 access_pcic_retry_sot_mem_unc_err_cnt),
4373 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0,
4374                                 CNTR_NORMAL,
4375                                 access_pcic_retry_mem_unc_err),
4376 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0,
4377                                 CNTR_NORMAL,
4378                                 access_pcic_n_post_dat_q_parity_err_cnt),
4379 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0,
4380                                 CNTR_NORMAL,
4381                                 access_pcic_n_post_h_q_parity_err_cnt),
4382 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0,
4383                                 CNTR_NORMAL,
4384                                 access_pcic_cpl_dat_q_cor_err_cnt),
4385 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0,
4386                                 CNTR_NORMAL,
4387                                 access_pcic_cpl_hd_q_cor_err_cnt),
4388 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0,
4389                                 CNTR_NORMAL,
4390                                 access_pcic_post_dat_q_cor_err_cnt),
4391 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0,
4392                                 CNTR_NORMAL,
4393                                 access_pcic_post_hd_q_cor_err_cnt),
4394 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0,
4395                                 CNTR_NORMAL,
4396                                 access_pcic_retry_sot_mem_cor_err_cnt),
4397 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0,
4398                                 CNTR_NORMAL,
4399                                 access_pcic_retry_mem_cor_err_cnt),
4400 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM(
4401                                 "CceCli1AsyncFifoDbgParityError", 0, 0,
4402                                 CNTR_NORMAL,
4403                                 access_cce_cli1_async_fifo_dbg_parity_err_cnt),
4404 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM(
4405                                 "CceCli1AsyncFifoRxdmaParityError", 0, 0,
4406                                 CNTR_NORMAL,
4407                                 access_cce_cli1_async_fifo_rxdma_parity_err_cnt
4408                                 ),
4409 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM(
4410                         "CceCli1AsyncFifoSdmaHdParityErr", 0, 0,
4411                         CNTR_NORMAL,
4412                         access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt),
4413 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM(
4414                         "CceCli1AsyncFifoPioCrdtParityErr", 0, 0,
4415                         CNTR_NORMAL,
4416                         access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt),
4417 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0,
4418                         0, CNTR_NORMAL,
4419                         access_cce_cli2_async_fifo_parity_err_cnt),
4420 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0,
4421                         CNTR_NORMAL,
4422                         access_cce_csr_cfg_bus_parity_err_cnt),
4423 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0,
4424                         0, CNTR_NORMAL,
4425                         access_cce_cli0_async_fifo_parity_err_cnt),
4426 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0,
4427                         CNTR_NORMAL,
4428                         access_cce_rspd_data_parity_err_cnt),
4429 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0,
4430                         CNTR_NORMAL,
4431                         access_cce_trgt_access_err_cnt),
4432 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0,
4433                         0, CNTR_NORMAL,
4434                         access_cce_trgt_async_fifo_parity_err_cnt),
4435 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0,
4436                         CNTR_NORMAL,
4437                         access_cce_csr_write_bad_addr_err_cnt),
4438 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0,
4439                         CNTR_NORMAL,
4440                         access_cce_csr_read_bad_addr_err_cnt),
4441 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0,
4442                         CNTR_NORMAL,
4443                         access_ccs_csr_parity_err_cnt),
4444
4445 /* RcvErrStatus */
4446 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0,
4447                         CNTR_NORMAL,
4448                         access_rx_csr_parity_err_cnt),
4449 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0,
4450                         CNTR_NORMAL,
4451                         access_rx_csr_write_bad_addr_err_cnt),
4452 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0,
4453                         CNTR_NORMAL,
4454                         access_rx_csr_read_bad_addr_err_cnt),
4455 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0,
4456                         CNTR_NORMAL,
4457                         access_rx_dma_csr_unc_err_cnt),
4458 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0,
4459                         CNTR_NORMAL,
4460                         access_rx_dma_dq_fsm_encoding_err_cnt),
4461 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0,
4462                         CNTR_NORMAL,
4463                         access_rx_dma_eq_fsm_encoding_err_cnt),
4464 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0,
4465                         CNTR_NORMAL,
4466                         access_rx_dma_csr_parity_err_cnt),
4467 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0,
4468                         CNTR_NORMAL,
4469                         access_rx_rbuf_data_cor_err_cnt),
4470 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0,
4471                         CNTR_NORMAL,
4472                         access_rx_rbuf_data_unc_err_cnt),
4473 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0,
4474                         CNTR_NORMAL,
4475                         access_rx_dma_data_fifo_rd_cor_err_cnt),
4476 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0,
4477                         CNTR_NORMAL,
4478                         access_rx_dma_data_fifo_rd_unc_err_cnt),
4479 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0,
4480                         CNTR_NORMAL,
4481                         access_rx_dma_hdr_fifo_rd_cor_err_cnt),
4482 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0,
4483                         CNTR_NORMAL,
4484                         access_rx_dma_hdr_fifo_rd_unc_err_cnt),
4485 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0,
4486                         CNTR_NORMAL,
4487                         access_rx_rbuf_desc_part2_cor_err_cnt),
4488 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0,
4489                         CNTR_NORMAL,
4490                         access_rx_rbuf_desc_part2_unc_err_cnt),
4491 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0,
4492                         CNTR_NORMAL,
4493                         access_rx_rbuf_desc_part1_cor_err_cnt),
4494 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0,
4495                         CNTR_NORMAL,
4496                         access_rx_rbuf_desc_part1_unc_err_cnt),
4497 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0,
4498                         CNTR_NORMAL,
4499                         access_rx_hq_intr_fsm_err_cnt),
4500 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0,
4501                         CNTR_NORMAL,
4502                         access_rx_hq_intr_csr_parity_err_cnt),
4503 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0,
4504                         CNTR_NORMAL,
4505                         access_rx_lookup_csr_parity_err_cnt),
4506 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0,
4507                         CNTR_NORMAL,
4508                         access_rx_lookup_rcv_array_cor_err_cnt),
4509 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0,
4510                         CNTR_NORMAL,
4511                         access_rx_lookup_rcv_array_unc_err_cnt),
4512 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0,
4513                         0, CNTR_NORMAL,
4514                         access_rx_lookup_des_part2_parity_err_cnt),
4515 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0,
4516                         0, CNTR_NORMAL,
4517                         access_rx_lookup_des_part1_unc_cor_err_cnt),
4518 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0,
4519                         CNTR_NORMAL,
4520                         access_rx_lookup_des_part1_unc_err_cnt),
4521 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0,
4522                         CNTR_NORMAL,
4523                         access_rx_rbuf_next_free_buf_cor_err_cnt),
4524 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0,
4525                         CNTR_NORMAL,
4526                         access_rx_rbuf_next_free_buf_unc_err_cnt),
4527 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM(
4528                         "RxRbufFlInitWrAddrParityErr", 0, 0,
4529                         CNTR_NORMAL,
4530                         access_rbuf_fl_init_wr_addr_parity_err_cnt),
4531 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0,
4532                         0, CNTR_NORMAL,
4533                         access_rx_rbuf_fl_initdone_parity_err_cnt),
4534 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0,
4535                         0, CNTR_NORMAL,
4536                         access_rx_rbuf_fl_write_addr_parity_err_cnt),
4537 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0,
4538                         CNTR_NORMAL,
4539                         access_rx_rbuf_fl_rd_addr_parity_err_cnt),
4540 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0,
4541                         CNTR_NORMAL,
4542                         access_rx_rbuf_empty_err_cnt),
4543 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0,
4544                         CNTR_NORMAL,
4545                         access_rx_rbuf_full_err_cnt),
4546 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0,
4547                         CNTR_NORMAL,
4548                         access_rbuf_bad_lookup_err_cnt),
4549 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0,
4550                         CNTR_NORMAL,
4551                         access_rbuf_ctx_id_parity_err_cnt),
4552 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0,
4553                         CNTR_NORMAL,
4554                         access_rbuf_csr_qeopdw_parity_err_cnt),
4555 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM(
4556                         "RxRbufCsrQNumOfPktParityErr", 0, 0,
4557                         CNTR_NORMAL,
4558                         access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt),
4559 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM(
4560                         "RxRbufCsrQTlPtrParityErr", 0, 0,
4561                         CNTR_NORMAL,
4562                         access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt),
4563 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0,
4564                         0, CNTR_NORMAL,
4565                         access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt),
4566 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0,
4567                         0, CNTR_NORMAL,
4568                         access_rx_rbuf_csr_q_vld_bit_parity_err_cnt),
4569 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr",
4570                         0, 0, CNTR_NORMAL,
4571                         access_rx_rbuf_csr_q_next_buf_parity_err_cnt),
4572 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0,
4573                         0, CNTR_NORMAL,
4574                         access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt),
4575 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM(
4576                         "RxRbufCsrQHeadBufNumParityErr", 0, 0,
4577                         CNTR_NORMAL,
4578                         access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt),
4579 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0,
4580                         0, CNTR_NORMAL,
4581                         access_rx_rbuf_block_list_read_cor_err_cnt),
4582 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0,
4583                         0, CNTR_NORMAL,
4584                         access_rx_rbuf_block_list_read_unc_err_cnt),
4585 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0,
4586                         CNTR_NORMAL,
4587                         access_rx_rbuf_lookup_des_cor_err_cnt),
4588 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0,
4589                         CNTR_NORMAL,
4590                         access_rx_rbuf_lookup_des_unc_err_cnt),
4591 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM(
4592                         "RxRbufLookupDesRegUncCorErr", 0, 0,
4593                         CNTR_NORMAL,
4594                         access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt),
4595 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0,
4596                         CNTR_NORMAL,
4597                         access_rx_rbuf_lookup_des_reg_unc_err_cnt),
4598 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0,
4599                         CNTR_NORMAL,
4600                         access_rx_rbuf_free_list_cor_err_cnt),
4601 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0,
4602                         CNTR_NORMAL,
4603                         access_rx_rbuf_free_list_unc_err_cnt),
4604 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0,
4605                         CNTR_NORMAL,
4606                         access_rx_rcv_fsm_encoding_err_cnt),
4607 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0,
4608                         CNTR_NORMAL,
4609                         access_rx_dma_flag_cor_err_cnt),
4610 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0,
4611                         CNTR_NORMAL,
4612                         access_rx_dma_flag_unc_err_cnt),
4613 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0,
4614                         CNTR_NORMAL,
4615                         access_rx_dc_sop_eop_parity_err_cnt),
4616 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0,
4617                         CNTR_NORMAL,
4618                         access_rx_rcv_csr_parity_err_cnt),
4619 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0,
4620                         CNTR_NORMAL,
4621                         access_rx_rcv_qp_map_table_cor_err_cnt),
4622 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0,
4623                         CNTR_NORMAL,
4624                         access_rx_rcv_qp_map_table_unc_err_cnt),
4625 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0,
4626                         CNTR_NORMAL,
4627                         access_rx_rcv_data_cor_err_cnt),
4628 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0,
4629                         CNTR_NORMAL,
4630                         access_rx_rcv_data_unc_err_cnt),
4631 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0,
4632                         CNTR_NORMAL,
4633                         access_rx_rcv_hdr_cor_err_cnt),
4634 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0,
4635                         CNTR_NORMAL,
4636                         access_rx_rcv_hdr_unc_err_cnt),
4637 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0,
4638                         CNTR_NORMAL,
4639                         access_rx_dc_intf_parity_err_cnt),
4640 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0,
4641                         CNTR_NORMAL,
4642                         access_rx_dma_csr_cor_err_cnt),
4643 /* SendPioErrStatus */
4644 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0,
4645                         CNTR_NORMAL,
4646                         access_pio_pec_sop_head_parity_err_cnt),
4647 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0,
4648                         CNTR_NORMAL,
4649                         access_pio_pcc_sop_head_parity_err_cnt),
4650 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr",
4651                         0, 0, CNTR_NORMAL,
4652                         access_pio_last_returned_cnt_parity_err_cnt),
4653 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0,
4654                         0, CNTR_NORMAL,
4655                         access_pio_current_free_cnt_parity_err_cnt),
4656 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0,
4657                         CNTR_NORMAL,
4658                         access_pio_reserved_31_err_cnt),
4659 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0,
4660                         CNTR_NORMAL,
4661                         access_pio_reserved_30_err_cnt),
4662 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0,
4663                         CNTR_NORMAL,
4664                         access_pio_ppmc_sop_len_err_cnt),
4665 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0,
4666                         CNTR_NORMAL,
4667                         access_pio_ppmc_bqc_mem_parity_err_cnt),
4668 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0,
4669                         CNTR_NORMAL,
4670                         access_pio_vl_fifo_parity_err_cnt),
4671 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0,
4672                         CNTR_NORMAL,
4673                         access_pio_vlf_sop_parity_err_cnt),
4674 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0,
4675                         CNTR_NORMAL,
4676                         access_pio_vlf_v1_len_parity_err_cnt),
4677 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0,
4678                         CNTR_NORMAL,
4679                         access_pio_block_qw_count_parity_err_cnt),
4680 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0,
4681                         CNTR_NORMAL,
4682                         access_pio_write_qw_valid_parity_err_cnt),
4683 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0,
4684                         CNTR_NORMAL,
4685                         access_pio_state_machine_err_cnt),
4686 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0,
4687                         CNTR_NORMAL,
4688                         access_pio_write_data_parity_err_cnt),
4689 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0,
4690                         CNTR_NORMAL,
4691                         access_pio_host_addr_mem_cor_err_cnt),
4692 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0,
4693                         CNTR_NORMAL,
4694                         access_pio_host_addr_mem_unc_err_cnt),
4695 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0,
4696                         CNTR_NORMAL,
4697                         access_pio_pkt_evict_sm_or_arb_sm_err_cnt),
4698 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0,
4699                         CNTR_NORMAL,
4700                         access_pio_init_sm_in_err_cnt),
4701 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0,
4702                         CNTR_NORMAL,
4703                         access_pio_ppmc_pbl_fifo_err_cnt),
4704 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0,
4705                         0, CNTR_NORMAL,
4706                         access_pio_credit_ret_fifo_parity_err_cnt),
4707 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0,
4708                         CNTR_NORMAL,
4709                         access_pio_v1_len_mem_bank1_cor_err_cnt),
4710 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0,
4711                         CNTR_NORMAL,
4712                         access_pio_v1_len_mem_bank0_cor_err_cnt),
4713 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0,
4714                         CNTR_NORMAL,
4715                         access_pio_v1_len_mem_bank1_unc_err_cnt),
4716 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0,
4717                         CNTR_NORMAL,
4718                         access_pio_v1_len_mem_bank0_unc_err_cnt),
4719 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0,
4720                         CNTR_NORMAL,
4721                         access_pio_sm_pkt_reset_parity_err_cnt),
4722 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0,
4723                         CNTR_NORMAL,
4724                         access_pio_pkt_evict_fifo_parity_err_cnt),
4725 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM(
4726                         "PioSbrdctrlCrrelFifoParityErr", 0, 0,
4727                         CNTR_NORMAL,
4728                         access_pio_sbrdctrl_crrel_fifo_parity_err_cnt),
4729 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0,
4730                         CNTR_NORMAL,
4731                         access_pio_sbrdctl_crrel_parity_err_cnt),
4732 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0,
4733                         CNTR_NORMAL,
4734                         access_pio_pec_fifo_parity_err_cnt),
4735 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0,
4736                         CNTR_NORMAL,
4737                         access_pio_pcc_fifo_parity_err_cnt),
4738 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0,
4739                         CNTR_NORMAL,
4740                         access_pio_sb_mem_fifo1_err_cnt),
4741 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0,
4742                         CNTR_NORMAL,
4743                         access_pio_sb_mem_fifo0_err_cnt),
4744 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0,
4745                         CNTR_NORMAL,
4746                         access_pio_csr_parity_err_cnt),
4747 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0,
4748                         CNTR_NORMAL,
4749                         access_pio_write_addr_parity_err_cnt),
4750 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0,
4751                         CNTR_NORMAL,
4752                         access_pio_write_bad_ctxt_err_cnt),
4753 /* SendDmaErrStatus */
4754 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0,
4755                         0, CNTR_NORMAL,
4756                         access_sdma_pcie_req_tracking_cor_err_cnt),
4757 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0,
4758                         0, CNTR_NORMAL,
4759                         access_sdma_pcie_req_tracking_unc_err_cnt),
4760 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0,
4761                         CNTR_NORMAL,
4762                         access_sdma_csr_parity_err_cnt),
4763 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0,
4764                         CNTR_NORMAL,
4765                         access_sdma_rpy_tag_err_cnt),
4766 /* SendEgressErrStatus */
4767 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0,
4768                         CNTR_NORMAL,
4769                         access_tx_read_pio_memory_csr_unc_err_cnt),
4770 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0,
4771                         0, CNTR_NORMAL,
4772                         access_tx_read_sdma_memory_csr_err_cnt),
4773 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0,
4774                         CNTR_NORMAL,
4775                         access_tx_egress_fifo_cor_err_cnt),
4776 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0,
4777                         CNTR_NORMAL,
4778                         access_tx_read_pio_memory_cor_err_cnt),
4779 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0,
4780                         CNTR_NORMAL,
4781                         access_tx_read_sdma_memory_cor_err_cnt),
4782 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0,
4783                         CNTR_NORMAL,
4784                         access_tx_sb_hdr_cor_err_cnt),
4785 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0,
4786                         CNTR_NORMAL,
4787                         access_tx_credit_overrun_err_cnt),
4788 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0,
4789                         CNTR_NORMAL,
4790                         access_tx_launch_fifo8_cor_err_cnt),
4791 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0,
4792                         CNTR_NORMAL,
4793                         access_tx_launch_fifo7_cor_err_cnt),
4794 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0,
4795                         CNTR_NORMAL,
4796                         access_tx_launch_fifo6_cor_err_cnt),
4797 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0,
4798                         CNTR_NORMAL,
4799                         access_tx_launch_fifo5_cor_err_cnt),
4800 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0,
4801                         CNTR_NORMAL,
4802                         access_tx_launch_fifo4_cor_err_cnt),
4803 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0,
4804                         CNTR_NORMAL,
4805                         access_tx_launch_fifo3_cor_err_cnt),
4806 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0,
4807                         CNTR_NORMAL,
4808                         access_tx_launch_fifo2_cor_err_cnt),
4809 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0,
4810                         CNTR_NORMAL,
4811                         access_tx_launch_fifo1_cor_err_cnt),
4812 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0,
4813                         CNTR_NORMAL,
4814                         access_tx_launch_fifo0_cor_err_cnt),
4815 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0,
4816                         CNTR_NORMAL,
4817                         access_tx_credit_return_vl_err_cnt),
4818 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0,
4819                         CNTR_NORMAL,
4820                         access_tx_hcrc_insertion_err_cnt),
4821 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0,
4822                         CNTR_NORMAL,
4823                         access_tx_egress_fifo_unc_err_cnt),
4824 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0,
4825                         CNTR_NORMAL,
4826                         access_tx_read_pio_memory_unc_err_cnt),
4827 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0,
4828                         CNTR_NORMAL,
4829                         access_tx_read_sdma_memory_unc_err_cnt),
4830 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0,
4831                         CNTR_NORMAL,
4832                         access_tx_sb_hdr_unc_err_cnt),
4833 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0,
4834                         CNTR_NORMAL,
4835                         access_tx_credit_return_partiy_err_cnt),
4836 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr",
4837                         0, 0, CNTR_NORMAL,
4838                         access_tx_launch_fifo8_unc_or_parity_err_cnt),
4839 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr",
4840                         0, 0, CNTR_NORMAL,
4841                         access_tx_launch_fifo7_unc_or_parity_err_cnt),
4842 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr",
4843                         0, 0, CNTR_NORMAL,
4844                         access_tx_launch_fifo6_unc_or_parity_err_cnt),
4845 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr",
4846                         0, 0, CNTR_NORMAL,
4847                         access_tx_launch_fifo5_unc_or_parity_err_cnt),
4848 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr",
4849                         0, 0, CNTR_NORMAL,
4850                         access_tx_launch_fifo4_unc_or_parity_err_cnt),
4851 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr",
4852                         0, 0, CNTR_NORMAL,
4853                         access_tx_launch_fifo3_unc_or_parity_err_cnt),
4854 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr",
4855                         0, 0, CNTR_NORMAL,
4856                         access_tx_launch_fifo2_unc_or_parity_err_cnt),
4857 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr",
4858                         0, 0, CNTR_NORMAL,
4859                         access_tx_launch_fifo1_unc_or_parity_err_cnt),
4860 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr",
4861                         0, 0, CNTR_NORMAL,
4862                         access_tx_launch_fifo0_unc_or_parity_err_cnt),
4863 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr",
4864                         0, 0, CNTR_NORMAL,
4865                         access_tx_sdma15_disallowed_packet_err_cnt),
4866 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr",
4867                         0, 0, CNTR_NORMAL,
4868                         access_tx_sdma14_disallowed_packet_err_cnt),
4869 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr",
4870                         0, 0, CNTR_NORMAL,
4871                         access_tx_sdma13_disallowed_packet_err_cnt),
4872 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr",
4873                         0, 0, CNTR_NORMAL,
4874                         access_tx_sdma12_disallowed_packet_err_cnt),
4875 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr",
4876                         0, 0, CNTR_NORMAL,
4877                         access_tx_sdma11_disallowed_packet_err_cnt),
4878 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr",
4879                         0, 0, CNTR_NORMAL,
4880                         access_tx_sdma10_disallowed_packet_err_cnt),
4881 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr",
4882                         0, 0, CNTR_NORMAL,
4883                         access_tx_sdma9_disallowed_packet_err_cnt),
4884 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr",
4885                         0, 0, CNTR_NORMAL,
4886                         access_tx_sdma8_disallowed_packet_err_cnt),
4887 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr",
4888                         0, 0, CNTR_NORMAL,
4889                         access_tx_sdma7_disallowed_packet_err_cnt),
4890 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr",
4891                         0, 0, CNTR_NORMAL,
4892                         access_tx_sdma6_disallowed_packet_err_cnt),
4893 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr",
4894                         0, 0, CNTR_NORMAL,
4895                         access_tx_sdma5_disallowed_packet_err_cnt),
4896 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr",
4897                         0, 0, CNTR_NORMAL,
4898                         access_tx_sdma4_disallowed_packet_err_cnt),
4899 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr",
4900                         0, 0, CNTR_NORMAL,
4901                         access_tx_sdma3_disallowed_packet_err_cnt),
4902 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr",
4903                         0, 0, CNTR_NORMAL,
4904                         access_tx_sdma2_disallowed_packet_err_cnt),
4905 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr",
4906                         0, 0, CNTR_NORMAL,
4907                         access_tx_sdma1_disallowed_packet_err_cnt),
4908 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr",
4909                         0, 0, CNTR_NORMAL,
4910                         access_tx_sdma0_disallowed_packet_err_cnt),
4911 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0,
4912                         CNTR_NORMAL,
4913                         access_tx_config_parity_err_cnt),
4914 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0,
4915                         CNTR_NORMAL,
4916                         access_tx_sbrd_ctl_csr_parity_err_cnt),
4917 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0,
4918                         CNTR_NORMAL,
4919                         access_tx_launch_csr_parity_err_cnt),
4920 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0,
4921                         CNTR_NORMAL,
4922                         access_tx_illegal_vl_err_cnt),
4923 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM(
4924                         "TxSbrdCtlStateMachineParityErr", 0, 0,
4925                         CNTR_NORMAL,
4926                         access_tx_sbrd_ctl_state_machine_parity_err_cnt),
4927 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0,
4928                         CNTR_NORMAL,
4929                         access_egress_reserved_10_err_cnt),
4930 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0,
4931                         CNTR_NORMAL,
4932                         access_egress_reserved_9_err_cnt),
4933 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr",
4934                         0, 0, CNTR_NORMAL,
4935                         access_tx_sdma_launch_intf_parity_err_cnt),
4936 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0,
4937                         CNTR_NORMAL,
4938                         access_tx_pio_launch_intf_parity_err_cnt),
4939 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0,
4940                         CNTR_NORMAL,
4941                         access_egress_reserved_6_err_cnt),
4942 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0,
4943                         CNTR_NORMAL,
4944                         access_tx_incorrect_link_state_err_cnt),
4945 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0,
4946                         CNTR_NORMAL,
4947                         access_tx_linkdown_err_cnt),
4948 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM(
4949                         "EgressFifoUnderrunOrParityErr", 0, 0,
4950                         CNTR_NORMAL,
4951                         access_tx_egress_fifi_underrun_or_parity_err_cnt),
4952 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0,
4953                         CNTR_NORMAL,
4954                         access_egress_reserved_2_err_cnt),
4955 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0,
4956                         CNTR_NORMAL,
4957                         access_tx_pkt_integrity_mem_unc_err_cnt),
4958 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0,
4959                         CNTR_NORMAL,
4960                         access_tx_pkt_integrity_mem_cor_err_cnt),
4961 /* SendErrStatus */
4962 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0,
4963                         CNTR_NORMAL,
4964                         access_send_csr_write_bad_addr_err_cnt),
4965 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0,
4966                         CNTR_NORMAL,
4967                         access_send_csr_read_bad_addr_err_cnt),
4968 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0,
4969                         CNTR_NORMAL,
4970                         access_send_csr_parity_cnt),
4971 /* SendCtxtErrStatus */
4972 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0,
4973                         CNTR_NORMAL,
4974                         access_pio_write_out_of_bounds_err_cnt),
4975 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0,
4976                         CNTR_NORMAL,
4977                         access_pio_write_overflow_err_cnt),
4978 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr",
4979                         0, 0, CNTR_NORMAL,
4980                         access_pio_write_crosses_boundary_err_cnt),
4981 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0,
4982                         CNTR_NORMAL,
4983                         access_pio_disallowed_packet_err_cnt),
4984 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0,
4985                         CNTR_NORMAL,
4986                         access_pio_inconsistent_sop_err_cnt),
4987 /* SendDmaEngErrStatus */
4988 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr",
4989                         0, 0, CNTR_NORMAL,
4990                         access_sdma_header_request_fifo_cor_err_cnt),
4991 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0,
4992                         CNTR_NORMAL,
4993                         access_sdma_header_storage_cor_err_cnt),
4994 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0,
4995                         CNTR_NORMAL,
4996                         access_sdma_packet_tracking_cor_err_cnt),
4997 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0,
4998                         CNTR_NORMAL,
4999                         access_sdma_assembly_cor_err_cnt),
5000 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0,
5001                         CNTR_NORMAL,
5002                         access_sdma_desc_table_cor_err_cnt),
5003 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr",
5004                         0, 0, CNTR_NORMAL,
5005                         access_sdma_header_request_fifo_unc_err_cnt),
5006 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0,
5007                         CNTR_NORMAL,
5008                         access_sdma_header_storage_unc_err_cnt),
5009 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0,
5010                         CNTR_NORMAL,
5011                         access_sdma_packet_tracking_unc_err_cnt),
5012 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0,
5013                         CNTR_NORMAL,
5014                         access_sdma_assembly_unc_err_cnt),
5015 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0,
5016                         CNTR_NORMAL,
5017                         access_sdma_desc_table_unc_err_cnt),
5018 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0,
5019                         CNTR_NORMAL,
5020                         access_sdma_timeout_err_cnt),
5021 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0,
5022                         CNTR_NORMAL,
5023                         access_sdma_header_length_err_cnt),
5024 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0,
5025                         CNTR_NORMAL,
5026                         access_sdma_header_address_err_cnt),
5027 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0,
5028                         CNTR_NORMAL,
5029                         access_sdma_header_select_err_cnt),
5030 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0,
5031                         CNTR_NORMAL,
5032                         access_sdma_reserved_9_err_cnt),
5033 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0,
5034                         CNTR_NORMAL,
5035                         access_sdma_packet_desc_overflow_err_cnt),
5036 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0,
5037                         CNTR_NORMAL,
5038                         access_sdma_length_mismatch_err_cnt),
5039 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0,
5040                         CNTR_NORMAL,
5041                         access_sdma_halt_err_cnt),
5042 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0,
5043                         CNTR_NORMAL,
5044                         access_sdma_mem_read_err_cnt),
5045 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0,
5046                         CNTR_NORMAL,
5047                         access_sdma_first_desc_err_cnt),
5048 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0,
5049                         CNTR_NORMAL,
5050                         access_sdma_tail_out_of_bounds_err_cnt),
5051 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0,
5052                         CNTR_NORMAL,
5053                         access_sdma_too_long_err_cnt),
5054 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0,
5055                         CNTR_NORMAL,
5056                         access_sdma_gen_mismatch_err_cnt),
5057 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0,
5058                         CNTR_NORMAL,
5059                         access_sdma_wrong_dw_err_cnt),
5060 };
5061
5062 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = {
5063 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT,
5064                         CNTR_NORMAL),
5065 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT,
5066                         CNTR_NORMAL),
5067 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT,
5068                         CNTR_NORMAL),
5069 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT,
5070                         CNTR_NORMAL),
5071 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT,
5072                         CNTR_NORMAL),
5073 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT,
5074                         CNTR_NORMAL),
5075 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT,
5076                         CNTR_NORMAL),
5077 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL),
5078 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL),
5079 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH),
5080 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT,
5081                                       CNTR_SYNTH | CNTR_VL),
5082 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT,
5083                                      CNTR_SYNTH | CNTR_VL),
5084 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT,
5085                                       CNTR_SYNTH | CNTR_VL),
5086 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL),
5087 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL),
5088 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5089                              access_sw_link_dn_cnt),
5090 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5091                            access_sw_link_up_cnt),
5092 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL,
5093                                  access_sw_unknown_frame_cnt),
5094 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT,
5095                              access_sw_xmit_discards),
5096 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0,
5097                                 CNTR_SYNTH | CNTR_32BIT | CNTR_VL,
5098                                 access_sw_xmit_discards),
5099 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH,
5100                                  access_xmit_constraint_errs),
5101 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH,
5102                                 access_rcv_constraint_errs),
5103 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts),
5104 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends),
5105 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks),
5106 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks),
5107 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts),
5108 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops),
5109 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait),
5110 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak),
5111 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq),
5112 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq),
5113 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned),
5114 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks),
5115 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL,
5116                                access_sw_cpu_rc_acks),
5117 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL,
5118                                 access_sw_cpu_rc_qacks),
5119 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL,
5120                                        access_sw_cpu_rc_delayed_comp),
5121 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1),
5122 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3),
5123 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5),
5124 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7),
5125 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9),
5126 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11),
5127 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13),
5128 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15),
5129 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17),
5130 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19),
5131 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21),
5132 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23),
5133 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25),
5134 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27),
5135 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29),
5136 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31),
5137 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33),
5138 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35),
5139 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37),
5140 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39),
5141 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41),
5142 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43),
5143 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45),
5144 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47),
5145 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49),
5146 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51),
5147 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53),
5148 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55),
5149 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57),
5150 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59),
5151 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61),
5152 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63),
5153 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65),
5154 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67),
5155 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69),
5156 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71),
5157 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73),
5158 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75),
5159 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77),
5160 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79),
5161 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81),
5162 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83),
5163 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85),
5164 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87),
5165 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89),
5166 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91),
5167 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93),
5168 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95),
5169 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97),
5170 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99),
5171 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101),
5172 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103),
5173 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105),
5174 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107),
5175 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109),
5176 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111),
5177 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113),
5178 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115),
5179 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117),
5180 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119),
5181 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121),
5182 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123),
5183 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125),
5184 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127),
5185 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129),
5186 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131),
5187 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133),
5188 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135),
5189 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137),
5190 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139),
5191 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141),
5192 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143),
5193 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145),
5194 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147),
5195 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149),
5196 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151),
5197 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153),
5198 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155),
5199 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157),
5200 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159),
5201 };
5202
5203 /* ======================================================================== */
5204
5205 /* return true if this is chip revision revision a */
5206 int is_ax(struct hfi1_devdata *dd)
5207 {
5208         u8 chip_rev_minor =
5209                 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5210                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
5211         return (chip_rev_minor & 0xf0) == 0;
5212 }
5213
5214 /* return true if this is chip revision revision b */
5215 int is_bx(struct hfi1_devdata *dd)
5216 {
5217         u8 chip_rev_minor =
5218                 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT
5219                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
5220         return (chip_rev_minor & 0xF0) == 0x10;
5221 }
5222
5223 /*
5224  * Append string s to buffer buf.  Arguments curp and len are the current
5225  * position and remaining length, respectively.
5226  *
5227  * return 0 on success, 1 on out of room
5228  */
5229 static int append_str(char *buf, char **curp, int *lenp, const char *s)
5230 {
5231         char *p = *curp;
5232         int len = *lenp;
5233         int result = 0; /* success */
5234         char c;
5235
5236         /* add a comma, if first in the buffer */
5237         if (p != buf) {
5238                 if (len == 0) {
5239                         result = 1; /* out of room */
5240                         goto done;
5241                 }
5242                 *p++ = ',';
5243                 len--;
5244         }
5245
5246         /* copy the string */
5247         while ((c = *s++) != 0) {
5248                 if (len == 0) {
5249                         result = 1; /* out of room */
5250                         goto done;
5251                 }
5252                 *p++ = c;
5253                 len--;
5254         }
5255
5256 done:
5257         /* write return values */
5258         *curp = p;
5259         *lenp = len;
5260
5261         return result;
5262 }
5263
5264 /*
5265  * Using the given flag table, print a comma separated string into
5266  * the buffer.  End in '*' if the buffer is too short.
5267  */
5268 static char *flag_string(char *buf, int buf_len, u64 flags,
5269                          struct flag_table *table, int table_size)
5270 {
5271         char extra[32];
5272         char *p = buf;
5273         int len = buf_len;
5274         int no_room = 0;
5275         int i;
5276
5277         /* make sure there is at least 2 so we can form "*" */
5278         if (len < 2)
5279                 return "";
5280
5281         len--;  /* leave room for a nul */
5282         for (i = 0; i < table_size; i++) {
5283                 if (flags & table[i].flag) {
5284                         no_room = append_str(buf, &p, &len, table[i].str);
5285                         if (no_room)
5286                                 break;
5287                         flags &= ~table[i].flag;
5288                 }
5289         }
5290
5291         /* any undocumented bits left? */
5292         if (!no_room && flags) {
5293                 snprintf(extra, sizeof(extra), "bits 0x%llx", flags);
5294                 no_room = append_str(buf, &p, &len, extra);
5295         }
5296
5297         /* add * if ran out of room */
5298         if (no_room) {
5299                 /* may need to back up to add space for a '*' */
5300                 if (len == 0)
5301                         --p;
5302                 *p++ = '*';
5303         }
5304
5305         /* add final nul - space already allocated above */
5306         *p = 0;
5307         return buf;
5308 }
5309
5310 /* first 8 CCE error interrupt source names */
5311 static const char * const cce_misc_names[] = {
5312         "CceErrInt",            /* 0 */
5313         "RxeErrInt",            /* 1 */
5314         "MiscErrInt",           /* 2 */
5315         "Reserved3",            /* 3 */
5316         "PioErrInt",            /* 4 */
5317         "SDmaErrInt",           /* 5 */
5318         "EgressErrInt",         /* 6 */
5319         "TxeErrInt"             /* 7 */
5320 };
5321
5322 /*
5323  * Return the miscellaneous error interrupt name.
5324  */
5325 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source)
5326 {
5327         if (source < ARRAY_SIZE(cce_misc_names))
5328                 strncpy(buf, cce_misc_names[source], bsize);
5329         else
5330                 snprintf(buf, bsize, "Reserved%u",
5331                          source + IS_GENERAL_ERR_START);
5332
5333         return buf;
5334 }
5335
5336 /*
5337  * Return the SDMA engine error interrupt name.
5338  */
5339 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source)
5340 {
5341         snprintf(buf, bsize, "SDmaEngErrInt%u", source);
5342         return buf;
5343 }
5344
5345 /*
5346  * Return the send context error interrupt name.
5347  */
5348 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source)
5349 {
5350         snprintf(buf, bsize, "SendCtxtErrInt%u", source);
5351         return buf;
5352 }
5353
5354 static const char * const various_names[] = {
5355         "PbcInt",
5356         "GpioAssertInt",
5357         "Qsfp1Int",
5358         "Qsfp2Int",
5359         "TCritInt"
5360 };
5361
5362 /*
5363  * Return the various interrupt name.
5364  */
5365 static char *is_various_name(char *buf, size_t bsize, unsigned int source)
5366 {
5367         if (source < ARRAY_SIZE(various_names))
5368                 strncpy(buf, various_names[source], bsize);
5369         else
5370                 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START);
5371         return buf;
5372 }
5373
5374 /*
5375  * Return the DC interrupt name.
5376  */
5377 static char *is_dc_name(char *buf, size_t bsize, unsigned int source)
5378 {
5379         static const char * const dc_int_names[] = {
5380                 "common",
5381                 "lcb",
5382                 "8051",
5383                 "lbm"   /* local block merge */
5384         };
5385
5386         if (source < ARRAY_SIZE(dc_int_names))
5387                 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]);
5388         else
5389                 snprintf(buf, bsize, "DCInt%u", source);
5390         return buf;
5391 }
5392
5393 static const char * const sdma_int_names[] = {
5394         "SDmaInt",
5395         "SdmaIdleInt",
5396         "SdmaProgressInt",
5397 };
5398
5399 /*
5400  * Return the SDMA engine interrupt name.
5401  */
5402 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source)
5403 {
5404         /* what interrupt */
5405         unsigned int what  = source / TXE_NUM_SDMA_ENGINES;
5406         /* which engine */
5407         unsigned int which = source % TXE_NUM_SDMA_ENGINES;
5408
5409         if (likely(what < 3))
5410                 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which);
5411         else
5412                 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source);
5413         return buf;
5414 }
5415
5416 /*
5417  * Return the receive available interrupt name.
5418  */
5419 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source)
5420 {
5421         snprintf(buf, bsize, "RcvAvailInt%u", source);
5422         return buf;
5423 }
5424
5425 /*
5426  * Return the receive urgent interrupt name.
5427  */
5428 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source)
5429 {
5430         snprintf(buf, bsize, "RcvUrgentInt%u", source);
5431         return buf;
5432 }
5433
5434 /*
5435  * Return the send credit interrupt name.
5436  */
5437 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source)
5438 {
5439         snprintf(buf, bsize, "SendCreditInt%u", source);
5440         return buf;
5441 }
5442
5443 /*
5444  * Return the reserved interrupt name.
5445  */
5446 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source)
5447 {
5448         snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START);
5449         return buf;
5450 }
5451
5452 static char *cce_err_status_string(char *buf, int buf_len, u64 flags)
5453 {
5454         return flag_string(buf, buf_len, flags,
5455                            cce_err_status_flags,
5456                            ARRAY_SIZE(cce_err_status_flags));
5457 }
5458
5459 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags)
5460 {
5461         return flag_string(buf, buf_len, flags,
5462                            rxe_err_status_flags,
5463                            ARRAY_SIZE(rxe_err_status_flags));
5464 }
5465
5466 static char *misc_err_status_string(char *buf, int buf_len, u64 flags)
5467 {
5468         return flag_string(buf, buf_len, flags, misc_err_status_flags,
5469                            ARRAY_SIZE(misc_err_status_flags));
5470 }
5471
5472 static char *pio_err_status_string(char *buf, int buf_len, u64 flags)
5473 {
5474         return flag_string(buf, buf_len, flags,
5475                            pio_err_status_flags,
5476                            ARRAY_SIZE(pio_err_status_flags));
5477 }
5478
5479 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags)
5480 {
5481         return flag_string(buf, buf_len, flags,
5482                            sdma_err_status_flags,
5483                            ARRAY_SIZE(sdma_err_status_flags));
5484 }
5485
5486 static char *egress_err_status_string(char *buf, int buf_len, u64 flags)
5487 {
5488         return flag_string(buf, buf_len, flags,
5489                            egress_err_status_flags,
5490                            ARRAY_SIZE(egress_err_status_flags));
5491 }
5492
5493 static char *egress_err_info_string(char *buf, int buf_len, u64 flags)
5494 {
5495         return flag_string(buf, buf_len, flags,
5496                            egress_err_info_flags,
5497                            ARRAY_SIZE(egress_err_info_flags));
5498 }
5499
5500 static char *send_err_status_string(char *buf, int buf_len, u64 flags)
5501 {
5502         return flag_string(buf, buf_len, flags,
5503                            send_err_status_flags,
5504                            ARRAY_SIZE(send_err_status_flags));
5505 }
5506
5507 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5508 {
5509         char buf[96];
5510         int i = 0;
5511
5512         /*
5513          * For most these errors, there is nothing that can be done except
5514          * report or record it.
5515          */
5516         dd_dev_info(dd, "CCE Error: %s\n",
5517                     cce_err_status_string(buf, sizeof(buf), reg));
5518
5519         if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) &&
5520             is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) {
5521                 /* this error requires a manual drop into SPC freeze mode */
5522                 /* then a fix up */
5523                 start_freeze_handling(dd->pport, FREEZE_SELF);
5524         }
5525
5526         for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) {
5527                 if (reg & (1ull << i)) {
5528                         incr_cntr64(&dd->cce_err_status_cnt[i]);
5529                         /* maintain a counter over all cce_err_status errors */
5530                         incr_cntr64(&dd->sw_cce_err_status_aggregate);
5531                 }
5532         }
5533 }
5534
5535 /*
5536  * Check counters for receive errors that do not have an interrupt
5537  * associated with them.
5538  */
5539 #define RCVERR_CHECK_TIME 10
5540 static void update_rcverr_timer(struct timer_list *t)
5541 {
5542         struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer);
5543         struct hfi1_pportdata *ppd = dd->pport;
5544         u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL);
5545
5546         if (dd->rcv_ovfl_cnt < cur_ovfl_cnt &&
5547             ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) {
5548                 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__);
5549                 set_link_down_reason(
5550                 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0,
5551                 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN);
5552                 queue_work(ppd->link_wq, &ppd->link_bounce_work);
5553         }
5554         dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt;
5555
5556         mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5557 }
5558
5559 static int init_rcverr(struct hfi1_devdata *dd)
5560 {
5561         timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0);
5562         /* Assume the hardware counter has been reset */
5563         dd->rcv_ovfl_cnt = 0;
5564         return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME);
5565 }
5566
5567 static void free_rcverr(struct hfi1_devdata *dd)
5568 {
5569         if (dd->rcverr_timer.function)
5570                 del_timer_sync(&dd->rcverr_timer);
5571 }
5572
5573 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5574 {
5575         char buf[96];
5576         int i = 0;
5577
5578         dd_dev_info(dd, "Receive Error: %s\n",
5579                     rxe_err_status_string(buf, sizeof(buf), reg));
5580
5581         if (reg & ALL_RXE_FREEZE_ERR) {
5582                 int flags = 0;
5583
5584                 /*
5585                  * Freeze mode recovery is disabled for the errors
5586                  * in RXE_FREEZE_ABORT_MASK
5587                  */
5588                 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK))
5589                         flags = FREEZE_ABORT;
5590
5591                 start_freeze_handling(dd->pport, flags);
5592         }
5593
5594         for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) {
5595                 if (reg & (1ull << i))
5596                         incr_cntr64(&dd->rcv_err_status_cnt[i]);
5597         }
5598 }
5599
5600 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5601 {
5602         char buf[96];
5603         int i = 0;
5604
5605         dd_dev_info(dd, "Misc Error: %s",
5606                     misc_err_status_string(buf, sizeof(buf), reg));
5607         for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) {
5608                 if (reg & (1ull << i))
5609                         incr_cntr64(&dd->misc_err_status_cnt[i]);
5610         }
5611 }
5612
5613 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5614 {
5615         char buf[96];
5616         int i = 0;
5617
5618         dd_dev_info(dd, "PIO Error: %s\n",
5619                     pio_err_status_string(buf, sizeof(buf), reg));
5620
5621         if (reg & ALL_PIO_FREEZE_ERR)
5622                 start_freeze_handling(dd->pport, 0);
5623
5624         for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) {
5625                 if (reg & (1ull << i))
5626                         incr_cntr64(&dd->send_pio_err_status_cnt[i]);
5627         }
5628 }
5629
5630 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5631 {
5632         char buf[96];
5633         int i = 0;
5634
5635         dd_dev_info(dd, "SDMA Error: %s\n",
5636                     sdma_err_status_string(buf, sizeof(buf), reg));
5637
5638         if (reg & ALL_SDMA_FREEZE_ERR)
5639                 start_freeze_handling(dd->pport, 0);
5640
5641         for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) {
5642                 if (reg & (1ull << i))
5643                         incr_cntr64(&dd->send_dma_err_status_cnt[i]);
5644         }
5645 }
5646
5647 static inline void __count_port_discards(struct hfi1_pportdata *ppd)
5648 {
5649         incr_cntr64(&ppd->port_xmit_discards);
5650 }
5651
5652 static void count_port_inactive(struct hfi1_devdata *dd)
5653 {
5654         __count_port_discards(dd->pport);
5655 }
5656
5657 /*
5658  * We have had a "disallowed packet" error during egress. Determine the
5659  * integrity check which failed, and update relevant error counter, etc.
5660  *
5661  * Note that the SEND_EGRESS_ERR_INFO register has only a single
5662  * bit of state per integrity check, and so we can miss the reason for an
5663  * egress error if more than one packet fails the same integrity check
5664  * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO.
5665  */
5666 static void handle_send_egress_err_info(struct hfi1_devdata *dd,
5667                                         int vl)
5668 {
5669         struct hfi1_pportdata *ppd = dd->pport;
5670         u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */
5671         u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO);
5672         char buf[96];
5673
5674         /* clear down all observed info as quickly as possible after read */
5675         write_csr(dd, SEND_EGRESS_ERR_INFO, info);
5676
5677         dd_dev_info(dd,
5678                     "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n",
5679                     info, egress_err_info_string(buf, sizeof(buf), info), src);
5680
5681         /* Eventually add other counters for each bit */
5682         if (info & PORT_DISCARD_EGRESS_ERRS) {
5683                 int weight, i;
5684
5685                 /*
5686                  * Count all applicable bits as individual errors and
5687                  * attribute them to the packet that triggered this handler.
5688                  * This may not be completely accurate due to limitations
5689                  * on the available hardware error information.  There is
5690                  * a single information register and any number of error
5691                  * packets may have occurred and contributed to it before
5692                  * this routine is called.  This means that:
5693                  * a) If multiple packets with the same error occur before
5694                  *    this routine is called, earlier packets are missed.
5695                  *    There is only a single bit for each error type.
5696                  * b) Errors may not be attributed to the correct VL.
5697                  *    The driver is attributing all bits in the info register
5698                  *    to the packet that triggered this call, but bits
5699                  *    could be an accumulation of different packets with
5700                  *    different VLs.
5701                  * c) A single error packet may have multiple counts attached
5702                  *    to it.  There is no way for the driver to know if
5703                  *    multiple bits set in the info register are due to a
5704                  *    single packet or multiple packets.  The driver assumes
5705                  *    multiple packets.
5706                  */
5707                 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS);
5708                 for (i = 0; i < weight; i++) {
5709                         __count_port_discards(ppd);
5710                         if (vl >= 0 && vl < TXE_NUM_DATA_VL)
5711                                 incr_cntr64(&ppd->port_xmit_discards_vl[vl]);
5712                         else if (vl == 15)
5713                                 incr_cntr64(&ppd->port_xmit_discards_vl
5714                                             [C_VL_15]);
5715                 }
5716         }
5717 }
5718
5719 /*
5720  * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5721  * register. Does it represent a 'port inactive' error?
5722  */
5723 static inline int port_inactive_err(u64 posn)
5724 {
5725         return (posn >= SEES(TX_LINKDOWN) &&
5726                 posn <= SEES(TX_INCORRECT_LINK_STATE));
5727 }
5728
5729 /*
5730  * Input value is a bit position within the SEND_EGRESS_ERR_STATUS
5731  * register. Does it represent a 'disallowed packet' error?
5732  */
5733 static inline int disallowed_pkt_err(int posn)
5734 {
5735         return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) &&
5736                 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET));
5737 }
5738
5739 /*
5740  * Input value is a bit position of one of the SDMA engine disallowed
5741  * packet errors.  Return which engine.  Use of this must be guarded by
5742  * disallowed_pkt_err().
5743  */
5744 static inline int disallowed_pkt_engine(int posn)
5745 {
5746         return posn - SEES(TX_SDMA0_DISALLOWED_PACKET);
5747 }
5748
5749 /*
5750  * Translate an SDMA engine to a VL.  Return -1 if the tranlation cannot
5751  * be done.
5752  */
5753 static int engine_to_vl(struct hfi1_devdata *dd, int engine)
5754 {
5755         struct sdma_vl_map *m;
5756         int vl;
5757
5758         /* range check */
5759         if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES)
5760                 return -1;
5761
5762         rcu_read_lock();
5763         m = rcu_dereference(dd->sdma_map);
5764         vl = m->engine_to_vl[engine];
5765         rcu_read_unlock();
5766
5767         return vl;
5768 }
5769
5770 /*
5771  * Translate the send context (sofware index) into a VL.  Return -1 if the
5772  * translation cannot be done.
5773  */
5774 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index)
5775 {
5776         struct send_context_info *sci;
5777         struct send_context *sc;
5778         int i;
5779
5780         sci = &dd->send_contexts[sw_index];
5781
5782         /* there is no information for user (PSM) and ack contexts */
5783         if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15))
5784                 return -1;
5785
5786         sc = sci->sc;
5787         if (!sc)
5788                 return -1;
5789         if (dd->vld[15].sc == sc)
5790                 return 15;
5791         for (i = 0; i < num_vls; i++)
5792                 if (dd->vld[i].sc == sc)
5793                         return i;
5794
5795         return -1;
5796 }
5797
5798 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5799 {
5800         u64 reg_copy = reg, handled = 0;
5801         char buf[96];
5802         int i = 0;
5803
5804         if (reg & ALL_TXE_EGRESS_FREEZE_ERR)
5805                 start_freeze_handling(dd->pport, 0);
5806         else if (is_ax(dd) &&
5807                  (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) &&
5808                  (dd->icode != ICODE_FUNCTIONAL_SIMULATOR))
5809                 start_freeze_handling(dd->pport, 0);
5810
5811         while (reg_copy) {
5812                 int posn = fls64(reg_copy);
5813                 /* fls64() returns a 1-based offset, we want it zero based */
5814                 int shift = posn - 1;
5815                 u64 mask = 1ULL << shift;
5816
5817                 if (port_inactive_err(shift)) {
5818                         count_port_inactive(dd);
5819                         handled |= mask;
5820                 } else if (disallowed_pkt_err(shift)) {
5821                         int vl = engine_to_vl(dd, disallowed_pkt_engine(shift));
5822
5823                         handle_send_egress_err_info(dd, vl);
5824                         handled |= mask;
5825                 }
5826                 reg_copy &= ~mask;
5827         }
5828
5829         reg &= ~handled;
5830
5831         if (reg)
5832                 dd_dev_info(dd, "Egress Error: %s\n",
5833                             egress_err_status_string(buf, sizeof(buf), reg));
5834
5835         for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) {
5836                 if (reg & (1ull << i))
5837                         incr_cntr64(&dd->send_egress_err_status_cnt[i]);
5838         }
5839 }
5840
5841 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
5842 {
5843         char buf[96];
5844         int i = 0;
5845
5846         dd_dev_info(dd, "Send Error: %s\n",
5847                     send_err_status_string(buf, sizeof(buf), reg));
5848
5849         for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) {
5850                 if (reg & (1ull << i))
5851                         incr_cntr64(&dd->send_err_status_cnt[i]);
5852         }
5853 }
5854
5855 /*
5856  * The maximum number of times the error clear down will loop before
5857  * blocking a repeating error.  This value is arbitrary.
5858  */
5859 #define MAX_CLEAR_COUNT 20
5860
5861 /*
5862  * Clear and handle an error register.  All error interrupts are funneled
5863  * through here to have a central location to correctly handle single-
5864  * or multi-shot errors.
5865  *
5866  * For non per-context registers, call this routine with a context value
5867  * of 0 so the per-context offset is zero.
5868  *
5869  * If the handler loops too many times, assume that something is wrong
5870  * and can't be fixed, so mask the error bits.
5871  */
5872 static void interrupt_clear_down(struct hfi1_devdata *dd,
5873                                  u32 context,
5874                                  const struct err_reg_info *eri)
5875 {
5876         u64 reg;
5877         u32 count;
5878
5879         /* read in a loop until no more errors are seen */
5880         count = 0;
5881         while (1) {
5882                 reg = read_kctxt_csr(dd, context, eri->status);
5883                 if (reg == 0)
5884                         break;
5885                 write_kctxt_csr(dd, context, eri->clear, reg);
5886                 if (likely(eri->handler))
5887                         eri->handler(dd, context, reg);
5888                 count++;
5889                 if (count > MAX_CLEAR_COUNT) {
5890                         u64 mask;
5891
5892                         dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n",
5893                                    eri->desc, reg);
5894                         /*
5895                          * Read-modify-write so any other masked bits
5896                          * remain masked.
5897                          */
5898                         mask = read_kctxt_csr(dd, context, eri->mask);
5899                         mask &= ~reg;
5900                         write_kctxt_csr(dd, context, eri->mask, mask);
5901                         break;
5902                 }
5903         }
5904 }
5905
5906 /*
5907  * CCE block "misc" interrupt.  Source is < 16.
5908  */
5909 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source)
5910 {
5911         const struct err_reg_info *eri = &misc_errs[source];
5912
5913         if (eri->handler) {
5914                 interrupt_clear_down(dd, 0, eri);
5915         } else {
5916                 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n",
5917                            source);
5918         }
5919 }
5920
5921 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags)
5922 {
5923         return flag_string(buf, buf_len, flags,
5924                            sc_err_status_flags,
5925                            ARRAY_SIZE(sc_err_status_flags));
5926 }
5927
5928 /*
5929  * Send context error interrupt.  Source (hw_context) is < 160.
5930  *
5931  * All send context errors cause the send context to halt.  The normal
5932  * clear-down mechanism cannot be used because we cannot clear the
5933  * error bits until several other long-running items are done first.
5934  * This is OK because with the context halted, nothing else is going
5935  * to happen on it anyway.
5936  */
5937 static void is_sendctxt_err_int(struct hfi1_devdata *dd,
5938                                 unsigned int hw_context)
5939 {
5940         struct send_context_info *sci;
5941         struct send_context *sc;
5942         char flags[96];
5943         u64 status;
5944         u32 sw_index;
5945         int i = 0;
5946         unsigned long irq_flags;
5947
5948         sw_index = dd->hw_to_sw[hw_context];
5949         if (sw_index >= dd->num_send_contexts) {
5950                 dd_dev_err(dd,
5951                            "out of range sw index %u for send context %u\n",
5952                            sw_index, hw_context);
5953                 return;
5954         }
5955         sci = &dd->send_contexts[sw_index];
5956         spin_lock_irqsave(&dd->sc_lock, irq_flags);
5957         sc = sci->sc;
5958         if (!sc) {
5959                 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__,
5960                            sw_index, hw_context);
5961                 spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5962                 return;
5963         }
5964
5965         /* tell the software that a halt has begun */
5966         sc_stop(sc, SCF_HALTED);
5967
5968         status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS);
5969
5970         dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context,
5971                     send_context_err_status_string(flags, sizeof(flags),
5972                                                    status));
5973
5974         if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK)
5975                 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index));
5976
5977         /*
5978          * Automatically restart halted kernel contexts out of interrupt
5979          * context.  User contexts must ask the driver to restart the context.
5980          */
5981         if (sc->type != SC_USER)
5982                 queue_work(dd->pport->hfi1_wq, &sc->halt_work);
5983         spin_unlock_irqrestore(&dd->sc_lock, irq_flags);
5984
5985         /*
5986          * Update the counters for the corresponding status bits.
5987          * Note that these particular counters are aggregated over all
5988          * 160 contexts.
5989          */
5990         for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) {
5991                 if (status & (1ull << i))
5992                         incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]);
5993         }
5994 }
5995
5996 static void handle_sdma_eng_err(struct hfi1_devdata *dd,
5997                                 unsigned int source, u64 status)
5998 {
5999         struct sdma_engine *sde;
6000         int i = 0;
6001
6002         sde = &dd->per_sdma[source];
6003 #ifdef CONFIG_SDMA_VERBOSITY
6004         dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6005                    slashstrip(__FILE__), __LINE__, __func__);
6006         dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n",
6007                    sde->this_idx, source, (unsigned long long)status);
6008 #endif
6009         sde->err_cnt++;
6010         sdma_engine_error(sde, status);
6011
6012         /*
6013         * Update the counters for the corresponding status bits.
6014         * Note that these particular counters are aggregated over
6015         * all 16 DMA engines.
6016         */
6017         for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) {
6018                 if (status & (1ull << i))
6019                         incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]);
6020         }
6021 }
6022
6023 /*
6024  * CCE block SDMA error interrupt.  Source is < 16.
6025  */
6026 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source)
6027 {
6028 #ifdef CONFIG_SDMA_VERBOSITY
6029         struct sdma_engine *sde = &dd->per_sdma[source];
6030
6031         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
6032                    slashstrip(__FILE__), __LINE__, __func__);
6033         dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx,
6034                    source);
6035         sdma_dumpstate(sde);
6036 #endif
6037         interrupt_clear_down(dd, source, &sdma_eng_err);
6038 }
6039
6040 /*
6041  * CCE block "various" interrupt.  Source is < 8.
6042  */
6043 static void is_various_int(struct hfi1_devdata *dd, unsigned int source)
6044 {
6045         const struct err_reg_info *eri = &various_err[source];
6046
6047         /*
6048          * TCritInt cannot go through interrupt_clear_down()
6049          * because it is not a second tier interrupt. The handler
6050          * should be called directly.
6051          */
6052         if (source == TCRIT_INT_SOURCE)
6053                 handle_temp_err(dd);
6054         else if (eri->handler)
6055                 interrupt_clear_down(dd, 0, eri);
6056         else
6057                 dd_dev_info(dd,
6058                             "%s: Unimplemented/reserved interrupt %d\n",
6059                             __func__, source);
6060 }
6061
6062 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg)
6063 {
6064         /* src_ctx is always zero */
6065         struct hfi1_pportdata *ppd = dd->pport;
6066         unsigned long flags;
6067         u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
6068
6069         if (reg & QSFP_HFI0_MODPRST_N) {
6070                 if (!qsfp_mod_present(ppd)) {
6071                         dd_dev_info(dd, "%s: QSFP module removed\n",
6072                                     __func__);
6073
6074                         ppd->driver_link_ready = 0;
6075                         /*
6076                          * Cable removed, reset all our information about the
6077                          * cache and cable capabilities
6078                          */
6079
6080                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6081                         /*
6082                          * We don't set cache_refresh_required here as we expect
6083                          * an interrupt when a cable is inserted
6084                          */
6085                         ppd->qsfp_info.cache_valid = 0;
6086                         ppd->qsfp_info.reset_needed = 0;
6087                         ppd->qsfp_info.limiting_active = 0;
6088                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6089                                                flags);
6090                         /* Invert the ModPresent pin now to detect plug-in */
6091                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6092                                   ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6093
6094                         if ((ppd->offline_disabled_reason >
6095                           HFI1_ODR_MASK(
6096                           OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) ||
6097                           (ppd->offline_disabled_reason ==
6098                           HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)))
6099                                 ppd->offline_disabled_reason =
6100                                 HFI1_ODR_MASK(
6101                                 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED);
6102
6103                         if (ppd->host_link_state == HLS_DN_POLL) {
6104                                 /*
6105                                  * The link is still in POLL. This means
6106                                  * that the normal link down processing
6107                                  * will not happen. We have to do it here
6108                                  * before turning the DC off.
6109                                  */
6110                                 queue_work(ppd->link_wq, &ppd->link_down_work);
6111                         }
6112                 } else {
6113                         dd_dev_info(dd, "%s: QSFP module inserted\n",
6114                                     __func__);
6115
6116                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6117                         ppd->qsfp_info.cache_valid = 0;
6118                         ppd->qsfp_info.cache_refresh_required = 1;
6119                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
6120                                                flags);
6121
6122                         /*
6123                          * Stop inversion of ModPresent pin to detect
6124                          * removal of the cable
6125                          */
6126                         qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N;
6127                         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT :
6128                                   ASIC_QSFP1_INVERT, qsfp_int_mgmt);
6129
6130                         ppd->offline_disabled_reason =
6131                                 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
6132                 }
6133         }
6134
6135         if (reg & QSFP_HFI0_INT_N) {
6136                 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n",
6137                             __func__);
6138                 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
6139                 ppd->qsfp_info.check_interrupt_flags = 1;
6140                 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags);
6141         }
6142
6143         /* Schedule the QSFP work only if there is a cable attached. */
6144         if (qsfp_mod_present(ppd))
6145                 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work);
6146 }
6147
6148 static int request_host_lcb_access(struct hfi1_devdata *dd)
6149 {
6150         int ret;
6151
6152         ret = do_8051_command(dd, HCMD_MISC,
6153                               (u64)HCMD_MISC_REQUEST_LCB_ACCESS <<
6154                               LOAD_DATA_FIELD_ID_SHIFT, NULL);
6155         if (ret != HCMD_SUCCESS) {
6156                 dd_dev_err(dd, "%s: command failed with error %d\n",
6157                            __func__, ret);
6158         }
6159         return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6160 }
6161
6162 static int request_8051_lcb_access(struct hfi1_devdata *dd)
6163 {
6164         int ret;
6165
6166         ret = do_8051_command(dd, HCMD_MISC,
6167                               (u64)HCMD_MISC_GRANT_LCB_ACCESS <<
6168                               LOAD_DATA_FIELD_ID_SHIFT, NULL);
6169         if (ret != HCMD_SUCCESS) {
6170                 dd_dev_err(dd, "%s: command failed with error %d\n",
6171                            __func__, ret);
6172         }
6173         return ret == HCMD_SUCCESS ? 0 : -EBUSY;
6174 }
6175
6176 /*
6177  * Set the LCB selector - allow host access.  The DCC selector always
6178  * points to the host.
6179  */
6180 static inline void set_host_lcb_access(struct hfi1_devdata *dd)
6181 {
6182         write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6183                   DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK |
6184                   DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK);
6185 }
6186
6187 /*
6188  * Clear the LCB selector - allow 8051 access.  The DCC selector always
6189  * points to the host.
6190  */
6191 static inline void set_8051_lcb_access(struct hfi1_devdata *dd)
6192 {
6193         write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL,
6194                   DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK);
6195 }
6196
6197 /*
6198  * Acquire LCB access from the 8051.  If the host already has access,
6199  * just increment a counter.  Otherwise, inform the 8051 that the
6200  * host is taking access.
6201  *
6202  * Returns:
6203  *      0 on success
6204  *      -EBUSY if the 8051 has control and cannot be disturbed
6205  *      -errno if unable to acquire access from the 8051
6206  */
6207 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6208 {
6209         struct hfi1_pportdata *ppd = dd->pport;
6210         int ret = 0;
6211
6212         /*
6213          * Use the host link state lock so the operation of this routine
6214          * { link state check, selector change, count increment } can occur
6215          * as a unit against a link state change.  Otherwise there is a
6216          * race between the state change and the count increment.
6217          */
6218         if (sleep_ok) {
6219                 mutex_lock(&ppd->hls_lock);
6220         } else {
6221                 while (!mutex_trylock(&ppd->hls_lock))
6222                         udelay(1);
6223         }
6224
6225         /* this access is valid only when the link is up */
6226         if (ppd->host_link_state & HLS_DOWN) {
6227                 dd_dev_info(dd, "%s: link state %s not up\n",
6228                             __func__, link_state_name(ppd->host_link_state));
6229                 ret = -EBUSY;
6230                 goto done;
6231         }
6232
6233         if (dd->lcb_access_count == 0) {
6234                 ret = request_host_lcb_access(dd);
6235                 if (ret) {
6236                         dd_dev_err(dd,
6237                                    "%s: unable to acquire LCB access, err %d\n",
6238                                    __func__, ret);
6239                         goto done;
6240                 }
6241                 set_host_lcb_access(dd);
6242         }
6243         dd->lcb_access_count++;
6244 done:
6245         mutex_unlock(&ppd->hls_lock);
6246         return ret;
6247 }
6248
6249 /*
6250  * Release LCB access by decrementing the use count.  If the count is moving
6251  * from 1 to 0, inform 8051 that it has control back.
6252  *
6253  * Returns:
6254  *      0 on success
6255  *      -errno if unable to release access to the 8051
6256  */
6257 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok)
6258 {
6259         int ret = 0;
6260
6261         /*
6262          * Use the host link state lock because the acquire needed it.
6263          * Here, we only need to keep { selector change, count decrement }
6264          * as a unit.
6265          */
6266         if (sleep_ok) {
6267                 mutex_lock(&dd->pport->hls_lock);
6268         } else {
6269                 while (!mutex_trylock(&dd->pport->hls_lock))
6270                         udelay(1);
6271         }
6272
6273         if (dd->lcb_access_count == 0) {
6274                 dd_dev_err(dd, "%s: LCB access count is zero.  Skipping.\n",
6275                            __func__);
6276                 goto done;
6277         }
6278
6279         if (dd->lcb_access_count == 1) {
6280                 set_8051_lcb_access(dd);
6281                 ret = request_8051_lcb_access(dd);
6282                 if (ret) {
6283                         dd_dev_err(dd,
6284                                    "%s: unable to release LCB access, err %d\n",
6285                                    __func__, ret);
6286                         /* restore host access if the grant didn't work */
6287                         set_host_lcb_access(dd);
6288                         goto done;
6289                 }
6290         }
6291         dd->lcb_access_count--;
6292 done:
6293         mutex_unlock(&dd->pport->hls_lock);
6294         return ret;
6295 }
6296
6297 /*
6298  * Initialize LCB access variables and state.  Called during driver load,
6299  * after most of the initialization is finished.
6300  *
6301  * The DC default is LCB access on for the host.  The driver defaults to
6302  * leaving access to the 8051.  Assign access now - this constrains the call
6303  * to this routine to be after all LCB set-up is done.  In particular, after
6304  * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts()
6305  */
6306 static void init_lcb_access(struct hfi1_devdata *dd)
6307 {
6308         dd->lcb_access_count = 0;
6309 }
6310
6311 /*
6312  * Write a response back to a 8051 request.
6313  */
6314 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data)
6315 {
6316         write_csr(dd, DC_DC8051_CFG_EXT_DEV_0,
6317                   DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK |
6318                   (u64)return_code <<
6319                   DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT |
6320                   (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
6321 }
6322
6323 /*
6324  * Handle host requests from the 8051.
6325  */
6326 static void handle_8051_request(struct hfi1_pportdata *ppd)
6327 {
6328         struct hfi1_devdata *dd = ppd->dd;
6329         u64 reg;
6330         u16 data = 0;
6331         u8 type;
6332
6333         reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1);
6334         if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0)
6335                 return; /* no request */
6336
6337         /* zero out COMPLETED so the response is seen */
6338         write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0);
6339
6340         /* extract request details */
6341         type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT)
6342                         & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK;
6343         data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT)
6344                         & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK;
6345
6346         switch (type) {
6347         case HREQ_LOAD_CONFIG:
6348         case HREQ_SAVE_CONFIG:
6349         case HREQ_READ_CONFIG:
6350         case HREQ_SET_TX_EQ_ABS:
6351         case HREQ_SET_TX_EQ_REL:
6352         case HREQ_ENABLE:
6353                 dd_dev_info(dd, "8051 request: request 0x%x not supported\n",
6354                             type);
6355                 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6356                 break;
6357         case HREQ_LCB_RESET:
6358                 /* Put the LCB, RX FPE and TX FPE into reset */
6359                 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET);
6360                 /* Make sure the write completed */
6361                 (void)read_csr(dd, DCC_CFG_RESET);
6362                 /* Hold the reset long enough to take effect */
6363                 udelay(1);
6364                 /* Take the LCB, RX FPE and TX FPE out of reset */
6365                 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6366                 hreq_response(dd, HREQ_SUCCESS, 0);
6367
6368                 break;
6369         case HREQ_CONFIG_DONE:
6370                 hreq_response(dd, HREQ_SUCCESS, 0);
6371                 break;
6372
6373         case HREQ_INTERFACE_TEST:
6374                 hreq_response(dd, HREQ_SUCCESS, data);
6375                 break;
6376         default:
6377                 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type);
6378                 hreq_response(dd, HREQ_NOT_SUPPORTED, 0);
6379                 break;
6380         }
6381 }
6382
6383 /*
6384  * Set up allocation unit vaulue.
6385  */
6386 void set_up_vau(struct hfi1_devdata *dd, u8 vau)
6387 {
6388         u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6389
6390         /* do not modify other values in the register */
6391         reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK;
6392         reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT;
6393         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6394 }
6395
6396 /*
6397  * Set up initial VL15 credits of the remote.  Assumes the rest of
6398  * the CM credit registers are zero from a previous global or credit reset.
6399  * Shared limit for VL15 will always be 0.
6400  */
6401 void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf)
6402 {
6403         u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
6404
6405         /* set initial values for total and shared credit limit */
6406         reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK |
6407                  SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK);
6408
6409         /*
6410          * Set total limit to be equal to VL15 credits.
6411          * Leave shared limit at 0.
6412          */
6413         reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
6414         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
6415
6416         write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf
6417                   << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT);
6418 }
6419
6420 /*
6421  * Zero all credit details from the previous connection and
6422  * reset the CM manager's internal counters.
6423  */
6424 void reset_link_credits(struct hfi1_devdata *dd)
6425 {
6426         int i;
6427
6428         /* remove all previous VL credit limits */
6429         for (i = 0; i < TXE_NUM_DATA_VL; i++)
6430                 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
6431         write_csr(dd, SEND_CM_CREDIT_VL15, 0);
6432         write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0);
6433         /* reset the CM block */
6434         pio_send_control(dd, PSC_CM_RESET);
6435         /* reset cached value */
6436         dd->vl15buf_cached = 0;
6437 }
6438
6439 /* convert a vCU to a CU */
6440 static u32 vcu_to_cu(u8 vcu)
6441 {
6442         return 1 << vcu;
6443 }
6444
6445 /* convert a CU to a vCU */
6446 static u8 cu_to_vcu(u32 cu)
6447 {
6448         return ilog2(cu);
6449 }
6450
6451 /* convert a vAU to an AU */
6452 static u32 vau_to_au(u8 vau)
6453 {
6454         return 8 * (1 << vau);
6455 }
6456
6457 static void set_linkup_defaults(struct hfi1_pportdata *ppd)
6458 {
6459         ppd->sm_trap_qp = 0x0;
6460         ppd->sa_qp = 0x1;
6461 }
6462
6463 /*
6464  * Graceful LCB shutdown.  This leaves the LCB FIFOs in reset.
6465  */
6466 static void lcb_shutdown(struct hfi1_devdata *dd, int abort)
6467 {
6468         u64 reg;
6469
6470         /* clear lcb run: LCB_CFG_RUN.EN = 0 */
6471         write_csr(dd, DC_LCB_CFG_RUN, 0);
6472         /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */
6473         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET,
6474                   1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT);
6475         /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */
6476         dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN);
6477         reg = read_csr(dd, DCC_CFG_RESET);
6478         write_csr(dd, DCC_CFG_RESET, reg |
6479                   DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE);
6480         (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */
6481         if (!abort) {
6482                 udelay(1);    /* must hold for the longer of 16cclks or 20ns */
6483                 write_csr(dd, DCC_CFG_RESET, reg);
6484                 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6485         }
6486 }
6487
6488 /*
6489  * This routine should be called after the link has been transitioned to
6490  * OFFLINE (OFFLINE state has the side effect of putting the SerDes into
6491  * reset).
6492  *
6493  * The expectation is that the caller of this routine would have taken
6494  * care of properly transitioning the link into the correct state.
6495  * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6496  *       before calling this function.
6497  */
6498 static void _dc_shutdown(struct hfi1_devdata *dd)
6499 {
6500         lockdep_assert_held(&dd->dc8051_lock);
6501
6502         if (dd->dc_shutdown)
6503                 return;
6504
6505         dd->dc_shutdown = 1;
6506         /* Shutdown the LCB */
6507         lcb_shutdown(dd, 1);
6508         /*
6509          * Going to OFFLINE would have causes the 8051 to put the
6510          * SerDes into reset already. Just need to shut down the 8051,
6511          * itself.
6512          */
6513         write_csr(dd, DC_DC8051_CFG_RST, 0x1);
6514 }
6515
6516 static void dc_shutdown(struct hfi1_devdata *dd)
6517 {
6518         mutex_lock(&dd->dc8051_lock);
6519         _dc_shutdown(dd);
6520         mutex_unlock(&dd->dc8051_lock);
6521 }
6522
6523 /*
6524  * Calling this after the DC has been brought out of reset should not
6525  * do any damage.
6526  * NOTE: the caller needs to acquire the dd->dc8051_lock lock
6527  *       before calling this function.
6528  */
6529 static void _dc_start(struct hfi1_devdata *dd)
6530 {
6531         lockdep_assert_held(&dd->dc8051_lock);
6532
6533         if (!dd->dc_shutdown)
6534                 return;
6535
6536         /* Take the 8051 out of reset */
6537         write_csr(dd, DC_DC8051_CFG_RST, 0ull);
6538         /* Wait until 8051 is ready */
6539         if (wait_fm_ready(dd, TIMEOUT_8051_START))
6540                 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
6541                            __func__);
6542
6543         /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
6544         write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET);
6545         /* lcb_shutdown() with abort=1 does not restore these */
6546         write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en);
6547         dd->dc_shutdown = 0;
6548 }
6549
6550 static void dc_start(struct hfi1_devdata *dd)
6551 {
6552         mutex_lock(&dd->dc8051_lock);
6553         _dc_start(dd);
6554         mutex_unlock(&dd->dc8051_lock);
6555 }
6556
6557 /*
6558  * These LCB adjustments are for the Aurora SerDes core in the FPGA.
6559  */
6560 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd)
6561 {
6562         u64 rx_radr, tx_radr;
6563         u32 version;
6564
6565         if (dd->icode != ICODE_FPGA_EMULATION)
6566                 return;
6567
6568         /*
6569          * These LCB defaults on emulator _s are good, nothing to do here:
6570          *      LCB_CFG_TX_FIFOS_RADR
6571          *      LCB_CFG_RX_FIFOS_RADR
6572          *      LCB_CFG_LN_DCLK
6573          *      LCB_CFG_IGNORE_LOST_RCLK
6574          */
6575         if (is_emulator_s(dd))
6576                 return;
6577         /* else this is _p */
6578
6579         version = emulator_rev(dd);
6580         if (!is_ax(dd))
6581                 version = 0x2d; /* all B0 use 0x2d or higher settings */
6582
6583         if (version <= 0x12) {
6584                 /* release 0x12 and below */
6585
6586                 /*
6587                  * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9
6588                  * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9
6589                  * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa
6590                  */
6591                 rx_radr =
6592                       0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6593                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6594                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6595                 /*
6596                  * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default)
6597                  * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6
6598                  */
6599                 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6600         } else if (version <= 0x18) {
6601                 /* release 0x13 up to 0x18 */
6602                 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6603                 rx_radr =
6604                       0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6605                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6606                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6607                 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6608         } else if (version == 0x19) {
6609                 /* release 0x19 */
6610                 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */
6611                 rx_radr =
6612                       0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6613                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6614                     | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6615                 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6616         } else if (version == 0x1a) {
6617                 /* release 0x1a */
6618                 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */
6619                 rx_radr =
6620                       0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6621                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6622                     | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6623                 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6624                 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull);
6625         } else {
6626                 /* release 0x1b and higher */
6627                 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */
6628                 rx_radr =
6629                       0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT
6630                     | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT
6631                     | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT;
6632                 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT;
6633         }
6634
6635         write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr);
6636         /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */
6637         write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
6638                   DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
6639         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr);
6640 }
6641
6642 /*
6643  * Handle a SMA idle message
6644  *
6645  * This is a work-queue function outside of the interrupt.
6646  */
6647 void handle_sma_message(struct work_struct *work)
6648 {
6649         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6650                                                         sma_message_work);
6651         struct hfi1_devdata *dd = ppd->dd;
6652         u64 msg;
6653         int ret;
6654
6655         /*
6656          * msg is bytes 1-4 of the 40-bit idle message - the command code
6657          * is stripped off
6658          */
6659         ret = read_idle_sma(dd, &msg);
6660         if (ret)
6661                 return;
6662         dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg);
6663         /*
6664          * React to the SMA message.  Byte[1] (0 for us) is the command.
6665          */
6666         switch (msg & 0xff) {
6667         case SMA_IDLE_ARM:
6668                 /*
6669                  * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6670                  * State Transitions
6671                  *
6672                  * Only expected in INIT or ARMED, discard otherwise.
6673                  */
6674                 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED))
6675                         ppd->neighbor_normal = 1;
6676                 break;
6677         case SMA_IDLE_ACTIVE:
6678                 /*
6679                  * See OPAv1 table 9-14 - HFI and External Switch Ports Key
6680                  * State Transitions
6681                  *
6682                  * Can activate the node.  Discard otherwise.
6683                  */
6684                 if (ppd->host_link_state == HLS_UP_ARMED &&
6685                     ppd->is_active_optimize_enabled) {
6686                         ppd->neighbor_normal = 1;
6687                         ret = set_link_state(ppd, HLS_UP_ACTIVE);
6688                         if (ret)
6689                                 dd_dev_err(
6690                                         dd,
6691                                         "%s: received Active SMA idle message, couldn't set link to Active\n",
6692                                         __func__);
6693                 }
6694                 break;
6695         default:
6696                 dd_dev_err(dd,
6697                            "%s: received unexpected SMA idle message 0x%llx\n",
6698                            __func__, msg);
6699                 break;
6700         }
6701 }
6702
6703 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear)
6704 {
6705         u64 rcvctrl;
6706         unsigned long flags;
6707
6708         spin_lock_irqsave(&dd->rcvctrl_lock, flags);
6709         rcvctrl = read_csr(dd, RCV_CTRL);
6710         rcvctrl |= add;
6711         rcvctrl &= ~clear;
6712         write_csr(dd, RCV_CTRL, rcvctrl);
6713         spin_unlock_irqrestore(&dd->rcvctrl_lock, flags);
6714 }
6715
6716 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add)
6717 {
6718         adjust_rcvctrl(dd, add, 0);
6719 }
6720
6721 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear)
6722 {
6723         adjust_rcvctrl(dd, 0, clear);
6724 }
6725
6726 /*
6727  * Called from all interrupt handlers to start handling an SPC freeze.
6728  */
6729 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags)
6730 {
6731         struct hfi1_devdata *dd = ppd->dd;
6732         struct send_context *sc;
6733         int i;
6734         int sc_flags;
6735
6736         if (flags & FREEZE_SELF)
6737                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6738
6739         /* enter frozen mode */
6740         dd->flags |= HFI1_FROZEN;
6741
6742         /* notify all SDMA engines that they are going into a freeze */
6743         sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN));
6744
6745         sc_flags = SCF_FROZEN | SCF_HALTED | (flags & FREEZE_LINK_DOWN ?
6746                                               SCF_LINK_DOWN : 0);
6747         /* do halt pre-handling on all enabled send contexts */
6748         for (i = 0; i < dd->num_send_contexts; i++) {
6749                 sc = dd->send_contexts[i].sc;
6750                 if (sc && (sc->flags & SCF_ENABLED))
6751                         sc_stop(sc, sc_flags);
6752         }
6753
6754         /* Send context are frozen. Notify user space */
6755         hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT);
6756
6757         if (flags & FREEZE_ABORT) {
6758                 dd_dev_err(dd,
6759                            "Aborted freeze recovery. Please REBOOT system\n");
6760                 return;
6761         }
6762         /* queue non-interrupt handler */
6763         queue_work(ppd->hfi1_wq, &ppd->freeze_work);
6764 }
6765
6766 /*
6767  * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen,
6768  * depending on the "freeze" parameter.
6769  *
6770  * No need to return an error if it times out, our only option
6771  * is to proceed anyway.
6772  */
6773 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze)
6774 {
6775         unsigned long timeout;
6776         u64 reg;
6777
6778         timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT);
6779         while (1) {
6780                 reg = read_csr(dd, CCE_STATUS);
6781                 if (freeze) {
6782                         /* waiting until all indicators are set */
6783                         if ((reg & ALL_FROZE) == ALL_FROZE)
6784                                 return; /* all done */
6785                 } else {
6786                         /* waiting until all indicators are clear */
6787                         if ((reg & ALL_FROZE) == 0)
6788                                 return; /* all done */
6789                 }
6790
6791                 if (time_after(jiffies, timeout)) {
6792                         dd_dev_err(dd,
6793                                    "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing",
6794                                    freeze ? "" : "un", reg & ALL_FROZE,
6795                                    freeze ? ALL_FROZE : 0ull);
6796                         return;
6797                 }
6798                 usleep_range(80, 120);
6799         }
6800 }
6801
6802 /*
6803  * Do all freeze handling for the RXE block.
6804  */
6805 static void rxe_freeze(struct hfi1_devdata *dd)
6806 {
6807         int i;
6808         struct hfi1_ctxtdata *rcd;
6809
6810         /* disable port */
6811         clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6812
6813         /* disable all receive contexts */
6814         for (i = 0; i < dd->num_rcv_contexts; i++) {
6815                 rcd = hfi1_rcd_get_by_index(dd, i);
6816                 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd);
6817                 hfi1_rcd_put(rcd);
6818         }
6819 }
6820
6821 /*
6822  * Unfreeze handling for the RXE block - kernel contexts only.
6823  * This will also enable the port.  User contexts will do unfreeze
6824  * handling on a per-context basis as they call into the driver.
6825  *
6826  */
6827 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd)
6828 {
6829         u32 rcvmask;
6830         u16 i;
6831         struct hfi1_ctxtdata *rcd;
6832
6833         /* enable all kernel contexts */
6834         for (i = 0; i < dd->num_rcv_contexts; i++) {
6835                 rcd = hfi1_rcd_get_by_index(dd, i);
6836
6837                 /* Ensure all non-user contexts(including vnic) are enabled */
6838                 if (!rcd ||
6839                     (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) {
6840                         hfi1_rcd_put(rcd);
6841                         continue;
6842                 }
6843                 rcvmask = HFI1_RCVCTRL_CTXT_ENB;
6844                 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */
6845                 rcvmask |= rcd->rcvhdrtail_kvaddr ?
6846                         HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
6847                 hfi1_rcvctrl(dd, rcvmask, rcd);
6848                 hfi1_rcd_put(rcd);
6849         }
6850
6851         /* enable port */
6852         add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
6853 }
6854
6855 /*
6856  * Non-interrupt SPC freeze handling.
6857  *
6858  * This is a work-queue function outside of the triggering interrupt.
6859  */
6860 void handle_freeze(struct work_struct *work)
6861 {
6862         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6863                                                                 freeze_work);
6864         struct hfi1_devdata *dd = ppd->dd;
6865
6866         /* wait for freeze indicators on all affected blocks */
6867         wait_for_freeze_status(dd, 1);
6868
6869         /* SPC is now frozen */
6870
6871         /* do send PIO freeze steps */
6872         pio_freeze(dd);
6873
6874         /* do send DMA freeze steps */
6875         sdma_freeze(dd);
6876
6877         /* do send egress freeze steps - nothing to do */
6878
6879         /* do receive freeze steps */
6880         rxe_freeze(dd);
6881
6882         /*
6883          * Unfreeze the hardware - clear the freeze, wait for each
6884          * block's frozen bit to clear, then clear the frozen flag.
6885          */
6886         write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6887         wait_for_freeze_status(dd, 0);
6888
6889         if (is_ax(dd)) {
6890                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK);
6891                 wait_for_freeze_status(dd, 1);
6892                 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK);
6893                 wait_for_freeze_status(dd, 0);
6894         }
6895
6896         /* do send PIO unfreeze steps for kernel contexts */
6897         pio_kernel_unfreeze(dd);
6898
6899         /* do send DMA unfreeze steps */
6900         sdma_unfreeze(dd);
6901
6902         /* do send egress unfreeze steps - nothing to do */
6903
6904         /* do receive unfreeze steps for kernel contexts */
6905         rxe_kernel_unfreeze(dd);
6906
6907         /*
6908          * The unfreeze procedure touches global device registers when
6909          * it disables and re-enables RXE. Mark the device unfrozen
6910          * after all that is done so other parts of the driver waiting
6911          * for the device to unfreeze don't do things out of order.
6912          *
6913          * The above implies that the meaning of HFI1_FROZEN flag is
6914          * "Device has gone into freeze mode and freeze mode handling
6915          * is still in progress."
6916          *
6917          * The flag will be removed when freeze mode processing has
6918          * completed.
6919          */
6920         dd->flags &= ~HFI1_FROZEN;
6921         wake_up(&dd->event_queue);
6922
6923         /* no longer frozen */
6924 }
6925
6926 /**
6927  * update_xmit_counters - update PortXmitWait/PortVlXmitWait
6928  * counters.
6929  * @ppd: info of physical Hfi port
6930  * @link_width: new link width after link up or downgrade
6931  *
6932  * Update the PortXmitWait and PortVlXmitWait counters after
6933  * a link up or downgrade event to reflect a link width change.
6934  */
6935 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width)
6936 {
6937         int i;
6938         u16 tx_width;
6939         u16 link_speed;
6940
6941         tx_width = tx_link_width(link_width);
6942         link_speed = get_link_speed(ppd->link_speed_active);
6943
6944         /*
6945          * There are C_VL_COUNT number of PortVLXmitWait counters.
6946          * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
6947          */
6948         for (i = 0; i < C_VL_COUNT + 1; i++)
6949                 get_xmit_wait_counters(ppd, tx_width, link_speed, i);
6950 }
6951
6952 /*
6953  * Handle a link up interrupt from the 8051.
6954  *
6955  * This is a work-queue function outside of the interrupt.
6956  */
6957 void handle_link_up(struct work_struct *work)
6958 {
6959         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
6960                                                   link_up_work);
6961         struct hfi1_devdata *dd = ppd->dd;
6962
6963         set_link_state(ppd, HLS_UP_INIT);
6964
6965         /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */
6966         read_ltp_rtt(dd);
6967         /*
6968          * OPA specifies that certain counters are cleared on a transition
6969          * to link up, so do that.
6970          */
6971         clear_linkup_counters(dd);
6972         /*
6973          * And (re)set link up default values.
6974          */
6975         set_linkup_defaults(ppd);
6976
6977         /*
6978          * Set VL15 credits. Use cached value from verify cap interrupt.
6979          * In case of quick linkup or simulator, vl15 value will be set by
6980          * handle_linkup_change. VerifyCap interrupt handler will not be
6981          * called in those scenarios.
6982          */
6983         if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR))
6984                 set_up_vl15(dd, dd->vl15buf_cached);
6985
6986         /* enforce link speed enabled */
6987         if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) {
6988                 /* oops - current speed is not enabled, bounce */
6989                 dd_dev_err(dd,
6990                            "Link speed active 0x%x is outside enabled 0x%x, downing link\n",
6991                            ppd->link_speed_active, ppd->link_speed_enabled);
6992                 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0,
6993                                      OPA_LINKDOWN_REASON_SPEED_POLICY);
6994                 set_link_state(ppd, HLS_DN_OFFLINE);
6995                 start_link(ppd);
6996         }
6997 }
6998
6999 /*
7000  * Several pieces of LNI information were cached for SMA in ppd.
7001  * Reset these on link down
7002  */
7003 static void reset_neighbor_info(struct hfi1_pportdata *ppd)
7004 {
7005         ppd->neighbor_guid = 0;
7006         ppd->neighbor_port_number = 0;
7007         ppd->neighbor_type = 0;
7008         ppd->neighbor_fm_security = 0;
7009 }
7010
7011 static const char * const link_down_reason_strs[] = {
7012         [OPA_LINKDOWN_REASON_NONE] = "None",
7013         [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0",
7014         [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length",
7015         [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long",
7016         [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short",
7017         [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID",
7018         [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID",
7019         [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2",
7020         [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC",
7021         [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8",
7022         [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail",
7023         [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10",
7024         [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error",
7025         [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15",
7026         [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker",
7027         [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14",
7028         [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15",
7029         [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance",
7030         [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance",
7031         [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance",
7032         [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack",
7033         [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker",
7034         [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt",
7035         [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit",
7036         [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit",
7037         [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24",
7038         [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25",
7039         [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26",
7040         [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27",
7041         [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28",
7042         [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29",
7043         [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30",
7044         [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] =
7045                                         "Excessive buffer overrun",
7046         [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown",
7047         [OPA_LINKDOWN_REASON_REBOOT] = "Reboot",
7048         [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown",
7049         [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce",
7050         [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy",
7051         [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy",
7052         [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected",
7053         [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] =
7054                                         "Local media not installed",
7055         [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed",
7056         [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config",
7057         [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] =
7058                                         "End to end not installed",
7059         [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy",
7060         [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy",
7061         [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy",
7062         [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management",
7063         [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled",
7064         [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient"
7065 };
7066
7067 /* return the neighbor link down reason string */
7068 static const char *link_down_reason_str(u8 reason)
7069 {
7070         const char *str = NULL;
7071
7072         if (reason < ARRAY_SIZE(link_down_reason_strs))
7073                 str = link_down_reason_strs[reason];
7074         if (!str)
7075                 str = "(invalid)";
7076
7077         return str;
7078 }
7079
7080 /*
7081  * Handle a link down interrupt from the 8051.
7082  *
7083  * This is a work-queue function outside of the interrupt.
7084  */
7085 void handle_link_down(struct work_struct *work)
7086 {
7087         u8 lcl_reason, neigh_reason = 0;
7088         u8 link_down_reason;
7089         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7090                                                   link_down_work);
7091         int was_up;
7092         static const char ldr_str[] = "Link down reason: ";
7093
7094         if ((ppd->host_link_state &
7095              (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) &&
7096              ppd->port_type == PORT_TYPE_FIXED)
7097                 ppd->offline_disabled_reason =
7098                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED);
7099
7100         /* Go offline first, then deal with reading/writing through 8051 */
7101         was_up = !!(ppd->host_link_state & HLS_UP);
7102         set_link_state(ppd, HLS_DN_OFFLINE);
7103         xchg(&ppd->is_link_down_queued, 0);
7104
7105         if (was_up) {
7106                 lcl_reason = 0;
7107                 /* link down reason is only valid if the link was up */
7108                 read_link_down_reason(ppd->dd, &link_down_reason);
7109                 switch (link_down_reason) {
7110                 case LDR_LINK_TRANSFER_ACTIVE_LOW:
7111                         /* the link went down, no idle message reason */
7112                         dd_dev_info(ppd->dd, "%sUnexpected link down\n",
7113                                     ldr_str);
7114                         break;
7115                 case LDR_RECEIVED_LINKDOWN_IDLE_MSG:
7116                         /*
7117                          * The neighbor reason is only valid if an idle message
7118                          * was received for it.
7119                          */
7120                         read_planned_down_reason_code(ppd->dd, &neigh_reason);
7121                         dd_dev_info(ppd->dd,
7122                                     "%sNeighbor link down message %d, %s\n",
7123                                     ldr_str, neigh_reason,
7124                                     link_down_reason_str(neigh_reason));
7125                         break;
7126                 case LDR_RECEIVED_HOST_OFFLINE_REQ:
7127                         dd_dev_info(ppd->dd,
7128                                     "%sHost requested link to go offline\n",
7129                                     ldr_str);
7130                         break;
7131                 default:
7132                         dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n",
7133                                     ldr_str, link_down_reason);
7134                         break;
7135                 }
7136
7137                 /*
7138                  * If no reason, assume peer-initiated but missed
7139                  * LinkGoingDown idle flits.
7140                  */
7141                 if (neigh_reason == 0)
7142                         lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN;
7143         } else {
7144                 /* went down while polling or going up */
7145                 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT;
7146         }
7147
7148         set_link_down_reason(ppd, lcl_reason, neigh_reason, 0);
7149
7150         /* inform the SMA when the link transitions from up to down */
7151         if (was_up && ppd->local_link_down_reason.sma == 0 &&
7152             ppd->neigh_link_down_reason.sma == 0) {
7153                 ppd->local_link_down_reason.sma =
7154                                         ppd->local_link_down_reason.latest;
7155                 ppd->neigh_link_down_reason.sma =
7156                                         ppd->neigh_link_down_reason.latest;
7157         }
7158
7159         reset_neighbor_info(ppd);
7160
7161         /* disable the port */
7162         clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
7163
7164         /*
7165          * If there is no cable attached, turn the DC off. Otherwise,
7166          * start the link bring up.
7167          */
7168         if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd))
7169                 dc_shutdown(ppd->dd);
7170         else
7171                 start_link(ppd);
7172 }
7173
7174 void handle_link_bounce(struct work_struct *work)
7175 {
7176         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7177                                                         link_bounce_work);
7178
7179         /*
7180          * Only do something if the link is currently up.
7181          */
7182         if (ppd->host_link_state & HLS_UP) {
7183                 set_link_state(ppd, HLS_DN_OFFLINE);
7184                 start_link(ppd);
7185         } else {
7186                 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n",
7187                             __func__, link_state_name(ppd->host_link_state));
7188         }
7189 }
7190
7191 /*
7192  * Mask conversion: Capability exchange to Port LTP.  The capability
7193  * exchange has an implicit 16b CRC that is mandatory.
7194  */
7195 static int cap_to_port_ltp(int cap)
7196 {
7197         int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */
7198
7199         if (cap & CAP_CRC_14B)
7200                 port_ltp |= PORT_LTP_CRC_MODE_14;
7201         if (cap & CAP_CRC_48B)
7202                 port_ltp |= PORT_LTP_CRC_MODE_48;
7203         if (cap & CAP_CRC_12B_16B_PER_LANE)
7204                 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE;
7205
7206         return port_ltp;
7207 }
7208
7209 /*
7210  * Convert an OPA Port LTP mask to capability mask
7211  */
7212 int port_ltp_to_cap(int port_ltp)
7213 {
7214         int cap_mask = 0;
7215
7216         if (port_ltp & PORT_LTP_CRC_MODE_14)
7217                 cap_mask |= CAP_CRC_14B;
7218         if (port_ltp & PORT_LTP_CRC_MODE_48)
7219                 cap_mask |= CAP_CRC_48B;
7220         if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE)
7221                 cap_mask |= CAP_CRC_12B_16B_PER_LANE;
7222
7223         return cap_mask;
7224 }
7225
7226 /*
7227  * Convert a single DC LCB CRC mode to an OPA Port LTP mask.
7228  */
7229 static int lcb_to_port_ltp(int lcb_crc)
7230 {
7231         int port_ltp = 0;
7232
7233         if (lcb_crc == LCB_CRC_12B_16B_PER_LANE)
7234                 port_ltp = PORT_LTP_CRC_MODE_PER_LANE;
7235         else if (lcb_crc == LCB_CRC_48B)
7236                 port_ltp = PORT_LTP_CRC_MODE_48;
7237         else if (lcb_crc == LCB_CRC_14B)
7238                 port_ltp = PORT_LTP_CRC_MODE_14;
7239         else
7240                 port_ltp = PORT_LTP_CRC_MODE_16;
7241
7242         return port_ltp;
7243 }
7244
7245 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd)
7246 {
7247         if (ppd->pkeys[2] != 0) {
7248                 ppd->pkeys[2] = 0;
7249                 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0);
7250                 hfi1_event_pkey_change(ppd->dd, ppd->port);
7251         }
7252 }
7253
7254 /*
7255  * Convert the given link width to the OPA link width bitmask.
7256  */
7257 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width)
7258 {
7259         switch (width) {
7260         case 0:
7261                 /*
7262                  * Simulator and quick linkup do not set the width.
7263                  * Just set it to 4x without complaint.
7264                  */
7265                 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup)
7266                         return OPA_LINK_WIDTH_4X;
7267                 return 0; /* no lanes up */
7268         case 1: return OPA_LINK_WIDTH_1X;
7269         case 2: return OPA_LINK_WIDTH_2X;
7270         case 3: return OPA_LINK_WIDTH_3X;
7271         default:
7272                 dd_dev_info(dd, "%s: invalid width %d, using 4\n",
7273                             __func__, width);
7274                 /* fall through */
7275         case 4: return OPA_LINK_WIDTH_4X;
7276         }
7277 }
7278
7279 /*
7280  * Do a population count on the bottom nibble.
7281  */
7282 static const u8 bit_counts[16] = {
7283         0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4
7284 };
7285
7286 static inline u8 nibble_to_count(u8 nibble)
7287 {
7288         return bit_counts[nibble & 0xf];
7289 }
7290
7291 /*
7292  * Read the active lane information from the 8051 registers and return
7293  * their widths.
7294  *
7295  * Active lane information is found in these 8051 registers:
7296  *      enable_lane_tx
7297  *      enable_lane_rx
7298  */
7299 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width,
7300                             u16 *rx_width)
7301 {
7302         u16 tx, rx;
7303         u8 enable_lane_rx;
7304         u8 enable_lane_tx;
7305         u8 tx_polarity_inversion;
7306         u8 rx_polarity_inversion;
7307         u8 max_rate;
7308
7309         /* read the active lanes */
7310         read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
7311                          &rx_polarity_inversion, &max_rate);
7312         read_local_lni(dd, &enable_lane_rx);
7313
7314         /* convert to counts */
7315         tx = nibble_to_count(enable_lane_tx);
7316         rx = nibble_to_count(enable_lane_rx);
7317
7318         /*
7319          * Set link_speed_active here, overriding what was set in
7320          * handle_verify_cap().  The ASIC 8051 firmware does not correctly
7321          * set the max_rate field in handle_verify_cap until v0.19.
7322          */
7323         if ((dd->icode == ICODE_RTL_SILICON) &&
7324             (dd->dc8051_ver < dc8051_ver(0, 19, 0))) {
7325                 /* max_rate: 0 = 12.5G, 1 = 25G */
7326                 switch (max_rate) {
7327                 case 0:
7328                         dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G;
7329                         break;
7330                 default:
7331                         dd_dev_err(dd,
7332                                    "%s: unexpected max rate %d, using 25Gb\n",
7333                                    __func__, (int)max_rate);
7334                         /* fall through */
7335                 case 1:
7336                         dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G;
7337                         break;
7338                 }
7339         }
7340
7341         dd_dev_info(dd,
7342                     "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n",
7343                     enable_lane_tx, tx, enable_lane_rx, rx);
7344         *tx_width = link_width_to_bits(dd, tx);
7345         *rx_width = link_width_to_bits(dd, rx);
7346 }
7347
7348 /*
7349  * Read verify_cap_local_fm_link_width[1] to obtain the link widths.
7350  * Valid after the end of VerifyCap and during LinkUp.  Does not change
7351  * after link up.  I.e. look elsewhere for downgrade information.
7352  *
7353  * Bits are:
7354  *      + bits [7:4] contain the number of active transmitters
7355  *      + bits [3:0] contain the number of active receivers
7356  * These are numbers 1 through 4 and can be different values if the
7357  * link is asymmetric.
7358  *
7359  * verify_cap_local_fm_link_width[0] retains its original value.
7360  */
7361 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width,
7362                               u16 *rx_width)
7363 {
7364         u16 widths, tx, rx;
7365         u8 misc_bits, local_flags;
7366         u16 active_tx, active_rx;
7367
7368         read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths);
7369         tx = widths >> 12;
7370         rx = (widths >> 8) & 0xf;
7371
7372         *tx_width = link_width_to_bits(dd, tx);
7373         *rx_width = link_width_to_bits(dd, rx);
7374
7375         /* print the active widths */
7376         get_link_widths(dd, &active_tx, &active_rx);
7377 }
7378
7379 /*
7380  * Set ppd->link_width_active and ppd->link_width_downgrade_active using
7381  * hardware information when the link first comes up.
7382  *
7383  * The link width is not available until after VerifyCap.AllFramesReceived
7384  * (the trigger for handle_verify_cap), so this is outside that routine
7385  * and should be called when the 8051 signals linkup.
7386  */
7387 void get_linkup_link_widths(struct hfi1_pportdata *ppd)
7388 {
7389         u16 tx_width, rx_width;
7390
7391         /* get end-of-LNI link widths */
7392         get_linkup_widths(ppd->dd, &tx_width, &rx_width);
7393
7394         /* use tx_width as the link is supposed to be symmetric on link up */
7395         ppd->link_width_active = tx_width;
7396         /* link width downgrade active (LWD.A) starts out matching LW.A */
7397         ppd->link_width_downgrade_tx_active = ppd->link_width_active;
7398         ppd->link_width_downgrade_rx_active = ppd->link_width_active;
7399         /* per OPA spec, on link up LWD.E resets to LWD.S */
7400         ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported;
7401         /* cache the active egress rate (units {10^6 bits/sec]) */
7402         ppd->current_egress_rate = active_egress_rate(ppd);
7403 }
7404
7405 /*
7406  * Handle a verify capabilities interrupt from the 8051.
7407  *
7408  * This is a work-queue function outside of the interrupt.
7409  */
7410 void handle_verify_cap(struct work_struct *work)
7411 {
7412         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7413                                                                 link_vc_work);
7414         struct hfi1_devdata *dd = ppd->dd;
7415         u64 reg;
7416         u8 power_management;
7417         u8 continuous;
7418         u8 vcu;
7419         u8 vau;
7420         u8 z;
7421         u16 vl15buf;
7422         u16 link_widths;
7423         u16 crc_mask;
7424         u16 crc_val;
7425         u16 device_id;
7426         u16 active_tx, active_rx;
7427         u8 partner_supported_crc;
7428         u8 remote_tx_rate;
7429         u8 device_rev;
7430
7431         set_link_state(ppd, HLS_VERIFY_CAP);
7432
7433         lcb_shutdown(dd, 0);
7434         adjust_lcb_for_fpga_serdes(dd);
7435
7436         read_vc_remote_phy(dd, &power_management, &continuous);
7437         read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
7438                               &partner_supported_crc);
7439         read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths);
7440         read_remote_device_id(dd, &device_id, &device_rev);
7441
7442         /* print the active widths */
7443         get_link_widths(dd, &active_tx, &active_rx);
7444         dd_dev_info(dd,
7445                     "Peer PHY: power management 0x%x, continuous updates 0x%x\n",
7446                     (int)power_management, (int)continuous);
7447         dd_dev_info(dd,
7448                     "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n",
7449                     (int)vau, (int)z, (int)vcu, (int)vl15buf,
7450                     (int)partner_supported_crc);
7451         dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n",
7452                     (u32)remote_tx_rate, (u32)link_widths);
7453         dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n",
7454                     (u32)device_id, (u32)device_rev);
7455         /*
7456          * The peer vAU value just read is the peer receiver value.  HFI does
7457          * not support a transmit vAU of 0 (AU == 8).  We advertised that
7458          * with Z=1 in the fabric capabilities sent to the peer.  The peer
7459          * will see our Z=1, and, if it advertised a vAU of 0, will move its
7460          * receive to vAU of 1 (AU == 16).  Do the same here.  We do not care
7461          * about the peer Z value - our sent vAU is 3 (hardwired) and is not
7462          * subject to the Z value exception.
7463          */
7464         if (vau == 0)
7465                 vau = 1;
7466         set_up_vau(dd, vau);
7467
7468         /*
7469          * Set VL15 credits to 0 in global credit register. Cache remote VL15
7470          * credits value and wait for link-up interrupt ot set it.
7471          */
7472         set_up_vl15(dd, 0);
7473         dd->vl15buf_cached = vl15buf;
7474
7475         /* set up the LCB CRC mode */
7476         crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc;
7477
7478         /* order is important: use the lowest bit in common */
7479         if (crc_mask & CAP_CRC_14B)
7480                 crc_val = LCB_CRC_14B;
7481         else if (crc_mask & CAP_CRC_48B)
7482                 crc_val = LCB_CRC_48B;
7483         else if (crc_mask & CAP_CRC_12B_16B_PER_LANE)
7484                 crc_val = LCB_CRC_12B_16B_PER_LANE;
7485         else
7486                 crc_val = LCB_CRC_16B;
7487
7488         dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val);
7489         write_csr(dd, DC_LCB_CFG_CRC_MODE,
7490                   (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT);
7491
7492         /* set (14b only) or clear sideband credit */
7493         reg = read_csr(dd, SEND_CM_CTRL);
7494         if (crc_val == LCB_CRC_14B && crc_14b_sideband) {
7495                 write_csr(dd, SEND_CM_CTRL,
7496                           reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7497         } else {
7498                 write_csr(dd, SEND_CM_CTRL,
7499                           reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK);
7500         }
7501
7502         ppd->link_speed_active = 0;     /* invalid value */
7503         if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
7504                 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */
7505                 switch (remote_tx_rate) {
7506                 case 0:
7507                         ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7508                         break;
7509                 case 1:
7510                         ppd->link_speed_active = OPA_LINK_SPEED_25G;
7511                         break;
7512                 }
7513         } else {
7514                 /* actual rate is highest bit of the ANDed rates */
7515                 u8 rate = remote_tx_rate & ppd->local_tx_rate;
7516
7517                 if (rate & 2)
7518                         ppd->link_speed_active = OPA_LINK_SPEED_25G;
7519                 else if (rate & 1)
7520                         ppd->link_speed_active = OPA_LINK_SPEED_12_5G;
7521         }
7522         if (ppd->link_speed_active == 0) {
7523                 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n",
7524                            __func__, (int)remote_tx_rate);
7525                 ppd->link_speed_active = OPA_LINK_SPEED_25G;
7526         }
7527
7528         /*
7529          * Cache the values of the supported, enabled, and active
7530          * LTP CRC modes to return in 'portinfo' queries. But the bit
7531          * flags that are returned in the portinfo query differ from
7532          * what's in the link_crc_mask, crc_sizes, and crc_val
7533          * variables. Convert these here.
7534          */
7535         ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
7536                 /* supported crc modes */
7537         ppd->port_ltp_crc_mode |=
7538                 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4;
7539                 /* enabled crc modes */
7540         ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val);
7541                 /* active crc mode */
7542
7543         /* set up the remote credit return table */
7544         assign_remote_cm_au_table(dd, vcu);
7545
7546         /*
7547          * The LCB is reset on entry to handle_verify_cap(), so this must
7548          * be applied on every link up.
7549          *
7550          * Adjust LCB error kill enable to kill the link if
7551          * these RBUF errors are seen:
7552          *      REPLAY_BUF_MBE_SMASK
7553          *      FLIT_INPUT_BUF_MBE_SMASK
7554          */
7555         if (is_ax(dd)) {                        /* fixed in B0 */
7556                 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN);
7557                 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK
7558                         | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK;
7559                 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg);
7560         }
7561
7562         /* pull LCB fifos out of reset - all fifo clocks must be stable */
7563         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
7564
7565         /* give 8051 access to the LCB CSRs */
7566         write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
7567         set_8051_lcb_access(dd);
7568
7569         /* tell the 8051 to go to LinkUp */
7570         set_link_state(ppd, HLS_GOING_UP);
7571 }
7572
7573 /**
7574  * apply_link_downgrade_policy - Apply the link width downgrade enabled
7575  * policy against the current active link widths.
7576  * @ppd: info of physical Hfi port
7577  * @refresh_widths: True indicates link downgrade event
7578  * @return: True indicates a successful link downgrade. False indicates
7579  *          link downgrade event failed and the link will bounce back to
7580  *          default link width.
7581  *
7582  * Called when the enabled policy changes or the active link widths
7583  * change.
7584  * Refresh_widths indicates that a link downgrade occurred. The
7585  * link_downgraded variable is set by refresh_widths and
7586  * determines the success/failure of the policy application.
7587  */
7588 bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
7589                                  bool refresh_widths)
7590 {
7591         int do_bounce = 0;
7592         int tries;
7593         u16 lwde;
7594         u16 tx, rx;
7595         bool link_downgraded = refresh_widths;
7596
7597         /* use the hls lock to avoid a race with actual link up */
7598         tries = 0;
7599 retry:
7600         mutex_lock(&ppd->hls_lock);
7601         /* only apply if the link is up */
7602         if (ppd->host_link_state & HLS_DOWN) {
7603                 /* still going up..wait and retry */
7604                 if (ppd->host_link_state & HLS_GOING_UP) {
7605                         if (++tries < 1000) {
7606                                 mutex_unlock(&ppd->hls_lock);
7607                                 usleep_range(100, 120); /* arbitrary */
7608                                 goto retry;
7609                         }
7610                         dd_dev_err(ppd->dd,
7611                                    "%s: giving up waiting for link state change\n",
7612                                    __func__);
7613                 }
7614                 goto done;
7615         }
7616
7617         lwde = ppd->link_width_downgrade_enabled;
7618
7619         if (refresh_widths) {
7620                 get_link_widths(ppd->dd, &tx, &rx);
7621                 ppd->link_width_downgrade_tx_active = tx;
7622                 ppd->link_width_downgrade_rx_active = rx;
7623         }
7624
7625         if (ppd->link_width_downgrade_tx_active == 0 ||
7626             ppd->link_width_downgrade_rx_active == 0) {
7627                 /* the 8051 reported a dead link as a downgrade */
7628                 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
7629                 link_downgraded = false;
7630         } else if (lwde == 0) {
7631                 /* downgrade is disabled */
7632
7633                 /* bounce if not at starting active width */
7634                 if ((ppd->link_width_active !=
7635                      ppd->link_width_downgrade_tx_active) ||
7636                     (ppd->link_width_active !=
7637                      ppd->link_width_downgrade_rx_active)) {
7638                         dd_dev_err(ppd->dd,
7639                                    "Link downgrade is disabled and link has downgraded, downing link\n");
7640                         dd_dev_err(ppd->dd,
7641                                    "  original 0x%x, tx active 0x%x, rx active 0x%x\n",
7642                                    ppd->link_width_active,
7643                                    ppd->link_width_downgrade_tx_active,
7644                                    ppd->link_width_downgrade_rx_active);
7645                         do_bounce = 1;
7646                         link_downgraded = false;
7647                 }
7648         } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
7649                    (lwde & ppd->link_width_downgrade_rx_active) == 0) {
7650                 /* Tx or Rx is outside the enabled policy */
7651                 dd_dev_err(ppd->dd,
7652                            "Link is outside of downgrade allowed, downing link\n");
7653                 dd_dev_err(ppd->dd,
7654                            "  enabled 0x%x, tx active 0x%x, rx active 0x%x\n",
7655                            lwde, ppd->link_width_downgrade_tx_active,
7656                            ppd->link_width_downgrade_rx_active);
7657                 do_bounce = 1;
7658                 link_downgraded = false;
7659         }
7660
7661 done:
7662         mutex_unlock(&ppd->hls_lock);
7663
7664         if (do_bounce) {
7665                 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0,
7666                                      OPA_LINKDOWN_REASON_WIDTH_POLICY);
7667                 set_link_state(ppd, HLS_DN_OFFLINE);
7668                 start_link(ppd);
7669         }
7670
7671         return link_downgraded;
7672 }
7673
7674 /*
7675  * Handle a link downgrade interrupt from the 8051.
7676  *
7677  * This is a work-queue function outside of the interrupt.
7678  */
7679 void handle_link_downgrade(struct work_struct *work)
7680 {
7681         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
7682                                                         link_downgrade_work);
7683
7684         dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
7685         if (apply_link_downgrade_policy(ppd, true))
7686                 update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active);
7687 }
7688
7689 static char *dcc_err_string(char *buf, int buf_len, u64 flags)
7690 {
7691         return flag_string(buf, buf_len, flags, dcc_err_flags,
7692                 ARRAY_SIZE(dcc_err_flags));
7693 }
7694
7695 static char *lcb_err_string(char *buf, int buf_len, u64 flags)
7696 {
7697         return flag_string(buf, buf_len, flags, lcb_err_flags,
7698                 ARRAY_SIZE(lcb_err_flags));
7699 }
7700
7701 static char *dc8051_err_string(char *buf, int buf_len, u64 flags)
7702 {
7703         return flag_string(buf, buf_len, flags, dc8051_err_flags,
7704                 ARRAY_SIZE(dc8051_err_flags));
7705 }
7706
7707 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags)
7708 {
7709         return flag_string(buf, buf_len, flags, dc8051_info_err_flags,
7710                 ARRAY_SIZE(dc8051_info_err_flags));
7711 }
7712
7713 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags)
7714 {
7715         return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags,
7716                 ARRAY_SIZE(dc8051_info_host_msg_flags));
7717 }
7718
7719 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg)
7720 {
7721         struct hfi1_pportdata *ppd = dd->pport;
7722         u64 info, err, host_msg;
7723         int queue_link_down = 0;
7724         char buf[96];
7725
7726         /* look at the flags */
7727         if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) {
7728                 /* 8051 information set by firmware */
7729                 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */
7730                 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051);
7731                 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT)
7732                         & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK;
7733                 host_msg = (info >>
7734                         DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT)
7735                         & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK;
7736
7737                 /*
7738                  * Handle error flags.
7739                  */
7740                 if (err & FAILED_LNI) {
7741                         /*
7742                          * LNI error indications are cleared by the 8051
7743                          * only when starting polling.  Only pay attention
7744                          * to them when in the states that occur during
7745                          * LNI.
7746                          */
7747                         if (ppd->host_link_state
7748                             & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
7749                                 queue_link_down = 1;
7750                                 dd_dev_info(dd, "Link error: %s\n",
7751                                             dc8051_info_err_string(buf,
7752                                                                    sizeof(buf),
7753                                                                    err &
7754                                                                    FAILED_LNI));
7755                         }
7756                         err &= ~(u64)FAILED_LNI;
7757                 }
7758                 /* unknown frames can happen durning LNI, just count */
7759                 if (err & UNKNOWN_FRAME) {
7760                         ppd->unknown_frame_count++;
7761                         err &= ~(u64)UNKNOWN_FRAME;
7762                 }
7763                 if (err) {
7764                         /* report remaining errors, but do not do anything */
7765                         dd_dev_err(dd, "8051 info error: %s\n",
7766                                    dc8051_info_err_string(buf, sizeof(buf),
7767                                                           err));
7768                 }
7769
7770                 /*
7771                  * Handle host message flags.
7772                  */
7773                 if (host_msg & HOST_REQ_DONE) {
7774                         /*
7775                          * Presently, the driver does a busy wait for
7776                          * host requests to complete.  This is only an
7777                          * informational message.
7778                          * NOTE: The 8051 clears the host message
7779                          * information *on the next 8051 command*.
7780                          * Therefore, when linkup is achieved,
7781                          * this flag will still be set.
7782                          */
7783                         host_msg &= ~(u64)HOST_REQ_DONE;
7784                 }
7785                 if (host_msg & BC_SMA_MSG) {
7786                         queue_work(ppd->link_wq, &ppd->sma_message_work);
7787                         host_msg &= ~(u64)BC_SMA_MSG;
7788                 }
7789                 if (host_msg & LINKUP_ACHIEVED) {
7790                         dd_dev_info(dd, "8051: Link up\n");
7791                         queue_work(ppd->link_wq, &ppd->link_up_work);
7792                         host_msg &= ~(u64)LINKUP_ACHIEVED;
7793                 }
7794                 if (host_msg & EXT_DEVICE_CFG_REQ) {
7795                         handle_8051_request(ppd);
7796                         host_msg &= ~(u64)EXT_DEVICE_CFG_REQ;
7797                 }
7798                 if (host_msg & VERIFY_CAP_FRAME) {
7799                         queue_work(ppd->link_wq, &ppd->link_vc_work);
7800                         host_msg &= ~(u64)VERIFY_CAP_FRAME;
7801                 }
7802                 if (host_msg & LINK_GOING_DOWN) {
7803                         const char *extra = "";
7804                         /* no downgrade action needed if going down */
7805                         if (host_msg & LINK_WIDTH_DOWNGRADED) {
7806                                 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7807                                 extra = " (ignoring downgrade)";
7808                         }
7809                         dd_dev_info(dd, "8051: Link down%s\n", extra);
7810                         queue_link_down = 1;
7811                         host_msg &= ~(u64)LINK_GOING_DOWN;
7812                 }
7813                 if (host_msg & LINK_WIDTH_DOWNGRADED) {
7814                         queue_work(ppd->link_wq, &ppd->link_downgrade_work);
7815                         host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED;
7816                 }
7817                 if (host_msg) {
7818                         /* report remaining messages, but do not do anything */
7819                         dd_dev_info(dd, "8051 info host message: %s\n",
7820                                     dc8051_info_host_msg_string(buf,
7821                                                                 sizeof(buf),
7822                                                                 host_msg));
7823                 }
7824
7825                 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK;
7826         }
7827         if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) {
7828                 /*
7829                  * Lost the 8051 heartbeat.  If this happens, we
7830                  * receive constant interrupts about it.  Disable
7831                  * the interrupt after the first.
7832                  */
7833                 dd_dev_err(dd, "Lost 8051 heartbeat\n");
7834                 write_csr(dd, DC_DC8051_ERR_EN,
7835                           read_csr(dd, DC_DC8051_ERR_EN) &
7836                           ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK);
7837
7838                 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK;
7839         }
7840         if (reg) {
7841                 /* report the error, but do not do anything */
7842                 dd_dev_err(dd, "8051 error: %s\n",
7843                            dc8051_err_string(buf, sizeof(buf), reg));
7844         }
7845
7846         if (queue_link_down) {
7847                 /*
7848                  * if the link is already going down or disabled, do not
7849                  * queue another. If there's a link down entry already
7850                  * queued, don't queue another one.
7851                  */
7852                 if ((ppd->host_link_state &
7853                     (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) ||
7854                     ppd->link_enabled == 0) {
7855                         dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n",
7856                                     __func__, ppd->host_link_state,
7857                                     ppd->link_enabled);
7858                 } else {
7859                         if (xchg(&ppd->is_link_down_queued, 1) == 1)
7860                                 dd_dev_info(dd,
7861                                             "%s: link down request already queued\n",
7862                                             __func__);
7863                         else
7864                                 queue_work(ppd->link_wq, &ppd->link_down_work);
7865                 }
7866         }
7867 }
7868
7869 static const char * const fm_config_txt[] = {
7870 [0] =
7871         "BadHeadDist: Distance violation between two head flits",
7872 [1] =
7873         "BadTailDist: Distance violation between two tail flits",
7874 [2] =
7875         "BadCtrlDist: Distance violation between two credit control flits",
7876 [3] =
7877         "BadCrdAck: Credits return for unsupported VL",
7878 [4] =
7879         "UnsupportedVLMarker: Received VL Marker",
7880 [5] =
7881         "BadPreempt: Exceeded the preemption nesting level",
7882 [6] =
7883         "BadControlFlit: Received unsupported control flit",
7884 /* no 7 */
7885 [8] =
7886         "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL",
7887 };
7888
7889 static const char * const port_rcv_txt[] = {
7890 [1] =
7891         "BadPktLen: Illegal PktLen",
7892 [2] =
7893         "PktLenTooLong: Packet longer than PktLen",
7894 [3] =
7895         "PktLenTooShort: Packet shorter than PktLen",
7896 [4] =
7897         "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)",
7898 [5] =
7899         "BadDLID: Illegal DLID (0, doesn't match HFI)",
7900 [6] =
7901         "BadL2: Illegal L2 opcode",
7902 [7] =
7903         "BadSC: Unsupported SC",
7904 [9] =
7905         "BadRC: Illegal RC",
7906 [11] =
7907         "PreemptError: Preempting with same VL",
7908 [12] =
7909         "PreemptVL15: Preempting a VL15 packet",
7910 };
7911
7912 #define OPA_LDR_FMCONFIG_OFFSET 16
7913 #define OPA_LDR_PORTRCV_OFFSET 0
7914 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
7915 {
7916         u64 info, hdr0, hdr1;
7917         const char *extra;
7918         char buf[96];
7919         struct hfi1_pportdata *ppd = dd->pport;
7920         u8 lcl_reason = 0;
7921         int do_bounce = 0;
7922
7923         if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) {
7924                 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) {
7925                         info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE);
7926                         dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK;
7927                         /* set status bit */
7928                         dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK;
7929                 }
7930                 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK;
7931         }
7932
7933         if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) {
7934                 struct hfi1_pportdata *ppd = dd->pport;
7935                 /* this counter saturates at (2^32) - 1 */
7936                 if (ppd->link_downed < (u32)UINT_MAX)
7937                         ppd->link_downed++;
7938                 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK;
7939         }
7940
7941         if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) {
7942                 u8 reason_valid = 1;
7943
7944                 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG);
7945                 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) {
7946                         dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK;
7947                         /* set status bit */
7948                         dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK;
7949                 }
7950                 switch (info) {
7951                 case 0:
7952                 case 1:
7953                 case 2:
7954                 case 3:
7955                 case 4:
7956                 case 5:
7957                 case 6:
7958                         extra = fm_config_txt[info];
7959                         break;
7960                 case 8:
7961                         extra = fm_config_txt[info];
7962                         if (ppd->port_error_action &
7963                             OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) {
7964                                 do_bounce = 1;
7965                                 /*
7966                                  * lcl_reason cannot be derived from info
7967                                  * for this error
7968                                  */
7969                                 lcl_reason =
7970                                   OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER;
7971                         }
7972                         break;
7973                 default:
7974                         reason_valid = 0;
7975                         snprintf(buf, sizeof(buf), "reserved%lld", info);
7976                         extra = buf;
7977                         break;
7978                 }
7979
7980                 if (reason_valid && !do_bounce) {
7981                         do_bounce = ppd->port_error_action &
7982                                         (1 << (OPA_LDR_FMCONFIG_OFFSET + info));
7983                         lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST;
7984                 }
7985
7986                 /* just report this */
7987                 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n",
7988                                         extra);
7989                 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK;
7990         }
7991
7992         if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) {
7993                 u8 reason_valid = 1;
7994
7995                 info = read_csr(dd, DCC_ERR_INFO_PORTRCV);
7996                 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0);
7997                 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1);
7998                 if (!(dd->err_info_rcvport.status_and_code &
7999                       OPA_EI_STATUS_SMASK)) {
8000                         dd->err_info_rcvport.status_and_code =
8001                                 info & OPA_EI_CODE_SMASK;
8002                         /* set status bit */
8003                         dd->err_info_rcvport.status_and_code |=
8004                                 OPA_EI_STATUS_SMASK;
8005                         /*
8006                          * save first 2 flits in the packet that caused
8007                          * the error
8008                          */
8009                         dd->err_info_rcvport.packet_flit1 = hdr0;
8010                         dd->err_info_rcvport.packet_flit2 = hdr1;
8011                 }
8012                 switch (info) {
8013                 case 1:
8014                 case 2:
8015                 case 3:
8016                 case 4:
8017                 case 5:
8018                 case 6:
8019                 case 7:
8020                 case 9:
8021                 case 11:
8022                 case 12:
8023                         extra = port_rcv_txt[info];
8024                         break;
8025                 default:
8026                         reason_valid = 0;
8027                         snprintf(buf, sizeof(buf), "reserved%lld", info);
8028                         extra = buf;
8029                         break;
8030                 }
8031
8032                 if (reason_valid && !do_bounce) {
8033                         do_bounce = ppd->port_error_action &
8034                                         (1 << (OPA_LDR_PORTRCV_OFFSET + info));
8035                         lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0;
8036                 }
8037
8038                 /* just report this */
8039                 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n"
8040                                         "               hdr0 0x%llx, hdr1 0x%llx\n",
8041                                         extra, hdr0, hdr1);
8042
8043                 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK;
8044         }
8045
8046         if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) {
8047                 /* informative only */
8048                 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n");
8049                 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK;
8050         }
8051         if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) {
8052                 /* informative only */
8053                 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n");
8054                 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK;
8055         }
8056
8057         if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev)))
8058                 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK;
8059
8060         /* report any remaining errors */
8061         if (reg)
8062                 dd_dev_info_ratelimited(dd, "DCC Error: %s\n",
8063                                         dcc_err_string(buf, sizeof(buf), reg));
8064
8065         if (lcl_reason == 0)
8066                 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN;
8067
8068         if (do_bounce) {
8069                 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n",
8070                                         __func__);
8071                 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason);
8072                 queue_work(ppd->link_wq, &ppd->link_bounce_work);
8073         }
8074 }
8075
8076 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
8077 {
8078         char buf[96];
8079
8080         dd_dev_info(dd, "LCB Error: %s\n",
8081                     lcb_err_string(buf, sizeof(buf), reg));
8082 }
8083
8084 /*
8085  * CCE block DC interrupt.  Source is < 8.
8086  */
8087 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source)
8088 {
8089         const struct err_reg_info *eri = &dc_errs[source];
8090
8091         if (eri->handler) {
8092                 interrupt_clear_down(dd, 0, eri);
8093         } else if (source == 3 /* dc_lbm_int */) {
8094                 /*
8095                  * This indicates that a parity error has occurred on the
8096                  * address/control lines presented to the LBM.  The error
8097                  * is a single pulse, there is no associated error flag,
8098                  * and it is non-maskable.  This is because if a parity
8099                  * error occurs on the request the request is dropped.
8100                  * This should never occur, but it is nice to know if it
8101                  * ever does.
8102                  */
8103                 dd_dev_err(dd, "Parity error in DC LBM block\n");
8104         } else {
8105                 dd_dev_err(dd, "Invalid DC interrupt %u\n", source);
8106         }
8107 }
8108
8109 /*
8110  * TX block send credit interrupt.  Source is < 160.
8111  */
8112 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source)
8113 {
8114         sc_group_release_update(dd, source);
8115 }
8116
8117 /*
8118  * TX block SDMA interrupt.  Source is < 48.
8119  *
8120  * SDMA interrupts are grouped by type:
8121  *
8122  *       0 -  N-1 = SDma
8123  *       N - 2N-1 = SDmaProgress
8124  *      2N - 3N-1 = SDmaIdle
8125  */
8126 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source)
8127 {
8128         /* what interrupt */
8129         unsigned int what  = source / TXE_NUM_SDMA_ENGINES;
8130         /* which engine */
8131         unsigned int which = source % TXE_NUM_SDMA_ENGINES;
8132
8133 #ifdef CONFIG_SDMA_VERBOSITY
8134         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which,
8135                    slashstrip(__FILE__), __LINE__, __func__);
8136         sdma_dumpstate(&dd->per_sdma[which]);
8137 #endif
8138
8139         if (likely(what < 3 && which < dd->num_sdma)) {
8140                 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source);
8141         } else {
8142                 /* should not happen */
8143                 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source);
8144         }
8145 }
8146
8147 /**
8148  * is_rcv_avail_int() - User receive context available IRQ handler
8149  * @dd: valid dd
8150  * @source: logical IRQ source (offset from IS_RCVAVAIL_START)
8151  *
8152  * RX block receive available interrupt.  Source is < 160.
8153  *
8154  * This is the general interrupt handler for user (PSM) receive contexts,
8155  * and can only be used for non-threaded IRQs.
8156  */
8157 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source)
8158 {
8159         struct hfi1_ctxtdata *rcd;
8160         char *err_detail;
8161
8162         if (likely(source < dd->num_rcv_contexts)) {
8163                 rcd = hfi1_rcd_get_by_index(dd, source);
8164                 if (rcd) {
8165                         handle_user_interrupt(rcd);
8166                         hfi1_rcd_put(rcd);
8167                         return; /* OK */
8168                 }
8169                 /* received an interrupt, but no rcd */
8170                 err_detail = "dataless";
8171         } else {
8172                 /* received an interrupt, but are not using that context */
8173                 err_detail = "out of range";
8174         }
8175         dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n",
8176                    err_detail, source);
8177 }
8178
8179 /**
8180  * is_rcv_urgent_int() - User receive context urgent IRQ handler
8181  * @dd: valid dd
8182  * @source: logical IRQ source (offset from IS_RCVURGENT_START)
8183  *
8184  * RX block receive urgent interrupt.  Source is < 160.
8185  *
8186  * NOTE: kernel receive contexts specifically do NOT enable this IRQ.
8187  */
8188 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source)
8189 {
8190         struct hfi1_ctxtdata *rcd;
8191         char *err_detail;
8192
8193         if (likely(source < dd->num_rcv_contexts)) {
8194                 rcd = hfi1_rcd_get_by_index(dd, source);
8195                 if (rcd) {
8196                         handle_user_interrupt(rcd);
8197                         hfi1_rcd_put(rcd);
8198                         return; /* OK */
8199                 }
8200                 /* received an interrupt, but no rcd */
8201                 err_detail = "dataless";
8202         } else {
8203                 /* received an interrupt, but are not using that context */
8204                 err_detail = "out of range";
8205         }
8206         dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n",
8207                    err_detail, source);
8208 }
8209
8210 /*
8211  * Reserved range interrupt.  Should not be called in normal operation.
8212  */
8213 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source)
8214 {
8215         char name[64];
8216
8217         dd_dev_err(dd, "unexpected %s interrupt\n",
8218                    is_reserved_name(name, sizeof(name), source));
8219 }
8220
8221 static const struct is_table is_table[] = {
8222 /*
8223  * start                 end
8224  *                              name func               interrupt func
8225  */
8226 { IS_GENERAL_ERR_START,  IS_GENERAL_ERR_END,
8227                                 is_misc_err_name,       is_misc_err_int },
8228 { IS_SDMAENG_ERR_START,  IS_SDMAENG_ERR_END,
8229                                 is_sdma_eng_err_name,   is_sdma_eng_err_int },
8230 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END,
8231                                 is_sendctxt_err_name,   is_sendctxt_err_int },
8232 { IS_SDMA_START,             IS_SDMA_IDLE_END,
8233                                 is_sdma_eng_name,       is_sdma_eng_int },
8234 { IS_VARIOUS_START,          IS_VARIOUS_END,
8235                                 is_various_name,        is_various_int },
8236 { IS_DC_START,       IS_DC_END,
8237                                 is_dc_name,             is_dc_int },
8238 { IS_RCVAVAIL_START,     IS_RCVAVAIL_END,
8239                                 is_rcv_avail_name,      is_rcv_avail_int },
8240 { IS_RCVURGENT_START,    IS_RCVURGENT_END,
8241                                 is_rcv_urgent_name,     is_rcv_urgent_int },
8242 { IS_SENDCREDIT_START,   IS_SENDCREDIT_END,
8243                                 is_send_credit_name,    is_send_credit_int},
8244 { IS_RESERVED_START,     IS_RESERVED_END,
8245                                 is_reserved_name,       is_reserved_int},
8246 };
8247
8248 /*
8249  * Interrupt source interrupt - called when the given source has an interrupt.
8250  * Source is a bit index into an array of 64-bit integers.
8251  */
8252 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source)
8253 {
8254         const struct is_table *entry;
8255
8256         /* avoids a double compare by walking the table in-order */
8257         for (entry = &is_table[0]; entry->is_name; entry++) {
8258                 if (source <= entry->end) {
8259                         trace_hfi1_interrupt(dd, entry, source);
8260                         entry->is_int(dd, source - entry->start);
8261                         return;
8262                 }
8263         }
8264         /* fell off the end */
8265         dd_dev_err(dd, "invalid interrupt source %u\n", source);
8266 }
8267
8268 /**
8269  * gerneral_interrupt() -  General interrupt handler
8270  * @irq: MSIx IRQ vector
8271  * @data: hfi1 devdata
8272  *
8273  * This is able to correctly handle all non-threaded interrupts.  Receive
8274  * context DATA IRQs are threaded and are not supported by this handler.
8275  *
8276  */
8277 irqreturn_t general_interrupt(int irq, void *data)
8278 {
8279         struct hfi1_devdata *dd = data;
8280         u64 regs[CCE_NUM_INT_CSRS];
8281         u32 bit;
8282         int i;
8283         irqreturn_t handled = IRQ_NONE;
8284
8285         this_cpu_inc(*dd->int_counter);
8286
8287         /* phase 1: scan and clear all handled interrupts */
8288         for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
8289                 if (dd->gi_mask[i] == 0) {
8290                         regs[i] = 0;    /* used later */
8291                         continue;
8292                 }
8293                 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) &
8294                                 dd->gi_mask[i];
8295                 /* only clear if anything is set */
8296                 if (regs[i])
8297                         write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]);
8298         }
8299
8300         /* phase 2: call the appropriate handler */
8301         for_each_set_bit(bit, (unsigned long *)&regs[0],
8302                          CCE_NUM_INT_CSRS * 64) {
8303                 is_interrupt(dd, bit);
8304                 handled = IRQ_HANDLED;
8305         }
8306
8307         return handled;
8308 }
8309
8310 irqreturn_t sdma_interrupt(int irq, void *data)
8311 {
8312         struct sdma_engine *sde = data;
8313         struct hfi1_devdata *dd = sde->dd;
8314         u64 status;
8315
8316 #ifdef CONFIG_SDMA_VERBOSITY
8317         dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
8318                    slashstrip(__FILE__), __LINE__, __func__);
8319         sdma_dumpstate(sde);
8320 #endif
8321
8322         this_cpu_inc(*dd->int_counter);
8323
8324         /* This read_csr is really bad in the hot path */
8325         status = read_csr(dd,
8326                           CCE_INT_STATUS + (8 * (IS_SDMA_START / 64)))
8327                           & sde->imask;
8328         if (likely(status)) {
8329                 /* clear the interrupt(s) */
8330                 write_csr(dd,
8331                           CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)),
8332                           status);
8333
8334                 /* handle the interrupt(s) */
8335                 sdma_engine_interrupt(sde, status);
8336         } else {
8337                 dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
8338                                         sde->this_idx);
8339         }
8340         return IRQ_HANDLED;
8341 }
8342
8343 /*
8344  * Clear the receive interrupt.  Use a read of the interrupt clear CSR
8345  * to insure that the write completed.  This does NOT guarantee that
8346  * queued DMA writes to memory from the chip are pushed.
8347  */
8348 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd)
8349 {
8350         struct hfi1_devdata *dd = rcd->dd;
8351         u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg);
8352
8353         mmiowb();       /* make sure everything before is written */
8354         write_csr(dd, addr, rcd->imask);
8355         /* force the above write on the chip and get a value back */
8356         (void)read_csr(dd, addr);
8357 }
8358
8359 /* force the receive interrupt */
8360 void force_recv_intr(struct hfi1_ctxtdata *rcd)
8361 {
8362         write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask);
8363 }
8364
8365 /*
8366  * Return non-zero if a packet is present.
8367  *
8368  * This routine is called when rechecking for packets after the RcvAvail
8369  * interrupt has been cleared down.  First, do a quick check of memory for
8370  * a packet present.  If not found, use an expensive CSR read of the context
8371  * tail to determine the actual tail.  The CSR read is necessary because there
8372  * is no method to push pending DMAs to memory other than an interrupt and we
8373  * are trying to determine if we need to force an interrupt.
8374  */
8375 static inline int check_packet_present(struct hfi1_ctxtdata *rcd)
8376 {
8377         u32 tail;
8378         int present;
8379
8380         if (!rcd->rcvhdrtail_kvaddr)
8381                 present = (rcd->seq_cnt ==
8382                                 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd))));
8383         else /* is RDMA rtail */
8384                 present = (rcd->head != get_rcvhdrtail(rcd));
8385
8386         if (present)
8387                 return 1;
8388
8389         /* fall back to a CSR read, correct indpendent of DMA_RTAIL */
8390         tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
8391         return rcd->head != tail;
8392 }
8393
8394 /*
8395  * Receive packet IRQ handler.  This routine expects to be on its own IRQ.
8396  * This routine will try to handle packets immediately (latency), but if
8397  * it finds too many, it will invoke the thread handler (bandwitdh).  The
8398  * chip receive interrupt is *not* cleared down until this or the thread (if
8399  * invoked) is finished.  The intent is to avoid extra interrupts while we
8400  * are processing packets anyway.
8401  */
8402 irqreturn_t receive_context_interrupt(int irq, void *data)
8403 {
8404         struct hfi1_ctxtdata *rcd = data;
8405         struct hfi1_devdata *dd = rcd->dd;
8406         int disposition;
8407         int present;
8408
8409         trace_hfi1_receive_interrupt(dd, rcd);
8410         this_cpu_inc(*dd->int_counter);
8411         aspm_ctx_disable(rcd);
8412
8413         /* receive interrupt remains blocked while processing packets */
8414         disposition = rcd->do_interrupt(rcd, 0);
8415
8416         /*
8417          * Too many packets were seen while processing packets in this
8418          * IRQ handler.  Invoke the handler thread.  The receive interrupt
8419          * remains blocked.
8420          */
8421         if (disposition == RCV_PKT_LIMIT)
8422                 return IRQ_WAKE_THREAD;
8423
8424         /*
8425          * The packet processor detected no more packets.  Clear the receive
8426          * interrupt and recheck for a packet packet that may have arrived
8427          * after the previous check and interrupt clear.  If a packet arrived,
8428          * force another interrupt.
8429          */
8430         clear_recv_intr(rcd);
8431         present = check_packet_present(rcd);
8432         if (present)
8433                 force_recv_intr(rcd);
8434
8435         return IRQ_HANDLED;
8436 }
8437
8438 /*
8439  * Receive packet thread handler.  This expects to be invoked with the
8440  * receive interrupt still blocked.
8441  */
8442 irqreturn_t receive_context_thread(int irq, void *data)
8443 {
8444         struct hfi1_ctxtdata *rcd = data;
8445         int present;
8446
8447         /* receive interrupt is still blocked from the IRQ handler */
8448         (void)rcd->do_interrupt(rcd, 1);
8449
8450         /*
8451          * The packet processor will only return if it detected no more
8452          * packets.  Hold IRQs here so we can safely clear the interrupt and
8453          * recheck for a packet that may have arrived after the previous
8454          * check and the interrupt clear.  If a packet arrived, force another
8455          * interrupt.
8456          */
8457         local_irq_disable();
8458         clear_recv_intr(rcd);
8459         present = check_packet_present(rcd);
8460         if (present)
8461                 force_recv_intr(rcd);
8462         local_irq_enable();
8463
8464         return IRQ_HANDLED;
8465 }
8466
8467 /* ========================================================================= */
8468
8469 u32 read_physical_state(struct hfi1_devdata *dd)
8470 {
8471         u64 reg;
8472
8473         reg = read_csr(dd, DC_DC8051_STS_CUR_STATE);
8474         return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT)
8475                                 & DC_DC8051_STS_CUR_STATE_PORT_MASK;
8476 }
8477
8478 u32 read_logical_state(struct hfi1_devdata *dd)
8479 {
8480         u64 reg;
8481
8482         reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8483         return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT)
8484                                 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK;
8485 }
8486
8487 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate)
8488 {
8489         u64 reg;
8490
8491         reg = read_csr(dd, DCC_CFG_PORT_CONFIG);
8492         /* clear current state, set new state */
8493         reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK;
8494         reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT;
8495         write_csr(dd, DCC_CFG_PORT_CONFIG, reg);
8496 }
8497
8498 /*
8499  * Use the 8051 to read a LCB CSR.
8500  */
8501 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data)
8502 {
8503         u32 regno;
8504         int ret;
8505
8506         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
8507                 if (acquire_lcb_access(dd, 0) == 0) {
8508                         *data = read_csr(dd, addr);
8509                         release_lcb_access(dd, 0);
8510                         return 0;
8511                 }
8512                 return -EBUSY;
8513         }
8514
8515         /* register is an index of LCB registers: (offset - base) / 8 */
8516         regno = (addr - DC_LCB_CFG_RUN) >> 3;
8517         ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data);
8518         if (ret != HCMD_SUCCESS)
8519                 return -EBUSY;
8520         return 0;
8521 }
8522
8523 /*
8524  * Provide a cache for some of the LCB registers in case the LCB is
8525  * unavailable.
8526  * (The LCB is unavailable in certain link states, for example.)
8527  */
8528 struct lcb_datum {
8529         u32 off;
8530         u64 val;
8531 };
8532
8533 static struct lcb_datum lcb_cache[] = {
8534         { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0},
8535         { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 },
8536         { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 },
8537 };
8538
8539 static void update_lcb_cache(struct hfi1_devdata *dd)
8540 {
8541         int i;
8542         int ret;
8543         u64 val;
8544
8545         for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8546                 ret = read_lcb_csr(dd, lcb_cache[i].off, &val);
8547
8548                 /* Update if we get good data */
8549                 if (likely(ret != -EBUSY))
8550                         lcb_cache[i].val = val;
8551         }
8552 }
8553
8554 static int read_lcb_cache(u32 off, u64 *val)
8555 {
8556         int i;
8557
8558         for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) {
8559                 if (lcb_cache[i].off == off) {
8560                         *val = lcb_cache[i].val;
8561                         return 0;
8562                 }
8563         }
8564
8565         pr_warn("%s bad offset 0x%x\n", __func__, off);
8566         return -1;
8567 }
8568
8569 /*
8570  * Read an LCB CSR.  Access may not be in host control, so check.
8571  * Return 0 on success, -EBUSY on failure.
8572  */
8573 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data)
8574 {
8575         struct hfi1_pportdata *ppd = dd->pport;
8576
8577         /* if up, go through the 8051 for the value */
8578         if (ppd->host_link_state & HLS_UP)
8579                 return read_lcb_via_8051(dd, addr, data);
8580         /* if going up or down, check the cache, otherwise, no access */
8581         if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) {
8582                 if (read_lcb_cache(addr, data))
8583                         return -EBUSY;
8584                 return 0;
8585         }
8586
8587         /* otherwise, host has access */
8588         *data = read_csr(dd, addr);
8589         return 0;
8590 }
8591
8592 /*
8593  * Use the 8051 to write a LCB CSR.
8594  */
8595 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data)
8596 {
8597         u32 regno;
8598         int ret;
8599
8600         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR ||
8601             (dd->dc8051_ver < dc8051_ver(0, 20, 0))) {
8602                 if (acquire_lcb_access(dd, 0) == 0) {
8603                         write_csr(dd, addr, data);
8604                         release_lcb_access(dd, 0);
8605                         return 0;
8606                 }
8607                 return -EBUSY;
8608         }
8609
8610         /* register is an index of LCB registers: (offset - base) / 8 */
8611         regno = (addr - DC_LCB_CFG_RUN) >> 3;
8612         ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data);
8613         if (ret != HCMD_SUCCESS)
8614                 return -EBUSY;
8615         return 0;
8616 }
8617
8618 /*
8619  * Write an LCB CSR.  Access may not be in host control, so check.
8620  * Return 0 on success, -EBUSY on failure.
8621  */
8622 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
8623 {
8624         struct hfi1_pportdata *ppd = dd->pport;
8625
8626         /* if up, go through the 8051 for the value */
8627         if (ppd->host_link_state & HLS_UP)
8628                 return write_lcb_via_8051(dd, addr, data);
8629         /* if going up or down, no access */
8630         if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE))
8631                 return -EBUSY;
8632         /* otherwise, host has access */
8633         write_csr(dd, addr, data);
8634         return 0;
8635 }
8636
8637 /*
8638  * Returns:
8639  *      < 0 = Linux error, not able to get access
8640  *      > 0 = 8051 command RETURN_CODE
8641  */
8642 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
8643                            u64 *out_data)
8644 {
8645         u64 reg, completed;
8646         int return_code;
8647         unsigned long timeout;
8648
8649         hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
8650
8651         mutex_lock(&dd->dc8051_lock);
8652
8653         /* We can't send any commands to the 8051 if it's in reset */
8654         if (dd->dc_shutdown) {
8655                 return_code = -ENODEV;
8656                 goto fail;
8657         }
8658
8659         /*
8660          * If an 8051 host command timed out previously, then the 8051 is
8661          * stuck.
8662          *
8663          * On first timeout, attempt to reset and restart the entire DC
8664          * block (including 8051). (Is this too big of a hammer?)
8665          *
8666          * If the 8051 times out a second time, the reset did not bring it
8667          * back to healthy life. In that case, fail any subsequent commands.
8668          */
8669         if (dd->dc8051_timed_out) {
8670                 if (dd->dc8051_timed_out > 1) {
8671                         dd_dev_err(dd,
8672                                    "Previous 8051 host command timed out, skipping command %u\n",
8673                                    type);
8674                         return_code = -ENXIO;
8675                         goto fail;
8676                 }
8677                 _dc_shutdown(dd);
8678                 _dc_start(dd);
8679         }
8680
8681         /*
8682          * If there is no timeout, then the 8051 command interface is
8683          * waiting for a command.
8684          */
8685
8686         /*
8687          * When writing a LCB CSR, out_data contains the full value to
8688          * to be written, while in_data contains the relative LCB
8689          * address in 7:0.  Do the work here, rather than the caller,
8690          * of distrubting the write data to where it needs to go:
8691          *
8692          * Write data
8693          *   39:00 -> in_data[47:8]
8694          *   47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE
8695          *   63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA
8696          */
8697         if (type == HCMD_WRITE_LCB_CSR) {
8698                 in_data |= ((*out_data) & 0xffffffffffull) << 8;
8699                 /* must preserve COMPLETED - it is tied to hardware */
8700                 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0);
8701                 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK;
8702                 reg |= ((((*out_data) >> 40) & 0xff) <<
8703                                 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT)
8704                       | ((((*out_data) >> 48) & 0xffff) <<
8705                                 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT);
8706                 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg);
8707         }
8708
8709         /*
8710          * Do two writes: the first to stabilize the type and req_data, the
8711          * second to activate.
8712          */
8713         reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK)
8714                         << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT
8715                 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK)
8716                         << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT;
8717         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8718         reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK;
8719         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg);
8720
8721         /* wait for completion, alternate: interrupt */
8722         timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT);
8723         while (1) {
8724                 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1);
8725                 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK;
8726                 if (completed)
8727                         break;
8728                 if (time_after(jiffies, timeout)) {
8729                         dd->dc8051_timed_out++;
8730                         dd_dev_err(dd, "8051 host command %u timeout\n", type);
8731                         if (out_data)
8732                                 *out_data = 0;
8733                         return_code = -ETIMEDOUT;
8734                         goto fail;
8735                 }
8736                 udelay(2);
8737         }
8738
8739         if (out_data) {
8740                 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT)
8741                                 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK;
8742                 if (type == HCMD_READ_LCB_CSR) {
8743                         /* top 16 bits are in a different register */
8744                         *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1)
8745                                 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK)
8746                                 << (48
8747                                     - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT);
8748                 }
8749         }
8750         return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT)
8751                                 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK;
8752         dd->dc8051_timed_out = 0;
8753         /*
8754          * Clear command for next user.
8755          */
8756         write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
8757
8758 fail:
8759         mutex_unlock(&dd->dc8051_lock);
8760         return return_code;
8761 }
8762
8763 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
8764 {
8765         return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
8766 }
8767
8768 int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
8769                      u8 lane_id, u32 config_data)
8770 {
8771         u64 data;
8772         int ret;
8773
8774         data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
8775                 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
8776                 | (u64)config_data << LOAD_DATA_DATA_SHIFT;
8777         ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
8778         if (ret != HCMD_SUCCESS) {
8779                 dd_dev_err(dd,
8780                            "load 8051 config: field id %d, lane %d, err %d\n",
8781                            (int)field_id, (int)lane_id, ret);
8782         }
8783         return ret;
8784 }
8785
8786 /*
8787  * Read the 8051 firmware "registers".  Use the RAM directly.  Always
8788  * set the result, even on error.
8789  * Return 0 on success, -errno on failure
8790  */
8791 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id,
8792                      u32 *result)
8793 {
8794         u64 big_data;
8795         u32 addr;
8796         int ret;
8797
8798         /* address start depends on the lane_id */
8799         if (lane_id < 4)
8800                 addr = (4 * NUM_GENERAL_FIELDS)
8801                         + (lane_id * 4 * NUM_LANE_FIELDS);
8802         else
8803                 addr = 0;
8804         addr += field_id * 4;
8805
8806         /* read is in 8-byte chunks, hardware will truncate the address down */
8807         ret = read_8051_data(dd, addr, 8, &big_data);
8808
8809         if (ret == 0) {
8810                 /* extract the 4 bytes we want */
8811                 if (addr & 0x4)
8812                         *result = (u32)(big_data >> 32);
8813                 else
8814                         *result = (u32)big_data;
8815         } else {
8816                 *result = 0;
8817                 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n",
8818                            __func__, lane_id, field_id);
8819         }
8820
8821         return ret;
8822 }
8823
8824 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management,
8825                               u8 continuous)
8826 {
8827         u32 frame;
8828
8829         frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT
8830                 | power_management << POWER_MANAGEMENT_SHIFT;
8831         return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY,
8832                                 GENERAL_CONFIG, frame);
8833 }
8834
8835 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu,
8836                                  u16 vl15buf, u8 crc_sizes)
8837 {
8838         u32 frame;
8839
8840         frame = (u32)vau << VAU_SHIFT
8841                 | (u32)z << Z_SHIFT
8842                 | (u32)vcu << VCU_SHIFT
8843                 | (u32)vl15buf << VL15BUF_SHIFT
8844                 | (u32)crc_sizes << CRC_SIZES_SHIFT;
8845         return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC,
8846                                 GENERAL_CONFIG, frame);
8847 }
8848
8849 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits,
8850                                     u8 *flag_bits, u16 *link_widths)
8851 {
8852         u32 frame;
8853
8854         read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8855                          &frame);
8856         *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK;
8857         *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK;
8858         *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8859 }
8860
8861 static int write_vc_local_link_mode(struct hfi1_devdata *dd,
8862                                     u8 misc_bits,
8863                                     u8 flag_bits,
8864                                     u16 link_widths)
8865 {
8866         u32 frame;
8867
8868         frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT
8869                 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT
8870                 | (u32)link_widths << LINK_WIDTH_SHIFT;
8871         return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG,
8872                      frame);
8873 }
8874
8875 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id,
8876                                  u8 device_rev)
8877 {
8878         u32 frame;
8879
8880         frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT)
8881                 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT);
8882         return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame);
8883 }
8884
8885 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id,
8886                                   u8 *device_rev)
8887 {
8888         u32 frame;
8889
8890         read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame);
8891         *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK;
8892         *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT)
8893                         & REMOTE_DEVICE_REV_MASK;
8894 }
8895
8896 int write_host_interface_version(struct hfi1_devdata *dd, u8 version)
8897 {
8898         u32 frame;
8899         u32 mask;
8900
8901         mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT);
8902         read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame);
8903         /* Clear, then set field */
8904         frame &= ~mask;
8905         frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT);
8906         return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
8907                                 frame);
8908 }
8909
8910 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
8911                       u8 *ver_patch)
8912 {
8913         u32 frame;
8914
8915         read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame);
8916         *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) &
8917                 STS_FM_VERSION_MAJOR_MASK;
8918         *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) &
8919                 STS_FM_VERSION_MINOR_MASK;
8920
8921         read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame);
8922         *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) &
8923                 STS_FM_VERSION_PATCH_MASK;
8924 }
8925
8926 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management,
8927                                u8 *continuous)
8928 {
8929         u32 frame;
8930
8931         read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame);
8932         *power_management = (frame >> POWER_MANAGEMENT_SHIFT)
8933                                         & POWER_MANAGEMENT_MASK;
8934         *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT)
8935                                         & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK;
8936 }
8937
8938 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z,
8939                                   u8 *vcu, u16 *vl15buf, u8 *crc_sizes)
8940 {
8941         u32 frame;
8942
8943         read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame);
8944         *vau = (frame >> VAU_SHIFT) & VAU_MASK;
8945         *z = (frame >> Z_SHIFT) & Z_MASK;
8946         *vcu = (frame >> VCU_SHIFT) & VCU_MASK;
8947         *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK;
8948         *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK;
8949 }
8950
8951 static void read_vc_remote_link_width(struct hfi1_devdata *dd,
8952                                       u8 *remote_tx_rate,
8953                                       u16 *link_widths)
8954 {
8955         u32 frame;
8956
8957         read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG,
8958                          &frame);
8959         *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT)
8960                                 & REMOTE_TX_RATE_MASK;
8961         *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK;
8962 }
8963
8964 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx)
8965 {
8966         u32 frame;
8967
8968         read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame);
8969         *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK;
8970 }
8971
8972 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls)
8973 {
8974         read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls);
8975 }
8976
8977 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs)
8978 {
8979         read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs);
8980 }
8981
8982 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality)
8983 {
8984         u32 frame;
8985         int ret;
8986
8987         *link_quality = 0;
8988         if (dd->pport->host_link_state & HLS_UP) {
8989                 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG,
8990                                        &frame);
8991                 if (ret == 0)
8992                         *link_quality = (frame >> LINK_QUALITY_SHIFT)
8993                                                 & LINK_QUALITY_MASK;
8994         }
8995 }
8996
8997 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc)
8998 {
8999         u32 frame;
9000
9001         read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame);
9002         *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK;
9003 }
9004
9005 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr)
9006 {
9007         u32 frame;
9008
9009         read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame);
9010         *ldr = (frame & 0xff);
9011 }
9012
9013 static int read_tx_settings(struct hfi1_devdata *dd,
9014                             u8 *enable_lane_tx,
9015                             u8 *tx_polarity_inversion,
9016                             u8 *rx_polarity_inversion,
9017                             u8 *max_rate)
9018 {
9019         u32 frame;
9020         int ret;
9021
9022         ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame);
9023         *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT)
9024                                 & ENABLE_LANE_TX_MASK;
9025         *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT)
9026                                 & TX_POLARITY_INVERSION_MASK;
9027         *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT)
9028                                 & RX_POLARITY_INVERSION_MASK;
9029         *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK;
9030         return ret;
9031 }
9032
9033 static int write_tx_settings(struct hfi1_devdata *dd,
9034                              u8 enable_lane_tx,
9035                              u8 tx_polarity_inversion,
9036                              u8 rx_polarity_inversion,
9037                              u8 max_rate)
9038 {
9039         u32 frame;
9040
9041         /* no need to mask, all variable sizes match field widths */
9042         frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT
9043                 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT
9044                 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT
9045                 | max_rate << MAX_RATE_SHIFT;
9046         return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame);
9047 }
9048
9049 /*
9050  * Read an idle LCB message.
9051  *
9052  * Returns 0 on success, -EINVAL on error
9053  */
9054 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out)
9055 {
9056         int ret;
9057
9058         ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out);
9059         if (ret != HCMD_SUCCESS) {
9060                 dd_dev_err(dd, "read idle message: type %d, err %d\n",
9061                            (u32)type, ret);
9062                 return -EINVAL;
9063         }
9064         dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out);
9065         /* return only the payload as we already know the type */
9066         *data_out >>= IDLE_PAYLOAD_SHIFT;
9067         return 0;
9068 }
9069
9070 /*
9071  * Read an idle SMA message.  To be done in response to a notification from
9072  * the 8051.
9073  *
9074  * Returns 0 on success, -EINVAL on error
9075  */
9076 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data)
9077 {
9078         return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT,
9079                                  data);
9080 }
9081
9082 /*
9083  * Send an idle LCB message.
9084  *
9085  * Returns 0 on success, -EINVAL on error
9086  */
9087 static int send_idle_message(struct hfi1_devdata *dd, u64 data)
9088 {
9089         int ret;
9090
9091         dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data);
9092         ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL);
9093         if (ret != HCMD_SUCCESS) {
9094                 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n",
9095                            data, ret);
9096                 return -EINVAL;
9097         }
9098         return 0;
9099 }
9100
9101 /*
9102  * Send an idle SMA message.
9103  *
9104  * Returns 0 on success, -EINVAL on error
9105  */
9106 int send_idle_sma(struct hfi1_devdata *dd, u64 message)
9107 {
9108         u64 data;
9109
9110         data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) |
9111                 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT);
9112         return send_idle_message(dd, data);
9113 }
9114
9115 /*
9116  * Initialize the LCB then do a quick link up.  This may or may not be
9117  * in loopback.
9118  *
9119  * return 0 on success, -errno on error
9120  */
9121 static int do_quick_linkup(struct hfi1_devdata *dd)
9122 {
9123         int ret;
9124
9125         lcb_shutdown(dd, 0);
9126
9127         if (loopback) {
9128                 /* LCB_CFG_LOOPBACK.VAL = 2 */
9129                 /* LCB_CFG_LANE_WIDTH.VAL = 0 */
9130                 write_csr(dd, DC_LCB_CFG_LOOPBACK,
9131                           IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT);
9132                 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
9133         }
9134
9135         /* start the LCBs */
9136         /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */
9137         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
9138
9139         /* simulator only loopback steps */
9140         if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
9141                 /* LCB_CFG_RUN.EN = 1 */
9142                 write_csr(dd, DC_LCB_CFG_RUN,
9143                           1ull << DC_LCB_CFG_RUN_EN_SHIFT);
9144
9145                 ret = wait_link_transfer_active(dd, 10);
9146                 if (ret)
9147                         return ret;
9148
9149                 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP,
9150                           1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT);
9151         }
9152
9153         if (!loopback) {
9154                 /*
9155                  * When doing quick linkup and not in loopback, both
9156                  * sides must be done with LCB set-up before either
9157                  * starts the quick linkup.  Put a delay here so that
9158                  * both sides can be started and have a chance to be
9159                  * done with LCB set up before resuming.
9160                  */
9161                 dd_dev_err(dd,
9162                            "Pausing for peer to be finished with LCB set up\n");
9163                 msleep(5000);
9164                 dd_dev_err(dd, "Continuing with quick linkup\n");
9165         }
9166
9167         write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
9168         set_8051_lcb_access(dd);
9169
9170         /*
9171          * State "quick" LinkUp request sets the physical link state to
9172          * LinkUp without a verify capability sequence.
9173          * This state is in simulator v37 and later.
9174          */
9175         ret = set_physical_link_state(dd, PLS_QUICK_LINKUP);
9176         if (ret != HCMD_SUCCESS) {
9177                 dd_dev_err(dd,
9178                            "%s: set physical link state to quick LinkUp failed with return %d\n",
9179                            __func__, ret);
9180
9181                 set_host_lcb_access(dd);
9182                 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
9183
9184                 if (ret >= 0)
9185                         ret = -EINVAL;
9186                 return ret;
9187         }
9188
9189         return 0; /* success */
9190 }
9191
9192 /*
9193  * Do all special steps to set up loopback.
9194  */
9195 static int init_loopback(struct hfi1_devdata *dd)
9196 {
9197         dd_dev_info(dd, "Entering loopback mode\n");
9198
9199         /* all loopbacks should disable self GUID check */
9200         write_csr(dd, DC_DC8051_CFG_MODE,
9201                   (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK));
9202
9203         /*
9204          * The simulator has only one loopback option - LCB.  Switch
9205          * to that option, which includes quick link up.
9206          *
9207          * Accept all valid loopback values.
9208          */
9209         if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) &&
9210             (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB ||
9211              loopback == LOOPBACK_CABLE)) {
9212                 loopback = LOOPBACK_LCB;
9213                 quick_linkup = 1;
9214                 return 0;
9215         }
9216
9217         /*
9218          * SerDes loopback init sequence is handled in set_local_link_attributes
9219          */
9220         if (loopback == LOOPBACK_SERDES)
9221                 return 0;
9222
9223         /* LCB loopback - handled at poll time */
9224         if (loopback == LOOPBACK_LCB) {
9225                 quick_linkup = 1; /* LCB is always quick linkup */
9226
9227                 /* not supported in emulation due to emulation RTL changes */
9228                 if (dd->icode == ICODE_FPGA_EMULATION) {
9229                         dd_dev_err(dd,
9230                                    "LCB loopback not supported in emulation\n");
9231                         return -EINVAL;
9232                 }
9233                 return 0;
9234         }
9235
9236         /* external cable loopback requires no extra steps */
9237         if (loopback == LOOPBACK_CABLE)
9238                 return 0;
9239
9240         dd_dev_err(dd, "Invalid loopback mode %d\n", loopback);
9241         return -EINVAL;
9242 }
9243
9244 /*
9245  * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits
9246  * used in the Verify Capability link width attribute.
9247  */
9248 static u16 opa_to_vc_link_widths(u16 opa_widths)
9249 {
9250         int i;
9251         u16 result = 0;
9252
9253         static const struct link_bits {
9254                 u16 from;
9255                 u16 to;
9256         } opa_link_xlate[] = {
9257                 { OPA_LINK_WIDTH_1X, 1 << (1 - 1)  },
9258                 { OPA_LINK_WIDTH_2X, 1 << (2 - 1)  },
9259                 { OPA_LINK_WIDTH_3X, 1 << (3 - 1)  },
9260                 { OPA_LINK_WIDTH_4X, 1 << (4 - 1)  },
9261         };
9262
9263         for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) {
9264                 if (opa_widths & opa_link_xlate[i].from)
9265                         result |= opa_link_xlate[i].to;
9266         }
9267         return result;
9268 }
9269
9270 /*
9271  * Set link attributes before moving to polling.
9272  */
9273 static int set_local_link_attributes(struct hfi1_pportdata *ppd)
9274 {
9275         struct hfi1_devdata *dd = ppd->dd;
9276         u8 enable_lane_tx;
9277         u8 tx_polarity_inversion;
9278         u8 rx_polarity_inversion;
9279         int ret;
9280         u32 misc_bits = 0;
9281         /* reset our fabric serdes to clear any lingering problems */
9282         fabric_serdes_reset(dd);
9283
9284         /* set the local tx rate - need to read-modify-write */
9285         ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion,
9286                                &rx_polarity_inversion, &ppd->local_tx_rate);
9287         if (ret)
9288                 goto set_local_link_attributes_fail;
9289
9290         if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) {
9291                 /* set the tx rate to the fastest enabled */
9292                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9293                         ppd->local_tx_rate = 1;
9294                 else
9295                         ppd->local_tx_rate = 0;
9296         } else {
9297                 /* set the tx rate to all enabled */
9298                 ppd->local_tx_rate = 0;
9299                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G)
9300                         ppd->local_tx_rate |= 2;
9301                 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G)
9302                         ppd->local_tx_rate |= 1;
9303         }
9304
9305         enable_lane_tx = 0xF; /* enable all four lanes */
9306         ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion,
9307                                 rx_polarity_inversion, ppd->local_tx_rate);
9308         if (ret != HCMD_SUCCESS)
9309                 goto set_local_link_attributes_fail;
9310
9311         ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
9312         if (ret != HCMD_SUCCESS) {
9313                 dd_dev_err(dd,
9314                            "Failed to set host interface version, return 0x%x\n",
9315                            ret);
9316                 goto set_local_link_attributes_fail;
9317         }
9318
9319         /*
9320          * DC supports continuous updates.
9321          */
9322         ret = write_vc_local_phy(dd,
9323                                  0 /* no power management */,
9324                                  1 /* continuous updates */);
9325         if (ret != HCMD_SUCCESS)
9326                 goto set_local_link_attributes_fail;
9327
9328         /* z=1 in the next call: AU of 0 is not supported by the hardware */
9329         ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init,
9330                                     ppd->port_crc_mode_enabled);
9331         if (ret != HCMD_SUCCESS)
9332                 goto set_local_link_attributes_fail;
9333
9334         /*
9335          * SerDes loopback init sequence requires
9336          * setting bit 0 of MISC_CONFIG_BITS
9337          */
9338         if (loopback == LOOPBACK_SERDES)
9339                 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT;
9340
9341         /*
9342          * An external device configuration request is used to reset the LCB
9343          * to retry to obtain operational lanes when the first attempt is
9344          * unsuccesful.
9345          */
9346         if (dd->dc8051_ver >= dc8051_ver(1, 25, 0))
9347                 misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT;
9348
9349         ret = write_vc_local_link_mode(dd, misc_bits, 0,
9350                                        opa_to_vc_link_widths(
9351                                                 ppd->link_width_enabled));
9352         if (ret != HCMD_SUCCESS)
9353                 goto set_local_link_attributes_fail;
9354
9355         /* let peer know who we are */
9356         ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev);
9357         if (ret == HCMD_SUCCESS)
9358                 return 0;
9359
9360 set_local_link_attributes_fail:
9361         dd_dev_err(dd,
9362                    "Failed to set local link attributes, return 0x%x\n",
9363                    ret);
9364         return ret;
9365 }
9366
9367 /*
9368  * Call this to start the link.
9369  * Do not do anything if the link is disabled.
9370  * Returns 0 if link is disabled, moved to polling, or the driver is not ready.
9371  */
9372 int start_link(struct hfi1_pportdata *ppd)
9373 {
9374         /*
9375          * Tune the SerDes to a ballpark setting for optimal signal and bit
9376          * error rate.  Needs to be done before starting the link.
9377          */
9378         tune_serdes(ppd);
9379
9380         if (!ppd->driver_link_ready) {
9381                 dd_dev_info(ppd->dd,
9382                             "%s: stopping link start because driver is not ready\n",
9383                             __func__);
9384                 return 0;
9385         }
9386
9387         /*
9388          * FULL_MGMT_P_KEY is cleared from the pkey table, so that the
9389          * pkey table can be configured properly if the HFI unit is connected
9390          * to switch port with MgmtAllowed=NO
9391          */
9392         clear_full_mgmt_pkey(ppd);
9393
9394         return set_link_state(ppd, HLS_DN_POLL);
9395 }
9396
9397 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd)
9398 {
9399         struct hfi1_devdata *dd = ppd->dd;
9400         u64 mask;
9401         unsigned long timeout;
9402
9403         /*
9404          * Some QSFP cables have a quirk that asserts the IntN line as a side
9405          * effect of power up on plug-in. We ignore this false positive
9406          * interrupt until the module has finished powering up by waiting for
9407          * a minimum timeout of the module inrush initialization time of
9408          * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the
9409          * module have stabilized.
9410          */
9411         msleep(500);
9412
9413         /*
9414          * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1)
9415          */
9416         timeout = jiffies + msecs_to_jiffies(2000);
9417         while (1) {
9418                 mask = read_csr(dd, dd->hfi1_id ?
9419                                 ASIC_QSFP2_IN : ASIC_QSFP1_IN);
9420                 if (!(mask & QSFP_HFI0_INT_N))
9421                         break;
9422                 if (time_after(jiffies, timeout)) {
9423                         dd_dev_info(dd, "%s: No IntN detected, reset complete\n",
9424                                     __func__);
9425                         break;
9426                 }
9427                 udelay(2);
9428         }
9429 }
9430
9431 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable)
9432 {
9433         struct hfi1_devdata *dd = ppd->dd;
9434         u64 mask;
9435
9436         mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK);
9437         if (enable) {
9438                 /*
9439                  * Clear the status register to avoid an immediate interrupt
9440                  * when we re-enable the IntN pin
9441                  */
9442                 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9443                           QSFP_HFI0_INT_N);
9444                 mask |= (u64)QSFP_HFI0_INT_N;
9445         } else {
9446                 mask &= ~(u64)QSFP_HFI0_INT_N;
9447         }
9448         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask);
9449 }
9450
9451 int reset_qsfp(struct hfi1_pportdata *ppd)
9452 {
9453         struct hfi1_devdata *dd = ppd->dd;
9454         u64 mask, qsfp_mask;
9455
9456         /* Disable INT_N from triggering QSFP interrupts */
9457         set_qsfp_int_n(ppd, 0);
9458
9459         /* Reset the QSFP */
9460         mask = (u64)QSFP_HFI0_RESET_N;
9461
9462         qsfp_mask = read_csr(dd,
9463                              dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT);
9464         qsfp_mask &= ~mask;
9465         write_csr(dd,
9466                   dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9467
9468         udelay(10);
9469
9470         qsfp_mask |= mask;
9471         write_csr(dd,
9472                   dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask);
9473
9474         wait_for_qsfp_init(ppd);
9475
9476         /*
9477          * Allow INT_N to trigger the QSFP interrupt to watch
9478          * for alarms and warnings
9479          */
9480         set_qsfp_int_n(ppd, 1);
9481
9482         /*
9483          * After the reset, AOC transmitters are enabled by default. They need
9484          * to be turned off to complete the QSFP setup before they can be
9485          * enabled again.
9486          */
9487         return set_qsfp_tx(ppd, 0);
9488 }
9489
9490 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd,
9491                                         u8 *qsfp_interrupt_status)
9492 {
9493         struct hfi1_devdata *dd = ppd->dd;
9494
9495         if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) ||
9496             (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING))
9497                 dd_dev_err(dd, "%s: QSFP cable temperature too high\n",
9498                            __func__);
9499
9500         if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) ||
9501             (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING))
9502                 dd_dev_err(dd, "%s: QSFP cable temperature too low\n",
9503                            __func__);
9504
9505         /*
9506          * The remaining alarms/warnings don't matter if the link is down.
9507          */
9508         if (ppd->host_link_state & HLS_DOWN)
9509                 return 0;
9510
9511         if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) ||
9512             (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING))
9513                 dd_dev_err(dd, "%s: QSFP supply voltage too high\n",
9514                            __func__);
9515
9516         if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) ||
9517             (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING))
9518                 dd_dev_err(dd, "%s: QSFP supply voltage too low\n",
9519                            __func__);
9520
9521         /* Byte 2 is vendor specific */
9522
9523         if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) ||
9524             (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING))
9525                 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n",
9526                            __func__);
9527
9528         if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) ||
9529             (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING))
9530                 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n",
9531                            __func__);
9532
9533         if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) ||
9534             (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING))
9535                 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n",
9536                            __func__);
9537
9538         if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) ||
9539             (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING))
9540                 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n",
9541                            __func__);
9542
9543         if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) ||
9544             (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING))
9545                 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n",
9546                            __func__);
9547
9548         if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) ||
9549             (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING))
9550                 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n",
9551                            __func__);
9552
9553         if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) ||
9554             (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING))
9555                 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n",
9556                            __func__);
9557
9558         if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) ||
9559             (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING))
9560                 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n",
9561                            __func__);
9562
9563         if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) ||
9564             (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING))
9565                 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n",
9566                            __func__);
9567
9568         if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) ||
9569             (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING))
9570                 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n",
9571                            __func__);
9572
9573         if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) ||
9574             (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING))
9575                 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n",
9576                            __func__);
9577
9578         if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) ||
9579             (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING))
9580                 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n",
9581                            __func__);
9582
9583         /* Bytes 9-10 and 11-12 are reserved */
9584         /* Bytes 13-15 are vendor specific */
9585
9586         return 0;
9587 }
9588
9589 /* This routine will only be scheduled if the QSFP module present is asserted */
9590 void qsfp_event(struct work_struct *work)
9591 {
9592         struct qsfp_data *qd;
9593         struct hfi1_pportdata *ppd;
9594         struct hfi1_devdata *dd;
9595
9596         qd = container_of(work, struct qsfp_data, qsfp_work);
9597         ppd = qd->ppd;
9598         dd = ppd->dd;
9599
9600         /* Sanity check */
9601         if (!qsfp_mod_present(ppd))
9602                 return;
9603
9604         if (ppd->host_link_state == HLS_DN_DISABLE) {
9605                 dd_dev_info(ppd->dd,
9606                             "%s: stopping link start because link is disabled\n",
9607                             __func__);
9608                 return;
9609         }
9610
9611         /*
9612          * Turn DC back on after cable has been re-inserted. Up until
9613          * now, the DC has been in reset to save power.
9614          */
9615         dc_start(dd);
9616
9617         if (qd->cache_refresh_required) {
9618                 set_qsfp_int_n(ppd, 0);
9619
9620                 wait_for_qsfp_init(ppd);
9621
9622                 /*
9623                  * Allow INT_N to trigger the QSFP interrupt to watch
9624                  * for alarms and warnings
9625                  */
9626                 set_qsfp_int_n(ppd, 1);
9627
9628                 start_link(ppd);
9629         }
9630
9631         if (qd->check_interrupt_flags) {
9632                 u8 qsfp_interrupt_status[16] = {0,};
9633
9634                 if (one_qsfp_read(ppd, dd->hfi1_id, 6,
9635                                   &qsfp_interrupt_status[0], 16) != 16) {
9636                         dd_dev_info(dd,
9637                                     "%s: Failed to read status of QSFP module\n",
9638                                     __func__);
9639                 } else {
9640                         unsigned long flags;
9641
9642                         handle_qsfp_error_conditions(
9643                                         ppd, qsfp_interrupt_status);
9644                         spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags);
9645                         ppd->qsfp_info.check_interrupt_flags = 0;
9646                         spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock,
9647                                                flags);
9648                 }
9649         }
9650 }
9651
9652 void init_qsfp_int(struct hfi1_devdata *dd)
9653 {
9654         struct hfi1_pportdata *ppd = dd->pport;
9655         u64 qsfp_mask;
9656
9657         qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N);
9658         /* Clear current status to avoid spurious interrupts */
9659         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR,
9660                   qsfp_mask);
9661         write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK,
9662                   qsfp_mask);
9663
9664         set_qsfp_int_n(ppd, 0);
9665
9666         /* Handle active low nature of INT_N and MODPRST_N pins */
9667         if (qsfp_mod_present(ppd))
9668                 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N;
9669         write_csr(dd,
9670                   dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT,
9671                   qsfp_mask);
9672
9673         /* Enable the appropriate QSFP IRQ source */
9674         if (!dd->hfi1_id)
9675                 set_intr_bits(dd, QSFP1_INT, QSFP1_INT, true);
9676         else
9677                 set_intr_bits(dd, QSFP2_INT, QSFP2_INT, true);
9678 }
9679
9680 /*
9681  * Do a one-time initialize of the LCB block.
9682  */
9683 static void init_lcb(struct hfi1_devdata *dd)
9684 {
9685         /* simulator does not correctly handle LCB cclk loopback, skip */
9686         if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
9687                 return;
9688
9689         /* the DC has been reset earlier in the driver load */
9690
9691         /* set LCB for cclk loopback on the port */
9692         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01);
9693         write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00);
9694         write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00);
9695         write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
9696         write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08);
9697         write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02);
9698         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00);
9699 }
9700
9701 /*
9702  * Perform a test read on the QSFP.  Return 0 on success, -ERRNO
9703  * on error.
9704  */
9705 static int test_qsfp_read(struct hfi1_pportdata *ppd)
9706 {
9707         int ret;
9708         u8 status;
9709
9710         /*
9711          * Report success if not a QSFP or, if it is a QSFP, but the cable is
9712          * not present
9713          */
9714         if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd))
9715                 return 0;
9716
9717         /* read byte 2, the status byte */
9718         ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1);
9719         if (ret < 0)
9720                 return ret;
9721         if (ret != 1)
9722                 return -EIO;
9723
9724         return 0; /* success */
9725 }
9726
9727 /*
9728  * Values for QSFP retry.
9729  *
9730  * Give up after 10s (20 x 500ms).  The overall timeout was empirically
9731  * arrived at from experience on a large cluster.
9732  */
9733 #define MAX_QSFP_RETRIES 20
9734 #define QSFP_RETRY_WAIT 500 /* msec */
9735
9736 /*
9737  * Try a QSFP read.  If it fails, schedule a retry for later.
9738  * Called on first link activation after driver load.
9739  */
9740 static void try_start_link(struct hfi1_pportdata *ppd)
9741 {
9742         if (test_qsfp_read(ppd)) {
9743                 /* read failed */
9744                 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) {
9745                         dd_dev_err(ppd->dd, "QSFP not responding, giving up\n");
9746                         return;
9747                 }
9748                 dd_dev_info(ppd->dd,
9749                             "QSFP not responding, waiting and retrying %d\n",
9750                             (int)ppd->qsfp_retry_count);
9751                 ppd->qsfp_retry_count++;
9752                 queue_delayed_work(ppd->link_wq, &ppd->start_link_work,
9753                                    msecs_to_jiffies(QSFP_RETRY_WAIT));
9754                 return;
9755         }
9756         ppd->qsfp_retry_count = 0;
9757
9758         start_link(ppd);
9759 }
9760
9761 /*
9762  * Workqueue function to start the link after a delay.
9763  */
9764 void handle_start_link(struct work_struct *work)
9765 {
9766         struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
9767                                                   start_link_work.work);
9768         try_start_link(ppd);
9769 }
9770
9771 int bringup_serdes(struct hfi1_pportdata *ppd)
9772 {
9773         struct hfi1_devdata *dd = ppd->dd;
9774         u64 guid;
9775         int ret;
9776
9777         if (HFI1_CAP_IS_KSET(EXTENDED_PSN))
9778                 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK);
9779
9780         guid = ppd->guids[HFI1_PORT_GUID_INDEX];
9781         if (!guid) {
9782                 if (dd->base_guid)
9783                         guid = dd->base_guid + ppd->port - 1;
9784                 ppd->guids[HFI1_PORT_GUID_INDEX] = guid;
9785         }
9786
9787         /* Set linkinit_reason on power up per OPA spec */
9788         ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP;
9789
9790         /* one-time init of the LCB */
9791         init_lcb(dd);
9792
9793         if (loopback) {
9794                 ret = init_loopback(dd);
9795                 if (ret < 0)
9796                         return ret;
9797         }
9798
9799         get_port_type(ppd);
9800         if (ppd->port_type == PORT_TYPE_QSFP) {
9801                 set_qsfp_int_n(ppd, 0);
9802                 wait_for_qsfp_init(ppd);
9803                 set_qsfp_int_n(ppd, 1);
9804         }
9805
9806         try_start_link(ppd);
9807         return 0;
9808 }
9809
9810 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd)
9811 {
9812         struct hfi1_devdata *dd = ppd->dd;
9813
9814         /*
9815          * Shut down the link and keep it down.   First turn off that the
9816          * driver wants to allow the link to be up (driver_link_ready).
9817          * Then make sure the link is not automatically restarted
9818          * (link_enabled).  Cancel any pending restart.  And finally
9819          * go offline.
9820          */
9821         ppd->driver_link_ready = 0;
9822         ppd->link_enabled = 0;
9823
9824         ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */
9825         flush_delayed_work(&ppd->start_link_work);
9826         cancel_delayed_work_sync(&ppd->start_link_work);
9827
9828         ppd->offline_disabled_reason =
9829                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT);
9830         set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0,
9831                              OPA_LINKDOWN_REASON_REBOOT);
9832         set_link_state(ppd, HLS_DN_OFFLINE);
9833
9834         /* disable the port */
9835         clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
9836 }
9837
9838 static inline int init_cpu_counters(struct hfi1_devdata *dd)
9839 {
9840         struct hfi1_pportdata *ppd;
9841         int i;
9842
9843         ppd = (struct hfi1_pportdata *)(dd + 1);
9844         for (i = 0; i < dd->num_pports; i++, ppd++) {
9845                 ppd->ibport_data.rvp.rc_acks = NULL;
9846                 ppd->ibport_data.rvp.rc_qacks = NULL;
9847                 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
9848                 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
9849                 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
9850                 if (!ppd->ibport_data.rvp.rc_acks ||
9851                     !ppd->ibport_data.rvp.rc_delayed_comp ||
9852                     !ppd->ibport_data.rvp.rc_qacks)
9853                         return -ENOMEM;
9854         }
9855
9856         return 0;
9857 }
9858
9859 /*
9860  * index is the index into the receive array
9861  */
9862 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index,
9863                   u32 type, unsigned long pa, u16 order)
9864 {
9865         u64 reg;
9866
9867         if (!(dd->flags & HFI1_PRESENT))
9868                 goto done;
9869
9870         if (type == PT_INVALID || type == PT_INVALID_FLUSH) {
9871                 pa = 0;
9872                 order = 0;
9873         } else if (type > PT_INVALID) {
9874                 dd_dev_err(dd,
9875                            "unexpected receive array type %u for index %u, not handled\n",
9876                            type, index);
9877                 goto done;
9878         }
9879         trace_hfi1_put_tid(dd, index, type, pa, order);
9880
9881 #define RT_ADDR_SHIFT 12        /* 4KB kernel address boundary */
9882         reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK
9883                 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT
9884                 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK)
9885                                         << RCV_ARRAY_RT_ADDR_SHIFT;
9886         trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg);
9887         writeq(reg, dd->rcvarray_wc + (index * 8));
9888
9889         if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3)
9890                 /*
9891                  * Eager entries are written and flushed
9892                  *
9893                  * Expected entries are flushed every 4 writes
9894                  */
9895                 flush_wc();
9896 done:
9897         return;
9898 }
9899
9900 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd)
9901 {
9902         struct hfi1_devdata *dd = rcd->dd;
9903         u32 i;
9904
9905         /* this could be optimized */
9906         for (i = rcd->eager_base; i < rcd->eager_base +
9907                      rcd->egrbufs.alloced; i++)
9908                 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9909
9910         for (i = rcd->expected_base;
9911                         i < rcd->expected_base + rcd->expected_count; i++)
9912                 hfi1_put_tid(dd, i, PT_INVALID, 0, 0);
9913 }
9914
9915 static const char * const ib_cfg_name_strings[] = {
9916         "HFI1_IB_CFG_LIDLMC",
9917         "HFI1_IB_CFG_LWID_DG_ENB",
9918         "HFI1_IB_CFG_LWID_ENB",
9919         "HFI1_IB_CFG_LWID",
9920         "HFI1_IB_CFG_SPD_ENB",
9921         "HFI1_IB_CFG_SPD",
9922         "HFI1_IB_CFG_RXPOL_ENB",
9923         "HFI1_IB_CFG_LREV_ENB",
9924         "HFI1_IB_CFG_LINKLATENCY",
9925         "HFI1_IB_CFG_HRTBT",
9926         "HFI1_IB_CFG_OP_VLS",
9927         "HFI1_IB_CFG_VL_HIGH_CAP",
9928         "HFI1_IB_CFG_VL_LOW_CAP",
9929         "HFI1_IB_CFG_OVERRUN_THRESH",
9930         "HFI1_IB_CFG_PHYERR_THRESH",
9931         "HFI1_IB_CFG_LINKDEFAULT",
9932         "HFI1_IB_CFG_PKEYS",
9933         "HFI1_IB_CFG_MTU",
9934         "HFI1_IB_CFG_LSTATE",
9935         "HFI1_IB_CFG_VL_HIGH_LIMIT",
9936         "HFI1_IB_CFG_PMA_TICKS",
9937         "HFI1_IB_CFG_PORT"
9938 };
9939
9940 static const char *ib_cfg_name(int which)
9941 {
9942         if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings))
9943                 return "invalid";
9944         return ib_cfg_name_strings[which];
9945 }
9946
9947 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which)
9948 {
9949         struct hfi1_devdata *dd = ppd->dd;
9950         int val = 0;
9951
9952         switch (which) {
9953         case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */
9954                 val = ppd->link_width_enabled;
9955                 break;
9956         case HFI1_IB_CFG_LWID: /* currently active Link-width */
9957                 val = ppd->link_width_active;
9958                 break;
9959         case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
9960                 val = ppd->link_speed_enabled;
9961                 break;
9962         case HFI1_IB_CFG_SPD: /* current Link speed */
9963                 val = ppd->link_speed_active;
9964                 break;
9965
9966         case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */
9967         case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */
9968         case HFI1_IB_CFG_LINKLATENCY:
9969                 goto unimplemented;
9970
9971         case HFI1_IB_CFG_OP_VLS:
9972                 val = ppd->actual_vls_operational;
9973                 break;
9974         case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */
9975                 val = VL_ARB_HIGH_PRIO_TABLE_SIZE;
9976                 break;
9977         case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */
9978                 val = VL_ARB_LOW_PRIO_TABLE_SIZE;
9979                 break;
9980         case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
9981                 val = ppd->overrun_threshold;
9982                 break;
9983         case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
9984                 val = ppd->phy_error_threshold;
9985                 break;
9986         case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
9987                 val = HLS_DEFAULT;
9988                 break;
9989
9990         case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */
9991         case HFI1_IB_CFG_PMA_TICKS:
9992         default:
9993 unimplemented:
9994                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
9995                         dd_dev_info(
9996                                 dd,
9997                                 "%s: which %s: not implemented\n",
9998                                 __func__,
9999                                 ib_cfg_name(which));
10000                 break;
10001         }
10002
10003         return val;
10004 }
10005
10006 /*
10007  * The largest MAD packet size.
10008  */
10009 #define MAX_MAD_PACKET 2048
10010
10011 /*
10012  * Return the maximum header bytes that can go on the _wire_
10013  * for this device. This count includes the ICRC which is
10014  * not part of the packet held in memory but it is appended
10015  * by the HW.
10016  * This is dependent on the device's receive header entry size.
10017  * HFI allows this to be set per-receive context, but the
10018  * driver presently enforces a global value.
10019  */
10020 u32 lrh_max_header_bytes(struct hfi1_devdata *dd)
10021 {
10022         /*
10023          * The maximum non-payload (MTU) bytes in LRH.PktLen are
10024          * the Receive Header Entry Size minus the PBC (or RHF) size
10025          * plus one DW for the ICRC appended by HW.
10026          *
10027          * dd->rcd[0].rcvhdrqentsize is in DW.
10028          * We use rcd[0] as all context will have the same value. Also,
10029          * the first kernel context would have been allocated by now so
10030          * we are guaranteed a valid value.
10031          */
10032         return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2;
10033 }
10034
10035 /*
10036  * Set Send Length
10037  * @ppd - per port data
10038  *
10039  * Set the MTU by limiting how many DWs may be sent.  The SendLenCheck*
10040  * registers compare against LRH.PktLen, so use the max bytes included
10041  * in the LRH.
10042  *
10043  * This routine changes all VL values except VL15, which it maintains at
10044  * the same value.
10045  */
10046 static void set_send_length(struct hfi1_pportdata *ppd)
10047 {
10048         struct hfi1_devdata *dd = ppd->dd;
10049         u32 max_hb = lrh_max_header_bytes(dd), dcmtu;
10050         u32 maxvlmtu = dd->vld[15].mtu;
10051         u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2)
10052                               & SEND_LEN_CHECK1_LEN_VL15_MASK) <<
10053                 SEND_LEN_CHECK1_LEN_VL15_SHIFT;
10054         int i, j;
10055         u32 thres;
10056
10057         for (i = 0; i < ppd->vls_supported; i++) {
10058                 if (dd->vld[i].mtu > maxvlmtu)
10059                         maxvlmtu = dd->vld[i].mtu;
10060                 if (i <= 3)
10061                         len1 |= (((dd->vld[i].mtu + max_hb) >> 2)
10062                                  & SEND_LEN_CHECK0_LEN_VL0_MASK) <<
10063                                 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT);
10064                 else
10065                         len2 |= (((dd->vld[i].mtu + max_hb) >> 2)
10066                                  & SEND_LEN_CHECK1_LEN_VL4_MASK) <<
10067                                 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT);
10068         }
10069         write_csr(dd, SEND_LEN_CHECK0, len1);
10070         write_csr(dd, SEND_LEN_CHECK1, len2);
10071         /* adjust kernel credit return thresholds based on new MTUs */
10072         /* all kernel receive contexts have the same hdrqentsize */
10073         for (i = 0; i < ppd->vls_supported; i++) {
10074                 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50),
10075                             sc_mtu_to_threshold(dd->vld[i].sc,
10076                                                 dd->vld[i].mtu,
10077                                                 dd->rcd[0]->rcvhdrqentsize));
10078                 for (j = 0; j < INIT_SC_PER_VL; j++)
10079                         sc_set_cr_threshold(
10080                                         pio_select_send_context_vl(dd, j, i),
10081                                             thres);
10082         }
10083         thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50),
10084                     sc_mtu_to_threshold(dd->vld[15].sc,
10085                                         dd->vld[15].mtu,
10086                                         dd->rcd[0]->rcvhdrqentsize));
10087         sc_set_cr_threshold(dd->vld[15].sc, thres);
10088
10089         /* Adjust maximum MTU for the port in DC */
10090         dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 :
10091                 (ilog2(maxvlmtu >> 8) + 1);
10092         len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG);
10093         len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK;
10094         len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) <<
10095                 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT;
10096         write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1);
10097 }
10098
10099 static void set_lidlmc(struct hfi1_pportdata *ppd)
10100 {
10101         int i;
10102         u64 sreg = 0;
10103         struct hfi1_devdata *dd = ppd->dd;
10104         u32 mask = ~((1U << ppd->lmc) - 1);
10105         u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1);
10106         u32 lid;
10107
10108         /*
10109          * Program 0 in CSR if port lid is extended. This prevents
10110          * 9B packets being sent out for large lids.
10111          */
10112         lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid;
10113         c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK
10114                 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK);
10115         c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK)
10116                         << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) |
10117               ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK)
10118                         << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT);
10119         write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1);
10120
10121         /*
10122          * Iterate over all the send contexts and set their SLID check
10123          */
10124         sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) <<
10125                         SEND_CTXT_CHECK_SLID_MASK_SHIFT) |
10126                (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) <<
10127                         SEND_CTXT_CHECK_SLID_VALUE_SHIFT);
10128
10129         for (i = 0; i < chip_send_contexts(dd); i++) {
10130                 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x",
10131                           i, (u32)sreg);
10132                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg);
10133         }
10134
10135         /* Now we have to do the same thing for the sdma engines */
10136         sdma_update_lmc(dd, mask, lid);
10137 }
10138
10139 static const char *state_completed_string(u32 completed)
10140 {
10141         static const char * const state_completed[] = {
10142                 "EstablishComm",
10143                 "OptimizeEQ",
10144                 "VerifyCap"
10145         };
10146
10147         if (completed < ARRAY_SIZE(state_completed))
10148                 return state_completed[completed];
10149
10150         return "unknown";
10151 }
10152
10153 static const char all_lanes_dead_timeout_expired[] =
10154         "All lanes were inactive – was the interconnect media removed?";
10155 static const char tx_out_of_policy[] =
10156         "Passing lanes on local port do not meet the local link width policy";
10157 static const char no_state_complete[] =
10158         "State timeout occurred before link partner completed the state";
10159 static const char * const state_complete_reasons[] = {
10160         [0x00] = "Reason unknown",
10161         [0x01] = "Link was halted by driver, refer to LinkDownReason",
10162         [0x02] = "Link partner reported failure",
10163         [0x10] = "Unable to achieve frame sync on any lane",
10164         [0x11] =
10165           "Unable to find a common bit rate with the link partner",
10166         [0x12] =
10167           "Unable to achieve frame sync on sufficient lanes to meet the local link width policy",
10168         [0x13] =
10169           "Unable to identify preset equalization on sufficient lanes to meet the local link width policy",
10170         [0x14] = no_state_complete,
10171         [0x15] =
10172           "State timeout occurred before link partner identified equalization presets",
10173         [0x16] =
10174           "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy",
10175         [0x17] = tx_out_of_policy,
10176         [0x20] = all_lanes_dead_timeout_expired,
10177         [0x21] =
10178           "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy",
10179         [0x22] = no_state_complete,
10180         [0x23] =
10181           "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy",
10182         [0x24] = tx_out_of_policy,
10183         [0x30] = all_lanes_dead_timeout_expired,
10184         [0x31] =
10185           "State timeout occurred waiting for host to process received frames",
10186         [0x32] = no_state_complete,
10187         [0x33] =
10188           "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy",
10189         [0x34] = tx_out_of_policy,
10190         [0x35] = "Negotiated link width is mutually exclusive",
10191         [0x36] =
10192           "Timed out before receiving verifycap frames in VerifyCap.Exchange",
10193         [0x37] = "Unable to resolve secure data exchange",
10194 };
10195
10196 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd,
10197                                                      u32 code)
10198 {
10199         const char *str = NULL;
10200
10201         if (code < ARRAY_SIZE(state_complete_reasons))
10202                 str = state_complete_reasons[code];
10203
10204         if (str)
10205                 return str;
10206         return "Reserved";
10207 }
10208
10209 /* describe the given last state complete frame */
10210 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame,
10211                                   const char *prefix)
10212 {
10213         struct hfi1_devdata *dd = ppd->dd;
10214         u32 success;
10215         u32 state;
10216         u32 reason;
10217         u32 lanes;
10218
10219         /*
10220          * Decode frame:
10221          *  [ 0: 0] - success
10222          *  [ 3: 1] - state
10223          *  [ 7: 4] - next state timeout
10224          *  [15: 8] - reason code
10225          *  [31:16] - lanes
10226          */
10227         success = frame & 0x1;
10228         state = (frame >> 1) & 0x7;
10229         reason = (frame >> 8) & 0xff;
10230         lanes = (frame >> 16) & 0xffff;
10231
10232         dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n",
10233                    prefix, frame);
10234         dd_dev_err(dd, "    last reported state state: %s (0x%x)\n",
10235                    state_completed_string(state), state);
10236         dd_dev_err(dd, "    state successfully completed: %s\n",
10237                    success ? "yes" : "no");
10238         dd_dev_err(dd, "    fail reason 0x%x: %s\n",
10239                    reason, state_complete_reason_code_string(ppd, reason));
10240         dd_dev_err(dd, "    passing lane mask: 0x%x", lanes);
10241 }
10242
10243 /*
10244  * Read the last state complete frames and explain them.  This routine
10245  * expects to be called if the link went down during link negotiation
10246  * and initialization (LNI).  That is, anywhere between polling and link up.
10247  */
10248 static void check_lni_states(struct hfi1_pportdata *ppd)
10249 {
10250         u32 last_local_state;
10251         u32 last_remote_state;
10252
10253         read_last_local_state(ppd->dd, &last_local_state);
10254         read_last_remote_state(ppd->dd, &last_remote_state);
10255
10256         /*
10257          * Don't report anything if there is nothing to report.  A value of
10258          * 0 means the link was taken down while polling and there was no
10259          * training in-process.
10260          */
10261         if (last_local_state == 0 && last_remote_state == 0)
10262                 return;
10263
10264         decode_state_complete(ppd, last_local_state, "transmitted");
10265         decode_state_complete(ppd, last_remote_state, "received");
10266 }
10267
10268 /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */
10269 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms)
10270 {
10271         u64 reg;
10272         unsigned long timeout;
10273
10274         /* watch LCB_STS_LINK_TRANSFER_ACTIVE */
10275         timeout = jiffies + msecs_to_jiffies(wait_ms);
10276         while (1) {
10277                 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE);
10278                 if (reg)
10279                         break;
10280                 if (time_after(jiffies, timeout)) {
10281                         dd_dev_err(dd,
10282                                    "timeout waiting for LINK_TRANSFER_ACTIVE\n");
10283                         return -ETIMEDOUT;
10284                 }
10285                 udelay(2);
10286         }
10287         return 0;
10288 }
10289
10290 /* called when the logical link state is not down as it should be */
10291 static void force_logical_link_state_down(struct hfi1_pportdata *ppd)
10292 {
10293         struct hfi1_devdata *dd = ppd->dd;
10294
10295         /*
10296          * Bring link up in LCB loopback
10297          */
10298         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10299         write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK,
10300                   DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK);
10301
10302         write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0);
10303         write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0);
10304         write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110);
10305         write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2);
10306
10307         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0);
10308         (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET);
10309         udelay(3);
10310         write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1);
10311         write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT);
10312
10313         wait_link_transfer_active(dd, 100);
10314
10315         /*
10316          * Bring the link down again.
10317          */
10318         write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1);
10319         write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0);
10320         write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0);
10321
10322         dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n");
10323 }
10324
10325 /*
10326  * Helper for set_link_state().  Do not call except from that routine.
10327  * Expects ppd->hls_mutex to be held.
10328  *
10329  * @rem_reason value to be sent to the neighbor
10330  *
10331  * LinkDownReasons only set if transition succeeds.
10332  */
10333 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)
10334 {
10335         struct hfi1_devdata *dd = ppd->dd;
10336         u32 previous_state;
10337         int offline_state_ret;
10338         int ret;
10339
10340         update_lcb_cache(dd);
10341
10342         previous_state = ppd->host_link_state;
10343         ppd->host_link_state = HLS_GOING_OFFLINE;
10344
10345         /* start offline transition */
10346         ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE);
10347
10348         if (ret != HCMD_SUCCESS) {
10349                 dd_dev_err(dd,
10350                            "Failed to transition to Offline link state, return %d\n",
10351                            ret);
10352                 return -EINVAL;
10353         }
10354         if (ppd->offline_disabled_reason ==
10355                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))
10356                 ppd->offline_disabled_reason =
10357                 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT);
10358
10359         offline_state_ret = wait_phys_link_offline_substates(ppd, 10000);
10360         if (offline_state_ret < 0)
10361                 return offline_state_ret;
10362
10363         /* Disabling AOC transmitters */
10364         if (ppd->port_type == PORT_TYPE_QSFP &&
10365             ppd->qsfp_info.limiting_active &&
10366             qsfp_mod_present(ppd)) {
10367                 int ret;
10368
10369                 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT);
10370                 if (ret == 0) {
10371                         set_qsfp_tx(ppd, 0);
10372                         release_chip_resource(dd, qsfp_resource(dd));
10373                 } else {
10374                         /* not fatal, but should warn */
10375                         dd_dev_err(dd,
10376                                    "Unable to acquire lock to turn off QSFP TX\n");
10377                 }
10378         }
10379
10380         /*
10381          * Wait for the offline.Quiet transition if it hasn't happened yet. It
10382          * can take a while for the link to go down.
10383          */
10384         if (offline_state_ret != PLS_OFFLINE_QUIET) {
10385                 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000);
10386                 if (ret < 0)
10387                         return ret;
10388         }
10389
10390         /*
10391          * Now in charge of LCB - must be after the physical state is
10392          * offline.quiet and before host_link_state is changed.
10393          */
10394         set_host_lcb_access(dd);
10395         write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */
10396
10397         /* make sure the logical state is also down */
10398         ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000);
10399         if (ret)
10400                 force_logical_link_state_down(ppd);
10401
10402         ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */
10403         update_statusp(ppd, IB_PORT_DOWN);
10404
10405         /*
10406          * The LNI has a mandatory wait time after the physical state
10407          * moves to Offline.Quiet.  The wait time may be different
10408          * depending on how the link went down.  The 8051 firmware
10409          * will observe the needed wait time and only move to ready
10410          * when that is completed.  The largest of the quiet timeouts
10411          * is 6s, so wait that long and then at least 0.5s more for
10412          * other transitions, and another 0.5s for a buffer.
10413          */
10414         ret = wait_fm_ready(dd, 7000);
10415         if (ret) {
10416                 dd_dev_err(dd,
10417                            "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n");
10418                 /* state is really offline, so make it so */
10419                 ppd->host_link_state = HLS_DN_OFFLINE;
10420                 return ret;
10421         }
10422
10423         /*
10424          * The state is now offline and the 8051 is ready to accept host
10425          * requests.
10426          *      - change our state
10427          *      - notify others if we were previously in a linkup state
10428          */
10429         ppd->host_link_state = HLS_DN_OFFLINE;
10430         if (previous_state & HLS_UP) {
10431                 /* went down while link was up */
10432                 handle_linkup_change(dd, 0);
10433         } else if (previous_state
10434                         & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) {
10435                 /* went down while attempting link up */
10436                 check_lni_states(ppd);
10437
10438                 /* The QSFP doesn't need to be reset on LNI failure */
10439                 ppd->qsfp_info.reset_needed = 0;
10440         }
10441
10442         /* the active link width (downgrade) is 0 on link down */
10443         ppd->link_width_active = 0;
10444         ppd->link_width_downgrade_tx_active = 0;
10445         ppd->link_width_downgrade_rx_active = 0;
10446         ppd->current_egress_rate = 0;
10447         return 0;
10448 }
10449
10450 /* return the link state name */
10451 static const char *link_state_name(u32 state)
10452 {
10453         const char *name;
10454         int n = ilog2(state);
10455         static const char * const names[] = {
10456                 [__HLS_UP_INIT_BP]       = "INIT",
10457                 [__HLS_UP_ARMED_BP]      = "ARMED",
10458                 [__HLS_UP_ACTIVE_BP]     = "ACTIVE",
10459                 [__HLS_DN_DOWNDEF_BP]    = "DOWNDEF",
10460                 [__HLS_DN_POLL_BP]       = "POLL",
10461                 [__HLS_DN_DISABLE_BP]    = "DISABLE",
10462                 [__HLS_DN_OFFLINE_BP]    = "OFFLINE",
10463                 [__HLS_VERIFY_CAP_BP]    = "VERIFY_CAP",
10464                 [__HLS_GOING_UP_BP]      = "GOING_UP",
10465                 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE",
10466                 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN"
10467         };
10468
10469         name = n < ARRAY_SIZE(names) ? names[n] : NULL;
10470         return name ? name : "unknown";
10471 }
10472
10473 /* return the link state reason name */
10474 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state)
10475 {
10476         if (state == HLS_UP_INIT) {
10477                 switch (ppd->linkinit_reason) {
10478                 case OPA_LINKINIT_REASON_LINKUP:
10479                         return "(LINKUP)";
10480                 case OPA_LINKINIT_REASON_FLAPPING:
10481                         return "(FLAPPING)";
10482                 case OPA_LINKINIT_OUTSIDE_POLICY:
10483                         return "(OUTSIDE_POLICY)";
10484                 case OPA_LINKINIT_QUARANTINED:
10485                         return "(QUARANTINED)";
10486                 case OPA_LINKINIT_INSUFIC_CAPABILITY:
10487                         return "(INSUFIC_CAPABILITY)";
10488                 default:
10489                         break;
10490                 }
10491         }
10492         return "";
10493 }
10494
10495 /*
10496  * driver_pstate - convert the driver's notion of a port's
10497  * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*).
10498  * Return -1 (converted to a u32) to indicate error.
10499  */
10500 u32 driver_pstate(struct hfi1_pportdata *ppd)
10501 {
10502         switch (ppd->host_link_state) {
10503         case HLS_UP_INIT:
10504         case HLS_UP_ARMED:
10505         case HLS_UP_ACTIVE:
10506                 return IB_PORTPHYSSTATE_LINKUP;
10507         case HLS_DN_POLL:
10508                 return IB_PORTPHYSSTATE_POLLING;
10509         case HLS_DN_DISABLE:
10510                 return IB_PORTPHYSSTATE_DISABLED;
10511         case HLS_DN_OFFLINE:
10512                 return OPA_PORTPHYSSTATE_OFFLINE;
10513         case HLS_VERIFY_CAP:
10514                 return IB_PORTPHYSSTATE_TRAINING;
10515         case HLS_GOING_UP:
10516                 return IB_PORTPHYSSTATE_TRAINING;
10517         case HLS_GOING_OFFLINE:
10518                 return OPA_PORTPHYSSTATE_OFFLINE;
10519         case HLS_LINK_COOLDOWN:
10520                 return OPA_PORTPHYSSTATE_OFFLINE;
10521         case HLS_DN_DOWNDEF:
10522         default:
10523                 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10524                            ppd->host_link_state);
10525                 return  -1;
10526         }
10527 }
10528
10529 /*
10530  * driver_lstate - convert the driver's notion of a port's
10531  * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1
10532  * (converted to a u32) to indicate error.
10533  */
10534 u32 driver_lstate(struct hfi1_pportdata *ppd)
10535 {
10536         if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN))
10537                 return IB_PORT_DOWN;
10538
10539         switch (ppd->host_link_state & HLS_UP) {
10540         case HLS_UP_INIT:
10541                 return IB_PORT_INIT;
10542         case HLS_UP_ARMED:
10543                 return IB_PORT_ARMED;
10544         case HLS_UP_ACTIVE:
10545                 return IB_PORT_ACTIVE;
10546         default:
10547                 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n",
10548                            ppd->host_link_state);
10549         return -1;
10550         }
10551 }
10552
10553 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
10554                           u8 neigh_reason, u8 rem_reason)
10555 {
10556         if (ppd->local_link_down_reason.latest == 0 &&
10557             ppd->neigh_link_down_reason.latest == 0) {
10558                 ppd->local_link_down_reason.latest = lcl_reason;
10559                 ppd->neigh_link_down_reason.latest = neigh_reason;
10560                 ppd->remote_link_down_reason = rem_reason;
10561         }
10562 }
10563
10564 /**
10565  * data_vls_operational() - Verify if data VL BCT credits and MTU
10566  *                          are both set.
10567  * @ppd: pointer to hfi1_pportdata structure
10568  *
10569  * Return: true - Ok, false -otherwise.
10570  */
10571 static inline bool data_vls_operational(struct hfi1_pportdata *ppd)
10572 {
10573         int i;
10574         u64 reg;
10575
10576         if (!ppd->actual_vls_operational)
10577                 return false;
10578
10579         for (i = 0; i < ppd->vls_supported; i++) {
10580                 reg = read_csr(ppd->dd, SEND_CM_CREDIT_VL + (8 * i));
10581                 if ((reg && !ppd->dd->vld[i].mtu) ||
10582                     (!reg && ppd->dd->vld[i].mtu))
10583                         return false;
10584         }
10585
10586         return true;
10587 }
10588
10589 /*
10590  * Change the physical and/or logical link state.
10591  *
10592  * Do not call this routine while inside an interrupt.  It contains
10593  * calls to routines that can take multiple seconds to finish.
10594  *
10595  * Returns 0 on success, -errno on failure.
10596  */
10597 int set_link_state(struct hfi1_pportdata *ppd, u32 state)
10598 {
10599         struct hfi1_devdata *dd = ppd->dd;
10600         struct ib_event event = {.device = NULL};
10601         int ret1, ret = 0;
10602         int orig_new_state, poll_bounce;
10603
10604         mutex_lock(&ppd->hls_lock);
10605
10606         orig_new_state = state;
10607         if (state == HLS_DN_DOWNDEF)
10608                 state = HLS_DEFAULT;
10609
10610         /* interpret poll -> poll as a link bounce */
10611         poll_bounce = ppd->host_link_state == HLS_DN_POLL &&
10612                       state == HLS_DN_POLL;
10613
10614         dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__,
10615                     link_state_name(ppd->host_link_state),
10616                     link_state_name(orig_new_state),
10617                     poll_bounce ? "(bounce) " : "",
10618                     link_state_reason_name(ppd, state));
10619
10620         /*
10621          * If we're going to a (HLS_*) link state that implies the logical
10622          * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then
10623          * reset is_sm_config_started to 0.
10624          */
10625         if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE)))
10626                 ppd->is_sm_config_started = 0;
10627
10628         /*
10629          * Do nothing if the states match.  Let a poll to poll link bounce
10630          * go through.
10631          */
10632         if (ppd->host_link_state == state && !poll_bounce)
10633                 goto done;
10634
10635         switch (state) {
10636         case HLS_UP_INIT:
10637                 if (ppd->host_link_state == HLS_DN_POLL &&
10638                     (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) {
10639                         /*
10640                          * Quick link up jumps from polling to here.
10641                          *
10642                          * Whether in normal or loopback mode, the
10643                          * simulator jumps from polling to link up.
10644                          * Accept that here.
10645                          */
10646                         /* OK */
10647                 } else if (ppd->host_link_state != HLS_GOING_UP) {
10648                         goto unexpected;
10649                 }
10650
10651                 /*
10652                  * Wait for Link_Up physical state.
10653                  * Physical and Logical states should already be
10654                  * be transitioned to LinkUp and LinkInit respectively.
10655                  */
10656                 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000);
10657                 if (ret) {
10658                         dd_dev_err(dd,
10659                                    "%s: physical state did not change to LINK-UP\n",
10660                                    __func__);
10661                         break;
10662                 }
10663
10664                 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
10665                 if (ret) {
10666                         dd_dev_err(dd,
10667                                    "%s: logical state did not change to INIT\n",
10668                                    __func__);
10669                         break;
10670                 }
10671
10672                 /* clear old transient LINKINIT_REASON code */
10673                 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR)
10674                         ppd->linkinit_reason =
10675                                 OPA_LINKINIT_REASON_LINKUP;
10676
10677                 /* enable the port */
10678                 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
10679
10680                 handle_linkup_change(dd, 1);
10681                 pio_kernel_linkup(dd);
10682
10683                 /*
10684                  * After link up, a new link width will have been set.
10685                  * Update the xmit counters with regards to the new
10686                  * link width.
10687                  */
10688                 update_xmit_counters(ppd, ppd->link_width_active);
10689
10690                 ppd->host_link_state = HLS_UP_INIT;
10691                 update_statusp(ppd, IB_PORT_INIT);
10692                 break;
10693         case HLS_UP_ARMED:
10694                 if (ppd->host_link_state != HLS_UP_INIT)
10695                         goto unexpected;
10696
10697                 if (!data_vls_operational(ppd)) {
10698                         dd_dev_err(dd,
10699                                    "%s: Invalid data VL credits or mtu\n",
10700                                    __func__);
10701                         ret = -EINVAL;
10702                         break;
10703                 }
10704
10705                 set_logical_state(dd, LSTATE_ARMED);
10706                 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000);
10707                 if (ret) {
10708                         dd_dev_err(dd,
10709                                    "%s: logical state did not change to ARMED\n",
10710                                    __func__);
10711                         break;
10712                 }
10713                 ppd->host_link_state = HLS_UP_ARMED;
10714                 update_statusp(ppd, IB_PORT_ARMED);
10715                 /*
10716                  * The simulator does not currently implement SMA messages,
10717                  * so neighbor_normal is not set.  Set it here when we first
10718                  * move to Armed.
10719                  */
10720                 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR)
10721                         ppd->neighbor_normal = 1;
10722                 break;
10723         case HLS_UP_ACTIVE:
10724                 if (ppd->host_link_state != HLS_UP_ARMED)
10725                         goto unexpected;
10726
10727                 set_logical_state(dd, LSTATE_ACTIVE);
10728                 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000);
10729                 if (ret) {
10730                         dd_dev_err(dd,
10731                                    "%s: logical state did not change to ACTIVE\n",
10732                                    __func__);
10733                 } else {
10734                         /* tell all engines to go running */
10735                         sdma_all_running(dd);
10736                         ppd->host_link_state = HLS_UP_ACTIVE;
10737                         update_statusp(ppd, IB_PORT_ACTIVE);
10738
10739                         /* Signal the IB layer that the port has went active */
10740                         event.device = &dd->verbs_dev.rdi.ibdev;
10741                         event.element.port_num = ppd->port;
10742                         event.event = IB_EVENT_PORT_ACTIVE;
10743                 }
10744                 break;
10745         case HLS_DN_POLL:
10746                 if ((ppd->host_link_state == HLS_DN_DISABLE ||
10747                      ppd->host_link_state == HLS_DN_OFFLINE) &&
10748                     dd->dc_shutdown)
10749                         dc_start(dd);
10750                 /* Hand LED control to the DC */
10751                 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
10752
10753                 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10754                         u8 tmp = ppd->link_enabled;
10755
10756                         ret = goto_offline(ppd, ppd->remote_link_down_reason);
10757                         if (ret) {
10758                                 ppd->link_enabled = tmp;
10759                                 break;
10760                         }
10761                         ppd->remote_link_down_reason = 0;
10762
10763                         if (ppd->driver_link_ready)
10764                                 ppd->link_enabled = 1;
10765                 }
10766
10767                 set_all_slowpath(ppd->dd);
10768                 ret = set_local_link_attributes(ppd);
10769                 if (ret)
10770                         break;
10771
10772                 ppd->port_error_action = 0;
10773                 ppd->host_link_state = HLS_DN_POLL;
10774
10775                 if (quick_linkup) {
10776                         /* quick linkup does not go into polling */
10777                         ret = do_quick_linkup(dd);
10778                 } else {
10779                         ret1 = set_physical_link_state(dd, PLS_POLLING);
10780                         if (ret1 != HCMD_SUCCESS) {
10781                                 dd_dev_err(dd,
10782                                            "Failed to transition to Polling link state, return 0x%x\n",
10783                                            ret1);
10784                                 ret = -EINVAL;
10785                         }
10786                 }
10787                 ppd->offline_disabled_reason =
10788                         HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE);
10789                 /*
10790                  * If an error occurred above, go back to offline.  The
10791                  * caller may reschedule another attempt.
10792                  */
10793                 if (ret)
10794                         goto_offline(ppd, 0);
10795                 else
10796                         log_physical_state(ppd, PLS_POLLING);
10797                 break;
10798         case HLS_DN_DISABLE:
10799                 /* link is disabled */
10800                 ppd->link_enabled = 0;
10801
10802                 /* allow any state to transition to disabled */
10803
10804                 /* must transition to offline first */
10805                 if (ppd->host_link_state != HLS_DN_OFFLINE) {
10806                         ret = goto_offline(ppd, ppd->remote_link_down_reason);
10807                         if (ret)
10808                                 break;
10809                         ppd->remote_link_down_reason = 0;
10810                 }
10811
10812                 if (!dd->dc_shutdown) {
10813                         ret1 = set_physical_link_state(dd, PLS_DISABLED);
10814                         if (ret1 != HCMD_SUCCESS) {
10815                                 dd_dev_err(dd,
10816                                            "Failed to transition to Disabled link state, return 0x%x\n",
10817                                            ret1);
10818                                 ret = -EINVAL;
10819                                 break;
10820                         }
10821                         ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000);
10822                         if (ret) {
10823                                 dd_dev_err(dd,
10824                                            "%s: physical state did not change to DISABLED\n",
10825                                            __func__);
10826                                 break;
10827                         }
10828                         dc_shutdown(dd);
10829                 }
10830                 ppd->host_link_state = HLS_DN_DISABLE;
10831                 break;
10832         case HLS_DN_OFFLINE:
10833                 if (ppd->host_link_state == HLS_DN_DISABLE)
10834                         dc_start(dd);
10835
10836                 /* allow any state to transition to offline */
10837                 ret = goto_offline(ppd, ppd->remote_link_down_reason);
10838                 if (!ret)
10839                         ppd->remote_link_down_reason = 0;
10840                 break;
10841         case HLS_VERIFY_CAP:
10842                 if (ppd->host_link_state != HLS_DN_POLL)
10843                         goto unexpected;
10844                 ppd->host_link_state = HLS_VERIFY_CAP;
10845                 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP);
10846                 break;
10847         case HLS_GOING_UP:
10848                 if (ppd->host_link_state != HLS_VERIFY_CAP)
10849                         goto unexpected;
10850
10851                 ret1 = set_physical_link_state(dd, PLS_LINKUP);
10852                 if (ret1 != HCMD_SUCCESS) {
10853                         dd_dev_err(dd,
10854                                    "Failed to transition to link up state, return 0x%x\n",
10855                                    ret1);
10856                         ret = -EINVAL;
10857                         break;
10858                 }
10859                 ppd->host_link_state = HLS_GOING_UP;
10860                 break;
10861
10862         case HLS_GOING_OFFLINE:         /* transient within goto_offline() */
10863         case HLS_LINK_COOLDOWN:         /* transient within goto_offline() */
10864         default:
10865                 dd_dev_info(dd, "%s: state 0x%x: not supported\n",
10866                             __func__, state);
10867                 ret = -EINVAL;
10868                 break;
10869         }
10870
10871         goto done;
10872
10873 unexpected:
10874         dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n",
10875                    __func__, link_state_name(ppd->host_link_state),
10876                    link_state_name(state));
10877         ret = -EINVAL;
10878
10879 done:
10880         mutex_unlock(&ppd->hls_lock);
10881
10882         if (event.device)
10883                 ib_dispatch_event(&event);
10884
10885         return ret;
10886 }
10887
10888 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val)
10889 {
10890         u64 reg;
10891         int ret = 0;
10892
10893         switch (which) {
10894         case HFI1_IB_CFG_LIDLMC:
10895                 set_lidlmc(ppd);
10896                 break;
10897         case HFI1_IB_CFG_VL_HIGH_LIMIT:
10898                 /*
10899                  * The VL Arbitrator high limit is sent in units of 4k
10900                  * bytes, while HFI stores it in units of 64 bytes.
10901                  */
10902                 val *= 4096 / 64;
10903                 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK)
10904                         << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT;
10905                 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg);
10906                 break;
10907         case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
10908                 /* HFI only supports POLL as the default link down state */
10909                 if (val != HLS_DN_POLL)
10910                         ret = -EINVAL;
10911                 break;
10912         case HFI1_IB_CFG_OP_VLS:
10913                 if (ppd->vls_operational != val) {
10914                         ppd->vls_operational = val;
10915                         if (!ppd->port)
10916                                 ret = -EINVAL;
10917                 }
10918                 break;
10919         /*
10920          * For link width, link width downgrade, and speed enable, always AND
10921          * the setting with what is actually supported.  This has two benefits.
10922          * First, enabled can't have unsupported values, no matter what the
10923          * SM or FM might want.  Second, the ALL_SUPPORTED wildcards that mean
10924          * "fill in with your supported value" have all the bits in the
10925          * field set, so simply ANDing with supported has the desired result.
10926          */
10927         case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */
10928                 ppd->link_width_enabled = val & ppd->link_width_supported;
10929                 break;
10930         case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */
10931                 ppd->link_width_downgrade_enabled =
10932                                 val & ppd->link_width_downgrade_supported;
10933                 break;
10934         case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */
10935                 ppd->link_speed_enabled = val & ppd->link_speed_supported;
10936                 break;
10937         case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
10938                 /*
10939                  * HFI does not follow IB specs, save this value
10940                  * so we can report it, if asked.
10941                  */
10942                 ppd->overrun_threshold = val;
10943                 break;
10944         case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
10945                 /*
10946                  * HFI does not follow IB specs, save this value
10947                  * so we can report it, if asked.
10948                  */
10949                 ppd->phy_error_threshold = val;
10950                 break;
10951
10952         case HFI1_IB_CFG_MTU:
10953                 set_send_length(ppd);
10954                 break;
10955
10956         case HFI1_IB_CFG_PKEYS:
10957                 if (HFI1_CAP_IS_KSET(PKEY_CHECK))
10958                         set_partition_keys(ppd);
10959                 break;
10960
10961         default:
10962                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
10963                         dd_dev_info(ppd->dd,
10964                                     "%s: which %s, val 0x%x: not implemented\n",
10965                                     __func__, ib_cfg_name(which), val);
10966                 break;
10967         }
10968         return ret;
10969 }
10970
10971 /* begin functions related to vl arbitration table caching */
10972 static void init_vl_arb_caches(struct hfi1_pportdata *ppd)
10973 {
10974         int i;
10975
10976         BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10977                         VL_ARB_LOW_PRIO_TABLE_SIZE);
10978         BUILD_BUG_ON(VL_ARB_TABLE_SIZE !=
10979                         VL_ARB_HIGH_PRIO_TABLE_SIZE);
10980
10981         /*
10982          * Note that we always return values directly from the
10983          * 'vl_arb_cache' (and do no CSR reads) in response to a
10984          * 'Get(VLArbTable)'. This is obviously correct after a
10985          * 'Set(VLArbTable)', since the cache will then be up to
10986          * date. But it's also correct prior to any 'Set(VLArbTable)'
10987          * since then both the cache, and the relevant h/w registers
10988          * will be zeroed.
10989          */
10990
10991         for (i = 0; i < MAX_PRIO_TABLE; i++)
10992                 spin_lock_init(&ppd->vl_arb_cache[i].lock);
10993 }
10994
10995 /*
10996  * vl_arb_lock_cache
10997  *
10998  * All other vl_arb_* functions should be called only after locking
10999  * the cache.
11000  */
11001 static inline struct vl_arb_cache *
11002 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx)
11003 {
11004         if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE)
11005                 return NULL;
11006         spin_lock(&ppd->vl_arb_cache[idx].lock);
11007         return &ppd->vl_arb_cache[idx];
11008 }
11009
11010 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx)
11011 {
11012         spin_unlock(&ppd->vl_arb_cache[idx].lock);
11013 }
11014
11015 static void vl_arb_get_cache(struct vl_arb_cache *cache,
11016                              struct ib_vl_weight_elem *vl)
11017 {
11018         memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl));
11019 }
11020
11021 static void vl_arb_set_cache(struct vl_arb_cache *cache,
11022                              struct ib_vl_weight_elem *vl)
11023 {
11024         memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11025 }
11026
11027 static int vl_arb_match_cache(struct vl_arb_cache *cache,
11028                               struct ib_vl_weight_elem *vl)
11029 {
11030         return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl));
11031 }
11032
11033 /* end functions related to vl arbitration table caching */
11034
11035 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target,
11036                           u32 size, struct ib_vl_weight_elem *vl)
11037 {
11038         struct hfi1_devdata *dd = ppd->dd;
11039         u64 reg;
11040         unsigned int i, is_up = 0;
11041         int drain, ret = 0;
11042
11043         mutex_lock(&ppd->hls_lock);
11044
11045         if (ppd->host_link_state & HLS_UP)
11046                 is_up = 1;
11047
11048         drain = !is_ax(dd) && is_up;
11049
11050         if (drain)
11051                 /*
11052                  * Before adjusting VL arbitration weights, empty per-VL
11053                  * FIFOs, otherwise a packet whose VL weight is being
11054                  * set to 0 could get stuck in a FIFO with no chance to
11055                  * egress.
11056                  */
11057                 ret = stop_drain_data_vls(dd);
11058
11059         if (ret) {
11060                 dd_dev_err(
11061                         dd,
11062                         "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n",
11063                         __func__);
11064                 goto err;
11065         }
11066
11067         for (i = 0; i < size; i++, vl++) {
11068                 /*
11069                  * NOTE: The low priority shift and mask are used here, but
11070                  * they are the same for both the low and high registers.
11071                  */
11072                 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK)
11073                                 << SEND_LOW_PRIORITY_LIST_VL_SHIFT)
11074                       | (((u64)vl->weight
11075                                 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK)
11076                                 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT);
11077                 write_csr(dd, target + (i * 8), reg);
11078         }
11079         pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE);
11080
11081         if (drain)
11082                 open_fill_data_vls(dd); /* reopen all VLs */
11083
11084 err:
11085         mutex_unlock(&ppd->hls_lock);
11086
11087         return ret;
11088 }
11089
11090 /*
11091  * Read one credit merge VL register.
11092  */
11093 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr,
11094                            struct vl_limit *vll)
11095 {
11096         u64 reg = read_csr(dd, csr);
11097
11098         vll->dedicated = cpu_to_be16(
11099                 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT)
11100                 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK);
11101         vll->shared = cpu_to_be16(
11102                 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT)
11103                 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK);
11104 }
11105
11106 /*
11107  * Read the current credit merge limits.
11108  */
11109 static int get_buffer_control(struct hfi1_devdata *dd,
11110                               struct buffer_control *bc, u16 *overall_limit)
11111 {
11112         u64 reg;
11113         int i;
11114
11115         /* not all entries are filled in */
11116         memset(bc, 0, sizeof(*bc));
11117
11118         /* OPA and HFI have a 1-1 mapping */
11119         for (i = 0; i < TXE_NUM_DATA_VL; i++)
11120                 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]);
11121
11122         /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */
11123         read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]);
11124
11125         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11126         bc->overall_shared_limit = cpu_to_be16(
11127                 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT)
11128                 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK);
11129         if (overall_limit)
11130                 *overall_limit = (reg
11131                         >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT)
11132                         & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK;
11133         return sizeof(struct buffer_control);
11134 }
11135
11136 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11137 {
11138         u64 reg;
11139         int i;
11140
11141         /* each register contains 16 SC->VLnt mappings, 4 bits each */
11142         reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0);
11143         for (i = 0; i < sizeof(u64); i++) {
11144                 u8 byte = *(((u8 *)&reg) + i);
11145
11146                 dp->vlnt[2 * i] = byte & 0xf;
11147                 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4;
11148         }
11149
11150         reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16);
11151         for (i = 0; i < sizeof(u64); i++) {
11152                 u8 byte = *(((u8 *)&reg) + i);
11153
11154                 dp->vlnt[16 + (2 * i)] = byte & 0xf;
11155                 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4;
11156         }
11157         return sizeof(struct sc2vlnt);
11158 }
11159
11160 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems,
11161                               struct ib_vl_weight_elem *vl)
11162 {
11163         unsigned int i;
11164
11165         for (i = 0; i < nelems; i++, vl++) {
11166                 vl->vl = 0xf;
11167                 vl->weight = 0;
11168         }
11169 }
11170
11171 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp)
11172 {
11173         write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0,
11174                   DC_SC_VL_VAL(15_0,
11175                                0, dp->vlnt[0] & 0xf,
11176                                1, dp->vlnt[1] & 0xf,
11177                                2, dp->vlnt[2] & 0xf,
11178                                3, dp->vlnt[3] & 0xf,
11179                                4, dp->vlnt[4] & 0xf,
11180                                5, dp->vlnt[5] & 0xf,
11181                                6, dp->vlnt[6] & 0xf,
11182                                7, dp->vlnt[7] & 0xf,
11183                                8, dp->vlnt[8] & 0xf,
11184                                9, dp->vlnt[9] & 0xf,
11185                                10, dp->vlnt[10] & 0xf,
11186                                11, dp->vlnt[11] & 0xf,
11187                                12, dp->vlnt[12] & 0xf,
11188                                13, dp->vlnt[13] & 0xf,
11189                                14, dp->vlnt[14] & 0xf,
11190                                15, dp->vlnt[15] & 0xf));
11191         write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16,
11192                   DC_SC_VL_VAL(31_16,
11193                                16, dp->vlnt[16] & 0xf,
11194                                17, dp->vlnt[17] & 0xf,
11195                                18, dp->vlnt[18] & 0xf,
11196                                19, dp->vlnt[19] & 0xf,
11197                                20, dp->vlnt[20] & 0xf,
11198                                21, dp->vlnt[21] & 0xf,
11199                                22, dp->vlnt[22] & 0xf,
11200                                23, dp->vlnt[23] & 0xf,
11201                                24, dp->vlnt[24] & 0xf,
11202                                25, dp->vlnt[25] & 0xf,
11203                                26, dp->vlnt[26] & 0xf,
11204                                27, dp->vlnt[27] & 0xf,
11205                                28, dp->vlnt[28] & 0xf,
11206                                29, dp->vlnt[29] & 0xf,
11207                                30, dp->vlnt[30] & 0xf,
11208                                31, dp->vlnt[31] & 0xf));
11209 }
11210
11211 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what,
11212                         u16 limit)
11213 {
11214         if (limit != 0)
11215                 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n",
11216                             what, (int)limit, idx);
11217 }
11218
11219 /* change only the shared limit portion of SendCmGLobalCredit */
11220 static void set_global_shared(struct hfi1_devdata *dd, u16 limit)
11221 {
11222         u64 reg;
11223
11224         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11225         reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK;
11226         reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT;
11227         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11228 }
11229
11230 /* change only the total credit limit portion of SendCmGLobalCredit */
11231 static void set_global_limit(struct hfi1_devdata *dd, u16 limit)
11232 {
11233         u64 reg;
11234
11235         reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT);
11236         reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK;
11237         reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT;
11238         write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg);
11239 }
11240
11241 /* set the given per-VL shared limit */
11242 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit)
11243 {
11244         u64 reg;
11245         u32 addr;
11246
11247         if (vl < TXE_NUM_DATA_VL)
11248                 addr = SEND_CM_CREDIT_VL + (8 * vl);
11249         else
11250                 addr = SEND_CM_CREDIT_VL15;
11251
11252         reg = read_csr(dd, addr);
11253         reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK;
11254         reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT;
11255         write_csr(dd, addr, reg);
11256 }
11257
11258 /* set the given per-VL dedicated limit */
11259 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit)
11260 {
11261         u64 reg;
11262         u32 addr;
11263
11264         if (vl < TXE_NUM_DATA_VL)
11265                 addr = SEND_CM_CREDIT_VL + (8 * vl);
11266         else
11267                 addr = SEND_CM_CREDIT_VL15;
11268
11269         reg = read_csr(dd, addr);
11270         reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK;
11271         reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT;
11272         write_csr(dd, addr, reg);
11273 }
11274
11275 /* spin until the given per-VL status mask bits clear */
11276 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask,
11277                                      const char *which)
11278 {
11279         unsigned long timeout;
11280         u64 reg;
11281
11282         timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT);
11283         while (1) {
11284                 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask;
11285
11286                 if (reg == 0)
11287                         return; /* success */
11288                 if (time_after(jiffies, timeout))
11289                         break;          /* timed out */
11290                 udelay(1);
11291         }
11292
11293         dd_dev_err(dd,
11294                    "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n",
11295                    which, VL_STATUS_CLEAR_TIMEOUT, mask, reg);
11296         /*
11297          * If this occurs, it is likely there was a credit loss on the link.
11298          * The only recovery from that is a link bounce.
11299          */
11300         dd_dev_err(dd,
11301                    "Continuing anyway.  A credit loss may occur.  Suggest a link bounce\n");
11302 }
11303
11304 /*
11305  * The number of credits on the VLs may be changed while everything
11306  * is "live", but the following algorithm must be followed due to
11307  * how the hardware is actually implemented.  In particular,
11308  * Return_Credit_Status[] is the only correct status check.
11309  *
11310  * if (reducing Global_Shared_Credit_Limit or any shared limit changing)
11311  *     set Global_Shared_Credit_Limit = 0
11312  *     use_all_vl = 1
11313  * mask0 = all VLs that are changing either dedicated or shared limits
11314  * set Shared_Limit[mask0] = 0
11315  * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0
11316  * if (changing any dedicated limit)
11317  *     mask1 = all VLs that are lowering dedicated limits
11318  *     lower Dedicated_Limit[mask1]
11319  *     spin until Return_Credit_Status[mask1] == 0
11320  *     raise Dedicated_Limits
11321  * raise Shared_Limits
11322  * raise Global_Shared_Credit_Limit
11323  *
11324  * lower = if the new limit is lower, set the limit to the new value
11325  * raise = if the new limit is higher than the current value (may be changed
11326  *      earlier in the algorithm), set the new limit to the new value
11327  */
11328 int set_buffer_control(struct hfi1_pportdata *ppd,
11329                        struct buffer_control *new_bc)
11330 {
11331         struct hfi1_devdata *dd = ppd->dd;
11332         u64 changing_mask, ld_mask, stat_mask;
11333         int change_count;
11334         int i, use_all_mask;
11335         int this_shared_changing;
11336         int vl_count = 0, ret;
11337         /*
11338          * A0: add the variable any_shared_limit_changing below and in the
11339          * algorithm above.  If removing A0 support, it can be removed.
11340          */
11341         int any_shared_limit_changing;
11342         struct buffer_control cur_bc;
11343         u8 changing[OPA_MAX_VLS];
11344         u8 lowering_dedicated[OPA_MAX_VLS];
11345         u16 cur_total;
11346         u32 new_total = 0;
11347         const u64 all_mask =
11348         SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK
11349          | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK
11350          | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK
11351          | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK
11352          | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK
11353          | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK
11354          | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK
11355          | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK
11356          | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK;
11357
11358 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15)
11359 #define NUM_USABLE_VLS 16       /* look at VL15 and less */
11360
11361         /* find the new total credits, do sanity check on unused VLs */
11362         for (i = 0; i < OPA_MAX_VLS; i++) {
11363                 if (valid_vl(i)) {
11364                         new_total += be16_to_cpu(new_bc->vl[i].dedicated);
11365                         continue;
11366                 }
11367                 nonzero_msg(dd, i, "dedicated",
11368                             be16_to_cpu(new_bc->vl[i].dedicated));
11369                 nonzero_msg(dd, i, "shared",
11370                             be16_to_cpu(new_bc->vl[i].shared));
11371                 new_bc->vl[i].dedicated = 0;
11372                 new_bc->vl[i].shared = 0;
11373         }
11374         new_total += be16_to_cpu(new_bc->overall_shared_limit);
11375
11376         /* fetch the current values */
11377         get_buffer_control(dd, &cur_bc, &cur_total);
11378
11379         /*
11380          * Create the masks we will use.
11381          */
11382         memset(changing, 0, sizeof(changing));
11383         memset(lowering_dedicated, 0, sizeof(lowering_dedicated));
11384         /*
11385          * NOTE: Assumes that the individual VL bits are adjacent and in
11386          * increasing order
11387          */
11388         stat_mask =
11389                 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK;
11390         changing_mask = 0;
11391         ld_mask = 0;
11392         change_count = 0;
11393         any_shared_limit_changing = 0;
11394         for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) {
11395                 if (!valid_vl(i))
11396                         continue;
11397                 this_shared_changing = new_bc->vl[i].shared
11398                                                 != cur_bc.vl[i].shared;
11399                 if (this_shared_changing)
11400                         any_shared_limit_changing = 1;
11401                 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated ||
11402                     this_shared_changing) {
11403                         changing[i] = 1;
11404                         changing_mask |= stat_mask;
11405                         change_count++;
11406                 }
11407                 if (be16_to_cpu(new_bc->vl[i].dedicated) <
11408                                         be16_to_cpu(cur_bc.vl[i].dedicated)) {
11409                         lowering_dedicated[i] = 1;
11410                         ld_mask |= stat_mask;
11411                 }
11412         }
11413
11414         /* bracket the credit change with a total adjustment */
11415         if (new_total > cur_total)
11416                 set_global_limit(dd, new_total);
11417
11418         /*
11419          * Start the credit change algorithm.
11420          */
11421         use_all_mask = 0;
11422         if ((be16_to_cpu(new_bc->overall_shared_limit) <
11423              be16_to_cpu(cur_bc.overall_shared_limit)) ||
11424             (is_ax(dd) && any_shared_limit_changing)) {
11425                 set_global_shared(dd, 0);
11426                 cur_bc.overall_shared_limit = 0;
11427                 use_all_mask = 1;
11428         }
11429
11430         for (i = 0; i < NUM_USABLE_VLS; i++) {
11431                 if (!valid_vl(i))
11432                         continue;
11433
11434                 if (changing[i]) {
11435                         set_vl_shared(dd, i, 0);
11436                         cur_bc.vl[i].shared = 0;
11437                 }
11438         }
11439
11440         wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask,
11441                                  "shared");
11442
11443         if (change_count > 0) {
11444                 for (i = 0; i < NUM_USABLE_VLS; i++) {
11445                         if (!valid_vl(i))
11446                                 continue;
11447
11448                         if (lowering_dedicated[i]) {
11449                                 set_vl_dedicated(dd, i,
11450                                                  be16_to_cpu(new_bc->
11451                                                              vl[i].dedicated));
11452                                 cur_bc.vl[i].dedicated =
11453                                                 new_bc->vl[i].dedicated;
11454                         }
11455                 }
11456
11457                 wait_for_vl_status_clear(dd, ld_mask, "dedicated");
11458
11459                 /* now raise all dedicated that are going up */
11460                 for (i = 0; i < NUM_USABLE_VLS; i++) {
11461                         if (!valid_vl(i))
11462                                 continue;
11463
11464                         if (be16_to_cpu(new_bc->vl[i].dedicated) >
11465                                         be16_to_cpu(cur_bc.vl[i].dedicated))
11466                                 set_vl_dedicated(dd, i,
11467                                                  be16_to_cpu(new_bc->
11468                                                              vl[i].dedicated));
11469                 }
11470         }
11471
11472         /* next raise all shared that are going up */
11473         for (i = 0; i < NUM_USABLE_VLS; i++) {
11474                 if (!valid_vl(i))
11475                         continue;
11476
11477                 if (be16_to_cpu(new_bc->vl[i].shared) >
11478                                 be16_to_cpu(cur_bc.vl[i].shared))
11479                         set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared));
11480         }
11481
11482         /* finally raise the global shared */
11483         if (be16_to_cpu(new_bc->overall_shared_limit) >
11484             be16_to_cpu(cur_bc.overall_shared_limit))
11485                 set_global_shared(dd,
11486                                   be16_to_cpu(new_bc->overall_shared_limit));
11487
11488         /* bracket the credit change with a total adjustment */
11489         if (new_total < cur_total)
11490                 set_global_limit(dd, new_total);
11491
11492         /*
11493          * Determine the actual number of operational VLS using the number of
11494          * dedicated and shared credits for each VL.
11495          */
11496         if (change_count > 0) {
11497                 for (i = 0; i < TXE_NUM_DATA_VL; i++)
11498                         if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 ||
11499                             be16_to_cpu(new_bc->vl[i].shared) > 0)
11500                                 vl_count++;
11501                 ppd->actual_vls_operational = vl_count;
11502                 ret = sdma_map_init(dd, ppd->port - 1, vl_count ?
11503                                     ppd->actual_vls_operational :
11504                                     ppd->vls_operational,
11505                                     NULL);
11506                 if (ret == 0)
11507                         ret = pio_map_init(dd, ppd->port - 1, vl_count ?
11508                                            ppd->actual_vls_operational :
11509                                            ppd->vls_operational, NULL);
11510                 if (ret)
11511                         return ret;
11512         }
11513         return 0;
11514 }
11515
11516 /*
11517  * Read the given fabric manager table. Return the size of the
11518  * table (in bytes) on success, and a negative error code on
11519  * failure.
11520  */
11521 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t)
11522
11523 {
11524         int size;
11525         struct vl_arb_cache *vlc;
11526
11527         switch (which) {
11528         case FM_TBL_VL_HIGH_ARB:
11529                 size = 256;
11530                 /*
11531                  * OPA specifies 128 elements (of 2 bytes each), though
11532                  * HFI supports only 16 elements in h/w.
11533                  */
11534                 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11535                 vl_arb_get_cache(vlc, t);
11536                 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11537                 break;
11538         case FM_TBL_VL_LOW_ARB:
11539                 size = 256;
11540                 /*
11541                  * OPA specifies 128 elements (of 2 bytes each), though
11542                  * HFI supports only 16 elements in h/w.
11543                  */
11544                 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11545                 vl_arb_get_cache(vlc, t);
11546                 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11547                 break;
11548         case FM_TBL_BUFFER_CONTROL:
11549                 size = get_buffer_control(ppd->dd, t, NULL);
11550                 break;
11551         case FM_TBL_SC2VLNT:
11552                 size = get_sc2vlnt(ppd->dd, t);
11553                 break;
11554         case FM_TBL_VL_PREEMPT_ELEMS:
11555                 size = 256;
11556                 /* OPA specifies 128 elements, of 2 bytes each */
11557                 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t);
11558                 break;
11559         case FM_TBL_VL_PREEMPT_MATRIX:
11560                 size = 256;
11561                 /*
11562                  * OPA specifies that this is the same size as the VL
11563                  * arbitration tables (i.e., 256 bytes).
11564                  */
11565                 break;
11566         default:
11567                 return -EINVAL;
11568         }
11569         return size;
11570 }
11571
11572 /*
11573  * Write the given fabric manager table.
11574  */
11575 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t)
11576 {
11577         int ret = 0;
11578         struct vl_arb_cache *vlc;
11579
11580         switch (which) {
11581         case FM_TBL_VL_HIGH_ARB:
11582                 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE);
11583                 if (vl_arb_match_cache(vlc, t)) {
11584                         vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11585                         break;
11586                 }
11587                 vl_arb_set_cache(vlc, t);
11588                 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE);
11589                 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST,
11590                                      VL_ARB_HIGH_PRIO_TABLE_SIZE, t);
11591                 break;
11592         case FM_TBL_VL_LOW_ARB:
11593                 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE);
11594                 if (vl_arb_match_cache(vlc, t)) {
11595                         vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11596                         break;
11597                 }
11598                 vl_arb_set_cache(vlc, t);
11599                 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE);
11600                 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST,
11601                                      VL_ARB_LOW_PRIO_TABLE_SIZE, t);
11602                 break;
11603         case FM_TBL_BUFFER_CONTROL:
11604                 ret = set_buffer_control(ppd, t);
11605                 break;
11606         case FM_TBL_SC2VLNT:
11607                 set_sc2vlnt(ppd->dd, t);
11608                 break;
11609         default:
11610                 ret = -EINVAL;
11611         }
11612         return ret;
11613 }
11614
11615 /*
11616  * Disable all data VLs.
11617  *
11618  * Return 0 if disabled, non-zero if the VLs cannot be disabled.
11619  */
11620 static int disable_data_vls(struct hfi1_devdata *dd)
11621 {
11622         if (is_ax(dd))
11623                 return 1;
11624
11625         pio_send_control(dd, PSC_DATA_VL_DISABLE);
11626
11627         return 0;
11628 }
11629
11630 /*
11631  * open_fill_data_vls() - the counterpart to stop_drain_data_vls().
11632  * Just re-enables all data VLs (the "fill" part happens
11633  * automatically - the name was chosen for symmetry with
11634  * stop_drain_data_vls()).
11635  *
11636  * Return 0 if successful, non-zero if the VLs cannot be enabled.
11637  */
11638 int open_fill_data_vls(struct hfi1_devdata *dd)
11639 {
11640         if (is_ax(dd))
11641                 return 1;
11642
11643         pio_send_control(dd, PSC_DATA_VL_ENABLE);
11644
11645         return 0;
11646 }
11647
11648 /*
11649  * drain_data_vls() - assumes that disable_data_vls() has been called,
11650  * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA
11651  * engines to drop to 0.
11652  */
11653 static void drain_data_vls(struct hfi1_devdata *dd)
11654 {
11655         sc_wait(dd);
11656         sdma_wait(dd);
11657         pause_for_credit_return(dd);
11658 }
11659
11660 /*
11661  * stop_drain_data_vls() - disable, then drain all per-VL fifos.
11662  *
11663  * Use open_fill_data_vls() to resume using data VLs.  This pair is
11664  * meant to be used like this:
11665  *
11666  * stop_drain_data_vls(dd);
11667  * // do things with per-VL resources
11668  * open_fill_data_vls(dd);
11669  */
11670 int stop_drain_data_vls(struct hfi1_devdata *dd)
11671 {
11672         int ret;
11673
11674         ret = disable_data_vls(dd);
11675         if (ret == 0)
11676                 drain_data_vls(dd);
11677
11678         return ret;
11679 }
11680
11681 /*
11682  * Convert a nanosecond time to a cclock count.  No matter how slow
11683  * the cclock, a non-zero ns will always have a non-zero result.
11684  */
11685 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns)
11686 {
11687         u32 cclocks;
11688
11689         if (dd->icode == ICODE_FPGA_EMULATION)
11690                 cclocks = (ns * 1000) / FPGA_CCLOCK_PS;
11691         else  /* simulation pretends to be ASIC */
11692                 cclocks = (ns * 1000) / ASIC_CCLOCK_PS;
11693         if (ns && !cclocks)     /* if ns nonzero, must be at least 1 */
11694                 cclocks = 1;
11695         return cclocks;
11696 }
11697
11698 /*
11699  * Convert a cclock count to nanoseconds. Not matter how slow
11700  * the cclock, a non-zero cclocks will always have a non-zero result.
11701  */
11702 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks)
11703 {
11704         u32 ns;
11705
11706         if (dd->icode == ICODE_FPGA_EMULATION)
11707                 ns = (cclocks * FPGA_CCLOCK_PS) / 1000;
11708         else  /* simulation pretends to be ASIC */
11709                 ns = (cclocks * ASIC_CCLOCK_PS) / 1000;
11710         if (cclocks && !ns)
11711                 ns = 1;
11712         return ns;
11713 }
11714
11715 /*
11716  * Dynamically adjust the receive interrupt timeout for a context based on
11717  * incoming packet rate.
11718  *
11719  * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero.
11720  */
11721 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts)
11722 {
11723         struct hfi1_devdata *dd = rcd->dd;
11724         u32 timeout = rcd->rcvavail_timeout;
11725
11726         /*
11727          * This algorithm doubles or halves the timeout depending on whether
11728          * the number of packets received in this interrupt were less than or
11729          * greater equal the interrupt count.
11730          *
11731          * The calculations below do not allow a steady state to be achieved.
11732          * Only at the endpoints it is possible to have an unchanging
11733          * timeout.
11734          */
11735         if (npkts < rcv_intr_count) {
11736                 /*
11737                  * Not enough packets arrived before the timeout, adjust
11738                  * timeout downward.
11739                  */
11740                 if (timeout < 2) /* already at minimum? */
11741                         return;
11742                 timeout >>= 1;
11743         } else {
11744                 /*
11745                  * More than enough packets arrived before the timeout, adjust
11746                  * timeout upward.
11747                  */
11748                 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */
11749                         return;
11750                 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr);
11751         }
11752
11753         rcd->rcvavail_timeout = timeout;
11754         /*
11755          * timeout cannot be larger than rcv_intr_timeout_csr which has already
11756          * been verified to be in range
11757          */
11758         write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT,
11759                         (u64)timeout <<
11760                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
11761 }
11762
11763 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
11764                     u32 intr_adjust, u32 npkts)
11765 {
11766         struct hfi1_devdata *dd = rcd->dd;
11767         u64 reg;
11768         u32 ctxt = rcd->ctxt;
11769
11770         /*
11771          * Need to write timeout register before updating RcvHdrHead to ensure
11772          * that a new value is used when the HW decides to restart counting.
11773          */
11774         if (intr_adjust)
11775                 adjust_rcv_timeout(rcd, npkts);
11776         if (updegr) {
11777                 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK)
11778                         << RCV_EGR_INDEX_HEAD_HEAD_SHIFT;
11779                 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg);
11780         }
11781         mmiowb();
11782         reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) |
11783                 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK)
11784                         << RCV_HDR_HEAD_HEAD_SHIFT);
11785         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
11786         mmiowb();
11787 }
11788
11789 u32 hdrqempty(struct hfi1_ctxtdata *rcd)
11790 {
11791         u32 head, tail;
11792
11793         head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD)
11794                 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT;
11795
11796         if (rcd->rcvhdrtail_kvaddr)
11797                 tail = get_rcvhdrtail(rcd);
11798         else
11799                 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL);
11800
11801         return head == tail;
11802 }
11803
11804 /*
11805  * Context Control and Receive Array encoding for buffer size:
11806  *      0x0 invalid
11807  *      0x1   4 KB
11808  *      0x2   8 KB
11809  *      0x3  16 KB
11810  *      0x4  32 KB
11811  *      0x5  64 KB
11812  *      0x6 128 KB
11813  *      0x7 256 KB
11814  *      0x8 512 KB (Receive Array only)
11815  *      0x9   1 MB (Receive Array only)
11816  *      0xa   2 MB (Receive Array only)
11817  *
11818  *      0xB-0xF - reserved (Receive Array only)
11819  *
11820  *
11821  * This routine assumes that the value has already been sanity checked.
11822  */
11823 static u32 encoded_size(u32 size)
11824 {
11825         switch (size) {
11826         case   4 * 1024: return 0x1;
11827         case   8 * 1024: return 0x2;
11828         case  16 * 1024: return 0x3;
11829         case  32 * 1024: return 0x4;
11830         case  64 * 1024: return 0x5;
11831         case 128 * 1024: return 0x6;
11832         case 256 * 1024: return 0x7;
11833         case 512 * 1024: return 0x8;
11834         case   1 * 1024 * 1024: return 0x9;
11835         case   2 * 1024 * 1024: return 0xa;
11836         }
11837         return 0x1;     /* if invalid, go with the minimum size */
11838 }
11839
11840 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op,
11841                   struct hfi1_ctxtdata *rcd)
11842 {
11843         u64 rcvctrl, reg;
11844         int did_enable = 0;
11845         u16 ctxt;
11846
11847         if (!rcd)
11848                 return;
11849
11850         ctxt = rcd->ctxt;
11851
11852         hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op);
11853
11854         rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL);
11855         /* if the context already enabled, don't do the extra steps */
11856         if ((op & HFI1_RCVCTRL_CTXT_ENB) &&
11857             !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) {
11858                 /* reset the tail and hdr addresses, and sequence count */
11859                 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR,
11860                                 rcd->rcvhdrq_dma);
11861                 if (rcd->rcvhdrtail_kvaddr)
11862                         write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11863                                         rcd->rcvhdrqtailaddr_dma);
11864                 rcd->seq_cnt = 1;
11865
11866                 /* reset the cached receive header queue head value */
11867                 rcd->head = 0;
11868
11869                 /*
11870                  * Zero the receive header queue so we don't get false
11871                  * positives when checking the sequence number.  The
11872                  * sequence numbers could land exactly on the same spot.
11873                  * E.g. a rcd restart before the receive header wrapped.
11874                  */
11875                 memset(rcd->rcvhdrq, 0, rcvhdrq_size(rcd));
11876
11877                 /* starting timeout */
11878                 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr;
11879
11880                 /* enable the context */
11881                 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK;
11882
11883                 /* clean the egr buffer size first */
11884                 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11885                 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size)
11886                                 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK)
11887                                         << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT;
11888
11889                 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */
11890                 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0);
11891                 did_enable = 1;
11892
11893                 /* zero RcvEgrIndexHead */
11894                 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0);
11895
11896                 /* set eager count and base index */
11897                 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT)
11898                         & RCV_EGR_CTRL_EGR_CNT_MASK)
11899                        << RCV_EGR_CTRL_EGR_CNT_SHIFT) |
11900                         (((rcd->eager_base >> RCV_SHIFT)
11901                           & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK)
11902                          << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT);
11903                 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg);
11904
11905                 /*
11906                  * Set TID (expected) count and base index.
11907                  * rcd->expected_count is set to individual RcvArray entries,
11908                  * not pairs, and the CSR takes a pair-count in groups of
11909                  * four, so divide by 8.
11910                  */
11911                 reg = (((rcd->expected_count >> RCV_SHIFT)
11912                                         & RCV_TID_CTRL_TID_PAIR_CNT_MASK)
11913                                 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) |
11914                       (((rcd->expected_base >> RCV_SHIFT)
11915                                         & RCV_TID_CTRL_TID_BASE_INDEX_MASK)
11916                                 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT);
11917                 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg);
11918                 if (ctxt == HFI1_CTRL_CTXT)
11919                         write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT);
11920         }
11921         if (op & HFI1_RCVCTRL_CTXT_DIS) {
11922                 write_csr(dd, RCV_VL15, 0);
11923                 /*
11924                  * When receive context is being disabled turn on tail
11925                  * update with a dummy tail address and then disable
11926                  * receive context.
11927                  */
11928                 if (dd->rcvhdrtail_dummy_dma) {
11929                         write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
11930                                         dd->rcvhdrtail_dummy_dma);
11931                         /* Enabling RcvCtxtCtrl.TailUpd is intentional. */
11932                         rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11933                 }
11934
11935                 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK;
11936         }
11937         if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) {
11938                 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
11939                               IS_RCVAVAIL_START + rcd->ctxt, true);
11940                 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11941         }
11942         if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) {
11943                 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt,
11944                               IS_RCVAVAIL_START + rcd->ctxt, false);
11945                 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK;
11946         }
11947         if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && rcd->rcvhdrtail_kvaddr)
11948                 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11949         if (op & HFI1_RCVCTRL_TAILUPD_DIS) {
11950                 /* See comment on RcvCtxtCtrl.TailUpd above */
11951                 if (!(op & HFI1_RCVCTRL_CTXT_DIS))
11952                         rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK;
11953         }
11954         if (op & HFI1_RCVCTRL_TIDFLOW_ENB)
11955                 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11956         if (op & HFI1_RCVCTRL_TIDFLOW_DIS)
11957                 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK;
11958         if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) {
11959                 /*
11960                  * In one-packet-per-eager mode, the size comes from
11961                  * the RcvArray entry.
11962                  */
11963                 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK;
11964                 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11965         }
11966         if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS)
11967                 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK;
11968         if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB)
11969                 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11970         if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS)
11971                 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK;
11972         if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB)
11973                 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11974         if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS)
11975                 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK;
11976         if (op & HFI1_RCVCTRL_URGENT_ENB)
11977                 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
11978                               IS_RCVURGENT_START + rcd->ctxt, true);
11979         if (op & HFI1_RCVCTRL_URGENT_DIS)
11980                 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt,
11981                               IS_RCVURGENT_START + rcd->ctxt, false);
11982
11983         hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl);
11984         write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl);
11985
11986         /* work around sticky RcvCtxtStatus.BlockedRHQFull */
11987         if (did_enable &&
11988             (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) {
11989                 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11990                 if (reg != 0) {
11991                         dd_dev_info(dd, "ctxt %d status %lld (blocked)\n",
11992                                     ctxt, reg);
11993                         read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11994                         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10);
11995                         write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00);
11996                         read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD);
11997                         reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS);
11998                         dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n",
11999                                     ctxt, reg, reg == 0 ? "not" : "still");
12000                 }
12001         }
12002
12003         if (did_enable) {
12004                 /*
12005                  * The interrupt timeout and count must be set after
12006                  * the context is enabled to take effect.
12007                  */
12008                 /* set interrupt timeout */
12009                 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT,
12010                                 (u64)rcd->rcvavail_timeout <<
12011                                 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT);
12012
12013                 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */
12014                 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT;
12015                 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg);
12016         }
12017
12018         if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS))
12019                 /*
12020                  * If the context has been disabled and the Tail Update has
12021                  * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address
12022                  * so it doesn't contain an address that is invalid.
12023                  */
12024                 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR,
12025                                 dd->rcvhdrtail_dummy_dma);
12026 }
12027
12028 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp)
12029 {
12030         int ret;
12031         u64 val = 0;
12032
12033         if (namep) {
12034                 ret = dd->cntrnameslen;
12035                 *namep = dd->cntrnames;
12036         } else {
12037                 const struct cntr_entry *entry;
12038                 int i, j;
12039
12040                 ret = (dd->ndevcntrs) * sizeof(u64);
12041
12042                 /* Get the start of the block of counters */
12043                 *cntrp = dd->cntrs;
12044
12045                 /*
12046                  * Now go and fill in each counter in the block.
12047                  */
12048                 for (i = 0; i < DEV_CNTR_LAST; i++) {
12049                         entry = &dev_cntrs[i];
12050                         hfi1_cdbg(CNTR, "reading %s", entry->name);
12051                         if (entry->flags & CNTR_DISABLED) {
12052                                 /* Nothing */
12053                                 hfi1_cdbg(CNTR, "\tDisabled\n");
12054                         } else {
12055                                 if (entry->flags & CNTR_VL) {
12056                                         hfi1_cdbg(CNTR, "\tPer VL\n");
12057                                         for (j = 0; j < C_VL_COUNT; j++) {
12058                                                 val = entry->rw_cntr(entry,
12059                                                                   dd, j,
12060                                                                   CNTR_MODE_R,
12061                                                                   0);
12062                                                 hfi1_cdbg(
12063                                                    CNTR,
12064                                                    "\t\tRead 0x%llx for %d\n",
12065                                                    val, j);
12066                                                 dd->cntrs[entry->offset + j] =
12067                                                                             val;
12068                                         }
12069                                 } else if (entry->flags & CNTR_SDMA) {
12070                                         hfi1_cdbg(CNTR,
12071                                                   "\t Per SDMA Engine\n");
12072                                         for (j = 0; j < chip_sdma_engines(dd);
12073                                              j++) {
12074                                                 val =
12075                                                 entry->rw_cntr(entry, dd, j,
12076                                                                CNTR_MODE_R, 0);
12077                                                 hfi1_cdbg(CNTR,
12078                                                           "\t\tRead 0x%llx for %d\n",
12079                                                           val, j);
12080                                                 dd->cntrs[entry->offset + j] =
12081                                                                         val;
12082                                         }
12083                                 } else {
12084                                         val = entry->rw_cntr(entry, dd,
12085                                                         CNTR_INVALID_VL,
12086                                                         CNTR_MODE_R, 0);
12087                                         dd->cntrs[entry->offset] = val;
12088                                         hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12089                                 }
12090                         }
12091                 }
12092         }
12093         return ret;
12094 }
12095
12096 /*
12097  * Used by sysfs to create files for hfi stats to read
12098  */
12099 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp)
12100 {
12101         int ret;
12102         u64 val = 0;
12103
12104         if (namep) {
12105                 ret = ppd->dd->portcntrnameslen;
12106                 *namep = ppd->dd->portcntrnames;
12107         } else {
12108                 const struct cntr_entry *entry;
12109                 int i, j;
12110
12111                 ret = ppd->dd->nportcntrs * sizeof(u64);
12112                 *cntrp = ppd->cntrs;
12113
12114                 for (i = 0; i < PORT_CNTR_LAST; i++) {
12115                         entry = &port_cntrs[i];
12116                         hfi1_cdbg(CNTR, "reading %s", entry->name);
12117                         if (entry->flags & CNTR_DISABLED) {
12118                                 /* Nothing */
12119                                 hfi1_cdbg(CNTR, "\tDisabled\n");
12120                                 continue;
12121                         }
12122
12123                         if (entry->flags & CNTR_VL) {
12124                                 hfi1_cdbg(CNTR, "\tPer VL");
12125                                 for (j = 0; j < C_VL_COUNT; j++) {
12126                                         val = entry->rw_cntr(entry, ppd, j,
12127                                                                CNTR_MODE_R,
12128                                                                0);
12129                                         hfi1_cdbg(
12130                                            CNTR,
12131                                            "\t\tRead 0x%llx for %d",
12132                                            val, j);
12133                                         ppd->cntrs[entry->offset + j] = val;
12134                                 }
12135                         } else {
12136                                 val = entry->rw_cntr(entry, ppd,
12137                                                        CNTR_INVALID_VL,
12138                                                        CNTR_MODE_R,
12139                                                        0);
12140                                 ppd->cntrs[entry->offset] = val;
12141                                 hfi1_cdbg(CNTR, "\tRead 0x%llx", val);
12142                         }
12143                 }
12144         }
12145         return ret;
12146 }
12147
12148 static void free_cntrs(struct hfi1_devdata *dd)
12149 {
12150         struct hfi1_pportdata *ppd;
12151         int i;
12152
12153         if (dd->synth_stats_timer.function)
12154                 del_timer_sync(&dd->synth_stats_timer);
12155         ppd = (struct hfi1_pportdata *)(dd + 1);
12156         for (i = 0; i < dd->num_pports; i++, ppd++) {
12157                 kfree(ppd->cntrs);
12158                 kfree(ppd->scntrs);
12159                 free_percpu(ppd->ibport_data.rvp.rc_acks);
12160                 free_percpu(ppd->ibport_data.rvp.rc_qacks);
12161                 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
12162                 ppd->cntrs = NULL;
12163                 ppd->scntrs = NULL;
12164                 ppd->ibport_data.rvp.rc_acks = NULL;
12165                 ppd->ibport_data.rvp.rc_qacks = NULL;
12166                 ppd->ibport_data.rvp.rc_delayed_comp = NULL;
12167         }
12168         kfree(dd->portcntrnames);
12169         dd->portcntrnames = NULL;
12170         kfree(dd->cntrs);
12171         dd->cntrs = NULL;
12172         kfree(dd->scntrs);
12173         dd->scntrs = NULL;
12174         kfree(dd->cntrnames);
12175         dd->cntrnames = NULL;
12176         if (dd->update_cntr_wq) {
12177                 destroy_workqueue(dd->update_cntr_wq);
12178                 dd->update_cntr_wq = NULL;
12179         }
12180 }
12181
12182 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry,
12183                               u64 *psval, void *context, int vl)
12184 {
12185         u64 val;
12186         u64 sval = *psval;
12187
12188         if (entry->flags & CNTR_DISABLED) {
12189                 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12190                 return 0;
12191         }
12192
12193         hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12194
12195         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0);
12196
12197         /* If its a synthetic counter there is more work we need to do */
12198         if (entry->flags & CNTR_SYNTH) {
12199                 if (sval == CNTR_MAX) {
12200                         /* No need to read already saturated */
12201                         return CNTR_MAX;
12202                 }
12203
12204                 if (entry->flags & CNTR_32BIT) {
12205                         /* 32bit counters can wrap multiple times */
12206                         u64 upper = sval >> 32;
12207                         u64 lower = (sval << 32) >> 32;
12208
12209                         if (lower > val) { /* hw wrapped */
12210                                 if (upper == CNTR_32BIT_MAX)
12211                                         val = CNTR_MAX;
12212                                 else
12213                                         upper++;
12214                         }
12215
12216                         if (val != CNTR_MAX)
12217                                 val = (upper << 32) | val;
12218
12219                 } else {
12220                         /* If we rolled we are saturated */
12221                         if ((val < sval) || (val > CNTR_MAX))
12222                                 val = CNTR_MAX;
12223                 }
12224         }
12225
12226         *psval = val;
12227
12228         hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12229
12230         return val;
12231 }
12232
12233 static u64 write_dev_port_cntr(struct hfi1_devdata *dd,
12234                                struct cntr_entry *entry,
12235                                u64 *psval, void *context, int vl, u64 data)
12236 {
12237         u64 val;
12238
12239         if (entry->flags & CNTR_DISABLED) {
12240                 dd_dev_err(dd, "Counter %s not enabled", entry->name);
12241                 return 0;
12242         }
12243
12244         hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval);
12245
12246         if (entry->flags & CNTR_SYNTH) {
12247                 *psval = data;
12248                 if (entry->flags & CNTR_32BIT) {
12249                         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12250                                              (data << 32) >> 32);
12251                         val = data; /* return the full 64bit value */
12252                 } else {
12253                         val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W,
12254                                              data);
12255                 }
12256         } else {
12257                 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data);
12258         }
12259
12260         *psval = val;
12261
12262         hfi1_cdbg(CNTR, "\tNew val=0x%llx", val);
12263
12264         return val;
12265 }
12266
12267 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl)
12268 {
12269         struct cntr_entry *entry;
12270         u64 *sval;
12271
12272         entry = &dev_cntrs[index];
12273         sval = dd->scntrs + entry->offset;
12274
12275         if (vl != CNTR_INVALID_VL)
12276                 sval += vl;
12277
12278         return read_dev_port_cntr(dd, entry, sval, dd, vl);
12279 }
12280
12281 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data)
12282 {
12283         struct cntr_entry *entry;
12284         u64 *sval;
12285
12286         entry = &dev_cntrs[index];
12287         sval = dd->scntrs + entry->offset;
12288
12289         if (vl != CNTR_INVALID_VL)
12290                 sval += vl;
12291
12292         return write_dev_port_cntr(dd, entry, sval, dd, vl, data);
12293 }
12294
12295 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl)
12296 {
12297         struct cntr_entry *entry;
12298         u64 *sval;
12299
12300         entry = &port_cntrs[index];
12301         sval = ppd->scntrs + entry->offset;
12302
12303         if (vl != CNTR_INVALID_VL)
12304                 sval += vl;
12305
12306         if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12307             (index <= C_RCV_HDR_OVF_LAST)) {
12308                 /* We do not want to bother for disabled contexts */
12309                 return 0;
12310         }
12311
12312         return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl);
12313 }
12314
12315 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data)
12316 {
12317         struct cntr_entry *entry;
12318         u64 *sval;
12319
12320         entry = &port_cntrs[index];
12321         sval = ppd->scntrs + entry->offset;
12322
12323         if (vl != CNTR_INVALID_VL)
12324                 sval += vl;
12325
12326         if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) &&
12327             (index <= C_RCV_HDR_OVF_LAST)) {
12328                 /* We do not want to bother for disabled contexts */
12329                 return 0;
12330         }
12331
12332         return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data);
12333 }
12334
12335 static void do_update_synth_timer(struct work_struct *work)
12336 {
12337         u64 cur_tx;
12338         u64 cur_rx;
12339         u64 total_flits;
12340         u8 update = 0;
12341         int i, j, vl;
12342         struct hfi1_pportdata *ppd;
12343         struct cntr_entry *entry;
12344         struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata,
12345                                                update_cntr_work);
12346
12347         /*
12348          * Rather than keep beating on the CSRs pick a minimal set that we can
12349          * check to watch for potential roll over. We can do this by looking at
12350          * the number of flits sent/recv. If the total flits exceeds 32bits then
12351          * we have to iterate all the counters and update.
12352          */
12353         entry = &dev_cntrs[C_DC_RCV_FLITS];
12354         cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12355
12356         entry = &dev_cntrs[C_DC_XMIT_FLITS];
12357         cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0);
12358
12359         hfi1_cdbg(
12360             CNTR,
12361             "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n",
12362             dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx);
12363
12364         if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) {
12365                 /*
12366                  * May not be strictly necessary to update but it won't hurt and
12367                  * simplifies the logic here.
12368                  */
12369                 update = 1;
12370                 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating",
12371                           dd->unit);
12372         } else {
12373                 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx);
12374                 hfi1_cdbg(CNTR,
12375                           "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit,
12376                           total_flits, (u64)CNTR_32BIT_MAX);
12377                 if (total_flits >= CNTR_32BIT_MAX) {
12378                         hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating",
12379                                   dd->unit);
12380                         update = 1;
12381                 }
12382         }
12383
12384         if (update) {
12385                 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit);
12386                 for (i = 0; i < DEV_CNTR_LAST; i++) {
12387                         entry = &dev_cntrs[i];
12388                         if (entry->flags & CNTR_VL) {
12389                                 for (vl = 0; vl < C_VL_COUNT; vl++)
12390                                         read_dev_cntr(dd, i, vl);
12391                         } else {
12392                                 read_dev_cntr(dd, i, CNTR_INVALID_VL);
12393                         }
12394                 }
12395                 ppd = (struct hfi1_pportdata *)(dd + 1);
12396                 for (i = 0; i < dd->num_pports; i++, ppd++) {
12397                         for (j = 0; j < PORT_CNTR_LAST; j++) {
12398                                 entry = &port_cntrs[j];
12399                                 if (entry->flags & CNTR_VL) {
12400                                         for (vl = 0; vl < C_VL_COUNT; vl++)
12401                                                 read_port_cntr(ppd, j, vl);
12402                                 } else {
12403                                         read_port_cntr(ppd, j, CNTR_INVALID_VL);
12404                                 }
12405                         }
12406                 }
12407
12408                 /*
12409                  * We want the value in the register. The goal is to keep track
12410                  * of the number of "ticks" not the counter value. In other
12411                  * words if the register rolls we want to notice it and go ahead
12412                  * and force an update.
12413                  */
12414                 entry = &dev_cntrs[C_DC_XMIT_FLITS];
12415                 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12416                                                 CNTR_MODE_R, 0);
12417
12418                 entry = &dev_cntrs[C_DC_RCV_FLITS];
12419                 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL,
12420                                                 CNTR_MODE_R, 0);
12421
12422                 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx",
12423                           dd->unit, dd->last_tx, dd->last_rx);
12424
12425         } else {
12426                 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit);
12427         }
12428 }
12429
12430 static void update_synth_timer(struct timer_list *t)
12431 {
12432         struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer);
12433
12434         queue_work(dd->update_cntr_wq, &dd->update_cntr_work);
12435         mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12436 }
12437
12438 #define C_MAX_NAME 16 /* 15 chars + one for /0 */
12439 static int init_cntrs(struct hfi1_devdata *dd)
12440 {
12441         int i, rcv_ctxts, j;
12442         size_t sz;
12443         char *p;
12444         char name[C_MAX_NAME];
12445         struct hfi1_pportdata *ppd;
12446         const char *bit_type_32 = ",32";
12447         const int bit_type_32_sz = strlen(bit_type_32);
12448         u32 sdma_engines = chip_sdma_engines(dd);
12449
12450         /* set up the stats timer; the add_timer is done at the end */
12451         timer_setup(&dd->synth_stats_timer, update_synth_timer, 0);
12452
12453         /***********************/
12454         /* per device counters */
12455         /***********************/
12456
12457         /* size names and determine how many we have*/
12458         dd->ndevcntrs = 0;
12459         sz = 0;
12460
12461         for (i = 0; i < DEV_CNTR_LAST; i++) {
12462                 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12463                         hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name);
12464                         continue;
12465                 }
12466
12467                 if (dev_cntrs[i].flags & CNTR_VL) {
12468                         dev_cntrs[i].offset = dd->ndevcntrs;
12469                         for (j = 0; j < C_VL_COUNT; j++) {
12470                                 snprintf(name, C_MAX_NAME, "%s%d",
12471                                          dev_cntrs[i].name, vl_from_idx(j));
12472                                 sz += strlen(name);
12473                                 /* Add ",32" for 32-bit counters */
12474                                 if (dev_cntrs[i].flags & CNTR_32BIT)
12475                                         sz += bit_type_32_sz;
12476                                 sz++;
12477                                 dd->ndevcntrs++;
12478                         }
12479                 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12480                         dev_cntrs[i].offset = dd->ndevcntrs;
12481                         for (j = 0; j < sdma_engines; j++) {
12482                                 snprintf(name, C_MAX_NAME, "%s%d",
12483                                          dev_cntrs[i].name, j);
12484                                 sz += strlen(name);
12485                                 /* Add ",32" for 32-bit counters */
12486                                 if (dev_cntrs[i].flags & CNTR_32BIT)
12487                                         sz += bit_type_32_sz;
12488                                 sz++;
12489                                 dd->ndevcntrs++;
12490                         }
12491                 } else {
12492                         /* +1 for newline. */
12493                         sz += strlen(dev_cntrs[i].name) + 1;
12494                         /* Add ",32" for 32-bit counters */
12495                         if (dev_cntrs[i].flags & CNTR_32BIT)
12496                                 sz += bit_type_32_sz;
12497                         dev_cntrs[i].offset = dd->ndevcntrs;
12498                         dd->ndevcntrs++;
12499                 }
12500         }
12501
12502         /* allocate space for the counter values */
12503         dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12504         if (!dd->cntrs)
12505                 goto bail;
12506
12507         dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL);
12508         if (!dd->scntrs)
12509                 goto bail;
12510
12511         /* allocate space for the counter names */
12512         dd->cntrnameslen = sz;
12513         dd->cntrnames = kmalloc(sz, GFP_KERNEL);
12514         if (!dd->cntrnames)
12515                 goto bail;
12516
12517         /* fill in the names */
12518         for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) {
12519                 if (dev_cntrs[i].flags & CNTR_DISABLED) {
12520                         /* Nothing */
12521                 } else if (dev_cntrs[i].flags & CNTR_VL) {
12522                         for (j = 0; j < C_VL_COUNT; j++) {
12523                                 snprintf(name, C_MAX_NAME, "%s%d",
12524                                          dev_cntrs[i].name,
12525                                          vl_from_idx(j));
12526                                 memcpy(p, name, strlen(name));
12527                                 p += strlen(name);
12528
12529                                 /* Counter is 32 bits */
12530                                 if (dev_cntrs[i].flags & CNTR_32BIT) {
12531                                         memcpy(p, bit_type_32, bit_type_32_sz);
12532                                         p += bit_type_32_sz;
12533                                 }
12534
12535                                 *p++ = '\n';
12536                         }
12537                 } else if (dev_cntrs[i].flags & CNTR_SDMA) {
12538                         for (j = 0; j < sdma_engines; j++) {
12539                                 snprintf(name, C_MAX_NAME, "%s%d",
12540                                          dev_cntrs[i].name, j);
12541                                 memcpy(p, name, strlen(name));
12542                                 p += strlen(name);
12543
12544                                 /* Counter is 32 bits */
12545                                 if (dev_cntrs[i].flags & CNTR_32BIT) {
12546                                         memcpy(p, bit_type_32, bit_type_32_sz);
12547                                         p += bit_type_32_sz;
12548                                 }
12549
12550                                 *p++ = '\n';
12551                         }
12552                 } else {
12553                         memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name));
12554                         p += strlen(dev_cntrs[i].name);
12555
12556                         /* Counter is 32 bits */
12557                         if (dev_cntrs[i].flags & CNTR_32BIT) {
12558                                 memcpy(p, bit_type_32, bit_type_32_sz);
12559                                 p += bit_type_32_sz;
12560                         }
12561
12562                         *p++ = '\n';
12563                 }
12564         }
12565
12566         /*********************/
12567         /* per port counters */
12568         /*********************/
12569
12570         /*
12571          * Go through the counters for the overflows and disable the ones we
12572          * don't need. This varies based on platform so we need to do it
12573          * dynamically here.
12574          */
12575         rcv_ctxts = dd->num_rcv_contexts;
12576         for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts;
12577              i <= C_RCV_HDR_OVF_LAST; i++) {
12578                 port_cntrs[i].flags |= CNTR_DISABLED;
12579         }
12580
12581         /* size port counter names and determine how many we have*/
12582         sz = 0;
12583         dd->nportcntrs = 0;
12584         for (i = 0; i < PORT_CNTR_LAST; i++) {
12585                 if (port_cntrs[i].flags & CNTR_DISABLED) {
12586                         hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name);
12587                         continue;
12588                 }
12589
12590                 if (port_cntrs[i].flags & CNTR_VL) {
12591                         port_cntrs[i].offset = dd->nportcntrs;
12592                         for (j = 0; j < C_VL_COUNT; j++) {
12593                                 snprintf(name, C_MAX_NAME, "%s%d",
12594                                          port_cntrs[i].name, vl_from_idx(j));
12595                                 sz += strlen(name);
12596                                 /* Add ",32" for 32-bit counters */
12597                                 if (port_cntrs[i].flags & CNTR_32BIT)
12598                                         sz += bit_type_32_sz;
12599                                 sz++;
12600                                 dd->nportcntrs++;
12601                         }
12602                 } else {
12603                         /* +1 for newline */
12604                         sz += strlen(port_cntrs[i].name) + 1;
12605                         /* Add ",32" for 32-bit counters */
12606                         if (port_cntrs[i].flags & CNTR_32BIT)
12607                                 sz += bit_type_32_sz;
12608                         port_cntrs[i].offset = dd->nportcntrs;
12609                         dd->nportcntrs++;
12610                 }
12611         }
12612
12613         /* allocate space for the counter names */
12614         dd->portcntrnameslen = sz;
12615         dd->portcntrnames = kmalloc(sz, GFP_KERNEL);
12616         if (!dd->portcntrnames)
12617                 goto bail;
12618
12619         /* fill in port cntr names */
12620         for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) {
12621                 if (port_cntrs[i].flags & CNTR_DISABLED)
12622                         continue;
12623
12624                 if (port_cntrs[i].flags & CNTR_VL) {
12625                         for (j = 0; j < C_VL_COUNT; j++) {
12626                                 snprintf(name, C_MAX_NAME, "%s%d",
12627                                          port_cntrs[i].name, vl_from_idx(j));
12628                                 memcpy(p, name, strlen(name));
12629                                 p += strlen(name);
12630
12631                                 /* Counter is 32 bits */
12632                                 if (port_cntrs[i].flags & CNTR_32BIT) {
12633                                         memcpy(p, bit_type_32, bit_type_32_sz);
12634                                         p += bit_type_32_sz;
12635                                 }
12636
12637                                 *p++ = '\n';
12638                         }
12639                 } else {
12640                         memcpy(p, port_cntrs[i].name,
12641                                strlen(port_cntrs[i].name));
12642                         p += strlen(port_cntrs[i].name);
12643
12644                         /* Counter is 32 bits */
12645                         if (port_cntrs[i].flags & CNTR_32BIT) {
12646                                 memcpy(p, bit_type_32, bit_type_32_sz);
12647                                 p += bit_type_32_sz;
12648                         }
12649
12650                         *p++ = '\n';
12651                 }
12652         }
12653
12654         /* allocate per port storage for counter values */
12655         ppd = (struct hfi1_pportdata *)(dd + 1);
12656         for (i = 0; i < dd->num_pports; i++, ppd++) {
12657                 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12658                 if (!ppd->cntrs)
12659                         goto bail;
12660
12661                 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL);
12662                 if (!ppd->scntrs)
12663                         goto bail;
12664         }
12665
12666         /* CPU counters need to be allocated and zeroed */
12667         if (init_cpu_counters(dd))
12668                 goto bail;
12669
12670         dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d",
12671                                                      WQ_MEM_RECLAIM, dd->unit);
12672         if (!dd->update_cntr_wq)
12673                 goto bail;
12674
12675         INIT_WORK(&dd->update_cntr_work, do_update_synth_timer);
12676
12677         mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME);
12678         return 0;
12679 bail:
12680         free_cntrs(dd);
12681         return -ENOMEM;
12682 }
12683
12684 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate)
12685 {
12686         switch (chip_lstate) {
12687         default:
12688                 dd_dev_err(dd,
12689                            "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n",
12690                            chip_lstate);
12691                 /* fall through */
12692         case LSTATE_DOWN:
12693                 return IB_PORT_DOWN;
12694         case LSTATE_INIT:
12695                 return IB_PORT_INIT;
12696         case LSTATE_ARMED:
12697                 return IB_PORT_ARMED;
12698         case LSTATE_ACTIVE:
12699                 return IB_PORT_ACTIVE;
12700         }
12701 }
12702
12703 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)
12704 {
12705         /* look at the HFI meta-states only */
12706         switch (chip_pstate & 0xf0) {
12707         default:
12708                 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n",
12709                            chip_pstate);
12710                 /* fall through */
12711         case PLS_DISABLED:
12712                 return IB_PORTPHYSSTATE_DISABLED;
12713         case PLS_OFFLINE:
12714                 return OPA_PORTPHYSSTATE_OFFLINE;
12715         case PLS_POLLING:
12716                 return IB_PORTPHYSSTATE_POLLING;
12717         case PLS_CONFIGPHY:
12718                 return IB_PORTPHYSSTATE_TRAINING;
12719         case PLS_LINKUP:
12720                 return IB_PORTPHYSSTATE_LINKUP;
12721         case PLS_PHYTEST:
12722                 return IB_PORTPHYSSTATE_PHY_TEST;
12723         }
12724 }
12725
12726 /* return the OPA port logical state name */
12727 const char *opa_lstate_name(u32 lstate)
12728 {
12729         static const char * const port_logical_names[] = {
12730                 "PORT_NOP",
12731                 "PORT_DOWN",
12732                 "PORT_INIT",
12733                 "PORT_ARMED",
12734                 "PORT_ACTIVE",
12735                 "PORT_ACTIVE_DEFER",
12736         };
12737         if (lstate < ARRAY_SIZE(port_logical_names))
12738                 return port_logical_names[lstate];
12739         return "unknown";
12740 }
12741
12742 /* return the OPA port physical state name */
12743 const char *opa_pstate_name(u32 pstate)
12744 {
12745         static const char * const port_physical_names[] = {
12746                 "PHYS_NOP",
12747                 "reserved1",
12748                 "PHYS_POLL",
12749                 "PHYS_DISABLED",
12750                 "PHYS_TRAINING",
12751                 "PHYS_LINKUP",
12752                 "PHYS_LINK_ERR_RECOVER",
12753                 "PHYS_PHY_TEST",
12754                 "reserved8",
12755                 "PHYS_OFFLINE",
12756                 "PHYS_GANGED",
12757                 "PHYS_TEST",
12758         };
12759         if (pstate < ARRAY_SIZE(port_physical_names))
12760                 return port_physical_names[pstate];
12761         return "unknown";
12762 }
12763
12764 /**
12765  * update_statusp - Update userspace status flag
12766  * @ppd: Port data structure
12767  * @state: port state information
12768  *
12769  * Actual port status is determined by the host_link_state value
12770  * in the ppd.
12771  *
12772  * host_link_state MUST be updated before updating the user space
12773  * statusp.
12774  */
12775 static void update_statusp(struct hfi1_pportdata *ppd, u32 state)
12776 {
12777         /*
12778          * Set port status flags in the page mapped into userspace
12779          * memory. Do it here to ensure a reliable state - this is
12780          * the only function called by all state handling code.
12781          * Always set the flags due to the fact that the cache value
12782          * might have been changed explicitly outside of this
12783          * function.
12784          */
12785         if (ppd->statusp) {
12786                 switch (state) {
12787                 case IB_PORT_DOWN:
12788                 case IB_PORT_INIT:
12789                         *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
12790                                            HFI1_STATUS_IB_READY);
12791                         break;
12792                 case IB_PORT_ARMED:
12793                         *ppd->statusp |= HFI1_STATUS_IB_CONF;
12794                         break;
12795                 case IB_PORT_ACTIVE:
12796                         *ppd->statusp |= HFI1_STATUS_IB_READY;
12797                         break;
12798                 }
12799         }
12800         dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n",
12801                     opa_lstate_name(state), state);
12802 }
12803
12804 /**
12805  * wait_logical_linkstate - wait for an IB link state change to occur
12806  * @ppd: port device
12807  * @state: the state to wait for
12808  * @msecs: the number of milliseconds to wait
12809  *
12810  * Wait up to msecs milliseconds for IB link state change to occur.
12811  * For now, take the easy polling route.
12812  * Returns 0 if state reached, otherwise -ETIMEDOUT.
12813  */
12814 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12815                                   int msecs)
12816 {
12817         unsigned long timeout;
12818         u32 new_state;
12819
12820         timeout = jiffies + msecs_to_jiffies(msecs);
12821         while (1) {
12822                 new_state = chip_to_opa_lstate(ppd->dd,
12823                                                read_logical_state(ppd->dd));
12824                 if (new_state == state)
12825                         break;
12826                 if (time_after(jiffies, timeout)) {
12827                         dd_dev_err(ppd->dd,
12828                                    "timeout waiting for link state 0x%x\n",
12829                                    state);
12830                         return -ETIMEDOUT;
12831                 }
12832                 msleep(20);
12833         }
12834
12835         return 0;
12836 }
12837
12838 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state)
12839 {
12840         u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state);
12841
12842         dd_dev_info(ppd->dd,
12843                     "physical state changed to %s (0x%x), phy 0x%x\n",
12844                     opa_pstate_name(ib_pstate), ib_pstate, state);
12845 }
12846
12847 /*
12848  * Read the physical hardware link state and check if it matches host
12849  * drivers anticipated state.
12850  */
12851 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state)
12852 {
12853         u32 read_state = read_physical_state(ppd->dd);
12854
12855         if (read_state == state) {
12856                 log_state_transition(ppd, state);
12857         } else {
12858                 dd_dev_err(ppd->dd,
12859                            "anticipated phy link state 0x%x, read 0x%x\n",
12860                            state, read_state);
12861         }
12862 }
12863
12864 /*
12865  * wait_physical_linkstate - wait for an physical link state change to occur
12866  * @ppd: port device
12867  * @state: the state to wait for
12868  * @msecs: the number of milliseconds to wait
12869  *
12870  * Wait up to msecs milliseconds for physical link state change to occur.
12871  * Returns 0 if state reached, otherwise -ETIMEDOUT.
12872  */
12873 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state,
12874                                    int msecs)
12875 {
12876         u32 read_state;
12877         unsigned long timeout;
12878
12879         timeout = jiffies + msecs_to_jiffies(msecs);
12880         while (1) {
12881                 read_state = read_physical_state(ppd->dd);
12882                 if (read_state == state)
12883                         break;
12884                 if (time_after(jiffies, timeout)) {
12885                         dd_dev_err(ppd->dd,
12886                                    "timeout waiting for phy link state 0x%x\n",
12887                                    state);
12888                         return -ETIMEDOUT;
12889                 }
12890                 usleep_range(1950, 2050); /* sleep 2ms-ish */
12891         }
12892
12893         log_state_transition(ppd, state);
12894         return 0;
12895 }
12896
12897 /*
12898  * wait_phys_link_offline_quiet_substates - wait for any offline substate
12899  * @ppd: port device
12900  * @msecs: the number of milliseconds to wait
12901  *
12902  * Wait up to msecs milliseconds for any offline physical link
12903  * state change to occur.
12904  * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT.
12905  */
12906 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd,
12907                                             int msecs)
12908 {
12909         u32 read_state;
12910         unsigned long timeout;
12911
12912         timeout = jiffies + msecs_to_jiffies(msecs);
12913         while (1) {
12914                 read_state = read_physical_state(ppd->dd);
12915                 if ((read_state & 0xF0) == PLS_OFFLINE)
12916                         break;
12917                 if (time_after(jiffies, timeout)) {
12918                         dd_dev_err(ppd->dd,
12919                                    "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n",
12920                                    read_state, msecs);
12921                         return -ETIMEDOUT;
12922                 }
12923                 usleep_range(1950, 2050); /* sleep 2ms-ish */
12924         }
12925
12926         log_state_transition(ppd, read_state);
12927         return read_state;
12928 }
12929
12930 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \
12931 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12932
12933 #define SET_STATIC_RATE_CONTROL_SMASK(r) \
12934 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK)
12935
12936 void hfi1_init_ctxt(struct send_context *sc)
12937 {
12938         if (sc) {
12939                 struct hfi1_devdata *dd = sc->dd;
12940                 u64 reg;
12941                 u8 set = (sc->type == SC_USER ?
12942                           HFI1_CAP_IS_USET(STATIC_RATE_CTRL) :
12943                           HFI1_CAP_IS_KSET(STATIC_RATE_CTRL));
12944                 reg = read_kctxt_csr(dd, sc->hw_context,
12945                                      SEND_CTXT_CHECK_ENABLE);
12946                 if (set)
12947                         CLEAR_STATIC_RATE_CONTROL_SMASK(reg);
12948                 else
12949                         SET_STATIC_RATE_CONTROL_SMASK(reg);
12950                 write_kctxt_csr(dd, sc->hw_context,
12951                                 SEND_CTXT_CHECK_ENABLE, reg);
12952         }
12953 }
12954
12955 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)
12956 {
12957         int ret = 0;
12958         u64 reg;
12959
12960         if (dd->icode != ICODE_RTL_SILICON) {
12961                 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL))
12962                         dd_dev_info(dd, "%s: tempsense not supported by HW\n",
12963                                     __func__);
12964                 return -EINVAL;
12965         }
12966         reg = read_csr(dd, ASIC_STS_THERM);
12967         temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) &
12968                       ASIC_STS_THERM_CURR_TEMP_MASK);
12969         temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) &
12970                         ASIC_STS_THERM_LO_TEMP_MASK);
12971         temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) &
12972                         ASIC_STS_THERM_HI_TEMP_MASK);
12973         temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) &
12974                           ASIC_STS_THERM_CRIT_TEMP_MASK);
12975         /* triggers is a 3-bit value - 1 bit per trigger. */
12976         temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7);
12977
12978         return ret;
12979 }
12980
12981 /* ========================================================================= */
12982
12983 /**
12984  * read_mod_write() - Calculate the IRQ register index and set/clear the bits
12985  * @dd: valid devdata
12986  * @src: IRQ source to determine register index from
12987  * @bits: the bits to set or clear
12988  * @set: true == set the bits, false == clear the bits
12989  *
12990  */
12991 static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits,
12992                            bool set)
12993 {
12994         u64 reg;
12995         u16 idx = src / BITS_PER_REGISTER;
12996
12997         spin_lock(&dd->irq_src_lock);
12998         reg = read_csr(dd, CCE_INT_MASK + (8 * idx));
12999         if (set)
13000                 reg |= bits;
13001         else
13002                 reg &= ~bits;
13003         write_csr(dd, CCE_INT_MASK + (8 * idx), reg);
13004         spin_unlock(&dd->irq_src_lock);
13005 }
13006
13007 /**
13008  * set_intr_bits() - Enable/disable a range (one or more) IRQ sources
13009  * @dd: valid devdata
13010  * @first: first IRQ source to set/clear
13011  * @last: last IRQ source (inclusive) to set/clear
13012  * @set: true == set the bits, false == clear the bits
13013  *
13014  * If first == last, set the exact source.
13015  */
13016 int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set)
13017 {
13018         u64 bits = 0;
13019         u64 bit;
13020         u16 src;
13021
13022         if (first > NUM_INTERRUPT_SOURCES || last > NUM_INTERRUPT_SOURCES)
13023                 return -EINVAL;
13024
13025         if (last < first)
13026                 return -ERANGE;
13027
13028         for (src = first; src <= last; src++) {
13029                 bit = src % BITS_PER_REGISTER;
13030                 /* wrapped to next register? */
13031                 if (!bit && bits) {
13032                         read_mod_write(dd, src - 1, bits, set);
13033                         bits = 0;
13034                 }
13035                 bits |= BIT_ULL(bit);
13036         }
13037         read_mod_write(dd, last, bits, set);
13038
13039         return 0;
13040 }
13041
13042 /*
13043  * Clear all interrupt sources on the chip.
13044  */
13045 void clear_all_interrupts(struct hfi1_devdata *dd)
13046 {
13047         int i;
13048
13049         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13050                 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0);
13051
13052         write_csr(dd, CCE_ERR_CLEAR, ~(u64)0);
13053         write_csr(dd, MISC_ERR_CLEAR, ~(u64)0);
13054         write_csr(dd, RCV_ERR_CLEAR, ~(u64)0);
13055         write_csr(dd, SEND_ERR_CLEAR, ~(u64)0);
13056         write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0);
13057         write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0);
13058         write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0);
13059         for (i = 0; i < chip_send_contexts(dd); i++)
13060                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0);
13061         for (i = 0; i < chip_sdma_engines(dd); i++)
13062                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0);
13063
13064         write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0);
13065         write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0);
13066         write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0);
13067 }
13068
13069 /*
13070  * Remap the interrupt source from the general handler to the given MSI-X
13071  * interrupt.
13072  */
13073 void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr)
13074 {
13075         u64 reg;
13076         int m, n;
13077
13078         /* clear from the handled mask of the general interrupt */
13079         m = isrc / 64;
13080         n = isrc % 64;
13081         if (likely(m < CCE_NUM_INT_CSRS)) {
13082                 dd->gi_mask[m] &= ~((u64)1 << n);
13083         } else {
13084                 dd_dev_err(dd, "remap interrupt err\n");
13085                 return;
13086         }
13087
13088         /* direct the chip source to the given MSI-X interrupt */
13089         m = isrc / 8;
13090         n = isrc % 8;
13091         reg = read_csr(dd, CCE_INT_MAP + (8 * m));
13092         reg &= ~((u64)0xff << (8 * n));
13093         reg |= ((u64)msix_intr & 0xff) << (8 * n);
13094         write_csr(dd, CCE_INT_MAP + (8 * m), reg);
13095 }
13096
13097 void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr)
13098 {
13099         /*
13100          * SDMA engine interrupt sources grouped by type, rather than
13101          * engine.  Per-engine interrupts are as follows:
13102          *      SDMA
13103          *      SDMAProgress
13104          *      SDMAIdle
13105          */
13106         remap_intr(dd, IS_SDMA_START + engine, msix_intr);
13107         remap_intr(dd, IS_SDMA_PROGRESS_START + engine, msix_intr);
13108         remap_intr(dd, IS_SDMA_IDLE_START + engine, msix_intr);
13109 }
13110
13111 /*
13112  * Set the general handler to accept all interrupts, remap all
13113  * chip interrupts back to MSI-X 0.
13114  */
13115 void reset_interrupts(struct hfi1_devdata *dd)
13116 {
13117         int i;
13118
13119         /* all interrupts handled by the general handler */
13120         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13121                 dd->gi_mask[i] = ~(u64)0;
13122
13123         /* all chip interrupts map to MSI-X 0 */
13124         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13125                 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13126 }
13127
13128 /**
13129  * set_up_interrupts() - Initialize the IRQ resources and state
13130  * @dd: valid devdata
13131  *
13132  */
13133 static int set_up_interrupts(struct hfi1_devdata *dd)
13134 {
13135         int ret;
13136
13137         /* mask all interrupts */
13138         set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false);
13139
13140         /* clear all pending interrupts */
13141         clear_all_interrupts(dd);
13142
13143         /* reset general handler mask, chip MSI-X mappings */
13144         reset_interrupts(dd);
13145
13146         /* ask for MSI-X interrupts */
13147         ret = msix_initialize(dd);
13148         if (ret)
13149                 return ret;
13150
13151         ret = msix_request_irqs(dd);
13152         if (ret)
13153                 msix_clean_up_interrupts(dd);
13154
13155         return ret;
13156 }
13157
13158 /*
13159  * Set up context values in dd.  Sets:
13160  *
13161  *      num_rcv_contexts - number of contexts being used
13162  *      n_krcv_queues - number of kernel contexts
13163  *      first_dyn_alloc_ctxt - first dynamically allocated context
13164  *                             in array of contexts
13165  *      freectxts  - number of free user contexts
13166  *      num_send_contexts - number of PIO send contexts being used
13167  *      num_vnic_contexts - number of contexts reserved for VNIC
13168  */
13169 static int set_up_context_variables(struct hfi1_devdata *dd)
13170 {
13171         unsigned long num_kernel_contexts;
13172         u16 num_vnic_contexts = HFI1_NUM_VNIC_CTXT;
13173         int total_contexts;
13174         int ret;
13175         unsigned ngroups;
13176         int qos_rmt_count;
13177         int user_rmt_reduced;
13178         u32 n_usr_ctxts;
13179         u32 send_contexts = chip_send_contexts(dd);
13180         u32 rcv_contexts = chip_rcv_contexts(dd);
13181
13182         /*
13183          * Kernel receive contexts:
13184          * - Context 0 - control context (VL15/multicast/error)
13185          * - Context 1 - first kernel context
13186          * - Context 2 - second kernel context
13187          * ...
13188          */
13189         if (n_krcvqs)
13190                 /*
13191                  * n_krcvqs is the sum of module parameter kernel receive
13192                  * contexts, krcvqs[].  It does not include the control
13193                  * context, so add that.
13194                  */
13195                 num_kernel_contexts = n_krcvqs + 1;
13196         else
13197                 num_kernel_contexts = DEFAULT_KRCVQS + 1;
13198         /*
13199          * Every kernel receive context needs an ACK send context.
13200          * one send context is allocated for each VL{0-7} and VL15
13201          */
13202         if (num_kernel_contexts > (send_contexts - num_vls - 1)) {
13203                 dd_dev_err(dd,
13204                            "Reducing # kernel rcv contexts to: %d, from %lu\n",
13205                            send_contexts - num_vls - 1,
13206                            num_kernel_contexts);
13207                 num_kernel_contexts = send_contexts - num_vls - 1;
13208         }
13209
13210         /* Accommodate VNIC contexts if possible */
13211         if ((num_kernel_contexts + num_vnic_contexts) > rcv_contexts) {
13212                 dd_dev_err(dd, "No receive contexts available for VNIC\n");
13213                 num_vnic_contexts = 0;
13214         }
13215         total_contexts = num_kernel_contexts + num_vnic_contexts;
13216
13217         /*
13218          * User contexts:
13219          *      - default to 1 user context per real (non-HT) CPU core if
13220          *        num_user_contexts is negative
13221          */
13222         if (num_user_contexts < 0)
13223                 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask);
13224         else
13225                 n_usr_ctxts = num_user_contexts;
13226         /*
13227          * Adjust the counts given a global max.
13228          */
13229         if (total_contexts + n_usr_ctxts > rcv_contexts) {
13230                 dd_dev_err(dd,
13231                            "Reducing # user receive contexts to: %d, from %u\n",
13232                            rcv_contexts - total_contexts,
13233                            n_usr_ctxts);
13234                 /* recalculate */
13235                 n_usr_ctxts = rcv_contexts - total_contexts;
13236         }
13237
13238         /* each user context requires an entry in the RMT */
13239         qos_rmt_count = qos_rmt_entries(dd, NULL, NULL);
13240         if (qos_rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
13241                 user_rmt_reduced = NUM_MAP_ENTRIES - qos_rmt_count;
13242                 dd_dev_err(dd,
13243                            "RMT size is reducing the number of user receive contexts from %u to %d\n",
13244                            n_usr_ctxts,
13245                            user_rmt_reduced);
13246                 /* recalculate */
13247                 n_usr_ctxts = user_rmt_reduced;
13248         }
13249
13250         total_contexts += n_usr_ctxts;
13251
13252         /* the first N are kernel contexts, the rest are user/vnic contexts */
13253         dd->num_rcv_contexts = total_contexts;
13254         dd->n_krcv_queues = num_kernel_contexts;
13255         dd->first_dyn_alloc_ctxt = num_kernel_contexts;
13256         dd->num_vnic_contexts = num_vnic_contexts;
13257         dd->num_user_contexts = n_usr_ctxts;
13258         dd->freectxts = n_usr_ctxts;
13259         dd_dev_info(dd,
13260                     "rcv contexts: chip %d, used %d (kernel %d, vnic %u, user %u)\n",
13261                     rcv_contexts,
13262                     (int)dd->num_rcv_contexts,
13263                     (int)dd->n_krcv_queues,
13264                     dd->num_vnic_contexts,
13265                     dd->num_user_contexts);
13266
13267         /*
13268          * Receive array allocation:
13269          *   All RcvArray entries are divided into groups of 8. This
13270          *   is required by the hardware and will speed up writes to
13271          *   consecutive entries by using write-combining of the entire
13272          *   cacheline.
13273          *
13274          *   The number of groups are evenly divided among all contexts.
13275          *   any left over groups will be given to the first N user
13276          *   contexts.
13277          */
13278         dd->rcv_entries.group_size = RCV_INCREMENT;
13279         ngroups = chip_rcv_array_count(dd) / dd->rcv_entries.group_size;
13280         dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts;
13281         dd->rcv_entries.nctxt_extra = ngroups -
13282                 (dd->num_rcv_contexts * dd->rcv_entries.ngroups);
13283         dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n",
13284                     dd->rcv_entries.ngroups,
13285                     dd->rcv_entries.nctxt_extra);
13286         if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size >
13287             MAX_EAGER_ENTRIES * 2) {
13288                 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) /
13289                         dd->rcv_entries.group_size;
13290                 dd_dev_info(dd,
13291                             "RcvArray group count too high, change to %u\n",
13292                             dd->rcv_entries.ngroups);
13293                 dd->rcv_entries.nctxt_extra = 0;
13294         }
13295         /*
13296          * PIO send contexts
13297          */
13298         ret = init_sc_pools_and_sizes(dd);
13299         if (ret >= 0) { /* success */
13300                 dd->num_send_contexts = ret;
13301                 dd_dev_info(
13302                         dd,
13303                         "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n",
13304                         send_contexts,
13305                         dd->num_send_contexts,
13306                         dd->sc_sizes[SC_KERNEL].count,
13307                         dd->sc_sizes[SC_ACK].count,
13308                         dd->sc_sizes[SC_USER].count,
13309                         dd->sc_sizes[SC_VL15].count);
13310                 ret = 0;        /* success */
13311         }
13312
13313         return ret;
13314 }
13315
13316 /*
13317  * Set the device/port partition key table. The MAD code
13318  * will ensure that, at least, the partial management
13319  * partition key is present in the table.
13320  */
13321 static void set_partition_keys(struct hfi1_pportdata *ppd)
13322 {
13323         struct hfi1_devdata *dd = ppd->dd;
13324         u64 reg = 0;
13325         int i;
13326
13327         dd_dev_info(dd, "Setting partition keys\n");
13328         for (i = 0; i < hfi1_get_npkeys(dd); i++) {
13329                 reg |= (ppd->pkeys[i] &
13330                         RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) <<
13331                         ((i % 4) *
13332                          RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT);
13333                 /* Each register holds 4 PKey values. */
13334                 if ((i % 4) == 3) {
13335                         write_csr(dd, RCV_PARTITION_KEY +
13336                                   ((i - 3) * 2), reg);
13337                         reg = 0;
13338                 }
13339         }
13340
13341         /* Always enable HW pkeys check when pkeys table is set */
13342         add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK);
13343 }
13344
13345 /*
13346  * These CSRs and memories are uninitialized on reset and must be
13347  * written before reading to set the ECC/parity bits.
13348  *
13349  * NOTE: All user context CSRs that are not mmaped write-only
13350  * (e.g. the TID flows) must be initialized even if the driver never
13351  * reads them.
13352  */
13353 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd)
13354 {
13355         int i, j;
13356
13357         /* CceIntMap */
13358         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13359                 write_csr(dd, CCE_INT_MAP + (8 * i), 0);
13360
13361         /* SendCtxtCreditReturnAddr */
13362         for (i = 0; i < chip_send_contexts(dd); i++)
13363                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13364
13365         /* PIO Send buffers */
13366         /* SDMA Send buffers */
13367         /*
13368          * These are not normally read, and (presently) have no method
13369          * to be read, so are not pre-initialized
13370          */
13371
13372         /* RcvHdrAddr */
13373         /* RcvHdrTailAddr */
13374         /* RcvTidFlowTable */
13375         for (i = 0; i < chip_rcv_contexts(dd); i++) {
13376                 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13377                 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13378                 for (j = 0; j < RXE_NUM_TID_FLOWS; j++)
13379                         write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0);
13380         }
13381
13382         /* RcvArray */
13383         for (i = 0; i < chip_rcv_array_count(dd); i++)
13384                 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0);
13385
13386         /* RcvQPMapTable */
13387         for (i = 0; i < 32; i++)
13388                 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13389 }
13390
13391 /*
13392  * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus.
13393  */
13394 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits,
13395                              u64 ctrl_bits)
13396 {
13397         unsigned long timeout;
13398         u64 reg;
13399
13400         /* is the condition present? */
13401         reg = read_csr(dd, CCE_STATUS);
13402         if ((reg & status_bits) == 0)
13403                 return;
13404
13405         /* clear the condition */
13406         write_csr(dd, CCE_CTRL, ctrl_bits);
13407
13408         /* wait for the condition to clear */
13409         timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT);
13410         while (1) {
13411                 reg = read_csr(dd, CCE_STATUS);
13412                 if ((reg & status_bits) == 0)
13413                         return;
13414                 if (time_after(jiffies, timeout)) {
13415                         dd_dev_err(dd,
13416                                    "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n",
13417                                    status_bits, reg & status_bits);
13418                         return;
13419                 }
13420                 udelay(1);
13421         }
13422 }
13423
13424 /* set CCE CSRs to chip reset defaults */
13425 static void reset_cce_csrs(struct hfi1_devdata *dd)
13426 {
13427         int i;
13428
13429         /* CCE_REVISION read-only */
13430         /* CCE_REVISION2 read-only */
13431         /* CCE_CTRL - bits clear automatically */
13432         /* CCE_STATUS read-only, use CceCtrl to clear */
13433         clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK);
13434         clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK);
13435         clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK);
13436         for (i = 0; i < CCE_NUM_SCRATCH; i++)
13437                 write_csr(dd, CCE_SCRATCH + (8 * i), 0);
13438         /* CCE_ERR_STATUS read-only */
13439         write_csr(dd, CCE_ERR_MASK, 0);
13440         write_csr(dd, CCE_ERR_CLEAR, ~0ull);
13441         /* CCE_ERR_FORCE leave alone */
13442         for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++)
13443                 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0);
13444         write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR);
13445         /* CCE_PCIE_CTRL leave alone */
13446         for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) {
13447                 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0);
13448                 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i),
13449                           CCE_MSIX_TABLE_UPPER_RESETCSR);
13450         }
13451         for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) {
13452                 /* CCE_MSIX_PBA read-only */
13453                 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull);
13454                 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull);
13455         }
13456         for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++)
13457                 write_csr(dd, CCE_INT_MAP, 0);
13458         for (i = 0; i < CCE_NUM_INT_CSRS; i++) {
13459                 /* CCE_INT_STATUS read-only */
13460                 write_csr(dd, CCE_INT_MASK + (8 * i), 0);
13461                 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull);
13462                 /* CCE_INT_FORCE leave alone */
13463                 /* CCE_INT_BLOCKED read-only */
13464         }
13465         for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++)
13466                 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0);
13467 }
13468
13469 /* set MISC CSRs to chip reset defaults */
13470 static void reset_misc_csrs(struct hfi1_devdata *dd)
13471 {
13472         int i;
13473
13474         for (i = 0; i < 32; i++) {
13475                 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0);
13476                 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0);
13477                 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0);
13478         }
13479         /*
13480          * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can
13481          * only be written 128-byte chunks
13482          */
13483         /* init RSA engine to clear lingering errors */
13484         write_csr(dd, MISC_CFG_RSA_CMD, 1);
13485         write_csr(dd, MISC_CFG_RSA_MU, 0);
13486         write_csr(dd, MISC_CFG_FW_CTRL, 0);
13487         /* MISC_STS_8051_DIGEST read-only */
13488         /* MISC_STS_SBM_DIGEST read-only */
13489         /* MISC_STS_PCIE_DIGEST read-only */
13490         /* MISC_STS_FAB_DIGEST read-only */
13491         /* MISC_ERR_STATUS read-only */
13492         write_csr(dd, MISC_ERR_MASK, 0);
13493         write_csr(dd, MISC_ERR_CLEAR, ~0ull);
13494         /* MISC_ERR_FORCE leave alone */
13495 }
13496
13497 /* set TXE CSRs to chip reset defaults */
13498 static void reset_txe_csrs(struct hfi1_devdata *dd)
13499 {
13500         int i;
13501
13502         /*
13503          * TXE Kernel CSRs
13504          */
13505         write_csr(dd, SEND_CTRL, 0);
13506         __cm_reset(dd, 0);      /* reset CM internal state */
13507         /* SEND_CONTEXTS read-only */
13508         /* SEND_DMA_ENGINES read-only */
13509         /* SEND_PIO_MEM_SIZE read-only */
13510         /* SEND_DMA_MEM_SIZE read-only */
13511         write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0);
13512         pio_reset_all(dd);      /* SEND_PIO_INIT_CTXT */
13513         /* SEND_PIO_ERR_STATUS read-only */
13514         write_csr(dd, SEND_PIO_ERR_MASK, 0);
13515         write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull);
13516         /* SEND_PIO_ERR_FORCE leave alone */
13517         /* SEND_DMA_ERR_STATUS read-only */
13518         write_csr(dd, SEND_DMA_ERR_MASK, 0);
13519         write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull);
13520         /* SEND_DMA_ERR_FORCE leave alone */
13521         /* SEND_EGRESS_ERR_STATUS read-only */
13522         write_csr(dd, SEND_EGRESS_ERR_MASK, 0);
13523         write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull);
13524         /* SEND_EGRESS_ERR_FORCE leave alone */
13525         write_csr(dd, SEND_BTH_QP, 0);
13526         write_csr(dd, SEND_STATIC_RATE_CONTROL, 0);
13527         write_csr(dd, SEND_SC2VLT0, 0);
13528         write_csr(dd, SEND_SC2VLT1, 0);
13529         write_csr(dd, SEND_SC2VLT2, 0);
13530         write_csr(dd, SEND_SC2VLT3, 0);
13531         write_csr(dd, SEND_LEN_CHECK0, 0);
13532         write_csr(dd, SEND_LEN_CHECK1, 0);
13533         /* SEND_ERR_STATUS read-only */
13534         write_csr(dd, SEND_ERR_MASK, 0);
13535         write_csr(dd, SEND_ERR_CLEAR, ~0ull);
13536         /* SEND_ERR_FORCE read-only */
13537         for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++)
13538                 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0);
13539         for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++)
13540                 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0);
13541         for (i = 0; i < chip_send_contexts(dd) / NUM_CONTEXTS_PER_SET; i++)
13542                 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0);
13543         for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++)
13544                 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0);
13545         for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++)
13546                 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0);
13547         write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR);
13548         write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR);
13549         /* SEND_CM_CREDIT_USED_STATUS read-only */
13550         write_csr(dd, SEND_CM_TIMER_CTRL, 0);
13551         write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0);
13552         write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0);
13553         write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0);
13554         write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0);
13555         for (i = 0; i < TXE_NUM_DATA_VL; i++)
13556                 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0);
13557         write_csr(dd, SEND_CM_CREDIT_VL15, 0);
13558         /* SEND_CM_CREDIT_USED_VL read-only */
13559         /* SEND_CM_CREDIT_USED_VL15 read-only */
13560         /* SEND_EGRESS_CTXT_STATUS read-only */
13561         /* SEND_EGRESS_SEND_DMA_STATUS read-only */
13562         write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull);
13563         /* SEND_EGRESS_ERR_INFO read-only */
13564         /* SEND_EGRESS_ERR_SOURCE read-only */
13565
13566         /*
13567          * TXE Per-Context CSRs
13568          */
13569         for (i = 0; i < chip_send_contexts(dd); i++) {
13570                 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13571                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0);
13572                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0);
13573                 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0);
13574                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0);
13575                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull);
13576                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0);
13577                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0);
13578                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0);
13579                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0);
13580                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0);
13581                 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0);
13582         }
13583
13584         /*
13585          * TXE Per-SDMA CSRs
13586          */
13587         for (i = 0; i < chip_sdma_engines(dd); i++) {
13588                 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13589                 /* SEND_DMA_STATUS read-only */
13590                 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0);
13591                 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0);
13592                 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0);
13593                 /* SEND_DMA_HEAD read-only */
13594                 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0);
13595                 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0);
13596                 /* SEND_DMA_IDLE_CNT read-only */
13597                 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0);
13598                 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0);
13599                 /* SEND_DMA_DESC_FETCHED_CNT read-only */
13600                 /* SEND_DMA_ENG_ERR_STATUS read-only */
13601                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0);
13602                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull);
13603                 /* SEND_DMA_ENG_ERR_FORCE leave alone */
13604                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0);
13605                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0);
13606                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0);
13607                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0);
13608                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0);
13609                 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0);
13610                 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0);
13611         }
13612 }
13613
13614 /*
13615  * Expect on entry:
13616  * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0
13617  */
13618 static void init_rbufs(struct hfi1_devdata *dd)
13619 {
13620         u64 reg;
13621         int count;
13622
13623         /*
13624          * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are
13625          * clear.
13626          */
13627         count = 0;
13628         while (1) {
13629                 reg = read_csr(dd, RCV_STATUS);
13630                 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK
13631                             | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0)
13632                         break;
13633                 /*
13634                  * Give up after 1ms - maximum wait time.
13635                  *
13636                  * RBuf size is 136KiB.  Slowest possible is PCIe Gen1 x1 at
13637                  * 250MB/s bandwidth.  Lower rate to 66% for overhead to get:
13638                  *      136 KB / (66% * 250MB/s) = 844us
13639                  */
13640                 if (count++ > 500) {
13641                         dd_dev_err(dd,
13642                                    "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n",
13643                                    __func__, reg);
13644                         break;
13645                 }
13646                 udelay(2); /* do not busy-wait the CSR */
13647         }
13648
13649         /* start the init - expect RcvCtrl to be 0 */
13650         write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK);
13651
13652         /*
13653          * Read to force the write of Rcvtrl.RxRbufInit.  There is a brief
13654          * period after the write before RcvStatus.RxRbufInitDone is valid.
13655          * The delay in the first run through the loop below is sufficient and
13656          * required before the first read of RcvStatus.RxRbufInintDone.
13657          */
13658         read_csr(dd, RCV_CTRL);
13659
13660         /* wait for the init to finish */
13661         count = 0;
13662         while (1) {
13663                 /* delay is required first time through - see above */
13664                 udelay(2); /* do not busy-wait the CSR */
13665                 reg = read_csr(dd, RCV_STATUS);
13666                 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK))
13667                         break;
13668
13669                 /* give up after 100us - slowest possible at 33MHz is 73us */
13670                 if (count++ > 50) {
13671                         dd_dev_err(dd,
13672                                    "%s: RcvStatus.RxRbufInit not set, continuing\n",
13673                                    __func__);
13674                         break;
13675                 }
13676         }
13677 }
13678
13679 /* set RXE CSRs to chip reset defaults */
13680 static void reset_rxe_csrs(struct hfi1_devdata *dd)
13681 {
13682         int i, j;
13683
13684         /*
13685          * RXE Kernel CSRs
13686          */
13687         write_csr(dd, RCV_CTRL, 0);
13688         init_rbufs(dd);
13689         /* RCV_STATUS read-only */
13690         /* RCV_CONTEXTS read-only */
13691         /* RCV_ARRAY_CNT read-only */
13692         /* RCV_BUF_SIZE read-only */
13693         write_csr(dd, RCV_BTH_QP, 0);
13694         write_csr(dd, RCV_MULTICAST, 0);
13695         write_csr(dd, RCV_BYPASS, 0);
13696         write_csr(dd, RCV_VL15, 0);
13697         /* this is a clear-down */
13698         write_csr(dd, RCV_ERR_INFO,
13699                   RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK);
13700         /* RCV_ERR_STATUS read-only */
13701         write_csr(dd, RCV_ERR_MASK, 0);
13702         write_csr(dd, RCV_ERR_CLEAR, ~0ull);
13703         /* RCV_ERR_FORCE leave alone */
13704         for (i = 0; i < 32; i++)
13705                 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0);
13706         for (i = 0; i < 4; i++)
13707                 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0);
13708         for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++)
13709                 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0);
13710         for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++)
13711                 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0);
13712         for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++)
13713                 clear_rsm_rule(dd, i);
13714         for (i = 0; i < 32; i++)
13715                 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0);
13716
13717         /*
13718          * RXE Kernel and User Per-Context CSRs
13719          */
13720         for (i = 0; i < chip_rcv_contexts(dd); i++) {
13721                 /* kernel */
13722                 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0);
13723                 /* RCV_CTXT_STATUS read-only */
13724                 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0);
13725                 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0);
13726                 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0);
13727                 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0);
13728                 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0);
13729                 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0);
13730                 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0);
13731                 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0);
13732                 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0);
13733                 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0);
13734
13735                 /* user */
13736                 /* RCV_HDR_TAIL read-only */
13737                 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0);
13738                 /* RCV_EGR_INDEX_TAIL read-only */
13739                 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0);
13740                 /* RCV_EGR_OFFSET_TAIL read-only */
13741                 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) {
13742                         write_uctxt_csr(dd, i,
13743                                         RCV_TID_FLOW_TABLE + (8 * j), 0);
13744                 }
13745         }
13746 }
13747
13748 /*
13749  * Set sc2vl tables.
13750  *
13751  * They power on to zeros, so to avoid send context errors
13752  * they need to be set:
13753  *
13754  * SC 0-7 -> VL 0-7 (respectively)
13755  * SC 15  -> VL 15
13756  * otherwise
13757  *        -> VL 0
13758  */
13759 static void init_sc2vl_tables(struct hfi1_devdata *dd)
13760 {
13761         int i;
13762         /* init per architecture spec, constrained by hardware capability */
13763
13764         /* HFI maps sent packets */
13765         write_csr(dd, SEND_SC2VLT0, SC2VL_VAL(
13766                 0,
13767                 0, 0, 1, 1,
13768                 2, 2, 3, 3,
13769                 4, 4, 5, 5,
13770                 6, 6, 7, 7));
13771         write_csr(dd, SEND_SC2VLT1, SC2VL_VAL(
13772                 1,
13773                 8, 0, 9, 0,
13774                 10, 0, 11, 0,
13775                 12, 0, 13, 0,
13776                 14, 0, 15, 15));
13777         write_csr(dd, SEND_SC2VLT2, SC2VL_VAL(
13778                 2,
13779                 16, 0, 17, 0,
13780                 18, 0, 19, 0,
13781                 20, 0, 21, 0,
13782                 22, 0, 23, 0));
13783         write_csr(dd, SEND_SC2VLT3, SC2VL_VAL(
13784                 3,
13785                 24, 0, 25, 0,
13786                 26, 0, 27, 0,
13787                 28, 0, 29, 0,
13788                 30, 0, 31, 0));
13789
13790         /* DC maps received packets */
13791         write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL(
13792                 15_0,
13793                 0, 0, 1, 1,  2, 2,  3, 3,  4, 4,  5, 5,  6, 6,  7,  7,
13794                 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15));
13795         write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL(
13796                 31_16,
13797                 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0,
13798                 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0));
13799
13800         /* initialize the cached sc2vl values consistently with h/w */
13801         for (i = 0; i < 32; i++) {
13802                 if (i < 8 || i == 15)
13803                         *((u8 *)(dd->sc2vl) + i) = (u8)i;
13804                 else
13805                         *((u8 *)(dd->sc2vl) + i) = 0;
13806         }
13807 }
13808
13809 /*
13810  * Read chip sizes and then reset parts to sane, disabled, values.  We cannot
13811  * depend on the chip going through a power-on reset - a driver may be loaded
13812  * and unloaded many times.
13813  *
13814  * Do not write any CSR values to the chip in this routine - there may be
13815  * a reset following the (possible) FLR in this routine.
13816  *
13817  */
13818 static int init_chip(struct hfi1_devdata *dd)
13819 {
13820         int i;
13821         int ret = 0;
13822
13823         /*
13824          * Put the HFI CSRs in a known state.
13825          * Combine this with a DC reset.
13826          *
13827          * Stop the device from doing anything while we do a
13828          * reset.  We know there are no other active users of
13829          * the device since we are now in charge.  Turn off
13830          * off all outbound and inbound traffic and make sure
13831          * the device does not generate any interrupts.
13832          */
13833
13834         /* disable send contexts and SDMA engines */
13835         write_csr(dd, SEND_CTRL, 0);
13836         for (i = 0; i < chip_send_contexts(dd); i++)
13837                 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0);
13838         for (i = 0; i < chip_sdma_engines(dd); i++)
13839                 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0);
13840         /* disable port (turn off RXE inbound traffic) and contexts */
13841         write_csr(dd, RCV_CTRL, 0);
13842         for (i = 0; i < chip_rcv_contexts(dd); i++)
13843                 write_csr(dd, RCV_CTXT_CTRL, 0);
13844         /* mask all interrupt sources */
13845         for (i = 0; i < CCE_NUM_INT_CSRS; i++)
13846                 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull);
13847
13848         /*
13849          * DC Reset: do a full DC reset before the register clear.
13850          * A recommended length of time to hold is one CSR read,
13851          * so reread the CceDcCtrl.  Then, hold the DC in reset
13852          * across the clear.
13853          */
13854         write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK);
13855         (void)read_csr(dd, CCE_DC_CTRL);
13856
13857         if (use_flr) {
13858                 /*
13859                  * A FLR will reset the SPC core and part of the PCIe.
13860                  * The parts that need to be restored have already been
13861                  * saved.
13862                  */
13863                 dd_dev_info(dd, "Resetting CSRs with FLR\n");
13864
13865                 /* do the FLR, the DC reset will remain */
13866                 pcie_flr(dd->pcidev);
13867
13868                 /* restore command and BARs */
13869                 ret = restore_pci_variables(dd);
13870                 if (ret) {
13871                         dd_dev_err(dd, "%s: Could not restore PCI variables\n",
13872                                    __func__);
13873                         return ret;
13874                 }
13875
13876                 if (is_ax(dd)) {
13877                         dd_dev_info(dd, "Resetting CSRs with FLR\n");
13878                         pcie_flr(dd->pcidev);
13879                         ret = restore_pci_variables(dd);
13880                         if (ret) {
13881                                 dd_dev_err(dd, "%s: Could not restore PCI variables\n",
13882                                            __func__);
13883                                 return ret;
13884                         }
13885                 }
13886         } else {
13887                 dd_dev_info(dd, "Resetting CSRs with writes\n");
13888                 reset_cce_csrs(dd);
13889                 reset_txe_csrs(dd);
13890                 reset_rxe_csrs(dd);
13891                 reset_misc_csrs(dd);
13892         }
13893         /* clear the DC reset */
13894         write_csr(dd, CCE_DC_CTRL, 0);
13895
13896         /* Set the LED off */
13897         setextled(dd, 0);
13898
13899         /*
13900          * Clear the QSFP reset.
13901          * An FLR enforces a 0 on all out pins. The driver does not touch
13902          * ASIC_QSFPn_OUT otherwise.  This leaves RESET_N low and
13903          * anything plugged constantly in reset, if it pays attention
13904          * to RESET_N.
13905          * Prime examples of this are optical cables. Set all pins high.
13906          * I2CCLK and I2CDAT will change per direction, and INT_N and
13907          * MODPRS_N are input only and their value is ignored.
13908          */
13909         write_csr(dd, ASIC_QSFP1_OUT, 0x1f);
13910         write_csr(dd, ASIC_QSFP2_OUT, 0x1f);
13911         init_chip_resources(dd);
13912         return ret;
13913 }
13914
13915 static void init_early_variables(struct hfi1_devdata *dd)
13916 {
13917         int i;
13918
13919         /* assign link credit variables */
13920         dd->vau = CM_VAU;
13921         dd->link_credits = CM_GLOBAL_CREDITS;
13922         if (is_ax(dd))
13923                 dd->link_credits--;
13924         dd->vcu = cu_to_vcu(hfi1_cu);
13925         /* enough room for 8 MAD packets plus header - 17K */
13926         dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau);
13927         if (dd->vl15_init > dd->link_credits)
13928                 dd->vl15_init = dd->link_credits;
13929
13930         write_uninitialized_csrs_and_memories(dd);
13931
13932         if (HFI1_CAP_IS_KSET(PKEY_CHECK))
13933                 for (i = 0; i < dd->num_pports; i++) {
13934                         struct hfi1_pportdata *ppd = &dd->pport[i];
13935
13936                         set_partition_keys(ppd);
13937                 }
13938         init_sc2vl_tables(dd);
13939 }
13940
13941 static void init_kdeth_qp(struct hfi1_devdata *dd)
13942 {
13943         /* user changed the KDETH_QP */
13944         if (kdeth_qp != 0 && kdeth_qp >= 0xff) {
13945                 /* out of range or illegal value */
13946                 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring");
13947                 kdeth_qp = 0;
13948         }
13949         if (kdeth_qp == 0)      /* not set, or failed range check */
13950                 kdeth_qp = DEFAULT_KDETH_QP;
13951
13952         write_csr(dd, SEND_BTH_QP,
13953                   (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) <<
13954                   SEND_BTH_QP_KDETH_QP_SHIFT);
13955
13956         write_csr(dd, RCV_BTH_QP,
13957                   (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) <<
13958                   RCV_BTH_QP_KDETH_QP_SHIFT);
13959 }
13960
13961 /**
13962  * init_qpmap_table
13963  * @dd - device data
13964  * @first_ctxt - first context
13965  * @last_ctxt - first context
13966  *
13967  * This return sets the qpn mapping table that
13968  * is indexed by qpn[8:1].
13969  *
13970  * The routine will round robin the 256 settings
13971  * from first_ctxt to last_ctxt.
13972  *
13973  * The first/last looks ahead to having specialized
13974  * receive contexts for mgmt and bypass.  Normal
13975  * verbs traffic will assumed to be on a range
13976  * of receive contexts.
13977  */
13978 static void init_qpmap_table(struct hfi1_devdata *dd,
13979                              u32 first_ctxt,
13980                              u32 last_ctxt)
13981 {
13982         u64 reg = 0;
13983         u64 regno = RCV_QP_MAP_TABLE;
13984         int i;
13985         u64 ctxt = first_ctxt;
13986
13987         for (i = 0; i < 256; i++) {
13988                 reg |= ctxt << (8 * (i % 8));
13989                 ctxt++;
13990                 if (ctxt > last_ctxt)
13991                         ctxt = first_ctxt;
13992                 if (i % 8 == 7) {
13993                         write_csr(dd, regno, reg);
13994                         reg = 0;
13995                         regno += 8;
13996                 }
13997         }
13998
13999         add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK
14000                         | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK);
14001 }
14002
14003 struct rsm_map_table {
14004         u64 map[NUM_MAP_REGS];
14005         unsigned int used;
14006 };
14007
14008 struct rsm_rule_data {
14009         u8 offset;
14010         u8 pkt_type;
14011         u32 field1_off;
14012         u32 field2_off;
14013         u32 index1_off;
14014         u32 index1_width;
14015         u32 index2_off;
14016         u32 index2_width;
14017         u32 mask1;
14018         u32 value1;
14019         u32 mask2;
14020         u32 value2;
14021 };
14022
14023 /*
14024  * Return an initialized RMT map table for users to fill in.  OK if it
14025  * returns NULL, indicating no table.
14026  */
14027 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd)
14028 {
14029         struct rsm_map_table *rmt;
14030         u8 rxcontext = is_ax(dd) ? 0 : 0xff;  /* 0 is default if a0 ver. */
14031
14032         rmt = kmalloc(sizeof(*rmt), GFP_KERNEL);
14033         if (rmt) {
14034                 memset(rmt->map, rxcontext, sizeof(rmt->map));
14035                 rmt->used = 0;
14036         }
14037
14038         return rmt;
14039 }
14040
14041 /*
14042  * Write the final RMT map table to the chip and free the table.  OK if
14043  * table is NULL.
14044  */
14045 static void complete_rsm_map_table(struct hfi1_devdata *dd,
14046                                    struct rsm_map_table *rmt)
14047 {
14048         int i;
14049
14050         if (rmt) {
14051                 /* write table to chip */
14052                 for (i = 0; i < NUM_MAP_REGS; i++)
14053                         write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]);
14054
14055                 /* enable RSM */
14056                 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14057         }
14058 }
14059
14060 /*
14061  * Add a receive side mapping rule.
14062  */
14063 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index,
14064                          struct rsm_rule_data *rrd)
14065 {
14066         write_csr(dd, RCV_RSM_CFG + (8 * rule_index),
14067                   (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT |
14068                   1ull << rule_index | /* enable bit */
14069                   (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT);
14070         write_csr(dd, RCV_RSM_SELECT + (8 * rule_index),
14071                   (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT |
14072                   (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT |
14073                   (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT |
14074                   (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT |
14075                   (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT |
14076                   (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT);
14077         write_csr(dd, RCV_RSM_MATCH + (8 * rule_index),
14078                   (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT |
14079                   (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT |
14080                   (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT |
14081                   (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT);
14082 }
14083
14084 /*
14085  * Clear a receive side mapping rule.
14086  */
14087 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index)
14088 {
14089         write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0);
14090         write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0);
14091         write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0);
14092 }
14093
14094 /* return the number of RSM map table entries that will be used for QOS */
14095 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
14096                            unsigned int *np)
14097 {
14098         int i;
14099         unsigned int m, n;
14100         u8 max_by_vl = 0;
14101
14102         /* is QOS active at all? */
14103         if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS ||
14104             num_vls == 1 ||
14105             krcvqsset <= 1)
14106                 goto no_qos;
14107
14108         /* determine bits for qpn */
14109         for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++)
14110                 if (krcvqs[i] > max_by_vl)
14111                         max_by_vl = krcvqs[i];
14112         if (max_by_vl > 32)
14113                 goto no_qos;
14114         m = ilog2(__roundup_pow_of_two(max_by_vl));
14115
14116         /* determine bits for vl */
14117         n = ilog2(__roundup_pow_of_two(num_vls));
14118
14119         /* reject if too much is used */
14120         if ((m + n) > 7)
14121                 goto no_qos;
14122
14123         if (mp)
14124                 *mp = m;
14125         if (np)
14126                 *np = n;
14127
14128         return 1 << (m + n);
14129
14130 no_qos:
14131         if (mp)
14132                 *mp = 0;
14133         if (np)
14134                 *np = 0;
14135         return 0;
14136 }
14137
14138 /**
14139  * init_qos - init RX qos
14140  * @dd - device data
14141  * @rmt - RSM map table
14142  *
14143  * This routine initializes Rule 0 and the RSM map table to implement
14144  * quality of service (qos).
14145  *
14146  * If all of the limit tests succeed, qos is applied based on the array
14147  * interpretation of krcvqs where entry 0 is VL0.
14148  *
14149  * The number of vl bits (n) and the number of qpn bits (m) are computed to
14150  * feed both the RSM map table and the single rule.
14151  */
14152 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt)
14153 {
14154         struct rsm_rule_data rrd;
14155         unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m;
14156         unsigned int rmt_entries;
14157         u64 reg;
14158
14159         if (!rmt)
14160                 goto bail;
14161         rmt_entries = qos_rmt_entries(dd, &m, &n);
14162         if (rmt_entries == 0)
14163                 goto bail;
14164         qpns_per_vl = 1 << m;
14165
14166         /* enough room in the map table? */
14167         rmt_entries = 1 << (m + n);
14168         if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES)
14169                 goto bail;
14170
14171         /* add qos entries to the the RSM map table */
14172         for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) {
14173                 unsigned tctxt;
14174
14175                 for (qpn = 0, tctxt = ctxt;
14176                      krcvqs[i] && qpn < qpns_per_vl; qpn++) {
14177                         unsigned idx, regoff, regidx;
14178
14179                         /* generate the index the hardware will produce */
14180                         idx = rmt->used + ((qpn << n) ^ i);
14181                         regoff = (idx % 8) * 8;
14182                         regidx = idx / 8;
14183                         /* replace default with context number */
14184                         reg = rmt->map[regidx];
14185                         reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK
14186                                 << regoff);
14187                         reg |= (u64)(tctxt++) << regoff;
14188                         rmt->map[regidx] = reg;
14189                         if (tctxt == ctxt + krcvqs[i])
14190                                 tctxt = ctxt;
14191                 }
14192                 ctxt += krcvqs[i];
14193         }
14194
14195         rrd.offset = rmt->used;
14196         rrd.pkt_type = 2;
14197         rrd.field1_off = LRH_BTH_MATCH_OFFSET;
14198         rrd.field2_off = LRH_SC_MATCH_OFFSET;
14199         rrd.index1_off = LRH_SC_SELECT_OFFSET;
14200         rrd.index1_width = n;
14201         rrd.index2_off = QPN_SELECT_OFFSET;
14202         rrd.index2_width = m + n;
14203         rrd.mask1 = LRH_BTH_MASK;
14204         rrd.value1 = LRH_BTH_VALUE;
14205         rrd.mask2 = LRH_SC_MASK;
14206         rrd.value2 = LRH_SC_VALUE;
14207
14208         /* add rule 0 */
14209         add_rsm_rule(dd, RSM_INS_VERBS, &rrd);
14210
14211         /* mark RSM map entries as used */
14212         rmt->used += rmt_entries;
14213         /* map everything else to the mcast/err/vl15 context */
14214         init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT);
14215         dd->qos_shift = n + 1;
14216         return;
14217 bail:
14218         dd->qos_shift = 1;
14219         init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
14220 }
14221
14222 static void init_user_fecn_handling(struct hfi1_devdata *dd,
14223                                     struct rsm_map_table *rmt)
14224 {
14225         struct rsm_rule_data rrd;
14226         u64 reg;
14227         int i, idx, regoff, regidx;
14228         u8 offset;
14229
14230         /* there needs to be enough room in the map table */
14231         if (rmt->used + dd->num_user_contexts >= NUM_MAP_ENTRIES) {
14232                 dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n");
14233                 return;
14234         }
14235
14236         /*
14237          * RSM will extract the destination context as an index into the
14238          * map table.  The destination contexts are a sequential block
14239          * in the range first_dyn_alloc_ctxt...num_rcv_contexts-1 (inclusive).
14240          * Map entries are accessed as offset + extracted value.  Adjust
14241          * the added offset so this sequence can be placed anywhere in
14242          * the table - as long as the entries themselves do not wrap.
14243          * There are only enough bits in offset for the table size, so
14244          * start with that to allow for a "negative" offset.
14245          */
14246         offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used -
14247                                                 (int)dd->first_dyn_alloc_ctxt);
14248
14249         for (i = dd->first_dyn_alloc_ctxt, idx = rmt->used;
14250                                 i < dd->num_rcv_contexts; i++, idx++) {
14251                 /* replace with identity mapping */
14252                 regoff = (idx % 8) * 8;
14253                 regidx = idx / 8;
14254                 reg = rmt->map[regidx];
14255                 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff);
14256                 reg |= (u64)i << regoff;
14257                 rmt->map[regidx] = reg;
14258         }
14259
14260         /*
14261          * For RSM intercept of Expected FECN packets:
14262          * o packet type 0 - expected
14263          * o match on F (bit 95), using select/match 1, and
14264          * o match on SH (bit 133), using select/match 2.
14265          *
14266          * Use index 1 to extract the 8-bit receive context from DestQP
14267          * (start at bit 64).  Use that as the RSM map table index.
14268          */
14269         rrd.offset = offset;
14270         rrd.pkt_type = 0;
14271         rrd.field1_off = 95;
14272         rrd.field2_off = 133;
14273         rrd.index1_off = 64;
14274         rrd.index1_width = 8;
14275         rrd.index2_off = 0;
14276         rrd.index2_width = 0;
14277         rrd.mask1 = 1;
14278         rrd.value1 = 1;
14279         rrd.mask2 = 1;
14280         rrd.value2 = 1;
14281
14282         /* add rule 1 */
14283         add_rsm_rule(dd, RSM_INS_FECN, &rrd);
14284
14285         rmt->used += dd->num_user_contexts;
14286 }
14287
14288 /* Initialize RSM for VNIC */
14289 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd)
14290 {
14291         u8 i, j;
14292         u8 ctx_id = 0;
14293         u64 reg;
14294         u32 regoff;
14295         struct rsm_rule_data rrd;
14296
14297         if (hfi1_vnic_is_rsm_full(dd, NUM_VNIC_MAP_ENTRIES)) {
14298                 dd_dev_err(dd, "Vnic RSM disabled, rmt entries used = %d\n",
14299                            dd->vnic.rmt_start);
14300                 return;
14301         }
14302
14303         dev_dbg(&(dd)->pcidev->dev, "Vnic rsm start = %d, end %d\n",
14304                 dd->vnic.rmt_start,
14305                 dd->vnic.rmt_start + NUM_VNIC_MAP_ENTRIES);
14306
14307         /* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */
14308         regoff = RCV_RSM_MAP_TABLE + (dd->vnic.rmt_start / 8) * 8;
14309         reg = read_csr(dd, regoff);
14310         for (i = 0; i < NUM_VNIC_MAP_ENTRIES; i++) {
14311                 /* Update map register with vnic context */
14312                 j = (dd->vnic.rmt_start + i) % 8;
14313                 reg &= ~(0xffllu << (j * 8));
14314                 reg |= (u64)dd->vnic.ctxt[ctx_id++]->ctxt << (j * 8);
14315                 /* Wrap up vnic ctx index */
14316                 ctx_id %= dd->vnic.num_ctxt;
14317                 /* Write back map register */
14318                 if (j == 7 || ((i + 1) == NUM_VNIC_MAP_ENTRIES)) {
14319                         dev_dbg(&(dd)->pcidev->dev,
14320                                 "Vnic rsm map reg[%d] =0x%llx\n",
14321                                 regoff - RCV_RSM_MAP_TABLE, reg);
14322
14323                         write_csr(dd, regoff, reg);
14324                         regoff += 8;
14325                         if (i < (NUM_VNIC_MAP_ENTRIES - 1))
14326                                 reg = read_csr(dd, regoff);
14327                 }
14328         }
14329
14330         /* Add rule for vnic */
14331         rrd.offset = dd->vnic.rmt_start;
14332         rrd.pkt_type = 4;
14333         /* Match 16B packets */
14334         rrd.field1_off = L2_TYPE_MATCH_OFFSET;
14335         rrd.mask1 = L2_TYPE_MASK;
14336         rrd.value1 = L2_16B_VALUE;
14337         /* Match ETH L4 packets */
14338         rrd.field2_off = L4_TYPE_MATCH_OFFSET;
14339         rrd.mask2 = L4_16B_TYPE_MASK;
14340         rrd.value2 = L4_16B_ETH_VALUE;
14341         /* Calc context from veswid and entropy */
14342         rrd.index1_off = L4_16B_HDR_VESWID_OFFSET;
14343         rrd.index1_width = ilog2(NUM_VNIC_MAP_ENTRIES);
14344         rrd.index2_off = L2_16B_ENTROPY_OFFSET;
14345         rrd.index2_width = ilog2(NUM_VNIC_MAP_ENTRIES);
14346         add_rsm_rule(dd, RSM_INS_VNIC, &rrd);
14347
14348         /* Enable RSM if not already enabled */
14349         add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14350 }
14351
14352 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd)
14353 {
14354         clear_rsm_rule(dd, RSM_INS_VNIC);
14355
14356         /* Disable RSM if used only by vnic */
14357         if (dd->vnic.rmt_start == 0)
14358                 clear_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK);
14359 }
14360
14361 static void init_rxe(struct hfi1_devdata *dd)
14362 {
14363         struct rsm_map_table *rmt;
14364         u64 val;
14365
14366         /* enable all receive errors */
14367         write_csr(dd, RCV_ERR_MASK, ~0ull);
14368
14369         rmt = alloc_rsm_map_table(dd);
14370         /* set up QOS, including the QPN map table */
14371         init_qos(dd, rmt);
14372         init_user_fecn_handling(dd, rmt);
14373         complete_rsm_map_table(dd, rmt);
14374         /* record number of used rsm map entries for vnic */
14375         dd->vnic.rmt_start = rmt->used;
14376         kfree(rmt);
14377
14378         /*
14379          * make sure RcvCtrl.RcvWcb <= PCIe Device Control
14380          * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config
14381          * space, PciCfgCap2.MaxPayloadSize in HFI).  There is only one
14382          * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and
14383          * Max_PayLoad_Size set to its minimum of 128.
14384          *
14385          * Presently, RcvCtrl.RcvWcb is not modified from its default of 0
14386          * (64 bytes).  Max_Payload_Size is possibly modified upward in
14387          * tune_pcie_caps() which is called after this routine.
14388          */
14389
14390         /* Have 16 bytes (4DW) of bypass header available in header queue */
14391         val = read_csr(dd, RCV_BYPASS);
14392         val &= ~RCV_BYPASS_HDR_SIZE_SMASK;
14393         val |= ((4ull & RCV_BYPASS_HDR_SIZE_MASK) <<
14394                 RCV_BYPASS_HDR_SIZE_SHIFT);
14395         write_csr(dd, RCV_BYPASS, val);
14396 }
14397
14398 static void init_other(struct hfi1_devdata *dd)
14399 {
14400         /* enable all CCE errors */
14401         write_csr(dd, CCE_ERR_MASK, ~0ull);
14402         /* enable *some* Misc errors */
14403         write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK);
14404         /* enable all DC errors, except LCB */
14405         write_csr(dd, DCC_ERR_FLG_EN, ~0ull);
14406         write_csr(dd, DC_DC8051_ERR_EN, ~0ull);
14407 }
14408
14409 /*
14410  * Fill out the given AU table using the given CU.  A CU is defined in terms
14411  * AUs.  The table is a an encoding: given the index, how many AUs does that
14412  * represent?
14413  *
14414  * NOTE: Assumes that the register layout is the same for the
14415  * local and remote tables.
14416  */
14417 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu,
14418                                u32 csr0to3, u32 csr4to7)
14419 {
14420         write_csr(dd, csr0to3,
14421                   0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT |
14422                   1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT |
14423                   2ull * cu <<
14424                   SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT |
14425                   4ull * cu <<
14426                   SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT);
14427         write_csr(dd, csr4to7,
14428                   8ull * cu <<
14429                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT |
14430                   16ull * cu <<
14431                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT |
14432                   32ull * cu <<
14433                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT |
14434                   64ull * cu <<
14435                   SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT);
14436 }
14437
14438 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14439 {
14440         assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3,
14441                            SEND_CM_LOCAL_AU_TABLE4_TO7);
14442 }
14443
14444 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu)
14445 {
14446         assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3,
14447                            SEND_CM_REMOTE_AU_TABLE4_TO7);
14448 }
14449
14450 static void init_txe(struct hfi1_devdata *dd)
14451 {
14452         int i;
14453
14454         /* enable all PIO, SDMA, general, and Egress errors */
14455         write_csr(dd, SEND_PIO_ERR_MASK, ~0ull);
14456         write_csr(dd, SEND_DMA_ERR_MASK, ~0ull);
14457         write_csr(dd, SEND_ERR_MASK, ~0ull);
14458         write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull);
14459
14460         /* enable all per-context and per-SDMA engine errors */
14461         for (i = 0; i < chip_send_contexts(dd); i++)
14462                 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull);
14463         for (i = 0; i < chip_sdma_engines(dd); i++)
14464                 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull);
14465
14466         /* set the local CU to AU mapping */
14467         assign_local_cm_au_table(dd, dd->vcu);
14468
14469         /*
14470          * Set reasonable default for Credit Return Timer
14471          * Don't set on Simulator - causes it to choke.
14472          */
14473         if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)
14474                 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE);
14475 }
14476
14477 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14478                        u16 jkey)
14479 {
14480         u8 hw_ctxt;
14481         u64 reg;
14482
14483         if (!rcd || !rcd->sc)
14484                 return -EINVAL;
14485
14486         hw_ctxt = rcd->sc->hw_context;
14487         reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */
14488                 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) <<
14489                  SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT);
14490         /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */
14491         if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY))
14492                 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK;
14493         write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg);
14494         /*
14495          * Enable send-side J_KEY integrity check, unless this is A0 h/w
14496          */
14497         if (!is_ax(dd)) {
14498                 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14499                 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14500                 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14501         }
14502
14503         /* Enable J_KEY check on receive context. */
14504         reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK |
14505                 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) <<
14506                  RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT);
14507         write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg);
14508
14509         return 0;
14510 }
14511
14512 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
14513 {
14514         u8 hw_ctxt;
14515         u64 reg;
14516
14517         if (!rcd || !rcd->sc)
14518                 return -EINVAL;
14519
14520         hw_ctxt = rcd->sc->hw_context;
14521         write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0);
14522         /*
14523          * Disable send-side J_KEY integrity check, unless this is A0 h/w.
14524          * This check would not have been enabled for A0 h/w, see
14525          * set_ctxt_jkey().
14526          */
14527         if (!is_ax(dd)) {
14528                 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14529                 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK;
14530                 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14531         }
14532         /* Turn off the J_KEY on the receive side */
14533         write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0);
14534
14535         return 0;
14536 }
14537
14538 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd,
14539                        u16 pkey)
14540 {
14541         u8 hw_ctxt;
14542         u64 reg;
14543
14544         if (!rcd || !rcd->sc)
14545                 return -EINVAL;
14546
14547         hw_ctxt = rcd->sc->hw_context;
14548         reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) <<
14549                 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT;
14550         write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg);
14551         reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14552         reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14553         reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK;
14554         write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14555
14556         return 0;
14557 }
14558
14559 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt)
14560 {
14561         u8 hw_ctxt;
14562         u64 reg;
14563
14564         if (!ctxt || !ctxt->sc)
14565                 return -EINVAL;
14566
14567         hw_ctxt = ctxt->sc->hw_context;
14568         reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE);
14569         reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK;
14570         write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg);
14571         write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0);
14572
14573         return 0;
14574 }
14575
14576 /*
14577  * Start doing the clean up the the chip. Our clean up happens in multiple
14578  * stages and this is just the first.
14579  */
14580 void hfi1_start_cleanup(struct hfi1_devdata *dd)
14581 {
14582         aspm_exit(dd);
14583         free_cntrs(dd);
14584         free_rcverr(dd);
14585         finish_chip_resources(dd);
14586 }
14587
14588 #define HFI_BASE_GUID(dev) \
14589         ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT))
14590
14591 /*
14592  * Information can be shared between the two HFIs on the same ASIC
14593  * in the same OS.  This function finds the peer device and sets
14594  * up a shared structure.
14595  */
14596 static int init_asic_data(struct hfi1_devdata *dd)
14597 {
14598         unsigned long flags;
14599         struct hfi1_devdata *tmp, *peer = NULL;
14600         struct hfi1_asic_data *asic_data;
14601         int ret = 0;
14602
14603         /* pre-allocate the asic structure in case we are the first device */
14604         asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL);
14605         if (!asic_data)
14606                 return -ENOMEM;
14607
14608         spin_lock_irqsave(&hfi1_devs_lock, flags);
14609         /* Find our peer device */
14610         list_for_each_entry(tmp, &hfi1_dev_list, list) {
14611                 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
14612                     dd->unit != tmp->unit) {
14613                         peer = tmp;
14614                         break;
14615                 }
14616         }
14617
14618         if (peer) {
14619                 /* use already allocated structure */
14620                 dd->asic_data = peer->asic_data;
14621                 kfree(asic_data);
14622         } else {
14623                 dd->asic_data = asic_data;
14624                 mutex_init(&dd->asic_data->asic_resource_mutex);
14625         }
14626         dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
14627         spin_unlock_irqrestore(&hfi1_devs_lock, flags);
14628
14629         /* first one through - set up i2c devices */
14630         if (!peer)
14631                 ret = set_up_i2c(dd, dd->asic_data);
14632
14633         return ret;
14634 }
14635
14636 /*
14637  * Set dd->boardname.  Use a generic name if a name is not returned from
14638  * EFI variable space.
14639  *
14640  * Return 0 on success, -ENOMEM if space could not be allocated.
14641  */
14642 static int obtain_boardname(struct hfi1_devdata *dd)
14643 {
14644         /* generic board description */
14645         const char generic[] =
14646                 "Intel Omni-Path Host Fabric Interface Adapter 100 Series";
14647         unsigned long size;
14648         int ret;
14649
14650         ret = read_hfi1_efi_var(dd, "description", &size,
14651                                 (void **)&dd->boardname);
14652         if (ret) {
14653                 dd_dev_info(dd, "Board description not found\n");
14654                 /* use generic description */
14655                 dd->boardname = kstrdup(generic, GFP_KERNEL);
14656                 if (!dd->boardname)
14657                         return -ENOMEM;
14658         }
14659         return 0;
14660 }
14661
14662 /*
14663  * Check the interrupt registers to make sure that they are mapped correctly.
14664  * It is intended to help user identify any mismapping by VMM when the driver
14665  * is running in a VM. This function should only be called before interrupt
14666  * is set up properly.
14667  *
14668  * Return 0 on success, -EINVAL on failure.
14669  */
14670 static int check_int_registers(struct hfi1_devdata *dd)
14671 {
14672         u64 reg;
14673         u64 all_bits = ~(u64)0;
14674         u64 mask;
14675
14676         /* Clear CceIntMask[0] to avoid raising any interrupts */
14677         mask = read_csr(dd, CCE_INT_MASK);
14678         write_csr(dd, CCE_INT_MASK, 0ull);
14679         reg = read_csr(dd, CCE_INT_MASK);
14680         if (reg)
14681                 goto err_exit;
14682
14683         /* Clear all interrupt status bits */
14684         write_csr(dd, CCE_INT_CLEAR, all_bits);
14685         reg = read_csr(dd, CCE_INT_STATUS);
14686         if (reg)
14687                 goto err_exit;
14688
14689         /* Set all interrupt status bits */
14690         write_csr(dd, CCE_INT_FORCE, all_bits);
14691         reg = read_csr(dd, CCE_INT_STATUS);
14692         if (reg != all_bits)
14693                 goto err_exit;
14694
14695         /* Restore the interrupt mask */
14696         write_csr(dd, CCE_INT_CLEAR, all_bits);
14697         write_csr(dd, CCE_INT_MASK, mask);
14698
14699         return 0;
14700 err_exit:
14701         write_csr(dd, CCE_INT_MASK, mask);
14702         dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n");
14703         return -EINVAL;
14704 }
14705
14706 /**
14707  * hfi1_init_dd() - Initialize most of the dd structure.
14708  * @dev: the pci_dev for hfi1_ib device
14709  * @ent: pci_device_id struct for this dev
14710  *
14711  * This is global, and is called directly at init to set up the
14712  * chip-specific function pointers for later use.
14713  */
14714 int hfi1_init_dd(struct hfi1_devdata *dd)
14715 {
14716         struct pci_dev *pdev = dd->pcidev;
14717         struct hfi1_pportdata *ppd;
14718         u64 reg;
14719         int i, ret;
14720         static const char * const inames[] = { /* implementation names */
14721                 "RTL silicon",
14722                 "RTL VCS simulation",
14723                 "RTL FPGA emulation",
14724                 "Functional simulator"
14725         };
14726         struct pci_dev *parent = pdev->bus->self;
14727         u32 sdma_engines = chip_sdma_engines(dd);
14728
14729         ppd = dd->pport;
14730         for (i = 0; i < dd->num_pports; i++, ppd++) {
14731                 int vl;
14732                 /* init common fields */
14733                 hfi1_init_pportdata(pdev, ppd, dd, 0, 1);
14734                 /* DC supports 4 link widths */
14735                 ppd->link_width_supported =
14736                         OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X |
14737                         OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X;
14738                 ppd->link_width_downgrade_supported =
14739                         ppd->link_width_supported;
14740                 /* start out enabling only 4X */
14741                 ppd->link_width_enabled = OPA_LINK_WIDTH_4X;
14742                 ppd->link_width_downgrade_enabled =
14743                                         ppd->link_width_downgrade_supported;
14744                 /* link width active is 0 when link is down */
14745                 /* link width downgrade active is 0 when link is down */
14746
14747                 if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
14748                     num_vls > HFI1_MAX_VLS_SUPPORTED) {
14749                         dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n",
14750                                    num_vls, HFI1_MAX_VLS_SUPPORTED);
14751                         num_vls = HFI1_MAX_VLS_SUPPORTED;
14752                 }
14753                 ppd->vls_supported = num_vls;
14754                 ppd->vls_operational = ppd->vls_supported;
14755                 /* Set the default MTU. */
14756                 for (vl = 0; vl < num_vls; vl++)
14757                         dd->vld[vl].mtu = hfi1_max_mtu;
14758                 dd->vld[15].mtu = MAX_MAD_PACKET;
14759                 /*
14760                  * Set the initial values to reasonable default, will be set
14761                  * for real when link is up.
14762                  */
14763                 ppd->overrun_threshold = 0x4;
14764                 ppd->phy_error_threshold = 0xf;
14765                 ppd->port_crc_mode_enabled = link_crc_mask;
14766                 /* initialize supported LTP CRC mode */
14767                 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8;
14768                 /* initialize enabled LTP CRC mode */
14769                 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4;
14770                 /* start in offline */
14771                 ppd->host_link_state = HLS_DN_OFFLINE;
14772                 init_vl_arb_caches(ppd);
14773         }
14774
14775         /*
14776          * Do remaining PCIe setup and save PCIe values in dd.
14777          * Any error printing is already done by the init code.
14778          * On return, we have the chip mapped.
14779          */
14780         ret = hfi1_pcie_ddinit(dd, pdev);
14781         if (ret < 0)
14782                 goto bail_free;
14783
14784         /* Save PCI space registers to rewrite after device reset */
14785         ret = save_pci_variables(dd);
14786         if (ret < 0)
14787                 goto bail_cleanup;
14788
14789         dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT)
14790                         & CCE_REVISION_CHIP_REV_MAJOR_MASK;
14791         dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT)
14792                         & CCE_REVISION_CHIP_REV_MINOR_MASK;
14793
14794         /*
14795          * Check interrupt registers mapping if the driver has no access to
14796          * the upstream component. In this case, it is likely that the driver
14797          * is running in a VM.
14798          */
14799         if (!parent) {
14800                 ret = check_int_registers(dd);
14801                 if (ret)
14802                         goto bail_cleanup;
14803         }
14804
14805         /*
14806          * obtain the hardware ID - NOT related to unit, which is a
14807          * software enumeration
14808          */
14809         reg = read_csr(dd, CCE_REVISION2);
14810         dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT)
14811                                         & CCE_REVISION2_HFI_ID_MASK;
14812         /* the variable size will remove unwanted bits */
14813         dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT;
14814         dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT;
14815         dd_dev_info(dd, "Implementation: %s, revision 0x%x\n",
14816                     dd->icode < ARRAY_SIZE(inames) ?
14817                     inames[dd->icode] : "unknown", (int)dd->irev);
14818
14819         /* speeds the hardware can support */
14820         dd->pport->link_speed_supported = OPA_LINK_SPEED_25G;
14821         /* speeds allowed to run at */
14822         dd->pport->link_speed_enabled = dd->pport->link_speed_supported;
14823         /* give a reasonable active value, will be set on link up */
14824         dd->pport->link_speed_active = OPA_LINK_SPEED_25G;
14825
14826         /* fix up link widths for emulation _p */
14827         ppd = dd->pport;
14828         if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) {
14829                 ppd->link_width_supported =
14830                         ppd->link_width_enabled =
14831                         ppd->link_width_downgrade_supported =
14832                         ppd->link_width_downgrade_enabled =
14833                                 OPA_LINK_WIDTH_1X;
14834         }
14835         /* insure num_vls isn't larger than number of sdma engines */
14836         if (HFI1_CAP_IS_KSET(SDMA) && num_vls > sdma_engines) {
14837                 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n",
14838                            num_vls, sdma_engines);
14839                 num_vls = sdma_engines;
14840                 ppd->vls_supported = sdma_engines;
14841                 ppd->vls_operational = ppd->vls_supported;
14842         }
14843
14844         /*
14845          * Convert the ns parameter to the 64 * cclocks used in the CSR.
14846          * Limit the max if larger than the field holds.  If timeout is
14847          * non-zero, then the calculated field will be at least 1.
14848          *
14849          * Must be after icode is set up - the cclock rate depends
14850          * on knowing the hardware being used.
14851          */
14852         dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64;
14853         if (dd->rcv_intr_timeout_csr >
14854                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK)
14855                 dd->rcv_intr_timeout_csr =
14856                         RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK;
14857         else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout)
14858                 dd->rcv_intr_timeout_csr = 1;
14859
14860         /* needs to be done before we look for the peer device */
14861         read_guid(dd);
14862
14863         /* set up shared ASIC data with peer device */
14864         ret = init_asic_data(dd);
14865         if (ret)
14866                 goto bail_cleanup;
14867
14868         /* obtain chip sizes, reset chip CSRs */
14869         ret = init_chip(dd);
14870         if (ret)
14871                 goto bail_cleanup;
14872
14873         /* read in the PCIe link speed information */
14874         ret = pcie_speeds(dd);
14875         if (ret)
14876                 goto bail_cleanup;
14877
14878         /* call before get_platform_config(), after init_chip_resources() */
14879         ret = eprom_init(dd);
14880         if (ret)
14881                 goto bail_free_rcverr;
14882
14883         /* Needs to be called before hfi1_firmware_init */
14884         get_platform_config(dd);
14885
14886         /* read in firmware */
14887         ret = hfi1_firmware_init(dd);
14888         if (ret)
14889                 goto bail_cleanup;
14890
14891         /*
14892          * In general, the PCIe Gen3 transition must occur after the
14893          * chip has been idled (so it won't initiate any PCIe transactions
14894          * e.g. an interrupt) and before the driver changes any registers
14895          * (the transition will reset the registers).
14896          *
14897          * In particular, place this call after:
14898          * - init_chip()     - the chip will not initiate any PCIe transactions
14899          * - pcie_speeds()   - reads the current link speed
14900          * - hfi1_firmware_init() - the needed firmware is ready to be
14901          *                          downloaded
14902          */
14903         ret = do_pcie_gen3_transition(dd);
14904         if (ret)
14905                 goto bail_cleanup;
14906
14907         /*
14908          * This should probably occur in hfi1_pcie_init(), but historically
14909          * occurs after the do_pcie_gen3_transition() code.
14910          */
14911         tune_pcie_caps(dd);
14912
14913         /* start setting dd values and adjusting CSRs */
14914         init_early_variables(dd);
14915
14916         parse_platform_config(dd);
14917
14918         ret = obtain_boardname(dd);
14919         if (ret)
14920                 goto bail_cleanup;
14921
14922         snprintf(dd->boardversion, BOARD_VERS_MAX,
14923                  "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n",
14924                  HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN,
14925                  (u32)dd->majrev,
14926                  (u32)dd->minrev,
14927                  (dd->revision >> CCE_REVISION_SW_SHIFT)
14928                     & CCE_REVISION_SW_MASK);
14929
14930         ret = set_up_context_variables(dd);
14931         if (ret)
14932                 goto bail_cleanup;
14933
14934         /* set initial RXE CSRs */
14935         init_rxe(dd);
14936         /* set initial TXE CSRs */
14937         init_txe(dd);
14938         /* set initial non-RXE, non-TXE CSRs */
14939         init_other(dd);
14940         /* set up KDETH QP prefix in both RX and TX CSRs */
14941         init_kdeth_qp(dd);
14942
14943         ret = hfi1_dev_affinity_init(dd);
14944         if (ret)
14945                 goto bail_cleanup;
14946
14947         /* send contexts must be set up before receive contexts */
14948         ret = init_send_contexts(dd);
14949         if (ret)
14950                 goto bail_cleanup;
14951
14952         ret = hfi1_create_kctxts(dd);
14953         if (ret)
14954                 goto bail_cleanup;
14955
14956         /*
14957          * Initialize aspm, to be done after gen3 transition and setting up
14958          * contexts and before enabling interrupts
14959          */
14960         aspm_init(dd);
14961
14962         ret = init_pervl_scs(dd);
14963         if (ret)
14964                 goto bail_cleanup;
14965
14966         /* sdma init */
14967         for (i = 0; i < dd->num_pports; ++i) {
14968                 ret = sdma_init(dd, i);
14969                 if (ret)
14970                         goto bail_cleanup;
14971         }
14972
14973         /* use contexts created by hfi1_create_kctxts */
14974         ret = set_up_interrupts(dd);
14975         if (ret)
14976                 goto bail_cleanup;
14977
14978         ret = hfi1_comp_vectors_set_up(dd);
14979         if (ret)
14980                 goto bail_clear_intr;
14981
14982         /* set up LCB access - must be after set_up_interrupts() */
14983         init_lcb_access(dd);
14984
14985         /*
14986          * Serial number is created from the base guid:
14987          * [27:24] = base guid [38:35]
14988          * [23: 0] = base guid [23: 0]
14989          */
14990         snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n",
14991                  (dd->base_guid & 0xFFFFFF) |
14992                      ((dd->base_guid >> 11) & 0xF000000));
14993
14994         dd->oui1 = dd->base_guid >> 56 & 0xFF;
14995         dd->oui2 = dd->base_guid >> 48 & 0xFF;
14996         dd->oui3 = dd->base_guid >> 40 & 0xFF;
14997
14998         ret = load_firmware(dd); /* asymmetric with dispose_firmware() */
14999         if (ret)
15000                 goto bail_clear_intr;
15001
15002         thermal_init(dd);
15003
15004         ret = init_cntrs(dd);
15005         if (ret)
15006                 goto bail_clear_intr;
15007
15008         ret = init_rcverr(dd);
15009         if (ret)
15010                 goto bail_free_cntrs;
15011
15012         init_completion(&dd->user_comp);
15013
15014         /* The user refcount starts with one to inidicate an active device */
15015         atomic_set(&dd->user_refcount, 1);
15016
15017         goto bail;
15018
15019 bail_free_rcverr:
15020         free_rcverr(dd);
15021 bail_free_cntrs:
15022         free_cntrs(dd);
15023 bail_clear_intr:
15024         hfi1_comp_vectors_clean_up(dd);
15025         msix_clean_up_interrupts(dd);
15026 bail_cleanup:
15027         hfi1_pcie_ddcleanup(dd);
15028 bail_free:
15029         hfi1_free_devdata(dd);
15030 bail:
15031         return ret;
15032 }
15033
15034 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate,
15035                         u32 dw_len)
15036 {
15037         u32 delta_cycles;
15038         u32 current_egress_rate = ppd->current_egress_rate;
15039         /* rates here are in units of 10^6 bits/sec */
15040
15041         if (desired_egress_rate == -1)
15042                 return 0; /* shouldn't happen */
15043
15044         if (desired_egress_rate >= current_egress_rate)
15045                 return 0; /* we can't help go faster, only slower */
15046
15047         delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) -
15048                         egress_cycles(dw_len * 4, current_egress_rate);
15049
15050         return (u16)delta_cycles;
15051 }
15052
15053 /**
15054  * create_pbc - build a pbc for transmission
15055  * @flags: special case flags or-ed in built pbc
15056  * @srate: static rate
15057  * @vl: vl
15058  * @dwlen: dword length (header words + data words + pbc words)
15059  *
15060  * Create a PBC with the given flags, rate, VL, and length.
15061  *
15062  * NOTE: The PBC created will not insert any HCRC - all callers but one are
15063  * for verbs, which does not use this PSM feature.  The lone other caller
15064  * is for the diagnostic interface which calls this if the user does not
15065  * supply their own PBC.
15066  */
15067 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl,
15068                u32 dw_len)
15069 {
15070         u64 pbc, delay = 0;
15071
15072         if (unlikely(srate_mbs))
15073                 delay = delay_cycles(ppd, srate_mbs, dw_len);
15074
15075         pbc = flags
15076                 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT)
15077                 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT)
15078                 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT
15079                 | (dw_len & PBC_LENGTH_DWS_MASK)
15080                         << PBC_LENGTH_DWS_SHIFT;
15081
15082         return pbc;
15083 }
15084
15085 #define SBUS_THERMAL    0x4f
15086 #define SBUS_THERM_MONITOR_MODE 0x1
15087
15088 #define THERM_FAILURE(dev, ret, reason) \
15089         dd_dev_err((dd),                                                \
15090                    "Thermal sensor initialization failed: %s (%d)\n",   \
15091                    (reason), (ret))
15092
15093 /*
15094  * Initialize the thermal sensor.
15095  *
15096  * After initialization, enable polling of thermal sensor through
15097  * SBus interface. In order for this to work, the SBus Master
15098  * firmware has to be loaded due to the fact that the HW polling
15099  * logic uses SBus interrupts, which are not supported with
15100  * default firmware. Otherwise, no data will be returned through
15101  * the ASIC_STS_THERM CSR.
15102  */
15103 static int thermal_init(struct hfi1_devdata *dd)
15104 {
15105         int ret = 0;
15106
15107         if (dd->icode != ICODE_RTL_SILICON ||
15108             check_chip_resource(dd, CR_THERM_INIT, NULL))
15109                 return ret;
15110
15111         ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT);
15112         if (ret) {
15113                 THERM_FAILURE(dd, ret, "Acquire SBus");
15114                 return ret;
15115         }
15116
15117         dd_dev_info(dd, "Initializing thermal sensor\n");
15118         /* Disable polling of thermal readings */
15119         write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0);
15120         msleep(100);
15121         /* Thermal Sensor Initialization */
15122         /*    Step 1: Reset the Thermal SBus Receiver */
15123         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15124                                 RESET_SBUS_RECEIVER, 0);
15125         if (ret) {
15126                 THERM_FAILURE(dd, ret, "Bus Reset");
15127                 goto done;
15128         }
15129         /*    Step 2: Set Reset bit in Thermal block */
15130         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15131                                 WRITE_SBUS_RECEIVER, 0x1);
15132         if (ret) {
15133                 THERM_FAILURE(dd, ret, "Therm Block Reset");
15134                 goto done;
15135         }
15136         /*    Step 3: Write clock divider value (100MHz -> 2MHz) */
15137         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1,
15138                                 WRITE_SBUS_RECEIVER, 0x32);
15139         if (ret) {
15140                 THERM_FAILURE(dd, ret, "Write Clock Div");
15141                 goto done;
15142         }
15143         /*    Step 4: Select temperature mode */
15144         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3,
15145                                 WRITE_SBUS_RECEIVER,
15146                                 SBUS_THERM_MONITOR_MODE);
15147         if (ret) {
15148                 THERM_FAILURE(dd, ret, "Write Mode Sel");
15149                 goto done;
15150         }
15151         /*    Step 5: De-assert block reset and start conversion */
15152         ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0,
15153                                 WRITE_SBUS_RECEIVER, 0x2);
15154         if (ret) {
15155                 THERM_FAILURE(dd, ret, "Write Reset Deassert");
15156                 goto done;
15157         }
15158         /*    Step 5.1: Wait for first conversion (21.5ms per spec) */
15159         msleep(22);
15160
15161         /* Enable polling of thermal readings */
15162         write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1);
15163
15164         /* Set initialized flag */
15165         ret = acquire_chip_resource(dd, CR_THERM_INIT, 0);
15166         if (ret)
15167                 THERM_FAILURE(dd, ret, "Unable to set thermal init flag");
15168
15169 done:
15170         release_chip_resource(dd, CR_SBUS);
15171         return ret;
15172 }
15173
15174 static void handle_temp_err(struct hfi1_devdata *dd)
15175 {
15176         struct hfi1_pportdata *ppd = &dd->pport[0];
15177         /*
15178          * Thermal Critical Interrupt
15179          * Put the device into forced freeze mode, take link down to
15180          * offline, and put DC into reset.
15181          */
15182         dd_dev_emerg(dd,
15183                      "Critical temperature reached! Forcing device into freeze mode!\n");
15184         dd->flags |= HFI1_FORCED_FREEZE;
15185         start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT);
15186         /*
15187          * Shut DC down as much and as quickly as possible.
15188          *
15189          * Step 1: Take the link down to OFFLINE. This will cause the
15190          *         8051 to put the Serdes in reset. However, we don't want to
15191          *         go through the entire link state machine since we want to
15192          *         shutdown ASAP. Furthermore, this is not a graceful shutdown
15193          *         but rather an attempt to save the chip.
15194          *         Code below is almost the same as quiet_serdes() but avoids
15195          *         all the extra work and the sleeps.
15196          */
15197         ppd->driver_link_ready = 0;
15198         ppd->link_enabled = 0;
15199         set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) |
15200                                 PLS_OFFLINE);
15201         /*
15202          * Step 2: Shutdown LCB and 8051
15203          *         After shutdown, do not restore DC_CFG_RESET value.
15204          */
15205         dc_shutdown(dd);
15206 }