drivers/scsi/hisi_sas/hisi_sas_v3_hw.c

   1 /*
   2  * Copyright (c) 2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  */
  10
  11 #include "hisi_sas.h"
  12 #define DRV_NAME "hisi_sas_v3_hw"
  13
  14 /* global registers need init*/
  15 #define DLVRY_QUEUE_ENABLE              0x0
  16 #define IOST_BASE_ADDR_LO               0x8
  17 #define IOST_BASE_ADDR_HI               0xc
  18 #define ITCT_BASE_ADDR_LO               0x10
  19 #define ITCT_BASE_ADDR_HI               0x14
  20 #define IO_BROKEN_MSG_ADDR_LO           0x18
  21 #define IO_BROKEN_MSG_ADDR_HI           0x1c
  22 #define PHY_CONTEXT                     0x20
  23 #define PHY_STATE                       0x24
  24 #define PHY_PORT_NUM_MA                 0x28
  25 #define PHY_CONN_RATE                   0x30
  26 #define ITCT_CLR                        0x44
  27 #define ITCT_CLR_EN_OFF                 16
  28 #define ITCT_CLR_EN_MSK                 (0x1 << ITCT_CLR_EN_OFF)
  29 #define ITCT_DEV_OFF                    0
  30 #define ITCT_DEV_MSK                    (0x7ff << ITCT_DEV_OFF)
  31 #define IO_SATA_BROKEN_MSG_ADDR_LO      0x58
  32 #define IO_SATA_BROKEN_MSG_ADDR_HI      0x5c
  33 #define SATA_INITI_D2H_STORE_ADDR_LO    0x60
  34 #define SATA_INITI_D2H_STORE_ADDR_HI    0x64
  35 #define CFG_MAX_TAG                     0x68
  36 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
  37 #define HGC_SAS_TXFAIL_RETRY_CTRL       0x88
  38 #define HGC_GET_ITV_TIME                0x90
  39 #define DEVICE_MSG_WORK_MODE            0x94
  40 #define OPENA_WT_CONTI_TIME             0x9c
  41 #define I_T_NEXUS_LOSS_TIME             0xa0
  42 #define MAX_CON_TIME_LIMIT_TIME         0xa4
  43 #define BUS_INACTIVE_LIMIT_TIME         0xa8
  44 #define REJECT_TO_OPEN_LIMIT_TIME       0xac
  45 #define CFG_AGING_TIME                  0xbc
  46 #define HGC_DFX_CFG2                    0xc0
  47 #define CFG_ABT_SET_QUERY_IPTT  0xd4
  48 #define CFG_SET_ABORTED_IPTT_OFF        0
  49 #define CFG_SET_ABORTED_IPTT_MSK        (0xfff << CFG_SET_ABORTED_IPTT_OFF)
  50 #define CFG_SET_ABORTED_EN_OFF  12
  51 #define CFG_ABT_SET_IPTT_DONE   0xd8
  52 #define CFG_ABT_SET_IPTT_DONE_OFF       0
  53 #define HGC_IOMB_PROC1_STATUS   0x104
  54 #define CFG_1US_TIMER_TRSH              0xcc
  55 #define CHNL_INT_STATUS                 0x148
  56 #define HGC_AXI_FIFO_ERR_INFO  0x154
  57 #define AXI_ERR_INFO_OFF               0
  58 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
  59 #define FIFO_ERR_INFO_OFF              8
  60 #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
  61 #define INT_COAL_EN                     0x19c
  62 #define OQ_INT_COAL_TIME                0x1a0
  63 #define OQ_INT_COAL_CNT                 0x1a4
  64 #define ENT_INT_COAL_TIME               0x1a8
  65 #define ENT_INT_COAL_CNT                0x1ac
  66 #define OQ_INT_SRC                      0x1b0
  67 #define OQ_INT_SRC_MSK                  0x1b4
  68 #define ENT_INT_SRC1                    0x1b8
  69 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF    0
  70 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
  71 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF    8
  72 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
  73 #define ENT_INT_SRC2                    0x1bc
  74 #define ENT_INT_SRC3                    0x1c0
  75 #define ENT_INT_SRC3_WP_DEPTH_OFF               8
  76 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF      9
  77 #define ENT_INT_SRC3_RP_DEPTH_OFF               10
  78 #define ENT_INT_SRC3_AXI_OFF                    11
  79 #define ENT_INT_SRC3_FIFO_OFF                   12
  80 #define ENT_INT_SRC3_LM_OFF                             14
  81 #define ENT_INT_SRC3_ITC_INT_OFF        15
  82 #define ENT_INT_SRC3_ITC_INT_MSK        (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
  83 #define ENT_INT_SRC3_ABT_OFF            16
  84 #define ENT_INT_SRC_MSK1                0x1c4
  85 #define ENT_INT_SRC_MSK2                0x1c8
  86 #define ENT_INT_SRC_MSK3                0x1cc
  87 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF  31
  88 #define CHNL_PHYUPDOWN_INT_MSK          0x1d0
  89 #define CHNL_ENT_INT_MSK                        0x1d4
  90 #define HGC_COM_INT_MSK                         0x1d8
  91 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK  (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
  92 #define SAS_ECC_INTR                    0x1e8
  93 #define SAS_ECC_INTR_MSK                0x1ec
  94 #define HGC_ERR_STAT_EN                 0x238
  95 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
  96 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
  97 #define DLVRY_Q_0_DEPTH                 0x268
  98 #define DLVRY_Q_0_WR_PTR                0x26c
  99 #define DLVRY_Q_0_RD_PTR                0x270
 100 #define HYPER_STREAM_ID_EN_CFG          0xc80
 101 #define OQ0_INT_SRC_MSK                 0xc90
 102 #define COMPL_Q_0_BASE_ADDR_LO          0x4e0
 103 #define COMPL_Q_0_BASE_ADDR_HI          0x4e4
 104 #define COMPL_Q_0_DEPTH                 0x4e8
 105 #define COMPL_Q_0_WR_PTR                0x4ec
 106 #define COMPL_Q_0_RD_PTR                0x4f0
 107 #define AWQOS_AWCACHE_CFG       0xc84
 108 #define ARQOS_ARCACHE_CFG       0xc88
 109
 110 /* phy registers requiring init */
 111 #define PORT_BASE                       (0x2000)
 112 #define PHY_CFG                         (PORT_BASE + 0x0)
 113 #define HARD_PHY_LINKRATE               (PORT_BASE + 0x4)
 114 #define PHY_CFG_ENA_OFF                 0
 115 #define PHY_CFG_ENA_MSK                 (0x1 << PHY_CFG_ENA_OFF)
 116 #define PHY_CFG_DC_OPT_OFF              2
 117 #define PHY_CFG_DC_OPT_MSK              (0x1 << PHY_CFG_DC_OPT_OFF)
 118 #define PROG_PHY_LINK_RATE              (PORT_BASE + 0x8)
 119 #define PHY_CTRL                        (PORT_BASE + 0x14)
 120 #define PHY_CTRL_RESET_OFF              0
 121 #define PHY_CTRL_RESET_MSK              (0x1 << PHY_CTRL_RESET_OFF)
 122 #define SL_CFG                          (PORT_BASE + 0x84)
 123 #define SL_CONTROL                      (PORT_BASE + 0x94)
 124 #define SL_CONTROL_NOTIFY_EN_OFF        0
 125 #define SL_CONTROL_NOTIFY_EN_MSK        (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 126 #define SL_CTA_OFF              17
 127 #define SL_CTA_MSK              (0x1 << SL_CTA_OFF)
 128 #define TX_ID_DWORD0                    (PORT_BASE + 0x9c)
 129 #define TX_ID_DWORD1                    (PORT_BASE + 0xa0)
 130 #define TX_ID_DWORD2                    (PORT_BASE + 0xa4)
 131 #define TX_ID_DWORD3                    (PORT_BASE + 0xa8)
 132 #define TX_ID_DWORD4                    (PORT_BASE + 0xaC)
 133 #define TX_ID_DWORD5                    (PORT_BASE + 0xb0)
 134 #define TX_ID_DWORD6                    (PORT_BASE + 0xb4)
 135 #define TXID_AUTO                               (PORT_BASE + 0xb8)
 136 #define CT3_OFF         1
 137 #define CT3_MSK         (0x1 << CT3_OFF)
 138 #define TX_HARDRST_OFF          2
 139 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 140 #define RX_IDAF_DWORD0                  (PORT_BASE + 0xc4)
 141 #define RXOP_CHECK_CFG_H                (PORT_BASE + 0xfc)
 142 #define STP_LINK_TIMER                  (PORT_BASE + 0x120)
 143 #define STP_LINK_TIMEOUT_STATE          (PORT_BASE + 0x124)
 144 #define CON_CFG_DRIVER                  (PORT_BASE + 0x130)
 145 #define SAS_SSP_CON_TIMER_CFG           (PORT_BASE + 0x134)
 146 #define SAS_SMP_CON_TIMER_CFG           (PORT_BASE + 0x138)
 147 #define SAS_STP_CON_TIMER_CFG           (PORT_BASE + 0x13c)
 148 #define CHL_INT0                        (PORT_BASE + 0x1b4)
 149 #define CHL_INT0_HOTPLUG_TOUT_OFF       0
 150 #define CHL_INT0_HOTPLUG_TOUT_MSK       (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
 151 #define CHL_INT0_SL_RX_BCST_ACK_OFF     1
 152 #define CHL_INT0_SL_RX_BCST_ACK_MSK     (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
 153 #define CHL_INT0_SL_PHY_ENABLE_OFF      2
 154 #define CHL_INT0_SL_PHY_ENABLE_MSK      (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
 155 #define CHL_INT0_NOT_RDY_OFF            4
 156 #define CHL_INT0_NOT_RDY_MSK            (0x1 << CHL_INT0_NOT_RDY_OFF)
 157 #define CHL_INT0_PHY_RDY_OFF            5
 158 #define CHL_INT0_PHY_RDY_MSK            (0x1 << CHL_INT0_PHY_RDY_OFF)
 159 #define CHL_INT1                        (PORT_BASE + 0x1b8)
 160 #define CHL_INT1_DMAC_TX_ECC_ERR_OFF    15
 161 #define CHL_INT1_DMAC_TX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF)
 162 #define CHL_INT1_DMAC_RX_ECC_ERR_OFF    17
 163 #define CHL_INT1_DMAC_RX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF)
 164 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
 165 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
 166 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
 167 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 168 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 169 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
 170 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 171 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 172 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
 173 #define CHL_INT2_MSK                    (PORT_BASE + 0x1c8)
 174 #define CHL_INT_COAL_EN                 (PORT_BASE + 0x1d0)
 175 #define SAS_RX_TRAIN_TIMER              (PORT_BASE + 0x2a4)
 176 #define PHY_CTRL_RDY_MSK                (PORT_BASE + 0x2b0)
 177 #define PHYCTRL_NOT_RDY_MSK             (PORT_BASE + 0x2b4)
 178 #define PHYCTRL_DWS_RESET_MSK           (PORT_BASE + 0x2b8)
 179 #define PHYCTRL_PHY_ENA_MSK             (PORT_BASE + 0x2bc)
 180 #define SL_RX_BCAST_CHK_MSK             (PORT_BASE + 0x2c0)
 181 #define PHYCTRL_OOB_RESTART_MSK         (PORT_BASE + 0x2c4)
 182 #define DMA_TX_STATUS                   (PORT_BASE + 0x2d0)
 183 #define DMA_TX_STATUS_BUSY_OFF          0
 184 #define DMA_TX_STATUS_BUSY_MSK          (0x1 << DMA_TX_STATUS_BUSY_OFF)
 185 #define DMA_RX_STATUS                   (PORT_BASE + 0x2e8)
 186 #define DMA_RX_STATUS_BUSY_OFF          0
 187 #define DMA_RX_STATUS_BUSY_MSK          (0x1 << DMA_RX_STATUS_BUSY_OFF)
 188
 189 #define COARSETUNE_TIME                 (PORT_BASE + 0x304)
 190 #define ERR_CNT_DWS_LOST                (PORT_BASE + 0x380)
 191 #define ERR_CNT_RESET_PROB              (PORT_BASE + 0x384)
 192 #define ERR_CNT_INVLD_DW                (PORT_BASE + 0x390)
 193 #define ERR_CNT_DISP_ERR                (PORT_BASE + 0x398)
 194
 195 #define DEFAULT_ITCT_HW         2048 /* reset value, not reprogrammed */
 196 #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
 197 #error Max ITCT exceeded
 198 #endif
 199
 200 #define AXI_MASTER_CFG_BASE             (0x5000)
 201 #define AM_CTRL_GLOBAL                  (0x0)
 202 #define AM_CURR_TRANS_RETURN    (0x150)
 203
 204 #define AM_CFG_MAX_TRANS                (0x5010)
 205 #define AM_CFG_SINGLE_PORT_MAX_TRANS    (0x5014)
 206 #define AXI_CFG                                 (0x5100)
 207 #define AM_ROB_ECC_ERR_ADDR             (0x510c)
 208 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF  0
 209 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
 210 #define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF  8
 211 #define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
 212
 213 /* RAS registers need init */
 214 #define RAS_BASE                (0x6000)
 215 #define SAS_RAS_INTR0                   (RAS_BASE)
 216 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 217 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 218 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
 219 #define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
 220 #define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
 221 #define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 222
 223 /* HW dma structures */
 224 /* Delivery queue header */
 225 /* dw0 */
 226 #define CMD_HDR_ABORT_FLAG_OFF          0
 227 #define CMD_HDR_ABORT_FLAG_MSK          (0x3 << CMD_HDR_ABORT_FLAG_OFF)
 228 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF   2
 229 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK   (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 230 #define CMD_HDR_RESP_REPORT_OFF         5
 231 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 232 #define CMD_HDR_TLR_CTRL_OFF            6
 233 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
 234 #define CMD_HDR_PORT_OFF                18
 235 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 236 #define CMD_HDR_PRIORITY_OFF            27
 237 #define CMD_HDR_PRIORITY_MSK            (0x1 << CMD_HDR_PRIORITY_OFF)
 238 #define CMD_HDR_CMD_OFF                 29
 239 #define CMD_HDR_CMD_MSK                 (0x7 << CMD_HDR_CMD_OFF)
 240 /* dw1 */
 241 #define CMD_HDR_UNCON_CMD_OFF   3
 242 #define CMD_HDR_DIR_OFF                 5
 243 #define CMD_HDR_DIR_MSK                 (0x3 << CMD_HDR_DIR_OFF)
 244 #define CMD_HDR_RESET_OFF               7
 245 #define CMD_HDR_RESET_MSK               (0x1 << CMD_HDR_RESET_OFF)
 246 #define CMD_HDR_VDTL_OFF                10
 247 #define CMD_HDR_VDTL_MSK                (0x1 << CMD_HDR_VDTL_OFF)
 248 #define CMD_HDR_FRAME_TYPE_OFF          11
 249 #define CMD_HDR_FRAME_TYPE_MSK          (0x1f << CMD_HDR_FRAME_TYPE_OFF)
 250 #define CMD_HDR_DEV_ID_OFF              16
 251 #define CMD_HDR_DEV_ID_MSK              (0xffff << CMD_HDR_DEV_ID_OFF)
 252 /* dw2 */
 253 #define CMD_HDR_CFL_OFF                 0
 254 #define CMD_HDR_CFL_MSK                 (0x1ff << CMD_HDR_CFL_OFF)
 255 #define CMD_HDR_NCQ_TAG_OFF             10
 256 #define CMD_HDR_NCQ_TAG_MSK             (0x1f << CMD_HDR_NCQ_TAG_OFF)
 257 #define CMD_HDR_MRFL_OFF                15
 258 #define CMD_HDR_MRFL_MSK                (0x1ff << CMD_HDR_MRFL_OFF)
 259 #define CMD_HDR_SG_MOD_OFF              24
 260 #define CMD_HDR_SG_MOD_MSK              (0x3 << CMD_HDR_SG_MOD_OFF)
 261 /* dw3 */
 262 #define CMD_HDR_IPTT_OFF                0
 263 #define CMD_HDR_IPTT_MSK                (0xffff << CMD_HDR_IPTT_OFF)
 264 /* dw6 */
 265 #define CMD_HDR_DIF_SGL_LEN_OFF         0
 266 #define CMD_HDR_DIF_SGL_LEN_MSK         (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 267 #define CMD_HDR_DATA_SGL_LEN_OFF        16
 268 #define CMD_HDR_DATA_SGL_LEN_MSK        (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
 269 /* dw7 */
 270 #define CMD_HDR_ADDR_MODE_SEL_OFF               15
 271 #define CMD_HDR_ADDR_MODE_SEL_MSK               (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
 272 #define CMD_HDR_ABORT_IPTT_OFF          16
 273 #define CMD_HDR_ABORT_IPTT_MSK          (0xffff << CMD_HDR_ABORT_IPTT_OFF)
 274
 275 /* Completion header */
 276 /* dw0 */
 277 #define CMPLT_HDR_CMPLT_OFF             0
 278 #define CMPLT_HDR_CMPLT_MSK             (0x3 << CMPLT_HDR_CMPLT_OFF)
 279 #define CMPLT_HDR_ERROR_PHASE_OFF   2
 280 #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
 281 #define CMPLT_HDR_RSPNS_XFRD_OFF        10
 282 #define CMPLT_HDR_RSPNS_XFRD_MSK        (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 283 #define CMPLT_HDR_ERX_OFF               12
 284 #define CMPLT_HDR_ERX_MSK               (0x1 << CMPLT_HDR_ERX_OFF)
 285 #define CMPLT_HDR_ABORT_STAT_OFF        13
 286 #define CMPLT_HDR_ABORT_STAT_MSK        (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
 287 /* abort_stat */
 288 #define STAT_IO_NOT_VALID               0x1
 289 #define STAT_IO_NO_DEVICE               0x2
 290 #define STAT_IO_COMPLETE                0x3
 291 #define STAT_IO_ABORTED                 0x4
 292 /* dw1 */
 293 #define CMPLT_HDR_IPTT_OFF              0
 294 #define CMPLT_HDR_IPTT_MSK              (0xffff << CMPLT_HDR_IPTT_OFF)
 295 #define CMPLT_HDR_DEV_ID_OFF            16
 296 #define CMPLT_HDR_DEV_ID_MSK            (0xffff << CMPLT_HDR_DEV_ID_OFF)
 297 /* dw3 */
 298 #define CMPLT_HDR_IO_IN_TARGET_OFF      17
 299 #define CMPLT_HDR_IO_IN_TARGET_MSK      (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
 300
 301 /* ITCT header */
 302 /* qw0 */
 303 #define ITCT_HDR_DEV_TYPE_OFF           0
 304 #define ITCT_HDR_DEV_TYPE_MSK           (0x3 << ITCT_HDR_DEV_TYPE_OFF)
 305 #define ITCT_HDR_VALID_OFF              2
 306 #define ITCT_HDR_VALID_MSK              (0x1 << ITCT_HDR_VALID_OFF)
 307 #define ITCT_HDR_MCR_OFF                5
 308 #define ITCT_HDR_MCR_MSK                (0xf << ITCT_HDR_MCR_OFF)
 309 #define ITCT_HDR_VLN_OFF                9
 310 #define ITCT_HDR_VLN_MSK                (0xf << ITCT_HDR_VLN_OFF)
 311 #define ITCT_HDR_SMP_TIMEOUT_OFF        16
 312 #define ITCT_HDR_AWT_CONTINUE_OFF       25
 313 #define ITCT_HDR_PORT_ID_OFF            28
 314 #define ITCT_HDR_PORT_ID_MSK            (0xf << ITCT_HDR_PORT_ID_OFF)
 315 /* qw2 */
 316 #define ITCT_HDR_INLT_OFF               0
 317 #define ITCT_HDR_INLT_MSK               (0xffffULL << ITCT_HDR_INLT_OFF)
 318 #define ITCT_HDR_RTOLT_OFF              48
 319 #define ITCT_HDR_RTOLT_MSK              (0xffffULL << ITCT_HDR_RTOLT_OFF)
 320
 321 struct hisi_sas_complete_v3_hdr {
 322         __le32 dw0;
 323         __le32 dw1;
 324         __le32 act;
 325         __le32 dw3;
 326 };
 327
 328 struct hisi_sas_err_record_v3 {
 329         /* dw0 */
 330         __le32 trans_tx_fail_type;
 331
 332         /* dw1 */
 333         __le32 trans_rx_fail_type;
 334
 335         /* dw2 */
 336         __le16 dma_tx_err_type;
 337         __le16 sipc_rx_err_type;
 338
 339         /* dw3 */
 340         __le32 dma_rx_err_type;
 341 };
 342
 343 #define RX_DATA_LEN_UNDERFLOW_OFF       6
 344 #define RX_DATA_LEN_UNDERFLOW_MSK       (1 << RX_DATA_LEN_UNDERFLOW_OFF)
 345
 346 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
 347 #define HISI_SAS_MSI_COUNT_V3_HW 32
 348
 349 #define DIR_NO_DATA 0
 350 #define DIR_TO_INI 1
 351 #define DIR_TO_DEVICE 2
 352 #define DIR_RESERVED 3
 353
 354 #define CMD_IS_UNCONSTRAINT(cmd) \
 355         ((cmd == ATA_CMD_READ_LOG_EXT) || \
 356         (cmd == ATA_CMD_READ_LOG_DMA_EXT) || \
 357         (cmd == ATA_CMD_DEV_RESET))
 358
 359 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 360 {
 361         void __iomem *regs = hisi_hba->regs + off;
 362
 363         return readl(regs);
 364 }
 365
 366 static u32 hisi_sas_read32_relaxed(struct hisi_hba *hisi_hba, u32 off)
 367 {
 368         void __iomem *regs = hisi_hba->regs + off;
 369
 370         return readl_relaxed(regs);
 371 }
 372
 373 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
 374 {
 375         void __iomem *regs = hisi_hba->regs + off;
 376
 377         writel(val, regs);
 378 }
 379
 380 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
 381                                  u32 off, u32 val)
 382 {
 383         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 384
 385         writel(val, regs);
 386 }
 387
 388 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 389                                       int phy_no, u32 off)
 390 {
 391         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 392
 393         return readl(regs);
 394 }
 395
 396 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 397 {
 398         struct pci_dev *pdev = hisi_hba->pci_dev;
 399         int i;
 400
 401         /* Global registers init */
 402         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 403                          (u32)((1ULL << hisi_hba->queue_count) - 1));
 404         hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
 405         hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 406         hisi_sas_write32(hisi_hba, CFG_1US_TIMER_TRSH, 0xd);
 407         hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
 408         hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
 409         hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
 410         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
 411         hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 412         hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 413         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 414         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 415         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
 416         if (pdev->revision >= 0x21)
 417                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7fff);
 418         else
 419                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
 420         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 421         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 422         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
 423         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
 424         hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 425         hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 426         for (i = 0; i < hisi_hba->queue_count; i++)
 427                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 428
 429         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 430
 431         for (i = 0; i < hisi_hba->n_phy; i++) {
 432                 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE, 0x855);
 433                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 434                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 435                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 436                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 437                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 438                 if (pdev->revision >= 0x21)
 439                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 440                                         0xffffffff);
 441                 else
 442                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 443                                         0xff87ffff);
 444                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 445                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 446                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 447                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
 448                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
 449                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 450                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 451                 hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 452
 453                 /* used for 12G negotiate */
 454                 hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
 455         }
 456
 457         for (i = 0; i < hisi_hba->queue_count; i++) {
 458                 /* Delivery queue */
 459                 hisi_sas_write32(hisi_hba,
 460                                  DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
 461                                  upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
 462
 463                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
 464                                  lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
 465
 466                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
 467                                  HISI_SAS_QUEUE_SLOTS);
 468
 469                 /* Completion queue */
 470                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
 471                                  upper_32_bits(hisi_hba->complete_hdr_dma[i]));
 472
 473                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
 474                                  lower_32_bits(hisi_hba->complete_hdr_dma[i]));
 475
 476                 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
 477                                  HISI_SAS_QUEUE_SLOTS);
 478         }
 479
 480         /* itct */
 481         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
 482                          lower_32_bits(hisi_hba->itct_dma));
 483
 484         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
 485                          upper_32_bits(hisi_hba->itct_dma));
 486
 487         /* iost */
 488         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
 489                          lower_32_bits(hisi_hba->iost_dma));
 490
 491         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
 492                          upper_32_bits(hisi_hba->iost_dma));
 493
 494         /* breakpoint */
 495         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
 496                          lower_32_bits(hisi_hba->breakpoint_dma));
 497
 498         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
 499                          upper_32_bits(hisi_hba->breakpoint_dma));
 500
 501         /* SATA broken msg */
 502         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
 503                          lower_32_bits(hisi_hba->sata_breakpoint_dma));
 504
 505         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
 506                          upper_32_bits(hisi_hba->sata_breakpoint_dma));
 507
 508         /* SATA initial fis */
 509         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
 510                          lower_32_bits(hisi_hba->initial_fis_dma));
 511
 512         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
 513                          upper_32_bits(hisi_hba->initial_fis_dma));
 514
 515         /* RAS registers init */
 516         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 517         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
 518         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
 519         hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
 520 }
 521
 522 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 523 {
 524         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 525
 526         cfg &= ~PHY_CFG_DC_OPT_MSK;
 527         cfg |= 1 << PHY_CFG_DC_OPT_OFF;
 528         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 529 }
 530
 531 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 532 {
 533         struct sas_identify_frame identify_frame;
 534         u32 *identify_buffer;
 535
 536         memset(&identify_frame, 0, sizeof(identify_frame));
 537         identify_frame.dev_type = SAS_END_DEVICE;
 538         identify_frame.frame_type = 0;
 539         identify_frame._un1 = 1;
 540         identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
 541         identify_frame.target_bits = SAS_PROTOCOL_NONE;
 542         memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 543         memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 544         identify_frame.phy_id = phy_no;
 545         identify_buffer = (u32 *)(&identify_frame);
 546
 547         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 548                         __swab32(identify_buffer[0]));
 549         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
 550                         __swab32(identify_buffer[1]));
 551         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
 552                         __swab32(identify_buffer[2]));
 553         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
 554                         __swab32(identify_buffer[3]));
 555         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
 556                         __swab32(identify_buffer[4]));
 557         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 558                         __swab32(identify_buffer[5]));
 559 }
 560
 561 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 562                              struct hisi_sas_device *sas_dev)
 563 {
 564         struct domain_device *device = sas_dev->sas_device;
 565         struct device *dev = hisi_hba->dev;
 566         u64 qw0, device_id = sas_dev->device_id;
 567         struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
 568         struct domain_device *parent_dev = device->parent;
 569         struct asd_sas_port *sas_port = device->port;
 570         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 571
 572         memset(itct, 0, sizeof(*itct));
 573
 574         /* qw0 */
 575         qw0 = 0;
 576         switch (sas_dev->dev_type) {
 577         case SAS_END_DEVICE:
 578         case SAS_EDGE_EXPANDER_DEVICE:
 579         case SAS_FANOUT_EXPANDER_DEVICE:
 580                 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 581                 break;
 582         case SAS_SATA_DEV:
 583         case SAS_SATA_PENDING:
 584                 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
 585                         qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 586                 else
 587                         qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 588                 break;
 589         default:
 590                 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
 591                          sas_dev->dev_type);
 592         }
 593
 594         qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
 595                 (device->linkrate << ITCT_HDR_MCR_OFF) |
 596                 (1 << ITCT_HDR_VLN_OFF) |
 597                 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
 598                 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
 599                 (port->id << ITCT_HDR_PORT_ID_OFF));
 600         itct->qw0 = cpu_to_le64(qw0);
 601
 602         /* qw1 */
 603         memcpy(&itct->sas_addr, device->sas_addr, SAS_ADDR_SIZE);
 604         itct->sas_addr = __swab64(itct->sas_addr);
 605
 606         /* qw2 */
 607         if (!dev_is_sata(device))
 608                 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
 609                                         (0x1ULL << ITCT_HDR_RTOLT_OFF));
 610 }
 611
 612 static void clear_itct_v3_hw(struct hisi_hba *hisi_hba,
 613                               struct hisi_sas_device *sas_dev)
 614 {
 615         DECLARE_COMPLETION_ONSTACK(completion);
 616         u64 dev_id = sas_dev->device_id;
 617         struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 618         u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 619
 620         sas_dev->completion = &completion;
 621
 622         /* clear the itct interrupt state */
 623         if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
 624                 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 625                                  ENT_INT_SRC3_ITC_INT_MSK);
 626
 627         /* clear the itct table*/
 628         reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 629         hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
 630
 631         wait_for_completion(sas_dev->completion);
 632         memset(itct, 0, sizeof(struct hisi_sas_itct));
 633 }
 634
 635 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 636                                 struct domain_device *device)
 637 {
 638         struct hisi_sas_slot *slot, *slot2;
 639         struct hisi_sas_device *sas_dev = device->lldd_dev;
 640         u32 cfg_abt_set_query_iptt;
 641
 642         cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 643                 CFG_ABT_SET_QUERY_IPTT);
 644         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 645                 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 646                 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
 647                         (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
 648                 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 649                         cfg_abt_set_query_iptt);
 650         }
 651         cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 652         hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 653                 cfg_abt_set_query_iptt);
 654         hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
 655                                         1 << CFG_ABT_SET_IPTT_DONE_OFF);
 656 }
 657
 658 static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 659 {
 660         struct device *dev = hisi_hba->dev;
 661         int ret;
 662         u32 val;
 663
 664         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 665
 666         /* Disable all of the PHYs */
 667         hisi_sas_stop_phys(hisi_hba);
 668         udelay(50);
 669
 670         /* Ensure axi bus idle */
 671         ret = readl_poll_timeout(hisi_hba->regs + AXI_CFG, val, !val,
 672                         20000, 1000000);
 673         if (ret) {
 674                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 675                 return -EIO;
 676         }
 677
 678         if (ACPI_HANDLE(dev)) {
 679                 acpi_status s;
 680
 681                 s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
 682                 if (ACPI_FAILURE(s)) {
 683                         dev_err(dev, "Reset failed\n");
 684                         return -EIO;
 685                 }
 686         } else {
 687                 dev_err(dev, "no reset method!\n");
 688                 return -EINVAL;
 689         }
 690
 691         return 0;
 692 }
 693
 694 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 695 {
 696         struct device *dev = hisi_hba->dev;
 697         int rc;
 698
 699         rc = reset_hw_v3_hw(hisi_hba);
 700         if (rc) {
 701                 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
 702                 return rc;
 703         }
 704
 705         msleep(100);
 706         init_reg_v3_hw(hisi_hba);
 707
 708         return 0;
 709 }
 710
 711 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 712 {
 713         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 714
 715         cfg |= PHY_CFG_ENA_MSK;
 716         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 717 }
 718
 719 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 720 {
 721         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 722
 723         cfg &= ~PHY_CFG_ENA_MSK;
 724         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 725 }
 726
 727 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 728 {
 729         config_id_frame_v3_hw(hisi_hba, phy_no);
 730         config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
 731         enable_phy_v3_hw(hisi_hba, phy_no);
 732 }
 733
 734 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 735 {
 736         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 737         u32 txid_auto;
 738
 739         disable_phy_v3_hw(hisi_hba, phy_no);
 740         if (phy->identify.device_type == SAS_END_DEVICE) {
 741                 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 742                 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 743                                         txid_auto | TX_HARDRST_MSK);
 744         }
 745         msleep(100);
 746         start_phy_v3_hw(hisi_hba, phy_no);
 747 }
 748
 749 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 750 {
 751         return SAS_LINK_RATE_12_0_GBPS;
 752 }
 753
 754 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 755 {
 756         int i;
 757
 758         for (i = 0; i < hisi_hba->n_phy; i++) {
 759                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 760                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 761
 762                 if (!sas_phy->phy->enabled)
 763                         continue;
 764
 765                 start_phy_v3_hw(hisi_hba, i);
 766         }
 767 }
 768
 769 static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 770 {
 771         u32 sl_control;
 772
 773         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 774         sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
 775         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 776         msleep(1);
 777         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 778         sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
 779         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 780 }
 781
 782 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
 783 {
 784         int i, bitmap = 0;
 785         u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
 786         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
 787
 788         for (i = 0; i < hisi_hba->n_phy; i++)
 789                 if (phy_state & BIT(i))
 790                         if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
 791                                 bitmap |= BIT(i);
 792
 793         return bitmap;
 794 }
 795
 796 /**
 797  * The callpath to this function and upto writing the write
 798  * queue pointer should be safe from interruption.
 799  */
 800 static int
 801 get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 802 {
 803         struct device *dev = hisi_hba->dev;
 804         int queue = dq->id;
 805         u32 r, w;
 806
 807         w = dq->wr_point;
 808         r = hisi_sas_read32_relaxed(hisi_hba,
 809                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
 810         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
 811                 dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
 812                                 queue, r, w);
 813                 return -EAGAIN;
 814         }
 815
 816         return 0;
 817 }
 818
 819 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 820 {
 821         struct hisi_hba *hisi_hba = dq->hisi_hba;
 822         int dlvry_queue = dq->slot_prep->dlvry_queue;
 823         int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
 824
 825         dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
 826         hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
 827                          dq->wr_point);
 828 }
 829
 830 static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 831                               struct hisi_sas_slot *slot,
 832                               struct hisi_sas_cmd_hdr *hdr,
 833                               struct scatterlist *scatter,
 834                               int n_elem)
 835 {
 836         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
 837         struct device *dev = hisi_hba->dev;
 838         struct scatterlist *sg;
 839         int i;
 840
 841         if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
 842                 dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
 843                         n_elem);
 844                 return -EINVAL;
 845         }
 846
 847         for_each_sg(scatter, sg, n_elem, i) {
 848                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 849
 850                 entry->addr = cpu_to_le64(sg_dma_address(sg));
 851                 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
 852                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
 853                 entry->data_off = 0;
 854         }
 855
 856         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 857
 858         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
 859
 860         return 0;
 861 }
 862
 863 static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 864                           struct hisi_sas_slot *slot, int is_tmf,
 865                           struct hisi_sas_tmf_task *tmf)
 866 {
 867         struct sas_task *task = slot->task;
 868         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 869         struct domain_device *device = task->dev;
 870         struct hisi_sas_device *sas_dev = device->lldd_dev;
 871         struct hisi_sas_port *port = slot->port;
 872         struct sas_ssp_task *ssp_task = &task->ssp_task;
 873         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 874         int has_data = 0, rc, priority = is_tmf;
 875         u8 *buf_cmd;
 876         u32 dw1 = 0, dw2 = 0;
 877
 878         hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
 879                                (2 << CMD_HDR_TLR_CTRL_OFF) |
 880                                (port->id << CMD_HDR_PORT_OFF) |
 881                                (priority << CMD_HDR_PRIORITY_OFF) |
 882                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 883
 884         dw1 = 1 << CMD_HDR_VDTL_OFF;
 885         if (is_tmf) {
 886                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 887                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 888         } else {
 889                 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
 890                 switch (scsi_cmnd->sc_data_direction) {
 891                 case DMA_TO_DEVICE:
 892                         has_data = 1;
 893                         dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
 894                         break;
 895                 case DMA_FROM_DEVICE:
 896                         has_data = 1;
 897                         dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
 898                         break;
 899                 default:
 900                         dw1 &= ~CMD_HDR_DIR_MSK;
 901                 }
 902         }
 903
 904         /* map itct entry */
 905         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 906         hdr->dw1 = cpu_to_le32(dw1);
 907
 908         dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
 909               + 3) / 4) << CMD_HDR_CFL_OFF) |
 910               ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
 911               (2 << CMD_HDR_SG_MOD_OFF);
 912         hdr->dw2 = cpu_to_le32(dw2);
 913         hdr->transfer_tags = cpu_to_le32(slot->idx);
 914
 915         if (has_data) {
 916                 rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 917                                         slot->n_elem);
 918                 if (rc)
 919                         return rc;
 920         }
 921
 922         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 923         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
 924         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 925
 926         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
 927                 sizeof(struct ssp_frame_hdr);
 928
 929         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
 930         if (!is_tmf) {
 931                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
 932                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
 933         } else {
 934                 buf_cmd[10] = tmf->tmf;
 935                 switch (tmf->tmf) {
 936                 case TMF_ABORT_TASK:
 937                 case TMF_QUERY_TASK:
 938                         buf_cmd[12] =
 939                                 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
 940                         buf_cmd[13] =
 941                                 tmf->tag_of_task_to_be_managed & 0xff;
 942                         break;
 943                 default:
 944                         break;
 945                 }
 946         }
 947
 948         return 0;
 949 }
 950
 951 static int prep_smp_v3_hw(struct hisi_hba *hisi_hba,
 952                           struct hisi_sas_slot *slot)
 953 {
 954         struct sas_task *task = slot->task;
 955         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 956         struct domain_device *device = task->dev;
 957         struct device *dev = hisi_hba->dev;
 958         struct hisi_sas_port *port = slot->port;
 959         struct scatterlist *sg_req, *sg_resp;
 960         struct hisi_sas_device *sas_dev = device->lldd_dev;
 961         dma_addr_t req_dma_addr;
 962         unsigned int req_len, resp_len;
 963         int elem, rc;
 964
 965         /*
 966          * DMA-map SMP request, response buffers
 967          */
 968         /* req */
 969         sg_req = &task->smp_task.smp_req;
 970         elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
 971         if (!elem)
 972                 return -ENOMEM;
 973         req_len = sg_dma_len(sg_req);
 974         req_dma_addr = sg_dma_address(sg_req);
 975
 976         /* resp */
 977         sg_resp = &task->smp_task.smp_resp;
 978         elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
 979         if (!elem) {
 980                 rc = -ENOMEM;
 981                 goto err_out_req;
 982         }
 983         resp_len = sg_dma_len(sg_resp);
 984         if ((req_len & 0x3) || (resp_len & 0x3)) {
 985                 rc = -EINVAL;
 986                 goto err_out_resp;
 987         }
 988
 989         /* create header */
 990         /* dw0 */
 991         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
 992                                (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
 993                                (2 << CMD_HDR_CMD_OFF)); /* smp */
 994
 995         /* map itct entry */
 996         hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
 997                                (1 << CMD_HDR_FRAME_TYPE_OFF) |
 998                                (DIR_NO_DATA << CMD_HDR_DIR_OFF));
 999
1000         /* dw2 */
1001         hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1002                                (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1003                                CMD_HDR_MRFL_OFF));
1004
1005         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1006
1007         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1008         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1009
1010         return 0;
1011
1012 err_out_resp:
1013         dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
1014                      DMA_FROM_DEVICE);
1015 err_out_req:
1016         dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
1017                      DMA_TO_DEVICE);
1018         return rc;
1019 }
1020
1021 static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1022                           struct hisi_sas_slot *slot)
1023 {
1024         struct sas_task *task = slot->task;
1025         struct domain_device *device = task->dev;
1026         struct domain_device *parent_dev = device->parent;
1027         struct hisi_sas_device *sas_dev = device->lldd_dev;
1028         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1029         struct asd_sas_port *sas_port = device->port;
1030         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1031         u8 *buf_cmd;
1032         int has_data = 0, rc = 0, hdr_tag = 0;
1033         u32 dw1 = 0, dw2 = 0;
1034
1035         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1036         if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
1037                 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1038         else
1039                 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
1040
1041         switch (task->data_dir) {
1042         case DMA_TO_DEVICE:
1043                 has_data = 1;
1044                 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1045                 break;
1046         case DMA_FROM_DEVICE:
1047                 has_data = 1;
1048                 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1049                 break;
1050         default:
1051                 dw1 &= ~CMD_HDR_DIR_MSK;
1052         }
1053
1054         if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1055                         (task->ata_task.fis.control & ATA_SRST))
1056                 dw1 |= 1 << CMD_HDR_RESET_OFF;
1057
1058         dw1 |= (hisi_sas_get_ata_protocol(
1059                 &task->ata_task.fis, task->data_dir))
1060                 << CMD_HDR_FRAME_TYPE_OFF;
1061         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1062
1063         if (CMD_IS_UNCONSTRAINT(task->ata_task.fis.command))
1064                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1065
1066         hdr->dw1 = cpu_to_le32(dw1);
1067
1068         /* dw2 */
1069         if (task->ata_task.use_ncq && hisi_sas_get_ncq_tag(task, &hdr_tag)) {
1070                 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1071                 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1072         }
1073
1074         dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1075                         2 << CMD_HDR_SG_MOD_OFF;
1076         hdr->dw2 = cpu_to_le32(dw2);
1077
1078         /* dw3 */
1079         hdr->transfer_tags = cpu_to_le32(slot->idx);
1080
1081         if (has_data) {
1082                 rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1083                                         slot->n_elem);
1084                 if (rc)
1085                         return rc;
1086         }
1087
1088         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1089         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1090         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1091
1092         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1093
1094         if (likely(!task->ata_task.device_control_reg_update))
1095                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1096         /* fill in command FIS */
1097         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1098
1099         return 0;
1100 }
1101
1102 static int prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1103                 struct hisi_sas_slot *slot,
1104                 int device_id, int abort_flag, int tag_to_abort)
1105 {
1106         struct sas_task *task = slot->task;
1107         struct domain_device *dev = task->dev;
1108         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1109         struct hisi_sas_port *port = slot->port;
1110
1111         /* dw0 */
1112         hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/
1113                                (port->id << CMD_HDR_PORT_OFF) |
1114                                    (dev_is_sata(dev)
1115                                         << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1116                                         (abort_flag
1117                                          << CMD_HDR_ABORT_FLAG_OFF));
1118
1119         /* dw1 */
1120         hdr->dw1 = cpu_to_le32(device_id
1121                         << CMD_HDR_DEV_ID_OFF);
1122
1123         /* dw7 */
1124         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
1125         hdr->transfer_tags = cpu_to_le32(slot->idx);
1126
1127         return 0;
1128 }
1129
1130 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1131 {
1132         int i, res;
1133         u32 context, port_id, link_rate;
1134         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1135         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1136         struct device *dev = hisi_hba->dev;
1137
1138         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1139
1140         port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1141         port_id = (port_id >> (4 * phy_no)) & 0xf;
1142         link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1143         link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1144
1145         if (port_id == 0xf) {
1146                 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1147                 res = IRQ_NONE;
1148                 goto end;
1149         }
1150         sas_phy->linkrate = link_rate;
1151         phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1152
1153         /* Check for SATA dev */
1154         context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1155         if (context & (1 << phy_no)) {
1156                 struct hisi_sas_initial_fis *initial_fis;
1157                 struct dev_to_host_fis *fis;
1158                 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1159
1160                 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1161                 initial_fis = &hisi_hba->initial_fis[phy_no];
1162                 fis = &initial_fis->fis;
1163                 sas_phy->oob_mode = SATA_OOB_MODE;
1164                 attached_sas_addr[0] = 0x50;
1165                 attached_sas_addr[7] = phy_no;
1166                 memcpy(sas_phy->attached_sas_addr,
1167                        attached_sas_addr,
1168                        SAS_ADDR_SIZE);
1169                 memcpy(sas_phy->frame_rcvd, fis,
1170                        sizeof(struct dev_to_host_fis));
1171                 phy->phy_type |= PORT_TYPE_SATA;
1172                 phy->identify.device_type = SAS_SATA_DEV;
1173                 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1174                 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1175         } else {
1176                 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1177                 struct sas_identify_frame *id =
1178                         (struct sas_identify_frame *)frame_rcvd;
1179
1180                 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1181                 for (i = 0; i < 6; i++) {
1182                         u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1183                                                RX_IDAF_DWORD0 + (i * 4));
1184                         frame_rcvd[i] = __swab32(idaf);
1185                 }
1186                 sas_phy->oob_mode = SAS_OOB_MODE;
1187                 memcpy(sas_phy->attached_sas_addr,
1188                        &id->sas_addr,
1189                        SAS_ADDR_SIZE);
1190                 phy->phy_type |= PORT_TYPE_SAS;
1191                 phy->identify.device_type = id->dev_type;
1192                 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1193                 if (phy->identify.device_type == SAS_END_DEVICE)
1194                         phy->identify.target_port_protocols =
1195                                 SAS_PROTOCOL_SSP;
1196                 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1197                         phy->identify.target_port_protocols =
1198                                 SAS_PROTOCOL_SMP;
1199         }
1200
1201         phy->port_id = port_id;
1202         phy->phy_attached = 1;
1203         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
1204         res = IRQ_HANDLED;
1205 end:
1206         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1207                              CHL_INT0_SL_PHY_ENABLE_MSK);
1208         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1209
1210         return res;
1211 }
1212
1213 static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1214 {
1215         u32 phy_state, sl_ctrl, txid_auto;
1216         struct device *dev = hisi_hba->dev;
1217
1218         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1219
1220         phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1221         dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1222         hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
1223
1224         sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1225         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1226                                                 sl_ctrl&(~SL_CTA_MSK));
1227
1228         txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1229         hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1230                                                 txid_auto | CT3_MSK);
1231
1232         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1233         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1234
1235         return IRQ_HANDLED;
1236 }
1237
1238 static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1239 {
1240         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1241         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1242         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1243
1244         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1245         sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
1246         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1247                              CHL_INT0_SL_RX_BCST_ACK_MSK);
1248         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1249
1250         return IRQ_HANDLED;
1251 }
1252
1253 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1254 {
1255         struct hisi_hba *hisi_hba = p;
1256         u32 irq_msk;
1257         int phy_no = 0;
1258         irqreturn_t res = IRQ_NONE;
1259
1260         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1261                                 & 0x11111111;
1262         while (irq_msk) {
1263                 if (irq_msk  & 1) {
1264                         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1265                                                             CHL_INT0);
1266                         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1267                         int rdy = phy_state & (1 << phy_no);
1268
1269                         if (rdy) {
1270                                 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1271                                         /* phy up */
1272                                         if (phy_up_v3_hw(phy_no, hisi_hba)
1273                                                         == IRQ_HANDLED)
1274                                                 res = IRQ_HANDLED;
1275                                 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1276                                         /* phy bcast */
1277                                         if (phy_bcast_v3_hw(phy_no, hisi_hba)
1278                                                         == IRQ_HANDLED)
1279                                                 res = IRQ_HANDLED;
1280                         } else {
1281                                 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1282                                         /* phy down */
1283                                         if (phy_down_v3_hw(phy_no, hisi_hba)
1284                                                         == IRQ_HANDLED)
1285                                                 res = IRQ_HANDLED;
1286                         }
1287                 }
1288                 irq_msk >>= 4;
1289                 phy_no++;
1290         }
1291
1292         return res;
1293 }
1294
1295 static const struct hisi_sas_hw_error port_axi_error[] = {
1296         {
1297                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1298                 .msg = "dma_tx_axi_wr_err",
1299         },
1300         {
1301                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1302                 .msg = "dma_tx_axi_rd_err",
1303         },
1304         {
1305                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1306                 .msg = "dma_rx_axi_wr_err",
1307         },
1308         {
1309                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1310                 .msg = "dma_rx_axi_rd_err",
1311         },
1312 };
1313
1314 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1315 {
1316         struct hisi_hba *hisi_hba = p;
1317         struct device *dev = hisi_hba->dev;
1318         u32 ent_msk, ent_tmp, irq_msk;
1319         int phy_no = 0;
1320
1321         ent_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1322         ent_tmp = ent_msk;
1323         ent_msk |= ENT_INT_SRC_MSK3_ENT95_MSK_MSK;
1324         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_msk);
1325
1326         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1327                                 & 0xeeeeeeee;
1328
1329         while (irq_msk) {
1330                 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no,
1331                                                      CHL_INT0);
1332                 u32 irq_value1 = hisi_sas_phy_read32(hisi_hba, phy_no,
1333                                                      CHL_INT1);
1334                 u32 irq_value2 = hisi_sas_phy_read32(hisi_hba, phy_no,
1335                                                      CHL_INT2);
1336                 u32 irq_msk1 = hisi_sas_phy_read32(hisi_hba, phy_no,
1337                                                         CHL_INT1_MSK);
1338                 u32 irq_msk2 = hisi_sas_phy_read32(hisi_hba, phy_no,
1339                                                         CHL_INT2_MSK);
1340
1341                 irq_value1 &= ~irq_msk1;
1342                 irq_value2 &= ~irq_msk2;
1343
1344                 if ((irq_msk & (4 << (phy_no * 4))) &&
1345                                                 irq_value1) {
1346                         int i;
1347
1348                         for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1349                                 const struct hisi_sas_hw_error *error =
1350                                                 &port_axi_error[i];
1351
1352                                 if (!(irq_value1 & error->irq_msk))
1353                                         continue;
1354
1355                                 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1356                                         error->msg, phy_no, irq_value1);
1357                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1358                         }
1359
1360                         hisi_sas_phy_write32(hisi_hba, phy_no,
1361                                              CHL_INT1, irq_value1);
1362                 }
1363
1364                 if (irq_msk & (8 << (phy_no * 4)) && irq_value2) {
1365                         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1366
1367                         if (irq_value2 & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1368                                 dev_warn(dev, "phy%d identify timeout\n",
1369                                                         phy_no);
1370                                 hisi_sas_notify_phy_event(phy,
1371                                         HISI_PHYE_LINK_RESET);
1372
1373                         }
1374
1375                         if (irq_value2 & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1376                                 u32 reg_value = hisi_sas_phy_read32(hisi_hba,
1377                                                 phy_no, STP_LINK_TIMEOUT_STATE);
1378
1379                                 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1380                                                         phy_no, reg_value);
1381                                 if (reg_value & BIT(4))
1382                                         hisi_sas_notify_phy_event(phy,
1383                                                 HISI_PHYE_LINK_RESET);
1384                         }
1385
1386                         hisi_sas_phy_write32(hisi_hba, phy_no,
1387                                              CHL_INT2, irq_value2);
1388                 }
1389
1390
1391                 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
1392                         hisi_sas_phy_write32(hisi_hba, phy_no,
1393                                         CHL_INT0, irq_value0
1394                                         & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1395                                         & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1396                                         & (~CHL_INT0_NOT_RDY_MSK));
1397                 }
1398                 irq_msk &= ~(0xe << (phy_no * 4));
1399                 phy_no++;
1400         }
1401
1402         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_tmp);
1403
1404         return IRQ_HANDLED;
1405 }
1406
1407 static const struct hisi_sas_hw_error axi_error[] = {
1408         { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
1409         { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
1410         { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
1411         { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
1412         { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
1413         { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
1414         { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
1415         { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
1416         {},
1417 };
1418
1419 static const struct hisi_sas_hw_error fifo_error[] = {
1420         { .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
1421         { .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
1422         { .msk = BIT(10), .msg = "GETDQE_FIFO" },
1423         { .msk = BIT(11), .msg = "CMDP_FIFO" },
1424         { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
1425         {},
1426 };
1427
1428 static const struct hisi_sas_hw_error fatal_axi_error[] = {
1429         {
1430                 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
1431                 .msg = "write pointer and depth",
1432         },
1433         {
1434                 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
1435                 .msg = "iptt no match slot",
1436         },
1437         {
1438                 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
1439                 .msg = "read pointer and depth",
1440         },
1441         {
1442                 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
1443                 .reg = HGC_AXI_FIFO_ERR_INFO,
1444                 .sub = axi_error,
1445         },
1446         {
1447                 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
1448                 .reg = HGC_AXI_FIFO_ERR_INFO,
1449                 .sub = fifo_error,
1450         },
1451         {
1452                 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
1453                 .msg = "LM add/fetch list",
1454         },
1455         {
1456                 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
1457                 .msg = "SAS_HGC_ABT fetch LM list",
1458         },
1459 };
1460
1461 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
1462 {
1463         u32 irq_value, irq_msk;
1464         struct hisi_hba *hisi_hba = p;
1465         struct device *dev = hisi_hba->dev;
1466         int i;
1467
1468         irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1469         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
1470
1471         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
1472         irq_value &= ~irq_msk;
1473
1474         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
1475                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
1476
1477                 if (!(irq_value & error->irq_msk))
1478                         continue;
1479
1480                 if (error->sub) {
1481                         const struct hisi_sas_hw_error *sub = error->sub;
1482                         u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
1483
1484                         for (; sub->msk || sub->msg; sub++) {
1485                                 if (!(err_value & sub->msk))
1486                                         continue;
1487
1488                                 dev_err(dev, "%s error (0x%x) found!\n",
1489                                         sub->msg, irq_value);
1490                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1491                         }
1492                 } else {
1493                         dev_err(dev, "%s error (0x%x) found!\n",
1494                                 error->msg, irq_value);
1495                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1496                 }
1497         }
1498
1499         if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
1500                 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
1501                 u32 dev_id = reg_val & ITCT_DEV_MSK;
1502                 struct hisi_sas_device *sas_dev =
1503                                 &hisi_hba->devices[dev_id];
1504
1505                 hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
1506                 dev_dbg(dev, "clear ITCT ok\n");
1507                 complete(sas_dev->completion);
1508         }
1509
1510         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
1511         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
1512
1513         return IRQ_HANDLED;
1514 }
1515
1516 static void
1517 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
1518                struct hisi_sas_slot *slot)
1519 {
1520         struct task_status_struct *ts = &task->task_status;
1521         struct hisi_sas_complete_v3_hdr *complete_queue =
1522                         hisi_hba->complete_hdr[slot->cmplt_queue];
1523         struct hisi_sas_complete_v3_hdr *complete_hdr =
1524                         &complete_queue[slot->cmplt_queue_slot];
1525         struct hisi_sas_err_record_v3 *record =
1526                         hisi_sas_status_buf_addr_mem(slot);
1527         u32 dma_rx_err_type = record->dma_rx_err_type;
1528         u32 trans_tx_fail_type = record->trans_tx_fail_type;
1529
1530         switch (task->task_proto) {
1531         case SAS_PROTOCOL_SSP:
1532                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1533                         ts->residual = trans_tx_fail_type;
1534                         ts->stat = SAS_DATA_UNDERRUN;
1535                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1536                         ts->stat = SAS_QUEUE_FULL;
1537                         slot->abort = 1;
1538                 } else {
1539                         ts->stat = SAS_OPEN_REJECT;
1540                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1541                 }
1542                 break;
1543         case SAS_PROTOCOL_SATA:
1544         case SAS_PROTOCOL_STP:
1545         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1546                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1547                         ts->residual = trans_tx_fail_type;
1548                         ts->stat = SAS_DATA_UNDERRUN;
1549                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1550                         ts->stat = SAS_PHY_DOWN;
1551                         slot->abort = 1;
1552                 } else {
1553                         ts->stat = SAS_OPEN_REJECT;
1554                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1555                 }
1556                 hisi_sas_sata_done(task, slot);
1557                 break;
1558         case SAS_PROTOCOL_SMP:
1559                 ts->stat = SAM_STAT_CHECK_CONDITION;
1560                 break;
1561         default:
1562                 break;
1563         }
1564 }
1565
1566 static int
1567 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
1568 {
1569         struct sas_task *task = slot->task;
1570         struct hisi_sas_device *sas_dev;
1571         struct device *dev = hisi_hba->dev;
1572         struct task_status_struct *ts;
1573         struct domain_device *device;
1574         struct sas_ha_struct *ha;
1575         enum exec_status sts;
1576         struct hisi_sas_complete_v3_hdr *complete_queue =
1577                         hisi_hba->complete_hdr[slot->cmplt_queue];
1578         struct hisi_sas_complete_v3_hdr *complete_hdr =
1579                         &complete_queue[slot->cmplt_queue_slot];
1580         unsigned long flags;
1581         bool is_internal = slot->is_internal;
1582
1583         if (unlikely(!task || !task->lldd_task || !task->dev))
1584                 return -EINVAL;
1585
1586         ts = &task->task_status;
1587         device = task->dev;
1588         ha = device->port->ha;
1589         sas_dev = device->lldd_dev;
1590
1591         spin_lock_irqsave(&task->task_state_lock, flags);
1592         task->task_state_flags &=
1593                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
1594         spin_unlock_irqrestore(&task->task_state_lock, flags);
1595
1596         memset(ts, 0, sizeof(*ts));
1597         ts->resp = SAS_TASK_COMPLETE;
1598
1599         if (unlikely(!sas_dev)) {
1600                 dev_dbg(dev, "slot complete: port has not device\n");
1601                 ts->stat = SAS_PHY_DOWN;
1602                 goto out;
1603         }
1604
1605         /*
1606          * Use SAS+TMF status codes
1607          */
1608         switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK)
1609                         >> CMPLT_HDR_ABORT_STAT_OFF) {
1610         case STAT_IO_ABORTED:
1611                 /* this IO has been aborted by abort command */
1612                 ts->stat = SAS_ABORTED_TASK;
1613                 goto out;
1614         case STAT_IO_COMPLETE:
1615                 /* internal abort command complete */
1616                 ts->stat = TMF_RESP_FUNC_SUCC;
1617                 goto out;
1618         case STAT_IO_NO_DEVICE:
1619                 ts->stat = TMF_RESP_FUNC_COMPLETE;
1620                 goto out;
1621         case STAT_IO_NOT_VALID:
1622                 /*
1623                  * abort single IO, the controller can't find the IO
1624                  */
1625                 ts->stat = TMF_RESP_FUNC_FAILED;
1626                 goto out;
1627         default:
1628                 break;
1629         }
1630
1631         /* check for erroneous completion */
1632         if ((complete_hdr->dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
1633                 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
1634
1635                 slot_err_v3_hw(hisi_hba, task, slot);
1636                 if (ts->stat != SAS_DATA_UNDERRUN)
1637                         dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
1638                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
1639                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
1640                                 slot->idx, task, sas_dev->device_id,
1641                                 complete_hdr->dw0, complete_hdr->dw1,
1642                                 complete_hdr->act, complete_hdr->dw3,
1643                                 error_info[0], error_info[1],
1644                                 error_info[2], error_info[3]);
1645                 if (unlikely(slot->abort))
1646                         return ts->stat;
1647                 goto out;
1648         }
1649
1650         switch (task->task_proto) {
1651         case SAS_PROTOCOL_SSP: {
1652                 struct ssp_response_iu *iu =
1653                         hisi_sas_status_buf_addr_mem(slot) +
1654                         sizeof(struct hisi_sas_err_record);
1655
1656                 sas_ssp_task_response(dev, task, iu);
1657                 break;
1658         }
1659         case SAS_PROTOCOL_SMP: {
1660                 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
1661                 void *to;
1662
1663                 ts->stat = SAM_STAT_GOOD;
1664                 to = kmap_atomic(sg_page(sg_resp));
1665
1666                 dma_unmap_sg(dev, &task->smp_task.smp_resp, 1,
1667                              DMA_FROM_DEVICE);
1668                 dma_unmap_sg(dev, &task->smp_task.smp_req, 1,
1669                              DMA_TO_DEVICE);
1670                 memcpy(to + sg_resp->offset,
1671                         hisi_sas_status_buf_addr_mem(slot) +
1672                        sizeof(struct hisi_sas_err_record),
1673                        sg_dma_len(sg_resp));
1674                 kunmap_atomic(to);
1675                 break;
1676         }
1677         case SAS_PROTOCOL_SATA:
1678         case SAS_PROTOCOL_STP:
1679         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1680                 ts->stat = SAM_STAT_GOOD;
1681                 hisi_sas_sata_done(task, slot);
1682                 break;
1683         default:
1684                 ts->stat = SAM_STAT_CHECK_CONDITION;
1685                 break;
1686         }
1687
1688         if (!slot->port->port_attached) {
1689                 dev_warn(dev, "slot complete: port %d has removed\n",
1690                         slot->port->sas_port.id);
1691                 ts->stat = SAS_PHY_DOWN;
1692         }
1693
1694 out:
1695         hisi_sas_slot_task_free(hisi_hba, task, slot);
1696         sts = ts->stat;
1697         spin_lock_irqsave(&task->task_state_lock, flags);
1698         if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
1699                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1700                 dev_info(dev, "slot complete: task(%p) aborted\n", task);
1701                 return SAS_ABORTED_TASK;
1702         }
1703         task->task_state_flags |= SAS_TASK_STATE_DONE;
1704         spin_unlock_irqrestore(&task->task_state_lock, flags);
1705
1706         if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
1707                 spin_lock_irqsave(&device->done_lock, flags);
1708                 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
1709                         spin_unlock_irqrestore(&device->done_lock, flags);
1710                         dev_info(dev, "slot complete: task(%p) ignored\n ",
1711                                  task);
1712                         return sts;
1713                 }
1714                 spin_unlock_irqrestore(&device->done_lock, flags);
1715         }
1716
1717         if (task->task_done)
1718                 task->task_done(task);
1719
1720         return sts;
1721 }
1722
1723 static void cq_tasklet_v3_hw(unsigned long val)
1724 {
1725         struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val;
1726         struct hisi_hba *hisi_hba = cq->hisi_hba;
1727         struct hisi_sas_slot *slot;
1728         struct hisi_sas_complete_v3_hdr *complete_queue;
1729         u32 rd_point = cq->rd_point, wr_point;
1730         int queue = cq->id;
1731
1732         complete_queue = hisi_hba->complete_hdr[queue];
1733
1734         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
1735                                    (0x14 * queue));
1736
1737         while (rd_point != wr_point) {
1738                 struct hisi_sas_complete_v3_hdr *complete_hdr;
1739                 struct device *dev = hisi_hba->dev;
1740                 int iptt;
1741
1742                 complete_hdr = &complete_queue[rd_point];
1743
1744                 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
1745                 if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
1746                         slot = &hisi_hba->slot_info[iptt];
1747                         slot->cmplt_queue_slot = rd_point;
1748                         slot->cmplt_queue = queue;
1749                         slot_complete_v3_hw(hisi_hba, slot);
1750                 } else
1751                         dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
1752
1753                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
1754                         rd_point = 0;
1755         }
1756
1757         /* update rd_point */
1758         cq->rd_point = rd_point;
1759         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
1760 }
1761
1762 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
1763 {
1764         struct hisi_sas_cq *cq = p;
1765         struct hisi_hba *hisi_hba = cq->hisi_hba;
1766         int queue = cq->id;
1767
1768         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
1769
1770         tasklet_schedule(&cq->tasklet);
1771
1772         return IRQ_HANDLED;
1773 }
1774
1775 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
1776 {
1777         struct device *dev = hisi_hba->dev;
1778         struct pci_dev *pdev = hisi_hba->pci_dev;
1779         int vectors, rc;
1780         int i, k;
1781         int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
1782
1783         vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1,
1784                                         max_msi, PCI_IRQ_MSI);
1785         if (vectors < max_msi) {
1786                 dev_err(dev, "could not allocate all msi (%d)\n", vectors);
1787                 return -ENOENT;
1788         }
1789
1790         rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
1791                               int_phy_up_down_bcast_v3_hw, 0,
1792                               DRV_NAME " phy", hisi_hba);
1793         if (rc) {
1794                 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
1795                 rc = -ENOENT;
1796                 goto free_irq_vectors;
1797         }
1798
1799         rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
1800                               int_chnl_int_v3_hw, 0,
1801                               DRV_NAME " channel", hisi_hba);
1802         if (rc) {
1803                 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
1804                 rc = -ENOENT;
1805                 goto free_phy_irq;
1806         }
1807
1808         rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
1809                               fatal_axi_int_v3_hw, 0,
1810                               DRV_NAME " fatal", hisi_hba);
1811         if (rc) {
1812                 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
1813                 rc = -ENOENT;
1814                 goto free_chnl_interrupt;
1815         }
1816
1817         /* Init tasklets for cq only */
1818         for (i = 0; i < hisi_hba->queue_count; i++) {
1819                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
1820                 struct tasklet_struct *t = &cq->tasklet;
1821
1822                 rc = devm_request_irq(dev, pci_irq_vector(pdev, i+16),
1823                                           cq_interrupt_v3_hw, 0,
1824                                           DRV_NAME " cq", cq);
1825                 if (rc) {
1826                         dev_err(dev,
1827                                 "could not request cq%d interrupt, rc=%d\n",
1828                                 i, rc);
1829                         rc = -ENOENT;
1830                         goto free_cq_irqs;
1831                 }
1832
1833                 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq);
1834         }
1835
1836         return 0;
1837
1838 free_cq_irqs:
1839         for (k = 0; k < i; k++) {
1840                 struct hisi_sas_cq *cq = &hisi_hba->cq[k];
1841
1842                 free_irq(pci_irq_vector(pdev, k+16), cq);
1843         }
1844         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
1845 free_chnl_interrupt:
1846         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
1847 free_phy_irq:
1848         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
1849 free_irq_vectors:
1850         pci_free_irq_vectors(pdev);
1851         return rc;
1852 }
1853
1854 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
1855 {
1856         int rc;
1857
1858         rc = hw_init_v3_hw(hisi_hba);
1859         if (rc)
1860                 return rc;
1861
1862         rc = interrupt_init_v3_hw(hisi_hba);
1863         if (rc)
1864                 return rc;
1865
1866         return 0;
1867 }
1868
1869 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
1870                 struct sas_phy_linkrates *r)
1871 {
1872         u32 prog_phy_link_rate =
1873                 hisi_sas_phy_read32(hisi_hba, phy_no, PROG_PHY_LINK_RATE);
1874         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1875         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1876         int i;
1877         enum sas_linkrate min, max;
1878         u32 rate_mask = 0;
1879
1880         if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
1881                 max = sas_phy->phy->maximum_linkrate;
1882                 min = r->minimum_linkrate;
1883         } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
1884                 max = r->maximum_linkrate;
1885                 min = sas_phy->phy->minimum_linkrate;
1886         } else
1887                 return;
1888
1889         sas_phy->phy->maximum_linkrate = max;
1890         sas_phy->phy->minimum_linkrate = min;
1891
1892         max -= SAS_LINK_RATE_1_5_GBPS;
1893
1894         for (i = 0; i <= max; i++)
1895                 rate_mask |= 1 << (i * 2);
1896
1897         prog_phy_link_rate &= ~0xff;
1898         prog_phy_link_rate |= rate_mask;
1899
1900         disable_phy_v3_hw(hisi_hba, phy_no);
1901         msleep(100);
1902         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
1903                         prog_phy_link_rate);
1904         start_phy_v3_hw(hisi_hba, phy_no);
1905 }
1906
1907 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
1908 {
1909         struct pci_dev *pdev = hisi_hba->pci_dev;
1910         int i;
1911
1912         synchronize_irq(pci_irq_vector(pdev, 1));
1913         synchronize_irq(pci_irq_vector(pdev, 2));
1914         synchronize_irq(pci_irq_vector(pdev, 11));
1915         for (i = 0; i < hisi_hba->queue_count; i++) {
1916                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
1917                 synchronize_irq(pci_irq_vector(pdev, i + 16));
1918         }
1919
1920         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
1921         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
1922         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
1923         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
1924
1925         for (i = 0; i < hisi_hba->n_phy; i++) {
1926                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
1927                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
1928                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
1929                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
1930                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
1931         }
1932 }
1933
1934 static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
1935 {
1936         return hisi_sas_read32(hisi_hba, PHY_STATE);
1937 }
1938
1939 static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1940 {
1941         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1942         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1943         struct sas_phy *sphy = sas_phy->phy;
1944         u32 reg_value;
1945
1946         /* loss dword sync */
1947         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1948         sphy->loss_of_dword_sync_count += reg_value;
1949
1950         /* phy reset problem */
1951         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1952         sphy->phy_reset_problem_count += reg_value;
1953
1954         /* invalid dword */
1955         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1956         sphy->invalid_dword_count += reg_value;
1957
1958         /* disparity err */
1959         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1960         sphy->running_disparity_error_count += reg_value;
1961
1962 }
1963
1964 static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
1965 {
1966         struct device *dev = hisi_hba->dev;
1967         int rc;
1968         u32 status;
1969
1970         interrupt_disable_v3_hw(hisi_hba);
1971         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
1972         hisi_sas_kill_tasklets(hisi_hba);
1973
1974         hisi_sas_stop_phys(hisi_hba);
1975
1976         mdelay(10);
1977
1978         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + AM_CTRL_GLOBAL, 0x1);
1979
1980         /* wait until bus idle */
1981         rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
1982                 AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
1983         if (rc) {
1984                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
1985                 return rc;
1986         }
1987
1988         hisi_sas_init_mem(hisi_hba);
1989
1990         return hw_init_v3_hw(hisi_hba);
1991 }
1992
1993 static const struct hisi_sas_hw hisi_sas_v3_hw = {
1994         .hw_init = hisi_sas_v3_init,
1995         .setup_itct = setup_itct_v3_hw,
1996         .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V3_HW,
1997         .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
1998         .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
1999         .clear_itct = clear_itct_v3_hw,
2000         .sl_notify = sl_notify_v3_hw,
2001         .prep_ssp = prep_ssp_v3_hw,
2002         .prep_smp = prep_smp_v3_hw,
2003         .prep_stp = prep_ata_v3_hw,
2004         .prep_abort = prep_abort_v3_hw,
2005         .get_free_slot = get_free_slot_v3_hw,
2006         .start_delivery = start_delivery_v3_hw,
2007         .slot_complete = slot_complete_v3_hw,
2008         .phys_init = phys_init_v3_hw,
2009         .phy_start = start_phy_v3_hw,
2010         .phy_disable = disable_phy_v3_hw,
2011         .phy_hard_reset = phy_hard_reset_v3_hw,
2012         .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
2013         .phy_set_linkrate = phy_set_linkrate_v3_hw,
2014         .dereg_device = dereg_device_v3_hw,
2015         .soft_reset = soft_reset_v3_hw,
2016         .get_phys_state = get_phys_state_v3_hw,
2017         .get_events = phy_get_events_v3_hw,
2018 };
2019
2020 static struct Scsi_Host *
2021 hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
2022 {
2023         struct Scsi_Host *shost;
2024         struct hisi_hba *hisi_hba;
2025         struct device *dev = &pdev->dev;
2026
2027         shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
2028         if (!shost) {
2029                 dev_err(dev, "shost alloc failed\n");
2030                 return NULL;
2031         }
2032         hisi_hba = shost_priv(shost);
2033
2034         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
2035         hisi_hba->hw = &hisi_sas_v3_hw;
2036         hisi_hba->pci_dev = pdev;
2037         hisi_hba->dev = dev;
2038         hisi_hba->shost = shost;
2039         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
2040
2041         timer_setup(&hisi_hba->timer, NULL, 0);
2042
2043         if (hisi_sas_get_fw_info(hisi_hba) < 0)
2044                 goto err_out;
2045
2046         if (hisi_sas_alloc(hisi_hba, shost)) {
2047                 hisi_sas_free(hisi_hba);
2048                 goto err_out;
2049         }
2050
2051         return shost;
2052 err_out:
2053         scsi_host_put(shost);
2054         dev_err(dev, "shost alloc failed\n");
2055         return NULL;
2056 }
2057
2058 static int
2059 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2060 {
2061         struct Scsi_Host *shost;
2062         struct hisi_hba *hisi_hba;
2063         struct device *dev = &pdev->dev;
2064         struct asd_sas_phy **arr_phy;
2065         struct asd_sas_port **arr_port;
2066         struct sas_ha_struct *sha;
2067         int rc, phy_nr, port_nr, i;
2068
2069         rc = pci_enable_device(pdev);
2070         if (rc)
2071                 goto err_out;
2072
2073         pci_set_master(pdev);
2074
2075         rc = pci_request_regions(pdev, DRV_NAME);
2076         if (rc)
2077                 goto err_out_disable_device;
2078
2079         if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) ||
2080             (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0)) {
2081                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) ||
2082                    (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0)) {
2083                         dev_err(dev, "No usable DMA addressing method\n");
2084                         rc = -EIO;
2085                         goto err_out_regions;
2086                 }
2087         }
2088
2089         shost = hisi_sas_shost_alloc_pci(pdev);
2090         if (!shost) {
2091                 rc = -ENOMEM;
2092                 goto err_out_regions;
2093         }
2094
2095         sha = SHOST_TO_SAS_HA(shost);
2096         hisi_hba = shost_priv(shost);
2097         dev_set_drvdata(dev, sha);
2098
2099         hisi_hba->regs = pcim_iomap(pdev, 5, 0);
2100         if (!hisi_hba->regs) {
2101                 dev_err(dev, "cannot map register.\n");
2102                 rc = -ENOMEM;
2103                 goto err_out_ha;
2104         }
2105
2106         phy_nr = port_nr = hisi_hba->n_phy;
2107
2108         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
2109         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
2110         if (!arr_phy || !arr_port) {
2111                 rc = -ENOMEM;
2112                 goto err_out_ha;
2113         }
2114
2115         sha->sas_phy = arr_phy;
2116         sha->sas_port = arr_port;
2117         sha->core.shost = shost;
2118         sha->lldd_ha = hisi_hba;
2119
2120         shost->transportt = hisi_sas_stt;
2121         shost->max_id = HISI_SAS_MAX_DEVICES;
2122         shost->max_lun = ~0;
2123         shost->max_channel = 1;
2124         shost->max_cmd_len = 16;
2125         shost->sg_tablesize = min_t(u16, SG_ALL, HISI_SAS_SGE_PAGE_CNT);
2126         shost->can_queue = hisi_hba->hw->max_command_entries;
2127         shost->cmd_per_lun = hisi_hba->hw->max_command_entries;
2128
2129         sha->sas_ha_name = DRV_NAME;
2130         sha->dev = dev;
2131         sha->lldd_module = THIS_MODULE;
2132         sha->sas_addr = &hisi_hba->sas_addr[0];
2133         sha->num_phys = hisi_hba->n_phy;
2134         sha->core.shost = hisi_hba->shost;
2135
2136         for (i = 0; i < hisi_hba->n_phy; i++) {
2137                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
2138                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
2139         }
2140
2141         rc = scsi_add_host(shost, dev);
2142         if (rc)
2143                 goto err_out_ha;
2144
2145         rc = sas_register_ha(sha);
2146         if (rc)
2147                 goto err_out_register_ha;
2148
2149         rc = hisi_hba->hw->hw_init(hisi_hba);
2150         if (rc)
2151                 goto err_out_register_ha;
2152
2153         scsi_scan_host(shost);
2154
2155         return 0;
2156
2157 err_out_register_ha:
2158         scsi_remove_host(shost);
2159 err_out_ha:
2160         scsi_host_put(shost);
2161 err_out_regions:
2162         pci_release_regions(pdev);
2163 err_out_disable_device:
2164         pci_disable_device(pdev);
2165 err_out:
2166         return rc;
2167 }
2168
2169 static void
2170 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
2171 {
2172         int i;
2173
2174         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
2175         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
2176         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
2177         for (i = 0; i < hisi_hba->queue_count; i++) {
2178                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2179
2180                 free_irq(pci_irq_vector(pdev, i+16), cq);
2181         }
2182         pci_free_irq_vectors(pdev);
2183 }
2184
2185 static void hisi_sas_v3_remove(struct pci_dev *pdev)
2186 {
2187         struct device *dev = &pdev->dev;
2188         struct sas_ha_struct *sha = dev_get_drvdata(dev);
2189         struct hisi_hba *hisi_hba = sha->lldd_ha;
2190         struct Scsi_Host *shost = sha->core.shost;
2191
2192         if (timer_pending(&hisi_hba->timer))
2193                 del_timer(&hisi_hba->timer);
2194
2195         sas_unregister_ha(sha);
2196         sas_remove_host(sha->core.shost);
2197
2198         hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
2199         hisi_sas_kill_tasklets(hisi_hba);
2200         pci_release_regions(pdev);
2201         pci_disable_device(pdev);
2202         hisi_sas_free(hisi_hba);
2203         scsi_host_put(shost);
2204 }
2205
2206 static const struct hisi_sas_hw_error sas_ras_intr0_nfe[] = {
2207         { .irq_msk = BIT(19), .msg = "HILINK_INT" },
2208         { .irq_msk = BIT(20), .msg = "HILINK_PLL0_OUT_OF_LOCK" },
2209         { .irq_msk = BIT(21), .msg = "HILINK_PLL1_OUT_OF_LOCK" },
2210         { .irq_msk = BIT(22), .msg = "HILINK_LOSS_OF_REFCLK0" },
2211         { .irq_msk = BIT(23), .msg = "HILINK_LOSS_OF_REFCLK1" },
2212         { .irq_msk = BIT(24), .msg = "DMAC0_TX_POISON" },
2213         { .irq_msk = BIT(25), .msg = "DMAC1_TX_POISON" },
2214         { .irq_msk = BIT(26), .msg = "DMAC2_TX_POISON" },
2215         { .irq_msk = BIT(27), .msg = "DMAC3_TX_POISON" },
2216         { .irq_msk = BIT(28), .msg = "DMAC4_TX_POISON" },
2217         { .irq_msk = BIT(29), .msg = "DMAC5_TX_POISON" },
2218         { .irq_msk = BIT(30), .msg = "DMAC6_TX_POISON" },
2219         { .irq_msk = BIT(31), .msg = "DMAC7_TX_POISON" },
2220 };
2221
2222 static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
2223         { .irq_msk = BIT(0), .msg = "RXM_CFG_MEM3_ECC2B_INTR" },
2224         { .irq_msk = BIT(1), .msg = "RXM_CFG_MEM2_ECC2B_INTR" },
2225         { .irq_msk = BIT(2), .msg = "RXM_CFG_MEM1_ECC2B_INTR" },
2226         { .irq_msk = BIT(3), .msg = "RXM_CFG_MEM0_ECC2B_INTR" },
2227         { .irq_msk = BIT(4), .msg = "HGC_CQE_ECC2B_INTR" },
2228         { .irq_msk = BIT(5), .msg = "LM_CFG_IOSTL_ECC2B_INTR" },
2229         { .irq_msk = BIT(6), .msg = "LM_CFG_ITCTL_ECC2B_INTR" },
2230         { .irq_msk = BIT(7), .msg = "HGC_ITCT_ECC2B_INTR" },
2231         { .irq_msk = BIT(8), .msg = "HGC_IOST_ECC2B_INTR" },
2232         { .irq_msk = BIT(9), .msg = "HGC_DQE_ECC2B_INTR" },
2233         { .irq_msk = BIT(10), .msg = "DMAC0_RAM_ECC2B_INTR" },
2234         { .irq_msk = BIT(11), .msg = "DMAC1_RAM_ECC2B_INTR" },
2235         { .irq_msk = BIT(12), .msg = "DMAC2_RAM_ECC2B_INTR" },
2236         { .irq_msk = BIT(13), .msg = "DMAC3_RAM_ECC2B_INTR" },
2237         { .irq_msk = BIT(14), .msg = "DMAC4_RAM_ECC2B_INTR" },
2238         { .irq_msk = BIT(15), .msg = "DMAC5_RAM_ECC2B_INTR" },
2239         { .irq_msk = BIT(16), .msg = "DMAC6_RAM_ECC2B_INTR" },
2240         { .irq_msk = BIT(17), .msg = "DMAC7_RAM_ECC2B_INTR" },
2241         { .irq_msk = BIT(18), .msg = "OOO_RAM_ECC2B_INTR" },
2242         { .irq_msk = BIT(20), .msg = "HGC_DQE_POISON_INTR" },
2243         { .irq_msk = BIT(21), .msg = "HGC_IOST_POISON_INTR" },
2244         { .irq_msk = BIT(22), .msg = "HGC_ITCT_POISON_INTR" },
2245         { .irq_msk = BIT(23), .msg = "HGC_ITCT_NCQ_POISON_INTR" },
2246         { .irq_msk = BIT(24), .msg = "DMAC0_RX_POISON" },
2247         { .irq_msk = BIT(25), .msg = "DMAC1_RX_POISON" },
2248         { .irq_msk = BIT(26), .msg = "DMAC2_RX_POISON" },
2249         { .irq_msk = BIT(27), .msg = "DMAC3_RX_POISON" },
2250         { .irq_msk = BIT(28), .msg = "DMAC4_RX_POISON" },
2251         { .irq_msk = BIT(29), .msg = "DMAC5_RX_POISON" },
2252         { .irq_msk = BIT(30), .msg = "DMAC6_RX_POISON" },
2253         { .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
2254 };
2255
2256 static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
2257         { .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
2258         { .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
2259         { .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
2260         { .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
2261         { .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
2262         { .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
2263         { .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
2264         { .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
2265         { .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
2266         { .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
2267         { .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
2268         { .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
2269         { .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
2270         { .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
2271         { .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
2272         { .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
2273         { .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
2274         { .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
2275         { .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
2276         { .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
2277 };
2278
2279 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
2280 {
2281         struct device *dev = hisi_hba->dev;
2282         const struct hisi_sas_hw_error *ras_error;
2283         bool need_reset = false;
2284         u32 irq_value;
2285         int i;
2286
2287         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR0);
2288         for (i = 0; i < ARRAY_SIZE(sas_ras_intr0_nfe); i++) {
2289                 ras_error = &sas_ras_intr0_nfe[i];
2290                 if (ras_error->irq_msk & irq_value) {
2291                         dev_warn(dev, "SAS_RAS_INTR0: %s(irq_value=0x%x) found.\n",
2292                                         ras_error->msg, irq_value);
2293                         need_reset = true;
2294                 }
2295         }
2296         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0, irq_value);
2297
2298         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR1);
2299         for (i = 0; i < ARRAY_SIZE(sas_ras_intr1_nfe); i++) {
2300                 ras_error = &sas_ras_intr1_nfe[i];
2301                 if (ras_error->irq_msk & irq_value) {
2302                         dev_warn(dev, "SAS_RAS_INTR1: %s(irq_value=0x%x) found.\n",
2303                                         ras_error->msg, irq_value);
2304                         need_reset = true;
2305                 }
2306         }
2307         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
2308
2309         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
2310         for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
2311                 ras_error = &sas_ras_intr2_nfe[i];
2312                 if (ras_error->irq_msk & irq_value) {
2313                         dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
2314                                         ras_error->msg, irq_value);
2315                         need_reset = true;
2316                 }
2317         }
2318         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
2319
2320         return need_reset;
2321 }
2322
2323 static pci_ers_result_t hisi_sas_error_detected_v3_hw(struct pci_dev *pdev,
2324                 pci_channel_state_t state)
2325 {
2326         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2327         struct hisi_hba *hisi_hba = sha->lldd_ha;
2328         struct device *dev = hisi_hba->dev;
2329
2330         dev_info(dev, "PCI error: detected callback, state(%d)!!\n", state);
2331         if (state == pci_channel_io_perm_failure)
2332                 return PCI_ERS_RESULT_DISCONNECT;
2333
2334         if (process_non_fatal_error_v3_hw(hisi_hba))
2335                 return PCI_ERS_RESULT_NEED_RESET;
2336
2337         return PCI_ERS_RESULT_CAN_RECOVER;
2338 }
2339
2340 static pci_ers_result_t hisi_sas_mmio_enabled_v3_hw(struct pci_dev *pdev)
2341 {
2342         return PCI_ERS_RESULT_RECOVERED;
2343 }
2344
2345 static pci_ers_result_t hisi_sas_slot_reset_v3_hw(struct pci_dev *pdev)
2346 {
2347         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2348         struct hisi_hba *hisi_hba = sha->lldd_ha;
2349         struct device *dev = hisi_hba->dev;
2350         HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
2351
2352         dev_info(dev, "PCI error: slot reset callback!!\n");
2353         queue_work(hisi_hba->wq, &r.work);
2354         wait_for_completion(r.completion);
2355         if (r.done)
2356                 return PCI_ERS_RESULT_RECOVERED;
2357
2358         return PCI_ERS_RESULT_DISCONNECT;
2359 }
2360
2361 enum {
2362         /* instances of the controller */
2363         hip08,
2364 };
2365
2366 static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
2367 {
2368         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2369         struct hisi_hba *hisi_hba = sha->lldd_ha;
2370         struct device *dev = hisi_hba->dev;
2371         struct Scsi_Host *shost = hisi_hba->shost;
2372         u32 device_state, status;
2373         int rc;
2374         u32 reg_val;
2375         unsigned long flags;
2376
2377         if (!pdev->pm_cap) {
2378                 dev_err(dev, "PCI PM not supported\n");
2379                 return -ENODEV;
2380         }
2381
2382         set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2383         scsi_block_requests(shost);
2384         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2385         flush_workqueue(hisi_hba->wq);
2386         /* disable DQ/PHY/bus */
2387         interrupt_disable_v3_hw(hisi_hba);
2388         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2389         hisi_sas_kill_tasklets(hisi_hba);
2390
2391         hisi_sas_stop_phys(hisi_hba);
2392
2393         reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2394                 AM_CTRL_GLOBAL);
2395         reg_val |= 0x1;
2396         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2397                 AM_CTRL_GLOBAL, reg_val);
2398
2399         /* wait until bus idle */
2400         rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
2401                 AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
2402         if (rc) {
2403                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
2404                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2405                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2406                 scsi_unblock_requests(shost);
2407                 return rc;
2408         }
2409
2410         hisi_sas_init_mem(hisi_hba);
2411
2412         device_state = pci_choose_state(pdev, state);
2413         dev_warn(dev, "entering operating state [D%d]\n",
2414                         device_state);
2415         pci_save_state(pdev);
2416         pci_disable_device(pdev);
2417         pci_set_power_state(pdev, device_state);
2418
2419         spin_lock_irqsave(&hisi_hba->lock, flags);
2420         hisi_sas_release_tasks(hisi_hba);
2421         spin_unlock_irqrestore(&hisi_hba->lock, flags);
2422
2423         sas_suspend_ha(sha);
2424         return 0;
2425 }
2426
2427 static int hisi_sas_v3_resume(struct pci_dev *pdev)
2428 {
2429         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2430         struct hisi_hba *hisi_hba = sha->lldd_ha;
2431         struct Scsi_Host *shost = hisi_hba->shost;
2432         struct device *dev = hisi_hba->dev;
2433         unsigned int rc;
2434         u32 device_state = pdev->current_state;
2435
2436         dev_warn(dev, "resuming from operating state [D%d]\n",
2437                         device_state);
2438         pci_set_power_state(pdev, PCI_D0);
2439         pci_enable_wake(pdev, PCI_D0, 0);
2440         pci_restore_state(pdev);
2441         rc = pci_enable_device(pdev);
2442         if (rc)
2443                 dev_err(dev, "enable device failed during resume (%d)\n", rc);
2444
2445         pci_set_master(pdev);
2446         scsi_unblock_requests(shost);
2447         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2448
2449         sas_prep_resume_ha(sha);
2450         init_reg_v3_hw(hisi_hba);
2451         hisi_hba->hw->phys_init(hisi_hba);
2452         sas_resume_ha(sha);
2453         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2454
2455         return 0;
2456 }
2457
2458 static const struct pci_device_id sas_v3_pci_table[] = {
2459         { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
2460         {}
2461 };
2462 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
2463
2464 static const struct pci_error_handlers hisi_sas_err_handler = {
2465         .error_detected = hisi_sas_error_detected_v3_hw,
2466         .mmio_enabled   = hisi_sas_mmio_enabled_v3_hw,
2467         .slot_reset     = hisi_sas_slot_reset_v3_hw,
2468 };
2469
2470 static struct pci_driver sas_v3_pci_driver = {
2471         .name           = DRV_NAME,
2472         .id_table       = sas_v3_pci_table,
2473         .probe          = hisi_sas_v3_probe,
2474         .remove         = hisi_sas_v3_remove,
2475         .suspend        = hisi_sas_v3_suspend,
2476         .resume         = hisi_sas_v3_resume,
2477         .err_handler    = &hisi_sas_err_handler,
2478 };
2479
2480 module_pci_driver(sas_v3_pci_driver);
2481
2482 MODULE_LICENSE("GPL");
2483 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
2484 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
2485 MODULE_ALIAS("pci:" DRV_NAME);