drivers/scsi/hisi_sas/hisi_sas_v3_hw.c

   1 /*
   2  * Copyright (c) 2017 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  */
  10
  11 #include "hisi_sas.h"
  12 #define DRV_NAME "hisi_sas_v3_hw"
  13
  14 /* global registers need init*/
  15 #define DLVRY_QUEUE_ENABLE              0x0
  16 #define IOST_BASE_ADDR_LO               0x8
  17 #define IOST_BASE_ADDR_HI               0xc
  18 #define ITCT_BASE_ADDR_LO               0x10
  19 #define ITCT_BASE_ADDR_HI               0x14
  20 #define IO_BROKEN_MSG_ADDR_LO           0x18
  21 #define IO_BROKEN_MSG_ADDR_HI           0x1c
  22 #define PHY_CONTEXT                     0x20
  23 #define PHY_STATE                       0x24
  24 #define PHY_PORT_NUM_MA                 0x28
  25 #define PHY_CONN_RATE                   0x30
  26 #define ITCT_CLR                        0x44
  27 #define ITCT_CLR_EN_OFF                 16
  28 #define ITCT_CLR_EN_MSK                 (0x1 << ITCT_CLR_EN_OFF)
  29 #define ITCT_DEV_OFF                    0
  30 #define ITCT_DEV_MSK                    (0x7ff << ITCT_DEV_OFF)
  31 #define IO_SATA_BROKEN_MSG_ADDR_LO      0x58
  32 #define IO_SATA_BROKEN_MSG_ADDR_HI      0x5c
  33 #define SATA_INITI_D2H_STORE_ADDR_LO    0x60
  34 #define SATA_INITI_D2H_STORE_ADDR_HI    0x64
  35 #define CFG_MAX_TAG                     0x68
  36 #define HGC_SAS_TX_OPEN_FAIL_RETRY_CTRL 0x84
  37 #define HGC_SAS_TXFAIL_RETRY_CTRL       0x88
  38 #define HGC_GET_ITV_TIME                0x90
  39 #define DEVICE_MSG_WORK_MODE            0x94
  40 #define OPENA_WT_CONTI_TIME             0x9c
  41 #define I_T_NEXUS_LOSS_TIME             0xa0
  42 #define MAX_CON_TIME_LIMIT_TIME         0xa4
  43 #define BUS_INACTIVE_LIMIT_TIME         0xa8
  44 #define REJECT_TO_OPEN_LIMIT_TIME       0xac
  45 #define CFG_AGING_TIME                  0xbc
  46 #define HGC_DFX_CFG2                    0xc0
  47 #define CFG_ABT_SET_QUERY_IPTT  0xd4
  48 #define CFG_SET_ABORTED_IPTT_OFF        0
  49 #define CFG_SET_ABORTED_IPTT_MSK        (0xfff << CFG_SET_ABORTED_IPTT_OFF)
  50 #define CFG_SET_ABORTED_EN_OFF  12
  51 #define CFG_ABT_SET_IPTT_DONE   0xd8
  52 #define CFG_ABT_SET_IPTT_DONE_OFF       0
  53 #define HGC_IOMB_PROC1_STATUS   0x104
  54 #define CFG_1US_TIMER_TRSH              0xcc
  55 #define CHNL_INT_STATUS                 0x148
  56 #define HGC_AXI_FIFO_ERR_INFO  0x154
  57 #define AXI_ERR_INFO_OFF               0
  58 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
  59 #define FIFO_ERR_INFO_OFF              8
  60 #define FIFO_ERR_INFO_MSK              (0xff << FIFO_ERR_INFO_OFF)
  61 #define INT_COAL_EN                     0x19c
  62 #define OQ_INT_COAL_TIME                0x1a0
  63 #define OQ_INT_COAL_CNT                 0x1a4
  64 #define ENT_INT_COAL_TIME               0x1a8
  65 #define ENT_INT_COAL_CNT                0x1ac
  66 #define OQ_INT_SRC                      0x1b0
  67 #define OQ_INT_SRC_MSK                  0x1b4
  68 #define ENT_INT_SRC1                    0x1b8
  69 #define ENT_INT_SRC1_D2H_FIS_CH0_OFF    0
  70 #define ENT_INT_SRC1_D2H_FIS_CH0_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH0_OFF)
  71 #define ENT_INT_SRC1_D2H_FIS_CH1_OFF    8
  72 #define ENT_INT_SRC1_D2H_FIS_CH1_MSK    (0x1 << ENT_INT_SRC1_D2H_FIS_CH1_OFF)
  73 #define ENT_INT_SRC2                    0x1bc
  74 #define ENT_INT_SRC3                    0x1c0
  75 #define ENT_INT_SRC3_WP_DEPTH_OFF               8
  76 #define ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF      9
  77 #define ENT_INT_SRC3_RP_DEPTH_OFF               10
  78 #define ENT_INT_SRC3_AXI_OFF                    11
  79 #define ENT_INT_SRC3_FIFO_OFF                   12
  80 #define ENT_INT_SRC3_LM_OFF                             14
  81 #define ENT_INT_SRC3_ITC_INT_OFF        15
  82 #define ENT_INT_SRC3_ITC_INT_MSK        (0x1 << ENT_INT_SRC3_ITC_INT_OFF)
  83 #define ENT_INT_SRC3_ABT_OFF            16
  84 #define ENT_INT_SRC_MSK1                0x1c4
  85 #define ENT_INT_SRC_MSK2                0x1c8
  86 #define ENT_INT_SRC_MSK3                0x1cc
  87 #define ENT_INT_SRC_MSK3_ENT95_MSK_OFF  31
  88 #define CHNL_PHYUPDOWN_INT_MSK          0x1d0
  89 #define CHNL_ENT_INT_MSK                        0x1d4
  90 #define HGC_COM_INT_MSK                         0x1d8
  91 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK  (0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
  92 #define SAS_ECC_INTR                    0x1e8
  93 #define SAS_ECC_INTR_MSK                0x1ec
  94 #define HGC_ERR_STAT_EN                 0x238
  95 #define DLVRY_Q_0_BASE_ADDR_LO          0x260
  96 #define DLVRY_Q_0_BASE_ADDR_HI          0x264
  97 #define DLVRY_Q_0_DEPTH                 0x268
  98 #define DLVRY_Q_0_WR_PTR                0x26c
  99 #define DLVRY_Q_0_RD_PTR                0x270
 100 #define HYPER_STREAM_ID_EN_CFG          0xc80
 101 #define OQ0_INT_SRC_MSK                 0xc90
 102 #define COMPL_Q_0_BASE_ADDR_LO          0x4e0
 103 #define COMPL_Q_0_BASE_ADDR_HI          0x4e4
 104 #define COMPL_Q_0_DEPTH                 0x4e8
 105 #define COMPL_Q_0_WR_PTR                0x4ec
 106 #define COMPL_Q_0_RD_PTR                0x4f0
 107 #define AWQOS_AWCACHE_CFG       0xc84
 108 #define ARQOS_ARCACHE_CFG       0xc88
 109
 110 /* phy registers requiring init */
 111 #define PORT_BASE                       (0x2000)
 112 #define PHY_CFG                         (PORT_BASE + 0x0)
 113 #define HARD_PHY_LINKRATE               (PORT_BASE + 0x4)
 114 #define PHY_CFG_ENA_OFF                 0
 115 #define PHY_CFG_ENA_MSK                 (0x1 << PHY_CFG_ENA_OFF)
 116 #define PHY_CFG_DC_OPT_OFF              2
 117 #define PHY_CFG_DC_OPT_MSK              (0x1 << PHY_CFG_DC_OPT_OFF)
 118 #define PROG_PHY_LINK_RATE              (PORT_BASE + 0x8)
 119 #define PHY_CTRL                        (PORT_BASE + 0x14)
 120 #define PHY_CTRL_RESET_OFF              0
 121 #define PHY_CTRL_RESET_MSK              (0x1 << PHY_CTRL_RESET_OFF)
 122 #define SL_CFG                          (PORT_BASE + 0x84)
 123 #define SL_CONTROL                      (PORT_BASE + 0x94)
 124 #define SL_CONTROL_NOTIFY_EN_OFF        0
 125 #define SL_CONTROL_NOTIFY_EN_MSK        (0x1 << SL_CONTROL_NOTIFY_EN_OFF)
 126 #define SL_CTA_OFF              17
 127 #define SL_CTA_MSK              (0x1 << SL_CTA_OFF)
 128 #define TX_ID_DWORD0                    (PORT_BASE + 0x9c)
 129 #define TX_ID_DWORD1                    (PORT_BASE + 0xa0)
 130 #define TX_ID_DWORD2                    (PORT_BASE + 0xa4)
 131 #define TX_ID_DWORD3                    (PORT_BASE + 0xa8)
 132 #define TX_ID_DWORD4                    (PORT_BASE + 0xaC)
 133 #define TX_ID_DWORD5                    (PORT_BASE + 0xb0)
 134 #define TX_ID_DWORD6                    (PORT_BASE + 0xb4)
 135 #define TXID_AUTO                               (PORT_BASE + 0xb8)
 136 #define CT3_OFF         1
 137 #define CT3_MSK         (0x1 << CT3_OFF)
 138 #define TX_HARDRST_OFF          2
 139 #define TX_HARDRST_MSK          (0x1 << TX_HARDRST_OFF)
 140 #define RX_IDAF_DWORD0                  (PORT_BASE + 0xc4)
 141 #define RXOP_CHECK_CFG_H                (PORT_BASE + 0xfc)
 142 #define STP_LINK_TIMER                  (PORT_BASE + 0x120)
 143 #define STP_LINK_TIMEOUT_STATE          (PORT_BASE + 0x124)
 144 #define CON_CFG_DRIVER                  (PORT_BASE + 0x130)
 145 #define SAS_SSP_CON_TIMER_CFG           (PORT_BASE + 0x134)
 146 #define SAS_SMP_CON_TIMER_CFG           (PORT_BASE + 0x138)
 147 #define SAS_STP_CON_TIMER_CFG           (PORT_BASE + 0x13c)
 148 #define CHL_INT0                        (PORT_BASE + 0x1b4)
 149 #define CHL_INT0_HOTPLUG_TOUT_OFF       0
 150 #define CHL_INT0_HOTPLUG_TOUT_MSK       (0x1 << CHL_INT0_HOTPLUG_TOUT_OFF)
 151 #define CHL_INT0_SL_RX_BCST_ACK_OFF     1
 152 #define CHL_INT0_SL_RX_BCST_ACK_MSK     (0x1 << CHL_INT0_SL_RX_BCST_ACK_OFF)
 153 #define CHL_INT0_SL_PHY_ENABLE_OFF      2
 154 #define CHL_INT0_SL_PHY_ENABLE_MSK      (0x1 << CHL_INT0_SL_PHY_ENABLE_OFF)
 155 #define CHL_INT0_NOT_RDY_OFF            4
 156 #define CHL_INT0_NOT_RDY_MSK            (0x1 << CHL_INT0_NOT_RDY_OFF)
 157 #define CHL_INT0_PHY_RDY_OFF            5
 158 #define CHL_INT0_PHY_RDY_MSK            (0x1 << CHL_INT0_PHY_RDY_OFF)
 159 #define CHL_INT1                        (PORT_BASE + 0x1b8)
 160 #define CHL_INT1_DMAC_TX_ECC_ERR_OFF    15
 161 #define CHL_INT1_DMAC_TX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF)
 162 #define CHL_INT1_DMAC_RX_ECC_ERR_OFF    17
 163 #define CHL_INT1_DMAC_RX_ECC_ERR_MSK    (0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF)
 164 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF 19
 165 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF 20
 166 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF 21
 167 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF 22
 168 #define CHL_INT2                        (PORT_BASE + 0x1bc)
 169 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF  0
 170 #define CHL_INT2_STP_LINK_TIMEOUT_OFF   31
 171 #define CHL_INT0_MSK                    (PORT_BASE + 0x1c0)
 172 #define CHL_INT1_MSK                    (PORT_BASE + 0x1c4)
 173 #define CHL_INT2_MSK                    (PORT_BASE + 0x1c8)
 174 #define CHL_INT_COAL_EN                 (PORT_BASE + 0x1d0)
 175 #define SAS_RX_TRAIN_TIMER              (PORT_BASE + 0x2a4)
 176 #define PHY_CTRL_RDY_MSK                (PORT_BASE + 0x2b0)
 177 #define PHYCTRL_NOT_RDY_MSK             (PORT_BASE + 0x2b4)
 178 #define PHYCTRL_DWS_RESET_MSK           (PORT_BASE + 0x2b8)
 179 #define PHYCTRL_PHY_ENA_MSK             (PORT_BASE + 0x2bc)
 180 #define SL_RX_BCAST_CHK_MSK             (PORT_BASE + 0x2c0)
 181 #define PHYCTRL_OOB_RESTART_MSK         (PORT_BASE + 0x2c4)
 182 #define DMA_TX_STATUS                   (PORT_BASE + 0x2d0)
 183 #define DMA_TX_STATUS_BUSY_OFF          0
 184 #define DMA_TX_STATUS_BUSY_MSK          (0x1 << DMA_TX_STATUS_BUSY_OFF)
 185 #define DMA_RX_STATUS                   (PORT_BASE + 0x2e8)
 186 #define DMA_RX_STATUS_BUSY_OFF          0
 187 #define DMA_RX_STATUS_BUSY_MSK          (0x1 << DMA_RX_STATUS_BUSY_OFF)
 188
 189 #define COARSETUNE_TIME                 (PORT_BASE + 0x304)
 190 #define ERR_CNT_DWS_LOST                (PORT_BASE + 0x380)
 191 #define ERR_CNT_RESET_PROB              (PORT_BASE + 0x384)
 192 #define ERR_CNT_INVLD_DW                (PORT_BASE + 0x390)
 193 #define ERR_CNT_DISP_ERR                (PORT_BASE + 0x398)
 194
 195 #define DEFAULT_ITCT_HW         2048 /* reset value, not reprogrammed */
 196 #if (HISI_SAS_MAX_DEVICES > DEFAULT_ITCT_HW)
 197 #error Max ITCT exceeded
 198 #endif
 199
 200 #define AXI_MASTER_CFG_BASE             (0x5000)
 201 #define AM_CTRL_GLOBAL                  (0x0)
 202 #define AM_CURR_TRANS_RETURN    (0x150)
 203
 204 #define AM_CFG_MAX_TRANS                (0x5010)
 205 #define AM_CFG_SINGLE_PORT_MAX_TRANS    (0x5014)
 206 #define AXI_CFG                                 (0x5100)
 207 #define AM_ROB_ECC_ERR_ADDR             (0x510c)
 208 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF  0
 209 #define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
 210 #define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF  8
 211 #define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK  (0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
 212
 213 /* RAS registers need init */
 214 #define RAS_BASE                (0x6000)
 215 #define SAS_RAS_INTR0                   (RAS_BASE)
 216 #define SAS_RAS_INTR1                   (RAS_BASE + 0x04)
 217 #define SAS_RAS_INTR0_MASK              (RAS_BASE + 0x08)
 218 #define SAS_RAS_INTR1_MASK              (RAS_BASE + 0x0c)
 219 #define CFG_SAS_RAS_INTR_MASK           (RAS_BASE + 0x1c)
 220 #define SAS_RAS_INTR2                   (RAS_BASE + 0x20)
 221 #define SAS_RAS_INTR2_MASK              (RAS_BASE + 0x24)
 222
 223 /* HW dma structures */
 224 /* Delivery queue header */
 225 /* dw0 */
 226 #define CMD_HDR_ABORT_FLAG_OFF          0
 227 #define CMD_HDR_ABORT_FLAG_MSK          (0x3 << CMD_HDR_ABORT_FLAG_OFF)
 228 #define CMD_HDR_ABORT_DEVICE_TYPE_OFF   2
 229 #define CMD_HDR_ABORT_DEVICE_TYPE_MSK   (0x1 << CMD_HDR_ABORT_DEVICE_TYPE_OFF)
 230 #define CMD_HDR_RESP_REPORT_OFF         5
 231 #define CMD_HDR_RESP_REPORT_MSK         (0x1 << CMD_HDR_RESP_REPORT_OFF)
 232 #define CMD_HDR_TLR_CTRL_OFF            6
 233 #define CMD_HDR_TLR_CTRL_MSK            (0x3 << CMD_HDR_TLR_CTRL_OFF)
 234 #define CMD_HDR_PORT_OFF                18
 235 #define CMD_HDR_PORT_MSK                (0xf << CMD_HDR_PORT_OFF)
 236 #define CMD_HDR_PRIORITY_OFF            27
 237 #define CMD_HDR_PRIORITY_MSK            (0x1 << CMD_HDR_PRIORITY_OFF)
 238 #define CMD_HDR_CMD_OFF                 29
 239 #define CMD_HDR_CMD_MSK                 (0x7 << CMD_HDR_CMD_OFF)
 240 /* dw1 */
 241 #define CMD_HDR_UNCON_CMD_OFF   3
 242 #define CMD_HDR_DIR_OFF                 5
 243 #define CMD_HDR_DIR_MSK                 (0x3 << CMD_HDR_DIR_OFF)
 244 #define CMD_HDR_RESET_OFF               7
 245 #define CMD_HDR_RESET_MSK               (0x1 << CMD_HDR_RESET_OFF)
 246 #define CMD_HDR_VDTL_OFF                10
 247 #define CMD_HDR_VDTL_MSK                (0x1 << CMD_HDR_VDTL_OFF)
 248 #define CMD_HDR_FRAME_TYPE_OFF          11
 249 #define CMD_HDR_FRAME_TYPE_MSK          (0x1f << CMD_HDR_FRAME_TYPE_OFF)
 250 #define CMD_HDR_DEV_ID_OFF              16
 251 #define CMD_HDR_DEV_ID_MSK              (0xffff << CMD_HDR_DEV_ID_OFF)
 252 /* dw2 */
 253 #define CMD_HDR_CFL_OFF                 0
 254 #define CMD_HDR_CFL_MSK                 (0x1ff << CMD_HDR_CFL_OFF)
 255 #define CMD_HDR_NCQ_TAG_OFF             10
 256 #define CMD_HDR_NCQ_TAG_MSK             (0x1f << CMD_HDR_NCQ_TAG_OFF)
 257 #define CMD_HDR_MRFL_OFF                15
 258 #define CMD_HDR_MRFL_MSK                (0x1ff << CMD_HDR_MRFL_OFF)
 259 #define CMD_HDR_SG_MOD_OFF              24
 260 #define CMD_HDR_SG_MOD_MSK              (0x3 << CMD_HDR_SG_MOD_OFF)
 261 /* dw3 */
 262 #define CMD_HDR_IPTT_OFF                0
 263 #define CMD_HDR_IPTT_MSK                (0xffff << CMD_HDR_IPTT_OFF)
 264 /* dw6 */
 265 #define CMD_HDR_DIF_SGL_LEN_OFF         0
 266 #define CMD_HDR_DIF_SGL_LEN_MSK         (0xffff << CMD_HDR_DIF_SGL_LEN_OFF)
 267 #define CMD_HDR_DATA_SGL_LEN_OFF        16
 268 #define CMD_HDR_DATA_SGL_LEN_MSK        (0xffff << CMD_HDR_DATA_SGL_LEN_OFF)
 269 /* dw7 */
 270 #define CMD_HDR_ADDR_MODE_SEL_OFF               15
 271 #define CMD_HDR_ADDR_MODE_SEL_MSK               (1 << CMD_HDR_ADDR_MODE_SEL_OFF)
 272 #define CMD_HDR_ABORT_IPTT_OFF          16
 273 #define CMD_HDR_ABORT_IPTT_MSK          (0xffff << CMD_HDR_ABORT_IPTT_OFF)
 274
 275 /* Completion header */
 276 /* dw0 */
 277 #define CMPLT_HDR_CMPLT_OFF             0
 278 #define CMPLT_HDR_CMPLT_MSK             (0x3 << CMPLT_HDR_CMPLT_OFF)
 279 #define CMPLT_HDR_ERROR_PHASE_OFF   2
 280 #define CMPLT_HDR_ERROR_PHASE_MSK   (0xff << CMPLT_HDR_ERROR_PHASE_OFF)
 281 #define CMPLT_HDR_RSPNS_XFRD_OFF        10
 282 #define CMPLT_HDR_RSPNS_XFRD_MSK        (0x1 << CMPLT_HDR_RSPNS_XFRD_OFF)
 283 #define CMPLT_HDR_ERX_OFF               12
 284 #define CMPLT_HDR_ERX_MSK               (0x1 << CMPLT_HDR_ERX_OFF)
 285 #define CMPLT_HDR_ABORT_STAT_OFF        13
 286 #define CMPLT_HDR_ABORT_STAT_MSK        (0x7 << CMPLT_HDR_ABORT_STAT_OFF)
 287 /* abort_stat */
 288 #define STAT_IO_NOT_VALID               0x1
 289 #define STAT_IO_NO_DEVICE               0x2
 290 #define STAT_IO_COMPLETE                0x3
 291 #define STAT_IO_ABORTED                 0x4
 292 /* dw1 */
 293 #define CMPLT_HDR_IPTT_OFF              0
 294 #define CMPLT_HDR_IPTT_MSK              (0xffff << CMPLT_HDR_IPTT_OFF)
 295 #define CMPLT_HDR_DEV_ID_OFF            16
 296 #define CMPLT_HDR_DEV_ID_MSK            (0xffff << CMPLT_HDR_DEV_ID_OFF)
 297 /* dw3 */
 298 #define CMPLT_HDR_IO_IN_TARGET_OFF      17
 299 #define CMPLT_HDR_IO_IN_TARGET_MSK      (0x1 << CMPLT_HDR_IO_IN_TARGET_OFF)
 300
 301 /* ITCT header */
 302 /* qw0 */
 303 #define ITCT_HDR_DEV_TYPE_OFF           0
 304 #define ITCT_HDR_DEV_TYPE_MSK           (0x3 << ITCT_HDR_DEV_TYPE_OFF)
 305 #define ITCT_HDR_VALID_OFF              2
 306 #define ITCT_HDR_VALID_MSK              (0x1 << ITCT_HDR_VALID_OFF)
 307 #define ITCT_HDR_MCR_OFF                5
 308 #define ITCT_HDR_MCR_MSK                (0xf << ITCT_HDR_MCR_OFF)
 309 #define ITCT_HDR_VLN_OFF                9
 310 #define ITCT_HDR_VLN_MSK                (0xf << ITCT_HDR_VLN_OFF)
 311 #define ITCT_HDR_SMP_TIMEOUT_OFF        16
 312 #define ITCT_HDR_AWT_CONTINUE_OFF       25
 313 #define ITCT_HDR_PORT_ID_OFF            28
 314 #define ITCT_HDR_PORT_ID_MSK            (0xf << ITCT_HDR_PORT_ID_OFF)
 315 /* qw2 */
 316 #define ITCT_HDR_INLT_OFF               0
 317 #define ITCT_HDR_INLT_MSK               (0xffffULL << ITCT_HDR_INLT_OFF)
 318 #define ITCT_HDR_RTOLT_OFF              48
 319 #define ITCT_HDR_RTOLT_MSK              (0xffffULL << ITCT_HDR_RTOLT_OFF)
 320
 321 struct hisi_sas_complete_v3_hdr {
 322         __le32 dw0;
 323         __le32 dw1;
 324         __le32 act;
 325         __le32 dw3;
 326 };
 327
 328 struct hisi_sas_err_record_v3 {
 329         /* dw0 */
 330         __le32 trans_tx_fail_type;
 331
 332         /* dw1 */
 333         __le32 trans_rx_fail_type;
 334
 335         /* dw2 */
 336         __le16 dma_tx_err_type;
 337         __le16 sipc_rx_err_type;
 338
 339         /* dw3 */
 340         __le32 dma_rx_err_type;
 341 };
 342
 343 #define RX_DATA_LEN_UNDERFLOW_OFF       6
 344 #define RX_DATA_LEN_UNDERFLOW_MSK       (1 << RX_DATA_LEN_UNDERFLOW_OFF)
 345
 346 #define HISI_SAS_COMMAND_ENTRIES_V3_HW 4096
 347 #define HISI_SAS_MSI_COUNT_V3_HW 32
 348
 349 #define DIR_NO_DATA 0
 350 #define DIR_TO_INI 1
 351 #define DIR_TO_DEVICE 2
 352 #define DIR_RESERVED 3
 353
 354 #define CMD_IS_UNCONSTRAINT(cmd) \
 355         ((cmd == ATA_CMD_READ_LOG_EXT) || \
 356         (cmd == ATA_CMD_READ_LOG_DMA_EXT) || \
 357         (cmd == ATA_CMD_DEV_RESET))
 358
 359 static u32 hisi_sas_read32(struct hisi_hba *hisi_hba, u32 off)
 360 {
 361         void __iomem *regs = hisi_hba->regs + off;
 362
 363         return readl(regs);
 364 }
 365
 366 static u32 hisi_sas_read32_relaxed(struct hisi_hba *hisi_hba, u32 off)
 367 {
 368         void __iomem *regs = hisi_hba->regs + off;
 369
 370         return readl_relaxed(regs);
 371 }
 372
 373 static void hisi_sas_write32(struct hisi_hba *hisi_hba, u32 off, u32 val)
 374 {
 375         void __iomem *regs = hisi_hba->regs + off;
 376
 377         writel(val, regs);
 378 }
 379
 380 static void hisi_sas_phy_write32(struct hisi_hba *hisi_hba, int phy_no,
 381                                  u32 off, u32 val)
 382 {
 383         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 384
 385         writel(val, regs);
 386 }
 387
 388 static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 389                                       int phy_no, u32 off)
 390 {
 391         void __iomem *regs = hisi_hba->regs + (0x400 * phy_no) + off;
 392
 393         return readl(regs);
 394 }
 395
 396 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 397 {
 398         struct pci_dev *pdev = hisi_hba->pci_dev;
 399         int i;
 400
 401         /* Global registers init */
 402         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE,
 403                          (u32)((1ULL << hisi_hba->queue_count) - 1));
 404         hisi_sas_write32(hisi_hba, CFG_MAX_TAG, 0xfff0400);
 405         hisi_sas_write32(hisi_hba, HGC_SAS_TXFAIL_RETRY_CTRL, 0x108);
 406         hisi_sas_write32(hisi_hba, CFG_1US_TIMER_TRSH, 0xd);
 407         hisi_sas_write32(hisi_hba, INT_COAL_EN, 0x1);
 408         hisi_sas_write32(hisi_hba, OQ_INT_COAL_TIME, 0x1);
 409         hisi_sas_write32(hisi_hba, OQ_INT_COAL_CNT, 0x1);
 410         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 0xffff);
 411         hisi_sas_write32(hisi_hba, ENT_INT_SRC1, 0xffffffff);
 412         hisi_sas_write32(hisi_hba, ENT_INT_SRC2, 0xffffffff);
 413         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 414         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 415         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
 416         if (pdev->revision >= 0x21)
 417                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7fff);
 418         else
 419                 hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
 420         hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 421         hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 422         hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
 423         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
 424         hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 425         hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 426         for (i = 0; i < hisi_hba->queue_count; i++)
 427                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK+0x4*i, 0);
 428
 429         hisi_sas_write32(hisi_hba, HYPER_STREAM_ID_EN_CFG, 1);
 430
 431         for (i = 0; i < hisi_hba->n_phy; i++) {
 432                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 433                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 434                 u32 prog_phy_link_rate = 0x800;
 435
 436                 if (!sas_phy->phy || (sas_phy->phy->maximum_linkrate <
 437                                 SAS_LINK_RATE_1_5_GBPS)) {
 438                         prog_phy_link_rate = 0x855;
 439                 } else {
 440                         enum sas_linkrate max = sas_phy->phy->maximum_linkrate;
 441
 442                         prog_phy_link_rate =
 443                                 hisi_sas_get_prog_phy_linkrate_mask(max) |
 444                                 0x800;
 445                 }
 446                 hisi_sas_phy_write32(hisi_hba, i, PROG_PHY_LINK_RATE,
 447                         prog_phy_link_rate);
 448                 hisi_sas_phy_write32(hisi_hba, i, SAS_RX_TRAIN_TIMER, 0x13e80);
 449                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT0, 0xffffffff);
 450                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 451                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 452                 hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
 453                 if (pdev->revision >= 0x21)
 454                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 455                                         0xffffffff);
 456                 else
 457                         hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
 458                                         0xff87ffff);
 459                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 460                 hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 461                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
 462                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_DWS_RESET_MSK, 0x0);
 463                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x0);
 464                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x0);
 465                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_OOB_RESTART_MSK, 0x1);
 466                 hisi_sas_phy_write32(hisi_hba, i, STP_LINK_TIMER, 0x7f7a120);
 467
 468                 /* used for 12G negotiate */
 469                 hisi_sas_phy_write32(hisi_hba, i, COARSETUNE_TIME, 0x1e);
 470         }
 471
 472         for (i = 0; i < hisi_hba->queue_count; i++) {
 473                 /* Delivery queue */
 474                 hisi_sas_write32(hisi_hba,
 475                                  DLVRY_Q_0_BASE_ADDR_HI + (i * 0x14),
 476                                  upper_32_bits(hisi_hba->cmd_hdr_dma[i]));
 477
 478                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_BASE_ADDR_LO + (i * 0x14),
 479                                  lower_32_bits(hisi_hba->cmd_hdr_dma[i]));
 480
 481                 hisi_sas_write32(hisi_hba, DLVRY_Q_0_DEPTH + (i * 0x14),
 482                                  HISI_SAS_QUEUE_SLOTS);
 483
 484                 /* Completion queue */
 485                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_HI + (i * 0x14),
 486                                  upper_32_bits(hisi_hba->complete_hdr_dma[i]));
 487
 488                 hisi_sas_write32(hisi_hba, COMPL_Q_0_BASE_ADDR_LO + (i * 0x14),
 489                                  lower_32_bits(hisi_hba->complete_hdr_dma[i]));
 490
 491                 hisi_sas_write32(hisi_hba, COMPL_Q_0_DEPTH + (i * 0x14),
 492                                  HISI_SAS_QUEUE_SLOTS);
 493         }
 494
 495         /* itct */
 496         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_LO,
 497                          lower_32_bits(hisi_hba->itct_dma));
 498
 499         hisi_sas_write32(hisi_hba, ITCT_BASE_ADDR_HI,
 500                          upper_32_bits(hisi_hba->itct_dma));
 501
 502         /* iost */
 503         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_LO,
 504                          lower_32_bits(hisi_hba->iost_dma));
 505
 506         hisi_sas_write32(hisi_hba, IOST_BASE_ADDR_HI,
 507                          upper_32_bits(hisi_hba->iost_dma));
 508
 509         /* breakpoint */
 510         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_LO,
 511                          lower_32_bits(hisi_hba->breakpoint_dma));
 512
 513         hisi_sas_write32(hisi_hba, IO_BROKEN_MSG_ADDR_HI,
 514                          upper_32_bits(hisi_hba->breakpoint_dma));
 515
 516         /* SATA broken msg */
 517         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_LO,
 518                          lower_32_bits(hisi_hba->sata_breakpoint_dma));
 519
 520         hisi_sas_write32(hisi_hba, IO_SATA_BROKEN_MSG_ADDR_HI,
 521                          upper_32_bits(hisi_hba->sata_breakpoint_dma));
 522
 523         /* SATA initial fis */
 524         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_LO,
 525                          lower_32_bits(hisi_hba->initial_fis_dma));
 526
 527         hisi_sas_write32(hisi_hba, SATA_INITI_D2H_STORE_ADDR_HI,
 528                          upper_32_bits(hisi_hba->initial_fis_dma));
 529
 530         /* RAS registers init */
 531         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0_MASK, 0x0);
 532         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1_MASK, 0x0);
 533         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2_MASK, 0x0);
 534         hisi_sas_write32(hisi_hba, CFG_SAS_RAS_INTR_MASK, 0x0);
 535 }
 536
 537 static void config_phy_opt_mode_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 538 {
 539         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 540
 541         cfg &= ~PHY_CFG_DC_OPT_MSK;
 542         cfg |= 1 << PHY_CFG_DC_OPT_OFF;
 543         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 544 }
 545
 546 static void config_id_frame_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 547 {
 548         struct sas_identify_frame identify_frame;
 549         u32 *identify_buffer;
 550
 551         memset(&identify_frame, 0, sizeof(identify_frame));
 552         identify_frame.dev_type = SAS_END_DEVICE;
 553         identify_frame.frame_type = 0;
 554         identify_frame._un1 = 1;
 555         identify_frame.initiator_bits = SAS_PROTOCOL_ALL;
 556         identify_frame.target_bits = SAS_PROTOCOL_NONE;
 557         memcpy(&identify_frame._un4_11[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 558         memcpy(&identify_frame.sas_addr[0], hisi_hba->sas_addr, SAS_ADDR_SIZE);
 559         identify_frame.phy_id = phy_no;
 560         identify_buffer = (u32 *)(&identify_frame);
 561
 562         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD0,
 563                         __swab32(identify_buffer[0]));
 564         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD1,
 565                         __swab32(identify_buffer[1]));
 566         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD2,
 567                         __swab32(identify_buffer[2]));
 568         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD3,
 569                         __swab32(identify_buffer[3]));
 570         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD4,
 571                         __swab32(identify_buffer[4]));
 572         hisi_sas_phy_write32(hisi_hba, phy_no, TX_ID_DWORD5,
 573                         __swab32(identify_buffer[5]));
 574 }
 575
 576 static void setup_itct_v3_hw(struct hisi_hba *hisi_hba,
 577                              struct hisi_sas_device *sas_dev)
 578 {
 579         struct domain_device *device = sas_dev->sas_device;
 580         struct device *dev = hisi_hba->dev;
 581         u64 qw0, device_id = sas_dev->device_id;
 582         struct hisi_sas_itct *itct = &hisi_hba->itct[device_id];
 583         struct domain_device *parent_dev = device->parent;
 584         struct asd_sas_port *sas_port = device->port;
 585         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
 586
 587         memset(itct, 0, sizeof(*itct));
 588
 589         /* qw0 */
 590         qw0 = 0;
 591         switch (sas_dev->dev_type) {
 592         case SAS_END_DEVICE:
 593         case SAS_EDGE_EXPANDER_DEVICE:
 594         case SAS_FANOUT_EXPANDER_DEVICE:
 595                 qw0 = HISI_SAS_DEV_TYPE_SSP << ITCT_HDR_DEV_TYPE_OFF;
 596                 break;
 597         case SAS_SATA_DEV:
 598         case SAS_SATA_PENDING:
 599                 if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
 600                         qw0 = HISI_SAS_DEV_TYPE_STP << ITCT_HDR_DEV_TYPE_OFF;
 601                 else
 602                         qw0 = HISI_SAS_DEV_TYPE_SATA << ITCT_HDR_DEV_TYPE_OFF;
 603                 break;
 604         default:
 605                 dev_warn(dev, "setup itct: unsupported dev type (%d)\n",
 606                          sas_dev->dev_type);
 607         }
 608
 609         qw0 |= ((1 << ITCT_HDR_VALID_OFF) |
 610                 (device->linkrate << ITCT_HDR_MCR_OFF) |
 611                 (1 << ITCT_HDR_VLN_OFF) |
 612                 (0xfa << ITCT_HDR_SMP_TIMEOUT_OFF) |
 613                 (1 << ITCT_HDR_AWT_CONTINUE_OFF) |
 614                 (port->id << ITCT_HDR_PORT_ID_OFF));
 615         itct->qw0 = cpu_to_le64(qw0);
 616
 617         /* qw1 */
 618         memcpy(&itct->sas_addr, device->sas_addr, SAS_ADDR_SIZE);
 619         itct->sas_addr = __swab64(itct->sas_addr);
 620
 621         /* qw2 */
 622         if (!dev_is_sata(device))
 623                 itct->qw2 = cpu_to_le64((5000ULL << ITCT_HDR_INLT_OFF) |
 624                                         (0x1ULL << ITCT_HDR_RTOLT_OFF));
 625 }
 626
 627 static void clear_itct_v3_hw(struct hisi_hba *hisi_hba,
 628                               struct hisi_sas_device *sas_dev)
 629 {
 630         DECLARE_COMPLETION_ONSTACK(completion);
 631         u64 dev_id = sas_dev->device_id;
 632         struct hisi_sas_itct *itct = &hisi_hba->itct[dev_id];
 633         u32 reg_val = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
 634
 635         sas_dev->completion = &completion;
 636
 637         /* clear the itct interrupt state */
 638         if (ENT_INT_SRC3_ITC_INT_MSK & reg_val)
 639                 hisi_sas_write32(hisi_hba, ENT_INT_SRC3,
 640                                  ENT_INT_SRC3_ITC_INT_MSK);
 641
 642         /* clear the itct table*/
 643         reg_val = ITCT_CLR_EN_MSK | (dev_id & ITCT_DEV_MSK);
 644         hisi_sas_write32(hisi_hba, ITCT_CLR, reg_val);
 645
 646         wait_for_completion(sas_dev->completion);
 647         memset(itct, 0, sizeof(struct hisi_sas_itct));
 648 }
 649
 650 static void dereg_device_v3_hw(struct hisi_hba *hisi_hba,
 651                                 struct domain_device *device)
 652 {
 653         struct hisi_sas_slot *slot, *slot2;
 654         struct hisi_sas_device *sas_dev = device->lldd_dev;
 655         u32 cfg_abt_set_query_iptt;
 656
 657         cfg_abt_set_query_iptt = hisi_sas_read32(hisi_hba,
 658                 CFG_ABT_SET_QUERY_IPTT);
 659         list_for_each_entry_safe(slot, slot2, &sas_dev->list, entry) {
 660                 cfg_abt_set_query_iptt &= ~CFG_SET_ABORTED_IPTT_MSK;
 661                 cfg_abt_set_query_iptt |= (1 << CFG_SET_ABORTED_EN_OFF) |
 662                         (slot->idx << CFG_SET_ABORTED_IPTT_OFF);
 663                 hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 664                         cfg_abt_set_query_iptt);
 665         }
 666         cfg_abt_set_query_iptt &= ~(1 << CFG_SET_ABORTED_EN_OFF);
 667         hisi_sas_write32(hisi_hba, CFG_ABT_SET_QUERY_IPTT,
 668                 cfg_abt_set_query_iptt);
 669         hisi_sas_write32(hisi_hba, CFG_ABT_SET_IPTT_DONE,
 670                                         1 << CFG_ABT_SET_IPTT_DONE_OFF);
 671 }
 672
 673 static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 674 {
 675         struct device *dev = hisi_hba->dev;
 676         int ret;
 677         u32 val;
 678
 679         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0);
 680
 681         /* Disable all of the PHYs */
 682         hisi_sas_stop_phys(hisi_hba);
 683         udelay(50);
 684
 685         /* Ensure axi bus idle */
 686         ret = readl_poll_timeout(hisi_hba->regs + AXI_CFG, val, !val,
 687                         20000, 1000000);
 688         if (ret) {
 689                 dev_err(dev, "axi bus is not idle, ret = %d!\n", ret);
 690                 return -EIO;
 691         }
 692
 693         if (ACPI_HANDLE(dev)) {
 694                 acpi_status s;
 695
 696                 s = acpi_evaluate_object(ACPI_HANDLE(dev), "_RST", NULL, NULL);
 697                 if (ACPI_FAILURE(s)) {
 698                         dev_err(dev, "Reset failed\n");
 699                         return -EIO;
 700                 }
 701         } else {
 702                 dev_err(dev, "no reset method!\n");
 703                 return -EINVAL;
 704         }
 705
 706         return 0;
 707 }
 708
 709 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 710 {
 711         struct device *dev = hisi_hba->dev;
 712         int rc;
 713
 714         rc = reset_hw_v3_hw(hisi_hba);
 715         if (rc) {
 716                 dev_err(dev, "hisi_sas_reset_hw failed, rc=%d", rc);
 717                 return rc;
 718         }
 719
 720         msleep(100);
 721         init_reg_v3_hw(hisi_hba);
 722
 723         return 0;
 724 }
 725
 726 static void enable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 727 {
 728         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 729
 730         cfg |= PHY_CFG_ENA_MSK;
 731         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 732 }
 733
 734 static void disable_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 735 {
 736         u32 cfg = hisi_sas_phy_read32(hisi_hba, phy_no, PHY_CFG);
 737
 738         cfg &= ~PHY_CFG_ENA_MSK;
 739         hisi_sas_phy_write32(hisi_hba, phy_no, PHY_CFG, cfg);
 740 }
 741
 742 static void start_phy_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 743 {
 744         config_id_frame_v3_hw(hisi_hba, phy_no);
 745         config_phy_opt_mode_v3_hw(hisi_hba, phy_no);
 746         enable_phy_v3_hw(hisi_hba, phy_no);
 747 }
 748
 749 static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 750 {
 751         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 752         u32 txid_auto;
 753
 754         disable_phy_v3_hw(hisi_hba, phy_no);
 755         if (phy->identify.device_type == SAS_END_DEVICE) {
 756                 txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 757                 hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 758                                         txid_auto | TX_HARDRST_MSK);
 759         }
 760         msleep(100);
 761         start_phy_v3_hw(hisi_hba, phy_no);
 762 }
 763
 764 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
 765 {
 766         return SAS_LINK_RATE_12_0_GBPS;
 767 }
 768
 769 static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 770 {
 771         int i;
 772
 773         for (i = 0; i < hisi_hba->n_phy; i++) {
 774                 struct hisi_sas_phy *phy = &hisi_hba->phy[i];
 775                 struct asd_sas_phy *sas_phy = &phy->sas_phy;
 776
 777                 if (!sas_phy->phy->enabled)
 778                         continue;
 779
 780                 start_phy_v3_hw(hisi_hba, i);
 781         }
 782 }
 783
 784 static void sl_notify_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 785 {
 786         u32 sl_control;
 787
 788         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 789         sl_control |= SL_CONTROL_NOTIFY_EN_MSK;
 790         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 791         msleep(1);
 792         sl_control = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
 793         sl_control &= ~SL_CONTROL_NOTIFY_EN_MSK;
 794         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL, sl_control);
 795 }
 796
 797 static int get_wideport_bitmap_v3_hw(struct hisi_hba *hisi_hba, int port_id)
 798 {
 799         int i, bitmap = 0;
 800         u32 phy_port_num_ma = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
 801         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
 802
 803         for (i = 0; i < hisi_hba->n_phy; i++)
 804                 if (phy_state & BIT(i))
 805                         if (((phy_port_num_ma >> (i * 4)) & 0xf) == port_id)
 806                                 bitmap |= BIT(i);
 807
 808         return bitmap;
 809 }
 810
 811 /**
 812  * The callpath to this function and upto writing the write
 813  * queue pointer should be safe from interruption.
 814  */
 815 static int
 816 get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 817 {
 818         struct device *dev = hisi_hba->dev;
 819         int queue = dq->id;
 820         u32 r, w;
 821
 822         w = dq->wr_point;
 823         r = hisi_sas_read32_relaxed(hisi_hba,
 824                                 DLVRY_Q_0_RD_PTR + (queue * 0x14));
 825         if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
 826                 dev_warn(dev, "full queue=%d r=%d w=%d\n\n",
 827                                 queue, r, w);
 828                 return -EAGAIN;
 829         }
 830
 831         return 0;
 832 }
 833
 834 static void start_delivery_v3_hw(struct hisi_sas_dq *dq)
 835 {
 836         struct hisi_hba *hisi_hba = dq->hisi_hba;
 837         int dlvry_queue = dq->slot_prep->dlvry_queue;
 838         int dlvry_queue_slot = dq->slot_prep->dlvry_queue_slot;
 839
 840         dq->wr_point = ++dlvry_queue_slot % HISI_SAS_QUEUE_SLOTS;
 841         hisi_sas_write32(hisi_hba, DLVRY_Q_0_WR_PTR + (dlvry_queue * 0x14),
 842                          dq->wr_point);
 843 }
 844
 845 static int prep_prd_sge_v3_hw(struct hisi_hba *hisi_hba,
 846                               struct hisi_sas_slot *slot,
 847                               struct hisi_sas_cmd_hdr *hdr,
 848                               struct scatterlist *scatter,
 849                               int n_elem)
 850 {
 851         struct hisi_sas_sge_page *sge_page = hisi_sas_sge_addr_mem(slot);
 852         struct device *dev = hisi_hba->dev;
 853         struct scatterlist *sg;
 854         int i;
 855
 856         if (n_elem > HISI_SAS_SGE_PAGE_CNT) {
 857                 dev_err(dev, "prd err: n_elem(%d) > HISI_SAS_SGE_PAGE_CNT",
 858                         n_elem);
 859                 return -EINVAL;
 860         }
 861
 862         for_each_sg(scatter, sg, n_elem, i) {
 863                 struct hisi_sas_sge *entry = &sge_page->sge[i];
 864
 865                 entry->addr = cpu_to_le64(sg_dma_address(sg));
 866                 entry->page_ctrl_0 = entry->page_ctrl_1 = 0;
 867                 entry->data_len = cpu_to_le32(sg_dma_len(sg));
 868                 entry->data_off = 0;
 869         }
 870
 871         hdr->prd_table_addr = cpu_to_le64(hisi_sas_sge_addr_dma(slot));
 872
 873         hdr->sg_len = cpu_to_le32(n_elem << CMD_HDR_DATA_SGL_LEN_OFF);
 874
 875         return 0;
 876 }
 877
 878 static int prep_ssp_v3_hw(struct hisi_hba *hisi_hba,
 879                           struct hisi_sas_slot *slot, int is_tmf,
 880                           struct hisi_sas_tmf_task *tmf)
 881 {
 882         struct sas_task *task = slot->task;
 883         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 884         struct domain_device *device = task->dev;
 885         struct hisi_sas_device *sas_dev = device->lldd_dev;
 886         struct hisi_sas_port *port = slot->port;
 887         struct sas_ssp_task *ssp_task = &task->ssp_task;
 888         struct scsi_cmnd *scsi_cmnd = ssp_task->cmd;
 889         int has_data = 0, rc, priority = is_tmf;
 890         u8 *buf_cmd;
 891         u32 dw1 = 0, dw2 = 0;
 892
 893         hdr->dw0 = cpu_to_le32((1 << CMD_HDR_RESP_REPORT_OFF) |
 894                                (2 << CMD_HDR_TLR_CTRL_OFF) |
 895                                (port->id << CMD_HDR_PORT_OFF) |
 896                                (priority << CMD_HDR_PRIORITY_OFF) |
 897                                (1 << CMD_HDR_CMD_OFF)); /* ssp */
 898
 899         dw1 = 1 << CMD_HDR_VDTL_OFF;
 900         if (is_tmf) {
 901                 dw1 |= 2 << CMD_HDR_FRAME_TYPE_OFF;
 902                 dw1 |= DIR_NO_DATA << CMD_HDR_DIR_OFF;
 903         } else {
 904                 dw1 |= 1 << CMD_HDR_FRAME_TYPE_OFF;
 905                 switch (scsi_cmnd->sc_data_direction) {
 906                 case DMA_TO_DEVICE:
 907                         has_data = 1;
 908                         dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
 909                         break;
 910                 case DMA_FROM_DEVICE:
 911                         has_data = 1;
 912                         dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
 913                         break;
 914                 default:
 915                         dw1 &= ~CMD_HDR_DIR_MSK;
 916                 }
 917         }
 918
 919         /* map itct entry */
 920         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
 921         hdr->dw1 = cpu_to_le32(dw1);
 922
 923         dw2 = (((sizeof(struct ssp_command_iu) + sizeof(struct ssp_frame_hdr)
 924               + 3) / 4) << CMD_HDR_CFL_OFF) |
 925               ((HISI_SAS_MAX_SSP_RESP_SZ / 4) << CMD_HDR_MRFL_OFF) |
 926               (2 << CMD_HDR_SG_MOD_OFF);
 927         hdr->dw2 = cpu_to_le32(dw2);
 928         hdr->transfer_tags = cpu_to_le32(slot->idx);
 929
 930         if (has_data) {
 931                 rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
 932                                         slot->n_elem);
 933                 if (rc)
 934                         return rc;
 935         }
 936
 937         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
 938         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
 939         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
 940
 941         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot) +
 942                 sizeof(struct ssp_frame_hdr);
 943
 944         memcpy(buf_cmd, &task->ssp_task.LUN, 8);
 945         if (!is_tmf) {
 946                 buf_cmd[9] = ssp_task->task_attr | (ssp_task->task_prio << 3);
 947                 memcpy(buf_cmd + 12, scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
 948         } else {
 949                 buf_cmd[10] = tmf->tmf;
 950                 switch (tmf->tmf) {
 951                 case TMF_ABORT_TASK:
 952                 case TMF_QUERY_TASK:
 953                         buf_cmd[12] =
 954                                 (tmf->tag_of_task_to_be_managed >> 8) & 0xff;
 955                         buf_cmd[13] =
 956                                 tmf->tag_of_task_to_be_managed & 0xff;
 957                         break;
 958                 default:
 959                         break;
 960                 }
 961         }
 962
 963         return 0;
 964 }
 965
 966 static int prep_smp_v3_hw(struct hisi_hba *hisi_hba,
 967                           struct hisi_sas_slot *slot)
 968 {
 969         struct sas_task *task = slot->task;
 970         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
 971         struct domain_device *device = task->dev;
 972         struct device *dev = hisi_hba->dev;
 973         struct hisi_sas_port *port = slot->port;
 974         struct scatterlist *sg_req, *sg_resp;
 975         struct hisi_sas_device *sas_dev = device->lldd_dev;
 976         dma_addr_t req_dma_addr;
 977         unsigned int req_len, resp_len;
 978         int elem, rc;
 979
 980         /*
 981          * DMA-map SMP request, response buffers
 982          */
 983         /* req */
 984         sg_req = &task->smp_task.smp_req;
 985         elem = dma_map_sg(dev, sg_req, 1, DMA_TO_DEVICE);
 986         if (!elem)
 987                 return -ENOMEM;
 988         req_len = sg_dma_len(sg_req);
 989         req_dma_addr = sg_dma_address(sg_req);
 990
 991         /* resp */
 992         sg_resp = &task->smp_task.smp_resp;
 993         elem = dma_map_sg(dev, sg_resp, 1, DMA_FROM_DEVICE);
 994         if (!elem) {
 995                 rc = -ENOMEM;
 996                 goto err_out_req;
 997         }
 998         resp_len = sg_dma_len(sg_resp);
 999         if ((req_len & 0x3) || (resp_len & 0x3)) {
1000                 rc = -EINVAL;
1001                 goto err_out_resp;
1002         }
1003
1004         /* create header */
1005         /* dw0 */
1006         hdr->dw0 = cpu_to_le32((port->id << CMD_HDR_PORT_OFF) |
1007                                (1 << CMD_HDR_PRIORITY_OFF) | /* high pri */
1008                                (2 << CMD_HDR_CMD_OFF)); /* smp */
1009
1010         /* map itct entry */
1011         hdr->dw1 = cpu_to_le32((sas_dev->device_id << CMD_HDR_DEV_ID_OFF) |
1012                                (1 << CMD_HDR_FRAME_TYPE_OFF) |
1013                                (DIR_NO_DATA << CMD_HDR_DIR_OFF));
1014
1015         /* dw2 */
1016         hdr->dw2 = cpu_to_le32((((req_len - 4) / 4) << CMD_HDR_CFL_OFF) |
1017                                (HISI_SAS_MAX_SMP_RESP_SZ / 4 <<
1018                                CMD_HDR_MRFL_OFF));
1019
1020         hdr->transfer_tags = cpu_to_le32(slot->idx << CMD_HDR_IPTT_OFF);
1021
1022         hdr->cmd_table_addr = cpu_to_le64(req_dma_addr);
1023         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1024
1025         return 0;
1026
1027 err_out_resp:
1028         dma_unmap_sg(dev, &slot->task->smp_task.smp_resp, 1,
1029                      DMA_FROM_DEVICE);
1030 err_out_req:
1031         dma_unmap_sg(dev, &slot->task->smp_task.smp_req, 1,
1032                      DMA_TO_DEVICE);
1033         return rc;
1034 }
1035
1036 static int prep_ata_v3_hw(struct hisi_hba *hisi_hba,
1037                           struct hisi_sas_slot *slot)
1038 {
1039         struct sas_task *task = slot->task;
1040         struct domain_device *device = task->dev;
1041         struct domain_device *parent_dev = device->parent;
1042         struct hisi_sas_device *sas_dev = device->lldd_dev;
1043         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1044         struct asd_sas_port *sas_port = device->port;
1045         struct hisi_sas_port *port = to_hisi_sas_port(sas_port);
1046         u8 *buf_cmd;
1047         int has_data = 0, rc = 0, hdr_tag = 0;
1048         u32 dw1 = 0, dw2 = 0;
1049
1050         hdr->dw0 = cpu_to_le32(port->id << CMD_HDR_PORT_OFF);
1051         if (parent_dev && DEV_IS_EXPANDER(parent_dev->dev_type))
1052                 hdr->dw0 |= cpu_to_le32(3 << CMD_HDR_CMD_OFF);
1053         else
1054                 hdr->dw0 |= cpu_to_le32(4 << CMD_HDR_CMD_OFF);
1055
1056         switch (task->data_dir) {
1057         case DMA_TO_DEVICE:
1058                 has_data = 1;
1059                 dw1 |= DIR_TO_DEVICE << CMD_HDR_DIR_OFF;
1060                 break;
1061         case DMA_FROM_DEVICE:
1062                 has_data = 1;
1063                 dw1 |= DIR_TO_INI << CMD_HDR_DIR_OFF;
1064                 break;
1065         default:
1066                 dw1 &= ~CMD_HDR_DIR_MSK;
1067         }
1068
1069         if ((task->ata_task.fis.command == ATA_CMD_DEV_RESET) &&
1070                         (task->ata_task.fis.control & ATA_SRST))
1071                 dw1 |= 1 << CMD_HDR_RESET_OFF;
1072
1073         dw1 |= (hisi_sas_get_ata_protocol(
1074                 &task->ata_task.fis, task->data_dir))
1075                 << CMD_HDR_FRAME_TYPE_OFF;
1076         dw1 |= sas_dev->device_id << CMD_HDR_DEV_ID_OFF;
1077
1078         if (CMD_IS_UNCONSTRAINT(task->ata_task.fis.command))
1079                 dw1 |= 1 << CMD_HDR_UNCON_CMD_OFF;
1080
1081         hdr->dw1 = cpu_to_le32(dw1);
1082
1083         /* dw2 */
1084         if (task->ata_task.use_ncq && hisi_sas_get_ncq_tag(task, &hdr_tag)) {
1085                 task->ata_task.fis.sector_count |= (u8) (hdr_tag << 3);
1086                 dw2 |= hdr_tag << CMD_HDR_NCQ_TAG_OFF;
1087         }
1088
1089         dw2 |= (HISI_SAS_MAX_STP_RESP_SZ / 4) << CMD_HDR_CFL_OFF |
1090                         2 << CMD_HDR_SG_MOD_OFF;
1091         hdr->dw2 = cpu_to_le32(dw2);
1092
1093         /* dw3 */
1094         hdr->transfer_tags = cpu_to_le32(slot->idx);
1095
1096         if (has_data) {
1097                 rc = prep_prd_sge_v3_hw(hisi_hba, slot, hdr, task->scatter,
1098                                         slot->n_elem);
1099                 if (rc)
1100                         return rc;
1101         }
1102
1103         hdr->data_transfer_len = cpu_to_le32(task->total_xfer_len);
1104         hdr->cmd_table_addr = cpu_to_le64(hisi_sas_cmd_hdr_addr_dma(slot));
1105         hdr->sts_buffer_addr = cpu_to_le64(hisi_sas_status_buf_addr_dma(slot));
1106
1107         buf_cmd = hisi_sas_cmd_hdr_addr_mem(slot);
1108
1109         if (likely(!task->ata_task.device_control_reg_update))
1110                 task->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
1111         /* fill in command FIS */
1112         memcpy(buf_cmd, &task->ata_task.fis, sizeof(struct host_to_dev_fis));
1113
1114         return 0;
1115 }
1116
1117 static int prep_abort_v3_hw(struct hisi_hba *hisi_hba,
1118                 struct hisi_sas_slot *slot,
1119                 int device_id, int abort_flag, int tag_to_abort)
1120 {
1121         struct sas_task *task = slot->task;
1122         struct domain_device *dev = task->dev;
1123         struct hisi_sas_cmd_hdr *hdr = slot->cmd_hdr;
1124         struct hisi_sas_port *port = slot->port;
1125
1126         /* dw0 */
1127         hdr->dw0 = cpu_to_le32((5 << CMD_HDR_CMD_OFF) | /*abort*/
1128                                (port->id << CMD_HDR_PORT_OFF) |
1129                                    (dev_is_sata(dev)
1130                                         << CMD_HDR_ABORT_DEVICE_TYPE_OFF) |
1131                                         (abort_flag
1132                                          << CMD_HDR_ABORT_FLAG_OFF));
1133
1134         /* dw1 */
1135         hdr->dw1 = cpu_to_le32(device_id
1136                         << CMD_HDR_DEV_ID_OFF);
1137
1138         /* dw7 */
1139         hdr->dw7 = cpu_to_le32(tag_to_abort << CMD_HDR_ABORT_IPTT_OFF);
1140         hdr->transfer_tags = cpu_to_le32(slot->idx);
1141
1142         return 0;
1143 }
1144
1145 static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1146 {
1147         int i, res;
1148         u32 context, port_id, link_rate;
1149         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1150         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1151         struct device *dev = hisi_hba->dev;
1152
1153         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 1);
1154
1155         port_id = hisi_sas_read32(hisi_hba, PHY_PORT_NUM_MA);
1156         port_id = (port_id >> (4 * phy_no)) & 0xf;
1157         link_rate = hisi_sas_read32(hisi_hba, PHY_CONN_RATE);
1158         link_rate = (link_rate >> (phy_no * 4)) & 0xf;
1159
1160         if (port_id == 0xf) {
1161                 dev_err(dev, "phyup: phy%d invalid portid\n", phy_no);
1162                 res = IRQ_NONE;
1163                 goto end;
1164         }
1165         sas_phy->linkrate = link_rate;
1166         phy->phy_type &= ~(PORT_TYPE_SAS | PORT_TYPE_SATA);
1167
1168         /* Check for SATA dev */
1169         context = hisi_sas_read32(hisi_hba, PHY_CONTEXT);
1170         if (context & (1 << phy_no)) {
1171                 struct hisi_sas_initial_fis *initial_fis;
1172                 struct dev_to_host_fis *fis;
1173                 u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
1174
1175                 dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
1176                 initial_fis = &hisi_hba->initial_fis[phy_no];
1177                 fis = &initial_fis->fis;
1178                 sas_phy->oob_mode = SATA_OOB_MODE;
1179                 attached_sas_addr[0] = 0x50;
1180                 attached_sas_addr[7] = phy_no;
1181                 memcpy(sas_phy->attached_sas_addr,
1182                        attached_sas_addr,
1183                        SAS_ADDR_SIZE);
1184                 memcpy(sas_phy->frame_rcvd, fis,
1185                        sizeof(struct dev_to_host_fis));
1186                 phy->phy_type |= PORT_TYPE_SATA;
1187                 phy->identify.device_type = SAS_SATA_DEV;
1188                 phy->frame_rcvd_size = sizeof(struct dev_to_host_fis);
1189                 phy->identify.target_port_protocols = SAS_PROTOCOL_SATA;
1190         } else {
1191                 u32 *frame_rcvd = (u32 *)sas_phy->frame_rcvd;
1192                 struct sas_identify_frame *id =
1193                         (struct sas_identify_frame *)frame_rcvd;
1194
1195                 dev_info(dev, "phyup: phy%d link_rate=%d\n", phy_no, link_rate);
1196                 for (i = 0; i < 6; i++) {
1197                         u32 idaf = hisi_sas_phy_read32(hisi_hba, phy_no,
1198                                                RX_IDAF_DWORD0 + (i * 4));
1199                         frame_rcvd[i] = __swab32(idaf);
1200                 }
1201                 sas_phy->oob_mode = SAS_OOB_MODE;
1202                 memcpy(sas_phy->attached_sas_addr,
1203                        &id->sas_addr,
1204                        SAS_ADDR_SIZE);
1205                 phy->phy_type |= PORT_TYPE_SAS;
1206                 phy->identify.device_type = id->dev_type;
1207                 phy->frame_rcvd_size = sizeof(struct sas_identify_frame);
1208                 if (phy->identify.device_type == SAS_END_DEVICE)
1209                         phy->identify.target_port_protocols =
1210                                 SAS_PROTOCOL_SSP;
1211                 else if (phy->identify.device_type != SAS_PHY_UNUSED)
1212                         phy->identify.target_port_protocols =
1213                                 SAS_PROTOCOL_SMP;
1214         }
1215
1216         phy->port_id = port_id;
1217         phy->phy_attached = 1;
1218         hisi_sas_notify_phy_event(phy, HISI_PHYE_PHY_UP);
1219         res = IRQ_HANDLED;
1220 end:
1221         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1222                              CHL_INT0_SL_PHY_ENABLE_MSK);
1223         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_PHY_ENA_MSK, 0);
1224
1225         return res;
1226 }
1227
1228 static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1229 {
1230         u32 phy_state, sl_ctrl, txid_auto;
1231         struct device *dev = hisi_hba->dev;
1232
1233         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 1);
1234
1235         phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1236         dev_info(dev, "phydown: phy%d phy_state=0x%x\n", phy_no, phy_state);
1237         hisi_sas_phy_down(hisi_hba, phy_no, (phy_state & 1 << phy_no) ? 1 : 0);
1238
1239         sl_ctrl = hisi_sas_phy_read32(hisi_hba, phy_no, SL_CONTROL);
1240         hisi_sas_phy_write32(hisi_hba, phy_no, SL_CONTROL,
1241                                                 sl_ctrl&(~SL_CTA_MSK));
1242
1243         txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
1244         hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
1245                                                 txid_auto | CT3_MSK);
1246
1247         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_NOT_RDY_MSK);
1248         hisi_sas_phy_write32(hisi_hba, phy_no, PHYCTRL_NOT_RDY_MSK, 0);
1249
1250         return IRQ_HANDLED;
1251 }
1252
1253 static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
1254 {
1255         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1256         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1257         struct sas_ha_struct *sas_ha = &hisi_hba->sha;
1258
1259         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1);
1260         sas_ha->notify_port_event(sas_phy, PORTE_BROADCAST_RCVD);
1261         hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0,
1262                              CHL_INT0_SL_RX_BCST_ACK_MSK);
1263         hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0);
1264
1265         return IRQ_HANDLED;
1266 }
1267
1268 static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
1269 {
1270         struct hisi_hba *hisi_hba = p;
1271         u32 irq_msk;
1272         int phy_no = 0;
1273         irqreturn_t res = IRQ_NONE;
1274
1275         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1276                                 & 0x11111111;
1277         while (irq_msk) {
1278                 if (irq_msk  & 1) {
1279                         u32 irq_value = hisi_sas_phy_read32(hisi_hba, phy_no,
1280                                                             CHL_INT0);
1281                         u32 phy_state = hisi_sas_read32(hisi_hba, PHY_STATE);
1282                         int rdy = phy_state & (1 << phy_no);
1283
1284                         if (rdy) {
1285                                 if (irq_value & CHL_INT0_SL_PHY_ENABLE_MSK)
1286                                         /* phy up */
1287                                         if (phy_up_v3_hw(phy_no, hisi_hba)
1288                                                         == IRQ_HANDLED)
1289                                                 res = IRQ_HANDLED;
1290                                 if (irq_value & CHL_INT0_SL_RX_BCST_ACK_MSK)
1291                                         /* phy bcast */
1292                                         if (phy_bcast_v3_hw(phy_no, hisi_hba)
1293                                                         == IRQ_HANDLED)
1294                                                 res = IRQ_HANDLED;
1295                         } else {
1296                                 if (irq_value & CHL_INT0_NOT_RDY_MSK)
1297                                         /* phy down */
1298                                         if (phy_down_v3_hw(phy_no, hisi_hba)
1299                                                         == IRQ_HANDLED)
1300                                                 res = IRQ_HANDLED;
1301                         }
1302                 }
1303                 irq_msk >>= 4;
1304                 phy_no++;
1305         }
1306
1307         return res;
1308 }
1309
1310 static const struct hisi_sas_hw_error port_axi_error[] = {
1311         {
1312                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
1313                 .msg = "dma_tx_axi_wr_err",
1314         },
1315         {
1316                 .irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF),
1317                 .msg = "dma_tx_axi_rd_err",
1318         },
1319         {
1320                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF),
1321                 .msg = "dma_rx_axi_wr_err",
1322         },
1323         {
1324                 .irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
1325                 .msg = "dma_rx_axi_rd_err",
1326         },
1327 };
1328
1329 static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
1330 {
1331         struct hisi_hba *hisi_hba = p;
1332         struct device *dev = hisi_hba->dev;
1333         u32 ent_msk, ent_tmp, irq_msk;
1334         int phy_no = 0;
1335
1336         ent_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1337         ent_tmp = ent_msk;
1338         ent_msk |= ENT_INT_SRC_MSK3_ENT95_MSK_MSK;
1339         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_msk);
1340
1341         irq_msk = hisi_sas_read32(hisi_hba, CHNL_INT_STATUS)
1342                                 & 0xeeeeeeee;
1343
1344         while (irq_msk) {
1345                 u32 irq_value0 = hisi_sas_phy_read32(hisi_hba, phy_no,
1346                                                      CHL_INT0);
1347                 u32 irq_value1 = hisi_sas_phy_read32(hisi_hba, phy_no,
1348                                                      CHL_INT1);
1349                 u32 irq_value2 = hisi_sas_phy_read32(hisi_hba, phy_no,
1350                                                      CHL_INT2);
1351                 u32 irq_msk1 = hisi_sas_phy_read32(hisi_hba, phy_no,
1352                                                         CHL_INT1_MSK);
1353                 u32 irq_msk2 = hisi_sas_phy_read32(hisi_hba, phy_no,
1354                                                         CHL_INT2_MSK);
1355
1356                 irq_value1 &= ~irq_msk1;
1357                 irq_value2 &= ~irq_msk2;
1358
1359                 if ((irq_msk & (4 << (phy_no * 4))) &&
1360                                                 irq_value1) {
1361                         int i;
1362
1363                         for (i = 0; i < ARRAY_SIZE(port_axi_error); i++) {
1364                                 const struct hisi_sas_hw_error *error =
1365                                                 &port_axi_error[i];
1366
1367                                 if (!(irq_value1 & error->irq_msk))
1368                                         continue;
1369
1370                                 dev_err(dev, "%s error (phy%d 0x%x) found!\n",
1371                                         error->msg, phy_no, irq_value1);
1372                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1373                         }
1374
1375                         hisi_sas_phy_write32(hisi_hba, phy_no,
1376                                              CHL_INT1, irq_value1);
1377                 }
1378
1379                 if (irq_msk & (8 << (phy_no * 4)) && irq_value2) {
1380                         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1381
1382                         if (irq_value2 & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
1383                                 dev_warn(dev, "phy%d identify timeout\n",
1384                                                         phy_no);
1385                                 hisi_sas_notify_phy_event(phy,
1386                                         HISI_PHYE_LINK_RESET);
1387
1388                         }
1389
1390                         if (irq_value2 & BIT(CHL_INT2_STP_LINK_TIMEOUT_OFF)) {
1391                                 u32 reg_value = hisi_sas_phy_read32(hisi_hba,
1392                                                 phy_no, STP_LINK_TIMEOUT_STATE);
1393
1394                                 dev_warn(dev, "phy%d stp link timeout (0x%x)\n",
1395                                                         phy_no, reg_value);
1396                                 if (reg_value & BIT(4))
1397                                         hisi_sas_notify_phy_event(phy,
1398                                                 HISI_PHYE_LINK_RESET);
1399                         }
1400
1401                         hisi_sas_phy_write32(hisi_hba, phy_no,
1402                                              CHL_INT2, irq_value2);
1403                 }
1404
1405
1406                 if (irq_msk & (2 << (phy_no * 4)) && irq_value0) {
1407                         hisi_sas_phy_write32(hisi_hba, phy_no,
1408                                         CHL_INT0, irq_value0
1409                                         & (~CHL_INT0_SL_RX_BCST_ACK_MSK)
1410                                         & (~CHL_INT0_SL_PHY_ENABLE_MSK)
1411                                         & (~CHL_INT0_NOT_RDY_MSK));
1412                 }
1413                 irq_msk &= ~(0xe << (phy_no * 4));
1414                 phy_no++;
1415         }
1416
1417         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, ent_tmp);
1418
1419         return IRQ_HANDLED;
1420 }
1421
1422 static const struct hisi_sas_hw_error axi_error[] = {
1423         { .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
1424         { .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
1425         { .msk = BIT(2), .msg = "ITCT_AXI_W_ERR" },
1426         { .msk = BIT(3), .msg = "ITCT_AXI_R_ERR" },
1427         { .msk = BIT(4), .msg = "SATA_AXI_W_ERR" },
1428         { .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
1429         { .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
1430         { .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
1431         {},
1432 };
1433
1434 static const struct hisi_sas_hw_error fifo_error[] = {
1435         { .msk = BIT(8),  .msg = "CQE_WINFO_FIFO" },
1436         { .msk = BIT(9),  .msg = "CQE_MSG_FIFIO" },
1437         { .msk = BIT(10), .msg = "GETDQE_FIFO" },
1438         { .msk = BIT(11), .msg = "CMDP_FIFO" },
1439         { .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
1440         {},
1441 };
1442
1443 static const struct hisi_sas_hw_error fatal_axi_error[] = {
1444         {
1445                 .irq_msk = BIT(ENT_INT_SRC3_WP_DEPTH_OFF),
1446                 .msg = "write pointer and depth",
1447         },
1448         {
1449                 .irq_msk = BIT(ENT_INT_SRC3_IPTT_SLOT_NOMATCH_OFF),
1450                 .msg = "iptt no match slot",
1451         },
1452         {
1453                 .irq_msk = BIT(ENT_INT_SRC3_RP_DEPTH_OFF),
1454                 .msg = "read pointer and depth",
1455         },
1456         {
1457                 .irq_msk = BIT(ENT_INT_SRC3_AXI_OFF),
1458                 .reg = HGC_AXI_FIFO_ERR_INFO,
1459                 .sub = axi_error,
1460         },
1461         {
1462                 .irq_msk = BIT(ENT_INT_SRC3_FIFO_OFF),
1463                 .reg = HGC_AXI_FIFO_ERR_INFO,
1464                 .sub = fifo_error,
1465         },
1466         {
1467                 .irq_msk = BIT(ENT_INT_SRC3_LM_OFF),
1468                 .msg = "LM add/fetch list",
1469         },
1470         {
1471                 .irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
1472                 .msg = "SAS_HGC_ABT fetch LM list",
1473         },
1474 };
1475
1476 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
1477 {
1478         u32 irq_value, irq_msk;
1479         struct hisi_hba *hisi_hba = p;
1480         struct device *dev = hisi_hba->dev;
1481         int i;
1482
1483         irq_msk = hisi_sas_read32(hisi_hba, ENT_INT_SRC_MSK3);
1484         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk | 0x1df00);
1485
1486         irq_value = hisi_sas_read32(hisi_hba, ENT_INT_SRC3);
1487         irq_value &= ~irq_msk;
1488
1489         for (i = 0; i < ARRAY_SIZE(fatal_axi_error); i++) {
1490                 const struct hisi_sas_hw_error *error = &fatal_axi_error[i];
1491
1492                 if (!(irq_value & error->irq_msk))
1493                         continue;
1494
1495                 if (error->sub) {
1496                         const struct hisi_sas_hw_error *sub = error->sub;
1497                         u32 err_value = hisi_sas_read32(hisi_hba, error->reg);
1498
1499                         for (; sub->msk || sub->msg; sub++) {
1500                                 if (!(err_value & sub->msk))
1501                                         continue;
1502
1503                                 dev_err(dev, "%s error (0x%x) found!\n",
1504                                         sub->msg, irq_value);
1505                                 queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1506                         }
1507                 } else {
1508                         dev_err(dev, "%s error (0x%x) found!\n",
1509                                 error->msg, irq_value);
1510                         queue_work(hisi_hba->wq, &hisi_hba->rst_work);
1511                 }
1512         }
1513
1514         if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
1515                 u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
1516                 u32 dev_id = reg_val & ITCT_DEV_MSK;
1517                 struct hisi_sas_device *sas_dev =
1518                                 &hisi_hba->devices[dev_id];
1519
1520                 hisi_sas_write32(hisi_hba, ITCT_CLR, 0);
1521                 dev_dbg(dev, "clear ITCT ok\n");
1522                 complete(sas_dev->completion);
1523         }
1524
1525         hisi_sas_write32(hisi_hba, ENT_INT_SRC3, irq_value & 0x1df00);
1526         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, irq_msk);
1527
1528         return IRQ_HANDLED;
1529 }
1530
1531 static void
1532 slot_err_v3_hw(struct hisi_hba *hisi_hba, struct sas_task *task,
1533                struct hisi_sas_slot *slot)
1534 {
1535         struct task_status_struct *ts = &task->task_status;
1536         struct hisi_sas_complete_v3_hdr *complete_queue =
1537                         hisi_hba->complete_hdr[slot->cmplt_queue];
1538         struct hisi_sas_complete_v3_hdr *complete_hdr =
1539                         &complete_queue[slot->cmplt_queue_slot];
1540         struct hisi_sas_err_record_v3 *record =
1541                         hisi_sas_status_buf_addr_mem(slot);
1542         u32 dma_rx_err_type = record->dma_rx_err_type;
1543         u32 trans_tx_fail_type = record->trans_tx_fail_type;
1544
1545         switch (task->task_proto) {
1546         case SAS_PROTOCOL_SSP:
1547                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1548                         ts->residual = trans_tx_fail_type;
1549                         ts->stat = SAS_DATA_UNDERRUN;
1550                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1551                         ts->stat = SAS_QUEUE_FULL;
1552                         slot->abort = 1;
1553                 } else {
1554                         ts->stat = SAS_OPEN_REJECT;
1555                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1556                 }
1557                 break;
1558         case SAS_PROTOCOL_SATA:
1559         case SAS_PROTOCOL_STP:
1560         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1561                 if (dma_rx_err_type & RX_DATA_LEN_UNDERFLOW_MSK) {
1562                         ts->residual = trans_tx_fail_type;
1563                         ts->stat = SAS_DATA_UNDERRUN;
1564                 } else if (complete_hdr->dw3 & CMPLT_HDR_IO_IN_TARGET_MSK) {
1565                         ts->stat = SAS_PHY_DOWN;
1566                         slot->abort = 1;
1567                 } else {
1568                         ts->stat = SAS_OPEN_REJECT;
1569                         ts->open_rej_reason = SAS_OREJ_RSVD_RETRY;
1570                 }
1571                 hisi_sas_sata_done(task, slot);
1572                 break;
1573         case SAS_PROTOCOL_SMP:
1574                 ts->stat = SAM_STAT_CHECK_CONDITION;
1575                 break;
1576         default:
1577                 break;
1578         }
1579 }
1580
1581 static int
1582 slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
1583 {
1584         struct sas_task *task = slot->task;
1585         struct hisi_sas_device *sas_dev;
1586         struct device *dev = hisi_hba->dev;
1587         struct task_status_struct *ts;
1588         struct domain_device *device;
1589         struct sas_ha_struct *ha;
1590         enum exec_status sts;
1591         struct hisi_sas_complete_v3_hdr *complete_queue =
1592                         hisi_hba->complete_hdr[slot->cmplt_queue];
1593         struct hisi_sas_complete_v3_hdr *complete_hdr =
1594                         &complete_queue[slot->cmplt_queue_slot];
1595         unsigned long flags;
1596         bool is_internal = slot->is_internal;
1597
1598         if (unlikely(!task || !task->lldd_task || !task->dev))
1599                 return -EINVAL;
1600
1601         ts = &task->task_status;
1602         device = task->dev;
1603         ha = device->port->ha;
1604         sas_dev = device->lldd_dev;
1605
1606         spin_lock_irqsave(&task->task_state_lock, flags);
1607         task->task_state_flags &=
1608                 ~(SAS_TASK_STATE_PENDING | SAS_TASK_AT_INITIATOR);
1609         spin_unlock_irqrestore(&task->task_state_lock, flags);
1610
1611         memset(ts, 0, sizeof(*ts));
1612         ts->resp = SAS_TASK_COMPLETE;
1613
1614         if (unlikely(!sas_dev)) {
1615                 dev_dbg(dev, "slot complete: port has not device\n");
1616                 ts->stat = SAS_PHY_DOWN;
1617                 goto out;
1618         }
1619
1620         /*
1621          * Use SAS+TMF status codes
1622          */
1623         switch ((complete_hdr->dw0 & CMPLT_HDR_ABORT_STAT_MSK)
1624                         >> CMPLT_HDR_ABORT_STAT_OFF) {
1625         case STAT_IO_ABORTED:
1626                 /* this IO has been aborted by abort command */
1627                 ts->stat = SAS_ABORTED_TASK;
1628                 goto out;
1629         case STAT_IO_COMPLETE:
1630                 /* internal abort command complete */
1631                 ts->stat = TMF_RESP_FUNC_SUCC;
1632                 goto out;
1633         case STAT_IO_NO_DEVICE:
1634                 ts->stat = TMF_RESP_FUNC_COMPLETE;
1635                 goto out;
1636         case STAT_IO_NOT_VALID:
1637                 /*
1638                  * abort single IO, the controller can't find the IO
1639                  */
1640                 ts->stat = TMF_RESP_FUNC_FAILED;
1641                 goto out;
1642         default:
1643                 break;
1644         }
1645
1646         /* check for erroneous completion */
1647         if ((complete_hdr->dw0 & CMPLT_HDR_CMPLT_MSK) == 0x3) {
1648                 u32 *error_info = hisi_sas_status_buf_addr_mem(slot);
1649
1650                 slot_err_v3_hw(hisi_hba, task, slot);
1651                 if (ts->stat != SAS_DATA_UNDERRUN)
1652                         dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
1653                                 "CQ hdr: 0x%x 0x%x 0x%x 0x%x "
1654                                 "Error info: 0x%x 0x%x 0x%x 0x%x\n",
1655                                 slot->idx, task, sas_dev->device_id,
1656                                 complete_hdr->dw0, complete_hdr->dw1,
1657                                 complete_hdr->act, complete_hdr->dw3,
1658                                 error_info[0], error_info[1],
1659                                 error_info[2], error_info[3]);
1660                 if (unlikely(slot->abort))
1661                         return ts->stat;
1662                 goto out;
1663         }
1664
1665         switch (task->task_proto) {
1666         case SAS_PROTOCOL_SSP: {
1667                 struct ssp_response_iu *iu =
1668                         hisi_sas_status_buf_addr_mem(slot) +
1669                         sizeof(struct hisi_sas_err_record);
1670
1671                 sas_ssp_task_response(dev, task, iu);
1672                 break;
1673         }
1674         case SAS_PROTOCOL_SMP: {
1675                 struct scatterlist *sg_resp = &task->smp_task.smp_resp;
1676                 void *to;
1677
1678                 ts->stat = SAM_STAT_GOOD;
1679                 to = kmap_atomic(sg_page(sg_resp));
1680
1681                 dma_unmap_sg(dev, &task->smp_task.smp_resp, 1,
1682                              DMA_FROM_DEVICE);
1683                 dma_unmap_sg(dev, &task->smp_task.smp_req, 1,
1684                              DMA_TO_DEVICE);
1685                 memcpy(to + sg_resp->offset,
1686                         hisi_sas_status_buf_addr_mem(slot) +
1687                        sizeof(struct hisi_sas_err_record),
1688                        sg_dma_len(sg_resp));
1689                 kunmap_atomic(to);
1690                 break;
1691         }
1692         case SAS_PROTOCOL_SATA:
1693         case SAS_PROTOCOL_STP:
1694         case SAS_PROTOCOL_SATA | SAS_PROTOCOL_STP:
1695                 ts->stat = SAM_STAT_GOOD;
1696                 hisi_sas_sata_done(task, slot);
1697                 break;
1698         default:
1699                 ts->stat = SAM_STAT_CHECK_CONDITION;
1700                 break;
1701         }
1702
1703         if (!slot->port->port_attached) {
1704                 dev_warn(dev, "slot complete: port %d has removed\n",
1705                         slot->port->sas_port.id);
1706                 ts->stat = SAS_PHY_DOWN;
1707         }
1708
1709 out:
1710         hisi_sas_slot_task_free(hisi_hba, task, slot);
1711         sts = ts->stat;
1712         spin_lock_irqsave(&task->task_state_lock, flags);
1713         if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
1714                 spin_unlock_irqrestore(&task->task_state_lock, flags);
1715                 dev_info(dev, "slot complete: task(%p) aborted\n", task);
1716                 return SAS_ABORTED_TASK;
1717         }
1718         task->task_state_flags |= SAS_TASK_STATE_DONE;
1719         spin_unlock_irqrestore(&task->task_state_lock, flags);
1720
1721         if (!is_internal && (task->task_proto != SAS_PROTOCOL_SMP)) {
1722                 spin_lock_irqsave(&device->done_lock, flags);
1723                 if (test_bit(SAS_HA_FROZEN, &ha->state)) {
1724                         spin_unlock_irqrestore(&device->done_lock, flags);
1725                         dev_info(dev, "slot complete: task(%p) ignored\n ",
1726                                  task);
1727                         return sts;
1728                 }
1729                 spin_unlock_irqrestore(&device->done_lock, flags);
1730         }
1731
1732         if (task->task_done)
1733                 task->task_done(task);
1734
1735         return sts;
1736 }
1737
1738 static void cq_tasklet_v3_hw(unsigned long val)
1739 {
1740         struct hisi_sas_cq *cq = (struct hisi_sas_cq *)val;
1741         struct hisi_hba *hisi_hba = cq->hisi_hba;
1742         struct hisi_sas_slot *slot;
1743         struct hisi_sas_complete_v3_hdr *complete_queue;
1744         u32 rd_point = cq->rd_point, wr_point;
1745         int queue = cq->id;
1746
1747         complete_queue = hisi_hba->complete_hdr[queue];
1748
1749         wr_point = hisi_sas_read32(hisi_hba, COMPL_Q_0_WR_PTR +
1750                                    (0x14 * queue));
1751
1752         while (rd_point != wr_point) {
1753                 struct hisi_sas_complete_v3_hdr *complete_hdr;
1754                 struct device *dev = hisi_hba->dev;
1755                 int iptt;
1756
1757                 complete_hdr = &complete_queue[rd_point];
1758
1759                 iptt = (complete_hdr->dw1) & CMPLT_HDR_IPTT_MSK;
1760                 if (likely(iptt < HISI_SAS_COMMAND_ENTRIES_V3_HW)) {
1761                         slot = &hisi_hba->slot_info[iptt];
1762                         slot->cmplt_queue_slot = rd_point;
1763                         slot->cmplt_queue = queue;
1764                         slot_complete_v3_hw(hisi_hba, slot);
1765                 } else
1766                         dev_err(dev, "IPTT %d is invalid, discard it.\n", iptt);
1767
1768                 if (++rd_point >= HISI_SAS_QUEUE_SLOTS)
1769                         rd_point = 0;
1770         }
1771
1772         /* update rd_point */
1773         cq->rd_point = rd_point;
1774         hisi_sas_write32(hisi_hba, COMPL_Q_0_RD_PTR + (0x14 * queue), rd_point);
1775 }
1776
1777 static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
1778 {
1779         struct hisi_sas_cq *cq = p;
1780         struct hisi_hba *hisi_hba = cq->hisi_hba;
1781         int queue = cq->id;
1782
1783         hisi_sas_write32(hisi_hba, OQ_INT_SRC, 1 << queue);
1784
1785         tasklet_schedule(&cq->tasklet);
1786
1787         return IRQ_HANDLED;
1788 }
1789
1790 static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
1791 {
1792         struct device *dev = hisi_hba->dev;
1793         struct pci_dev *pdev = hisi_hba->pci_dev;
1794         int vectors, rc;
1795         int i, k;
1796         int max_msi = HISI_SAS_MSI_COUNT_V3_HW;
1797
1798         vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, 1,
1799                                         max_msi, PCI_IRQ_MSI);
1800         if (vectors < max_msi) {
1801                 dev_err(dev, "could not allocate all msi (%d)\n", vectors);
1802                 return -ENOENT;
1803         }
1804
1805         rc = devm_request_irq(dev, pci_irq_vector(pdev, 1),
1806                               int_phy_up_down_bcast_v3_hw, 0,
1807                               DRV_NAME " phy", hisi_hba);
1808         if (rc) {
1809                 dev_err(dev, "could not request phy interrupt, rc=%d\n", rc);
1810                 rc = -ENOENT;
1811                 goto free_irq_vectors;
1812         }
1813
1814         rc = devm_request_irq(dev, pci_irq_vector(pdev, 2),
1815                               int_chnl_int_v3_hw, 0,
1816                               DRV_NAME " channel", hisi_hba);
1817         if (rc) {
1818                 dev_err(dev, "could not request chnl interrupt, rc=%d\n", rc);
1819                 rc = -ENOENT;
1820                 goto free_phy_irq;
1821         }
1822
1823         rc = devm_request_irq(dev, pci_irq_vector(pdev, 11),
1824                               fatal_axi_int_v3_hw, 0,
1825                               DRV_NAME " fatal", hisi_hba);
1826         if (rc) {
1827                 dev_err(dev, "could not request fatal interrupt, rc=%d\n", rc);
1828                 rc = -ENOENT;
1829                 goto free_chnl_interrupt;
1830         }
1831
1832         /* Init tasklets for cq only */
1833         for (i = 0; i < hisi_hba->queue_count; i++) {
1834                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
1835                 struct tasklet_struct *t = &cq->tasklet;
1836
1837                 rc = devm_request_irq(dev, pci_irq_vector(pdev, i+16),
1838                                           cq_interrupt_v3_hw, 0,
1839                                           DRV_NAME " cq", cq);
1840                 if (rc) {
1841                         dev_err(dev,
1842                                 "could not request cq%d interrupt, rc=%d\n",
1843                                 i, rc);
1844                         rc = -ENOENT;
1845                         goto free_cq_irqs;
1846                 }
1847
1848                 tasklet_init(t, cq_tasklet_v3_hw, (unsigned long)cq);
1849         }
1850
1851         return 0;
1852
1853 free_cq_irqs:
1854         for (k = 0; k < i; k++) {
1855                 struct hisi_sas_cq *cq = &hisi_hba->cq[k];
1856
1857                 free_irq(pci_irq_vector(pdev, k+16), cq);
1858         }
1859         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
1860 free_chnl_interrupt:
1861         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
1862 free_phy_irq:
1863         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
1864 free_irq_vectors:
1865         pci_free_irq_vectors(pdev);
1866         return rc;
1867 }
1868
1869 static int hisi_sas_v3_init(struct hisi_hba *hisi_hba)
1870 {
1871         int rc;
1872
1873         rc = hw_init_v3_hw(hisi_hba);
1874         if (rc)
1875                 return rc;
1876
1877         rc = interrupt_init_v3_hw(hisi_hba);
1878         if (rc)
1879                 return rc;
1880
1881         return 0;
1882 }
1883
1884 static void phy_set_linkrate_v3_hw(struct hisi_hba *hisi_hba, int phy_no,
1885                 struct sas_phy_linkrates *r)
1886 {
1887         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1888         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1889         enum sas_linkrate min, max;
1890         u32 prog_phy_link_rate = 0x800;
1891
1892         if (r->maximum_linkrate == SAS_LINK_RATE_UNKNOWN) {
1893                 max = sas_phy->phy->maximum_linkrate;
1894                 min = r->minimum_linkrate;
1895         } else if (r->minimum_linkrate == SAS_LINK_RATE_UNKNOWN) {
1896                 max = r->maximum_linkrate;
1897                 min = sas_phy->phy->minimum_linkrate;
1898         } else
1899                 return;
1900
1901         sas_phy->phy->maximum_linkrate = max;
1902         sas_phy->phy->minimum_linkrate = min;
1903         prog_phy_link_rate |= hisi_sas_get_prog_phy_linkrate_mask(max);
1904
1905         disable_phy_v3_hw(hisi_hba, phy_no);
1906         msleep(100);
1907         hisi_sas_phy_write32(hisi_hba, phy_no, PROG_PHY_LINK_RATE,
1908                         prog_phy_link_rate);
1909         start_phy_v3_hw(hisi_hba, phy_no);
1910 }
1911
1912 static void interrupt_disable_v3_hw(struct hisi_hba *hisi_hba)
1913 {
1914         struct pci_dev *pdev = hisi_hba->pci_dev;
1915         int i;
1916
1917         synchronize_irq(pci_irq_vector(pdev, 1));
1918         synchronize_irq(pci_irq_vector(pdev, 2));
1919         synchronize_irq(pci_irq_vector(pdev, 11));
1920         for (i = 0; i < hisi_hba->queue_count; i++) {
1921                 hisi_sas_write32(hisi_hba, OQ0_INT_SRC_MSK + 0x4 * i, 0x1);
1922                 synchronize_irq(pci_irq_vector(pdev, i + 16));
1923         }
1924
1925         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xffffffff);
1926         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xffffffff);
1927         hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffffffff);
1928         hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0xffffffff);
1929
1930         for (i = 0; i < hisi_hba->n_phy; i++) {
1931                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xffffffff);
1932                 hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffffff);
1933                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x1);
1934                 hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_PHY_ENA_MSK, 0x1);
1935                 hisi_sas_phy_write32(hisi_hba, i, SL_RX_BCAST_CHK_MSK, 0x1);
1936         }
1937 }
1938
1939 static u32 get_phys_state_v3_hw(struct hisi_hba *hisi_hba)
1940 {
1941         return hisi_sas_read32(hisi_hba, PHY_STATE);
1942 }
1943
1944 static void phy_get_events_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
1945 {
1946         struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
1947         struct asd_sas_phy *sas_phy = &phy->sas_phy;
1948         struct sas_phy *sphy = sas_phy->phy;
1949         u32 reg_value;
1950
1951         /* loss dword sync */
1952         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DWS_LOST);
1953         sphy->loss_of_dword_sync_count += reg_value;
1954
1955         /* phy reset problem */
1956         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_RESET_PROB);
1957         sphy->phy_reset_problem_count += reg_value;
1958
1959         /* invalid dword */
1960         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_INVLD_DW);
1961         sphy->invalid_dword_count += reg_value;
1962
1963         /* disparity err */
1964         reg_value = hisi_sas_phy_read32(hisi_hba, phy_no, ERR_CNT_DISP_ERR);
1965         sphy->running_disparity_error_count += reg_value;
1966
1967 }
1968
1969 static int soft_reset_v3_hw(struct hisi_hba *hisi_hba)
1970 {
1971         struct device *dev = hisi_hba->dev;
1972         int rc;
1973         u32 status;
1974
1975         interrupt_disable_v3_hw(hisi_hba);
1976         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
1977         hisi_sas_kill_tasklets(hisi_hba);
1978
1979         hisi_sas_stop_phys(hisi_hba);
1980
1981         mdelay(10);
1982
1983         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE + AM_CTRL_GLOBAL, 0x1);
1984
1985         /* wait until bus idle */
1986         rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
1987                 AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
1988         if (rc) {
1989                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
1990                 return rc;
1991         }
1992
1993         hisi_sas_init_mem(hisi_hba);
1994
1995         return hw_init_v3_hw(hisi_hba);
1996 }
1997
1998 static const struct hisi_sas_hw hisi_sas_v3_hw = {
1999         .hw_init = hisi_sas_v3_init,
2000         .setup_itct = setup_itct_v3_hw,
2001         .max_command_entries = HISI_SAS_COMMAND_ENTRIES_V3_HW,
2002         .get_wideport_bitmap = get_wideport_bitmap_v3_hw,
2003         .complete_hdr_size = sizeof(struct hisi_sas_complete_v3_hdr),
2004         .clear_itct = clear_itct_v3_hw,
2005         .sl_notify = sl_notify_v3_hw,
2006         .prep_ssp = prep_ssp_v3_hw,
2007         .prep_smp = prep_smp_v3_hw,
2008         .prep_stp = prep_ata_v3_hw,
2009         .prep_abort = prep_abort_v3_hw,
2010         .get_free_slot = get_free_slot_v3_hw,
2011         .start_delivery = start_delivery_v3_hw,
2012         .slot_complete = slot_complete_v3_hw,
2013         .phys_init = phys_init_v3_hw,
2014         .phy_start = start_phy_v3_hw,
2015         .phy_disable = disable_phy_v3_hw,
2016         .phy_hard_reset = phy_hard_reset_v3_hw,
2017         .phy_get_max_linkrate = phy_get_max_linkrate_v3_hw,
2018         .phy_set_linkrate = phy_set_linkrate_v3_hw,
2019         .dereg_device = dereg_device_v3_hw,
2020         .soft_reset = soft_reset_v3_hw,
2021         .get_phys_state = get_phys_state_v3_hw,
2022         .get_events = phy_get_events_v3_hw,
2023 };
2024
2025 static struct Scsi_Host *
2026 hisi_sas_shost_alloc_pci(struct pci_dev *pdev)
2027 {
2028         struct Scsi_Host *shost;
2029         struct hisi_hba *hisi_hba;
2030         struct device *dev = &pdev->dev;
2031
2032         shost = scsi_host_alloc(hisi_sas_sht, sizeof(*hisi_hba));
2033         if (!shost) {
2034                 dev_err(dev, "shost alloc failed\n");
2035                 return NULL;
2036         }
2037         hisi_hba = shost_priv(shost);
2038
2039         INIT_WORK(&hisi_hba->rst_work, hisi_sas_rst_work_handler);
2040         hisi_hba->hw = &hisi_sas_v3_hw;
2041         hisi_hba->pci_dev = pdev;
2042         hisi_hba->dev = dev;
2043         hisi_hba->shost = shost;
2044         SHOST_TO_SAS_HA(shost) = &hisi_hba->sha;
2045
2046         timer_setup(&hisi_hba->timer, NULL, 0);
2047
2048         if (hisi_sas_get_fw_info(hisi_hba) < 0)
2049                 goto err_out;
2050
2051         if (hisi_sas_alloc(hisi_hba, shost)) {
2052                 hisi_sas_free(hisi_hba);
2053                 goto err_out;
2054         }
2055
2056         return shost;
2057 err_out:
2058         scsi_host_put(shost);
2059         dev_err(dev, "shost alloc failed\n");
2060         return NULL;
2061 }
2062
2063 static int
2064 hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2065 {
2066         struct Scsi_Host *shost;
2067         struct hisi_hba *hisi_hba;
2068         struct device *dev = &pdev->dev;
2069         struct asd_sas_phy **arr_phy;
2070         struct asd_sas_port **arr_port;
2071         struct sas_ha_struct *sha;
2072         int rc, phy_nr, port_nr, i;
2073
2074         rc = pci_enable_device(pdev);
2075         if (rc)
2076                 goto err_out;
2077
2078         pci_set_master(pdev);
2079
2080         rc = pci_request_regions(pdev, DRV_NAME);
2081         if (rc)
2082                 goto err_out_disable_device;
2083
2084         if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) ||
2085             (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0)) {
2086                 if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) ||
2087                    (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)) != 0)) {
2088                         dev_err(dev, "No usable DMA addressing method\n");
2089                         rc = -EIO;
2090                         goto err_out_regions;
2091                 }
2092         }
2093
2094         shost = hisi_sas_shost_alloc_pci(pdev);
2095         if (!shost) {
2096                 rc = -ENOMEM;
2097                 goto err_out_regions;
2098         }
2099
2100         sha = SHOST_TO_SAS_HA(shost);
2101         hisi_hba = shost_priv(shost);
2102         dev_set_drvdata(dev, sha);
2103
2104         hisi_hba->regs = pcim_iomap(pdev, 5, 0);
2105         if (!hisi_hba->regs) {
2106                 dev_err(dev, "cannot map register.\n");
2107                 rc = -ENOMEM;
2108                 goto err_out_ha;
2109         }
2110
2111         phy_nr = port_nr = hisi_hba->n_phy;
2112
2113         arr_phy = devm_kcalloc(dev, phy_nr, sizeof(void *), GFP_KERNEL);
2114         arr_port = devm_kcalloc(dev, port_nr, sizeof(void *), GFP_KERNEL);
2115         if (!arr_phy || !arr_port) {
2116                 rc = -ENOMEM;
2117                 goto err_out_ha;
2118         }
2119
2120         sha->sas_phy = arr_phy;
2121         sha->sas_port = arr_port;
2122         sha->core.shost = shost;
2123         sha->lldd_ha = hisi_hba;
2124
2125         shost->transportt = hisi_sas_stt;
2126         shost->max_id = HISI_SAS_MAX_DEVICES;
2127         shost->max_lun = ~0;
2128         shost->max_channel = 1;
2129         shost->max_cmd_len = 16;
2130         shost->sg_tablesize = min_t(u16, SG_ALL, HISI_SAS_SGE_PAGE_CNT);
2131         shost->can_queue = hisi_hba->hw->max_command_entries;
2132         shost->cmd_per_lun = hisi_hba->hw->max_command_entries;
2133
2134         sha->sas_ha_name = DRV_NAME;
2135         sha->dev = dev;
2136         sha->lldd_module = THIS_MODULE;
2137         sha->sas_addr = &hisi_hba->sas_addr[0];
2138         sha->num_phys = hisi_hba->n_phy;
2139         sha->core.shost = hisi_hba->shost;
2140
2141         for (i = 0; i < hisi_hba->n_phy; i++) {
2142                 sha->sas_phy[i] = &hisi_hba->phy[i].sas_phy;
2143                 sha->sas_port[i] = &hisi_hba->port[i].sas_port;
2144         }
2145
2146         rc = scsi_add_host(shost, dev);
2147         if (rc)
2148                 goto err_out_ha;
2149
2150         rc = sas_register_ha(sha);
2151         if (rc)
2152                 goto err_out_register_ha;
2153
2154         rc = hisi_hba->hw->hw_init(hisi_hba);
2155         if (rc)
2156                 goto err_out_register_ha;
2157
2158         scsi_scan_host(shost);
2159
2160         return 0;
2161
2162 err_out_register_ha:
2163         scsi_remove_host(shost);
2164 err_out_ha:
2165         scsi_host_put(shost);
2166 err_out_regions:
2167         pci_release_regions(pdev);
2168 err_out_disable_device:
2169         pci_disable_device(pdev);
2170 err_out:
2171         return rc;
2172 }
2173
2174 static void
2175 hisi_sas_v3_destroy_irqs(struct pci_dev *pdev, struct hisi_hba *hisi_hba)
2176 {
2177         int i;
2178
2179         free_irq(pci_irq_vector(pdev, 1), hisi_hba);
2180         free_irq(pci_irq_vector(pdev, 2), hisi_hba);
2181         free_irq(pci_irq_vector(pdev, 11), hisi_hba);
2182         for (i = 0; i < hisi_hba->queue_count; i++) {
2183                 struct hisi_sas_cq *cq = &hisi_hba->cq[i];
2184
2185                 free_irq(pci_irq_vector(pdev, i+16), cq);
2186         }
2187         pci_free_irq_vectors(pdev);
2188 }
2189
2190 static void hisi_sas_v3_remove(struct pci_dev *pdev)
2191 {
2192         struct device *dev = &pdev->dev;
2193         struct sas_ha_struct *sha = dev_get_drvdata(dev);
2194         struct hisi_hba *hisi_hba = sha->lldd_ha;
2195         struct Scsi_Host *shost = sha->core.shost;
2196
2197         if (timer_pending(&hisi_hba->timer))
2198                 del_timer(&hisi_hba->timer);
2199
2200         sas_unregister_ha(sha);
2201         sas_remove_host(sha->core.shost);
2202
2203         hisi_sas_v3_destroy_irqs(pdev, hisi_hba);
2204         hisi_sas_kill_tasklets(hisi_hba);
2205         pci_release_regions(pdev);
2206         pci_disable_device(pdev);
2207         hisi_sas_free(hisi_hba);
2208         scsi_host_put(shost);
2209 }
2210
2211 static const struct hisi_sas_hw_error sas_ras_intr0_nfe[] = {
2212         { .irq_msk = BIT(19), .msg = "HILINK_INT" },
2213         { .irq_msk = BIT(20), .msg = "HILINK_PLL0_OUT_OF_LOCK" },
2214         { .irq_msk = BIT(21), .msg = "HILINK_PLL1_OUT_OF_LOCK" },
2215         { .irq_msk = BIT(22), .msg = "HILINK_LOSS_OF_REFCLK0" },
2216         { .irq_msk = BIT(23), .msg = "HILINK_LOSS_OF_REFCLK1" },
2217         { .irq_msk = BIT(24), .msg = "DMAC0_TX_POISON" },
2218         { .irq_msk = BIT(25), .msg = "DMAC1_TX_POISON" },
2219         { .irq_msk = BIT(26), .msg = "DMAC2_TX_POISON" },
2220         { .irq_msk = BIT(27), .msg = "DMAC3_TX_POISON" },
2221         { .irq_msk = BIT(28), .msg = "DMAC4_TX_POISON" },
2222         { .irq_msk = BIT(29), .msg = "DMAC5_TX_POISON" },
2223         { .irq_msk = BIT(30), .msg = "DMAC6_TX_POISON" },
2224         { .irq_msk = BIT(31), .msg = "DMAC7_TX_POISON" },
2225 };
2226
2227 static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
2228         { .irq_msk = BIT(0), .msg = "RXM_CFG_MEM3_ECC2B_INTR" },
2229         { .irq_msk = BIT(1), .msg = "RXM_CFG_MEM2_ECC2B_INTR" },
2230         { .irq_msk = BIT(2), .msg = "RXM_CFG_MEM1_ECC2B_INTR" },
2231         { .irq_msk = BIT(3), .msg = "RXM_CFG_MEM0_ECC2B_INTR" },
2232         { .irq_msk = BIT(4), .msg = "HGC_CQE_ECC2B_INTR" },
2233         { .irq_msk = BIT(5), .msg = "LM_CFG_IOSTL_ECC2B_INTR" },
2234         { .irq_msk = BIT(6), .msg = "LM_CFG_ITCTL_ECC2B_INTR" },
2235         { .irq_msk = BIT(7), .msg = "HGC_ITCT_ECC2B_INTR" },
2236         { .irq_msk = BIT(8), .msg = "HGC_IOST_ECC2B_INTR" },
2237         { .irq_msk = BIT(9), .msg = "HGC_DQE_ECC2B_INTR" },
2238         { .irq_msk = BIT(10), .msg = "DMAC0_RAM_ECC2B_INTR" },
2239         { .irq_msk = BIT(11), .msg = "DMAC1_RAM_ECC2B_INTR" },
2240         { .irq_msk = BIT(12), .msg = "DMAC2_RAM_ECC2B_INTR" },
2241         { .irq_msk = BIT(13), .msg = "DMAC3_RAM_ECC2B_INTR" },
2242         { .irq_msk = BIT(14), .msg = "DMAC4_RAM_ECC2B_INTR" },
2243         { .irq_msk = BIT(15), .msg = "DMAC5_RAM_ECC2B_INTR" },
2244         { .irq_msk = BIT(16), .msg = "DMAC6_RAM_ECC2B_INTR" },
2245         { .irq_msk = BIT(17), .msg = "DMAC7_RAM_ECC2B_INTR" },
2246         { .irq_msk = BIT(18), .msg = "OOO_RAM_ECC2B_INTR" },
2247         { .irq_msk = BIT(20), .msg = "HGC_DQE_POISON_INTR" },
2248         { .irq_msk = BIT(21), .msg = "HGC_IOST_POISON_INTR" },
2249         { .irq_msk = BIT(22), .msg = "HGC_ITCT_POISON_INTR" },
2250         { .irq_msk = BIT(23), .msg = "HGC_ITCT_NCQ_POISON_INTR" },
2251         { .irq_msk = BIT(24), .msg = "DMAC0_RX_POISON" },
2252         { .irq_msk = BIT(25), .msg = "DMAC1_RX_POISON" },
2253         { .irq_msk = BIT(26), .msg = "DMAC2_RX_POISON" },
2254         { .irq_msk = BIT(27), .msg = "DMAC3_RX_POISON" },
2255         { .irq_msk = BIT(28), .msg = "DMAC4_RX_POISON" },
2256         { .irq_msk = BIT(29), .msg = "DMAC5_RX_POISON" },
2257         { .irq_msk = BIT(30), .msg = "DMAC6_RX_POISON" },
2258         { .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
2259 };
2260
2261 static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
2262         { .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
2263         { .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
2264         { .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
2265         { .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
2266         { .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
2267         { .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
2268         { .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
2269         { .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
2270         { .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
2271         { .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
2272         { .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
2273         { .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
2274         { .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
2275         { .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
2276         { .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
2277         { .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
2278         { .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
2279         { .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
2280         { .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
2281         { .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
2282 };
2283
2284 static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
2285 {
2286         struct device *dev = hisi_hba->dev;
2287         const struct hisi_sas_hw_error *ras_error;
2288         bool need_reset = false;
2289         u32 irq_value;
2290         int i;
2291
2292         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR0);
2293         for (i = 0; i < ARRAY_SIZE(sas_ras_intr0_nfe); i++) {
2294                 ras_error = &sas_ras_intr0_nfe[i];
2295                 if (ras_error->irq_msk & irq_value) {
2296                         dev_warn(dev, "SAS_RAS_INTR0: %s(irq_value=0x%x) found.\n",
2297                                         ras_error->msg, irq_value);
2298                         need_reset = true;
2299                 }
2300         }
2301         hisi_sas_write32(hisi_hba, SAS_RAS_INTR0, irq_value);
2302
2303         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR1);
2304         for (i = 0; i < ARRAY_SIZE(sas_ras_intr1_nfe); i++) {
2305                 ras_error = &sas_ras_intr1_nfe[i];
2306                 if (ras_error->irq_msk & irq_value) {
2307                         dev_warn(dev, "SAS_RAS_INTR1: %s(irq_value=0x%x) found.\n",
2308                                         ras_error->msg, irq_value);
2309                         need_reset = true;
2310                 }
2311         }
2312         hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
2313
2314         irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
2315         for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
2316                 ras_error = &sas_ras_intr2_nfe[i];
2317                 if (ras_error->irq_msk & irq_value) {
2318                         dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
2319                                         ras_error->msg, irq_value);
2320                         need_reset = true;
2321                 }
2322         }
2323         hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
2324
2325         return need_reset;
2326 }
2327
2328 static pci_ers_result_t hisi_sas_error_detected_v3_hw(struct pci_dev *pdev,
2329                 pci_channel_state_t state)
2330 {
2331         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2332         struct hisi_hba *hisi_hba = sha->lldd_ha;
2333         struct device *dev = hisi_hba->dev;
2334
2335         dev_info(dev, "PCI error: detected callback, state(%d)!!\n", state);
2336         if (state == pci_channel_io_perm_failure)
2337                 return PCI_ERS_RESULT_DISCONNECT;
2338
2339         if (process_non_fatal_error_v3_hw(hisi_hba))
2340                 return PCI_ERS_RESULT_NEED_RESET;
2341
2342         return PCI_ERS_RESULT_CAN_RECOVER;
2343 }
2344
2345 static pci_ers_result_t hisi_sas_mmio_enabled_v3_hw(struct pci_dev *pdev)
2346 {
2347         return PCI_ERS_RESULT_RECOVERED;
2348 }
2349
2350 static pci_ers_result_t hisi_sas_slot_reset_v3_hw(struct pci_dev *pdev)
2351 {
2352         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2353         struct hisi_hba *hisi_hba = sha->lldd_ha;
2354         struct device *dev = hisi_hba->dev;
2355         HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
2356
2357         dev_info(dev, "PCI error: slot reset callback!!\n");
2358         queue_work(hisi_hba->wq, &r.work);
2359         wait_for_completion(r.completion);
2360         if (r.done)
2361                 return PCI_ERS_RESULT_RECOVERED;
2362
2363         return PCI_ERS_RESULT_DISCONNECT;
2364 }
2365
2366 enum {
2367         /* instances of the controller */
2368         hip08,
2369 };
2370
2371 static int hisi_sas_v3_suspend(struct pci_dev *pdev, pm_message_t state)
2372 {
2373         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2374         struct hisi_hba *hisi_hba = sha->lldd_ha;
2375         struct device *dev = hisi_hba->dev;
2376         struct Scsi_Host *shost = hisi_hba->shost;
2377         u32 device_state, status;
2378         int rc;
2379         u32 reg_val;
2380         unsigned long flags;
2381
2382         if (!pdev->pm_cap) {
2383                 dev_err(dev, "PCI PM not supported\n");
2384                 return -ENODEV;
2385         }
2386
2387         set_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2388         scsi_block_requests(shost);
2389         set_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2390         flush_workqueue(hisi_hba->wq);
2391         /* disable DQ/PHY/bus */
2392         interrupt_disable_v3_hw(hisi_hba);
2393         hisi_sas_write32(hisi_hba, DLVRY_QUEUE_ENABLE, 0x0);
2394         hisi_sas_kill_tasklets(hisi_hba);
2395
2396         hisi_sas_stop_phys(hisi_hba);
2397
2398         reg_val = hisi_sas_read32(hisi_hba, AXI_MASTER_CFG_BASE +
2399                 AM_CTRL_GLOBAL);
2400         reg_val |= 0x1;
2401         hisi_sas_write32(hisi_hba, AXI_MASTER_CFG_BASE +
2402                 AM_CTRL_GLOBAL, reg_val);
2403
2404         /* wait until bus idle */
2405         rc = readl_poll_timeout(hisi_hba->regs + AXI_MASTER_CFG_BASE +
2406                 AM_CURR_TRANS_RETURN, status, status == 0x3, 10, 100);
2407         if (rc) {
2408                 dev_err(dev, "axi bus is not idle, rc = %d\n", rc);
2409                 clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2410                 clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2411                 scsi_unblock_requests(shost);
2412                 return rc;
2413         }
2414
2415         hisi_sas_init_mem(hisi_hba);
2416
2417         device_state = pci_choose_state(pdev, state);
2418         dev_warn(dev, "entering operating state [D%d]\n",
2419                         device_state);
2420         pci_save_state(pdev);
2421         pci_disable_device(pdev);
2422         pci_set_power_state(pdev, device_state);
2423
2424         spin_lock_irqsave(&hisi_hba->lock, flags);
2425         hisi_sas_release_tasks(hisi_hba);
2426         spin_unlock_irqrestore(&hisi_hba->lock, flags);
2427
2428         sas_suspend_ha(sha);
2429         return 0;
2430 }
2431
2432 static int hisi_sas_v3_resume(struct pci_dev *pdev)
2433 {
2434         struct sas_ha_struct *sha = pci_get_drvdata(pdev);
2435         struct hisi_hba *hisi_hba = sha->lldd_ha;
2436         struct Scsi_Host *shost = hisi_hba->shost;
2437         struct device *dev = hisi_hba->dev;
2438         unsigned int rc;
2439         u32 device_state = pdev->current_state;
2440
2441         dev_warn(dev, "resuming from operating state [D%d]\n",
2442                         device_state);
2443         pci_set_power_state(pdev, PCI_D0);
2444         pci_enable_wake(pdev, PCI_D0, 0);
2445         pci_restore_state(pdev);
2446         rc = pci_enable_device(pdev);
2447         if (rc)
2448                 dev_err(dev, "enable device failed during resume (%d)\n", rc);
2449
2450         pci_set_master(pdev);
2451         scsi_unblock_requests(shost);
2452         clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
2453
2454         sas_prep_resume_ha(sha);
2455         init_reg_v3_hw(hisi_hba);
2456         hisi_hba->hw->phys_init(hisi_hba);
2457         sas_resume_ha(sha);
2458         clear_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags);
2459
2460         return 0;
2461 }
2462
2463 static const struct pci_device_id sas_v3_pci_table[] = {
2464         { PCI_VDEVICE(HUAWEI, 0xa230), hip08 },
2465         {}
2466 };
2467 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
2468
2469 static const struct pci_error_handlers hisi_sas_err_handler = {
2470         .error_detected = hisi_sas_error_detected_v3_hw,
2471         .mmio_enabled   = hisi_sas_mmio_enabled_v3_hw,
2472         .slot_reset     = hisi_sas_slot_reset_v3_hw,
2473 };
2474
2475 static struct pci_driver sas_v3_pci_driver = {
2476         .name           = DRV_NAME,
2477         .id_table       = sas_v3_pci_table,
2478         .probe          = hisi_sas_v3_probe,
2479         .remove         = hisi_sas_v3_remove,
2480         .suspend        = hisi_sas_v3_suspend,
2481         .resume         = hisi_sas_v3_resume,
2482         .err_handler    = &hisi_sas_err_handler,
2483 };
2484
2485 module_pci_driver(sas_v3_pci_driver);
2486
2487 MODULE_LICENSE("GPL");
2488 MODULE_AUTHOR("John Garry <john.garry@huawei.com>");
2489 MODULE_DESCRIPTION("HISILICON SAS controller v3 hw driver based on pci device");
2490 MODULE_ALIAS("pci:" DRV_NAME);