drivers/crypto/hisilicon/qm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 HiSilicon Limited. */
   3 #include <asm/page.h>
   4 #include <linux/acpi.h>
   5 #include <linux/aer.h>
   6 #include <linux/bitmap.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/idr.h>
   9 #include <linux/io.h>
  10 #include <linux/irqreturn.h>
  11 #include <linux/log2.h>
  12 #include <linux/pm_runtime.h>
  13 #include <linux/seq_file.h>
  14 #include <linux/slab.h>
  15 #include <linux/uacce.h>
  16 #include <linux/uaccess.h>
  17 #include <uapi/misc/uacce/hisi_qm.h>
  18 #include "qm.h"
  19
  20 /* eq/aeq irq enable */
  21 #define QM_VF_AEQ_INT_SOURCE            0x0
  22 #define QM_VF_AEQ_INT_MASK              0x4
  23 #define QM_VF_EQ_INT_SOURCE             0x8
  24 #define QM_VF_EQ_INT_MASK               0xc
  25 #define QM_IRQ_NUM_V1                   1
  26 #define QM_IRQ_NUM_PF_V2                4
  27 #define QM_IRQ_NUM_VF_V2                2
  28 #define QM_IRQ_NUM_VF_V3                3
  29
  30 #define QM_EQ_EVENT_IRQ_VECTOR          0
  31 #define QM_AEQ_EVENT_IRQ_VECTOR         1
  32 #define QM_CMD_EVENT_IRQ_VECTOR         2
  33 #define QM_ABNORMAL_EVENT_IRQ_VECTOR    3
  34
  35 /* mailbox */
  36 #define QM_MB_CMD_SQC                   0x0
  37 #define QM_MB_CMD_CQC                   0x1
  38 #define QM_MB_CMD_EQC                   0x2
  39 #define QM_MB_CMD_AEQC                  0x3
  40 #define QM_MB_CMD_SQC_BT                0x4
  41 #define QM_MB_CMD_CQC_BT                0x5
  42 #define QM_MB_CMD_SQC_VFT_V2            0x6
  43 #define QM_MB_CMD_STOP_QP               0x8
  44 #define QM_MB_CMD_SRC                   0xc
  45 #define QM_MB_CMD_DST                   0xd
  46
  47 #define QM_MB_CMD_SEND_BASE             0x300
  48 #define QM_MB_EVENT_SHIFT               8
  49 #define QM_MB_BUSY_SHIFT                13
  50 #define QM_MB_OP_SHIFT                  14
  51 #define QM_MB_CMD_DATA_ADDR_L           0x304
  52 #define QM_MB_CMD_DATA_ADDR_H           0x308
  53 #define QM_MB_PING_ALL_VFS              0xffff
  54 #define QM_MB_CMD_DATA_SHIFT            32
  55 #define QM_MB_CMD_DATA_MASK             GENMASK(31, 0)
  56
  57 /* sqc shift */
  58 #define QM_SQ_HOP_NUM_SHIFT             0
  59 #define QM_SQ_PAGE_SIZE_SHIFT           4
  60 #define QM_SQ_BUF_SIZE_SHIFT            8
  61 #define QM_SQ_SQE_SIZE_SHIFT            12
  62 #define QM_SQ_PRIORITY_SHIFT            0
  63 #define QM_SQ_ORDERS_SHIFT              4
  64 #define QM_SQ_TYPE_SHIFT                8
  65 #define QM_QC_PASID_ENABLE              0x1
  66 #define QM_QC_PASID_ENABLE_SHIFT        7
  67
  68 #define QM_SQ_TYPE_MASK                 GENMASK(3, 0)
  69 #define QM_SQ_TAIL_IDX(sqc)             ((le16_to_cpu((sqc)->w11) >> 6) & 0x1)
  70
  71 /* cqc shift */
  72 #define QM_CQ_HOP_NUM_SHIFT             0
  73 #define QM_CQ_PAGE_SIZE_SHIFT           4
  74 #define QM_CQ_BUF_SIZE_SHIFT            8
  75 #define QM_CQ_CQE_SIZE_SHIFT            12
  76 #define QM_CQ_PHASE_SHIFT               0
  77 #define QM_CQ_FLAG_SHIFT                1
  78
  79 #define QM_CQE_PHASE(cqe)               (le16_to_cpu((cqe)->w7) & 0x1)
  80 #define QM_QC_CQE_SIZE                  4
  81 #define QM_CQ_TAIL_IDX(cqc)             ((le16_to_cpu((cqc)->w11) >> 6) & 0x1)
  82
  83 /* eqc shift */
  84 #define QM_EQE_AEQE_SIZE                (2UL << 12)
  85 #define QM_EQC_PHASE_SHIFT              16
  86
  87 #define QM_EQE_PHASE(eqe)               ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)
  88 #define QM_EQE_CQN_MASK                 GENMASK(15, 0)
  89
  90 #define QM_AEQE_PHASE(aeqe)             ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
  91 #define QM_AEQE_TYPE_SHIFT              17
  92
  93 #define QM_DOORBELL_CMD_SQ              0
  94 #define QM_DOORBELL_CMD_CQ              1
  95 #define QM_DOORBELL_CMD_EQ              2
  96 #define QM_DOORBELL_CMD_AEQ             3
  97
  98 #define QM_DOORBELL_BASE_V1             0x340
  99 #define QM_DB_CMD_SHIFT_V1              16
 100 #define QM_DB_INDEX_SHIFT_V1            32
 101 #define QM_DB_PRIORITY_SHIFT_V1         48
 102 #define QM_DOORBELL_SQ_CQ_BASE_V2       0x1000
 103 #define QM_DOORBELL_EQ_AEQ_BASE_V2      0x2000
 104 #define QM_QUE_ISO_CFG_V                0x0030
 105 #define QM_PAGE_SIZE                    0x0034
 106 #define QM_QUE_ISO_EN                   0x100154
 107 #define QM_CAPBILITY                    0x100158
 108 #define QM_QP_NUN_MASK                  GENMASK(10, 0)
 109 #define QM_QP_DB_INTERVAL               0x10000
 110 #define QM_QP_MAX_NUM_SHIFT             11
 111 #define QM_DB_CMD_SHIFT_V2              12
 112 #define QM_DB_RAND_SHIFT_V2             16
 113 #define QM_DB_INDEX_SHIFT_V2            32
 114 #define QM_DB_PRIORITY_SHIFT_V2         48
 115
 116 #define QM_MEM_START_INIT               0x100040
 117 #define QM_MEM_INIT_DONE                0x100044
 118 #define QM_VFT_CFG_RDY                  0x10006c
 119 #define QM_VFT_CFG_OP_WR                0x100058
 120 #define QM_VFT_CFG_TYPE                 0x10005c
 121 #define QM_SQC_VFT                      0x0
 122 #define QM_CQC_VFT                      0x1
 123 #define QM_VFT_CFG                      0x100060
 124 #define QM_VFT_CFG_OP_ENABLE            0x100054
 125
 126 #define QM_VFT_CFG_DATA_L               0x100064
 127 #define QM_VFT_CFG_DATA_H               0x100068
 128 #define QM_SQC_VFT_BUF_SIZE             (7ULL << 8)
 129 #define QM_SQC_VFT_SQC_SIZE             (5ULL << 12)
 130 #define QM_SQC_VFT_INDEX_NUMBER         (1ULL << 16)
 131 #define QM_SQC_VFT_START_SQN_SHIFT      28
 132 #define QM_SQC_VFT_VALID                (1ULL << 44)
 133 #define QM_SQC_VFT_SQN_SHIFT            45
 134 #define QM_CQC_VFT_BUF_SIZE             (7ULL << 8)
 135 #define QM_CQC_VFT_SQC_SIZE             (5ULL << 12)
 136 #define QM_CQC_VFT_INDEX_NUMBER         (1ULL << 16)
 137 #define QM_CQC_VFT_VALID                (1ULL << 28)
 138
 139 #define QM_SQC_VFT_BASE_SHIFT_V2        28
 140 #define QM_SQC_VFT_BASE_MASK_V2         GENMASK(15, 0)
 141 #define QM_SQC_VFT_NUM_SHIFT_V2         45
 142 #define QM_SQC_VFT_NUM_MASK_v2          GENMASK(9, 0)
 143
 144 #define QM_DFX_CNT_CLR_CE               0x100118
 145
 146 #define QM_ABNORMAL_INT_SOURCE          0x100000
 147 #define QM_ABNORMAL_INT_SOURCE_CLR      GENMASK(14, 0)
 148 #define QM_ABNORMAL_INT_MASK            0x100004
 149 #define QM_ABNORMAL_INT_MASK_VALUE      0x7fff
 150 #define QM_ABNORMAL_INT_STATUS          0x100008
 151 #define QM_ABNORMAL_INT_SET             0x10000c
 152 #define QM_ABNORMAL_INF00               0x100010
 153 #define QM_FIFO_OVERFLOW_TYPE           0xc0
 154 #define QM_FIFO_OVERFLOW_TYPE_SHIFT     6
 155 #define QM_FIFO_OVERFLOW_VF             0x3f
 156 #define QM_ABNORMAL_INF01               0x100014
 157 #define QM_DB_TIMEOUT_TYPE              0xc0
 158 #define QM_DB_TIMEOUT_TYPE_SHIFT        6
 159 #define QM_DB_TIMEOUT_VF                0x3f
 160 #define QM_RAS_CE_ENABLE                0x1000ec
 161 #define QM_RAS_FE_ENABLE                0x1000f0
 162 #define QM_RAS_NFE_ENABLE               0x1000f4
 163 #define QM_RAS_CE_THRESHOLD             0x1000f8
 164 #define QM_RAS_CE_TIMES_PER_IRQ         1
 165 #define QM_RAS_MSI_INT_SEL              0x1040f4
 166 #define QM_OOO_SHUTDOWN_SEL             0x1040f8
 167
 168 #define QM_RESET_WAIT_TIMEOUT           400
 169 #define QM_PEH_VENDOR_ID                0x1000d8
 170 #define ACC_VENDOR_ID_VALUE             0x5a5a
 171 #define QM_PEH_DFX_INFO0                0x1000fc
 172 #define QM_PEH_DFX_INFO1                0x100100
 173 #define QM_PEH_DFX_MASK                 (BIT(0) | BIT(2))
 174 #define QM_PEH_MSI_FINISH_MASK          GENMASK(19, 16)
 175 #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3
 176 #define ACC_PEH_MSI_DISABLE             GENMASK(31, 0)
 177 #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1
 178 #define ACC_MASTER_TRANS_RETURN_RW      3
 179 #define ACC_MASTER_TRANS_RETURN         0x300150
 180 #define ACC_MASTER_GLOBAL_CTRL          0x300000
 181 #define ACC_AM_CFG_PORT_WR_EN           0x30001c
 182 #define QM_RAS_NFE_MBIT_DISABLE         ~QM_ECC_MBIT
 183 #define ACC_AM_ROB_ECC_INT_STS          0x300104
 184 #define ACC_ROB_ECC_ERR_MULTPL          BIT(1)
 185 #define QM_MSI_CAP_ENABLE               BIT(16)
 186
 187 /* interfunction communication */
 188 #define QM_IFC_READY_STATUS             0x100128
 189 #define QM_IFC_C_STS_M                  0x10012C
 190 #define QM_IFC_INT_SET_P                0x100130
 191 #define QM_IFC_INT_CFG                  0x100134
 192 #define QM_IFC_INT_SOURCE_P             0x100138
 193 #define QM_IFC_INT_SOURCE_V             0x0020
 194 #define QM_IFC_INT_MASK                 0x0024
 195 #define QM_IFC_INT_STATUS               0x0028
 196 #define QM_IFC_INT_SET_V                0x002C
 197 #define QM_IFC_SEND_ALL_VFS             GENMASK(6, 0)
 198 #define QM_IFC_INT_SOURCE_CLR           GENMASK(63, 0)
 199 #define QM_IFC_INT_SOURCE_MASK          BIT(0)
 200 #define QM_IFC_INT_DISABLE              BIT(0)
 201 #define QM_IFC_INT_STATUS_MASK          BIT(0)
 202 #define QM_IFC_INT_SET_MASK             BIT(0)
 203 #define QM_WAIT_DST_ACK                 10
 204 #define QM_MAX_PF_WAIT_COUNT            10
 205 #define QM_MAX_VF_WAIT_COUNT            40
 206 #define QM_VF_RESET_WAIT_US            20000
 207 #define QM_VF_RESET_WAIT_CNT           3000
 208 #define QM_VF_RESET_WAIT_TIMEOUT_US    \
 209         (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT)
 210
 211 #define QM_DFX_MB_CNT_VF                0x104010
 212 #define QM_DFX_DB_CNT_VF                0x104020
 213 #define QM_DFX_SQE_CNT_VF_SQN           0x104030
 214 #define QM_DFX_CQE_CNT_VF_CQN           0x104040
 215 #define QM_DFX_QN_SHIFT                 16
 216 #define CURRENT_FUN_MASK                GENMASK(5, 0)
 217 #define CURRENT_Q_MASK                  GENMASK(31, 16)
 218
 219 #define POLL_PERIOD                     10
 220 #define POLL_TIMEOUT                    1000
 221 #define WAIT_PERIOD_US_MAX              200
 222 #define WAIT_PERIOD_US_MIN              100
 223 #define MAX_WAIT_COUNTS                 1000
 224 #define QM_CACHE_WB_START               0x204
 225 #define QM_CACHE_WB_DONE                0x208
 226
 227 #define PCI_BAR_2                       2
 228 #define PCI_BAR_4                       4
 229 #define QM_SQE_DATA_ALIGN_MASK          GENMASK(6, 0)
 230 #define QMC_ALIGN(sz)                   ALIGN(sz, 32)
 231
 232 #define QM_DBG_READ_LEN         256
 233 #define QM_DBG_WRITE_LEN                1024
 234 #define QM_DBG_TMP_BUF_LEN              22
 235 #define QM_PCI_COMMAND_INVALID          ~0
 236
 237 #define WAIT_PERIOD                     20
 238 #define REMOVE_WAIT_DELAY               10
 239 #define QM_SQE_ADDR_MASK                GENMASK(7, 0)
 240 #define QM_EQ_DEPTH                     (1024 * 2)
 241
 242 #define QM_DRIVER_REMOVING              0
 243 #define QM_RST_SCHED                    1
 244 #define QM_RESETTING                    2
 245 #define QM_QOS_PARAM_NUM                2
 246 #define QM_QOS_VAL_NUM                  1
 247 #define QM_QOS_BDF_PARAM_NUM            4
 248 #define QM_QOS_MAX_VAL                  1000
 249 #define QM_QOS_RATE                     100
 250 #define QM_QOS_EXPAND_RATE              1000
 251 #define QM_SHAPER_CIR_B_MASK            GENMASK(7, 0)
 252 #define QM_SHAPER_CIR_U_MASK            GENMASK(10, 8)
 253 #define QM_SHAPER_CIR_S_MASK            GENMASK(14, 11)
 254 #define QM_SHAPER_FACTOR_CIR_U_SHIFT    8
 255 #define QM_SHAPER_FACTOR_CIR_S_SHIFT    11
 256 #define QM_SHAPER_FACTOR_CBS_B_SHIFT    15
 257 #define QM_SHAPER_FACTOR_CBS_S_SHIFT    19
 258 #define QM_SHAPER_CBS_B                 1
 259 #define QM_SHAPER_CBS_S                 16
 260 #define QM_SHAPER_VFT_OFFSET            6
 261 #define WAIT_FOR_QOS_VF                 100
 262 #define QM_QOS_MIN_ERROR_RATE           5
 263 #define QM_QOS_TYPICAL_NUM              8
 264 #define QM_SHAPER_MIN_CBS_S             8
 265 #define QM_QOS_TICK                     0x300U
 266 #define QM_QOS_DIVISOR_CLK              0x1f40U
 267 #define QM_QOS_MAX_CIR_B                200
 268 #define QM_QOS_MIN_CIR_B                100
 269 #define QM_QOS_MAX_CIR_U                6
 270 #define QM_QOS_MAX_CIR_S                11
 271 #define QM_QOS_VAL_MAX_LEN              32
 272
 273 #define QM_AUTOSUSPEND_DELAY            3000
 274
 275 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
 276         (((hop_num) << QM_CQ_HOP_NUM_SHIFT)     | \
 277         ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT)      | \
 278         ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT)      | \
 279         ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 280
 281 #define QM_MK_CQC_DW3_V2(cqe_sz) \
 282         ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 283
 284 #define QM_MK_SQC_W13(priority, orders, alg_type) \
 285         (((priority) << QM_SQ_PRIORITY_SHIFT)   | \
 286         ((orders) << QM_SQ_ORDERS_SHIFT)        | \
 287         (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
 288
 289 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
 290         (((hop_num) << QM_SQ_HOP_NUM_SHIFT)     | \
 291         ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT)      | \
 292         ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT)      | \
 293         ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 294
 295 #define QM_MK_SQC_DW3_V2(sqe_sz) \
 296         ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 297
 298 #define INIT_QC_COMMON(qc, base, pasid) do {                    \
 299         (qc)->head = 0;                                         \
 300         (qc)->tail = 0;                                         \
 301         (qc)->base_l = cpu_to_le32(lower_32_bits(base));        \
 302         (qc)->base_h = cpu_to_le32(upper_32_bits(base));        \
 303         (qc)->dw3 = 0;                                          \
 304         (qc)->w8 = 0;                                           \
 305         (qc)->rsvd0 = 0;                                        \
 306         (qc)->pasid = cpu_to_le16(pasid);                       \
 307         (qc)->w11 = 0;                                          \
 308         (qc)->rsvd1 = 0;                                        \
 309 } while (0)
 310
 311 enum vft_type {
 312         SQC_VFT = 0,
 313         CQC_VFT,
 314         SHAPER_VFT,
 315 };
 316
 317 enum acc_err_result {
 318         ACC_ERR_NONE,
 319         ACC_ERR_NEED_RESET,
 320         ACC_ERR_RECOVERED,
 321 };
 322
 323 enum qm_alg_type {
 324         ALG_TYPE_0,
 325         ALG_TYPE_1,
 326 };
 327
 328 enum qm_mb_cmd {
 329         QM_PF_FLR_PREPARE = 0x01,
 330         QM_PF_SRST_PREPARE,
 331         QM_PF_RESET_DONE,
 332         QM_VF_PREPARE_DONE,
 333         QM_VF_PREPARE_FAIL,
 334         QM_VF_START_DONE,
 335         QM_VF_START_FAIL,
 336         QM_PF_SET_QOS,
 337         QM_VF_GET_QOS,
 338 };
 339
 340 struct qm_cqe {
 341         __le32 rsvd0;
 342         __le16 cmd_id;
 343         __le16 rsvd1;
 344         __le16 sq_head;
 345         __le16 sq_num;
 346         __le16 rsvd2;
 347         __le16 w7;
 348 };
 349
 350 struct qm_eqe {
 351         __le32 dw0;
 352 };
 353
 354 struct qm_aeqe {
 355         __le32 dw0;
 356 };
 357
 358 struct qm_sqc {
 359         __le16 head;
 360         __le16 tail;
 361         __le32 base_l;
 362         __le32 base_h;
 363         __le32 dw3;
 364         __le16 w8;
 365         __le16 rsvd0;
 366         __le16 pasid;
 367         __le16 w11;
 368         __le16 cq_num;
 369         __le16 w13;
 370         __le32 rsvd1;
 371 };
 372
 373 struct qm_cqc {
 374         __le16 head;
 375         __le16 tail;
 376         __le32 base_l;
 377         __le32 base_h;
 378         __le32 dw3;
 379         __le16 w8;
 380         __le16 rsvd0;
 381         __le16 pasid;
 382         __le16 w11;
 383         __le32 dw6;
 384         __le32 rsvd1;
 385 };
 386
 387 struct qm_eqc {
 388         __le16 head;
 389         __le16 tail;
 390         __le32 base_l;
 391         __le32 base_h;
 392         __le32 dw3;
 393         __le32 rsvd[2];
 394         __le32 dw6;
 395 };
 396
 397 struct qm_aeqc {
 398         __le16 head;
 399         __le16 tail;
 400         __le32 base_l;
 401         __le32 base_h;
 402         __le32 dw3;
 403         __le32 rsvd[2];
 404         __le32 dw6;
 405 };
 406
 407 struct qm_mailbox {
 408         __le16 w0;
 409         __le16 queue_num;
 410         __le32 base_l;
 411         __le32 base_h;
 412         __le32 rsvd;
 413 };
 414
 415 struct qm_doorbell {
 416         __le16 queue_num;
 417         __le16 cmd;
 418         __le16 index;
 419         __le16 priority;
 420 };
 421
 422 struct hisi_qm_resource {
 423         struct hisi_qm *qm;
 424         int distance;
 425         struct list_head list;
 426 };
 427
 428 struct hisi_qm_hw_ops {
 429         int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
 430         void (*qm_db)(struct hisi_qm *qm, u16 qn,
 431                       u8 cmd, u16 index, u8 priority);
 432         u32 (*get_irq_num)(struct hisi_qm *qm);
 433         int (*debug_init)(struct hisi_qm *qm);
 434         void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe);
 435         void (*hw_error_uninit)(struct hisi_qm *qm);
 436         enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
 437         int (*stop_qp)(struct hisi_qp *qp);
 438         int (*set_msi)(struct hisi_qm *qm, bool set);
 439         int (*ping_all_vfs)(struct hisi_qm *qm, u64 cmd);
 440         int (*ping_pf)(struct hisi_qm *qm, u64 cmd);
 441 };
 442
 443 struct qm_dfx_item {
 444         const char *name;
 445         u32 offset;
 446 };
 447
 448 static struct qm_dfx_item qm_dfx_files[] = {
 449         {"err_irq", offsetof(struct qm_dfx, err_irq_cnt)},
 450         {"aeq_irq", offsetof(struct qm_dfx, aeq_irq_cnt)},
 451         {"abnormal_irq", offsetof(struct qm_dfx, abnormal_irq_cnt)},
 452         {"create_qp_err", offsetof(struct qm_dfx, create_qp_err_cnt)},
 453         {"mb_err", offsetof(struct qm_dfx, mb_err_cnt)},
 454 };
 455
 456 static const char * const qm_debug_file_name[] = {
 457         [CURRENT_QM]   = "current_qm",
 458         [CURRENT_Q]    = "current_q",
 459         [CLEAR_ENABLE] = "clear_enable",
 460 };
 461
 462 struct hisi_qm_hw_error {
 463         u32 int_msk;
 464         const char *msg;
 465 };
 466
 467 static const struct hisi_qm_hw_error qm_hw_error[] = {
 468         { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
 469         { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
 470         { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
 471         { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
 472         { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
 473         { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
 474         { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
 475         { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
 476         { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
 477         { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
 478         { .int_msk = BIT(10), .msg = "qm_db_timeout" },
 479         { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
 480         { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
 481         { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
 482         { .int_msk = BIT(14), .msg = "qm_flr_timeout" },
 483         { /* sentinel */ }
 484 };
 485
 486 static const char * const qm_db_timeout[] = {
 487         "sq", "cq", "eq", "aeq",
 488 };
 489
 490 static const char * const qm_fifo_overflow[] = {
 491         "cq", "eq", "aeq",
 492 };
 493
 494 static const char * const qm_s[] = {
 495         "init", "start", "close", "stop",
 496 };
 497
 498 static const char * const qp_s[] = {
 499         "none", "init", "start", "stop", "close",
 500 };
 501
 502 static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000,
 503                                                 10000, 25000, 50000, 100000};
 504 static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16,
 505                                                          17, 18, 19};
 506
 507 static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
 508 {
 509         enum qm_state curr = atomic_read(&qm->status.flags);
 510         bool avail = false;
 511
 512         switch (curr) {
 513         case QM_INIT:
 514                 if (new == QM_START || new == QM_CLOSE)
 515                         avail = true;
 516                 break;
 517         case QM_START:
 518                 if (new == QM_STOP)
 519                         avail = true;
 520                 break;
 521         case QM_STOP:
 522                 if (new == QM_CLOSE || new == QM_START)
 523                         avail = true;
 524                 break;
 525         default:
 526                 break;
 527         }
 528
 529         dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n",
 530                 qm_s[curr], qm_s[new]);
 531
 532         if (!avail)
 533                 dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n",
 534                          qm_s[curr], qm_s[new]);
 535
 536         return avail;
 537 }
 538
 539 static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
 540                               enum qp_state new)
 541 {
 542         enum qm_state qm_curr = atomic_read(&qm->status.flags);
 543         enum qp_state qp_curr = 0;
 544         bool avail = false;
 545
 546         if (qp)
 547                 qp_curr = atomic_read(&qp->qp_status.flags);
 548
 549         switch (new) {
 550         case QP_INIT:
 551                 if (qm_curr == QM_START || qm_curr == QM_INIT)
 552                         avail = true;
 553                 break;
 554         case QP_START:
 555                 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
 556                     (qm_curr == QM_START && qp_curr == QP_STOP))
 557                         avail = true;
 558                 break;
 559         case QP_STOP:
 560                 if ((qm_curr == QM_START && qp_curr == QP_START) ||
 561                     (qp_curr == QP_INIT))
 562                         avail = true;
 563                 break;
 564         case QP_CLOSE:
 565                 if ((qm_curr == QM_START && qp_curr == QP_INIT) ||
 566                     (qm_curr == QM_START && qp_curr == QP_STOP) ||
 567                     (qm_curr == QM_STOP && qp_curr == QP_STOP)  ||
 568                     (qm_curr == QM_STOP && qp_curr == QP_INIT))
 569                         avail = true;
 570                 break;
 571         default:
 572                 break;
 573         }
 574
 575         dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n",
 576                 qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
 577
 578         if (!avail)
 579                 dev_warn(&qm->pdev->dev,
 580                          "Can not change qp state from %s to %s in QM %s\n",
 581                          qp_s[qp_curr], qp_s[new], qm_s[qm_curr]);
 582
 583         return avail;
 584 }
 585
 586 static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
 587                            u64 base, u16 queue, bool op)
 588 {
 589         mailbox->w0 = cpu_to_le16((cmd) |
 590                 ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) |
 591                 (0x1 << QM_MB_BUSY_SHIFT));
 592         mailbox->queue_num = cpu_to_le16(queue);
 593         mailbox->base_l = cpu_to_le32(lower_32_bits(base));
 594         mailbox->base_h = cpu_to_le32(upper_32_bits(base));
 595         mailbox->rsvd = 0;
 596 }
 597
 598 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
 599 static int qm_wait_mb_ready(struct hisi_qm *qm)
 600 {
 601         u32 val;
 602
 603         return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
 604                                           val, !((val >> QM_MB_BUSY_SHIFT) &
 605                                           0x1), POLL_PERIOD, POLL_TIMEOUT);
 606 }
 607
 608 /* 128 bit should be written to hardware at one time to trigger a mailbox */
 609 static void qm_mb_write(struct hisi_qm *qm, const void *src)
 610 {
 611         void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
 612         unsigned long tmp0 = 0, tmp1 = 0;
 613
 614         if (!IS_ENABLED(CONFIG_ARM64)) {
 615                 memcpy_toio(fun_base, src, 16);
 616                 wmb();
 617                 return;
 618         }
 619
 620         asm volatile("ldp %0, %1, %3\n"
 621                      "stp %0, %1, %2\n"
 622                      "dsb sy\n"
 623                      : "=&r" (tmp0),
 624                        "=&r" (tmp1),
 625                        "+Q" (*((char __iomem *)fun_base))
 626                      : "Q" (*((char *)src))
 627                      : "memory");
 628 }
 629
 630 static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox)
 631 {
 632         if (unlikely(qm_wait_mb_ready(qm))) {
 633                 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
 634                 goto mb_busy;
 635         }
 636
 637         qm_mb_write(qm, mailbox);
 638
 639         if (unlikely(qm_wait_mb_ready(qm))) {
 640                 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
 641                 goto mb_busy;
 642         }
 643
 644         return 0;
 645
 646 mb_busy:
 647         atomic64_inc(&qm->debug.dfx.mb_err_cnt);
 648         return -EBUSY;
 649 }
 650
 651 static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
 652                  bool op)
 653 {
 654         struct qm_mailbox mailbox;
 655         int ret;
 656
 657         dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
 658                 queue, cmd, (unsigned long long)dma_addr);
 659
 660         qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op);
 661
 662         mutex_lock(&qm->mailbox_lock);
 663         ret = qm_mb_nolock(qm, &mailbox);
 664         mutex_unlock(&qm->mailbox_lock);
 665
 666         return ret;
 667 }
 668
 669 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 670 {
 671         u64 doorbell;
 672
 673         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
 674                    ((u64)index << QM_DB_INDEX_SHIFT_V1)  |
 675                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
 676
 677         writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
 678 }
 679
 680 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 681 {
 682         void __iomem *io_base = qm->io_base;
 683         u16 randata = 0;
 684         u64 doorbell;
 685
 686         if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
 687                 io_base = qm->db_io_base + (u64)qn * qm->db_interval +
 688                           QM_DOORBELL_SQ_CQ_BASE_V2;
 689         else
 690                 io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
 691
 692         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
 693                    ((u64)randata << QM_DB_RAND_SHIFT_V2) |
 694                    ((u64)index << QM_DB_INDEX_SHIFT_V2)  |
 695                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
 696
 697         writeq(doorbell, io_base);
 698 }
 699
 700 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 701 {
 702         dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
 703                 qn, cmd, index);
 704
 705         qm->ops->qm_db(qm, qn, cmd, index, priority);
 706 }
 707
 708 static int qm_dev_mem_reset(struct hisi_qm *qm)
 709 {
 710         u32 val;
 711
 712         writel(0x1, qm->io_base + QM_MEM_START_INIT);
 713         return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
 714                                           val & BIT(0), POLL_PERIOD,
 715                                           POLL_TIMEOUT);
 716 }
 717
 718 static u32 qm_get_irq_num_v1(struct hisi_qm *qm)
 719 {
 720         return QM_IRQ_NUM_V1;
 721 }
 722
 723 static u32 qm_get_irq_num_v2(struct hisi_qm *qm)
 724 {
 725         if (qm->fun_type == QM_HW_PF)
 726                 return QM_IRQ_NUM_PF_V2;
 727         else
 728                 return QM_IRQ_NUM_VF_V2;
 729 }
 730
 731 static u32 qm_get_irq_num_v3(struct hisi_qm *qm)
 732 {
 733         if (qm->fun_type == QM_HW_PF)
 734                 return QM_IRQ_NUM_PF_V2;
 735
 736         return QM_IRQ_NUM_VF_V3;
 737 }
 738
 739 static int qm_pm_get_sync(struct hisi_qm *qm)
 740 {
 741         struct device *dev = &qm->pdev->dev;
 742         int ret;
 743
 744         if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3)
 745                 return 0;
 746
 747         ret = pm_runtime_resume_and_get(dev);
 748         if (ret < 0) {
 749                 dev_err(dev, "failed to get_sync(%d).\n", ret);
 750                 return ret;
 751         }
 752
 753         return 0;
 754 }
 755
 756 static void qm_pm_put_sync(struct hisi_qm *qm)
 757 {
 758         struct device *dev = &qm->pdev->dev;
 759
 760         if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3)
 761                 return;
 762
 763         pm_runtime_mark_last_busy(dev);
 764         pm_runtime_put_autosuspend(dev);
 765 }
 766
 767 static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe)
 768 {
 769         u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;
 770
 771         return &qm->qp_array[cqn];
 772 }
 773
 774 static void qm_cq_head_update(struct hisi_qp *qp)
 775 {
 776         if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
 777                 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
 778                 qp->qp_status.cq_head = 0;
 779         } else {
 780                 qp->qp_status.cq_head++;
 781         }
 782 }
 783
 784 static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm)
 785 {
 786         if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))
 787                 return;
 788
 789         if (qp->event_cb) {
 790                 qp->event_cb(qp);
 791                 return;
 792         }
 793
 794         if (qp->req_cb) {
 795                 struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
 796
 797                 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
 798                         dma_rmb();
 799                         qp->req_cb(qp, qp->sqe + qm->sqe_size *
 800                                    le16_to_cpu(cqe->sq_head));
 801                         qm_cq_head_update(qp);
 802                         cqe = qp->cqe + qp->qp_status.cq_head;
 803                         qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 804                               qp->qp_status.cq_head, 0);
 805                         atomic_dec(&qp->qp_status.used);
 806                 }
 807
 808                 /* set c_flag */
 809                 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 810                       qp->qp_status.cq_head, 1);
 811         }
 812 }
 813
 814 static void qm_work_process(struct work_struct *work)
 815 {
 816         struct hisi_qm *qm = container_of(work, struct hisi_qm, work);
 817         struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
 818         struct hisi_qp *qp;
 819         int eqe_num = 0;
 820
 821         while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
 822                 eqe_num++;
 823                 qp = qm_to_hisi_qp(qm, eqe);
 824                 qm_poll_qp(qp, qm);
 825
 826                 if (qm->status.eq_head == QM_EQ_DEPTH - 1) {
 827                         qm->status.eqc_phase = !qm->status.eqc_phase;
 828                         eqe = qm->eqe;
 829                         qm->status.eq_head = 0;
 830                 } else {
 831                         eqe++;
 832                         qm->status.eq_head++;
 833                 }
 834
 835                 if (eqe_num == QM_EQ_DEPTH / 2 - 1) {
 836                         eqe_num = 0;
 837                         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 838                 }
 839         }
 840
 841         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 842 }
 843
 844 static irqreturn_t do_qm_irq(int irq, void *data)
 845 {
 846         struct hisi_qm *qm = (struct hisi_qm *)data;
 847
 848         /* the workqueue created by device driver of QM */
 849         if (qm->wq)
 850                 queue_work(qm->wq, &qm->work);
 851         else
 852                 schedule_work(&qm->work);
 853
 854         return IRQ_HANDLED;
 855 }
 856
 857 static irqreturn_t qm_irq(int irq, void *data)
 858 {
 859         struct hisi_qm *qm = data;
 860
 861         if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE))
 862                 return do_qm_irq(irq, data);
 863
 864         atomic64_inc(&qm->debug.dfx.err_irq_cnt);
 865         dev_err(&qm->pdev->dev, "invalid int source\n");
 866         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 867
 868         return IRQ_NONE;
 869 }
 870
 871 static irqreturn_t qm_mb_cmd_irq(int irq, void *data)
 872 {
 873         struct hisi_qm *qm = data;
 874         u32 val;
 875
 876         val = readl(qm->io_base + QM_IFC_INT_STATUS);
 877         val &= QM_IFC_INT_STATUS_MASK;
 878         if (!val)
 879                 return IRQ_NONE;
 880
 881         schedule_work(&qm->cmd_process);
 882
 883         return IRQ_HANDLED;
 884 }
 885
 886 static irqreturn_t qm_aeq_irq(int irq, void *data)
 887 {
 888         struct hisi_qm *qm = data;
 889         struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
 890         u32 type;
 891
 892         atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
 893         if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
 894                 return IRQ_NONE;
 895
 896         while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
 897                 type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
 898                 if (type < ARRAY_SIZE(qm_fifo_overflow))
 899                         dev_err(&qm->pdev->dev, "%s overflow\n",
 900                                 qm_fifo_overflow[type]);
 901                 else
 902                         dev_err(&qm->pdev->dev, "unknown error type %u\n",
 903                                 type);
 904
 905                 if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
 906                         qm->status.aeqc_phase = !qm->status.aeqc_phase;
 907                         aeqe = qm->aeqe;
 908                         qm->status.aeq_head = 0;
 909                 } else {
 910                         aeqe++;
 911                         qm->status.aeq_head++;
 912                 }
 913
 914                 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
 915         }
 916
 917         return IRQ_HANDLED;
 918 }
 919
 920 static void qm_irq_unregister(struct hisi_qm *qm)
 921 {
 922         struct pci_dev *pdev = qm->pdev;
 923
 924         free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
 925
 926         if (qm->ver > QM_HW_V1) {
 927                 free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
 928
 929                 if (qm->fun_type == QM_HW_PF)
 930                         free_irq(pci_irq_vector(pdev,
 931                                  QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
 932         }
 933
 934         if (qm->ver > QM_HW_V2)
 935                 free_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), qm);
 936 }
 937
 938 static void qm_init_qp_status(struct hisi_qp *qp)
 939 {
 940         struct hisi_qp_status *qp_status = &qp->qp_status;
 941
 942         qp_status->sq_tail = 0;
 943         qp_status->cq_head = 0;
 944         qp_status->cqc_phase = true;
 945         atomic_set(&qp_status->used, 0);
 946 }
 947
 948 static void qm_init_prefetch(struct hisi_qm *qm)
 949 {
 950         struct device *dev = &qm->pdev->dev;
 951         u32 page_type = 0x0;
 952
 953         if (qm->ver < QM_HW_V3)
 954                 return;
 955
 956         switch (PAGE_SIZE) {
 957         case SZ_4K:
 958                 page_type = 0x0;
 959                 break;
 960         case SZ_16K:
 961                 page_type = 0x1;
 962                 break;
 963         case SZ_64K:
 964                 page_type = 0x2;
 965                 break;
 966         default:
 967                 dev_err(dev, "system page size is not support: %lu, default set to 4KB",
 968                         PAGE_SIZE);
 969         }
 970
 971         writel(page_type, qm->io_base + QM_PAGE_SIZE);
 972 }
 973
 974 /*
 975  * the formula:
 976  * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
 977  *
 978  *                      IR_b * (2 ^ IR_u) * 8
 979  * IR(Mbps) * 10 ^ -3 = -------------------------
 980  *                      Tick * (2 ^ IR_s)
 981  */
 982 static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
 983 {
 984         return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) /
 985                                         (QM_QOS_TICK * (1 << cir_s));
 986 }
 987
 988 static u32 acc_shaper_calc_cbs_s(u32 ir)
 989 {
 990         int i;
 991
 992         if (ir < typical_qos_val[0])
 993                 return QM_SHAPER_MIN_CBS_S;
 994
 995         for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) {
 996                 if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i])
 997                         return typical_qos_cbs_s[i - 1];
 998         }
 999
1000         return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1];
1001 }
1002
1003 static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
1004 {
1005         u32 cir_b, cir_u, cir_s, ir_calc;
1006         u32 error_rate;
1007
1008         factor->cbs_s = acc_shaper_calc_cbs_s(ir);
1009
1010         for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
1011                 for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
1012                         for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) {
1013                                 /** the formula is changed to:
1014                                  *         IR_b * (2 ^ IR_u) * DIVISOR_CLK
1015                                  * IR(Mbps) = -------------------------
1016                                  *             768 * (2 ^ IR_s)
1017                                  */
1018                                 ir_calc = acc_shaper_para_calc(cir_b, cir_u,
1019                                                                cir_s);
1020                                 error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
1021                                 if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
1022                                         factor->cir_b = cir_b;
1023                                         factor->cir_u = cir_u;
1024                                         factor->cir_s = cir_s;
1025
1026                                         return 0;
1027                                 }
1028                         }
1029                 }
1030         }
1031
1032         return -EINVAL;
1033 }
1034
1035 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
1036                             u32 number, struct qm_shaper_factor *factor)
1037 {
1038         u64 tmp = 0;
1039
1040         if (number > 0) {
1041                 switch (type) {
1042                 case SQC_VFT:
1043                         if (qm->ver == QM_HW_V1) {
1044                                 tmp = QM_SQC_VFT_BUF_SIZE       |
1045                                       QM_SQC_VFT_SQC_SIZE       |
1046                                       QM_SQC_VFT_INDEX_NUMBER   |
1047                                       QM_SQC_VFT_VALID          |
1048                                       (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
1049                         } else {
1050                                 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
1051                                       QM_SQC_VFT_VALID |
1052                                       (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
1053                         }
1054                         break;
1055                 case CQC_VFT:
1056                         if (qm->ver == QM_HW_V1) {
1057                                 tmp = QM_CQC_VFT_BUF_SIZE       |
1058                                       QM_CQC_VFT_SQC_SIZE       |
1059                                       QM_CQC_VFT_INDEX_NUMBER   |
1060                                       QM_CQC_VFT_VALID;
1061                         } else {
1062                                 tmp = QM_CQC_VFT_VALID;
1063                         }
1064                         break;
1065                 case SHAPER_VFT:
1066                         if (qm->ver >= QM_HW_V3) {
1067                                 tmp = factor->cir_b |
1068                                 (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) |
1069                                 (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) |
1070                                 (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) |
1071                                 (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT);
1072                         }
1073                         break;
1074                 }
1075         }
1076
1077         writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
1078         writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
1079 }
1080
1081 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
1082                              u32 fun_num, u32 base, u32 number)
1083 {
1084         struct qm_shaper_factor *factor = &qm->factor[fun_num];
1085         unsigned int val;
1086         int ret;
1087
1088         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1089                                          val & BIT(0), POLL_PERIOD,
1090                                          POLL_TIMEOUT);
1091         if (ret)
1092                 return ret;
1093
1094         writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
1095         writel(type, qm->io_base + QM_VFT_CFG_TYPE);
1096         if (type == SHAPER_VFT)
1097                 fun_num |= base << QM_SHAPER_VFT_OFFSET;
1098
1099         writel(fun_num, qm->io_base + QM_VFT_CFG);
1100
1101         qm_vft_data_cfg(qm, type, base, number, factor);
1102
1103         writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
1104         writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
1105
1106         return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
1107                                           val & BIT(0), POLL_PERIOD,
1108                                           POLL_TIMEOUT);
1109 }
1110
1111 static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
1112 {
1113         int ret, i;
1114
1115         qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL;
1116         ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]);
1117         if (ret) {
1118                 dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
1119                 return ret;
1120         }
1121         writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG);
1122         for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
1123                 /* The base number of queue reuse for different alg type */
1124                 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1);
1125                 if (ret)
1126                         return ret;
1127         }
1128
1129         return 0;
1130 }
1131
1132 /* The config should be conducted after qm_dev_mem_reset() */
1133 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
1134                               u32 number)
1135 {
1136         int ret, i;
1137
1138         for (i = SQC_VFT; i <= CQC_VFT; i++) {
1139                 ret = qm_set_vft_common(qm, i, fun_num, base, number);
1140                 if (ret)
1141                         return ret;
1142         }
1143
1144         /* init default shaper qos val */
1145         if (qm->ver >= QM_HW_V3) {
1146                 ret = qm_shaper_init_vft(qm, fun_num);
1147                 if (ret)
1148                         goto back_sqc_cqc;
1149         }
1150
1151         return 0;
1152 back_sqc_cqc:
1153         for (i = SQC_VFT; i <= CQC_VFT; i++) {
1154                 ret = qm_set_vft_common(qm, i, fun_num, 0, 0);
1155                 if (ret)
1156                         return ret;
1157         }
1158         return ret;
1159 }
1160
1161 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
1162 {
1163         u64 sqc_vft;
1164         int ret;
1165
1166         ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
1167         if (ret)
1168                 return ret;
1169
1170         sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
1171                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
1172         *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
1173         *number = (QM_SQC_VFT_NUM_MASK_v2 &
1174                    (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
1175
1176         return 0;
1177 }
1178
1179 static int qm_get_vf_qp_num(struct hisi_qm *qm, u32 fun_num)
1180 {
1181         u32 remain_q_num, vfq_num;
1182         u32 num_vfs = qm->vfs_num;
1183
1184         vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs;
1185         if (vfq_num >= qm->max_qp_num)
1186                 return qm->max_qp_num;
1187
1188         remain_q_num = (qm->ctrl_qp_num - qm->qp_num) % num_vfs;
1189         if (vfq_num + remain_q_num <= qm->max_qp_num)
1190                 return fun_num == num_vfs ? vfq_num + remain_q_num : vfq_num;
1191
1192         /*
1193          * if vfq_num + remain_q_num > max_qp_num, the last VFs,
1194          * each with one more queue.
1195          */
1196         return fun_num + remain_q_num > num_vfs ? vfq_num + 1 : vfq_num;
1197 }
1198
1199 static struct hisi_qm *file_to_qm(struct debugfs_file *file)
1200 {
1201         struct qm_debug *debug = file->debug;
1202
1203         return container_of(debug, struct hisi_qm, debug);
1204 }
1205
1206 static u32 current_q_read(struct hisi_qm *qm)
1207 {
1208         return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT;
1209 }
1210
1211 static int current_q_write(struct hisi_qm *qm, u32 val)
1212 {
1213         u32 tmp;
1214
1215         if (val >= qm->debug.curr_qm_qp_num)
1216                 return -EINVAL;
1217
1218         tmp = val << QM_DFX_QN_SHIFT |
1219               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK);
1220         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
1221
1222         tmp = val << QM_DFX_QN_SHIFT |
1223               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK);
1224         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
1225
1226         return 0;
1227 }
1228
1229 static u32 clear_enable_read(struct hisi_qm *qm)
1230 {
1231         return readl(qm->io_base + QM_DFX_CNT_CLR_CE);
1232 }
1233
1234 /* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */
1235 static int clear_enable_write(struct hisi_qm *qm, u32 rd_clr_ctrl)
1236 {
1237         if (rd_clr_ctrl > 1)
1238                 return -EINVAL;
1239
1240         writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE);
1241
1242         return 0;
1243 }
1244
1245 static u32 current_qm_read(struct hisi_qm *qm)
1246 {
1247         return readl(qm->io_base + QM_DFX_MB_CNT_VF);
1248 }
1249
1250 static int current_qm_write(struct hisi_qm *qm, u32 val)
1251 {
1252         u32 tmp;
1253
1254         if (val > qm->vfs_num)
1255                 return -EINVAL;
1256
1257         /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
1258         if (!val)
1259                 qm->debug.curr_qm_qp_num = qm->qp_num;
1260         else
1261                 qm->debug.curr_qm_qp_num = qm_get_vf_qp_num(qm, val);
1262
1263         writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
1264         writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
1265
1266         tmp = val |
1267               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
1268         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
1269
1270         tmp = val |
1271               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
1272         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
1273
1274         return 0;
1275 }
1276
1277 static ssize_t qm_debug_read(struct file *filp, char __user *buf,
1278                              size_t count, loff_t *pos)
1279 {
1280         struct debugfs_file *file = filp->private_data;
1281         enum qm_debug_file index = file->index;
1282         struct hisi_qm *qm = file_to_qm(file);
1283         char tbuf[QM_DBG_TMP_BUF_LEN];
1284         u32 val;
1285         int ret;
1286
1287         ret = hisi_qm_get_dfx_access(qm);
1288         if (ret)
1289                 return ret;
1290
1291         mutex_lock(&file->lock);
1292         switch (index) {
1293         case CURRENT_QM:
1294                 val = current_qm_read(qm);
1295                 break;
1296         case CURRENT_Q:
1297                 val = current_q_read(qm);
1298                 break;
1299         case CLEAR_ENABLE:
1300                 val = clear_enable_read(qm);
1301                 break;
1302         default:
1303                 goto err_input;
1304         }
1305         mutex_unlock(&file->lock);
1306
1307         hisi_qm_put_dfx_access(qm);
1308         ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val);
1309         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
1310
1311 err_input:
1312         mutex_unlock(&file->lock);
1313         hisi_qm_put_dfx_access(qm);
1314         return -EINVAL;
1315 }
1316
1317 static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
1318                               size_t count, loff_t *pos)
1319 {
1320         struct debugfs_file *file = filp->private_data;
1321         enum qm_debug_file index = file->index;
1322         struct hisi_qm *qm = file_to_qm(file);
1323         unsigned long val;
1324         char tbuf[QM_DBG_TMP_BUF_LEN];
1325         int len, ret;
1326
1327         if (*pos != 0)
1328                 return 0;
1329
1330         if (count >= QM_DBG_TMP_BUF_LEN)
1331                 return -ENOSPC;
1332
1333         len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf,
1334                                      count);
1335         if (len < 0)
1336                 return len;
1337
1338         tbuf[len] = '\0';
1339         if (kstrtoul(tbuf, 0, &val))
1340                 return -EFAULT;
1341
1342         ret = hisi_qm_get_dfx_access(qm);
1343         if (ret)
1344                 return ret;
1345
1346         mutex_lock(&file->lock);
1347         switch (index) {
1348         case CURRENT_QM:
1349                 ret = current_qm_write(qm, val);
1350                 break;
1351         case CURRENT_Q:
1352                 ret = current_q_write(qm, val);
1353                 break;
1354         case CLEAR_ENABLE:
1355                 ret = clear_enable_write(qm, val);
1356                 break;
1357         default:
1358                 ret = -EINVAL;
1359         }
1360         mutex_unlock(&file->lock);
1361
1362         hisi_qm_put_dfx_access(qm);
1363
1364         if (ret)
1365                 return ret;
1366
1367         return count;
1368 }
1369
1370 static const struct file_operations qm_debug_fops = {
1371         .owner = THIS_MODULE,
1372         .open = simple_open,
1373         .read = qm_debug_read,
1374         .write = qm_debug_write,
1375 };
1376
1377 #define CNT_CYC_REGS_NUM                10
1378 static const struct debugfs_reg32 qm_dfx_regs[] = {
1379         /* XXX_CNT are reading clear register */
1380         {"QM_ECC_1BIT_CNT               ",  0x104000ull},
1381         {"QM_ECC_MBIT_CNT               ",  0x104008ull},
1382         {"QM_DFX_MB_CNT                 ",  0x104018ull},
1383         {"QM_DFX_DB_CNT                 ",  0x104028ull},
1384         {"QM_DFX_SQE_CNT                ",  0x104038ull},
1385         {"QM_DFX_CQE_CNT                ",  0x104048ull},
1386         {"QM_DFX_SEND_SQE_TO_ACC_CNT    ",  0x104050ull},
1387         {"QM_DFX_WB_SQE_FROM_ACC_CNT    ",  0x104058ull},
1388         {"QM_DFX_ACC_FINISH_CNT         ",  0x104060ull},
1389         {"QM_DFX_CQE_ERR_CNT            ",  0x1040b4ull},
1390         {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
1391         {"QM_ECC_1BIT_INF               ",  0x104004ull},
1392         {"QM_ECC_MBIT_INF               ",  0x10400cull},
1393         {"QM_DFX_ACC_RDY_VLD0           ",  0x1040a0ull},
1394         {"QM_DFX_ACC_RDY_VLD1           ",  0x1040a4ull},
1395         {"QM_DFX_AXI_RDY_VLD            ",  0x1040a8ull},
1396         {"QM_DFX_FF_ST0                 ",  0x1040c8ull},
1397         {"QM_DFX_FF_ST1                 ",  0x1040ccull},
1398         {"QM_DFX_FF_ST2                 ",  0x1040d0ull},
1399         {"QM_DFX_FF_ST3                 ",  0x1040d4ull},
1400         {"QM_DFX_FF_ST4                 ",  0x1040d8ull},
1401         {"QM_DFX_FF_ST5                 ",  0x1040dcull},
1402         {"QM_DFX_FF_ST6                 ",  0x1040e0ull},
1403         {"QM_IN_IDLE_ST                 ",  0x1040e4ull},
1404 };
1405
1406 static const struct debugfs_reg32 qm_vf_dfx_regs[] = {
1407         {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
1408 };
1409
1410 /**
1411  * hisi_qm_regs_dump() - Dump registers's value.
1412  * @s: debugfs file handle.
1413  * @regset: accelerator registers information.
1414  *
1415  * Dump accelerator registers.
1416  */
1417 void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset)
1418 {
1419         struct pci_dev *pdev = to_pci_dev(regset->dev);
1420         struct hisi_qm *qm = pci_get_drvdata(pdev);
1421         const struct debugfs_reg32 *regs = regset->regs;
1422         int regs_len = regset->nregs;
1423         int i, ret;
1424         u32 val;
1425
1426         ret = hisi_qm_get_dfx_access(qm);
1427         if (ret)
1428                 return;
1429
1430         for (i = 0; i < regs_len; i++) {
1431                 val = readl(regset->base + regs[i].offset);
1432                 seq_printf(s, "%s= 0x%08x\n", regs[i].name, val);
1433         }
1434
1435         hisi_qm_put_dfx_access(qm);
1436 }
1437 EXPORT_SYMBOL_GPL(hisi_qm_regs_dump);
1438
1439 static int qm_regs_show(struct seq_file *s, void *unused)
1440 {
1441         struct hisi_qm *qm = s->private;
1442         struct debugfs_regset32 regset;
1443
1444         if (qm->fun_type == QM_HW_PF) {
1445                 regset.regs = qm_dfx_regs;
1446                 regset.nregs = ARRAY_SIZE(qm_dfx_regs);
1447         } else {
1448                 regset.regs = qm_vf_dfx_regs;
1449                 regset.nregs = ARRAY_SIZE(qm_vf_dfx_regs);
1450         }
1451
1452         regset.base = qm->io_base;
1453         regset.dev = &qm->pdev->dev;
1454
1455         hisi_qm_regs_dump(s, &regset);
1456
1457         return 0;
1458 }
1459
1460 DEFINE_SHOW_ATTRIBUTE(qm_regs);
1461
1462 static ssize_t qm_cmd_read(struct file *filp, char __user *buffer,
1463                            size_t count, loff_t *pos)
1464 {
1465         char buf[QM_DBG_READ_LEN];
1466         int len;
1467
1468         len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n",
1469                         "Please echo help to cmd to get help information");
1470
1471         return simple_read_from_buffer(buffer, count, pos, buf, len);
1472 }
1473
1474 static void *qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size,
1475                           dma_addr_t *dma_addr)
1476 {
1477         struct device *dev = &qm->pdev->dev;
1478         void *ctx_addr;
1479
1480         ctx_addr = kzalloc(ctx_size, GFP_KERNEL);
1481         if (!ctx_addr)
1482                 return ERR_PTR(-ENOMEM);
1483
1484         *dma_addr = dma_map_single(dev, ctx_addr, ctx_size, DMA_FROM_DEVICE);
1485         if (dma_mapping_error(dev, *dma_addr)) {
1486                 dev_err(dev, "DMA mapping error!\n");
1487                 kfree(ctx_addr);
1488                 return ERR_PTR(-ENOMEM);
1489         }
1490
1491         return ctx_addr;
1492 }
1493
1494 static void qm_ctx_free(struct hisi_qm *qm, size_t ctx_size,
1495                         const void *ctx_addr, dma_addr_t *dma_addr)
1496 {
1497         struct device *dev = &qm->pdev->dev;
1498
1499         dma_unmap_single(dev, *dma_addr, ctx_size, DMA_FROM_DEVICE);
1500         kfree(ctx_addr);
1501 }
1502
1503 static int dump_show(struct hisi_qm *qm, void *info,
1504                      unsigned int info_size, char *info_name)
1505 {
1506         struct device *dev = &qm->pdev->dev;
1507         u8 *info_buf, *info_curr = info;
1508         u32 i;
1509 #define BYTE_PER_DW     4
1510
1511         info_buf = kzalloc(info_size, GFP_KERNEL);
1512         if (!info_buf)
1513                 return -ENOMEM;
1514
1515         for (i = 0; i < info_size; i++, info_curr++) {
1516                 if (i % BYTE_PER_DW == 0)
1517                         info_buf[i + 3UL] = *info_curr;
1518                 else if (i % BYTE_PER_DW == 1)
1519                         info_buf[i + 1UL] = *info_curr;
1520                 else if (i % BYTE_PER_DW == 2)
1521                         info_buf[i - 1] = *info_curr;
1522                 else if (i % BYTE_PER_DW == 3)
1523                         info_buf[i - 3] = *info_curr;
1524         }
1525
1526         dev_info(dev, "%s DUMP\n", info_name);
1527         for (i = 0; i < info_size; i += BYTE_PER_DW) {
1528                 pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW,
1529                         info_buf[i], info_buf[i + 1UL],
1530                         info_buf[i + 2UL], info_buf[i + 3UL]);
1531         }
1532
1533         kfree(info_buf);
1534
1535         return 0;
1536 }
1537
1538 static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1539 {
1540         return qm_mb(qm, QM_MB_CMD_SQC, dma_addr, qp_id, 1);
1541 }
1542
1543 static int qm_dump_cqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id)
1544 {
1545         return qm_mb(qm, QM_MB_CMD_CQC, dma_addr, qp_id, 1);
1546 }
1547
1548 static int qm_sqc_dump(struct hisi_qm *qm, const char *s)
1549 {
1550         struct device *dev = &qm->pdev->dev;
1551         struct qm_sqc *sqc, *sqc_curr;
1552         dma_addr_t sqc_dma;
1553         u32 qp_id;
1554         int ret;
1555
1556         if (!s)
1557                 return -EINVAL;
1558
1559         ret = kstrtou32(s, 0, &qp_id);
1560         if (ret || qp_id >= qm->qp_num) {
1561                 dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1);
1562                 return -EINVAL;
1563         }
1564
1565         sqc = qm_ctx_alloc(qm, sizeof(*sqc), &sqc_dma);
1566         if (IS_ERR(sqc))
1567                 return PTR_ERR(sqc);
1568
1569         ret = qm_dump_sqc_raw(qm, sqc_dma, qp_id);
1570         if (ret) {
1571                 down_read(&qm->qps_lock);
1572                 if (qm->sqc) {
1573                         sqc_curr = qm->sqc + qp_id;
1574
1575                         ret = dump_show(qm, sqc_curr, sizeof(*sqc),
1576                                         "SOFT SQC");
1577                         if (ret)
1578                                 dev_info(dev, "Show soft sqc failed!\n");
1579                 }
1580                 up_read(&qm->qps_lock);
1581
1582                 goto err_free_ctx;
1583         }
1584
1585         ret = dump_show(qm, sqc, sizeof(*sqc), "SQC");
1586         if (ret)
1587                 dev_info(dev, "Show hw sqc failed!\n");
1588
1589 err_free_ctx:
1590         qm_ctx_free(qm, sizeof(*sqc), sqc, &sqc_dma);
1591         return ret;
1592 }
1593
1594 static int qm_cqc_dump(struct hisi_qm *qm, const char *s)
1595 {
1596         struct device *dev = &qm->pdev->dev;
1597         struct qm_cqc *cqc, *cqc_curr;
1598         dma_addr_t cqc_dma;
1599         u32 qp_id;
1600         int ret;
1601
1602         if (!s)
1603                 return -EINVAL;
1604
1605         ret = kstrtou32(s, 0, &qp_id);
1606         if (ret || qp_id >= qm->qp_num) {
1607                 dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1);
1608                 return -EINVAL;
1609         }
1610
1611         cqc = qm_ctx_alloc(qm, sizeof(*cqc), &cqc_dma);
1612         if (IS_ERR(cqc))
1613                 return PTR_ERR(cqc);
1614
1615         ret = qm_dump_cqc_raw(qm, cqc_dma, qp_id);
1616         if (ret) {
1617                 down_read(&qm->qps_lock);
1618                 if (qm->cqc) {
1619                         cqc_curr = qm->cqc + qp_id;
1620
1621                         ret = dump_show(qm, cqc_curr, sizeof(*cqc),
1622                                         "SOFT CQC");
1623                         if (ret)
1624                                 dev_info(dev, "Show soft cqc failed!\n");
1625                 }
1626                 up_read(&qm->qps_lock);
1627
1628                 goto err_free_ctx;
1629         }
1630
1631         ret = dump_show(qm, cqc, sizeof(*cqc), "CQC");
1632         if (ret)
1633                 dev_info(dev, "Show hw cqc failed!\n");
1634
1635 err_free_ctx:
1636         qm_ctx_free(qm, sizeof(*cqc), cqc, &cqc_dma);
1637         return ret;
1638 }
1639
1640 static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size,
1641                             int cmd, char *name)
1642 {
1643         struct device *dev = &qm->pdev->dev;
1644         dma_addr_t xeqc_dma;
1645         void *xeqc;
1646         int ret;
1647
1648         if (strsep(&s, " ")) {
1649                 dev_err(dev, "Please do not input extra characters!\n");
1650                 return -EINVAL;
1651         }
1652
1653         xeqc = qm_ctx_alloc(qm, size, &xeqc_dma);
1654         if (IS_ERR(xeqc))
1655                 return PTR_ERR(xeqc);
1656
1657         ret = qm_mb(qm, cmd, xeqc_dma, 0, 1);
1658         if (ret)
1659                 goto err_free_ctx;
1660
1661         ret = dump_show(qm, xeqc, size, name);
1662         if (ret)
1663                 dev_info(dev, "Show hw %s failed!\n", name);
1664
1665 err_free_ctx:
1666         qm_ctx_free(qm, size, xeqc, &xeqc_dma);
1667         return ret;
1668 }
1669
1670 static int q_dump_param_parse(struct hisi_qm *qm, char *s,
1671                               u32 *e_id, u32 *q_id)
1672 {
1673         struct device *dev = &qm->pdev->dev;
1674         unsigned int qp_num = qm->qp_num;
1675         char *presult;
1676         int ret;
1677
1678         presult = strsep(&s, " ");
1679         if (!presult) {
1680                 dev_err(dev, "Please input qp number!\n");
1681                 return -EINVAL;
1682         }
1683
1684         ret = kstrtou32(presult, 0, q_id);
1685         if (ret || *q_id >= qp_num) {
1686                 dev_err(dev, "Please input qp num (0-%u)", qp_num - 1);
1687                 return -EINVAL;
1688         }
1689
1690         presult = strsep(&s, " ");
1691         if (!presult) {
1692                 dev_err(dev, "Please input sqe number!\n");
1693                 return -EINVAL;
1694         }
1695
1696         ret = kstrtou32(presult, 0, e_id);
1697         if (ret || *e_id >= QM_Q_DEPTH) {
1698                 dev_err(dev, "Please input sqe num (0-%d)", QM_Q_DEPTH - 1);
1699                 return -EINVAL;
1700         }
1701
1702         if (strsep(&s, " ")) {
1703                 dev_err(dev, "Please do not input extra characters!\n");
1704                 return -EINVAL;
1705         }
1706
1707         return 0;
1708 }
1709
1710 static int qm_sq_dump(struct hisi_qm *qm, char *s)
1711 {
1712         struct device *dev = &qm->pdev->dev;
1713         void *sqe, *sqe_curr;
1714         struct hisi_qp *qp;
1715         u32 qp_id, sqe_id;
1716         int ret;
1717
1718         ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id);
1719         if (ret)
1720                 return ret;
1721
1722         sqe = kzalloc(qm->sqe_size * QM_Q_DEPTH, GFP_KERNEL);
1723         if (!sqe)
1724                 return -ENOMEM;
1725
1726         qp = &qm->qp_array[qp_id];
1727         memcpy(sqe, qp->sqe, qm->sqe_size * QM_Q_DEPTH);
1728         sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size);
1729         memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK,
1730                qm->debug.sqe_mask_len);
1731
1732         ret = dump_show(qm, sqe_curr, qm->sqe_size, "SQE");
1733         if (ret)
1734                 dev_info(dev, "Show sqe failed!\n");
1735
1736         kfree(sqe);
1737
1738         return ret;
1739 }
1740
1741 static int qm_cq_dump(struct hisi_qm *qm, char *s)
1742 {
1743         struct device *dev = &qm->pdev->dev;
1744         struct qm_cqe *cqe_curr;
1745         struct hisi_qp *qp;
1746         u32 qp_id, cqe_id;
1747         int ret;
1748
1749         ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id);
1750         if (ret)
1751                 return ret;
1752
1753         qp = &qm->qp_array[qp_id];
1754         cqe_curr = qp->cqe + cqe_id;
1755         ret = dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE");
1756         if (ret)
1757                 dev_info(dev, "Show cqe failed!\n");
1758
1759         return ret;
1760 }
1761
1762 static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s,
1763                           size_t size, char *name)
1764 {
1765         struct device *dev = &qm->pdev->dev;
1766         void *xeqe;
1767         u32 xeqe_id;
1768         int ret;
1769
1770         if (!s)
1771                 return -EINVAL;
1772
1773         ret = kstrtou32(s, 0, &xeqe_id);
1774         if (ret)
1775                 return -EINVAL;
1776
1777         if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) {
1778                 dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1);
1779                 return -EINVAL;
1780         } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) {
1781                 dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1);
1782                 return -EINVAL;
1783         }
1784
1785         down_read(&qm->qps_lock);
1786
1787         if (qm->eqe && !strcmp(name, "EQE")) {
1788                 xeqe = qm->eqe + xeqe_id;
1789         } else if (qm->aeqe && !strcmp(name, "AEQE")) {
1790                 xeqe = qm->aeqe + xeqe_id;
1791         } else {
1792                 ret = -EINVAL;
1793                 goto err_unlock;
1794         }
1795
1796         ret = dump_show(qm, xeqe, size, name);
1797         if (ret)
1798                 dev_info(dev, "Show %s failed!\n", name);
1799
1800 err_unlock:
1801         up_read(&qm->qps_lock);
1802         return ret;
1803 }
1804
1805 static int qm_dbg_help(struct hisi_qm *qm, char *s)
1806 {
1807         struct device *dev = &qm->pdev->dev;
1808
1809         if (strsep(&s, " ")) {
1810                 dev_err(dev, "Please do not input extra characters!\n");
1811                 return -EINVAL;
1812         }
1813
1814         dev_info(dev, "available commands:\n");
1815         dev_info(dev, "sqc <num>\n");
1816         dev_info(dev, "cqc <num>\n");
1817         dev_info(dev, "eqc\n");
1818         dev_info(dev, "aeqc\n");
1819         dev_info(dev, "sq <num> <e>\n");
1820         dev_info(dev, "cq <num> <e>\n");
1821         dev_info(dev, "eq <e>\n");
1822         dev_info(dev, "aeq <e>\n");
1823
1824         return 0;
1825 }
1826
1827 static int qm_cmd_write_dump(struct hisi_qm *qm, const char *cmd_buf)
1828 {
1829         struct device *dev = &qm->pdev->dev;
1830         char *presult, *s, *s_tmp;
1831         int ret;
1832
1833         s = kstrdup(cmd_buf, GFP_KERNEL);
1834         if (!s)
1835                 return -ENOMEM;
1836
1837         s_tmp = s;
1838         presult = strsep(&s, " ");
1839         if (!presult) {
1840                 ret = -EINVAL;
1841                 goto err_buffer_free;
1842         }
1843
1844         if (!strcmp(presult, "sqc"))
1845                 ret = qm_sqc_dump(qm, s);
1846         else if (!strcmp(presult, "cqc"))
1847                 ret = qm_cqc_dump(qm, s);
1848         else if (!strcmp(presult, "eqc"))
1849                 ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_eqc),
1850                                        QM_MB_CMD_EQC, "EQC");
1851         else if (!strcmp(presult, "aeqc"))
1852                 ret = qm_eqc_aeqc_dump(qm, s, sizeof(struct qm_aeqc),
1853                                        QM_MB_CMD_AEQC, "AEQC");
1854         else if (!strcmp(presult, "sq"))
1855                 ret = qm_sq_dump(qm, s);
1856         else if (!strcmp(presult, "cq"))
1857                 ret = qm_cq_dump(qm, s);
1858         else if (!strcmp(presult, "eq"))
1859                 ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_eqe), "EQE");
1860         else if (!strcmp(presult, "aeq"))
1861                 ret = qm_eq_aeq_dump(qm, s, sizeof(struct qm_aeqe), "AEQE");
1862         else if (!strcmp(presult, "help"))
1863                 ret = qm_dbg_help(qm, s);
1864         else
1865                 ret = -EINVAL;
1866
1867         if (ret)
1868                 dev_info(dev, "Please echo help\n");
1869
1870 err_buffer_free:
1871         kfree(s_tmp);
1872
1873         return ret;
1874 }
1875
1876 static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer,
1877                             size_t count, loff_t *pos)
1878 {
1879         struct hisi_qm *qm = filp->private_data;
1880         char *cmd_buf, *cmd_buf_tmp;
1881         int ret;
1882
1883         if (*pos)
1884                 return 0;
1885
1886         ret = hisi_qm_get_dfx_access(qm);
1887         if (ret)
1888                 return ret;
1889
1890         /* Judge if the instance is being reset. */
1891         if (unlikely(atomic_read(&qm->status.flags) == QM_STOP))
1892                 return 0;
1893
1894         if (count > QM_DBG_WRITE_LEN) {
1895                 ret = -ENOSPC;
1896                 goto put_dfx_access;
1897         }
1898
1899         cmd_buf = memdup_user_nul(buffer, count);
1900         if (IS_ERR(cmd_buf)) {
1901                 ret = PTR_ERR(cmd_buf);
1902                 goto put_dfx_access;
1903         }
1904
1905         cmd_buf_tmp = strchr(cmd_buf, '\n');
1906         if (cmd_buf_tmp) {
1907                 *cmd_buf_tmp = '\0';
1908                 count = cmd_buf_tmp - cmd_buf + 1;
1909         }
1910
1911         ret = qm_cmd_write_dump(qm, cmd_buf);
1912         if (ret) {
1913                 kfree(cmd_buf);
1914                 goto put_dfx_access;
1915         }
1916
1917         kfree(cmd_buf);
1918
1919         ret = count;
1920
1921 put_dfx_access:
1922         hisi_qm_put_dfx_access(qm);
1923         return ret;
1924 }
1925
1926 static const struct file_operations qm_cmd_fops = {
1927         .owner = THIS_MODULE,
1928         .open = simple_open,
1929         .read = qm_cmd_read,
1930         .write = qm_cmd_write,
1931 };
1932
1933 static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
1934                                    enum qm_debug_file index)
1935 {
1936         struct debugfs_file *file = qm->debug.files + index;
1937
1938         debugfs_create_file(qm_debug_file_name[index], 0600, dir, file,
1939                             &qm_debug_fops);
1940
1941         file->index = index;
1942         mutex_init(&file->lock);
1943         file->debug = &qm->debug;
1944 }
1945
1946 static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1947 {
1948         writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1949 }
1950
1951 static void qm_hw_error_cfg(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1952 {
1953         qm->error_mask = ce | nfe | fe;
1954         /* clear QM hw residual error source */
1955         writel(QM_ABNORMAL_INT_SOURCE_CLR,
1956                qm->io_base + QM_ABNORMAL_INT_SOURCE);
1957
1958         /* configure error type */
1959         writel(ce, qm->io_base + QM_RAS_CE_ENABLE);
1960         writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1961         writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1962         writel(fe, qm->io_base + QM_RAS_FE_ENABLE);
1963 }
1964
1965 static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1966 {
1967         u32 irq_enable = ce | nfe | fe;
1968         u32 irq_unmask = ~irq_enable;
1969
1970         qm_hw_error_cfg(qm, ce, nfe, fe);
1971
1972         irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1973         writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1974 }
1975
1976 static void qm_hw_error_uninit_v2(struct hisi_qm *qm)
1977 {
1978         writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1979 }
1980
1981 static void qm_hw_error_init_v3(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe)
1982 {
1983         u32 irq_enable = ce | nfe | fe;
1984         u32 irq_unmask = ~irq_enable;
1985
1986         qm_hw_error_cfg(qm, ce, nfe, fe);
1987
1988         /* enable close master ooo when hardware error happened */
1989         writel(nfe & (~QM_DB_RANDOM_INVALID), qm->io_base + QM_OOO_SHUTDOWN_SEL);
1990
1991         irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1992         writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1993 }
1994
1995 static void qm_hw_error_uninit_v3(struct hisi_qm *qm)
1996 {
1997         writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
1998
1999         /* disable close master ooo when hardware error happened */
2000         writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL);
2001 }
2002
2003 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
2004 {
2005         const struct hisi_qm_hw_error *err;
2006         struct device *dev = &qm->pdev->dev;
2007         u32 reg_val, type, vf_num;
2008         int i;
2009
2010         for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
2011                 err = &qm_hw_error[i];
2012                 if (!(err->int_msk & error_status))
2013                         continue;
2014
2015                 dev_err(dev, "%s [error status=0x%x] found\n",
2016                         err->msg, err->int_msk);
2017
2018                 if (err->int_msk & QM_DB_TIMEOUT) {
2019                         reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);
2020                         type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
2021                                QM_DB_TIMEOUT_TYPE_SHIFT;
2022                         vf_num = reg_val & QM_DB_TIMEOUT_VF;
2023                         dev_err(dev, "qm %s doorbell timeout in function %u\n",
2024                                 qm_db_timeout[type], vf_num);
2025                 } else if (err->int_msk & QM_OF_FIFO_OF) {
2026                         reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
2027                         type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
2028                                QM_FIFO_OVERFLOW_TYPE_SHIFT;
2029                         vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
2030
2031                         if (type < ARRAY_SIZE(qm_fifo_overflow))
2032                                 dev_err(dev, "qm %s fifo overflow in function %u\n",
2033                                         qm_fifo_overflow[type], vf_num);
2034                         else
2035                                 dev_err(dev, "unknown error type\n");
2036                 }
2037         }
2038 }
2039
2040 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
2041 {
2042         u32 error_status, tmp, val;
2043
2044         /* read err sts */
2045         tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
2046         error_status = qm->error_mask & tmp;
2047
2048         if (error_status) {
2049                 if (error_status & QM_ECC_MBIT)
2050                         qm->err_status.is_qm_ecc_mbit = true;
2051
2052                 qm_log_hw_error(qm, error_status);
2053                 val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE;
2054                 /* ce error does not need to be reset */
2055                 if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) {
2056                         writel(error_status, qm->io_base +
2057                                QM_ABNORMAL_INT_SOURCE);
2058                         writel(qm->err_info.nfe,
2059                                qm->io_base + QM_RAS_NFE_ENABLE);
2060                         return ACC_ERR_RECOVERED;
2061                 }
2062
2063                 return ACC_ERR_NEED_RESET;
2064         }
2065
2066         return ACC_ERR_RECOVERED;
2067 }
2068
2069 static u32 qm_get_hw_error_status(struct hisi_qm *qm)
2070 {
2071         return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
2072 }
2073
2074 static u32 qm_get_dev_err_status(struct hisi_qm *qm)
2075 {
2076         return qm->err_ini->get_dev_hw_err_status(qm);
2077 }
2078
2079 /* Check if the error causes the master ooo block */
2080 static int qm_check_dev_error(struct hisi_qm *qm)
2081 {
2082         u32 val, dev_val;
2083
2084         if (qm->fun_type == QM_HW_VF)
2085                 return 0;
2086
2087         val = qm_get_hw_error_status(qm);
2088         dev_val = qm_get_dev_err_status(qm);
2089
2090         if (qm->ver < QM_HW_V3)
2091                 return (val & QM_ECC_MBIT) ||
2092                        (dev_val & qm->err_info.ecc_2bits_mask);
2093
2094         return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
2095                (dev_val & (~qm->err_info.dev_ce_mask));
2096 }
2097
2098 static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
2099 {
2100         struct qm_mailbox mailbox;
2101         int ret;
2102
2103         qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0);
2104         mutex_lock(&qm->mailbox_lock);
2105         ret = qm_mb_nolock(qm, &mailbox);
2106         if (ret)
2107                 goto err_unlock;
2108
2109         *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
2110                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
2111
2112 err_unlock:
2113         mutex_unlock(&qm->mailbox_lock);
2114         return ret;
2115 }
2116
2117 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask)
2118 {
2119         u32 val;
2120
2121         if (qm->fun_type == QM_HW_PF)
2122                 writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P);
2123
2124         val = readl(qm->io_base + QM_IFC_INT_SOURCE_V);
2125         val |= QM_IFC_INT_SOURCE_MASK;
2126         writel(val, qm->io_base + QM_IFC_INT_SOURCE_V);
2127 }
2128
2129 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id)
2130 {
2131         struct device *dev = &qm->pdev->dev;
2132         u32 cmd;
2133         u64 msg;
2134         int ret;
2135
2136         ret = qm_get_mb_cmd(qm, &msg, vf_id);
2137         if (ret) {
2138                 dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id);
2139                 return;
2140         }
2141
2142         cmd = msg & QM_MB_CMD_DATA_MASK;
2143         switch (cmd) {
2144         case QM_VF_PREPARE_FAIL:
2145                 dev_err(dev, "failed to stop VF(%u)!\n", vf_id);
2146                 break;
2147         case QM_VF_START_FAIL:
2148                 dev_err(dev, "failed to start VF(%u)!\n", vf_id);
2149                 break;
2150         case QM_VF_PREPARE_DONE:
2151         case QM_VF_START_DONE:
2152                 break;
2153         default:
2154                 dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id);
2155                 break;
2156         }
2157 }
2158
2159 static int qm_wait_vf_prepare_finish(struct hisi_qm *qm)
2160 {
2161         struct device *dev = &qm->pdev->dev;
2162         u32 vfs_num = qm->vfs_num;
2163         int cnt = 0;
2164         int ret = 0;
2165         u64 val;
2166         u32 i;
2167
2168         if (!qm->vfs_num || qm->ver < QM_HW_V3)
2169                 return 0;
2170
2171         while (true) {
2172                 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
2173                 /* All VFs send command to PF, break */
2174                 if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1))
2175                         break;
2176
2177                 if (++cnt > QM_MAX_PF_WAIT_COUNT) {
2178                         ret = -EBUSY;
2179                         break;
2180                 }
2181
2182                 msleep(QM_WAIT_DST_ACK);
2183         }
2184
2185         /* PF check VFs msg */
2186         for (i = 1; i <= vfs_num; i++) {
2187                 if (val & BIT(i))
2188                         qm_handle_vf_msg(qm, i);
2189                 else
2190                         dev_err(dev, "VF(%u) not ping PF!\n", i);
2191         }
2192
2193         /* PF clear interrupt to ack VFs */
2194         qm_clear_cmd_interrupt(qm, val);
2195
2196         return ret;
2197 }
2198
2199 static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num)
2200 {
2201         u32 val;
2202
2203         val = readl(qm->io_base + QM_IFC_INT_CFG);
2204         val &= ~QM_IFC_SEND_ALL_VFS;
2205         val |= fun_num;
2206         writel(val, qm->io_base + QM_IFC_INT_CFG);
2207
2208         val = readl(qm->io_base + QM_IFC_INT_SET_P);
2209         val |= QM_IFC_INT_SET_MASK;
2210         writel(val, qm->io_base + QM_IFC_INT_SET_P);
2211 }
2212
2213 static void qm_trigger_pf_interrupt(struct hisi_qm *qm)
2214 {
2215         u32 val;
2216
2217         val = readl(qm->io_base + QM_IFC_INT_SET_V);
2218         val |= QM_IFC_INT_SET_MASK;
2219         writel(val, qm->io_base + QM_IFC_INT_SET_V);
2220 }
2221
2222 static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num)
2223 {
2224         struct device *dev = &qm->pdev->dev;
2225         struct qm_mailbox mailbox;
2226         int cnt = 0;
2227         u64 val;
2228         int ret;
2229
2230         qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0);
2231         mutex_lock(&qm->mailbox_lock);
2232         ret = qm_mb_nolock(qm, &mailbox);
2233         if (ret) {
2234                 dev_err(dev, "failed to send command to vf(%u)!\n", fun_num);
2235                 goto err_unlock;
2236         }
2237
2238         qm_trigger_vf_interrupt(qm, fun_num);
2239         while (true) {
2240                 msleep(QM_WAIT_DST_ACK);
2241                 val = readq(qm->io_base + QM_IFC_READY_STATUS);
2242                 /* if VF respond, PF notifies VF successfully. */
2243                 if (!(val & BIT(fun_num)))
2244                         goto err_unlock;
2245
2246                 if (++cnt > QM_MAX_PF_WAIT_COUNT) {
2247                         dev_err(dev, "failed to get response from VF(%u)!\n", fun_num);
2248                         ret = -ETIMEDOUT;
2249                         break;
2250                 }
2251         }
2252
2253 err_unlock:
2254         mutex_unlock(&qm->mailbox_lock);
2255         return ret;
2256 }
2257
2258 static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd)
2259 {
2260         struct device *dev = &qm->pdev->dev;
2261         u32 vfs_num = qm->vfs_num;
2262         struct qm_mailbox mailbox;
2263         u64 val = 0;
2264         int cnt = 0;
2265         int ret;
2266         u32 i;
2267
2268         qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0);
2269         mutex_lock(&qm->mailbox_lock);
2270         /* PF sends command to all VFs by mailbox */
2271         ret = qm_mb_nolock(qm, &mailbox);
2272         if (ret) {
2273                 dev_err(dev, "failed to send command to VFs!\n");
2274                 mutex_unlock(&qm->mailbox_lock);
2275                 return ret;
2276         }
2277
2278         qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS);
2279         while (true) {
2280                 msleep(QM_WAIT_DST_ACK);
2281                 val = readq(qm->io_base + QM_IFC_READY_STATUS);
2282                 /* If all VFs acked, PF notifies VFs successfully. */
2283                 if (!(val & GENMASK(vfs_num, 1))) {
2284                         mutex_unlock(&qm->mailbox_lock);
2285                         return 0;
2286                 }
2287
2288                 if (++cnt > QM_MAX_PF_WAIT_COUNT)
2289                         break;
2290         }
2291
2292         mutex_unlock(&qm->mailbox_lock);
2293
2294         /* Check which vf respond timeout. */
2295         for (i = 1; i <= vfs_num; i++) {
2296                 if (val & BIT(i))
2297                         dev_err(dev, "failed to get response from VF(%u)!\n", i);
2298         }
2299
2300         return -ETIMEDOUT;
2301 }
2302
2303 static int qm_ping_pf(struct hisi_qm *qm, u64 cmd)
2304 {
2305         struct qm_mailbox mailbox;
2306         int cnt = 0;
2307         u32 val;
2308         int ret;
2309
2310         qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0);
2311         mutex_lock(&qm->mailbox_lock);
2312         ret = qm_mb_nolock(qm, &mailbox);
2313         if (ret) {
2314                 dev_err(&qm->pdev->dev, "failed to send command to PF!\n");
2315                 goto unlock;
2316         }
2317
2318         qm_trigger_pf_interrupt(qm);
2319         /* Waiting for PF response */
2320         while (true) {
2321                 msleep(QM_WAIT_DST_ACK);
2322                 val = readl(qm->io_base + QM_IFC_INT_SET_V);
2323                 if (!(val & QM_IFC_INT_STATUS_MASK))
2324                         break;
2325
2326                 if (++cnt > QM_MAX_VF_WAIT_COUNT) {
2327                         ret = -ETIMEDOUT;
2328                         break;
2329                 }
2330         }
2331
2332 unlock:
2333         mutex_unlock(&qm->mailbox_lock);
2334         return ret;
2335 }
2336
2337 static int qm_stop_qp(struct hisi_qp *qp)
2338 {
2339         return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
2340 }
2341
2342 static int qm_set_msi(struct hisi_qm *qm, bool set)
2343 {
2344         struct pci_dev *pdev = qm->pdev;
2345
2346         if (set) {
2347                 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
2348                                        0);
2349         } else {
2350                 pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,
2351                                        ACC_PEH_MSI_DISABLE);
2352                 if (qm->err_status.is_qm_ecc_mbit ||
2353                     qm->err_status.is_dev_ecc_mbit)
2354                         return 0;
2355
2356                 mdelay(1);
2357                 if (readl(qm->io_base + QM_PEH_DFX_INFO0))
2358                         return -EFAULT;
2359         }
2360
2361         return 0;
2362 }
2363
2364 static void qm_wait_msi_finish(struct hisi_qm *qm)
2365 {
2366         struct pci_dev *pdev = qm->pdev;
2367         u32 cmd = ~0;
2368         int cnt = 0;
2369         u32 val;
2370         int ret;
2371
2372         while (true) {
2373                 pci_read_config_dword(pdev, pdev->msi_cap +
2374                                       PCI_MSI_PENDING_64, &cmd);
2375                 if (!cmd)
2376                         break;
2377
2378                 if (++cnt > MAX_WAIT_COUNTS) {
2379                         pci_warn(pdev, "failed to empty MSI PENDING!\n");
2380                         break;
2381                 }
2382
2383                 udelay(1);
2384         }
2385
2386         ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0,
2387                                          val, !(val & QM_PEH_DFX_MASK),
2388                                          POLL_PERIOD, POLL_TIMEOUT);
2389         if (ret)
2390                 pci_warn(pdev, "failed to empty PEH MSI!\n");
2391
2392         ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1,
2393                                          val, !(val & QM_PEH_MSI_FINISH_MASK),
2394                                          POLL_PERIOD, POLL_TIMEOUT);
2395         if (ret)
2396                 pci_warn(pdev, "failed to finish MSI operation!\n");
2397 }
2398
2399 static int qm_set_msi_v3(struct hisi_qm *qm, bool set)
2400 {
2401         struct pci_dev *pdev = qm->pdev;
2402         int ret = -ETIMEDOUT;
2403         u32 cmd, i;
2404
2405         pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
2406         if (set)
2407                 cmd |= QM_MSI_CAP_ENABLE;
2408         else
2409                 cmd &= ~QM_MSI_CAP_ENABLE;
2410
2411         pci_write_config_dword(pdev, pdev->msi_cap, cmd);
2412         if (set) {
2413                 for (i = 0; i < MAX_WAIT_COUNTS; i++) {
2414                         pci_read_config_dword(pdev, pdev->msi_cap, &cmd);
2415                         if (cmd & QM_MSI_CAP_ENABLE)
2416                                 return 0;
2417
2418                         udelay(1);
2419                 }
2420         } else {
2421                 udelay(WAIT_PERIOD_US_MIN);
2422                 qm_wait_msi_finish(qm);
2423                 ret = 0;
2424         }
2425
2426         return ret;
2427 }
2428
2429 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
2430         .qm_db = qm_db_v1,
2431         .get_irq_num = qm_get_irq_num_v1,
2432         .hw_error_init = qm_hw_error_init_v1,
2433         .set_msi = qm_set_msi,
2434 };
2435
2436 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
2437         .get_vft = qm_get_vft_v2,
2438         .qm_db = qm_db_v2,
2439         .get_irq_num = qm_get_irq_num_v2,
2440         .hw_error_init = qm_hw_error_init_v2,
2441         .hw_error_uninit = qm_hw_error_uninit_v2,
2442         .hw_error_handle = qm_hw_error_handle_v2,
2443         .set_msi = qm_set_msi,
2444 };
2445
2446 static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
2447         .get_vft = qm_get_vft_v2,
2448         .qm_db = qm_db_v2,
2449         .get_irq_num = qm_get_irq_num_v3,
2450         .hw_error_init = qm_hw_error_init_v3,
2451         .hw_error_uninit = qm_hw_error_uninit_v3,
2452         .hw_error_handle = qm_hw_error_handle_v2,
2453         .stop_qp = qm_stop_qp,
2454         .set_msi = qm_set_msi_v3,
2455         .ping_all_vfs = qm_ping_all_vfs,
2456         .ping_pf = qm_ping_pf,
2457 };
2458
2459 static void *qm_get_avail_sqe(struct hisi_qp *qp)
2460 {
2461         struct hisi_qp_status *qp_status = &qp->qp_status;
2462         u16 sq_tail = qp_status->sq_tail;
2463
2464         if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1))
2465                 return NULL;
2466
2467         return qp->sqe + sq_tail * qp->qm->sqe_size;
2468 }
2469
2470 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)
2471 {
2472         struct device *dev = &qm->pdev->dev;
2473         struct hisi_qp *qp;
2474         int qp_id;
2475
2476         if (!qm_qp_avail_state(qm, NULL, QP_INIT))
2477                 return ERR_PTR(-EPERM);
2478
2479         if (qm->qp_in_used == qm->qp_num) {
2480                 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
2481                                      qm->qp_num);
2482                 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
2483                 return ERR_PTR(-EBUSY);
2484         }
2485
2486         qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);
2487         if (qp_id < 0) {
2488                 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",
2489                                     qm->qp_num);
2490                 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);
2491                 return ERR_PTR(-EBUSY);
2492         }
2493
2494         qp = &qm->qp_array[qp_id];
2495
2496         memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH);
2497
2498         qp->event_cb = NULL;
2499         qp->req_cb = NULL;
2500         qp->qp_id = qp_id;
2501         qp->alg_type = alg_type;
2502         qp->is_in_kernel = true;
2503         qm->qp_in_used++;
2504         atomic_set(&qp->qp_status.flags, QP_INIT);
2505
2506         return qp;
2507 }
2508
2509 /**
2510  * hisi_qm_create_qp() - Create a queue pair from qm.
2511  * @qm: The qm we create a qp from.
2512  * @alg_type: Accelerator specific algorithm type in sqc.
2513  *
2514  * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
2515  * qp memory fails.
2516  */
2517 struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
2518 {
2519         struct hisi_qp *qp;
2520         int ret;
2521
2522         ret = qm_pm_get_sync(qm);
2523         if (ret)
2524                 return ERR_PTR(ret);
2525
2526         down_write(&qm->qps_lock);
2527         qp = qm_create_qp_nolock(qm, alg_type);
2528         up_write(&qm->qps_lock);
2529
2530         if (IS_ERR(qp))
2531                 qm_pm_put_sync(qm);
2532
2533         return qp;
2534 }
2535 EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
2536
2537 /**
2538  * hisi_qm_release_qp() - Release a qp back to its qm.
2539  * @qp: The qp we want to release.
2540  *
2541  * This function releases the resource of a qp.
2542  */
2543 void hisi_qm_release_qp(struct hisi_qp *qp)
2544 {
2545         struct hisi_qm *qm = qp->qm;
2546
2547         down_write(&qm->qps_lock);
2548
2549         if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) {
2550                 up_write(&qm->qps_lock);
2551                 return;
2552         }
2553
2554         qm->qp_in_used--;
2555         idr_remove(&qm->qp_idr, qp->qp_id);
2556
2557         up_write(&qm->qps_lock);
2558
2559         qm_pm_put_sync(qm);
2560 }
2561 EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
2562
2563 static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2564 {
2565         struct hisi_qm *qm = qp->qm;
2566         struct device *dev = &qm->pdev->dev;
2567         enum qm_hw_ver ver = qm->ver;
2568         struct qm_sqc *sqc;
2569         dma_addr_t sqc_dma;
2570         int ret;
2571
2572         sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
2573         if (!sqc)
2574                 return -ENOMEM;
2575
2576         INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
2577         if (ver == QM_HW_V1) {
2578                 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));
2579                 sqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
2580         } else {
2581                 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size));
2582                 sqc->w8 = 0; /* rand_qc */
2583         }
2584         sqc->cq_num = cpu_to_le16(qp_id);
2585         sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));
2586
2587         if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2588                 sqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<
2589                                        QM_QC_PASID_ENABLE_SHIFT);
2590
2591         sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
2592                                  DMA_TO_DEVICE);
2593         if (dma_mapping_error(dev, sqc_dma)) {
2594                 kfree(sqc);
2595                 return -ENOMEM;
2596         }
2597
2598         ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
2599         dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
2600         kfree(sqc);
2601
2602         return ret;
2603 }
2604
2605 static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2606 {
2607         struct hisi_qm *qm = qp->qm;
2608         struct device *dev = &qm->pdev->dev;
2609         enum qm_hw_ver ver = qm->ver;
2610         struct qm_cqc *cqc;
2611         dma_addr_t cqc_dma;
2612         int ret;
2613
2614         cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
2615         if (!cqc)
2616                 return -ENOMEM;
2617
2618         INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
2619         if (ver == QM_HW_V1) {
2620                 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0,
2621                                                         QM_QC_CQE_SIZE));
2622                 cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1);
2623         } else {
2624                 cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE));
2625                 cqc->w8 = 0; /* rand_qc */
2626         }
2627         cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);
2628
2629         if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)
2630                 cqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE);
2631
2632         cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
2633                                  DMA_TO_DEVICE);
2634         if (dma_mapping_error(dev, cqc_dma)) {
2635                 kfree(cqc);
2636                 return -ENOMEM;
2637         }
2638
2639         ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
2640         dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
2641         kfree(cqc);
2642
2643         return ret;
2644 }
2645
2646 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)
2647 {
2648         int ret;
2649
2650         qm_init_qp_status(qp);
2651
2652         ret = qm_sq_ctx_cfg(qp, qp_id, pasid);
2653         if (ret)
2654                 return ret;
2655
2656         return qm_cq_ctx_cfg(qp, qp_id, pasid);
2657 }
2658
2659 static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)
2660 {
2661         struct hisi_qm *qm = qp->qm;
2662         struct device *dev = &qm->pdev->dev;
2663         int qp_id = qp->qp_id;
2664         u32 pasid = arg;
2665         int ret;
2666
2667         if (!qm_qp_avail_state(qm, qp, QP_START))
2668                 return -EPERM;
2669
2670         ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
2671         if (ret)
2672                 return ret;
2673
2674         atomic_set(&qp->qp_status.flags, QP_START);
2675         dev_dbg(dev, "queue %d started\n", qp_id);
2676
2677         return 0;
2678 }
2679
2680 /**
2681  * hisi_qm_start_qp() - Start a qp into running.
2682  * @qp: The qp we want to start to run.
2683  * @arg: Accelerator specific argument.
2684  *
2685  * After this function, qp can receive request from user. Return 0 if
2686  * successful, Return -EBUSY if failed.
2687  */
2688 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
2689 {
2690         struct hisi_qm *qm = qp->qm;
2691         int ret;
2692
2693         down_write(&qm->qps_lock);
2694         ret = qm_start_qp_nolock(qp, arg);
2695         up_write(&qm->qps_lock);
2696
2697         return ret;
2698 }
2699 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
2700
2701 /**
2702  * qp_stop_fail_cb() - call request cb.
2703  * @qp: stopped failed qp.
2704  *
2705  * Callback function should be called whether task completed or not.
2706  */
2707 static void qp_stop_fail_cb(struct hisi_qp *qp)
2708 {
2709         int qp_used = atomic_read(&qp->qp_status.used);
2710         u16 cur_tail = qp->qp_status.sq_tail;
2711         u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH;
2712         struct hisi_qm *qm = qp->qm;
2713         u16 pos;
2714         int i;
2715
2716         for (i = 0; i < qp_used; i++) {
2717                 pos = (i + cur_head) % QM_Q_DEPTH;
2718                 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
2719                 atomic_dec(&qp->qp_status.used);
2720         }
2721 }
2722
2723 /**
2724  * qm_drain_qp() - Drain a qp.
2725  * @qp: The qp we want to drain.
2726  *
2727  * Determine whether the queue is cleared by judging the tail pointers of
2728  * sq and cq.
2729  */
2730 static int qm_drain_qp(struct hisi_qp *qp)
2731 {
2732         size_t size = sizeof(struct qm_sqc) + sizeof(struct qm_cqc);
2733         struct hisi_qm *qm = qp->qm;
2734         struct device *dev = &qm->pdev->dev;
2735         struct qm_sqc *sqc;
2736         struct qm_cqc *cqc;
2737         dma_addr_t dma_addr;
2738         int ret = 0, i = 0;
2739         void *addr;
2740
2741         /* No need to judge if master OOO is blocked. */
2742         if (qm_check_dev_error(qm))
2743                 return 0;
2744
2745         /* Kunpeng930 supports drain qp by device */
2746         if (qm->ops->stop_qp) {
2747                 ret = qm->ops->stop_qp(qp);
2748                 if (ret)
2749                         dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id);
2750                 return ret;
2751         }
2752
2753         addr = qm_ctx_alloc(qm, size, &dma_addr);
2754         if (IS_ERR(addr)) {
2755                 dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n");
2756                 return -ENOMEM;
2757         }
2758
2759         while (++i) {
2760                 ret = qm_dump_sqc_raw(qm, dma_addr, qp->qp_id);
2761                 if (ret) {
2762                         dev_err_ratelimited(dev, "Failed to dump sqc!\n");
2763                         break;
2764                 }
2765                 sqc = addr;
2766
2767                 ret = qm_dump_cqc_raw(qm, (dma_addr + sizeof(struct qm_sqc)),
2768                                       qp->qp_id);
2769                 if (ret) {
2770                         dev_err_ratelimited(dev, "Failed to dump cqc!\n");
2771                         break;
2772                 }
2773                 cqc = addr + sizeof(struct qm_sqc);
2774
2775                 if ((sqc->tail == cqc->tail) &&
2776                     (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))
2777                         break;
2778
2779                 if (i == MAX_WAIT_COUNTS) {
2780                         dev_err(dev, "Fail to empty queue %u!\n", qp->qp_id);
2781                         ret = -EBUSY;
2782                         break;
2783                 }
2784
2785                 usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);
2786         }
2787
2788         qm_ctx_free(qm, size, addr, &dma_addr);
2789
2790         return ret;
2791 }
2792
2793 static int qm_stop_qp_nolock(struct hisi_qp *qp)
2794 {
2795         struct device *dev = &qp->qm->pdev->dev;
2796         int ret;
2797
2798         /*
2799          * It is allowed to stop and release qp when reset, If the qp is
2800          * stopped when reset but still want to be released then, the
2801          * is_resetting flag should be set negative so that this qp will not
2802          * be restarted after reset.
2803          */
2804         if (atomic_read(&qp->qp_status.flags) == QP_STOP) {
2805                 qp->is_resetting = false;
2806                 return 0;
2807         }
2808
2809         if (!qm_qp_avail_state(qp->qm, qp, QP_STOP))
2810                 return -EPERM;
2811
2812         atomic_set(&qp->qp_status.flags, QP_STOP);
2813
2814         ret = qm_drain_qp(qp);
2815         if (ret)
2816                 dev_err(dev, "Failed to drain out data for stopping!\n");
2817
2818         if (qp->qm->wq)
2819                 flush_workqueue(qp->qm->wq);
2820         else
2821                 flush_work(&qp->qm->work);
2822
2823         if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))
2824                 qp_stop_fail_cb(qp);
2825
2826         dev_dbg(dev, "stop queue %u!", qp->qp_id);
2827
2828         return 0;
2829 }
2830
2831 /**
2832  * hisi_qm_stop_qp() - Stop a qp in qm.
2833  * @qp: The qp we want to stop.
2834  *
2835  * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
2836  */
2837 int hisi_qm_stop_qp(struct hisi_qp *qp)
2838 {
2839         int ret;
2840
2841         down_write(&qp->qm->qps_lock);
2842         ret = qm_stop_qp_nolock(qp);
2843         up_write(&qp->qm->qps_lock);
2844
2845         return ret;
2846 }
2847 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
2848
2849 /**
2850  * hisi_qp_send() - Queue up a task in the hardware queue.
2851  * @qp: The qp in which to put the message.
2852  * @msg: The message.
2853  *
2854  * This function will return -EBUSY if qp is currently full, and -EAGAIN
2855  * if qp related qm is resetting.
2856  *
2857  * Note: This function may run with qm_irq_thread and ACC reset at same time.
2858  *       It has no race with qm_irq_thread. However, during hisi_qp_send, ACC
2859  *       reset may happen, we have no lock here considering performance. This
2860  *       causes current qm_db sending fail or can not receive sended sqe. QM
2861  *       sync/async receive function should handle the error sqe. ACC reset
2862  *       done function should clear used sqe to 0.
2863  */
2864 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
2865 {
2866         struct hisi_qp_status *qp_status = &qp->qp_status;
2867         u16 sq_tail = qp_status->sq_tail;
2868         u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
2869         void *sqe = qm_get_avail_sqe(qp);
2870
2871         if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||
2872                      atomic_read(&qp->qm->status.flags) == QM_STOP ||
2873                      qp->is_resetting)) {
2874                 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
2875                 return -EAGAIN;
2876         }
2877
2878         if (!sqe)
2879                 return -EBUSY;
2880
2881         memcpy(sqe, msg, qp->qm->sqe_size);
2882
2883         qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
2884         atomic_inc(&qp->qp_status.used);
2885         qp_status->sq_tail = sq_tail_next;
2886
2887         return 0;
2888 }
2889 EXPORT_SYMBOL_GPL(hisi_qp_send);
2890
2891 static void hisi_qm_cache_wb(struct hisi_qm *qm)
2892 {
2893         unsigned int val;
2894
2895         if (qm->ver == QM_HW_V1)
2896                 return;
2897
2898         writel(0x1, qm->io_base + QM_CACHE_WB_START);
2899         if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
2900                                        val, val & BIT(0), POLL_PERIOD,
2901                                        POLL_TIMEOUT))
2902                 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
2903 }
2904
2905 static void qm_qp_event_notifier(struct hisi_qp *qp)
2906 {
2907         wake_up_interruptible(&qp->uacce_q->wait);
2908 }
2909
2910 static int hisi_qm_get_available_instances(struct uacce_device *uacce)
2911 {
2912         return hisi_qm_get_free_qp_num(uacce->priv);
2913 }
2914
2915 static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,
2916                                    unsigned long arg,
2917                                    struct uacce_queue *q)
2918 {
2919         struct hisi_qm *qm = uacce->priv;
2920         struct hisi_qp *qp;
2921         u8 alg_type = 0;
2922
2923         qp = hisi_qm_create_qp(qm, alg_type);
2924         if (IS_ERR(qp))
2925                 return PTR_ERR(qp);
2926
2927         q->priv = qp;
2928         q->uacce = uacce;
2929         qp->uacce_q = q;
2930         qp->event_cb = qm_qp_event_notifier;
2931         qp->pasid = arg;
2932         qp->is_in_kernel = false;
2933
2934         return 0;
2935 }
2936
2937 static void hisi_qm_uacce_put_queue(struct uacce_queue *q)
2938 {
2939         struct hisi_qp *qp = q->priv;
2940
2941         hisi_qm_cache_wb(qp->qm);
2942         hisi_qm_release_qp(qp);
2943 }
2944
2945 /* map sq/cq/doorbell to user space */
2946 static int hisi_qm_uacce_mmap(struct uacce_queue *q,
2947                               struct vm_area_struct *vma,
2948                               struct uacce_qfile_region *qfr)
2949 {
2950         struct hisi_qp *qp = q->priv;
2951         struct hisi_qm *qm = qp->qm;
2952         resource_size_t phys_base = qm->db_phys_base +
2953                                     qp->qp_id * qm->db_interval;
2954         size_t sz = vma->vm_end - vma->vm_start;
2955         struct pci_dev *pdev = qm->pdev;
2956         struct device *dev = &pdev->dev;
2957         unsigned long vm_pgoff;
2958         int ret;
2959
2960         switch (qfr->type) {
2961         case UACCE_QFRT_MMIO:
2962                 if (qm->ver == QM_HW_V1) {
2963                         if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
2964                                 return -EINVAL;
2965                 } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) {
2966                         if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
2967                             QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
2968                                 return -EINVAL;
2969                 } else {
2970                         if (sz > qm->db_interval)
2971                                 return -EINVAL;
2972                 }
2973
2974                 vma->vm_flags |= VM_IO;
2975
2976                 return remap_pfn_range(vma, vma->vm_start,
2977                                        phys_base >> PAGE_SHIFT,
2978                                        sz, pgprot_noncached(vma->vm_page_prot));
2979         case UACCE_QFRT_DUS:
2980                 if (sz != qp->qdma.size)
2981                         return -EINVAL;
2982
2983                 /*
2984                  * dma_mmap_coherent() requires vm_pgoff as 0
2985                  * restore vm_pfoff to initial value for mmap()
2986                  */
2987                 vm_pgoff = vma->vm_pgoff;
2988                 vma->vm_pgoff = 0;
2989                 ret = dma_mmap_coherent(dev, vma, qp->qdma.va,
2990                                         qp->qdma.dma, sz);
2991                 vma->vm_pgoff = vm_pgoff;
2992                 return ret;
2993
2994         default:
2995                 return -EINVAL;
2996         }
2997 }
2998
2999 static int hisi_qm_uacce_start_queue(struct uacce_queue *q)
3000 {
3001         struct hisi_qp *qp = q->priv;
3002
3003         return hisi_qm_start_qp(qp, qp->pasid);
3004 }
3005
3006 static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)
3007 {
3008         hisi_qm_stop_qp(q->priv);
3009 }
3010
3011 static int hisi_qm_is_q_updated(struct uacce_queue *q)
3012 {
3013         struct hisi_qp *qp = q->priv;
3014         struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
3015         int updated = 0;
3016
3017         while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
3018                 /* make sure to read data from memory */
3019                 dma_rmb();
3020                 qm_cq_head_update(qp);
3021                 cqe = qp->cqe + qp->qp_status.cq_head;
3022                 updated = 1;
3023         }
3024
3025         return updated;
3026 }
3027
3028 static void qm_set_sqctype(struct uacce_queue *q, u16 type)
3029 {
3030         struct hisi_qm *qm = q->uacce->priv;
3031         struct hisi_qp *qp = q->priv;
3032
3033         down_write(&qm->qps_lock);
3034         qp->alg_type = type;
3035         up_write(&qm->qps_lock);
3036 }
3037
3038 static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,
3039                                 unsigned long arg)
3040 {
3041         struct hisi_qp *qp = q->priv;
3042         struct hisi_qp_ctx qp_ctx;
3043
3044         if (cmd == UACCE_CMD_QM_SET_QP_CTX) {
3045                 if (copy_from_user(&qp_ctx, (void __user *)arg,
3046                                    sizeof(struct hisi_qp_ctx)))
3047                         return -EFAULT;
3048
3049                 if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1)
3050                         return -EINVAL;
3051
3052                 qm_set_sqctype(q, qp_ctx.qc_type);
3053                 qp_ctx.id = qp->qp_id;
3054
3055                 if (copy_to_user((void __user *)arg, &qp_ctx,
3056                                  sizeof(struct hisi_qp_ctx)))
3057                         return -EFAULT;
3058         } else {
3059                 return -EINVAL;
3060         }
3061
3062         return 0;
3063 }
3064
3065 static const struct uacce_ops uacce_qm_ops = {
3066         .get_available_instances = hisi_qm_get_available_instances,
3067         .get_queue = hisi_qm_uacce_get_queue,
3068         .put_queue = hisi_qm_uacce_put_queue,
3069         .start_queue = hisi_qm_uacce_start_queue,
3070         .stop_queue = hisi_qm_uacce_stop_queue,
3071         .mmap = hisi_qm_uacce_mmap,
3072         .ioctl = hisi_qm_uacce_ioctl,
3073         .is_q_updated = hisi_qm_is_q_updated,
3074 };
3075
3076 static int qm_alloc_uacce(struct hisi_qm *qm)
3077 {
3078         struct pci_dev *pdev = qm->pdev;
3079         struct uacce_device *uacce;
3080         unsigned long mmio_page_nr;
3081         unsigned long dus_page_nr;
3082         struct uacce_interface interface = {
3083                 .flags = UACCE_DEV_SVA,
3084                 .ops = &uacce_qm_ops,
3085         };
3086         int ret;
3087
3088         ret = strscpy(interface.name, pdev->driver->name,
3089                       sizeof(interface.name));
3090         if (ret < 0)
3091                 return -ENAMETOOLONG;
3092
3093         uacce = uacce_alloc(&pdev->dev, &interface);
3094         if (IS_ERR(uacce))
3095                 return PTR_ERR(uacce);
3096
3097         if (uacce->flags & UACCE_DEV_SVA && qm->mode == UACCE_MODE_SVA) {
3098                 qm->use_sva = true;
3099         } else {
3100                 /* only consider sva case */
3101                 uacce_remove(uacce);
3102                 qm->uacce = NULL;
3103                 return -EINVAL;
3104         }
3105
3106         uacce->is_vf = pdev->is_virtfn;
3107         uacce->priv = qm;
3108         uacce->algs = qm->algs;
3109
3110         if (qm->ver == QM_HW_V1)
3111                 uacce->api_ver = HISI_QM_API_VER_BASE;
3112         else if (qm->ver == QM_HW_V2)
3113                 uacce->api_ver = HISI_QM_API_VER2_BASE;
3114         else
3115                 uacce->api_ver = HISI_QM_API_VER3_BASE;
3116
3117         if (qm->ver == QM_HW_V1)
3118                 mmio_page_nr = QM_DOORBELL_PAGE_NR;
3119         else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation)
3120                 mmio_page_nr = QM_DOORBELL_PAGE_NR +
3121                         QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
3122         else
3123                 mmio_page_nr = qm->db_interval / PAGE_SIZE;
3124
3125         dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH +
3126                        sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT;
3127
3128         uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;
3129         uacce->qf_pg_num[UACCE_QFRT_DUS]  = dus_page_nr;
3130
3131         qm->uacce = uacce;
3132
3133         return 0;
3134 }
3135
3136 /**
3137  * qm_frozen() - Try to froze QM to cut continuous queue request. If
3138  * there is user on the QM, return failure without doing anything.
3139  * @qm: The qm needed to be fronzen.
3140  *
3141  * This function frozes QM, then we can do SRIOV disabling.
3142  */
3143 static int qm_frozen(struct hisi_qm *qm)
3144 {
3145         if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))
3146                 return 0;
3147
3148         down_write(&qm->qps_lock);
3149
3150         if (!qm->qp_in_used) {
3151                 qm->qp_in_used = qm->qp_num;
3152                 up_write(&qm->qps_lock);
3153                 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);
3154                 return 0;
3155         }
3156
3157         up_write(&qm->qps_lock);
3158
3159         return -EBUSY;
3160 }
3161
3162 static int qm_try_frozen_vfs(struct pci_dev *pdev,
3163                              struct hisi_qm_list *qm_list)
3164 {
3165         struct hisi_qm *qm, *vf_qm;
3166         struct pci_dev *dev;
3167         int ret = 0;
3168
3169         if (!qm_list || !pdev)
3170                 return -EINVAL;
3171
3172         /* Try to frozen all the VFs as disable SRIOV */
3173         mutex_lock(&qm_list->lock);
3174         list_for_each_entry(qm, &qm_list->list, list) {
3175                 dev = qm->pdev;
3176                 if (dev == pdev)
3177                         continue;
3178                 if (pci_physfn(dev) == pdev) {
3179                         vf_qm = pci_get_drvdata(dev);
3180                         ret = qm_frozen(vf_qm);
3181                         if (ret)
3182                                 goto frozen_fail;
3183                 }
3184         }
3185
3186 frozen_fail:
3187         mutex_unlock(&qm_list->lock);
3188
3189         return ret;
3190 }
3191
3192 /**
3193  * hisi_qm_wait_task_finish() - Wait until the task is finished
3194  * when removing the driver.
3195  * @qm: The qm needed to wait for the task to finish.
3196  * @qm_list: The list of all available devices.
3197  */
3198 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
3199 {
3200         while (qm_frozen(qm) ||
3201                ((qm->fun_type == QM_HW_PF) &&
3202                qm_try_frozen_vfs(qm->pdev, qm_list))) {
3203                 msleep(WAIT_PERIOD);
3204         }
3205
3206         while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||
3207                test_bit(QM_RESETTING, &qm->misc_ctl))
3208                 msleep(WAIT_PERIOD);
3209
3210         udelay(REMOVE_WAIT_DELAY);
3211 }
3212 EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
3213
3214 /**
3215  * hisi_qm_get_free_qp_num() - Get free number of qp in qm.
3216  * @qm: The qm which want to get free qp.
3217  *
3218  * This function return free number of qp in qm.
3219  */
3220 int hisi_qm_get_free_qp_num(struct hisi_qm *qm)
3221 {
3222         int ret;
3223
3224         down_read(&qm->qps_lock);
3225         ret = qm->qp_num - qm->qp_in_used;
3226         up_read(&qm->qps_lock);
3227
3228         return ret;
3229 }
3230 EXPORT_SYMBOL_GPL(hisi_qm_get_free_qp_num);
3231
3232 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
3233 {
3234         struct device *dev = &qm->pdev->dev;
3235         struct qm_dma *qdma;
3236         int i;
3237
3238         for (i = num - 1; i >= 0; i--) {
3239                 qdma = &qm->qp_array[i].qdma;
3240                 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
3241         }
3242
3243         kfree(qm->qp_array);
3244 }
3245
3246 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id)
3247 {
3248         struct device *dev = &qm->pdev->dev;
3249         size_t off = qm->sqe_size * QM_Q_DEPTH;
3250         struct hisi_qp *qp;
3251
3252         qp = &qm->qp_array[id];
3253         qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
3254                                          GFP_KERNEL);
3255         if (!qp->qdma.va)
3256                 return -ENOMEM;
3257
3258         qp->sqe = qp->qdma.va;
3259         qp->sqe_dma = qp->qdma.dma;
3260         qp->cqe = qp->qdma.va + off;
3261         qp->cqe_dma = qp->qdma.dma + off;
3262         qp->qdma.size = dma_size;
3263         qp->qm = qm;
3264         qp->qp_id = id;
3265
3266         return 0;
3267 }
3268
3269 static void hisi_qm_pre_init(struct hisi_qm *qm)
3270 {
3271         struct pci_dev *pdev = qm->pdev;
3272
3273         if (qm->ver == QM_HW_V1)
3274                 qm->ops = &qm_hw_ops_v1;
3275         else if (qm->ver == QM_HW_V2)
3276                 qm->ops = &qm_hw_ops_v2;
3277         else
3278                 qm->ops = &qm_hw_ops_v3;
3279
3280         pci_set_drvdata(pdev, qm);
3281         mutex_init(&qm->mailbox_lock);
3282         init_rwsem(&qm->qps_lock);
3283         qm->qp_in_used = 0;
3284         qm->misc_ctl = false;
3285         if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V2) {
3286                 if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev)))
3287                         dev_info(&pdev->dev, "_PS0 and _PR0 are not defined");
3288         }
3289 }
3290
3291 static void qm_cmd_uninit(struct hisi_qm *qm)
3292 {
3293         u32 val;
3294
3295         if (qm->ver < QM_HW_V3)
3296                 return;
3297
3298         val = readl(qm->io_base + QM_IFC_INT_MASK);
3299         val |= QM_IFC_INT_DISABLE;
3300         writel(val, qm->io_base + QM_IFC_INT_MASK);
3301 }
3302
3303 static void qm_cmd_init(struct hisi_qm *qm)
3304 {
3305         u32 val;
3306
3307         if (qm->ver < QM_HW_V3)
3308                 return;
3309
3310         /* Clear communication interrupt source */
3311         qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR);
3312
3313         /* Enable pf to vf communication reg. */
3314         val = readl(qm->io_base + QM_IFC_INT_MASK);
3315         val &= ~QM_IFC_INT_DISABLE;
3316         writel(val, qm->io_base + QM_IFC_INT_MASK);
3317 }
3318
3319 static void qm_put_pci_res(struct hisi_qm *qm)
3320 {
3321         struct pci_dev *pdev = qm->pdev;
3322
3323         if (qm->use_db_isolation)
3324                 iounmap(qm->db_io_base);
3325
3326         iounmap(qm->io_base);
3327         pci_release_mem_regions(pdev);
3328 }
3329
3330 static void hisi_qm_pci_uninit(struct hisi_qm *qm)
3331 {
3332         struct pci_dev *pdev = qm->pdev;
3333
3334         pci_free_irq_vectors(pdev);
3335         qm_put_pci_res(qm);
3336         pci_disable_device(pdev);
3337 }
3338
3339 /**
3340  * hisi_qm_uninit() - Uninitialize qm.
3341  * @qm: The qm needed uninit.
3342  *
3343  * This function uninits qm related device resources.
3344  */
3345 void hisi_qm_uninit(struct hisi_qm *qm)
3346 {
3347         struct pci_dev *pdev = qm->pdev;
3348         struct device *dev = &pdev->dev;
3349
3350         qm_cmd_uninit(qm);
3351         kfree(qm->factor);
3352         down_write(&qm->qps_lock);
3353
3354         if (!qm_avail_state(qm, QM_CLOSE)) {
3355                 up_write(&qm->qps_lock);
3356                 return;
3357         }
3358
3359         hisi_qp_memory_uninit(qm, qm->qp_num);
3360         idr_destroy(&qm->qp_idr);
3361
3362         if (qm->qdma.va) {
3363                 hisi_qm_cache_wb(qm);
3364                 dma_free_coherent(dev, qm->qdma.size,
3365                                   qm->qdma.va, qm->qdma.dma);
3366         }
3367
3368         qm_irq_unregister(qm);
3369         hisi_qm_pci_uninit(qm);
3370         uacce_remove(qm->uacce);
3371         qm->uacce = NULL;
3372
3373         up_write(&qm->qps_lock);
3374 }
3375 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
3376
3377 /**
3378  * hisi_qm_get_vft() - Get vft from a qm.
3379  * @qm: The qm we want to get its vft.
3380  * @base: The base number of queue in vft.
3381  * @number: The number of queues in vft.
3382  *
3383  * We can allocate multiple queues to a qm by configuring virtual function
3384  * table. We get related configures by this function. Normally, we call this
3385  * function in VF driver to get the queue information.
3386  *
3387  * qm hw v1 does not support this interface.
3388  */
3389 int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
3390 {
3391         if (!base || !number)
3392                 return -EINVAL;
3393
3394         if (!qm->ops->get_vft) {
3395                 dev_err(&qm->pdev->dev, "Don't support vft read!\n");
3396                 return -EINVAL;
3397         }
3398
3399         return qm->ops->get_vft(qm, base, number);
3400 }
3401 EXPORT_SYMBOL_GPL(hisi_qm_get_vft);
3402
3403 /**
3404  * hisi_qm_set_vft() - Set vft to a qm.
3405  * @qm: The qm we want to set its vft.
3406  * @fun_num: The function number.
3407  * @base: The base number of queue in vft.
3408  * @number: The number of queues in vft.
3409  *
3410  * This function is alway called in PF driver, it is used to assign queues
3411  * among PF and VFs.
3412  *
3413  * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
3414  * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
3415  * (VF function number 0x2)
3416  */
3417 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
3418                     u32 number)
3419 {
3420         u32 max_q_num = qm->ctrl_qp_num;
3421
3422         if (base >= max_q_num || number > max_q_num ||
3423             (base + number) > max_q_num)
3424                 return -EINVAL;
3425
3426         return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
3427 }
3428
3429 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
3430 {
3431         struct hisi_qm_status *status = &qm->status;
3432
3433         status->eq_head = 0;
3434         status->aeq_head = 0;
3435         status->eqc_phase = true;
3436         status->aeqc_phase = true;
3437 }
3438
3439 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
3440 {
3441         struct device *dev = &qm->pdev->dev;
3442         struct qm_eqc *eqc;
3443         dma_addr_t eqc_dma;
3444         int ret;
3445
3446         eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
3447         if (!eqc)
3448                 return -ENOMEM;
3449
3450         eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));
3451         eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));
3452         if (qm->ver == QM_HW_V1)
3453                 eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);
3454         eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
3455
3456         eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
3457                                  DMA_TO_DEVICE);
3458         if (dma_mapping_error(dev, eqc_dma)) {
3459                 kfree(eqc);
3460                 return -ENOMEM;
3461         }
3462
3463         ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
3464         dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
3465         kfree(eqc);
3466
3467         return ret;
3468 }
3469
3470 static int qm_aeq_ctx_cfg(struct hisi_qm *qm)
3471 {
3472         struct device *dev = &qm->pdev->dev;
3473         struct qm_aeqc *aeqc;
3474         dma_addr_t aeqc_dma;
3475         int ret;
3476
3477         aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
3478         if (!aeqc)
3479                 return -ENOMEM;
3480
3481         aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));
3482         aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));
3483         aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT));
3484
3485         aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
3486                                   DMA_TO_DEVICE);
3487         if (dma_mapping_error(dev, aeqc_dma)) {
3488                 kfree(aeqc);
3489                 return -ENOMEM;
3490         }
3491
3492         ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
3493         dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
3494         kfree(aeqc);
3495
3496         return ret;
3497 }
3498
3499 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)
3500 {
3501         struct device *dev = &qm->pdev->dev;
3502         int ret;
3503
3504         qm_init_eq_aeq_status(qm);
3505
3506         ret = qm_eq_ctx_cfg(qm);
3507         if (ret) {
3508                 dev_err(dev, "Set eqc failed!\n");
3509                 return ret;
3510         }
3511
3512         return qm_aeq_ctx_cfg(qm);
3513 }
3514
3515 static int __hisi_qm_start(struct hisi_qm *qm)
3516 {
3517         int ret;
3518
3519         WARN_ON(!qm->qdma.va);
3520
3521         if (qm->fun_type == QM_HW_PF) {
3522                 ret = qm_dev_mem_reset(qm);
3523                 if (ret)
3524                         return ret;
3525
3526                 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
3527                 if (ret)
3528                         return ret;
3529         }
3530
3531         ret = qm_eq_aeq_ctx_cfg(qm);
3532         if (ret)
3533                 return ret;
3534
3535         ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
3536         if (ret)
3537                 return ret;
3538
3539         ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
3540         if (ret)
3541                 return ret;
3542
3543         qm_init_prefetch(qm);
3544
3545         writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
3546         writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
3547
3548         return 0;
3549 }
3550
3551 /**
3552  * hisi_qm_start() - start qm
3553  * @qm: The qm to be started.
3554  *
3555  * This function starts a qm, then we can allocate qp from this qm.
3556  */
3557 int hisi_qm_start(struct hisi_qm *qm)
3558 {
3559         struct device *dev = &qm->pdev->dev;
3560         int ret = 0;
3561
3562         down_write(&qm->qps_lock);
3563
3564         if (!qm_avail_state(qm, QM_START)) {
3565                 up_write(&qm->qps_lock);
3566                 return -EPERM;
3567         }
3568
3569         dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);
3570
3571         if (!qm->qp_num) {
3572                 dev_err(dev, "qp_num should not be 0\n");
3573                 ret = -EINVAL;
3574                 goto err_unlock;
3575         }
3576
3577         ret = __hisi_qm_start(qm);
3578         if (!ret)
3579                 atomic_set(&qm->status.flags, QM_START);
3580
3581 err_unlock:
3582         up_write(&qm->qps_lock);
3583         return ret;
3584 }
3585 EXPORT_SYMBOL_GPL(hisi_qm_start);
3586
3587 static int qm_restart(struct hisi_qm *qm)
3588 {
3589         struct device *dev = &qm->pdev->dev;
3590         struct hisi_qp *qp;
3591         int ret, i;
3592
3593         ret = hisi_qm_start(qm);
3594         if (ret < 0)
3595                 return ret;
3596
3597         down_write(&qm->qps_lock);
3598         for (i = 0; i < qm->qp_num; i++) {
3599                 qp = &qm->qp_array[i];
3600                 if (atomic_read(&qp->qp_status.flags) == QP_STOP &&
3601                     qp->is_resetting == true) {
3602                         ret = qm_start_qp_nolock(qp, 0);
3603                         if (ret < 0) {
3604                                 dev_err(dev, "Failed to start qp%d!\n", i);
3605
3606                                 up_write(&qm->qps_lock);
3607                                 return ret;
3608                         }
3609                         qp->is_resetting = false;
3610                 }
3611         }
3612         up_write(&qm->qps_lock);
3613
3614         return 0;
3615 }
3616
3617 /* Stop started qps in reset flow */
3618 static int qm_stop_started_qp(struct hisi_qm *qm)
3619 {
3620         struct device *dev = &qm->pdev->dev;
3621         struct hisi_qp *qp;
3622         int i, ret;
3623
3624         for (i = 0; i < qm->qp_num; i++) {
3625                 qp = &qm->qp_array[i];
3626                 if (qp && atomic_read(&qp->qp_status.flags) == QP_START) {
3627                         qp->is_resetting = true;
3628                         ret = qm_stop_qp_nolock(qp);
3629                         if (ret < 0) {
3630                                 dev_err(dev, "Failed to stop qp%d!\n", i);
3631                                 return ret;
3632                         }
3633                 }
3634         }
3635
3636         return 0;
3637 }
3638
3639
3640 /**
3641  * qm_clear_queues() - Clear all queues memory in a qm.
3642  * @qm: The qm in which the queues will be cleared.
3643  *
3644  * This function clears all queues memory in a qm. Reset of accelerator can
3645  * use this to clear queues.
3646  */
3647 static void qm_clear_queues(struct hisi_qm *qm)
3648 {
3649         struct hisi_qp *qp;
3650         int i;
3651
3652         for (i = 0; i < qm->qp_num; i++) {
3653                 qp = &qm->qp_array[i];
3654                 if (qp->is_resetting)
3655                         memset(qp->qdma.va, 0, qp->qdma.size);
3656         }
3657
3658         memset(qm->qdma.va, 0, qm->qdma.size);
3659 }
3660
3661 /**
3662  * hisi_qm_stop() - Stop a qm.
3663  * @qm: The qm which will be stopped.
3664  * @r: The reason to stop qm.
3665  *
3666  * This function stops qm and its qps, then qm can not accept request.
3667  * Related resources are not released at this state, we can use hisi_qm_start
3668  * to let qm start again.
3669  */
3670 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
3671 {
3672         struct device *dev = &qm->pdev->dev;
3673         int ret = 0;
3674
3675         down_write(&qm->qps_lock);
3676
3677         qm->status.stop_reason = r;
3678         if (!qm_avail_state(qm, QM_STOP)) {
3679                 ret = -EPERM;
3680                 goto err_unlock;
3681         }
3682
3683         if (qm->status.stop_reason == QM_SOFT_RESET ||
3684             qm->status.stop_reason == QM_FLR) {
3685                 ret = qm_stop_started_qp(qm);
3686                 if (ret < 0) {
3687                         dev_err(dev, "Failed to stop started qp!\n");
3688                         goto err_unlock;
3689                 }
3690         }
3691
3692         /* Mask eq and aeq irq */
3693         writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
3694         writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
3695
3696         if (qm->fun_type == QM_HW_PF) {
3697                 ret = hisi_qm_set_vft(qm, 0, 0, 0);
3698                 if (ret < 0) {
3699                         dev_err(dev, "Failed to set vft!\n");
3700                         ret = -EBUSY;
3701                         goto err_unlock;
3702                 }
3703         }
3704
3705         qm_clear_queues(qm);
3706         atomic_set(&qm->status.flags, QM_STOP);
3707
3708 err_unlock:
3709         up_write(&qm->qps_lock);
3710         return ret;
3711 }
3712 EXPORT_SYMBOL_GPL(hisi_qm_stop);
3713
3714 static ssize_t qm_status_read(struct file *filp, char __user *buffer,
3715                               size_t count, loff_t *pos)
3716 {
3717         struct hisi_qm *qm = filp->private_data;
3718         char buf[QM_DBG_READ_LEN];
3719         int val, len;
3720
3721         val = atomic_read(&qm->status.flags);
3722         len = scnprintf(buf, QM_DBG_READ_LEN, "%s\n", qm_s[val]);
3723
3724         return simple_read_from_buffer(buffer, count, pos, buf, len);
3725 }
3726
3727 static const struct file_operations qm_status_fops = {
3728         .owner = THIS_MODULE,
3729         .open = simple_open,
3730         .read = qm_status_read,
3731 };
3732
3733 static int qm_debugfs_atomic64_set(void *data, u64 val)
3734 {
3735         if (val)
3736                 return -EINVAL;
3737
3738         atomic64_set((atomic64_t *)data, 0);
3739
3740         return 0;
3741 }
3742
3743 static int qm_debugfs_atomic64_get(void *data, u64 *val)
3744 {
3745         *val = atomic64_read((atomic64_t *)data);
3746
3747         return 0;
3748 }
3749
3750 DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get,
3751                          qm_debugfs_atomic64_set, "%llu\n");
3752
3753 static void qm_hw_error_init(struct hisi_qm *qm)
3754 {
3755         struct hisi_qm_err_info *err_info = &qm->err_info;
3756
3757         if (!qm->ops->hw_error_init) {
3758                 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
3759                 return;
3760         }
3761
3762         qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, err_info->fe);
3763 }
3764
3765 static void qm_hw_error_uninit(struct hisi_qm *qm)
3766 {
3767         if (!qm->ops->hw_error_uninit) {
3768                 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");
3769                 return;
3770         }
3771
3772         qm->ops->hw_error_uninit(qm);
3773 }
3774
3775 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)
3776 {
3777         if (!qm->ops->hw_error_handle) {
3778                 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");
3779                 return ACC_ERR_NONE;
3780         }
3781
3782         return qm->ops->hw_error_handle(qm);
3783 }
3784
3785 /**
3786  * hisi_qm_dev_err_init() - Initialize device error configuration.
3787  * @qm: The qm for which we want to do error initialization.
3788  *
3789  * Initialize QM and device error related configuration.
3790  */
3791 void hisi_qm_dev_err_init(struct hisi_qm *qm)
3792 {
3793         if (qm->fun_type == QM_HW_VF)
3794                 return;
3795
3796         qm_hw_error_init(qm);
3797
3798         if (!qm->err_ini->hw_err_enable) {
3799                 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");
3800                 return;
3801         }
3802         qm->err_ini->hw_err_enable(qm);
3803 }
3804 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);
3805
3806 /**
3807  * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.
3808  * @qm: The qm for which we want to do error uninitialization.
3809  *
3810  * Uninitialize QM and device error related configuration.
3811  */
3812 void hisi_qm_dev_err_uninit(struct hisi_qm *qm)
3813 {
3814         if (qm->fun_type == QM_HW_VF)
3815                 return;
3816
3817         qm_hw_error_uninit(qm);
3818
3819         if (!qm->err_ini->hw_err_disable) {
3820                 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");
3821                 return;
3822         }
3823         qm->err_ini->hw_err_disable(qm);
3824 }
3825 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);
3826
3827 /**
3828  * hisi_qm_free_qps() - free multiple queue pairs.
3829  * @qps: The queue pairs need to be freed.
3830  * @qp_num: The num of queue pairs.
3831  */
3832 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)
3833 {
3834         int i;
3835
3836         if (!qps || qp_num <= 0)
3837                 return;
3838
3839         for (i = qp_num - 1; i >= 0; i--)
3840                 hisi_qm_release_qp(qps[i]);
3841 }
3842 EXPORT_SYMBOL_GPL(hisi_qm_free_qps);
3843
3844 static void free_list(struct list_head *head)
3845 {
3846         struct hisi_qm_resource *res, *tmp;
3847
3848         list_for_each_entry_safe(res, tmp, head, list) {
3849                 list_del(&res->list);
3850                 kfree(res);
3851         }
3852 }
3853
3854 static int hisi_qm_sort_devices(int node, struct list_head *head,
3855                                 struct hisi_qm_list *qm_list)
3856 {
3857         struct hisi_qm_resource *res, *tmp;
3858         struct hisi_qm *qm;
3859         struct list_head *n;
3860         struct device *dev;
3861         int dev_node = 0;
3862
3863         list_for_each_entry(qm, &qm_list->list, list) {
3864                 dev = &qm->pdev->dev;
3865
3866                 if (IS_ENABLED(CONFIG_NUMA)) {
3867                         dev_node = dev_to_node(dev);
3868                         if (dev_node < 0)
3869                                 dev_node = 0;
3870                 }
3871
3872                 res = kzalloc(sizeof(*res), GFP_KERNEL);
3873                 if (!res)
3874                         return -ENOMEM;
3875
3876                 res->qm = qm;
3877                 res->distance = node_distance(dev_node, node);
3878                 n = head;
3879                 list_for_each_entry(tmp, head, list) {
3880                         if (res->distance < tmp->distance) {
3881                                 n = &tmp->list;
3882                                 break;
3883                         }
3884                 }
3885                 list_add_tail(&res->list, n);
3886         }
3887
3888         return 0;
3889 }
3890
3891 /**
3892  * hisi_qm_alloc_qps_node() - Create multiple queue pairs.
3893  * @qm_list: The list of all available devices.
3894  * @qp_num: The number of queue pairs need created.
3895  * @alg_type: The algorithm type.
3896  * @node: The numa node.
3897  * @qps: The queue pairs need created.
3898  *
3899  * This function will sort all available device according to numa distance.
3900  * Then try to create all queue pairs from one device, if all devices do
3901  * not meet the requirements will return error.
3902  */
3903 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,
3904                            u8 alg_type, int node, struct hisi_qp **qps)
3905 {
3906         struct hisi_qm_resource *tmp;
3907         int ret = -ENODEV;
3908         LIST_HEAD(head);
3909         int i;
3910
3911         if (!qps || !qm_list || qp_num <= 0)
3912                 return -EINVAL;
3913
3914         mutex_lock(&qm_list->lock);
3915         if (hisi_qm_sort_devices(node, &head, qm_list)) {
3916                 mutex_unlock(&qm_list->lock);
3917                 goto err;
3918         }
3919
3920         list_for_each_entry(tmp, &head, list) {
3921                 for (i = 0; i < qp_num; i++) {
3922                         qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);
3923                         if (IS_ERR(qps[i])) {
3924                                 hisi_qm_free_qps(qps, i);
3925                                 break;
3926                         }
3927                 }
3928
3929                 if (i == qp_num) {
3930                         ret = 0;
3931                         break;
3932                 }
3933         }
3934
3935         mutex_unlock(&qm_list->lock);
3936         if (ret)
3937                 pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",
3938                         node, alg_type, qp_num);
3939
3940 err:
3941         free_list(&head);
3942         return ret;
3943 }
3944 EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
3945
3946 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
3947 {
3948         u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
3949         u32 max_qp_num = qm->max_qp_num;
3950         u32 q_base = qm->qp_num;
3951         int ret;
3952
3953         if (!num_vfs)
3954                 return -EINVAL;
3955
3956         vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
3957
3958         /* If vfs_q_num is less than num_vfs, return error. */
3959         if (vfs_q_num < num_vfs)
3960                 return -EINVAL;
3961
3962         q_num = vfs_q_num / num_vfs;
3963         remain_q_num = vfs_q_num % num_vfs;
3964
3965         for (i = num_vfs; i > 0; i--) {
3966                 /*
3967                  * if q_num + remain_q_num > max_qp_num in last vf, divide the
3968                  * remaining queues equally.
3969                  */
3970                 if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
3971                         act_q_num = q_num + remain_q_num;
3972                         remain_q_num = 0;
3973                 } else if (remain_q_num > 0) {
3974                         act_q_num = q_num + 1;
3975                         remain_q_num--;
3976                 } else {
3977                         act_q_num = q_num;
3978                 }
3979
3980                 act_q_num = min_t(int, act_q_num, max_qp_num);
3981                 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
3982                 if (ret) {
3983                         for (j = num_vfs; j > i; j--)
3984                                 hisi_qm_set_vft(qm, j, 0, 0);
3985                         return ret;
3986                 }
3987                 q_base += act_q_num;
3988         }
3989
3990         return 0;
3991 }
3992
3993 static int qm_clear_vft_config(struct hisi_qm *qm)
3994 {
3995         int ret;
3996         u32 i;
3997
3998         for (i = 1; i <= qm->vfs_num; i++) {
3999                 ret = hisi_qm_set_vft(qm, i, 0, 0);
4000                 if (ret)
4001                         return ret;
4002         }
4003         qm->vfs_num = 0;
4004
4005         return 0;
4006 }
4007
4008 static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos)
4009 {
4010         struct device *dev = &qm->pdev->dev;
4011         u32 ir = qos * QM_QOS_RATE;
4012         int ret, total_vfs, i;
4013
4014         total_vfs = pci_sriov_get_totalvfs(qm->pdev);
4015         if (fun_index > total_vfs)
4016                 return -EINVAL;
4017
4018         qm->factor[fun_index].func_qos = qos;
4019
4020         ret = qm_get_shaper_para(ir, &qm->factor[fun_index]);
4021         if (ret) {
4022                 dev_err(dev, "failed to calculate shaper parameter!\n");
4023                 return -EINVAL;
4024         }
4025
4026         for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {
4027                 /* The base number of queue reuse for different alg type */
4028                 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1);
4029                 if (ret) {
4030                         dev_err(dev, "type: %d, failed to set shaper vft!\n", i);
4031                         return -EINVAL;
4032                 }
4033         }
4034
4035         return 0;
4036 }
4037
4038 static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index)
4039 {
4040         u64 cir_u = 0, cir_b = 0, cir_s = 0;
4041         u64 shaper_vft, ir_calc, ir;
4042         unsigned int val;
4043         u32 error_rate;
4044         int ret;
4045
4046         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
4047                                          val & BIT(0), POLL_PERIOD,
4048                                          POLL_TIMEOUT);
4049         if (ret)
4050                 return 0;
4051
4052         writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);
4053         writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE);
4054         writel(fun_index, qm->io_base + QM_VFT_CFG);
4055
4056         writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
4057         writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
4058
4059         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
4060                                          val & BIT(0), POLL_PERIOD,
4061                                          POLL_TIMEOUT);
4062         if (ret)
4063                 return 0;
4064
4065         shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |
4066                   ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32);
4067
4068         cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK;
4069         cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK;
4070         cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT;
4071
4072         cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK;
4073         cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT;
4074
4075         ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
4076
4077         ir = qm->factor[fun_index].func_qos * QM_QOS_RATE;
4078
4079         error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
4080         if (error_rate > QM_QOS_MIN_ERROR_RATE) {
4081                 pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate);
4082                 return 0;
4083         }
4084
4085         return ir;
4086 }
4087
4088 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num)
4089 {
4090         struct device *dev = &qm->pdev->dev;
4091         u64 mb_cmd;
4092         u32 qos;
4093         int ret;
4094
4095         qos = qm_get_shaper_vft_qos(qm, fun_num);
4096         if (!qos) {
4097                 dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num);
4098                 return;
4099         }
4100
4101         mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT;
4102         ret = qm_ping_single_vf(qm, mb_cmd, fun_num);
4103         if (ret)
4104                 dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num);
4105 }
4106
4107 static int qm_vf_read_qos(struct hisi_qm *qm)
4108 {
4109         int cnt = 0;
4110         int ret;
4111
4112         /* reset mailbox qos val */
4113         qm->mb_qos = 0;
4114
4115         /* vf ping pf to get function qos */
4116         if (qm->ops->ping_pf) {
4117                 ret = qm->ops->ping_pf(qm, QM_VF_GET_QOS);
4118                 if (ret) {
4119                         pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n");
4120                         return ret;
4121                 }
4122         }
4123
4124         while (true) {
4125                 msleep(QM_WAIT_DST_ACK);
4126                 if (qm->mb_qos)
4127                         break;
4128
4129                 if (++cnt > QM_MAX_VF_WAIT_COUNT) {
4130                         pci_err(qm->pdev, "PF ping VF timeout!\n");
4131                         return  -ETIMEDOUT;
4132                 }
4133         }
4134
4135         return ret;
4136 }
4137
4138 static ssize_t qm_algqos_read(struct file *filp, char __user *buf,
4139                                size_t count, loff_t *pos)
4140 {
4141         struct hisi_qm *qm = filp->private_data;
4142         char tbuf[QM_DBG_READ_LEN];
4143         u32 qos_val, ir;
4144         int ret;
4145
4146         ret = hisi_qm_get_dfx_access(qm);
4147         if (ret)
4148                 return ret;
4149
4150         /* Mailbox and reset cannot be operated at the same time */
4151         if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
4152                 pci_err(qm->pdev, "dev resetting, read alg qos failed!\n");
4153                 ret = -EAGAIN;
4154                 goto err_put_dfx_access;
4155         }
4156
4157         if (qm->fun_type == QM_HW_PF) {
4158                 ir = qm_get_shaper_vft_qos(qm, 0);
4159         } else {
4160                 ret = qm_vf_read_qos(qm);
4161                 if (ret)
4162                         goto err_get_status;
4163                 ir = qm->mb_qos;
4164         }
4165
4166         qos_val = ir / QM_QOS_RATE;
4167         ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val);
4168
4169         ret =  simple_read_from_buffer(buf, count, pos, tbuf, ret);
4170
4171 err_get_status:
4172         clear_bit(QM_RESETTING, &qm->misc_ctl);
4173 err_put_dfx_access:
4174         hisi_qm_put_dfx_access(qm);
4175         return ret;
4176 }
4177
4178 static ssize_t qm_qos_value_init(const char *buf, unsigned long *val)
4179 {
4180         int buflen = strlen(buf);
4181         int ret, i;
4182
4183         for (i = 0; i < buflen; i++) {
4184                 if (!isdigit(buf[i]))
4185                         return -EINVAL;
4186         }
4187
4188         ret = sscanf(buf, "%ld", val);
4189         if (ret != QM_QOS_VAL_NUM)
4190                 return -EINVAL;
4191
4192         return 0;
4193 }
4194
4195 static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
4196                                size_t count, loff_t *pos)
4197 {
4198         struct hisi_qm *qm = filp->private_data;
4199         char tbuf[QM_DBG_READ_LEN];
4200         int tmp1, bus, device, function;
4201         char tbuf_bdf[QM_DBG_READ_LEN] = {0};
4202         char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
4203         unsigned int fun_index;
4204         unsigned long val = 0;
4205         int len, ret;
4206
4207         if (qm->fun_type == QM_HW_VF)
4208                 return -EINVAL;
4209
4210         /* Mailbox and reset cannot be operated at the same time */
4211         if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
4212                 pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
4213                 return -EAGAIN;
4214         }
4215
4216         if (*pos != 0) {
4217                 ret = 0;
4218                 goto err_get_status;
4219         }
4220
4221         if (count >= QM_DBG_READ_LEN) {
4222                 ret = -ENOSPC;
4223                 goto err_get_status;
4224         }
4225
4226         len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
4227         if (len < 0) {
4228                 ret = len;
4229                 goto err_get_status;
4230         }
4231
4232         tbuf[len] = '\0';
4233         ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf);
4234         if (ret != QM_QOS_PARAM_NUM) {
4235                 ret = -EINVAL;
4236                 goto err_get_status;
4237         }
4238
4239         ret = qm_qos_value_init(val_buf, &val);
4240         if (val == 0 || val > QM_QOS_MAX_VAL || ret) {
4241                 pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
4242                 ret = -EINVAL;
4243                 goto err_get_status;
4244         }
4245
4246         ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function);
4247         if (ret != QM_QOS_BDF_PARAM_NUM) {
4248                 pci_err(qm->pdev, "input pci bdf value is error!\n");
4249                 ret = -EINVAL;
4250                 goto err_get_status;
4251         }
4252
4253         fun_index = device * 8 + function;
4254
4255         ret = qm_pm_get_sync(qm);
4256         if (ret) {
4257                 ret = -EINVAL;
4258                 goto err_get_status;
4259         }
4260
4261         ret = qm_func_shaper_enable(qm, fun_index, val);
4262         if (ret) {
4263                 pci_err(qm->pdev, "failed to enable function shaper!\n");
4264                 ret = -EINVAL;
4265                 goto err_put_sync;
4266         }
4267
4268         ret = count;
4269
4270 err_put_sync:
4271         qm_pm_put_sync(qm);
4272 err_get_status:
4273         clear_bit(QM_RESETTING, &qm->misc_ctl);
4274         return ret;
4275 }
4276
4277 static const struct file_operations qm_algqos_fops = {
4278         .owner = THIS_MODULE,
4279         .open = simple_open,
4280         .read = qm_algqos_read,
4281         .write = qm_algqos_write,
4282 };
4283
4284 /**
4285  * hisi_qm_set_algqos_init() - Initialize function qos debugfs files.
4286  * @qm: The qm for which we want to add debugfs files.
4287  *
4288  * Create function qos debugfs files.
4289  */
4290 static void hisi_qm_set_algqos_init(struct hisi_qm *qm)
4291 {
4292         if (qm->fun_type == QM_HW_PF)
4293                 debugfs_create_file("alg_qos", 0644, qm->debug.debug_root,
4294                                     qm, &qm_algqos_fops);
4295         else
4296                 debugfs_create_file("alg_qos", 0444, qm->debug.debug_root,
4297                                     qm, &qm_algqos_fops);
4298 }
4299
4300 /**
4301  * hisi_qm_debug_init() - Initialize qm related debugfs files.
4302  * @qm: The qm for which we want to add debugfs files.
4303  *
4304  * Create qm related debugfs files.
4305  */
4306 void hisi_qm_debug_init(struct hisi_qm *qm)
4307 {
4308         struct qm_dfx *dfx = &qm->debug.dfx;
4309         struct dentry *qm_d;
4310         void *data;
4311         int i;
4312
4313         qm_d = debugfs_create_dir("qm", qm->debug.debug_root);
4314         qm->debug.qm_d = qm_d;
4315
4316         /* only show this in PF */
4317         if (qm->fun_type == QM_HW_PF) {
4318                 qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM);
4319                 for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
4320                         qm_create_debugfs_file(qm, qm->debug.qm_d, i);
4321         }
4322
4323         debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops);
4324
4325         debugfs_create_file("cmd", 0600, qm->debug.qm_d, qm, &qm_cmd_fops);
4326
4327         debugfs_create_file("status", 0444, qm->debug.qm_d, qm,
4328                         &qm_status_fops);
4329         for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) {
4330                 data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset);
4331                 debugfs_create_file(qm_dfx_files[i].name,
4332                         0644,
4333                         qm_d,
4334                         data,
4335                         &qm_atomic64_ops);
4336         }
4337
4338         if (qm->ver >= QM_HW_V3)
4339                 hisi_qm_set_algqos_init(qm);
4340 }
4341 EXPORT_SYMBOL_GPL(hisi_qm_debug_init);
4342
4343 /**
4344  * hisi_qm_debug_regs_clear() - clear qm debug related registers.
4345  * @qm: The qm for which we want to clear its debug registers.
4346  */
4347 void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
4348 {
4349         const struct debugfs_reg32 *regs;
4350         int i;
4351
4352         /* clear current_qm */
4353         writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
4354         writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
4355
4356         /* clear current_q */
4357         writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
4358         writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
4359
4360         /*
4361          * these registers are reading and clearing, so clear them after
4362          * reading them.
4363          */
4364         writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE);
4365
4366         regs = qm_dfx_regs;
4367         for (i = 0; i < CNT_CYC_REGS_NUM; i++) {
4368                 readl(qm->io_base + regs->offset);
4369                 regs++;
4370         }
4371
4372         /* clear clear_enable */
4373         writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE);
4374 }
4375 EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
4376
4377 /**
4378  * hisi_qm_sriov_enable() - enable virtual functions
4379  * @pdev: the PCIe device
4380  * @max_vfs: the number of virtual functions to enable
4381  *
4382  * Returns the number of enabled VFs. If there are VFs enabled already or
4383  * max_vfs is more than the total number of device can be enabled, returns
4384  * failure.
4385  */
4386 int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)
4387 {
4388         struct hisi_qm *qm = pci_get_drvdata(pdev);
4389         int pre_existing_vfs, num_vfs, total_vfs, ret;
4390
4391         ret = qm_pm_get_sync(qm);
4392         if (ret)
4393                 return ret;
4394
4395         total_vfs = pci_sriov_get_totalvfs(pdev);
4396         pre_existing_vfs = pci_num_vf(pdev);
4397         if (pre_existing_vfs) {
4398                 pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",
4399                         pre_existing_vfs);
4400                 goto err_put_sync;
4401         }
4402
4403         num_vfs = min_t(int, max_vfs, total_vfs);
4404         ret = qm_vf_q_assign(qm, num_vfs);
4405         if (ret) {
4406                 pci_err(pdev, "Can't assign queues for VF!\n");
4407                 goto err_put_sync;
4408         }
4409
4410         qm->vfs_num = num_vfs;
4411
4412         ret = pci_enable_sriov(pdev, num_vfs);
4413         if (ret) {
4414                 pci_err(pdev, "Can't enable VF!\n");
4415                 qm_clear_vft_config(qm);
4416                 goto err_put_sync;
4417         }
4418
4419         pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);
4420
4421         return num_vfs;
4422
4423 err_put_sync:
4424         qm_pm_put_sync(qm);
4425         return ret;
4426 }
4427 EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);
4428
4429 /**
4430  * hisi_qm_sriov_disable - disable virtual functions
4431  * @pdev: the PCI device.
4432  * @is_frozen: true when all the VFs are frozen.
4433  *
4434  * Return failure if there are VFs assigned already or VF is in used.
4435  */
4436 int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)
4437 {
4438         struct hisi_qm *qm = pci_get_drvdata(pdev);
4439         int total_vfs = pci_sriov_get_totalvfs(qm->pdev);
4440         int ret;
4441
4442         if (pci_vfs_assigned(pdev)) {
4443                 pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");
4444                 return -EPERM;
4445         }
4446
4447         /* While VF is in used, SRIOV cannot be disabled. */
4448         if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {
4449                 pci_err(pdev, "Task is using its VF!\n");
4450                 return -EBUSY;
4451         }
4452
4453         pci_disable_sriov(pdev);
4454         /* clear vf function shaper configure array */
4455         memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs);
4456         ret = qm_clear_vft_config(qm);
4457         if (ret)
4458                 return ret;
4459
4460         qm_pm_put_sync(qm);
4461
4462         return 0;
4463 }
4464 EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);
4465
4466 /**
4467  * hisi_qm_sriov_configure - configure the number of VFs
4468  * @pdev: The PCI device
4469  * @num_vfs: The number of VFs need enabled
4470  *
4471  * Enable SR-IOV according to num_vfs, 0 means disable.
4472  */
4473 int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)
4474 {
4475         if (num_vfs == 0)
4476                 return hisi_qm_sriov_disable(pdev, false);
4477         else
4478                 return hisi_qm_sriov_enable(pdev, num_vfs);
4479 }
4480 EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);
4481
4482 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
4483 {
4484         u32 err_sts;
4485
4486         if (!qm->err_ini->get_dev_hw_err_status) {
4487                 dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n");
4488                 return ACC_ERR_NONE;
4489         }
4490
4491         /* get device hardware error status */
4492         err_sts = qm->err_ini->get_dev_hw_err_status(qm);
4493         if (err_sts) {
4494                 if (err_sts & qm->err_info.ecc_2bits_mask)
4495                         qm->err_status.is_dev_ecc_mbit = true;
4496
4497                 if (qm->err_ini->log_dev_hw_err)
4498                         qm->err_ini->log_dev_hw_err(qm, err_sts);
4499
4500                 /* ce error does not need to be reset */
4501                 if ((err_sts | qm->err_info.dev_ce_mask) ==
4502                      qm->err_info.dev_ce_mask) {
4503                         if (qm->err_ini->clear_dev_hw_err_status)
4504                                 qm->err_ini->clear_dev_hw_err_status(qm,
4505                                                                 err_sts);
4506
4507                         return ACC_ERR_RECOVERED;
4508                 }
4509
4510                 return ACC_ERR_NEED_RESET;
4511         }
4512
4513         return ACC_ERR_RECOVERED;
4514 }
4515
4516 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)
4517 {
4518         enum acc_err_result qm_ret, dev_ret;
4519
4520         /* log qm error */
4521         qm_ret = qm_hw_error_handle(qm);
4522
4523         /* log device error */
4524         dev_ret = qm_dev_err_handle(qm);
4525
4526         return (qm_ret == ACC_ERR_NEED_RESET ||
4527                 dev_ret == ACC_ERR_NEED_RESET) ?
4528                 ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;
4529 }
4530
4531 /**
4532  * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
4533  * @pdev: The PCI device which need report error.
4534  * @state: The connectivity between CPU and device.
4535  *
4536  * We register this function into PCIe AER handlers, It will report device or
4537  * qm hardware error status when error occur.
4538  */
4539 pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,
4540                                           pci_channel_state_t state)
4541 {
4542         struct hisi_qm *qm = pci_get_drvdata(pdev);
4543         enum acc_err_result ret;
4544
4545         if (pdev->is_virtfn)
4546                 return PCI_ERS_RESULT_NONE;
4547
4548         pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);
4549         if (state == pci_channel_io_perm_failure)
4550                 return PCI_ERS_RESULT_DISCONNECT;
4551
4552         ret = qm_process_dev_error(qm);
4553         if (ret == ACC_ERR_NEED_RESET)
4554                 return PCI_ERS_RESULT_NEED_RESET;
4555
4556         return PCI_ERS_RESULT_RECOVERED;
4557 }
4558 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);
4559
4560 static int qm_check_req_recv(struct hisi_qm *qm)
4561 {
4562         struct pci_dev *pdev = qm->pdev;
4563         int ret;
4564         u32 val;
4565
4566         if (qm->ver >= QM_HW_V3)
4567                 return 0;
4568
4569         writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);
4570         ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4571                                          (val == ACC_VENDOR_ID_VALUE),
4572                                          POLL_PERIOD, POLL_TIMEOUT);
4573         if (ret) {
4574                 dev_err(&pdev->dev, "Fails to read QM reg!\n");
4575                 return ret;
4576         }
4577
4578         writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);
4579         ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,
4580                                          (val == PCI_VENDOR_ID_HUAWEI),
4581                                          POLL_PERIOD, POLL_TIMEOUT);
4582         if (ret)
4583                 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");
4584
4585         return ret;
4586 }
4587
4588 static int qm_set_pf_mse(struct hisi_qm *qm, bool set)
4589 {
4590         struct pci_dev *pdev = qm->pdev;
4591         u16 cmd;
4592         int i;
4593
4594         pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4595         if (set)
4596                 cmd |= PCI_COMMAND_MEMORY;
4597         else
4598                 cmd &= ~PCI_COMMAND_MEMORY;
4599
4600         pci_write_config_word(pdev, PCI_COMMAND, cmd);
4601         for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4602                 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
4603                 if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))
4604                         return 0;
4605
4606                 udelay(1);
4607         }
4608
4609         return -ETIMEDOUT;
4610 }
4611
4612 static int qm_set_vf_mse(struct hisi_qm *qm, bool set)
4613 {
4614         struct pci_dev *pdev = qm->pdev;
4615         u16 sriov_ctrl;
4616         int pos;
4617         int i;
4618
4619         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
4620         pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4621         if (set)
4622                 sriov_ctrl |= PCI_SRIOV_CTRL_MSE;
4623         else
4624                 sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;
4625         pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);
4626
4627         for (i = 0; i < MAX_WAIT_COUNTS; i++) {
4628                 pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);
4629                 if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>
4630                     ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)
4631                         return 0;
4632
4633                 udelay(1);
4634         }
4635
4636         return -ETIMEDOUT;
4637 }
4638
4639 static int qm_vf_reset_prepare(struct hisi_qm *qm,
4640                                enum qm_stop_reason stop_reason)
4641 {
4642         struct hisi_qm_list *qm_list = qm->qm_list;
4643         struct pci_dev *pdev = qm->pdev;
4644         struct pci_dev *virtfn;
4645         struct hisi_qm *vf_qm;
4646         int ret = 0;
4647
4648         mutex_lock(&qm_list->lock);
4649         list_for_each_entry(vf_qm, &qm_list->list, list) {
4650                 virtfn = vf_qm->pdev;
4651                 if (virtfn == pdev)
4652                         continue;
4653
4654                 if (pci_physfn(virtfn) == pdev) {
4655                         /* save VFs PCIE BAR configuration */
4656                         pci_save_state(virtfn);
4657
4658                         ret = hisi_qm_stop(vf_qm, stop_reason);
4659                         if (ret)
4660                                 goto stop_fail;
4661                 }
4662         }
4663
4664 stop_fail:
4665         mutex_unlock(&qm_list->lock);
4666         return ret;
4667 }
4668
4669 static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd,
4670                            enum qm_stop_reason stop_reason)
4671 {
4672         struct pci_dev *pdev = qm->pdev;
4673         int ret;
4674
4675         if (!qm->vfs_num)
4676                 return 0;
4677
4678         /* Kunpeng930 supports to notify VFs to stop before PF reset */
4679         if (qm->ops->ping_all_vfs) {
4680                 ret = qm->ops->ping_all_vfs(qm, cmd);
4681                 if (ret)
4682                         pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n");
4683         } else {
4684                 ret = qm_vf_reset_prepare(qm, stop_reason);
4685                 if (ret)
4686                         pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret);
4687         }
4688
4689         return ret;
4690 }
4691
4692 static int qm_wait_reset_finish(struct hisi_qm *qm)
4693 {
4694         int delay = 0;
4695
4696         /* All reset requests need to be queued for processing */
4697         while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
4698                 msleep(++delay);
4699                 if (delay > QM_RESET_WAIT_TIMEOUT)
4700                         return -EBUSY;
4701         }
4702
4703         return 0;
4704 }
4705
4706 static int qm_reset_prepare_ready(struct hisi_qm *qm)
4707 {
4708         struct pci_dev *pdev = qm->pdev;
4709         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4710
4711         /*
4712          * PF and VF on host doesnot support resetting at the
4713          * same time on Kunpeng920.
4714          */
4715         if (qm->ver < QM_HW_V3)
4716                 return qm_wait_reset_finish(pf_qm);
4717
4718         return qm_wait_reset_finish(qm);
4719 }
4720
4721 static void qm_reset_bit_clear(struct hisi_qm *qm)
4722 {
4723         struct pci_dev *pdev = qm->pdev;
4724         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
4725
4726         if (qm->ver < QM_HW_V3)
4727                 clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
4728
4729         clear_bit(QM_RESETTING, &qm->misc_ctl);
4730 }
4731
4732 static int qm_controller_reset_prepare(struct hisi_qm *qm)
4733 {
4734         struct pci_dev *pdev = qm->pdev;
4735         int ret;
4736
4737         ret = qm_reset_prepare_ready(qm);
4738         if (ret) {
4739                 pci_err(pdev, "Controller reset not ready!\n");
4740                 return ret;
4741         }
4742
4743         /* PF obtains the information of VF by querying the register. */
4744         qm_cmd_uninit(qm);
4745
4746         /* Whether VFs stop successfully, soft reset will continue. */
4747         ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET);
4748         if (ret)
4749                 pci_err(pdev, "failed to stop vfs by pf in soft reset.\n");
4750
4751         ret = hisi_qm_stop(qm, QM_SOFT_RESET);
4752         if (ret) {
4753                 pci_err(pdev, "Fails to stop QM!\n");
4754                 qm_reset_bit_clear(qm);
4755                 return ret;
4756         }
4757
4758         ret = qm_wait_vf_prepare_finish(qm);
4759         if (ret)
4760                 pci_err(pdev, "failed to stop by vfs in soft reset!\n");
4761
4762         clear_bit(QM_RST_SCHED, &qm->misc_ctl);
4763
4764         return 0;
4765 }
4766
4767 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)
4768 {
4769         u32 nfe_enb = 0;
4770
4771         /* Kunpeng930 hardware automatically close master ooo when NFE occurs */
4772         if (qm->ver >= QM_HW_V3)
4773                 return;
4774
4775         if (!qm->err_status.is_dev_ecc_mbit &&
4776             qm->err_status.is_qm_ecc_mbit &&
4777             qm->err_ini->close_axi_master_ooo) {
4778
4779                 qm->err_ini->close_axi_master_ooo(qm);
4780
4781         } else if (qm->err_status.is_dev_ecc_mbit &&
4782                    !qm->err_status.is_qm_ecc_mbit &&
4783                    !qm->err_ini->close_axi_master_ooo) {
4784
4785                 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);
4786                 writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,
4787                        qm->io_base + QM_RAS_NFE_ENABLE);
4788                 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);
4789         }
4790 }
4791
4792 static int qm_soft_reset(struct hisi_qm *qm)
4793 {
4794         struct pci_dev *pdev = qm->pdev;
4795         int ret;
4796         u32 val;
4797
4798         /* Ensure all doorbells and mailboxes received by QM */
4799         ret = qm_check_req_recv(qm);
4800         if (ret)
4801                 return ret;
4802
4803         if (qm->vfs_num) {
4804                 ret = qm_set_vf_mse(qm, false);
4805                 if (ret) {
4806                         pci_err(pdev, "Fails to disable vf MSE bit.\n");
4807                         return ret;
4808                 }
4809         }
4810
4811         ret = qm->ops->set_msi(qm, false);
4812         if (ret) {
4813                 pci_err(pdev, "Fails to disable PEH MSI bit.\n");
4814                 return ret;
4815         }
4816
4817         qm_dev_ecc_mbit_handle(qm);
4818
4819         /* OOO register set and check */
4820         writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
4821                qm->io_base + ACC_MASTER_GLOBAL_CTRL);
4822
4823         /* If bus lock, reset chip */
4824         ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
4825                                          val,
4826                                          (val == ACC_MASTER_TRANS_RETURN_RW),
4827                                          POLL_PERIOD, POLL_TIMEOUT);
4828         if (ret) {
4829                 pci_emerg(pdev, "Bus lock! Please reset system.\n");
4830                 return ret;
4831         }
4832
4833         if (qm->err_ini->close_sva_prefetch)
4834                 qm->err_ini->close_sva_prefetch(qm);
4835
4836         ret = qm_set_pf_mse(qm, false);
4837         if (ret) {
4838                 pci_err(pdev, "Fails to disable pf MSE bit.\n");
4839                 return ret;
4840         }
4841
4842         /* The reset related sub-control registers are not in PCI BAR */
4843         if (ACPI_HANDLE(&pdev->dev)) {
4844                 unsigned long long value = 0;
4845                 acpi_status s;
4846
4847                 s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
4848                                           qm->err_info.acpi_rst,
4849                                           NULL, &value);
4850                 if (ACPI_FAILURE(s)) {
4851                         pci_err(pdev, "NO controller reset method!\n");
4852                         return -EIO;
4853                 }
4854
4855                 if (value) {
4856                         pci_err(pdev, "Reset step %llu failed!\n", value);
4857                         return -EIO;
4858                 }
4859         } else {
4860                 pci_err(pdev, "No reset method!\n");
4861                 return -EINVAL;
4862         }
4863
4864         return 0;
4865 }
4866
4867 static int qm_vf_reset_done(struct hisi_qm *qm)
4868 {
4869         struct hisi_qm_list *qm_list = qm->qm_list;
4870         struct pci_dev *pdev = qm->pdev;
4871         struct pci_dev *virtfn;
4872         struct hisi_qm *vf_qm;
4873         int ret = 0;
4874
4875         mutex_lock(&qm_list->lock);
4876         list_for_each_entry(vf_qm, &qm_list->list, list) {
4877                 virtfn = vf_qm->pdev;
4878                 if (virtfn == pdev)
4879                         continue;
4880
4881                 if (pci_physfn(virtfn) == pdev) {
4882                         /* enable VFs PCIE BAR configuration */
4883                         pci_restore_state(virtfn);
4884
4885                         ret = qm_restart(vf_qm);
4886                         if (ret)
4887                                 goto restart_fail;
4888                 }
4889         }
4890
4891 restart_fail:
4892         mutex_unlock(&qm_list->lock);
4893         return ret;
4894 }
4895
4896 static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd)
4897 {
4898         struct pci_dev *pdev = qm->pdev;
4899         int ret;
4900
4901         if (!qm->vfs_num)
4902                 return 0;
4903
4904         ret = qm_vf_q_assign(qm, qm->vfs_num);
4905         if (ret) {
4906                 pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret);
4907                 return ret;
4908         }
4909
4910         /* Kunpeng930 supports to notify VFs to start after PF reset. */
4911         if (qm->ops->ping_all_vfs) {
4912                 ret = qm->ops->ping_all_vfs(qm, cmd);
4913                 if (ret)
4914                         pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n");
4915         } else {
4916                 ret = qm_vf_reset_done(qm);
4917                 if (ret)
4918                         pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret);
4919         }
4920
4921         return ret;
4922 }
4923
4924 static int qm_dev_hw_init(struct hisi_qm *qm)
4925 {
4926         return qm->err_ini->hw_init(qm);
4927 }
4928
4929 static void qm_restart_prepare(struct hisi_qm *qm)
4930 {
4931         u32 value;
4932
4933         if (qm->err_ini->open_sva_prefetch)
4934                 qm->err_ini->open_sva_prefetch(qm);
4935
4936         if (qm->ver >= QM_HW_V3)
4937                 return;
4938
4939         if (!qm->err_status.is_qm_ecc_mbit &&
4940             !qm->err_status.is_dev_ecc_mbit)
4941                 return;
4942
4943         /* temporarily close the OOO port used for PEH to write out MSI */
4944         value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4945         writel(value & ~qm->err_info.msi_wr_port,
4946                qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4947
4948         /* clear dev ecc 2bit error source if having */
4949         value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
4950         if (value && qm->err_ini->clear_dev_hw_err_status)
4951                 qm->err_ini->clear_dev_hw_err_status(qm, value);
4952
4953         /* clear QM ecc mbit error source */
4954         writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);
4955
4956         /* clear AM Reorder Buffer ecc mbit source */
4957         writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);
4958 }
4959
4960 static void qm_restart_done(struct hisi_qm *qm)
4961 {
4962         u32 value;
4963
4964         if (qm->ver >= QM_HW_V3)
4965                 goto clear_flags;
4966
4967         if (!qm->err_status.is_qm_ecc_mbit &&
4968             !qm->err_status.is_dev_ecc_mbit)
4969                 return;
4970
4971         /* open the OOO port for PEH to write out MSI */
4972         value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4973         value |= qm->err_info.msi_wr_port;
4974         writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
4975
4976 clear_flags:
4977         qm->err_status.is_qm_ecc_mbit = false;
4978         qm->err_status.is_dev_ecc_mbit = false;
4979 }
4980
4981 static int qm_controller_reset_done(struct hisi_qm *qm)
4982 {
4983         struct pci_dev *pdev = qm->pdev;
4984         int ret;
4985
4986         ret = qm->ops->set_msi(qm, true);
4987         if (ret) {
4988                 pci_err(pdev, "Fails to enable PEH MSI bit!\n");
4989                 return ret;
4990         }
4991
4992         ret = qm_set_pf_mse(qm, true);
4993         if (ret) {
4994                 pci_err(pdev, "Fails to enable pf MSE bit!\n");
4995                 return ret;
4996         }
4997
4998         if (qm->vfs_num) {
4999                 ret = qm_set_vf_mse(qm, true);
5000                 if (ret) {
5001                         pci_err(pdev, "Fails to enable vf MSE bit!\n");
5002                         return ret;
5003                 }
5004         }
5005
5006         ret = qm_dev_hw_init(qm);
5007         if (ret) {
5008                 pci_err(pdev, "Failed to init device\n");
5009                 return ret;
5010         }
5011
5012         qm_restart_prepare(qm);
5013         hisi_qm_dev_err_init(qm);
5014         if (qm->err_ini->open_axi_master_ooo)
5015                 qm->err_ini->open_axi_master_ooo(qm);
5016
5017         ret = qm_restart(qm);
5018         if (ret) {
5019                 pci_err(pdev, "Failed to start QM!\n");
5020                 return ret;
5021         }
5022
5023         ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
5024         if (ret)
5025                 pci_err(pdev, "failed to start vfs by pf in soft reset.\n");
5026
5027         ret = qm_wait_vf_prepare_finish(qm);
5028         if (ret)
5029                 pci_err(pdev, "failed to start by vfs in soft reset!\n");
5030
5031         qm_cmd_init(qm);
5032         qm_restart_done(qm);
5033
5034         qm_reset_bit_clear(qm);
5035
5036         return 0;
5037 }
5038
5039 static int qm_controller_reset(struct hisi_qm *qm)
5040 {
5041         struct pci_dev *pdev = qm->pdev;
5042         int ret;
5043
5044         pci_info(pdev, "Controller resetting...\n");
5045
5046         ret = qm_controller_reset_prepare(qm);
5047         if (ret) {
5048                 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
5049                 return ret;
5050         }
5051
5052         ret = qm_soft_reset(qm);
5053         if (ret) {
5054                 pci_err(pdev, "Controller reset failed (%d)\n", ret);
5055                 qm_reset_bit_clear(qm);
5056                 return ret;
5057         }
5058
5059         ret = qm_controller_reset_done(qm);
5060         if (ret) {
5061                 qm_reset_bit_clear(qm);
5062                 return ret;
5063         }
5064
5065         pci_info(pdev, "Controller reset complete\n");
5066
5067         return 0;
5068 }
5069
5070 /**
5071  * hisi_qm_dev_slot_reset() - slot reset
5072  * @pdev: the PCIe device
5073  *
5074  * This function offers QM relate PCIe device reset interface. Drivers which
5075  * use QM can use this function as slot_reset in its struct pci_error_handlers.
5076  */
5077 pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)
5078 {
5079         struct hisi_qm *qm = pci_get_drvdata(pdev);
5080         int ret;
5081
5082         if (pdev->is_virtfn)
5083                 return PCI_ERS_RESULT_RECOVERED;
5084
5085         pci_aer_clear_nonfatal_status(pdev);
5086
5087         /* reset pcie device controller */
5088         ret = qm_controller_reset(qm);
5089         if (ret) {
5090                 pci_err(pdev, "Controller reset failed (%d)\n", ret);
5091                 return PCI_ERS_RESULT_DISCONNECT;
5092         }
5093
5094         return PCI_ERS_RESULT_RECOVERED;
5095 }
5096 EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);
5097
5098 void hisi_qm_reset_prepare(struct pci_dev *pdev)
5099 {
5100         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
5101         struct hisi_qm *qm = pci_get_drvdata(pdev);
5102         u32 delay = 0;
5103         int ret;
5104
5105         hisi_qm_dev_err_uninit(pf_qm);
5106
5107         /*
5108          * Check whether there is an ECC mbit error, If it occurs, need to
5109          * wait for soft reset to fix it.
5110          */
5111         while (qm_check_dev_error(pf_qm)) {
5112                 msleep(++delay);
5113                 if (delay > QM_RESET_WAIT_TIMEOUT)
5114                         return;
5115         }
5116
5117         ret = qm_reset_prepare_ready(qm);
5118         if (ret) {
5119                 pci_err(pdev, "FLR not ready!\n");
5120                 return;
5121         }
5122
5123         /* PF obtains the information of VF by querying the register. */
5124         if (qm->fun_type == QM_HW_PF)
5125                 qm_cmd_uninit(qm);
5126
5127         ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_FLR);
5128         if (ret)
5129                 pci_err(pdev, "failed to stop vfs by pf in FLR.\n");
5130
5131         ret = hisi_qm_stop(qm, QM_FLR);
5132         if (ret) {
5133                 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);
5134                 return;
5135         }
5136
5137         ret = qm_wait_vf_prepare_finish(qm);
5138         if (ret)
5139                 pci_err(pdev, "failed to stop by vfs in FLR!\n");
5140
5141         pci_info(pdev, "FLR resetting...\n");
5142 }
5143 EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);
5144
5145 static bool qm_flr_reset_complete(struct pci_dev *pdev)
5146 {
5147         struct pci_dev *pf_pdev = pci_physfn(pdev);
5148         struct hisi_qm *qm = pci_get_drvdata(pf_pdev);
5149         u32 id;
5150
5151         pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);
5152         if (id == QM_PCI_COMMAND_INVALID) {
5153                 pci_err(pdev, "Device can not be used!\n");
5154                 return false;
5155         }
5156
5157         return true;
5158 }
5159
5160 void hisi_qm_reset_done(struct pci_dev *pdev)
5161 {
5162         struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
5163         struct hisi_qm *qm = pci_get_drvdata(pdev);
5164         int ret;
5165
5166         if (qm->fun_type == QM_HW_PF) {
5167                 ret = qm_dev_hw_init(qm);
5168                 if (ret) {
5169                         pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);
5170                         goto flr_done;
5171                 }
5172         }
5173
5174         hisi_qm_dev_err_init(pf_qm);
5175
5176         ret = qm_restart(qm);
5177         if (ret) {
5178                 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);
5179                 goto flr_done;
5180         }
5181
5182         ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);
5183         if (ret)
5184                 pci_err(pdev, "failed to start vfs by pf in FLR.\n");
5185
5186         ret = qm_wait_vf_prepare_finish(qm);
5187         if (ret)
5188                 pci_err(pdev, "failed to start by vfs in FLR!\n");
5189
5190 flr_done:
5191         if (qm->fun_type == QM_HW_PF)
5192                 qm_cmd_init(qm);
5193
5194         if (qm_flr_reset_complete(pdev))
5195                 pci_info(pdev, "FLR reset complete\n");
5196
5197         qm_reset_bit_clear(qm);
5198 }
5199 EXPORT_SYMBOL_GPL(hisi_qm_reset_done);
5200
5201 static irqreturn_t qm_abnormal_irq(int irq, void *data)
5202 {
5203         struct hisi_qm *qm = data;
5204         enum acc_err_result ret;
5205
5206         atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);
5207         ret = qm_process_dev_error(qm);
5208         if (ret == ACC_ERR_NEED_RESET &&
5209             !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&
5210             !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))
5211                 schedule_work(&qm->rst_work);
5212
5213         return IRQ_HANDLED;
5214 }
5215
5216 static int qm_irq_register(struct hisi_qm *qm)
5217 {
5218         struct pci_dev *pdev = qm->pdev;
5219         int ret;
5220
5221         ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR),
5222                           qm_irq, 0, qm->dev_name, qm);
5223         if (ret)
5224                 return ret;
5225
5226         if (qm->ver > QM_HW_V1) {
5227                 ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
5228                                   qm_aeq_irq, 0, qm->dev_name, qm);
5229                 if (ret)
5230                         goto err_aeq_irq;
5231
5232                 if (qm->fun_type == QM_HW_PF) {
5233                         ret = request_irq(pci_irq_vector(pdev,
5234                                           QM_ABNORMAL_EVENT_IRQ_VECTOR),
5235                                           qm_abnormal_irq, 0, qm->dev_name, qm);
5236                         if (ret)
5237                                 goto err_abonormal_irq;
5238                 }
5239         }
5240
5241         if (qm->ver > QM_HW_V2) {
5242                 ret = request_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR),
5243                                 qm_mb_cmd_irq, 0, qm->dev_name, qm);
5244                 if (ret)
5245                         goto err_mb_cmd_irq;
5246         }
5247
5248         return 0;
5249
5250 err_mb_cmd_irq:
5251         if (qm->fun_type == QM_HW_PF)
5252                 free_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
5253 err_abonormal_irq:
5254         free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
5255 err_aeq_irq:
5256         free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
5257         return ret;
5258 }
5259
5260 /**
5261  * hisi_qm_dev_shutdown() - Shutdown device.
5262  * @pdev: The device will be shutdown.
5263  *
5264  * This function will stop qm when OS shutdown or rebooting.
5265  */
5266 void hisi_qm_dev_shutdown(struct pci_dev *pdev)
5267 {
5268         struct hisi_qm *qm = pci_get_drvdata(pdev);
5269         int ret;
5270
5271         ret = hisi_qm_stop(qm, QM_NORMAL);
5272         if (ret)
5273                 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");
5274 }
5275 EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);
5276
5277 static void hisi_qm_controller_reset(struct work_struct *rst_work)
5278 {
5279         struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);
5280         int ret;
5281
5282         ret = qm_pm_get_sync(qm);
5283         if (ret) {
5284                 clear_bit(QM_RST_SCHED, &qm->misc_ctl);
5285                 return;
5286         }
5287
5288         /* reset pcie device controller */
5289         ret = qm_controller_reset(qm);
5290         if (ret)
5291                 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);
5292
5293         qm_pm_put_sync(qm);
5294 }
5295
5296 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm,
5297                                    enum qm_stop_reason stop_reason)
5298 {
5299         enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE;
5300         struct pci_dev *pdev = qm->pdev;
5301         int ret;
5302
5303         ret = qm_reset_prepare_ready(qm);
5304         if (ret) {
5305                 dev_err(&pdev->dev, "reset prepare not ready!\n");
5306                 atomic_set(&qm->status.flags, QM_STOP);
5307                 cmd = QM_VF_PREPARE_FAIL;
5308                 goto err_prepare;
5309         }
5310
5311         ret = hisi_qm_stop(qm, stop_reason);
5312         if (ret) {
5313                 dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret);
5314                 atomic_set(&qm->status.flags, QM_STOP);
5315                 cmd = QM_VF_PREPARE_FAIL;
5316                 goto err_prepare;
5317         }
5318
5319 err_prepare:
5320         pci_save_state(pdev);
5321         ret = qm->ops->ping_pf(qm, cmd);
5322         if (ret)
5323                 dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n");
5324 }
5325
5326 static void qm_pf_reset_vf_done(struct hisi_qm *qm)
5327 {
5328         enum qm_mb_cmd cmd = QM_VF_START_DONE;
5329         struct pci_dev *pdev = qm->pdev;
5330         int ret;
5331
5332         pci_restore_state(pdev);
5333         ret = hisi_qm_start(qm);
5334         if (ret) {
5335                 dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret);
5336                 cmd = QM_VF_START_FAIL;
5337         }
5338
5339         ret = qm->ops->ping_pf(qm, cmd);
5340         if (ret)
5341                 dev_warn(&pdev->dev, "PF responds timeout in reset done!\n");
5342
5343         qm_reset_bit_clear(qm);
5344 }
5345
5346 static int qm_wait_pf_reset_finish(struct hisi_qm *qm)
5347 {
5348         struct device *dev = &qm->pdev->dev;
5349         u32 val, cmd;
5350         u64 msg;
5351         int ret;
5352
5353         /* Wait for reset to finish */
5354         ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val,
5355                                          val == BIT(0), QM_VF_RESET_WAIT_US,
5356                                          QM_VF_RESET_WAIT_TIMEOUT_US);
5357         /* hardware completion status should be available by this time */
5358         if (ret) {
5359                 dev_err(dev, "couldn't get reset done status from PF, timeout!\n");
5360                 return -ETIMEDOUT;
5361         }
5362
5363         /*
5364          * Whether message is got successfully,
5365          * VF needs to ack PF by clearing the interrupt.
5366          */
5367         ret = qm_get_mb_cmd(qm, &msg, 0);
5368         qm_clear_cmd_interrupt(qm, 0);
5369         if (ret) {
5370                 dev_err(dev, "failed to get msg from PF in reset done!\n");
5371                 return ret;
5372         }
5373
5374         cmd = msg & QM_MB_CMD_DATA_MASK;
5375         if (cmd != QM_PF_RESET_DONE) {
5376                 dev_err(dev, "the cmd(%u) is not reset done!\n", cmd);
5377                 ret = -EINVAL;
5378         }
5379
5380         return ret;
5381 }
5382
5383 static void qm_pf_reset_vf_process(struct hisi_qm *qm,
5384                                    enum qm_stop_reason stop_reason)
5385 {
5386         struct device *dev = &qm->pdev->dev;
5387         int ret;
5388
5389         dev_info(dev, "device reset start...\n");
5390
5391         /* The message is obtained by querying the register during resetting */
5392         qm_cmd_uninit(qm);
5393         qm_pf_reset_vf_prepare(qm, stop_reason);
5394
5395         ret = qm_wait_pf_reset_finish(qm);
5396         if (ret)
5397                 goto err_get_status;
5398
5399         qm_pf_reset_vf_done(qm);
5400         qm_cmd_init(qm);
5401
5402         dev_info(dev, "device reset done.\n");
5403
5404         return;
5405
5406 err_get_status:
5407         qm_cmd_init(qm);
5408         qm_reset_bit_clear(qm);
5409 }
5410
5411 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num)
5412 {
5413         struct device *dev = &qm->pdev->dev;
5414         u64 msg;
5415         u32 cmd;
5416         int ret;
5417
5418         /*
5419          * Get the msg from source by sending mailbox. Whether message is got
5420          * successfully, destination needs to ack source by clearing the interrupt.
5421          */
5422         ret = qm_get_mb_cmd(qm, &msg, fun_num);
5423         qm_clear_cmd_interrupt(qm, BIT(fun_num));
5424         if (ret) {
5425                 dev_err(dev, "failed to get msg from source!\n");
5426                 return;
5427         }
5428
5429         cmd = msg & QM_MB_CMD_DATA_MASK;
5430         switch (cmd) {
5431         case QM_PF_FLR_PREPARE:
5432                 qm_pf_reset_vf_process(qm, QM_FLR);
5433                 break;
5434         case QM_PF_SRST_PREPARE:
5435                 qm_pf_reset_vf_process(qm, QM_SOFT_RESET);
5436                 break;
5437         case QM_VF_GET_QOS:
5438                 qm_vf_get_qos(qm, fun_num);
5439                 break;
5440         case QM_PF_SET_QOS:
5441                 qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT;
5442                 break;
5443         default:
5444                 dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num);
5445                 break;
5446         }
5447 }
5448
5449 static void qm_cmd_process(struct work_struct *cmd_process)
5450 {
5451         struct hisi_qm *qm = container_of(cmd_process,
5452                                         struct hisi_qm, cmd_process);
5453         u32 vfs_num = qm->vfs_num;
5454         u64 val;
5455         u32 i;
5456
5457         if (qm->fun_type == QM_HW_PF) {
5458                 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);
5459                 if (!val)
5460                         return;
5461
5462                 for (i = 1; i <= vfs_num; i++) {
5463                         if (val & BIT(i))
5464                                 qm_handle_cmd_msg(qm, i);
5465                 }
5466
5467                 return;
5468         }
5469
5470         qm_handle_cmd_msg(qm, 0);
5471 }
5472
5473 /**
5474  * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list.
5475  * @qm: The qm needs add.
5476  * @qm_list: The qm list.
5477  *
5478  * This function adds qm to qm list, and will register algorithm to
5479  * crypto when the qm list is empty.
5480  */
5481 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
5482 {
5483         struct device *dev = &qm->pdev->dev;
5484         int flag = 0;
5485         int ret = 0;
5486
5487         mutex_lock(&qm_list->lock);
5488         if (list_empty(&qm_list->list))
5489                 flag = 1;
5490         list_add_tail(&qm->list, &qm_list->list);
5491         mutex_unlock(&qm_list->lock);
5492
5493         if (qm->ver <= QM_HW_V2 && qm->use_sva) {
5494                 dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n");
5495                 return 0;
5496         }
5497
5498         if (flag) {
5499                 ret = qm_list->register_to_crypto(qm);
5500                 if (ret) {
5501                         mutex_lock(&qm_list->lock);
5502                         list_del(&qm->list);
5503                         mutex_unlock(&qm_list->lock);
5504                 }
5505         }
5506
5507         return ret;
5508 }
5509 EXPORT_SYMBOL_GPL(hisi_qm_alg_register);
5510
5511 /**
5512  * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from
5513  * qm list.
5514  * @qm: The qm needs delete.
5515  * @qm_list: The qm list.
5516  *
5517  * This function deletes qm from qm list, and will unregister algorithm
5518  * from crypto when the qm list is empty.
5519  */
5520 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
5521 {
5522         mutex_lock(&qm_list->lock);
5523         list_del(&qm->list);
5524         mutex_unlock(&qm_list->lock);
5525
5526         if (qm->ver <= QM_HW_V2 && qm->use_sva)
5527                 return;
5528
5529         if (list_empty(&qm_list->list))
5530                 qm_list->unregister_from_crypto(qm);
5531 }
5532 EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
5533
5534 static int qm_get_qp_num(struct hisi_qm *qm)
5535 {
5536         if (qm->ver == QM_HW_V1)
5537                 qm->ctrl_qp_num = QM_QNUM_V1;
5538         else if (qm->ver == QM_HW_V2)
5539                 qm->ctrl_qp_num = QM_QNUM_V2;
5540         else
5541                 qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) &
5542                                         QM_QP_NUN_MASK;
5543
5544         if (qm->use_db_isolation)
5545                 qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >>
5546                                   QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK;
5547         else
5548                 qm->max_qp_num = qm->ctrl_qp_num;
5549
5550         /* check if qp number is valid */
5551         if (qm->qp_num > qm->max_qp_num) {
5552                 dev_err(&qm->pdev->dev, "qp num(%u) is more than max qp num(%u)!\n",
5553                         qm->qp_num, qm->max_qp_num);
5554                 return -EINVAL;
5555         }
5556
5557         return 0;
5558 }
5559
5560 static int qm_get_pci_res(struct hisi_qm *qm)
5561 {
5562         struct pci_dev *pdev = qm->pdev;
5563         struct device *dev = &pdev->dev;
5564         int ret;
5565
5566         ret = pci_request_mem_regions(pdev, qm->dev_name);
5567         if (ret < 0) {
5568                 dev_err(dev, "Failed to request mem regions!\n");
5569                 return ret;
5570         }
5571
5572         qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
5573         qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));
5574         if (!qm->io_base) {
5575                 ret = -EIO;
5576                 goto err_request_mem_regions;
5577         }
5578
5579         if (qm->ver > QM_HW_V2) {
5580                 if (qm->fun_type == QM_HW_PF)
5581                         qm->use_db_isolation = readl(qm->io_base +
5582                                                      QM_QUE_ISO_EN) & BIT(0);
5583                 else
5584                         qm->use_db_isolation = readl(qm->io_base +
5585                                                      QM_QUE_ISO_CFG_V) & BIT(0);
5586         }
5587
5588         if (qm->use_db_isolation) {
5589                 qm->db_interval = QM_QP_DB_INTERVAL;
5590                 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
5591                 qm->db_io_base = ioremap(qm->db_phys_base,
5592                                          pci_resource_len(pdev, PCI_BAR_4));
5593                 if (!qm->db_io_base) {
5594                         ret = -EIO;
5595                         goto err_ioremap;
5596                 }
5597         } else {
5598                 qm->db_phys_base = qm->phys_base;
5599                 qm->db_io_base = qm->io_base;
5600                 qm->db_interval = 0;
5601         }
5602
5603         if (qm->fun_type == QM_HW_PF) {
5604                 ret = qm_get_qp_num(qm);
5605                 if (ret)
5606                         goto err_db_ioremap;
5607         }
5608
5609         return 0;
5610
5611 err_db_ioremap:
5612         if (qm->use_db_isolation)
5613                 iounmap(qm->db_io_base);
5614 err_ioremap:
5615         iounmap(qm->io_base);
5616 err_request_mem_regions:
5617         pci_release_mem_regions(pdev);
5618         return ret;
5619 }
5620
5621 static int hisi_qm_pci_init(struct hisi_qm *qm)
5622 {
5623         struct pci_dev *pdev = qm->pdev;
5624         struct device *dev = &pdev->dev;
5625         unsigned int num_vec;
5626         int ret;
5627
5628         ret = pci_enable_device_mem(pdev);
5629         if (ret < 0) {
5630                 dev_err(dev, "Failed to enable device mem!\n");
5631                 return ret;
5632         }
5633
5634         ret = qm_get_pci_res(qm);
5635         if (ret)
5636                 goto err_disable_pcidev;
5637
5638         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
5639         if (ret < 0)
5640                 goto err_get_pci_res;
5641         pci_set_master(pdev);
5642
5643         if (!qm->ops->get_irq_num) {
5644                 ret = -EOPNOTSUPP;
5645                 goto err_get_pci_res;
5646         }
5647         num_vec = qm->ops->get_irq_num(qm);
5648         ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
5649         if (ret < 0) {
5650                 dev_err(dev, "Failed to enable MSI vectors!\n");
5651                 goto err_get_pci_res;
5652         }
5653
5654         return 0;
5655
5656 err_get_pci_res:
5657         qm_put_pci_res(qm);
5658 err_disable_pcidev:
5659         pci_disable_device(pdev);
5660         return ret;
5661 }
5662
5663 static void hisi_qm_init_work(struct hisi_qm *qm)
5664 {
5665         INIT_WORK(&qm->work, qm_work_process);
5666         if (qm->fun_type == QM_HW_PF)
5667                 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);
5668
5669         if (qm->ver > QM_HW_V2)
5670                 INIT_WORK(&qm->cmd_process, qm_cmd_process);
5671 }
5672
5673 static int hisi_qp_alloc_memory(struct hisi_qm *qm)
5674 {
5675         struct device *dev = &qm->pdev->dev;
5676         size_t qp_dma_size;
5677         int i, ret;
5678
5679         qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);
5680         if (!qm->qp_array)
5681                 return -ENOMEM;
5682
5683         /* one more page for device or qp statuses */
5684         qp_dma_size = qm->sqe_size * QM_Q_DEPTH +
5685                       sizeof(struct qm_cqe) * QM_Q_DEPTH;
5686         qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE;
5687         for (i = 0; i < qm->qp_num; i++) {
5688                 ret = hisi_qp_memory_init(qm, qp_dma_size, i);
5689                 if (ret)
5690                         goto err_init_qp_mem;
5691
5692                 dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);
5693         }
5694
5695         return 0;
5696 err_init_qp_mem:
5697         hisi_qp_memory_uninit(qm, i);
5698
5699         return ret;
5700 }
5701
5702 static int hisi_qm_memory_init(struct hisi_qm *qm)
5703 {
5704         struct device *dev = &qm->pdev->dev;
5705         int ret, total_vfs;
5706         size_t off = 0;
5707
5708         total_vfs = pci_sriov_get_totalvfs(qm->pdev);
5709         qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL);
5710         if (!qm->factor)
5711                 return -ENOMEM;
5712
5713 #define QM_INIT_BUF(qm, type, num) do { \
5714         (qm)->type = ((qm)->qdma.va + (off)); \
5715         (qm)->type##_dma = (qm)->qdma.dma + (off); \
5716         off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
5717 } while (0)
5718
5719         idr_init(&qm->qp_idr);
5720         qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) +
5721                         QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
5722                         QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
5723                         QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
5724         qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,
5725                                          GFP_ATOMIC);
5726         dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);
5727         if (!qm->qdma.va) {
5728                 ret =  -ENOMEM;
5729                 goto err_alloc_qdma;
5730         }
5731
5732         QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH);
5733         QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
5734         QM_INIT_BUF(qm, sqc, qm->qp_num);
5735         QM_INIT_BUF(qm, cqc, qm->qp_num);
5736
5737         ret = hisi_qp_alloc_memory(qm);
5738         if (ret)
5739                 goto err_alloc_qp_array;
5740
5741         return 0;
5742
5743 err_alloc_qp_array:
5744         dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);
5745 err_alloc_qdma:
5746         kfree(qm->factor);
5747
5748         return ret;
5749 }
5750
5751 /**
5752  * hisi_qm_init() - Initialize configures about qm.
5753  * @qm: The qm needing init.
5754  *
5755  * This function init qm, then we can call hisi_qm_start to put qm into work.
5756  */
5757 int hisi_qm_init(struct hisi_qm *qm)
5758 {
5759         struct pci_dev *pdev = qm->pdev;
5760         struct device *dev = &pdev->dev;
5761         int ret;
5762
5763         hisi_qm_pre_init(qm);
5764
5765         ret = hisi_qm_pci_init(qm);
5766         if (ret)
5767                 return ret;
5768
5769         ret = qm_irq_register(qm);
5770         if (ret)
5771                 goto err_pci_init;
5772
5773         if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) {
5774                 /* v2 starts to support get vft by mailbox */
5775                 ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
5776                 if (ret)
5777                         goto err_irq_register;
5778         }
5779
5780         ret = qm_alloc_uacce(qm);
5781         if (ret < 0)
5782                 dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
5783
5784         ret = hisi_qm_memory_init(qm);
5785         if (ret)
5786                 goto err_alloc_uacce;
5787
5788         hisi_qm_init_work(qm);
5789         qm_cmd_init(qm);
5790         atomic_set(&qm->status.flags, QM_INIT);
5791
5792         return 0;
5793
5794 err_alloc_uacce:
5795         uacce_remove(qm->uacce);
5796         qm->uacce = NULL;
5797 err_irq_register:
5798         qm_irq_unregister(qm);
5799 err_pci_init:
5800         hisi_qm_pci_uninit(qm);
5801         return ret;
5802 }
5803 EXPORT_SYMBOL_GPL(hisi_qm_init);
5804
5805 /**
5806  * hisi_qm_get_dfx_access() - Try to get dfx access.
5807  * @qm: pointer to accelerator device.
5808  *
5809  * Try to get dfx access, then user can get message.
5810  *
5811  * If device is in suspended, return failure, otherwise
5812  * bump up the runtime PM usage counter.
5813  */
5814 int hisi_qm_get_dfx_access(struct hisi_qm *qm)
5815 {
5816         struct device *dev = &qm->pdev->dev;
5817
5818         if (pm_runtime_suspended(dev)) {
5819                 dev_info(dev, "can not read/write - device in suspended.\n");
5820                 return -EAGAIN;
5821         }
5822
5823         return qm_pm_get_sync(qm);
5824 }
5825 EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access);
5826
5827 /**
5828  * hisi_qm_put_dfx_access() - Put dfx access.
5829  * @qm: pointer to accelerator device.
5830  *
5831  * Put dfx access, drop runtime PM usage counter.
5832  */
5833 void hisi_qm_put_dfx_access(struct hisi_qm *qm)
5834 {
5835         qm_pm_put_sync(qm);
5836 }
5837 EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access);
5838
5839 /**
5840  * hisi_qm_pm_init() - Initialize qm runtime PM.
5841  * @qm: pointer to accelerator device.
5842  *
5843  * Function that initialize qm runtime PM.
5844  */
5845 void hisi_qm_pm_init(struct hisi_qm *qm)
5846 {
5847         struct device *dev = &qm->pdev->dev;
5848
5849         if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3)
5850                 return;
5851
5852         pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY);
5853         pm_runtime_use_autosuspend(dev);
5854         pm_runtime_put_noidle(dev);
5855 }
5856 EXPORT_SYMBOL_GPL(hisi_qm_pm_init);
5857
5858 /**
5859  * hisi_qm_pm_uninit() - Uninitialize qm runtime PM.
5860  * @qm: pointer to accelerator device.
5861  *
5862  * Function that uninitialize qm runtime PM.
5863  */
5864 void hisi_qm_pm_uninit(struct hisi_qm *qm)
5865 {
5866         struct device *dev = &qm->pdev->dev;
5867
5868         if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3)
5869                 return;
5870
5871         pm_runtime_get_noresume(dev);
5872         pm_runtime_dont_use_autosuspend(dev);
5873 }
5874 EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit);
5875
5876 static int qm_prepare_for_suspend(struct hisi_qm *qm)
5877 {
5878         struct pci_dev *pdev = qm->pdev;
5879         int ret;
5880         u32 val;
5881
5882         ret = qm->ops->set_msi(qm, false);
5883         if (ret) {
5884                 pci_err(pdev, "failed to disable MSI before suspending!\n");
5885                 return ret;
5886         }
5887
5888         /* shutdown OOO register */
5889         writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN,
5890                qm->io_base + ACC_MASTER_GLOBAL_CTRL);
5891
5892         ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,
5893                                          val,
5894                                          (val == ACC_MASTER_TRANS_RETURN_RW),
5895                                          POLL_PERIOD, POLL_TIMEOUT);
5896         if (ret) {
5897                 pci_emerg(pdev, "Bus lock! Please reset system.\n");
5898                 return ret;
5899         }
5900
5901         ret = qm_set_pf_mse(qm, false);
5902         if (ret)
5903                 pci_err(pdev, "failed to disable MSE before suspending!\n");
5904
5905         return ret;
5906 }
5907
5908 static int qm_rebuild_for_resume(struct hisi_qm *qm)
5909 {
5910         struct pci_dev *pdev = qm->pdev;
5911         int ret;
5912
5913         ret = qm_set_pf_mse(qm, true);
5914         if (ret) {
5915                 pci_err(pdev, "failed to enable MSE after resuming!\n");
5916                 return ret;
5917         }
5918
5919         ret = qm->ops->set_msi(qm, true);
5920         if (ret) {
5921                 pci_err(pdev, "failed to enable MSI after resuming!\n");
5922                 return ret;
5923         }
5924
5925         ret = qm_dev_hw_init(qm);
5926         if (ret) {
5927                 pci_err(pdev, "failed to init device after resuming\n");
5928                 return ret;
5929         }
5930
5931         qm_cmd_init(qm);
5932         hisi_qm_dev_err_init(qm);
5933
5934         return 0;
5935 }
5936
5937 /**
5938  * hisi_qm_suspend() - Runtime suspend of given device.
5939  * @dev: device to suspend.
5940  *
5941  * Function that suspend the device.
5942  */
5943 int hisi_qm_suspend(struct device *dev)
5944 {
5945         struct pci_dev *pdev = to_pci_dev(dev);
5946         struct hisi_qm *qm = pci_get_drvdata(pdev);
5947         int ret;
5948
5949         pci_info(pdev, "entering suspended state\n");
5950
5951         ret = hisi_qm_stop(qm, QM_NORMAL);
5952         if (ret) {
5953                 pci_err(pdev, "failed to stop qm(%d)\n", ret);
5954                 return ret;
5955         }
5956
5957         ret = qm_prepare_for_suspend(qm);
5958         if (ret)
5959                 pci_err(pdev, "failed to prepare suspended(%d)\n", ret);
5960
5961         return ret;
5962 }
5963 EXPORT_SYMBOL_GPL(hisi_qm_suspend);
5964
5965 /**
5966  * hisi_qm_resume() - Runtime resume of given device.
5967  * @dev: device to resume.
5968  *
5969  * Function that resume the device.
5970  */
5971 int hisi_qm_resume(struct device *dev)
5972 {
5973         struct pci_dev *pdev = to_pci_dev(dev);
5974         struct hisi_qm *qm = pci_get_drvdata(pdev);
5975         int ret;
5976
5977         pci_info(pdev, "resuming from suspend state\n");
5978
5979         ret = qm_rebuild_for_resume(qm);
5980         if (ret) {
5981                 pci_err(pdev, "failed to rebuild resume(%d)\n", ret);
5982                 return ret;
5983         }
5984
5985         ret = hisi_qm_start(qm);
5986         if (ret)
5987                 pci_err(pdev, "failed to start qm(%d)\n", ret);
5988
5989         return 0;
5990 }
5991 EXPORT_SYMBOL_GPL(hisi_qm_resume);
5992
5993 MODULE_LICENSE("GPL v2");
5994 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
5995 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");