drivers/crypto/hisilicon/hpre/hpre_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018-2019 HiSilicon Limited. */
   3 #include <linux/acpi.h>
   4 #include <linux/aer.h>
   5 #include <linux/bitops.h>
   6 #include <linux/debugfs.h>
   7 #include <linux/init.h>
   8 #include <linux/io.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/topology.h>
  13 #include "hpre.h"
  14
  15 #define HPRE_VF_NUM                     63
  16 #define HPRE_QUEUE_NUM_V2               1024
  17 #define HPRE_QM_ABNML_INT_MASK          0x100004
  18 #define HPRE_CTRL_CNT_CLR_CE_BIT        BIT(0)
  19 #define HPRE_COMM_CNT_CLR_CE            0x0
  20 #define HPRE_CTRL_CNT_CLR_CE            0x301000
  21 #define HPRE_FSM_MAX_CNT                0x301008
  22 #define HPRE_VFG_AXQOS                  0x30100c
  23 #define HPRE_VFG_AXCACHE                0x301010
  24 #define HPRE_RDCHN_INI_CFG              0x301014
  25 #define HPRE_AWUSR_FP_CFG               0x301018
  26 #define HPRE_BD_ENDIAN                  0x301020
  27 #define HPRE_ECC_BYPASS                 0x301024
  28 #define HPRE_RAS_WIDTH_CFG              0x301028
  29 #define HPRE_POISON_BYPASS              0x30102c
  30 #define HPRE_BD_ARUSR_CFG               0x301030
  31 #define HPRE_BD_AWUSR_CFG               0x301034
  32 #define HPRE_TYPES_ENB                  0x301038
  33 #define HPRE_DATA_RUSER_CFG             0x30103c
  34 #define HPRE_DATA_WUSER_CFG             0x301040
  35 #define HPRE_INT_MASK                   0x301400
  36 #define HPRE_INT_STATUS                 0x301800
  37 #define HPRE_CORE_INT_ENABLE            0
  38 #define HPRE_CORE_INT_DISABLE           0x003fffff
  39 #define HPRE_RAS_ECC_1BIT_TH            0x30140c
  40 #define HPRE_RDCHN_INI_ST               0x301a00
  41 #define HPRE_CLSTR_BASE                 0x302000
  42 #define HPRE_CORE_EN_OFFSET             0x04
  43 #define HPRE_CORE_INI_CFG_OFFSET        0x20
  44 #define HPRE_CORE_INI_STATUS_OFFSET     0x80
  45 #define HPRE_CORE_HTBT_WARN_OFFSET      0x8c
  46 #define HPRE_CORE_IS_SCHD_OFFSET        0x90
  47
  48 #define HPRE_RAS_CE_ENB                 0x301410
  49 #define HPRE_HAC_RAS_CE_ENABLE          0x3f
  50 #define HPRE_RAS_NFE_ENB                0x301414
  51 #define HPRE_HAC_RAS_NFE_ENABLE         0x3fffc0
  52 #define HPRE_RAS_FE_ENB                 0x301418
  53 #define HPRE_HAC_RAS_FE_ENABLE          0
  54
  55 #define HPRE_CORE_ENB           (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET)
  56 #define HPRE_CORE_INI_CFG       (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET)
  57 #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET)
  58 #define HPRE_HAC_ECC1_CNT               0x301a04
  59 #define HPRE_HAC_ECC2_CNT               0x301a08
  60 #define HPRE_HAC_INT_STATUS             0x301800
  61 #define HPRE_HAC_SOURCE_INT             0x301600
  62 #define HPRE_CLSTR_ADDR_INTRVL          0x1000
  63 #define HPRE_CLUSTER_INQURY             0x100
  64 #define HPRE_CLSTR_ADDR_INQRY_RSLT      0x104
  65 #define HPRE_TIMEOUT_ABNML_BIT          6
  66 #define HPRE_PASID_EN_BIT               9
  67 #define HPRE_REG_RD_INTVRL_US           10
  68 #define HPRE_REG_RD_TMOUT_US            1000
  69 #define HPRE_DBGFS_VAL_MAX_LEN          20
  70 #define HPRE_PCI_DEVICE_ID              0xa258
  71 #define HPRE_PCI_VF_DEVICE_ID           0xa259
  72 #define HPRE_ADDR(qm, offset)           ((qm)->io_base + (offset))
  73 #define HPRE_QM_USR_CFG_MASK            0xfffffffe
  74 #define HPRE_QM_AXI_CFG_MASK            0xffff
  75 #define HPRE_QM_VFG_AX_MASK             0xff
  76 #define HPRE_BD_USR_MASK                0x3
  77 #define HPRE_CLUSTER_CORE_MASK          0xf
  78
  79 #define HPRE_AM_OOO_SHUTDOWN_ENB        0x301044
  80 #define HPRE_AM_OOO_SHUTDOWN_ENABLE     BIT(0)
  81 #define HPRE_WR_MSI_PORT                BIT(2)
  82
  83 #define HPRE_CORE_ECC_2BIT_ERR          BIT(1)
  84 #define HPRE_OOO_ECC_2BIT_ERR           BIT(5)
  85
  86 #define HPRE_VIA_MSI_DSM                1
  87 #define HPRE_SQE_MASK_OFFSET            8
  88 #define HPRE_SQE_MASK_LEN               24
  89
  90 static struct hisi_qm_list hpre_devices;
  91 static const char hpre_name[] = "hisi_hpre";
  92 static struct dentry *hpre_debugfs_root;
  93 static const struct pci_device_id hpre_dev_ids[] = {
  94         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_DEVICE_ID) },
  95         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_VF_DEVICE_ID) },
  96         { 0, }
  97 };
  98
  99 MODULE_DEVICE_TABLE(pci, hpre_dev_ids);
 100
 101 struct hpre_hw_error {
 102         u32 int_msk;
 103         const char *msg;
 104 };
 105
 106 static const char * const hpre_debug_file_name[] = {
 107         [HPRE_CURRENT_QM]   = "current_qm",
 108         [HPRE_CLEAR_ENABLE] = "rdclr_en",
 109         [HPRE_CLUSTER_CTRL] = "cluster_ctrl",
 110 };
 111
 112 static const struct hpre_hw_error hpre_hw_errors[] = {
 113         { .int_msk = BIT(0), .msg = "core_ecc_1bit_err_int_set" },
 114         { .int_msk = BIT(1), .msg = "core_ecc_2bit_err_int_set" },
 115         { .int_msk = BIT(2), .msg = "dat_wb_poison_int_set" },
 116         { .int_msk = BIT(3), .msg = "dat_rd_poison_int_set" },
 117         { .int_msk = BIT(4), .msg = "bd_rd_poison_int_set" },
 118         { .int_msk = BIT(5), .msg = "ooo_ecc_2bit_err_int_set" },
 119         { .int_msk = BIT(6), .msg = "cluster1_shb_timeout_int_set" },
 120         { .int_msk = BIT(7), .msg = "cluster2_shb_timeout_int_set" },
 121         { .int_msk = BIT(8), .msg = "cluster3_shb_timeout_int_set" },
 122         { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" },
 123         { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" },
 124         { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" },
 125         { /* sentinel */ }
 126 };
 127
 128 static const u64 hpre_cluster_offsets[] = {
 129         [HPRE_CLUSTER0] =
 130                 HPRE_CLSTR_BASE + HPRE_CLUSTER0 * HPRE_CLSTR_ADDR_INTRVL,
 131         [HPRE_CLUSTER1] =
 132                 HPRE_CLSTR_BASE + HPRE_CLUSTER1 * HPRE_CLSTR_ADDR_INTRVL,
 133         [HPRE_CLUSTER2] =
 134                 HPRE_CLSTR_BASE + HPRE_CLUSTER2 * HPRE_CLSTR_ADDR_INTRVL,
 135         [HPRE_CLUSTER3] =
 136                 HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL,
 137 };
 138
 139 static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
 140         {"CORES_EN_STATUS          ",  HPRE_CORE_EN_OFFSET},
 141         {"CORES_INI_CFG              ",  HPRE_CORE_INI_CFG_OFFSET},
 142         {"CORES_INI_STATUS         ",  HPRE_CORE_INI_STATUS_OFFSET},
 143         {"CORES_HTBT_WARN         ",  HPRE_CORE_HTBT_WARN_OFFSET},
 144         {"CORES_IS_SCHD               ",  HPRE_CORE_IS_SCHD_OFFSET},
 145 };
 146
 147 static const struct debugfs_reg32 hpre_com_dfx_regs[] = {
 148         {"READ_CLR_EN          ",  HPRE_CTRL_CNT_CLR_CE},
 149         {"AXQOS                   ",  HPRE_VFG_AXQOS},
 150         {"AWUSR_CFG              ",  HPRE_AWUSR_FP_CFG},
 151         {"QM_ARUSR_MCFG1           ",  QM_ARUSER_M_CFG_1},
 152         {"QM_AWUSR_MCFG1           ",  QM_AWUSER_M_CFG_1},
 153         {"BD_ENDIAN               ",  HPRE_BD_ENDIAN},
 154         {"ECC_CHECK_CTRL       ",  HPRE_ECC_BYPASS},
 155         {"RAS_INT_WIDTH       ",  HPRE_RAS_WIDTH_CFG},
 156         {"POISON_BYPASS       ",  HPRE_POISON_BYPASS},
 157         {"BD_ARUSER               ",  HPRE_BD_ARUSR_CFG},
 158         {"BD_AWUSER               ",  HPRE_BD_AWUSR_CFG},
 159         {"DATA_ARUSER            ",  HPRE_DATA_RUSER_CFG},
 160         {"DATA_AWUSER           ",  HPRE_DATA_WUSER_CFG},
 161         {"INT_STATUS               ",  HPRE_INT_STATUS},
 162 };
 163
 164 static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = {
 165         "send_cnt",
 166         "recv_cnt",
 167         "send_fail_cnt",
 168         "send_busy_cnt",
 169         "over_thrhld_cnt",
 170         "overtime_thrhld",
 171         "invalid_req_cnt"
 172 };
 173
 174 static int pf_q_num_set(const char *val, const struct kernel_param *kp)
 175 {
 176         return q_num_set(val, kp, HPRE_PCI_DEVICE_ID);
 177 }
 178
 179 static const struct kernel_param_ops hpre_pf_q_num_ops = {
 180         .set = pf_q_num_set,
 181         .get = param_get_int,
 182 };
 183
 184 static u32 pf_q_num = HPRE_PF_DEF_Q_NUM;
 185 module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444);
 186 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(1-1024)");
 187
 188 static const struct kernel_param_ops vfs_num_ops = {
 189         .set = vfs_num_set,
 190         .get = param_get_int,
 191 };
 192
 193 static u32 vfs_num;
 194 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
 195 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
 196
 197 struct hisi_qp *hpre_create_qp(void)
 198 {
 199         int node = cpu_to_node(smp_processor_id());
 200         struct hisi_qp *qp = NULL;
 201         int ret;
 202
 203         ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, 0, node, &qp);
 204         if (!ret)
 205                 return qp;
 206
 207         return NULL;
 208 }
 209
 210 static int hpre_cfg_by_dsm(struct hisi_qm *qm)
 211 {
 212         struct device *dev = &qm->pdev->dev;
 213         union acpi_object *obj;
 214         guid_t guid;
 215
 216         if (guid_parse("b06b81ab-0134-4a45-9b0c-483447b95fa7", &guid)) {
 217                 dev_err(dev, "Hpre GUID failed\n");
 218                 return -EINVAL;
 219         }
 220
 221         /* Switch over to MSI handling due to non-standard PCI implementation */
 222         obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid,
 223                                 0, HPRE_VIA_MSI_DSM, NULL);
 224         if (!obj) {
 225                 dev_err(dev, "ACPI handle failed!\n");
 226                 return -EIO;
 227         }
 228
 229         ACPI_FREE(obj);
 230
 231         return 0;
 232 }
 233
 234 static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
 235 {
 236         struct device *dev = &qm->pdev->dev;
 237         unsigned long offset;
 238         int ret, i;
 239         u32 val;
 240
 241         writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE));
 242         writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE));
 243         writel_relaxed(HPRE_QM_AXI_CFG_MASK, HPRE_ADDR(qm, QM_AXI_M_CFG));
 244
 245         /* disable FLR triggered by BME(bus master enable) */
 246         writel(PEH_AXUSER_CFG, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG));
 247         writel(PEH_AXUSER_CFG_ENABLE, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG_ENABLE));
 248
 249         /* HPRE need more time, we close this interrupt */
 250         val = readl_relaxed(HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK));
 251         val |= BIT(HPRE_TIMEOUT_ABNML_BIT);
 252         writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK));
 253
 254         writel(0x1, HPRE_ADDR(qm, HPRE_TYPES_ENB));
 255         writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE));
 256         writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN));
 257         writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK));
 258         writel(0x0, HPRE_ADDR(qm, HPRE_RAS_ECC_1BIT_TH));
 259         writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS));
 260         writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE));
 261         writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS));
 262
 263         writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_ARUSR_CFG));
 264         writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_AWUSR_CFG));
 265         writel(0x1, HPRE_ADDR(qm, HPRE_RDCHN_INI_CFG));
 266         ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, HPRE_RDCHN_INI_ST), val,
 267                         val & BIT(0),
 268                         HPRE_REG_RD_INTVRL_US,
 269                         HPRE_REG_RD_TMOUT_US);
 270         if (ret) {
 271                 dev_err(dev, "read rd channel timeout fail!\n");
 272                 return -ETIMEDOUT;
 273         }
 274
 275         for (i = 0; i < HPRE_CLUSTERS_NUM; i++) {
 276                 offset = i * HPRE_CLSTR_ADDR_INTRVL;
 277
 278                 /* clusters initiating */
 279                 writel(HPRE_CLUSTER_CORE_MASK,
 280                        HPRE_ADDR(qm, offset + HPRE_CORE_ENB));
 281                 writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG));
 282                 ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset +
 283                                         HPRE_CORE_INI_STATUS), val,
 284                                         ((val & HPRE_CLUSTER_CORE_MASK) ==
 285                                         HPRE_CLUSTER_CORE_MASK),
 286                                         HPRE_REG_RD_INTVRL_US,
 287                                         HPRE_REG_RD_TMOUT_US);
 288                 if (ret) {
 289                         dev_err(dev,
 290                                 "cluster %d int st status timeout!\n", i);
 291                         return -ETIMEDOUT;
 292                 }
 293         }
 294
 295         ret = hpre_cfg_by_dsm(qm);
 296         if (ret)
 297                 dev_err(dev, "acpi_evaluate_dsm err.\n");
 298
 299         return ret;
 300 }
 301
 302 static void hpre_cnt_regs_clear(struct hisi_qm *qm)
 303 {
 304         unsigned long offset;
 305         int i;
 306
 307         /* clear current_qm */
 308         writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
 309         writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
 310
 311         /* clear clusterX/cluster_ctrl */
 312         for (i = 0; i < HPRE_CLUSTERS_NUM; i++) {
 313                 offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL;
 314                 writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY);
 315         }
 316
 317         /* clear rdclr_en */
 318         writel(0x0, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
 319
 320         hisi_qm_debug_regs_clear(qm);
 321 }
 322
 323 static void hpre_hw_error_disable(struct hisi_qm *qm)
 324 {
 325         u32 val;
 326
 327         /* disable hpre hw error interrupts */
 328         writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK);
 329
 330         /* disable HPRE block master OOO when m-bit error occur */
 331         val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 332         val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
 333         writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 334 }
 335
 336 static void hpre_hw_error_enable(struct hisi_qm *qm)
 337 {
 338         u32 val;
 339
 340         /* clear HPRE hw error source if having */
 341         writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT);
 342
 343         /* enable hpre hw error interrupts */
 344         writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK);
 345         writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB);
 346         writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB);
 347         writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB);
 348
 349         /* enable HPRE block master OOO when m-bit error occur */
 350         val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 351         val |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
 352         writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 353 }
 354
 355 static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
 356 {
 357         struct hpre *hpre = container_of(file->debug, struct hpre, debug);
 358
 359         return &hpre->qm;
 360 }
 361
 362 static u32 hpre_current_qm_read(struct hpre_debugfs_file *file)
 363 {
 364         struct hisi_qm *qm = hpre_file_to_qm(file);
 365
 366         return readl(qm->io_base + QM_DFX_MB_CNT_VF);
 367 }
 368
 369 static int hpre_current_qm_write(struct hpre_debugfs_file *file, u32 val)
 370 {
 371         struct hisi_qm *qm = hpre_file_to_qm(file);
 372         u32 num_vfs = qm->vfs_num;
 373         u32 vfq_num, tmp;
 374
 375
 376         if (val > num_vfs)
 377                 return -EINVAL;
 378
 379         /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
 380         if (val == 0) {
 381                 qm->debug.curr_qm_qp_num = qm->qp_num;
 382         } else {
 383                 vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs;
 384                 if (val == num_vfs) {
 385                         qm->debug.curr_qm_qp_num =
 386                         qm->ctrl_qp_num - qm->qp_num - (num_vfs - 1) * vfq_num;
 387                 } else {
 388                         qm->debug.curr_qm_qp_num = vfq_num;
 389                 }
 390         }
 391
 392         writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
 393         writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
 394
 395         tmp = val |
 396               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
 397         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 398
 399         tmp = val |
 400               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
 401         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 402
 403         return  0;
 404 }
 405
 406 static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file)
 407 {
 408         struct hisi_qm *qm = hpre_file_to_qm(file);
 409
 410         return readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
 411                HPRE_CTRL_CNT_CLR_CE_BIT;
 412 }
 413
 414 static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val)
 415 {
 416         struct hisi_qm *qm = hpre_file_to_qm(file);
 417         u32 tmp;
 418
 419         if (val != 1 && val != 0)
 420                 return -EINVAL;
 421
 422         tmp = (readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
 423                ~HPRE_CTRL_CNT_CLR_CE_BIT) | val;
 424         writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
 425
 426         return  0;
 427 }
 428
 429 static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file)
 430 {
 431         struct hisi_qm *qm = hpre_file_to_qm(file);
 432         int cluster_index = file->index - HPRE_CLUSTER_CTRL;
 433         unsigned long offset = HPRE_CLSTR_BASE +
 434                                cluster_index * HPRE_CLSTR_ADDR_INTRVL;
 435
 436         return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT);
 437 }
 438
 439 static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val)
 440 {
 441         struct hisi_qm *qm = hpre_file_to_qm(file);
 442         int cluster_index = file->index - HPRE_CLUSTER_CTRL;
 443         unsigned long offset = HPRE_CLSTR_BASE + cluster_index *
 444                                HPRE_CLSTR_ADDR_INTRVL;
 445
 446         writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY);
 447
 448         return  0;
 449 }
 450
 451 static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf,
 452                                size_t count, loff_t *pos)
 453 {
 454         struct hpre_debugfs_file *file = filp->private_data;
 455         char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
 456         u32 val;
 457         int ret;
 458
 459         spin_lock_irq(&file->lock);
 460         switch (file->type) {
 461         case HPRE_CURRENT_QM:
 462                 val = hpre_current_qm_read(file);
 463                 break;
 464         case HPRE_CLEAR_ENABLE:
 465                 val = hpre_clear_enable_read(file);
 466                 break;
 467         case HPRE_CLUSTER_CTRL:
 468                 val = hpre_cluster_inqry_read(file);
 469                 break;
 470         default:
 471                 spin_unlock_irq(&file->lock);
 472                 return -EINVAL;
 473         }
 474         spin_unlock_irq(&file->lock);
 475         ret = snprintf(tbuf, HPRE_DBGFS_VAL_MAX_LEN, "%u\n", val);
 476         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
 477 }
 478
 479 static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf,
 480                                 size_t count, loff_t *pos)
 481 {
 482         struct hpre_debugfs_file *file = filp->private_data;
 483         char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
 484         unsigned long val;
 485         int len, ret;
 486
 487         if (*pos != 0)
 488                 return 0;
 489
 490         if (count >= HPRE_DBGFS_VAL_MAX_LEN)
 491                 return -ENOSPC;
 492
 493         len = simple_write_to_buffer(tbuf, HPRE_DBGFS_VAL_MAX_LEN - 1,
 494                                      pos, buf, count);
 495         if (len < 0)
 496                 return len;
 497
 498         tbuf[len] = '\0';
 499         if (kstrtoul(tbuf, 0, &val))
 500                 return -EFAULT;
 501
 502         spin_lock_irq(&file->lock);
 503         switch (file->type) {
 504         case HPRE_CURRENT_QM:
 505                 ret = hpre_current_qm_write(file, val);
 506                 if (ret)
 507                         goto err_input;
 508                 break;
 509         case HPRE_CLEAR_ENABLE:
 510                 ret = hpre_clear_enable_write(file, val);
 511                 if (ret)
 512                         goto err_input;
 513                 break;
 514         case HPRE_CLUSTER_CTRL:
 515                 ret = hpre_cluster_inqry_write(file, val);
 516                 if (ret)
 517                         goto err_input;
 518                 break;
 519         default:
 520                 ret = -EINVAL;
 521                 goto err_input;
 522         }
 523         spin_unlock_irq(&file->lock);
 524
 525         return count;
 526
 527 err_input:
 528         spin_unlock_irq(&file->lock);
 529         return ret;
 530 }
 531
 532 static const struct file_operations hpre_ctrl_debug_fops = {
 533         .owner = THIS_MODULE,
 534         .open = simple_open,
 535         .read = hpre_ctrl_debug_read,
 536         .write = hpre_ctrl_debug_write,
 537 };
 538
 539 static int hpre_debugfs_atomic64_get(void *data, u64 *val)
 540 {
 541         struct hpre_dfx *dfx_item = data;
 542
 543         *val = atomic64_read(&dfx_item->value);
 544
 545         return 0;
 546 }
 547
 548 static int hpre_debugfs_atomic64_set(void *data, u64 val)
 549 {
 550         struct hpre_dfx *dfx_item = data;
 551         struct hpre_dfx *hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD;
 552
 553         if (val)
 554                 return -EINVAL;
 555
 556         if (dfx_item->type == HPRE_OVERTIME_THRHLD)
 557                 atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0);
 558         atomic64_set(&dfx_item->value, val);
 559
 560         return 0;
 561 }
 562
 563 DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get,
 564                          hpre_debugfs_atomic64_set, "%llu\n");
 565
 566 static int hpre_create_debugfs_file(struct hpre_debug *dbg, struct dentry *dir,
 567                                     enum hpre_ctrl_dbgfs_file type, int indx)
 568 {
 569         struct dentry *file_dir;
 570
 571         if (dir)
 572                 file_dir = dir;
 573         else
 574                 file_dir = dbg->debug_root;
 575
 576         if (type >= HPRE_DEBUG_FILE_NUM)
 577                 return -EINVAL;
 578
 579         spin_lock_init(&dbg->files[indx].lock);
 580         dbg->files[indx].debug = dbg;
 581         dbg->files[indx].type = type;
 582         dbg->files[indx].index = indx;
 583         debugfs_create_file(hpre_debug_file_name[type], 0600, file_dir,
 584                             dbg->files + indx, &hpre_ctrl_debug_fops);
 585
 586         return 0;
 587 }
 588
 589 static int hpre_pf_comm_regs_debugfs_init(struct hpre_debug *debug)
 590 {
 591         struct hpre *hpre = container_of(debug, struct hpre, debug);
 592         struct hisi_qm *qm = &hpre->qm;
 593         struct device *dev = &qm->pdev->dev;
 594         struct debugfs_regset32 *regset;
 595
 596         regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 597         if (!regset)
 598                 return -ENOMEM;
 599
 600         regset->regs = hpre_com_dfx_regs;
 601         regset->nregs = ARRAY_SIZE(hpre_com_dfx_regs);
 602         regset->base = qm->io_base;
 603
 604         debugfs_create_regset32("regs", 0444,  debug->debug_root, regset);
 605         return 0;
 606 }
 607
 608 static int hpre_cluster_debugfs_init(struct hpre_debug *debug)
 609 {
 610         struct hpre *hpre = container_of(debug, struct hpre, debug);
 611         struct hisi_qm *qm = &hpre->qm;
 612         struct device *dev = &qm->pdev->dev;
 613         char buf[HPRE_DBGFS_VAL_MAX_LEN];
 614         struct debugfs_regset32 *regset;
 615         struct dentry *tmp_d;
 616         int i, ret;
 617
 618         for (i = 0; i < HPRE_CLUSTERS_NUM; i++) {
 619                 ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i);
 620                 if (ret < 0)
 621                         return -EINVAL;
 622                 tmp_d = debugfs_create_dir(buf, debug->debug_root);
 623
 624                 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 625                 if (!regset)
 626                         return -ENOMEM;
 627
 628                 regset->regs = hpre_cluster_dfx_regs;
 629                 regset->nregs = ARRAY_SIZE(hpre_cluster_dfx_regs);
 630                 regset->base = qm->io_base + hpre_cluster_offsets[i];
 631
 632                 debugfs_create_regset32("regs", 0444, tmp_d, regset);
 633                 ret = hpre_create_debugfs_file(debug, tmp_d, HPRE_CLUSTER_CTRL,
 634                                                i + HPRE_CLUSTER_CTRL);
 635                 if (ret)
 636                         return ret;
 637         }
 638
 639         return 0;
 640 }
 641
 642 static int hpre_ctrl_debug_init(struct hpre_debug *debug)
 643 {
 644         int ret;
 645
 646         ret = hpre_create_debugfs_file(debug, NULL, HPRE_CURRENT_QM,
 647                                        HPRE_CURRENT_QM);
 648         if (ret)
 649                 return ret;
 650
 651         ret = hpre_create_debugfs_file(debug, NULL, HPRE_CLEAR_ENABLE,
 652                                        HPRE_CLEAR_ENABLE);
 653         if (ret)
 654                 return ret;
 655
 656         ret = hpre_pf_comm_regs_debugfs_init(debug);
 657         if (ret)
 658                 return ret;
 659
 660         return hpre_cluster_debugfs_init(debug);
 661 }
 662
 663 static void hpre_dfx_debug_init(struct hpre_debug *debug)
 664 {
 665         struct hpre *hpre = container_of(debug, struct hpre, debug);
 666         struct hpre_dfx *dfx = hpre->debug.dfx;
 667         struct hisi_qm *qm = &hpre->qm;
 668         struct dentry *parent;
 669         int i;
 670
 671         parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root);
 672         for (i = 0; i < HPRE_DFX_FILE_NUM; i++) {
 673                 dfx[i].type = i;
 674                 debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i],
 675                                     &hpre_atomic64_ops);
 676         }
 677 }
 678
 679 static int hpre_debugfs_init(struct hpre *hpre)
 680 {
 681         struct hisi_qm *qm = &hpre->qm;
 682         struct device *dev = &qm->pdev->dev;
 683         struct dentry *dir;
 684         int ret;
 685
 686         dir = debugfs_create_dir(dev_name(dev), hpre_debugfs_root);
 687         qm->debug.debug_root = dir;
 688         qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET;
 689         qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN;
 690
 691         ret = hisi_qm_debug_init(qm);
 692         if (ret)
 693                 goto failed_to_create;
 694
 695         if (qm->pdev->device == HPRE_PCI_DEVICE_ID) {
 696                 hpre->debug.debug_root = dir;
 697                 ret = hpre_ctrl_debug_init(&hpre->debug);
 698                 if (ret)
 699                         goto failed_to_create;
 700         }
 701
 702         hpre_dfx_debug_init(&hpre->debug);
 703
 704         return 0;
 705
 706 failed_to_create:
 707         debugfs_remove_recursive(qm->debug.debug_root);
 708         return ret;
 709 }
 710
 711 static void hpre_debugfs_exit(struct hpre *hpre)
 712 {
 713         struct hisi_qm *qm = &hpre->qm;
 714
 715         debugfs_remove_recursive(qm->debug.debug_root);
 716 }
 717
 718 static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
 719 {
 720         if (pdev->revision == QM_HW_V1) {
 721                 pci_warn(pdev, "HPRE version 1 is not supported!\n");
 722                 return -EINVAL;
 723         }
 724
 725         qm->pdev = pdev;
 726         qm->ver = pdev->revision;
 727         qm->sqe_size = HPRE_SQE_SIZE;
 728         qm->dev_name = hpre_name;
 729
 730         qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ?
 731                         QM_HW_PF : QM_HW_VF;
 732         if (qm->fun_type == QM_HW_PF) {
 733                 qm->qp_base = HPRE_PF_DEF_Q_BASE;
 734                 qm->qp_num = pf_q_num;
 735                 qm->qm_list = &hpre_devices;
 736         }
 737
 738         return hisi_qm_init(qm);
 739 }
 740
 741 static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
 742 {
 743         const struct hpre_hw_error *err = hpre_hw_errors;
 744         struct device *dev = &qm->pdev->dev;
 745
 746         while (err->msg) {
 747                 if (err->int_msk & err_sts)
 748                         dev_warn(dev, "%s [error status=0x%x] found\n",
 749                                  err->msg, err->int_msk);
 750                 err++;
 751         }
 752 }
 753
 754 static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
 755 {
 756         return readl(qm->io_base + HPRE_HAC_INT_STATUS);
 757 }
 758
 759 static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
 760 {
 761         writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT);
 762 }
 763
 764 static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
 765 {
 766         u32 value;
 767
 768         value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 769         writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE,
 770                HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
 771         writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE,
 772                HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
 773 }
 774
 775 static const struct hisi_qm_err_ini hpre_err_ini = {
 776         .hw_init                = hpre_set_user_domain_and_cache,
 777         .hw_err_enable          = hpre_hw_error_enable,
 778         .hw_err_disable         = hpre_hw_error_disable,
 779         .get_dev_hw_err_status  = hpre_get_hw_err_status,
 780         .clear_dev_hw_err_status = hpre_clear_hw_err_status,
 781         .log_dev_hw_err         = hpre_log_hw_error,
 782         .open_axi_master_ooo    = hpre_open_axi_master_ooo,
 783         .err_info               = {
 784                 .ce                     = QM_BASE_CE,
 785                 .nfe                    = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT,
 786                 .fe                     = 0,
 787                 .ecc_2bits_mask         = HPRE_CORE_ECC_2BIT_ERR |
 788                                           HPRE_OOO_ECC_2BIT_ERR,
 789                 .msi_wr_port            = HPRE_WR_MSI_PORT,
 790                 .acpi_rst               = "HRST",
 791         }
 792 };
 793
 794 static int hpre_pf_probe_init(struct hpre *hpre)
 795 {
 796         struct hisi_qm *qm = &hpre->qm;
 797         int ret;
 798
 799         qm->ctrl_qp_num = HPRE_QUEUE_NUM_V2;
 800
 801         ret = hpre_set_user_domain_and_cache(qm);
 802         if (ret)
 803                 return ret;
 804
 805         qm->err_ini = &hpre_err_ini;
 806         hisi_qm_dev_err_init(qm);
 807
 808         return 0;
 809 }
 810
 811 static int hpre_probe_init(struct hpre *hpre)
 812 {
 813         struct hisi_qm *qm = &hpre->qm;
 814         int ret;
 815
 816         if (qm->fun_type == QM_HW_PF) {
 817                 ret = hpre_pf_probe_init(hpre);
 818                 if (ret)
 819                         return ret;
 820         }
 821
 822         return 0;
 823 }
 824
 825 static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 826 {
 827         struct hisi_qm *qm;
 828         struct hpre *hpre;
 829         int ret;
 830
 831         hpre = devm_kzalloc(&pdev->dev, sizeof(*hpre), GFP_KERNEL);
 832         if (!hpre)
 833                 return -ENOMEM;
 834
 835         qm = &hpre->qm;
 836         ret = hpre_qm_init(qm, pdev);
 837         if (ret) {
 838                 pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret);
 839                 return ret;
 840         }
 841
 842         ret = hpre_probe_init(hpre);
 843         if (ret) {
 844                 pci_err(pdev, "Failed to probe (%d)!\n", ret);
 845                 goto err_with_qm_init;
 846         }
 847
 848         ret = hisi_qm_start(qm);
 849         if (ret)
 850                 goto err_with_err_init;
 851
 852         ret = hpre_debugfs_init(hpre);
 853         if (ret)
 854                 dev_warn(&pdev->dev, "init debugfs fail!\n");
 855
 856         hisi_qm_add_to_list(qm, &hpre_devices);
 857
 858         ret = hpre_algs_register();
 859         if (ret < 0) {
 860                 pci_err(pdev, "fail to register algs to crypto!\n");
 861                 goto err_with_qm_start;
 862         }
 863
 864         if (qm->fun_type == QM_HW_PF && vfs_num) {
 865                 ret = hisi_qm_sriov_enable(pdev, vfs_num);
 866                 if (ret < 0)
 867                         goto err_with_crypto_register;
 868         }
 869
 870         return 0;
 871
 872 err_with_crypto_register:
 873         hpre_algs_unregister();
 874
 875 err_with_qm_start:
 876         hisi_qm_del_from_list(qm, &hpre_devices);
 877         hisi_qm_stop(qm);
 878
 879 err_with_err_init:
 880         hisi_qm_dev_err_uninit(qm);
 881
 882 err_with_qm_init:
 883         hisi_qm_uninit(qm);
 884
 885         return ret;
 886 }
 887
 888 static void hpre_remove(struct pci_dev *pdev)
 889 {
 890         struct hpre *hpre = pci_get_drvdata(pdev);
 891         struct hisi_qm *qm = &hpre->qm;
 892         int ret;
 893
 894         hpre_algs_unregister();
 895         hisi_qm_del_from_list(qm, &hpre_devices);
 896         if (qm->fun_type == QM_HW_PF && qm->vfs_num) {
 897                 ret = hisi_qm_sriov_disable(pdev);
 898                 if (ret) {
 899                         pci_err(pdev, "Disable SRIOV fail!\n");
 900                         return;
 901                 }
 902         }
 903         if (qm->fun_type == QM_HW_PF) {
 904                 hpre_cnt_regs_clear(qm);
 905                 qm->debug.curr_qm_qp_num = 0;
 906         }
 907
 908         hpre_debugfs_exit(hpre);
 909         hisi_qm_stop(qm);
 910         hisi_qm_dev_err_uninit(qm);
 911         hisi_qm_uninit(qm);
 912 }
 913
 914
 915 static const struct pci_error_handlers hpre_err_handler = {
 916         .error_detected         = hisi_qm_dev_err_detected,
 917         .slot_reset             = hisi_qm_dev_slot_reset,
 918         .reset_prepare          = hisi_qm_reset_prepare,
 919         .reset_done             = hisi_qm_reset_done,
 920 };
 921
 922 static struct pci_driver hpre_pci_driver = {
 923         .name                   = hpre_name,
 924         .id_table               = hpre_dev_ids,
 925         .probe                  = hpre_probe,
 926         .remove                 = hpre_remove,
 927         .sriov_configure        = hisi_qm_sriov_configure,
 928         .err_handler            = &hpre_err_handler,
 929 };
 930
 931 static void hpre_register_debugfs(void)
 932 {
 933         if (!debugfs_initialized())
 934                 return;
 935
 936         hpre_debugfs_root = debugfs_create_dir(hpre_name, NULL);
 937 }
 938
 939 static void hpre_unregister_debugfs(void)
 940 {
 941         debugfs_remove_recursive(hpre_debugfs_root);
 942 }
 943
 944 static int __init hpre_init(void)
 945 {
 946         int ret;
 947
 948         hisi_qm_init_list(&hpre_devices);
 949         hpre_register_debugfs();
 950
 951         ret = pci_register_driver(&hpre_pci_driver);
 952         if (ret) {
 953                 hpre_unregister_debugfs();
 954                 pr_err("hpre: can't register hisi hpre driver.\n");
 955         }
 956
 957         return ret;
 958 }
 959
 960 static void __exit hpre_exit(void)
 961 {
 962         pci_unregister_driver(&hpre_pci_driver);
 963         hpre_unregister_debugfs();
 964 }
 965
 966 module_init(hpre_init);
 967 module_exit(hpre_exit);
 968
 969 MODULE_LICENSE("GPL v2");
 970 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
 971 MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator");