drivers/crypto/hisilicon/hpre/hpre_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018-2019 HiSilicon Limited. */
   3 #include <linux/acpi.h>
   4 #include <linux/aer.h>
   5 #include <linux/bitops.h>
   6 #include <linux/debugfs.h>
   7 #include <linux/init.h>
   8 #include <linux/io.h>
   9 #include <linux/kernel.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/topology.h>
  13 #include <linux/uacce.h>
  14 #include "hpre.h"
  15
  16 #define HPRE_QM_ABNML_INT_MASK          0x100004
  17 #define HPRE_CTRL_CNT_CLR_CE_BIT        BIT(0)
  18 #define HPRE_COMM_CNT_CLR_CE            0x0
  19 #define HPRE_CTRL_CNT_CLR_CE            0x301000
  20 #define HPRE_FSM_MAX_CNT                0x301008
  21 #define HPRE_VFG_AXQOS                  0x30100c
  22 #define HPRE_VFG_AXCACHE                0x301010
  23 #define HPRE_RDCHN_INI_CFG              0x301014
  24 #define HPRE_AWUSR_FP_CFG               0x301018
  25 #define HPRE_BD_ENDIAN                  0x301020
  26 #define HPRE_ECC_BYPASS                 0x301024
  27 #define HPRE_RAS_WIDTH_CFG              0x301028
  28 #define HPRE_POISON_BYPASS              0x30102c
  29 #define HPRE_BD_ARUSR_CFG               0x301030
  30 #define HPRE_BD_AWUSR_CFG               0x301034
  31 #define HPRE_TYPES_ENB                  0x301038
  32 #define HPRE_RSA_ENB                    BIT(0)
  33 #define HPRE_ECC_ENB                    BIT(1)
  34 #define HPRE_DATA_RUSER_CFG             0x30103c
  35 #define HPRE_DATA_WUSER_CFG             0x301040
  36 #define HPRE_INT_MASK                   0x301400
  37 #define HPRE_INT_STATUS                 0x301800
  38 #define HPRE_CORE_INT_ENABLE            0
  39 #define HPRE_CORE_INT_DISABLE           GENMASK(21, 0)
  40 #define HPRE_RDCHN_INI_ST               0x301a00
  41 #define HPRE_CLSTR_BASE                 0x302000
  42 #define HPRE_CORE_EN_OFFSET             0x04
  43 #define HPRE_CORE_INI_CFG_OFFSET        0x20
  44 #define HPRE_CORE_INI_STATUS_OFFSET     0x80
  45 #define HPRE_CORE_HTBT_WARN_OFFSET      0x8c
  46 #define HPRE_CORE_IS_SCHD_OFFSET        0x90
  47
  48 #define HPRE_RAS_CE_ENB                 0x301410
  49 #define HPRE_HAC_RAS_CE_ENABLE          (BIT(0) | BIT(22) | BIT(23))
  50 #define HPRE_RAS_NFE_ENB                0x301414
  51 #define HPRE_HAC_RAS_NFE_ENABLE         0x3ffffe
  52 #define HPRE_RAS_FE_ENB                 0x301418
  53 #define HPRE_OOO_SHUTDOWN_SEL           0x301a3c
  54 #define HPRE_HAC_RAS_FE_ENABLE          0
  55
  56 #define HPRE_CORE_ENB           (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET)
  57 #define HPRE_CORE_INI_CFG       (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET)
  58 #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET)
  59 #define HPRE_HAC_ECC1_CNT               0x301a04
  60 #define HPRE_HAC_ECC2_CNT               0x301a08
  61 #define HPRE_HAC_SOURCE_INT             0x301600
  62 #define HPRE_CLSTR_ADDR_INTRVL          0x1000
  63 #define HPRE_CLUSTER_INQURY             0x100
  64 #define HPRE_CLSTR_ADDR_INQRY_RSLT      0x104
  65 #define HPRE_TIMEOUT_ABNML_BIT          6
  66 #define HPRE_PASID_EN_BIT               9
  67 #define HPRE_REG_RD_INTVRL_US           10
  68 #define HPRE_REG_RD_TMOUT_US            1000
  69 #define HPRE_DBGFS_VAL_MAX_LEN          20
  70 #define HPRE_PCI_DEVICE_ID              0xa258
  71 #define HPRE_PCI_VF_DEVICE_ID           0xa259
  72 #define HPRE_QM_USR_CFG_MASK            GENMASK(31, 1)
  73 #define HPRE_QM_AXI_CFG_MASK            GENMASK(15, 0)
  74 #define HPRE_QM_VFG_AX_MASK             GENMASK(7, 0)
  75 #define HPRE_BD_USR_MASK                GENMASK(1, 0)
  76 #define HPRE_CLUSTER_CORE_MASK_V2       GENMASK(3, 0)
  77 #define HPRE_CLUSTER_CORE_MASK_V3       GENMASK(7, 0)
  78 #define HPRE_PREFETCH_CFG               0x301130
  79 #define HPRE_SVA_PREFTCH_DFX            0x30115C
  80 #define HPRE_PREFETCH_ENABLE            (~(BIT(0) | BIT(30)))
  81 #define HPRE_PREFETCH_DISABLE           BIT(30)
  82 #define HPRE_SVA_DISABLE_READY          (BIT(4) | BIT(8))
  83
  84 #define HPRE_AM_OOO_SHUTDOWN_ENB        0x301044
  85 #define HPRE_AM_OOO_SHUTDOWN_ENABLE     BIT(0)
  86 #define HPRE_WR_MSI_PORT                BIT(2)
  87
  88 #define HPRE_CORE_ECC_2BIT_ERR          BIT(1)
  89 #define HPRE_OOO_ECC_2BIT_ERR           BIT(5)
  90
  91 #define HPRE_QM_BME_FLR                 BIT(7)
  92 #define HPRE_QM_PM_FLR                  BIT(11)
  93 #define HPRE_QM_SRIOV_FLR               BIT(12)
  94
  95 #define HPRE_SHAPER_TYPE_RATE           128
  96 #define HPRE_VIA_MSI_DSM                1
  97 #define HPRE_SQE_MASK_OFFSET            8
  98 #define HPRE_SQE_MASK_LEN               24
  99
 100 static const char hpre_name[] = "hisi_hpre";
 101 static struct dentry *hpre_debugfs_root;
 102 static const struct pci_device_id hpre_dev_ids[] = {
 103         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_DEVICE_ID) },
 104         { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HPRE_PCI_VF_DEVICE_ID) },
 105         { 0, }
 106 };
 107
 108 MODULE_DEVICE_TABLE(pci, hpre_dev_ids);
 109
 110 struct hpre_hw_error {
 111         u32 int_msk;
 112         const char *msg;
 113 };
 114
 115 static struct hisi_qm_list hpre_devices = {
 116         .register_to_crypto     = hpre_algs_register,
 117         .unregister_from_crypto = hpre_algs_unregister,
 118 };
 119
 120 static const char * const hpre_debug_file_name[] = {
 121         [HPRE_CLEAR_ENABLE] = "rdclr_en",
 122         [HPRE_CLUSTER_CTRL] = "cluster_ctrl",
 123 };
 124
 125 static const struct hpre_hw_error hpre_hw_errors[] = {
 126         {
 127                 .int_msk = BIT(0),
 128                 .msg = "core_ecc_1bit_err_int_set"
 129         }, {
 130                 .int_msk = BIT(1),
 131                 .msg = "core_ecc_2bit_err_int_set"
 132         }, {
 133                 .int_msk = BIT(2),
 134                 .msg = "dat_wb_poison_int_set"
 135         }, {
 136                 .int_msk = BIT(3),
 137                 .msg = "dat_rd_poison_int_set"
 138         }, {
 139                 .int_msk = BIT(4),
 140                 .msg = "bd_rd_poison_int_set"
 141         }, {
 142                 .int_msk = BIT(5),
 143                 .msg = "ooo_ecc_2bit_err_int_set"
 144         }, {
 145                 .int_msk = BIT(6),
 146                 .msg = "cluster1_shb_timeout_int_set"
 147         }, {
 148                 .int_msk = BIT(7),
 149                 .msg = "cluster2_shb_timeout_int_set"
 150         }, {
 151                 .int_msk = BIT(8),
 152                 .msg = "cluster3_shb_timeout_int_set"
 153         }, {
 154                 .int_msk = BIT(9),
 155                 .msg = "cluster4_shb_timeout_int_set"
 156         }, {
 157                 .int_msk = GENMASK(15, 10),
 158                 .msg = "ooo_rdrsp_err_int_set"
 159         }, {
 160                 .int_msk = GENMASK(21, 16),
 161                 .msg = "ooo_wrrsp_err_int_set"
 162         }, {
 163                 .int_msk = BIT(22),
 164                 .msg = "pt_rng_timeout_int_set"
 165         }, {
 166                 .int_msk = BIT(23),
 167                 .msg = "sva_fsm_timeout_int_set"
 168         }, {
 169                 /* sentinel */
 170         }
 171 };
 172
 173 static const u64 hpre_cluster_offsets[] = {
 174         [HPRE_CLUSTER0] =
 175                 HPRE_CLSTR_BASE + HPRE_CLUSTER0 * HPRE_CLSTR_ADDR_INTRVL,
 176         [HPRE_CLUSTER1] =
 177                 HPRE_CLSTR_BASE + HPRE_CLUSTER1 * HPRE_CLSTR_ADDR_INTRVL,
 178         [HPRE_CLUSTER2] =
 179                 HPRE_CLSTR_BASE + HPRE_CLUSTER2 * HPRE_CLSTR_ADDR_INTRVL,
 180         [HPRE_CLUSTER3] =
 181                 HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL,
 182 };
 183
 184 static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
 185         {"CORES_EN_STATUS          ",  HPRE_CORE_EN_OFFSET},
 186         {"CORES_INI_CFG              ",  HPRE_CORE_INI_CFG_OFFSET},
 187         {"CORES_INI_STATUS         ",  HPRE_CORE_INI_STATUS_OFFSET},
 188         {"CORES_HTBT_WARN         ",  HPRE_CORE_HTBT_WARN_OFFSET},
 189         {"CORES_IS_SCHD               ",  HPRE_CORE_IS_SCHD_OFFSET},
 190 };
 191
 192 static const struct debugfs_reg32 hpre_com_dfx_regs[] = {
 193         {"READ_CLR_EN          ",  HPRE_CTRL_CNT_CLR_CE},
 194         {"AXQOS                   ",  HPRE_VFG_AXQOS},
 195         {"AWUSR_CFG              ",  HPRE_AWUSR_FP_CFG},
 196         {"QM_ARUSR_MCFG1           ",  QM_ARUSER_M_CFG_1},
 197         {"QM_AWUSR_MCFG1           ",  QM_AWUSER_M_CFG_1},
 198         {"BD_ENDIAN               ",  HPRE_BD_ENDIAN},
 199         {"ECC_CHECK_CTRL       ",  HPRE_ECC_BYPASS},
 200         {"RAS_INT_WIDTH       ",  HPRE_RAS_WIDTH_CFG},
 201         {"POISON_BYPASS       ",  HPRE_POISON_BYPASS},
 202         {"BD_ARUSER               ",  HPRE_BD_ARUSR_CFG},
 203         {"BD_AWUSER               ",  HPRE_BD_AWUSR_CFG},
 204         {"DATA_ARUSER            ",  HPRE_DATA_RUSER_CFG},
 205         {"DATA_AWUSER           ",  HPRE_DATA_WUSER_CFG},
 206         {"INT_STATUS               ",  HPRE_INT_STATUS},
 207 };
 208
 209 static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = {
 210         "send_cnt",
 211         "recv_cnt",
 212         "send_fail_cnt",
 213         "send_busy_cnt",
 214         "over_thrhld_cnt",
 215         "overtime_thrhld",
 216         "invalid_req_cnt"
 217 };
 218
 219 static const struct kernel_param_ops hpre_uacce_mode_ops = {
 220         .set = uacce_mode_set,
 221         .get = param_get_int,
 222 };
 223
 224 /*
 225  * uacce_mode = 0 means hpre only register to crypto,
 226  * uacce_mode = 1 means hpre both register to crypto and uacce.
 227  */
 228 static u32 uacce_mode = UACCE_MODE_NOUACCE;
 229 module_param_cb(uacce_mode, &hpre_uacce_mode_ops, &uacce_mode, 0444);
 230 MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
 231
 232 static int pf_q_num_set(const char *val, const struct kernel_param *kp)
 233 {
 234         return q_num_set(val, kp, HPRE_PCI_DEVICE_ID);
 235 }
 236
 237 static const struct kernel_param_ops hpre_pf_q_num_ops = {
 238         .set = pf_q_num_set,
 239         .get = param_get_int,
 240 };
 241
 242 static u32 pf_q_num = HPRE_PF_DEF_Q_NUM;
 243 module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444);
 244 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)");
 245
 246 static const struct kernel_param_ops vfs_num_ops = {
 247         .set = vfs_num_set,
 248         .get = param_get_int,
 249 };
 250
 251 static u32 vfs_num;
 252 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
 253 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
 254
 255 static inline int hpre_cluster_num(struct hisi_qm *qm)
 256 {
 257         return (qm->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 :
 258                 HPRE_CLUSTERS_NUM_V2;
 259 }
 260
 261 static inline int hpre_cluster_core_mask(struct hisi_qm *qm)
 262 {
 263         return (qm->ver >= QM_HW_V3) ?
 264                 HPRE_CLUSTER_CORE_MASK_V3 : HPRE_CLUSTER_CORE_MASK_V2;
 265 }
 266
 267 struct hisi_qp *hpre_create_qp(u8 type)
 268 {
 269         int node = cpu_to_node(smp_processor_id());
 270         struct hisi_qp *qp = NULL;
 271         int ret;
 272
 273         if (type != HPRE_V2_ALG_TYPE && type != HPRE_V3_ECC_ALG_TYPE)
 274                 return NULL;
 275
 276         /*
 277          * type: 0 - RSA/DH. algorithm supported in V2,
 278          *       1 - ECC algorithm in V3.
 279          */
 280         ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp);
 281         if (!ret)
 282                 return qp;
 283
 284         return NULL;
 285 }
 286
 287 static void hpre_config_pasid(struct hisi_qm *qm)
 288 {
 289         u32 val1, val2;
 290
 291         if (qm->ver >= QM_HW_V3)
 292                 return;
 293
 294         val1 = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG);
 295         val2 = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG);
 296         if (qm->use_sva) {
 297                 val1 |= BIT(HPRE_PASID_EN_BIT);
 298                 val2 |= BIT(HPRE_PASID_EN_BIT);
 299         } else {
 300                 val1 &= ~BIT(HPRE_PASID_EN_BIT);
 301                 val2 &= ~BIT(HPRE_PASID_EN_BIT);
 302         }
 303         writel_relaxed(val1, qm->io_base + HPRE_DATA_RUSER_CFG);
 304         writel_relaxed(val2, qm->io_base + HPRE_DATA_WUSER_CFG);
 305 }
 306
 307 static int hpre_cfg_by_dsm(struct hisi_qm *qm)
 308 {
 309         struct device *dev = &qm->pdev->dev;
 310         union acpi_object *obj;
 311         guid_t guid;
 312
 313         if (guid_parse("b06b81ab-0134-4a45-9b0c-483447b95fa7", &guid)) {
 314                 dev_err(dev, "Hpre GUID failed\n");
 315                 return -EINVAL;
 316         }
 317
 318         /* Switch over to MSI handling due to non-standard PCI implementation */
 319         obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid,
 320                                 0, HPRE_VIA_MSI_DSM, NULL);
 321         if (!obj) {
 322                 dev_err(dev, "ACPI handle failed!\n");
 323                 return -EIO;
 324         }
 325
 326         ACPI_FREE(obj);
 327
 328         return 0;
 329 }
 330
 331 static int hpre_set_cluster(struct hisi_qm *qm)
 332 {
 333         u32 cluster_core_mask = hpre_cluster_core_mask(qm);
 334         u8 clusters_num = hpre_cluster_num(qm);
 335         struct device *dev = &qm->pdev->dev;
 336         unsigned long offset;
 337         u32 val = 0;
 338         int ret, i;
 339
 340         for (i = 0; i < clusters_num; i++) {
 341                 offset = i * HPRE_CLSTR_ADDR_INTRVL;
 342
 343                 /* clusters initiating */
 344                 writel(cluster_core_mask,
 345                        qm->io_base + offset + HPRE_CORE_ENB);
 346                 writel(0x1, qm->io_base + offset + HPRE_CORE_INI_CFG);
 347                 ret = readl_relaxed_poll_timeout(qm->io_base + offset +
 348                                         HPRE_CORE_INI_STATUS, val,
 349                                         ((val & cluster_core_mask) ==
 350                                         cluster_core_mask),
 351                                         HPRE_REG_RD_INTVRL_US,
 352                                         HPRE_REG_RD_TMOUT_US);
 353                 if (ret) {
 354                         dev_err(dev,
 355                                 "cluster %d int st status timeout!\n", i);
 356                         return -ETIMEDOUT;
 357                 }
 358         }
 359
 360         return 0;
 361 }
 362
 363 /*
 364  * For Kunpeng 920, we should disable FLR triggered by hardware (BME/PM/SRIOV).
 365  * Or it may stay in D3 state when we bind and unbind hpre quickly,
 366  * as it does FLR triggered by hardware.
 367  */
 368 static void disable_flr_of_bme(struct hisi_qm *qm)
 369 {
 370         u32 val;
 371
 372         val = readl(qm->io_base + QM_PEH_AXUSER_CFG);
 373         val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR);
 374         val |= HPRE_QM_PM_FLR;
 375         writel(val, qm->io_base + QM_PEH_AXUSER_CFG);
 376         writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);
 377 }
 378
 379 static void hpre_open_sva_prefetch(struct hisi_qm *qm)
 380 {
 381         u32 val;
 382         int ret;
 383
 384         if (qm->ver < QM_HW_V3)
 385                 return;
 386
 387         /* Enable prefetch */
 388         val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG);
 389         val &= HPRE_PREFETCH_ENABLE;
 390         writel(val, qm->io_base + HPRE_PREFETCH_CFG);
 391
 392         ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG,
 393                                          val, !(val & HPRE_PREFETCH_DISABLE),
 394                                          HPRE_REG_RD_INTVRL_US,
 395                                          HPRE_REG_RD_TMOUT_US);
 396         if (ret)
 397                 pci_err(qm->pdev, "failed to open sva prefetch\n");
 398 }
 399
 400 static void hpre_close_sva_prefetch(struct hisi_qm *qm)
 401 {
 402         u32 val;
 403         int ret;
 404
 405         if (qm->ver < QM_HW_V3)
 406                 return;
 407
 408         val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG);
 409         val |= HPRE_PREFETCH_DISABLE;
 410         writel(val, qm->io_base + HPRE_PREFETCH_CFG);
 411
 412         ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX,
 413                                          val, !(val & HPRE_SVA_DISABLE_READY),
 414                                          HPRE_REG_RD_INTVRL_US,
 415                                          HPRE_REG_RD_TMOUT_US);
 416         if (ret)
 417                 pci_err(qm->pdev, "failed to close sva prefetch\n");
 418 }
 419
 420 static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
 421 {
 422         struct device *dev = &qm->pdev->dev;
 423         u32 val;
 424         int ret;
 425
 426         writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
 427         writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE);
 428         writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG);
 429
 430         /* HPRE need more time, we close this interrupt */
 431         val = readl_relaxed(qm->io_base + HPRE_QM_ABNML_INT_MASK);
 432         val |= BIT(HPRE_TIMEOUT_ABNML_BIT);
 433         writel_relaxed(val, qm->io_base + HPRE_QM_ABNML_INT_MASK);
 434
 435         if (qm->ver >= QM_HW_V3)
 436                 writel(HPRE_RSA_ENB | HPRE_ECC_ENB,
 437                         qm->io_base + HPRE_TYPES_ENB);
 438         else
 439                 writel(HPRE_RSA_ENB, qm->io_base + HPRE_TYPES_ENB);
 440
 441         writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE);
 442         writel(0x0, qm->io_base + HPRE_BD_ENDIAN);
 443         writel(0x0, qm->io_base + HPRE_INT_MASK);
 444         writel(0x0, qm->io_base + HPRE_POISON_BYPASS);
 445         writel(0x0, qm->io_base + HPRE_COMM_CNT_CLR_CE);
 446         writel(0x0, qm->io_base + HPRE_ECC_BYPASS);
 447
 448         writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG);
 449         writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_AWUSR_CFG);
 450         writel(0x1, qm->io_base + HPRE_RDCHN_INI_CFG);
 451         ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_RDCHN_INI_ST, val,
 452                         val & BIT(0),
 453                         HPRE_REG_RD_INTVRL_US,
 454                         HPRE_REG_RD_TMOUT_US);
 455         if (ret) {
 456                 dev_err(dev, "read rd channel timeout fail!\n");
 457                 return -ETIMEDOUT;
 458         }
 459
 460         ret = hpre_set_cluster(qm);
 461         if (ret)
 462                 return -ETIMEDOUT;
 463
 464         /* This setting is only needed by Kunpeng 920. */
 465         if (qm->ver == QM_HW_V2) {
 466                 ret = hpre_cfg_by_dsm(qm);
 467                 if (ret)
 468                         return ret;
 469
 470                 disable_flr_of_bme(qm);
 471         }
 472
 473         /* Config data buffer pasid needed by Kunpeng 920 */
 474         hpre_config_pasid(qm);
 475
 476         return ret;
 477 }
 478
 479 static void hpre_cnt_regs_clear(struct hisi_qm *qm)
 480 {
 481         u8 clusters_num = hpre_cluster_num(qm);
 482         unsigned long offset;
 483         int i;
 484
 485         /* clear clusterX/cluster_ctrl */
 486         for (i = 0; i < clusters_num; i++) {
 487                 offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL;
 488                 writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY);
 489         }
 490
 491         /* clear rdclr_en */
 492         writel(0x0, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
 493
 494         hisi_qm_debug_regs_clear(qm);
 495 }
 496
 497 static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
 498 {
 499         u32 val1, val2;
 500
 501         val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 502         if (enable) {
 503                 val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
 504                 val2 = HPRE_HAC_RAS_NFE_ENABLE;
 505         } else {
 506                 val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
 507                 val2 = 0x0;
 508         }
 509
 510         if (qm->ver > QM_HW_V2)
 511                 writel(val2, qm->io_base + HPRE_OOO_SHUTDOWN_SEL);
 512
 513         writel(val1, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 514 }
 515
 516 static void hpre_hw_error_disable(struct hisi_qm *qm)
 517 {
 518         /* disable hpre hw error interrupts */
 519         writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK);
 520
 521         /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */
 522         hpre_master_ooo_ctrl(qm, false);
 523 }
 524
 525 static void hpre_hw_error_enable(struct hisi_qm *qm)
 526 {
 527         /* clear HPRE hw error source if having */
 528         writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT);
 529
 530         /* configure error type */
 531         writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB);
 532         writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB);
 533         writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB);
 534
 535         /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */
 536         hpre_master_ooo_ctrl(qm, true);
 537
 538         /* enable hpre hw error interrupts */
 539         writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK);
 540 }
 541
 542 static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
 543 {
 544         struct hpre *hpre = container_of(file->debug, struct hpre, debug);
 545
 546         return &hpre->qm;
 547 }
 548
 549 static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file)
 550 {
 551         struct hisi_qm *qm = hpre_file_to_qm(file);
 552
 553         return readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
 554                HPRE_CTRL_CNT_CLR_CE_BIT;
 555 }
 556
 557 static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val)
 558 {
 559         struct hisi_qm *qm = hpre_file_to_qm(file);
 560         u32 tmp;
 561
 562         if (val != 1 && val != 0)
 563                 return -EINVAL;
 564
 565         tmp = (readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
 566                ~HPRE_CTRL_CNT_CLR_CE_BIT) | val;
 567         writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
 568
 569         return 0;
 570 }
 571
 572 static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file)
 573 {
 574         struct hisi_qm *qm = hpre_file_to_qm(file);
 575         int cluster_index = file->index - HPRE_CLUSTER_CTRL;
 576         unsigned long offset = HPRE_CLSTR_BASE +
 577                                cluster_index * HPRE_CLSTR_ADDR_INTRVL;
 578
 579         return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT);
 580 }
 581
 582 static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val)
 583 {
 584         struct hisi_qm *qm = hpre_file_to_qm(file);
 585         int cluster_index = file->index - HPRE_CLUSTER_CTRL;
 586         unsigned long offset = HPRE_CLSTR_BASE + cluster_index *
 587                                HPRE_CLSTR_ADDR_INTRVL;
 588
 589         writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY);
 590
 591         return 0;
 592 }
 593
 594 static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf,
 595                                     size_t count, loff_t *pos)
 596 {
 597         struct hpre_debugfs_file *file = filp->private_data;
 598         char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
 599         u32 val;
 600         int ret;
 601
 602         spin_lock_irq(&file->lock);
 603         switch (file->type) {
 604         case HPRE_CLEAR_ENABLE:
 605                 val = hpre_clear_enable_read(file);
 606                 break;
 607         case HPRE_CLUSTER_CTRL:
 608                 val = hpre_cluster_inqry_read(file);
 609                 break;
 610         default:
 611                 spin_unlock_irq(&file->lock);
 612                 return -EINVAL;
 613         }
 614         spin_unlock_irq(&file->lock);
 615         ret = snprintf(tbuf, HPRE_DBGFS_VAL_MAX_LEN, "%u\n", val);
 616         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
 617 }
 618
 619 static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf,
 620                                      size_t count, loff_t *pos)
 621 {
 622         struct hpre_debugfs_file *file = filp->private_data;
 623         char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
 624         unsigned long val;
 625         int len, ret;
 626
 627         if (*pos != 0)
 628                 return 0;
 629
 630         if (count >= HPRE_DBGFS_VAL_MAX_LEN)
 631                 return -ENOSPC;
 632
 633         len = simple_write_to_buffer(tbuf, HPRE_DBGFS_VAL_MAX_LEN - 1,
 634                                      pos, buf, count);
 635         if (len < 0)
 636                 return len;
 637
 638         tbuf[len] = '\0';
 639         if (kstrtoul(tbuf, 0, &val))
 640                 return -EFAULT;
 641
 642         spin_lock_irq(&file->lock);
 643         switch (file->type) {
 644         case HPRE_CLEAR_ENABLE:
 645                 ret = hpre_clear_enable_write(file, val);
 646                 if (ret)
 647                         goto err_input;
 648                 break;
 649         case HPRE_CLUSTER_CTRL:
 650                 ret = hpre_cluster_inqry_write(file, val);
 651                 if (ret)
 652                         goto err_input;
 653                 break;
 654         default:
 655                 ret = -EINVAL;
 656                 goto err_input;
 657         }
 658         spin_unlock_irq(&file->lock);
 659
 660         return count;
 661
 662 err_input:
 663         spin_unlock_irq(&file->lock);
 664         return ret;
 665 }
 666
 667 static const struct file_operations hpre_ctrl_debug_fops = {
 668         .owner = THIS_MODULE,
 669         .open = simple_open,
 670         .read = hpre_ctrl_debug_read,
 671         .write = hpre_ctrl_debug_write,
 672 };
 673
 674 static int hpre_debugfs_atomic64_get(void *data, u64 *val)
 675 {
 676         struct hpre_dfx *dfx_item = data;
 677
 678         *val = atomic64_read(&dfx_item->value);
 679
 680         return 0;
 681 }
 682
 683 static int hpre_debugfs_atomic64_set(void *data, u64 val)
 684 {
 685         struct hpre_dfx *dfx_item = data;
 686         struct hpre_dfx *hpre_dfx = NULL;
 687
 688         if (dfx_item->type == HPRE_OVERTIME_THRHLD) {
 689                 hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD;
 690                 atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0);
 691         } else if (val) {
 692                 return -EINVAL;
 693         }
 694
 695         atomic64_set(&dfx_item->value, val);
 696
 697         return 0;
 698 }
 699
 700 DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get,
 701                          hpre_debugfs_atomic64_set, "%llu\n");
 702
 703 static int hpre_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
 704                                     enum hpre_ctrl_dbgfs_file type, int indx)
 705 {
 706         struct hpre *hpre = container_of(qm, struct hpre, qm);
 707         struct hpre_debug *dbg = &hpre->debug;
 708         struct dentry *file_dir;
 709
 710         if (dir)
 711                 file_dir = dir;
 712         else
 713                 file_dir = qm->debug.debug_root;
 714
 715         if (type >= HPRE_DEBUG_FILE_NUM)
 716                 return -EINVAL;
 717
 718         spin_lock_init(&dbg->files[indx].lock);
 719         dbg->files[indx].debug = dbg;
 720         dbg->files[indx].type = type;
 721         dbg->files[indx].index = indx;
 722         debugfs_create_file(hpre_debug_file_name[type], 0600, file_dir,
 723                             dbg->files + indx, &hpre_ctrl_debug_fops);
 724
 725         return 0;
 726 }
 727
 728 static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm)
 729 {
 730         struct device *dev = &qm->pdev->dev;
 731         struct debugfs_regset32 *regset;
 732
 733         regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 734         if (!regset)
 735                 return -ENOMEM;
 736
 737         regset->regs = hpre_com_dfx_regs;
 738         regset->nregs = ARRAY_SIZE(hpre_com_dfx_regs);
 739         regset->base = qm->io_base;
 740
 741         debugfs_create_regset32("regs", 0444,  qm->debug.debug_root, regset);
 742         return 0;
 743 }
 744
 745 static int hpre_cluster_debugfs_init(struct hisi_qm *qm)
 746 {
 747         u8 clusters_num = hpre_cluster_num(qm);
 748         struct device *dev = &qm->pdev->dev;
 749         char buf[HPRE_DBGFS_VAL_MAX_LEN];
 750         struct debugfs_regset32 *regset;
 751         struct dentry *tmp_d;
 752         int i, ret;
 753
 754         for (i = 0; i < clusters_num; i++) {
 755                 ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i);
 756                 if (ret < 0)
 757                         return -EINVAL;
 758                 tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
 759
 760                 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
 761                 if (!regset)
 762                         return -ENOMEM;
 763
 764                 regset->regs = hpre_cluster_dfx_regs;
 765                 regset->nregs = ARRAY_SIZE(hpre_cluster_dfx_regs);
 766                 regset->base = qm->io_base + hpre_cluster_offsets[i];
 767
 768                 debugfs_create_regset32("regs", 0444, tmp_d, regset);
 769                 ret = hpre_create_debugfs_file(qm, tmp_d, HPRE_CLUSTER_CTRL,
 770                                                i + HPRE_CLUSTER_CTRL);
 771                 if (ret)
 772                         return ret;
 773         }
 774
 775         return 0;
 776 }
 777
 778 static int hpre_ctrl_debug_init(struct hisi_qm *qm)
 779 {
 780         int ret;
 781
 782         ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE,
 783                                        HPRE_CLEAR_ENABLE);
 784         if (ret)
 785                 return ret;
 786
 787         ret = hpre_pf_comm_regs_debugfs_init(qm);
 788         if (ret)
 789                 return ret;
 790
 791         return hpre_cluster_debugfs_init(qm);
 792 }
 793
 794 static void hpre_dfx_debug_init(struct hisi_qm *qm)
 795 {
 796         struct hpre *hpre = container_of(qm, struct hpre, qm);
 797         struct hpre_dfx *dfx = hpre->debug.dfx;
 798         struct dentry *parent;
 799         int i;
 800
 801         parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root);
 802         for (i = 0; i < HPRE_DFX_FILE_NUM; i++) {
 803                 dfx[i].type = i;
 804                 debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i],
 805                                     &hpre_atomic64_ops);
 806         }
 807 }
 808
 809 static int hpre_debugfs_init(struct hisi_qm *qm)
 810 {
 811         struct device *dev = &qm->pdev->dev;
 812         int ret;
 813
 814         qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
 815                                                   hpre_debugfs_root);
 816
 817         qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET;
 818         qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN;
 819         hisi_qm_debug_init(qm);
 820
 821         if (qm->pdev->device == HPRE_PCI_DEVICE_ID) {
 822                 ret = hpre_ctrl_debug_init(qm);
 823                 if (ret)
 824                         goto failed_to_create;
 825         }
 826
 827         hpre_dfx_debug_init(qm);
 828
 829         return 0;
 830
 831 failed_to_create:
 832         debugfs_remove_recursive(qm->debug.debug_root);
 833         return ret;
 834 }
 835
 836 static void hpre_debugfs_exit(struct hisi_qm *qm)
 837 {
 838         debugfs_remove_recursive(qm->debug.debug_root);
 839 }
 840
 841 static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
 842 {
 843         if (pdev->revision == QM_HW_V1) {
 844                 pci_warn(pdev, "HPRE version 1 is not supported!\n");
 845                 return -EINVAL;
 846         }
 847
 848         if (pdev->revision >= QM_HW_V3)
 849                 qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2";
 850         else
 851                 qm->algs = "rsa\ndh";
 852         qm->mode = uacce_mode;
 853         qm->pdev = pdev;
 854         qm->ver = pdev->revision;
 855         qm->sqe_size = HPRE_SQE_SIZE;
 856         qm->dev_name = hpre_name;
 857
 858         qm->fun_type = (pdev->device == HPRE_PCI_DEVICE_ID) ?
 859                         QM_HW_PF : QM_HW_VF;
 860         if (qm->fun_type == QM_HW_PF) {
 861                 qm->qp_base = HPRE_PF_DEF_Q_BASE;
 862                 qm->qp_num = pf_q_num;
 863                 qm->debug.curr_qm_qp_num = pf_q_num;
 864                 qm->qm_list = &hpre_devices;
 865         }
 866
 867         return hisi_qm_init(qm);
 868 }
 869
 870 static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
 871 {
 872         const struct hpre_hw_error *err = hpre_hw_errors;
 873         struct device *dev = &qm->pdev->dev;
 874
 875         while (err->msg) {
 876                 if (err->int_msk & err_sts)
 877                         dev_warn(dev, "%s [error status=0x%x] found\n",
 878                                  err->msg, err->int_msk);
 879                 err++;
 880         }
 881 }
 882
 883 static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
 884 {
 885         return readl(qm->io_base + HPRE_INT_STATUS);
 886 }
 887
 888 static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
 889 {
 890         writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT);
 891 }
 892
 893 static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
 894 {
 895         u32 value;
 896
 897         value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 898         writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE,
 899                qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 900         writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE,
 901                qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
 902 }
 903
 904 static void hpre_err_info_init(struct hisi_qm *qm)
 905 {
 906         struct hisi_qm_err_info *err_info = &qm->err_info;
 907
 908         err_info->ce = QM_BASE_CE;
 909         err_info->fe = 0;
 910         err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR |
 911                                    HPRE_OOO_ECC_2BIT_ERR;
 912         err_info->dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE;
 913         err_info->msi_wr_port = HPRE_WR_MSI_PORT;
 914         err_info->acpi_rst = "HRST";
 915         err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT;
 916 }
 917
 918 static const struct hisi_qm_err_ini hpre_err_ini = {
 919         .hw_init                = hpre_set_user_domain_and_cache,
 920         .hw_err_enable          = hpre_hw_error_enable,
 921         .hw_err_disable         = hpre_hw_error_disable,
 922         .get_dev_hw_err_status  = hpre_get_hw_err_status,
 923         .clear_dev_hw_err_status = hpre_clear_hw_err_status,
 924         .log_dev_hw_err         = hpre_log_hw_error,
 925         .open_axi_master_ooo    = hpre_open_axi_master_ooo,
 926         .open_sva_prefetch      = hpre_open_sva_prefetch,
 927         .close_sva_prefetch     = hpre_close_sva_prefetch,
 928         .err_info_init          = hpre_err_info_init,
 929 };
 930
 931 static int hpre_pf_probe_init(struct hpre *hpre)
 932 {
 933         struct hisi_qm *qm = &hpre->qm;
 934         int ret;
 935
 936         ret = hpre_set_user_domain_and_cache(qm);
 937         if (ret)
 938                 return ret;
 939
 940         hpre_open_sva_prefetch(qm);
 941
 942         qm->err_ini = &hpre_err_ini;
 943         qm->err_ini->err_info_init(qm);
 944         hisi_qm_dev_err_init(qm);
 945
 946         return 0;
 947 }
 948
 949 static int hpre_probe_init(struct hpre *hpre)
 950 {
 951         u32 type_rate = HPRE_SHAPER_TYPE_RATE;
 952         struct hisi_qm *qm = &hpre->qm;
 953         int ret;
 954
 955         if (qm->fun_type == QM_HW_PF) {
 956                 ret = hpre_pf_probe_init(hpre);
 957                 if (ret)
 958                         return ret;
 959                 /* Enable shaper type 0 */
 960                 if (qm->ver >= QM_HW_V3) {
 961                         type_rate |= QM_SHAPER_ENABLE;
 962                         qm->type_rate = type_rate;
 963                 }
 964         }
 965
 966         return 0;
 967 }
 968
 969 static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 970 {
 971         struct hisi_qm *qm;
 972         struct hpre *hpre;
 973         int ret;
 974
 975         hpre = devm_kzalloc(&pdev->dev, sizeof(*hpre), GFP_KERNEL);
 976         if (!hpre)
 977                 return -ENOMEM;
 978
 979         qm = &hpre->qm;
 980         ret = hpre_qm_init(qm, pdev);
 981         if (ret) {
 982                 pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret);
 983                 return ret;
 984         }
 985
 986         ret = hpre_probe_init(hpre);
 987         if (ret) {
 988                 pci_err(pdev, "Failed to probe (%d)!\n", ret);
 989                 goto err_with_qm_init;
 990         }
 991
 992         ret = hisi_qm_start(qm);
 993         if (ret)
 994                 goto err_with_err_init;
 995
 996         ret = hpre_debugfs_init(qm);
 997         if (ret)
 998                 dev_warn(&pdev->dev, "init debugfs fail!\n");
 999
1000         ret = hisi_qm_alg_register(qm, &hpre_devices);
1001         if (ret < 0) {
1002                 pci_err(pdev, "fail to register algs to crypto!\n");
1003                 goto err_with_qm_start;
1004         }
1005
1006         if (qm->uacce) {
1007                 ret = uacce_register(qm->uacce);
1008                 if (ret) {
1009                         pci_err(pdev, "failed to register uacce (%d)!\n", ret);
1010                         goto err_with_alg_register;
1011                 }
1012         }
1013
1014         if (qm->fun_type == QM_HW_PF && vfs_num) {
1015                 ret = hisi_qm_sriov_enable(pdev, vfs_num);
1016                 if (ret < 0)
1017                         goto err_with_alg_register;
1018         }
1019
1020         return 0;
1021
1022 err_with_alg_register:
1023         hisi_qm_alg_unregister(qm, &hpre_devices);
1024
1025 err_with_qm_start:
1026         hpre_debugfs_exit(qm);
1027         hisi_qm_stop(qm, QM_NORMAL);
1028
1029 err_with_err_init:
1030         hisi_qm_dev_err_uninit(qm);
1031
1032 err_with_qm_init:
1033         hisi_qm_uninit(qm);
1034
1035         return ret;
1036 }
1037
1038 static void hpre_remove(struct pci_dev *pdev)
1039 {
1040         struct hisi_qm *qm = pci_get_drvdata(pdev);
1041         int ret;
1042
1043         hisi_qm_wait_task_finish(qm, &hpre_devices);
1044         hisi_qm_alg_unregister(qm, &hpre_devices);
1045         if (qm->fun_type == QM_HW_PF && qm->vfs_num) {
1046                 ret = hisi_qm_sriov_disable(pdev, true);
1047                 if (ret) {
1048                         pci_err(pdev, "Disable SRIOV fail!\n");
1049                         return;
1050                 }
1051         }
1052
1053         hpre_debugfs_exit(qm);
1054         hisi_qm_stop(qm, QM_NORMAL);
1055
1056         if (qm->fun_type == QM_HW_PF) {
1057                 hpre_cnt_regs_clear(qm);
1058                 qm->debug.curr_qm_qp_num = 0;
1059                 hisi_qm_dev_err_uninit(qm);
1060         }
1061
1062         hisi_qm_uninit(qm);
1063 }
1064
1065 static const struct pci_error_handlers hpre_err_handler = {
1066         .error_detected         = hisi_qm_dev_err_detected,
1067         .slot_reset             = hisi_qm_dev_slot_reset,
1068         .reset_prepare          = hisi_qm_reset_prepare,
1069         .reset_done             = hisi_qm_reset_done,
1070 };
1071
1072 static struct pci_driver hpre_pci_driver = {
1073         .name                   = hpre_name,
1074         .id_table               = hpre_dev_ids,
1075         .probe                  = hpre_probe,
1076         .remove                 = hpre_remove,
1077         .sriov_configure        = IS_ENABLED(CONFIG_PCI_IOV) ?
1078                                   hisi_qm_sriov_configure : NULL,
1079         .err_handler            = &hpre_err_handler,
1080         .shutdown               = hisi_qm_dev_shutdown,
1081 };
1082
1083 static void hpre_register_debugfs(void)
1084 {
1085         if (!debugfs_initialized())
1086                 return;
1087
1088         hpre_debugfs_root = debugfs_create_dir(hpre_name, NULL);
1089 }
1090
1091 static void hpre_unregister_debugfs(void)
1092 {
1093         debugfs_remove_recursive(hpre_debugfs_root);
1094 }
1095
1096 static int __init hpre_init(void)
1097 {
1098         int ret;
1099
1100         hisi_qm_init_list(&hpre_devices);
1101         hpre_register_debugfs();
1102
1103         ret = pci_register_driver(&hpre_pci_driver);
1104         if (ret) {
1105                 hpre_unregister_debugfs();
1106                 pr_err("hpre: can't register hisi hpre driver.\n");
1107         }
1108
1109         return ret;
1110 }
1111
1112 static void __exit hpre_exit(void)
1113 {
1114         pci_unregister_driver(&hpre_pci_driver);
1115         hpre_unregister_debugfs();
1116 }
1117
1118 module_init(hpre_init);
1119 module_exit(hpre_exit);
1120
1121 MODULE_LICENSE("GPL v2");
1122 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
1123 MODULE_AUTHOR("Meng Yu <yumeng18@huawei.com>");
1124 MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator");