drivers/crypto/qat/qat_common/qat_hal.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/slab.h>
   4 #include <linux/delay.h>
   5 #include <linux/pci_ids.h>
   6
   7 #include "adf_accel_devices.h"
   8 #include "adf_common_drv.h"
   9 #include "icp_qat_hal.h"
  10 #include "icp_qat_uclo.h"
  11
  12 #define BAD_REGADDR            0xffff
  13 #define MAX_RETRY_TIMES    10000
  14 #define INIT_CTX_ARB_VALUE      0x0
  15 #define INIT_CTX_ENABLE_VALUE     0x0
  16 #define INIT_PC_VALUE        0x0
  17 #define INIT_WAKEUP_EVENTS_VALUE  0x1
  18 #define INIT_SIG_EVENTS_VALUE     0x1
  19 #define INIT_CCENABLE_VALUE       0x2000
  20 #define RST_CSR_QAT_LSB    20
  21 #define RST_CSR_AE_LSB            0
  22 #define MC_TIMESTAMP_ENABLE       (0x1 << 7)
  23
  24 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
  25         (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
  26         (~(1 << CE_REG_PAR_ERR_BITPOS)))
  27 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
  28         (inst = ((inst & 0xFFFF00C03FFull) | \
  29                 ((((const_val) << 12) & 0x0FF00000ull) | \
  30                 (((const_val) << 10) & 0x0003FC00ull))))
  31 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
  32         (inst = ((inst & 0xFFFF00FFF00ull) | \
  33                 ((((const_val) << 12) & 0x0FF00000ull) | \
  34                 (((const_val) <<  0) & 0x000000FFull))))
  35
  36 #define AE(handle, ae) ((handle)->hal_handle->aes[ae])
  37
  38 static const u64 inst_4b[] = {
  39         0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
  40         0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  41         0x0A021000000ull
  42 };
  43
  44 static const u64 inst[] = {
  45         0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
  46         0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  47         0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
  48         0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  49         0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
  50         0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
  51         0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
  52         0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
  53         0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
  54         0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
  55         0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
  56         0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
  57         0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
  58         0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
  59         0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
  60         0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
  61         0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
  62         0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
  63         0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
  64         0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
  65         0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
  66         0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
  67 };
  68
  69 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
  70                           unsigned char ae, unsigned int ctx_mask)
  71 {
  72         AE(handle, ae).live_ctx_mask = ctx_mask;
  73 }
  74
  75 #define CSR_RETRY_TIMES 500
  76 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
  77                              unsigned char ae, unsigned int csr)
  78 {
  79         unsigned int iterations = CSR_RETRY_TIMES;
  80         int value;
  81
  82         do {
  83                 value = GET_AE_CSR(handle, ae, csr);
  84                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
  85                         return value;
  86         } while (iterations--);
  87
  88         pr_err("QAT: Read CSR timeout\n");
  89         return 0;
  90 }
  91
  92 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
  93                              unsigned char ae, unsigned int csr,
  94                              unsigned int value)
  95 {
  96         unsigned int iterations = CSR_RETRY_TIMES;
  97
  98         do {
  99                 SET_AE_CSR(handle, ae, csr, value);
 100                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 101                         return 0;
 102         } while (iterations--);
 103
 104         pr_err("QAT: Write CSR Timeout\n");
 105         return -EFAULT;
 106 }
 107
 108 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 109                                      unsigned char ae, unsigned char ctx,
 110                                      unsigned int *events)
 111 {
 112         unsigned int cur_ctx;
 113
 114         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 115         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 116         *events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
 117         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 118 }
 119
 120 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 121                                unsigned char ae, unsigned int cycles,
 122                                int chk_inactive)
 123 {
 124         unsigned int base_cnt = 0, cur_cnt = 0;
 125         unsigned int csr = (1 << ACS_ABO_BITPOS);
 126         int times = MAX_RETRY_TIMES;
 127         int elapsed_cycles = 0;
 128
 129         base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 130         base_cnt &= 0xffff;
 131         while ((int)cycles > elapsed_cycles && times--) {
 132                 if (chk_inactive)
 133                         csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 134
 135                 cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 136                 cur_cnt &= 0xffff;
 137                 elapsed_cycles = cur_cnt - base_cnt;
 138
 139                 if (elapsed_cycles < 0)
 140                         elapsed_cycles += 0x10000;
 141
 142                 /* ensure at least 8 time cycles elapsed in wait_cycles */
 143                 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
 144                         return 0;
 145         }
 146         if (times < 0) {
 147                 pr_err("QAT: wait_num_cycles time out\n");
 148                 return -EFAULT;
 149         }
 150         return 0;
 151 }
 152
 153 #define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit)))
 154 #define SET_BIT(wrd, bit) ((wrd) | 1 << (bit))
 155
 156 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 157                             unsigned char ae, unsigned char mode)
 158 {
 159         unsigned int csr, new_csr;
 160
 161         if (mode != 4 && mode != 8) {
 162                 pr_err("QAT: bad ctx mode=%d\n", mode);
 163                 return -EINVAL;
 164         }
 165
 166         /* Sets the accelaration engine context mode to either four or eight */
 167         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 168         csr = IGNORE_W1C_MASK & csr;
 169         new_csr = (mode == 4) ?
 170                 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
 171                 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
 172         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 173         return 0;
 174 }
 175
 176 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 177                            unsigned char ae, unsigned char mode)
 178 {
 179         unsigned int csr, new_csr;
 180
 181         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 182         csr &= IGNORE_W1C_MASK;
 183
 184         new_csr = (mode) ?
 185                 SET_BIT(csr, CE_NN_MODE_BITPOS) :
 186                 CLR_BIT(csr, CE_NN_MODE_BITPOS);
 187
 188         if (new_csr != csr)
 189                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 190
 191         return 0;
 192 }
 193
 194 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 195                            unsigned char ae, enum icp_qat_uof_regtype lm_type,
 196                            unsigned char mode)
 197 {
 198         unsigned int csr, new_csr;
 199
 200         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 201         csr &= IGNORE_W1C_MASK;
 202         switch (lm_type) {
 203         case ICP_LMEM0:
 204                 new_csr = (mode) ?
 205                         SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
 206                         CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
 207                 break;
 208         case ICP_LMEM1:
 209                 new_csr = (mode) ?
 210                         SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
 211                         CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
 212                 break;
 213         case ICP_LMEM2:
 214                 new_csr = (mode) ?
 215                         SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) :
 216                         CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS);
 217                 break;
 218         case ICP_LMEM3:
 219                 new_csr = (mode) ?
 220                         SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) :
 221                         CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS);
 222                 break;
 223         default:
 224                 pr_err("QAT: lmType = 0x%x\n", lm_type);
 225                 return -EINVAL;
 226         }
 227
 228         if (new_csr != csr)
 229                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 230         return 0;
 231 }
 232
 233 void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle,
 234                                 unsigned char ae, unsigned char mode)
 235 {
 236         unsigned int csr, new_csr;
 237
 238         csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 239         csr &= IGNORE_W1C_MASK;
 240         new_csr = (mode) ?
 241                   SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) :
 242                   CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS);
 243         if (new_csr != csr)
 244                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 245 }
 246
 247 static unsigned short qat_hal_get_reg_addr(unsigned int type,
 248                                            unsigned short reg_num)
 249 {
 250         unsigned short reg_addr;
 251
 252         switch (type) {
 253         case ICP_GPA_ABS:
 254         case ICP_GPB_ABS:
 255                 reg_addr = 0x80 | (reg_num & 0x7f);
 256                 break;
 257         case ICP_GPA_REL:
 258         case ICP_GPB_REL:
 259                 reg_addr = reg_num & 0x1f;
 260                 break;
 261         case ICP_SR_RD_REL:
 262         case ICP_SR_WR_REL:
 263         case ICP_SR_REL:
 264                 reg_addr = 0x180 | (reg_num & 0x1f);
 265                 break;
 266         case ICP_SR_ABS:
 267                 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
 268                 break;
 269         case ICP_DR_RD_REL:
 270         case ICP_DR_WR_REL:
 271         case ICP_DR_REL:
 272                 reg_addr = 0x1c0 | (reg_num & 0x1f);
 273                 break;
 274         case ICP_DR_ABS:
 275                 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
 276                 break;
 277         case ICP_NEIGH_REL:
 278                 reg_addr = 0x280 | (reg_num & 0x1f);
 279                 break;
 280         case ICP_LMEM0:
 281                 reg_addr = 0x200;
 282                 break;
 283         case ICP_LMEM1:
 284                 reg_addr = 0x220;
 285                 break;
 286         case ICP_LMEM2:
 287                 reg_addr = 0x2c0;
 288                 break;
 289         case ICP_LMEM3:
 290                 reg_addr = 0x2e0;
 291                 break;
 292         case ICP_NO_DEST:
 293                 reg_addr = 0x300 | (reg_num & 0xff);
 294                 break;
 295         default:
 296                 reg_addr = BAD_REGADDR;
 297                 break;
 298         }
 299         return reg_addr;
 300 }
 301
 302 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
 303 {
 304         unsigned int reset_mask = handle->chip_info->icp_rst_mask;
 305         unsigned int reset_csr = handle->chip_info->icp_rst_csr;
 306         unsigned int csr_val;
 307
 308         csr_val = GET_CAP_CSR(handle, reset_csr);
 309         csr_val |= reset_mask;
 310         SET_CAP_CSR(handle, reset_csr, csr_val);
 311 }
 312
 313 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 314                                 unsigned char ae, unsigned int ctx_mask,
 315                                 unsigned int ae_csr, unsigned int csr_val)
 316 {
 317         unsigned int ctx, cur_ctx;
 318
 319         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 320
 321         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 322                 if (!(ctx_mask & (1 << ctx)))
 323                         continue;
 324                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 325                 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
 326         }
 327
 328         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 329 }
 330
 331 static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 332                                 unsigned char ae, unsigned char ctx,
 333                                 unsigned int ae_csr)
 334 {
 335         unsigned int cur_ctx, csr_val;
 336
 337         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 338         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 339         csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
 340         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 341
 342         return csr_val;
 343 }
 344
 345 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 346                                   unsigned char ae, unsigned int ctx_mask,
 347                                   unsigned int events)
 348 {
 349         unsigned int ctx, cur_ctx;
 350
 351         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 352         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 353                 if (!(ctx_mask & (1 << ctx)))
 354                         continue;
 355                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 356                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
 357         }
 358         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 359 }
 360
 361 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 362                                      unsigned char ae, unsigned int ctx_mask,
 363                                      unsigned int events)
 364 {
 365         unsigned int ctx, cur_ctx;
 366
 367         cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 368         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 369                 if (!(ctx_mask & (1 << ctx)))
 370                         continue;
 371                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 372                 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
 373                                   events);
 374         }
 375         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 376 }
 377
 378 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 379 {
 380         unsigned long ae_mask = handle->hal_handle->ae_mask;
 381         unsigned int base_cnt, cur_cnt;
 382         unsigned char ae;
 383         int times = MAX_RETRY_TIMES;
 384
 385         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 386                 base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 387                 base_cnt &= 0xffff;
 388
 389                 do {
 390                         cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 391                         cur_cnt &= 0xffff;
 392                 } while (times-- && (cur_cnt == base_cnt));
 393
 394                 if (times < 0) {
 395                         pr_err("QAT: AE%d is inactive!!\n", ae);
 396                         return -EFAULT;
 397                 }
 398         }
 399
 400         return 0;
 401 }
 402
 403 int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
 404                             unsigned int ae)
 405 {
 406         unsigned int enable = 0, active = 0;
 407
 408         enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 409         active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 410         if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
 411             (active & (1 << ACS_ABO_BITPOS)))
 412                 return 1;
 413         else
 414                 return 0;
 415 }
 416
 417 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
 418 {
 419         unsigned long ae_mask = handle->hal_handle->ae_mask;
 420         unsigned int misc_ctl_csr, misc_ctl;
 421         unsigned char ae;
 422
 423         misc_ctl_csr = handle->chip_info->misc_ctl_csr;
 424         /* stop the timestamp timers */
 425         misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr);
 426         if (misc_ctl & MC_TIMESTAMP_ENABLE)
 427                 SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl &
 428                             (~MC_TIMESTAMP_ENABLE));
 429
 430         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 431                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
 432                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
 433         }
 434         /* start timestamp timers */
 435         SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE);
 436 }
 437
 438 #define ESRAM_AUTO_TINIT        BIT(2)
 439 #define ESRAM_AUTO_TINIT_DONE   BIT(3)
 440 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
 441 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
 442 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 443 {
 444         void __iomem *csr_addr =
 445                         (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v +
 446                         ESRAM_AUTO_INIT_CSR_OFFSET);
 447         unsigned int csr_val;
 448         int times = 30;
 449
 450         if (handle->pci_dev->device != PCI_DEVICE_ID_INTEL_QAT_DH895XCC)
 451                 return 0;
 452
 453         csr_val = ADF_CSR_RD(csr_addr, 0);
 454         if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
 455                 return 0;
 456
 457         csr_val = ADF_CSR_RD(csr_addr, 0);
 458         csr_val |= ESRAM_AUTO_TINIT;
 459         ADF_CSR_WR(csr_addr, 0, csr_val);
 460
 461         do {
 462                 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 463                 csr_val = ADF_CSR_RD(csr_addr, 0);
 464         } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
 465         if (times < 0) {
 466                 pr_err("QAT: Fail to init eSram!\n");
 467                 return -EFAULT;
 468         }
 469         return 0;
 470 }
 471
 472 #define SHRAM_INIT_CYCLES 2060
 473 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
 474 {
 475         unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr;
 476         unsigned int reset_mask = handle->chip_info->icp_rst_mask;
 477         unsigned int reset_csr = handle->chip_info->icp_rst_csr;
 478         unsigned long ae_mask = handle->hal_handle->ae_mask;
 479         unsigned char ae = 0;
 480         unsigned int times = 100;
 481         unsigned int csr_val;
 482
 483         /* write to the reset csr */
 484         csr_val = GET_CAP_CSR(handle, reset_csr);
 485         csr_val &= ~reset_mask;
 486         do {
 487                 SET_CAP_CSR(handle, reset_csr, csr_val);
 488                 if (!(times--))
 489                         goto out_err;
 490                 csr_val = GET_CAP_CSR(handle, reset_csr);
 491                 csr_val &= reset_mask;
 492         } while (csr_val);
 493         /* enable clock */
 494         csr_val = GET_CAP_CSR(handle, clk_csr);
 495         csr_val |= reset_mask;
 496         SET_CAP_CSR(handle, clk_csr, csr_val);
 497         if (qat_hal_check_ae_alive(handle))
 498                 goto out_err;
 499
 500         /* Set undefined power-up/reset states to reasonable default values */
 501         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 502                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 503                                   INIT_CTX_ENABLE_VALUE);
 504                 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
 505                                     CTX_STS_INDIRECT,
 506                                     handle->hal_handle->upc_mask &
 507                                     INIT_PC_VALUE);
 508                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 509                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 510                 qat_hal_put_wakeup_event(handle, ae,
 511                                          ICP_QAT_UCLO_AE_ALL_CTX,
 512                                          INIT_WAKEUP_EVENTS_VALUE);
 513                 qat_hal_put_sig_event(handle, ae,
 514                                       ICP_QAT_UCLO_AE_ALL_CTX,
 515                                       INIT_SIG_EVENTS_VALUE);
 516         }
 517         if (qat_hal_init_esram(handle))
 518                 goto out_err;
 519         if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
 520                 goto out_err;
 521         qat_hal_reset_timestamp(handle);
 522
 523         return 0;
 524 out_err:
 525         pr_err("QAT: failed to get device out of reset\n");
 526         return -EFAULT;
 527 }
 528
 529 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 530                                 unsigned char ae, unsigned int ctx_mask)
 531 {
 532         unsigned int ctx;
 533
 534         ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 535         ctx &= IGNORE_W1C_MASK &
 536                 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 537         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 538 }
 539
 540 static u64 qat_hal_parity_64bit(u64 word)
 541 {
 542         word ^= word >> 1;
 543         word ^= word >> 2;
 544         word ^= word >> 4;
 545         word ^= word >> 8;
 546         word ^= word >> 16;
 547         word ^= word >> 32;
 548         return word & 1;
 549 }
 550
 551 static u64 qat_hal_set_uword_ecc(u64 uword)
 552 {
 553         u64 bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
 554                 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
 555                 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
 556                 bit6_mask = 0xdaf69a46910ULL;
 557
 558         /* clear the ecc bits */
 559         uword &= ~(0x7fULL << 0x2C);
 560         uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
 561         uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
 562         uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
 563         uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
 564         uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
 565         uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
 566         uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
 567         return uword;
 568 }
 569
 570 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 571                        unsigned char ae, unsigned int uaddr,
 572                        unsigned int words_num, u64 *uword)
 573 {
 574         unsigned int ustore_addr;
 575         unsigned int i;
 576
 577         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 578         uaddr |= UA_ECS;
 579         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 580         for (i = 0; i < words_num; i++) {
 581                 unsigned int uwrd_lo, uwrd_hi;
 582                 u64 tmp;
 583
 584                 tmp = qat_hal_set_uword_ecc(uword[i]);
 585                 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
 586                 uwrd_hi = (unsigned int)(tmp >> 0x20);
 587                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 588                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 589         }
 590         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 591 }
 592
 593 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 594                                unsigned char ae, unsigned int ctx_mask)
 595 {
 596         unsigned int ctx;
 597
 598         ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 599         ctx &= IGNORE_W1C_MASK;
 600         ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 601         ctx |= (ctx_mask << CE_ENABLE_BITPOS);
 602         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 603 }
 604
 605 static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle)
 606 {
 607         unsigned long ae_mask = handle->hal_handle->ae_mask;
 608         unsigned char ae;
 609         unsigned short reg;
 610
 611         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 612                 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
 613                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
 614                                              reg, 0);
 615                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
 616                                              reg, 0);
 617                 }
 618         }
 619 }
 620
 621 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 622 {
 623         unsigned long ae_mask = handle->hal_handle->ae_mask;
 624         unsigned char ae;
 625         unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
 626         int times = MAX_RETRY_TIMES;
 627         unsigned int csr_val = 0;
 628         unsigned int savctx = 0;
 629         int ret = 0;
 630
 631         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 632                 csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 633                 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 634                 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
 635                 csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 636                 csr_val &= IGNORE_W1C_MASK;
 637                 if (handle->chip_info->nn)
 638                         csr_val |= CE_NN_MODE;
 639
 640                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
 641                 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
 642                                   (u64 *)inst);
 643                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 644                                     handle->hal_handle->upc_mask &
 645                                     INIT_PC_VALUE);
 646                 savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 647                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 648                 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 649                 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
 650                                     CTX_SIG_EVENTS_INDIRECT, 0);
 651                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 652                 qat_hal_enable_ctx(handle, ae, ctx_mask);
 653         }
 654         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 655                 /* wait for AE to finish */
 656                 do {
 657                         ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 658                 } while (ret && times--);
 659
 660                 if (times < 0) {
 661                         pr_err("QAT: clear GPR of AE %d failed", ae);
 662                         return -EINVAL;
 663                 }
 664                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 665                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 666                                   savctx & ACS_ACNO);
 667                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 668                                   INIT_CTX_ENABLE_VALUE);
 669                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 670                                     handle->hal_handle->upc_mask &
 671                                     INIT_PC_VALUE);
 672                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 673                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 674                 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
 675                                          INIT_WAKEUP_EVENTS_VALUE);
 676                 qat_hal_put_sig_event(handle, ae, ctx_mask,
 677                                       INIT_SIG_EVENTS_VALUE);
 678         }
 679         return 0;
 680 }
 681
 682 static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
 683                              struct adf_accel_dev *accel_dev)
 684 {
 685         struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
 686         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 687         void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
 688         unsigned int max_en_ae_id = 0;
 689         struct adf_bar *sram_bar;
 690         unsigned int csr_val = 0;
 691         unsigned long ae_mask;
 692         unsigned char ae = 0;
 693         int ret = 0;
 694
 695         handle->pci_dev = pci_info->pci_dev;
 696         switch (handle->pci_dev->device) {
 697         case ADF_4XXX_PCI_DEVICE_ID:
 698         case ADF_401XX_PCI_DEVICE_ID:
 699         case ADF_402XX_PCI_DEVICE_ID:
 700                 handle->chip_info->mmp_sram_size = 0;
 701                 handle->chip_info->nn = false;
 702                 handle->chip_info->lm2lm3 = true;
 703                 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X;
 704                 handle->chip_info->icp_rst_csr = ICP_RESET_CPP0;
 705                 handle->chip_info->icp_rst_mask = 0x100015;
 706                 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0;
 707                 handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX;
 708                 handle->chip_info->wakeup_event_val = 0x80000000;
 709                 handle->chip_info->fw_auth = true;
 710                 handle->chip_info->css_3k = true;
 711                 handle->chip_info->tgroup_share_ustore = true;
 712                 handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX;
 713                 handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX;
 714                 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX;
 715                 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX;
 716                 handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX;
 717                 handle->chip_info->fcu_loaded_ae_pos = 0;
 718
 719                 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET_4XXX;
 720                 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET_4XXX;
 721                 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET_4XXX;
 722                 handle->hal_cap_ae_local_csr_addr_v =
 723                         (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 724                         + LOCAL_TO_XFER_REG_OFFSET);
 725                 break;
 726         case PCI_DEVICE_ID_INTEL_QAT_C62X:
 727         case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
 728                 handle->chip_info->mmp_sram_size = 0;
 729                 handle->chip_info->nn = true;
 730                 handle->chip_info->lm2lm3 = false;
 731                 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
 732                 handle->chip_info->icp_rst_csr = ICP_RESET;
 733                 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
 734                                                   (hw_data->accel_mask << RST_CSR_QAT_LSB);
 735                 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
 736                 handle->chip_info->misc_ctl_csr = MISC_CONTROL;
 737                 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
 738                 handle->chip_info->fw_auth = true;
 739                 handle->chip_info->css_3k = false;
 740                 handle->chip_info->tgroup_share_ustore = false;
 741                 handle->chip_info->fcu_ctl_csr = FCU_CONTROL;
 742                 handle->chip_info->fcu_sts_csr = FCU_STATUS;
 743                 handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI;
 744                 handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO;
 745                 handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS;
 746                 handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
 747                 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
 748                 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
 749                 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
 750                 handle->hal_cap_ae_local_csr_addr_v =
 751                         (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 752                         + LOCAL_TO_XFER_REG_OFFSET);
 753                 break;
 754         case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
 755                 handle->chip_info->mmp_sram_size = 0x40000;
 756                 handle->chip_info->nn = true;
 757                 handle->chip_info->lm2lm3 = false;
 758                 handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG;
 759                 handle->chip_info->icp_rst_csr = ICP_RESET;
 760                 handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) |
 761                                                   (hw_data->accel_mask << RST_CSR_QAT_LSB);
 762                 handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE;
 763                 handle->chip_info->misc_ctl_csr = MISC_CONTROL;
 764                 handle->chip_info->wakeup_event_val = WAKEUP_EVENT;
 765                 handle->chip_info->fw_auth = false;
 766                 handle->chip_info->css_3k = false;
 767                 handle->chip_info->tgroup_share_ustore = false;
 768                 handle->chip_info->fcu_ctl_csr = 0;
 769                 handle->chip_info->fcu_sts_csr = 0;
 770                 handle->chip_info->fcu_dram_addr_hi = 0;
 771                 handle->chip_info->fcu_dram_addr_lo = 0;
 772                 handle->chip_info->fcu_loaded_ae_csr = 0;
 773                 handle->chip_info->fcu_loaded_ae_pos = 0;
 774                 handle->hal_cap_g_ctl_csr_addr_v = pmisc_addr + ICP_QAT_CAP_OFFSET;
 775                 handle->hal_cap_ae_xfer_csr_addr_v = pmisc_addr + ICP_QAT_AE_OFFSET;
 776                 handle->hal_ep_csr_addr_v = pmisc_addr + ICP_QAT_EP_OFFSET;
 777                 handle->hal_cap_ae_local_csr_addr_v =
 778                         (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v
 779                         + LOCAL_TO_XFER_REG_OFFSET);
 780                 break;
 781         default:
 782                 ret = -EINVAL;
 783                 goto out_err;
 784         }
 785
 786         if (handle->chip_info->mmp_sram_size > 0) {
 787                 sram_bar =
 788                         &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)];
 789                 handle->hal_sram_addr_v = sram_bar->virt_addr;
 790         }
 791         handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 792         handle->hal_handle->ae_mask = hw_data->ae_mask;
 793         handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
 794         handle->hal_handle->slice_mask = hw_data->accel_mask;
 795         handle->cfg_ae_mask = ALL_AE_MASK;
 796         /* create AE objects */
 797         handle->hal_handle->upc_mask = 0x1ffff;
 798         handle->hal_handle->max_ustore = 0x4000;
 799
 800         ae_mask = handle->hal_handle->ae_mask;
 801         for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) {
 802                 handle->hal_handle->aes[ae].free_addr = 0;
 803                 handle->hal_handle->aes[ae].free_size =
 804                     handle->hal_handle->max_ustore;
 805                 handle->hal_handle->aes[ae].ustore_size =
 806                     handle->hal_handle->max_ustore;
 807                 handle->hal_handle->aes[ae].live_ctx_mask =
 808                                                 ICP_QAT_UCLO_AE_ALL_CTX;
 809                 max_en_ae_id = ae;
 810         }
 811         handle->hal_handle->ae_max_num = max_en_ae_id + 1;
 812
 813         /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
 814         for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 815                 csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
 816                 csr_val |= 0x1;
 817                 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 818         }
 819 out_err:
 820         return ret;
 821 }
 822
 823 int qat_hal_init(struct adf_accel_dev *accel_dev)
 824 {
 825         struct icp_qat_fw_loader_handle *handle;
 826         int ret = 0;
 827
 828         handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 829         if (!handle)
 830                 return -ENOMEM;
 831
 832         handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
 833         if (!handle->hal_handle) {
 834                 ret = -ENOMEM;
 835                 goto out_hal_handle;
 836         }
 837
 838         handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL);
 839         if (!handle->chip_info) {
 840                 ret = -ENOMEM;
 841                 goto out_chip_info;
 842         }
 843
 844         ret = qat_hal_chip_init(handle, accel_dev);
 845         if (ret) {
 846                 dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n");
 847                 goto out_err;
 848         }
 849
 850         /* take all AEs out of reset */
 851         ret = qat_hal_clr_reset(handle);
 852         if (ret) {
 853                 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
 854                 goto out_err;
 855         }
 856
 857         qat_hal_clear_xfer(handle);
 858         if (!handle->chip_info->fw_auth) {
 859                 ret = qat_hal_clear_gpr(handle);
 860                 if (ret)
 861                         goto out_err;
 862         }
 863
 864         accel_dev->fw_loader->fw_loader = handle;
 865         return 0;
 866
 867 out_err:
 868         kfree(handle->chip_info);
 869 out_chip_info:
 870         kfree(handle->hal_handle);
 871 out_hal_handle:
 872         kfree(handle);
 873         return ret;
 874 }
 875
 876 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
 877 {
 878         if (!handle)
 879                 return;
 880         kfree(handle->chip_info);
 881         kfree(handle->hal_handle);
 882         kfree(handle);
 883 }
 884
 885 int qat_hal_start(struct icp_qat_fw_loader_handle *handle)
 886 {
 887         unsigned long ae_mask = handle->hal_handle->ae_mask;
 888         u32 wakeup_val = handle->chip_info->wakeup_event_val;
 889         u32 fcu_ctl_csr, fcu_sts_csr;
 890         unsigned int fcu_sts;
 891         unsigned char ae;
 892         u32 ae_ctr = 0;
 893         int retry = 0;
 894
 895         if (handle->chip_info->fw_auth) {
 896                 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
 897                 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
 898                 ae_ctr = hweight32(ae_mask);
 899                 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START);
 900                 do {
 901                         msleep(FW_AUTH_WAIT_PERIOD);
 902                         fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
 903                         if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1))
 904                                 return ae_ctr;
 905                 } while (retry++ < FW_AUTH_MAX_RETRY);
 906                 pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts);
 907                 return 0;
 908         } else {
 909                 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
 910                         qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val);
 911                         qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX);
 912                         ae_ctr++;
 913                 }
 914                 return ae_ctr;
 915         }
 916 }
 917
 918 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 919                   unsigned int ctx_mask)
 920 {
 921         if (!handle->chip_info->fw_auth)
 922                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 923 }
 924
 925 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
 926                     unsigned char ae, unsigned int ctx_mask, unsigned int upc)
 927 {
 928         qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 929                             handle->hal_handle->upc_mask & upc);
 930 }
 931
 932 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 933                                unsigned char ae, unsigned int uaddr,
 934                                unsigned int words_num, u64 *uword)
 935 {
 936         unsigned int i, uwrd_lo, uwrd_hi;
 937         unsigned int ustore_addr, misc_control;
 938
 939         misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 940         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 941                           misc_control & 0xfffffffb);
 942         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 943         uaddr |= UA_ECS;
 944         for (i = 0; i < words_num; i++) {
 945                 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 946                 uaddr++;
 947                 uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
 948                 uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
 949                 uword[i] = uwrd_hi;
 950                 uword[i] = (uword[i] << 0x20) | uwrd_lo;
 951         }
 952         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
 953         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 954 }
 955
 956 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 957                      unsigned char ae, unsigned int uaddr,
 958                      unsigned int words_num, unsigned int *data)
 959 {
 960         unsigned int i, ustore_addr;
 961
 962         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 963         uaddr |= UA_ECS;
 964         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 965         for (i = 0; i < words_num; i++) {
 966                 unsigned int uwrd_lo, uwrd_hi, tmp;
 967
 968                 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
 969                           ((data[i] & 0xff00) << 2) |
 970                           (0x3 << 8) | (data[i] & 0xff);
 971                 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
 972                 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
 973                 tmp = ((data[i] >> 0x10) & 0xffff);
 974                 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
 975                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 976                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 977         }
 978         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 979 }
 980
 981 #define MAX_EXEC_INST 100
 982 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 983                                    unsigned char ae, unsigned char ctx,
 984                                    u64 *micro_inst, unsigned int inst_num,
 985                                    int code_off, unsigned int max_cycle,
 986                                    unsigned int *endpc)
 987 {
 988         unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0;
 989         unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0;
 990         unsigned int ind_t_index = 0, ind_t_index_byte = 0;
 991         unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0;
 992         unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0;
 993         u64 savuwords[MAX_EXEC_INST];
 994         unsigned int ind_cnt_sig;
 995         unsigned int ind_sig, act_sig;
 996         unsigned int csr_val = 0, newcsr_val;
 997         unsigned int savctx;
 998         unsigned int savcc, wakeup_events, savpc;
 999         unsigned int ctxarb_ctl, ctx_enables;
1000
1001         if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
1002                 pr_err("QAT: invalid instruction num %d\n", inst_num);
1003                 return -EINVAL;
1004         }
1005         /* save current context */
1006         ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
1007         ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
1008         ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
1009                                                 INDIRECT_LM_ADDR_0_BYTE_INDEX);
1010         ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
1011                                                 INDIRECT_LM_ADDR_1_BYTE_INDEX);
1012         if (handle->chip_info->lm2lm3) {
1013                 ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1014                                                    LM_ADDR_2_INDIRECT);
1015                 ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1016                                                    LM_ADDR_3_INDIRECT);
1017                 ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx,
1018                                                         INDIRECT_LM_ADDR_2_BYTE_INDEX);
1019                 ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx,
1020                                                         INDIRECT_LM_ADDR_3_BYTE_INDEX);
1021                 ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx,
1022                                                   INDIRECT_T_INDEX);
1023                 ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx,
1024                                                        INDIRECT_T_INDEX_BYTE_INDEX);
1025         }
1026         if (inst_num <= MAX_EXEC_INST)
1027                 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
1028         qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
1029         savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
1030         savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
1031         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1032         ctx_enables &= IGNORE_W1C_MASK;
1033         savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
1034         savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1035         ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1036         ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1037                                           FUTURE_COUNT_SIGNAL_INDIRECT);
1038         ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
1039                                       CTX_SIG_EVENTS_INDIRECT);
1040         act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
1041         /* execute micro codes */
1042         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1043         qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
1044         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
1045         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
1046         if (code_off)
1047                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
1048         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
1049         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
1050         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
1051         qat_hal_enable_ctx(handle, ae, (1 << ctx));
1052         /* wait for micro codes to finish */
1053         if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
1054                 return -EFAULT;
1055         if (endpc) {
1056                 unsigned int ctx_status;
1057
1058                 ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
1059                                                  CTX_STS_INDIRECT);
1060                 *endpc = ctx_status & handle->hal_handle->upc_mask;
1061         }
1062         /* retore to saved context */
1063         qat_hal_disable_ctx(handle, ae, (1 << ctx));
1064         if (inst_num <= MAX_EXEC_INST)
1065                 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
1066         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
1067         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
1068                             handle->hal_handle->upc_mask & savpc);
1069         csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
1070         newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
1071         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
1072         qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
1073         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
1074         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
1075         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1076                             LM_ADDR_0_INDIRECT, ind_lm_addr0);
1077         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1078                             LM_ADDR_1_INDIRECT, ind_lm_addr1);
1079         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1080                             INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
1081         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1082                             INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
1083         if (handle->chip_info->lm2lm3) {
1084                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT,
1085                                     ind_lm_addr2);
1086                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT,
1087                                     ind_lm_addr3);
1088                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1089                                     INDIRECT_LM_ADDR_2_BYTE_INDEX,
1090                                     ind_lm_addr_byte2);
1091                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1092                                     INDIRECT_LM_ADDR_3_BYTE_INDEX,
1093                                     ind_lm_addr_byte3);
1094                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1095                                     INDIRECT_T_INDEX, ind_t_index);
1096                 qat_hal_wr_indr_csr(handle, ae, BIT(ctx),
1097                                     INDIRECT_T_INDEX_BYTE_INDEX,
1098                                     ind_t_index_byte);
1099         }
1100         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1101                             FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
1102         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
1103                             CTX_SIG_EVENTS_INDIRECT, ind_sig);
1104         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
1105         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1106
1107         return 0;
1108 }
1109
1110 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
1111                               unsigned char ae, unsigned char ctx,
1112                               enum icp_qat_uof_regtype reg_type,
1113                               unsigned short reg_num, unsigned int *data)
1114 {
1115         unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
1116         unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
1117         unsigned short reg_addr;
1118         int status = 0;
1119         u64 insts, savuword;
1120
1121         reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1122         if (reg_addr == BAD_REGADDR) {
1123                 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
1124                 return -EINVAL;
1125         }
1126         switch (reg_type) {
1127         case ICP_GPA_REL:
1128                 insts = 0xA070000000ull | (reg_addr & 0x3ff);
1129                 break;
1130         default:
1131                 insts = (u64)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
1132                 break;
1133         }
1134         savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
1135         ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
1136         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1137         ctx_enables &= IGNORE_W1C_MASK;
1138         if (ctx != (savctx & ACS_ACNO))
1139                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1140                                   ctx & ACS_ACNO);
1141         qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
1142         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1143         ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
1144         uaddr = UA_ECS;
1145         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1146         insts = qat_hal_set_uword_ecc(insts);
1147         uwrd_lo = (unsigned int)(insts & 0xffffffff);
1148         uwrd_hi = (unsigned int)(insts >> 0x20);
1149         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
1150         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
1151         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
1152         /* delay for at least 8 cycles */
1153         qat_hal_wait_cycles(handle, ae, 0x8, 0);
1154         /*
1155          * read ALU output
1156          * the instruction should have been executed
1157          * prior to clearing the ECS in putUwords
1158          */
1159         *data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
1160         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
1161         qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
1162         if (ctx != (savctx & ACS_ACNO))
1163                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
1164                                   savctx & ACS_ACNO);
1165         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
1166         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1167
1168         return status;
1169 }
1170
1171 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
1172                               unsigned char ae, unsigned char ctx,
1173                               enum icp_qat_uof_regtype reg_type,
1174                               unsigned short reg_num, unsigned int data)
1175 {
1176         unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
1177         u64 insts[] = {
1178                 0x0F440000000ull,
1179                 0x0F040000000ull,
1180                 0x0F0000C0300ull,
1181                 0x0E000010000ull
1182         };
1183         const int num_inst = ARRAY_SIZE(insts), code_off = 1;
1184         const int imm_w1 = 0, imm_w0 = 1;
1185
1186         dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1187         if (dest_addr == BAD_REGADDR) {
1188                 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
1189                 return -EINVAL;
1190         }
1191
1192         data16lo = 0xffff & data;
1193         data16hi = 0xffff & (data >> 0x10);
1194         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1195                                           (0xff & data16hi));
1196         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1197                                            (0xff & data16lo));
1198         switch (reg_type) {
1199         case ICP_GPA_REL:
1200                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1201                     ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1202                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1203                     ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1204                 break;
1205         default:
1206                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1207                     ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1208
1209                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1210                     ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1211                 break;
1212         }
1213
1214         return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1215                                        code_off, num_inst * 0x5, NULL);
1216 }
1217
1218 int qat_hal_get_ins_num(void)
1219 {
1220         return ARRAY_SIZE(inst_4b);
1221 }
1222
1223 static int qat_hal_concat_micro_code(u64 *micro_inst,
1224                                      unsigned int inst_num, unsigned int size,
1225                                      unsigned int addr, unsigned int *value)
1226 {
1227         int i;
1228         unsigned int cur_value;
1229         const u64 *inst_arr;
1230         int fixup_offset;
1231         int usize = 0;
1232         int orig_num;
1233
1234         orig_num = inst_num;
1235         cur_value = value[0];
1236         inst_arr = inst_4b;
1237         usize = ARRAY_SIZE(inst_4b);
1238         fixup_offset = inst_num;
1239         for (i = 0; i < usize; i++)
1240                 micro_inst[inst_num++] = inst_arr[i];
1241         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1242         fixup_offset++;
1243         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1244         fixup_offset++;
1245         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1246         fixup_offset++;
1247         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1248
1249         return inst_num - orig_num;
1250 }
1251
1252 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1253                                       unsigned char ae, unsigned char ctx,
1254                                       int *pfirst_exec, u64 *micro_inst,
1255                                       unsigned int inst_num)
1256 {
1257         int stat = 0;
1258         unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1259         unsigned int gprb0 = 0, gprb1 = 0;
1260
1261         if (*pfirst_exec) {
1262                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1263                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1264                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1265                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1266                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1267                 *pfirst_exec = 0;
1268         }
1269         stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1270                                        inst_num * 0x5, NULL);
1271         if (stat != 0)
1272                 return -EFAULT;
1273         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1274         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1275         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1276         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1277         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1278
1279         return 0;
1280 }
1281
1282 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1283                         unsigned char ae,
1284                         struct icp_qat_uof_batch_init *lm_init_header)
1285 {
1286         struct icp_qat_uof_batch_init *plm_init;
1287         u64 *micro_inst_arry;
1288         int micro_inst_num;
1289         int alloc_inst_size;
1290         int first_exec = 1;
1291         int stat = 0;
1292
1293         plm_init = lm_init_header->next;
1294         alloc_inst_size = lm_init_header->size;
1295         if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1296                 alloc_inst_size = handle->hal_handle->max_ustore;
1297         micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(u64),
1298                                         GFP_KERNEL);
1299         if (!micro_inst_arry)
1300                 return -ENOMEM;
1301         micro_inst_num = 0;
1302         while (plm_init) {
1303                 unsigned int addr, *value, size;
1304
1305                 ae = plm_init->ae;
1306                 addr = plm_init->addr;
1307                 value = plm_init->value;
1308                 size = plm_init->size;
1309                 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1310                                                             micro_inst_num,
1311                                                             size, addr, value);
1312                 plm_init = plm_init->next;
1313         }
1314         /* exec micro codes */
1315         if (micro_inst_arry && micro_inst_num > 0) {
1316                 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1317                 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1318                                                   micro_inst_arry,
1319                                                   micro_inst_num);
1320         }
1321         kfree(micro_inst_arry);
1322         return stat;
1323 }
1324
1325 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1326                                    unsigned char ae, unsigned char ctx,
1327                                    enum icp_qat_uof_regtype reg_type,
1328                                    unsigned short reg_num, unsigned int val)
1329 {
1330         int status = 0;
1331         unsigned int reg_addr;
1332         unsigned int ctx_enables;
1333         unsigned short mask;
1334         unsigned short dr_offset = 0x10;
1335
1336         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1337         if (CE_INUSE_CONTEXTS & ctx_enables) {
1338                 if (ctx & 0x1) {
1339                         pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1340                         return -EINVAL;
1341                 }
1342                 mask = 0x1f;
1343                 dr_offset = 0x20;
1344         } else {
1345                 mask = 0x0f;
1346         }
1347         if (reg_num & ~mask)
1348                 return -EINVAL;
1349         reg_addr = reg_num + (ctx << 0x5);
1350         switch (reg_type) {
1351         case ICP_SR_RD_REL:
1352         case ICP_SR_REL:
1353                 SET_AE_XFER(handle, ae, reg_addr, val);
1354                 break;
1355         case ICP_DR_RD_REL:
1356         case ICP_DR_REL:
1357                 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1358                 break;
1359         default:
1360                 status = -EINVAL;
1361                 break;
1362         }
1363         return status;
1364 }
1365
1366 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1367                                    unsigned char ae, unsigned char ctx,
1368                                    enum icp_qat_uof_regtype reg_type,
1369                                    unsigned short reg_num, unsigned int data)
1370 {
1371         unsigned int gprval, ctx_enables;
1372         unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1373             data16low;
1374         unsigned short reg_mask;
1375         int status = 0;
1376         u64 micro_inst[] = {
1377                 0x0F440000000ull,
1378                 0x0F040000000ull,
1379                 0x0A000000000ull,
1380                 0x0F0000C0300ull,
1381                 0x0E000010000ull
1382         };
1383         const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1384         const unsigned short gprnum = 0, dly = num_inst * 0x5;
1385
1386         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1387         if (CE_INUSE_CONTEXTS & ctx_enables) {
1388                 if (ctx & 0x1) {
1389                         pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1390                         return -EINVAL;
1391                 }
1392                 reg_mask = (unsigned short)~0x1f;
1393         } else {
1394                 reg_mask = (unsigned short)~0xf;
1395         }
1396         if (reg_num & reg_mask)
1397                 return -EINVAL;
1398         xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1399         if (xfr_addr == BAD_REGADDR) {
1400                 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1401                 return -EINVAL;
1402         }
1403         status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1404         if (status) {
1405                 pr_err("QAT: failed to read register");
1406                 return status;
1407         }
1408         gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1409         data16low = 0xffff & data;
1410         data16hi = 0xffff & (data >> 0x10);
1411         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1412                                           (unsigned short)(0xff & data16hi));
1413         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1414                                            (unsigned short)(0xff & data16low));
1415         micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1416             ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1417         micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1418             ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1419         micro_inst[0x2] = micro_inst[0x2] |
1420             ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1421         status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1422                                          code_off, dly, NULL);
1423         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1424         return status;
1425 }
1426
1427 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1428                               unsigned char ae, unsigned char ctx,
1429                               unsigned short nn, unsigned int val)
1430 {
1431         unsigned int ctx_enables;
1432         int stat = 0;
1433
1434         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1435         ctx_enables &= IGNORE_W1C_MASK;
1436         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1437
1438         stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1439         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1440         return stat;
1441 }
1442
1443 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1444                                       *handle, unsigned char ae,
1445                                       unsigned short absreg_num,
1446                                       unsigned short *relreg,
1447                                       unsigned char *ctx)
1448 {
1449         unsigned int ctx_enables;
1450
1451         ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
1452         if (ctx_enables & CE_INUSE_CONTEXTS) {
1453                 /* 4-ctx mode */
1454                 *relreg = absreg_num & 0x1F;
1455                 *ctx = (absreg_num >> 0x4) & 0x6;
1456         } else {
1457                 /* 8-ctx mode */
1458                 *relreg = absreg_num & 0x0F;
1459                 *ctx = (absreg_num >> 0x4) & 0x7;
1460         }
1461         return 0;
1462 }
1463
1464 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1465                      unsigned char ae, unsigned long ctx_mask,
1466                      enum icp_qat_uof_regtype reg_type,
1467                      unsigned short reg_num, unsigned int regdata)
1468 {
1469         int stat = 0;
1470         unsigned short reg;
1471         unsigned char ctx = 0;
1472         enum icp_qat_uof_regtype type;
1473
1474         if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1475                 return -EINVAL;
1476
1477         do {
1478                 if (ctx_mask == 0) {
1479                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1480                                                    &ctx);
1481                         type = reg_type - 1;
1482                 } else {
1483                         reg = reg_num;
1484                         type = reg_type;
1485                         if (!test_bit(ctx, &ctx_mask))
1486                                 continue;
1487                 }
1488                 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1489                 if (stat) {
1490                         pr_err("QAT: write gpr fail\n");
1491                         return -EINVAL;
1492                 }
1493         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1494
1495         return 0;
1496 }
1497
1498 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1499                          unsigned char ae, unsigned long ctx_mask,
1500                          enum icp_qat_uof_regtype reg_type,
1501                          unsigned short reg_num, unsigned int regdata)
1502 {
1503         int stat = 0;
1504         unsigned short reg;
1505         unsigned char ctx = 0;
1506         enum icp_qat_uof_regtype type;
1507
1508         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1509                 return -EINVAL;
1510
1511         do {
1512                 if (ctx_mask == 0) {
1513                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1514                                                    &ctx);
1515                         type = reg_type - 3;
1516                 } else {
1517                         reg = reg_num;
1518                         type = reg_type;
1519                         if (!test_bit(ctx, &ctx_mask))
1520                                 continue;
1521                 }
1522                 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1523                                                regdata);
1524                 if (stat) {
1525                         pr_err("QAT: write wr xfer fail\n");
1526                         return -EINVAL;
1527                 }
1528         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1529
1530         return 0;
1531 }
1532
1533 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1534                          unsigned char ae, unsigned long ctx_mask,
1535                          enum icp_qat_uof_regtype reg_type,
1536                          unsigned short reg_num, unsigned int regdata)
1537 {
1538         int stat = 0;
1539         unsigned short reg;
1540         unsigned char ctx = 0;
1541         enum icp_qat_uof_regtype type;
1542
1543         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1544                 return -EINVAL;
1545
1546         do {
1547                 if (ctx_mask == 0) {
1548                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1549                                                    &ctx);
1550                         type = reg_type - 3;
1551                 } else {
1552                         reg = reg_num;
1553                         type = reg_type;
1554                         if (!test_bit(ctx, &ctx_mask))
1555                                 continue;
1556                 }
1557                 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1558                                                regdata);
1559                 if (stat) {
1560                         pr_err("QAT: write rd xfer fail\n");
1561                         return -EINVAL;
1562                 }
1563         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1564
1565         return 0;
1566 }
1567
1568 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1569                     unsigned char ae, unsigned long ctx_mask,
1570                     unsigned short reg_num, unsigned int regdata)
1571 {
1572         int stat = 0;
1573         unsigned char ctx;
1574         if (!handle->chip_info->nn) {
1575                 dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n",
1576                         handle->pci_dev->device);
1577                 return -EINVAL;
1578         }
1579
1580         if (ctx_mask == 0)
1581                 return -EINVAL;
1582
1583         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1584                 if (!test_bit(ctx, &ctx_mask))
1585                         continue;
1586                 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1587                 if (stat) {
1588                         pr_err("QAT: write neigh error\n");
1589                         return -EINVAL;
1590                 }
1591         }
1592
1593         return 0;
1594 }