drivers/crypto/qat/qat_common/qat_uclo.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2014 - 2020 Intel Corporation */
   3 #include <linux/slab.h>
   4 #include <linux/ctype.h>
   5 #include <linux/kernel.h>
   6 #include <linux/delay.h>
   7 #include "adf_accel_devices.h"
   8 #include "adf_common_drv.h"
   9 #include "icp_qat_uclo.h"
  10 #include "icp_qat_hal.h"
  11 #include "icp_qat_fw_loader_handle.h"
  12
  13 #define UWORD_CPYBUF_SIZE 1024
  14 #define INVLD_UWORD 0xffffffffffull
  15 #define PID_MINOR_REV 0xf
  16 #define PID_MAJOR_REV (0xf << 4)
  17
  18 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle,
  19                                  unsigned int ae, unsigned int image_num)
  20 {
  21         struct icp_qat_uclo_aedata *ae_data;
  22         struct icp_qat_uclo_encapme *encap_image;
  23         struct icp_qat_uclo_page *page = NULL;
  24         struct icp_qat_uclo_aeslice *ae_slice = NULL;
  25
  26         ae_data = &obj_handle->ae_data[ae];
  27         encap_image = &obj_handle->ae_uimage[image_num];
  28         ae_slice = &ae_data->ae_slices[ae_data->slice_num];
  29         ae_slice->encap_image = encap_image;
  30
  31         if (encap_image->img_ptr) {
  32                 ae_slice->ctx_mask_assigned =
  33                                         encap_image->img_ptr->ctx_assigned;
  34                 ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
  35         } else {
  36                 ae_slice->ctx_mask_assigned = 0;
  37         }
  38         ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL);
  39         if (!ae_slice->region)
  40                 return -ENOMEM;
  41         ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL);
  42         if (!ae_slice->page)
  43                 goto out_err;
  44         page = ae_slice->page;
  45         page->encap_page = encap_image->page;
  46         ae_slice->page->region = ae_slice->region;
  47         ae_data->slice_num++;
  48         return 0;
  49 out_err:
  50         kfree(ae_slice->region);
  51         ae_slice->region = NULL;
  52         return -ENOMEM;
  53 }
  54
  55 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data)
  56 {
  57         unsigned int i;
  58
  59         if (!ae_data) {
  60                 pr_err("QAT: bad argument, ae_data is NULL\n ");
  61                 return -EINVAL;
  62         }
  63
  64         for (i = 0; i < ae_data->slice_num; i++) {
  65                 kfree(ae_data->ae_slices[i].region);
  66                 ae_data->ae_slices[i].region = NULL;
  67                 kfree(ae_data->ae_slices[i].page);
  68                 ae_data->ae_slices[i].page = NULL;
  69         }
  70         return 0;
  71 }
  72
  73 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table,
  74                                  unsigned int str_offset)
  75 {
  76         if ((!str_table->table_len) || (str_offset > str_table->table_len))
  77                 return NULL;
  78         return (char *)(((uintptr_t)(str_table->strings)) + str_offset);
  79 }
  80
  81 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr)
  82 {
  83         int maj = hdr->maj_ver & 0xff;
  84         int min = hdr->min_ver & 0xff;
  85
  86         if (hdr->file_id != ICP_QAT_UOF_FID) {
  87                 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id);
  88                 return -EINVAL;
  89         }
  90         if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) {
  91                 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n",
  92                        maj, min);
  93                 return -EINVAL;
  94         }
  95         return 0;
  96 }
  97
  98 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr)
  99 {
 100         int maj = suof_hdr->maj_ver & 0xff;
 101         int min = suof_hdr->min_ver & 0xff;
 102
 103         if (suof_hdr->file_id != ICP_QAT_SUOF_FID) {
 104                 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id);
 105                 return -EINVAL;
 106         }
 107         if (suof_hdr->fw_type != 0) {
 108                 pr_err("QAT: unsupported firmware type\n");
 109                 return -EINVAL;
 110         }
 111         if (suof_hdr->num_chunks <= 0x1) {
 112                 pr_err("QAT: SUOF chunk amount is incorrect\n");
 113                 return -EINVAL;
 114         }
 115         if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) {
 116                 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n",
 117                        maj, min);
 118                 return -EINVAL;
 119         }
 120         return 0;
 121 }
 122
 123 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle,
 124                                       unsigned int addr, unsigned int *val,
 125                                       unsigned int num_in_bytes)
 126 {
 127         unsigned int outval;
 128         unsigned char *ptr = (unsigned char *)val;
 129
 130         while (num_in_bytes) {
 131                 memcpy(&outval, ptr, 4);
 132                 SRAM_WRITE(handle, addr, outval);
 133                 num_in_bytes -= 4;
 134                 ptr += 4;
 135                 addr += 4;
 136         }
 137 }
 138
 139 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle,
 140                                       unsigned char ae, unsigned int addr,
 141                                       unsigned int *val,
 142                                       unsigned int num_in_bytes)
 143 {
 144         unsigned int outval;
 145         unsigned char *ptr = (unsigned char *)val;
 146
 147         addr >>= 0x2; /* convert to uword address */
 148
 149         while (num_in_bytes) {
 150                 memcpy(&outval, ptr, 4);
 151                 qat_hal_wr_umem(handle, ae, addr++, 1, &outval);
 152                 num_in_bytes -= 4;
 153                 ptr += 4;
 154         }
 155 }
 156
 157 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle,
 158                                    unsigned char ae,
 159                                    struct icp_qat_uof_batch_init
 160                                    *umem_init_header)
 161 {
 162         struct icp_qat_uof_batch_init *umem_init;
 163
 164         if (!umem_init_header)
 165                 return;
 166         umem_init = umem_init_header->next;
 167         while (umem_init) {
 168                 unsigned int addr, *value, size;
 169
 170                 ae = umem_init->ae;
 171                 addr = umem_init->addr;
 172                 value = umem_init->value;
 173                 size = umem_init->size;
 174                 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size);
 175                 umem_init = umem_init->next;
 176         }
 177 }
 178
 179 static void
 180 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle,
 181                                  struct icp_qat_uof_batch_init **base)
 182 {
 183         struct icp_qat_uof_batch_init *umem_init;
 184
 185         umem_init = *base;
 186         while (umem_init) {
 187                 struct icp_qat_uof_batch_init *pre;
 188
 189                 pre = umem_init;
 190                 umem_init = umem_init->next;
 191                 kfree(pre);
 192         }
 193         *base = NULL;
 194 }
 195
 196 static int qat_uclo_parse_num(char *str, unsigned int *num)
 197 {
 198         char buf[16] = {0};
 199         unsigned long ae = 0;
 200         int i;
 201
 202         strncpy(buf, str, 15);
 203         for (i = 0; i < 16; i++) {
 204                 if (!isdigit(buf[i])) {
 205                         buf[i] = '\0';
 206                         break;
 207                 }
 208         }
 209         if ((kstrtoul(buf, 10, &ae)))
 210                 return -EFAULT;
 211
 212         *num = (unsigned int)ae;
 213         return 0;
 214 }
 215
 216 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle,
 217                                      struct icp_qat_uof_initmem *init_mem,
 218                                      unsigned int size_range, unsigned int *ae)
 219 {
 220         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 221         char *str;
 222
 223         if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) {
 224                 pr_err("QAT: initmem is out of range");
 225                 return -EINVAL;
 226         }
 227         if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) {
 228                 pr_err("QAT: Memory scope for init_mem error\n");
 229                 return -EINVAL;
 230         }
 231         str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name);
 232         if (!str) {
 233                 pr_err("QAT: AE name assigned in UOF init table is NULL\n");
 234                 return -EINVAL;
 235         }
 236         if (qat_uclo_parse_num(str, ae)) {
 237                 pr_err("QAT: Parse num for AE number failed\n");
 238                 return -EINVAL;
 239         }
 240         if (*ae >= ICP_QAT_UCLO_MAX_AE) {
 241                 pr_err("QAT: ae %d out of range\n", *ae);
 242                 return -EINVAL;
 243         }
 244         return 0;
 245 }
 246
 247 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle
 248                                            *handle, struct icp_qat_uof_initmem
 249                                            *init_mem, unsigned int ae,
 250                                            struct icp_qat_uof_batch_init
 251                                            **init_tab_base)
 252 {
 253         struct icp_qat_uof_batch_init *init_header, *tail;
 254         struct icp_qat_uof_batch_init *mem_init, *tail_old;
 255         struct icp_qat_uof_memvar_attr *mem_val_attr;
 256         unsigned int i, flag = 0;
 257
 258         mem_val_attr =
 259                 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem +
 260                 sizeof(struct icp_qat_uof_initmem));
 261
 262         init_header = *init_tab_base;
 263         if (!init_header) {
 264                 init_header = kzalloc(sizeof(*init_header), GFP_KERNEL);
 265                 if (!init_header)
 266                         return -ENOMEM;
 267                 init_header->size = 1;
 268                 *init_tab_base = init_header;
 269                 flag = 1;
 270         }
 271         tail_old = init_header;
 272         while (tail_old->next)
 273                 tail_old = tail_old->next;
 274         tail = tail_old;
 275         for (i = 0; i < init_mem->val_attr_num; i++) {
 276                 mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL);
 277                 if (!mem_init)
 278                         goto out_err;
 279                 mem_init->ae = ae;
 280                 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte;
 281                 mem_init->value = &mem_val_attr->value;
 282                 mem_init->size = 4;
 283                 mem_init->next = NULL;
 284                 tail->next = mem_init;
 285                 tail = mem_init;
 286                 init_header->size += qat_hal_get_ins_num();
 287                 mem_val_attr++;
 288         }
 289         return 0;
 290 out_err:
 291         /* Do not free the list head unless we allocated it. */
 292         tail_old = tail_old->next;
 293         if (flag) {
 294                 kfree(*init_tab_base);
 295                 *init_tab_base = NULL;
 296         }
 297
 298         while (tail_old) {
 299                 mem_init = tail_old->next;
 300                 kfree(tail_old);
 301                 tail_old = mem_init;
 302         }
 303         return -ENOMEM;
 304 }
 305
 306 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle,
 307                                   struct icp_qat_uof_initmem *init_mem)
 308 {
 309         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 310         unsigned int ae;
 311
 312         if (qat_uclo_fetch_initmem_ae(handle, init_mem,
 313                                       ICP_QAT_UCLO_MAX_LMEM_REG, &ae))
 314                 return -EINVAL;
 315         if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
 316                                             &obj_handle->lm_init_tab[ae]))
 317                 return -EINVAL;
 318         return 0;
 319 }
 320
 321 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
 322                                   struct icp_qat_uof_initmem *init_mem)
 323 {
 324         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 325         unsigned int ae, ustore_size, uaddr, i;
 326
 327         ustore_size = obj_handle->ustore_phy_size;
 328         if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae))
 329                 return -EINVAL;
 330         if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
 331                                             &obj_handle->umem_init_tab[ae]))
 332                 return -EINVAL;
 333         /* set the highest ustore address referenced */
 334         uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2;
 335         for (i = 0; i < obj_handle->ae_data[ae].slice_num; i++) {
 336                 if (obj_handle->ae_data[ae].ae_slices[i].
 337                     encap_image->uwords_num < uaddr)
 338                         obj_handle->ae_data[ae].ae_slices[i].
 339                         encap_image->uwords_num = uaddr;
 340         }
 341         return 0;
 342 }
 343
 344 #define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000
 345 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
 346                                    struct icp_qat_uof_initmem *init_mem)
 347 {
 348         switch (init_mem->region) {
 349         case ICP_QAT_UOF_LMEM_REGION:
 350                 if (qat_uclo_init_lmem_seg(handle, init_mem))
 351                         return -EINVAL;
 352                 break;
 353         case ICP_QAT_UOF_UMEM_REGION:
 354                 if (qat_uclo_init_umem_seg(handle, init_mem))
 355                         return -EINVAL;
 356                 break;
 357         default:
 358                 pr_err("QAT: initmem region error. region type=0x%x\n",
 359                        init_mem->region);
 360                 return -EINVAL;
 361         }
 362         return 0;
 363 }
 364
 365 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle,
 366                                 struct icp_qat_uclo_encapme *image)
 367 {
 368         unsigned int i;
 369         struct icp_qat_uclo_encap_page *page;
 370         struct icp_qat_uof_image *uof_image;
 371         unsigned char ae;
 372         unsigned int ustore_size;
 373         unsigned int patt_pos;
 374         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 375         u64 *fill_data;
 376
 377         uof_image = image->img_ptr;
 378         fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64),
 379                             GFP_KERNEL);
 380         if (!fill_data)
 381                 return -ENOMEM;
 382         for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++)
 383                 memcpy(&fill_data[i], &uof_image->fill_pattern,
 384                        sizeof(u64));
 385         page = image->page;
 386
 387         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 388                 if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned))
 389                         continue;
 390                 ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
 391                 patt_pos = page->beg_addr_p + page->micro_words_num;
 392
 393                 qat_hal_wr_uwords(handle, (unsigned char)ae, 0,
 394                                   page->beg_addr_p, &fill_data[0]);
 395                 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos,
 396                                   ustore_size - patt_pos + 1,
 397                                   &fill_data[page->beg_addr_p]);
 398         }
 399         kfree(fill_data);
 400         return 0;
 401 }
 402
 403 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle)
 404 {
 405         int i, ae;
 406         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 407         struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem;
 408
 409         for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) {
 410                 if (initmem->num_in_bytes) {
 411                         if (qat_uclo_init_ae_memory(handle, initmem))
 412                                 return -EINVAL;
 413                 }
 414                 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)(
 415                         (uintptr_t)initmem +
 416                         sizeof(struct icp_qat_uof_initmem)) +
 417                         (sizeof(struct icp_qat_uof_memvar_attr) *
 418                         initmem->val_attr_num));
 419         }
 420         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 421                 if (qat_hal_batch_wr_lm(handle, ae,
 422                                         obj_handle->lm_init_tab[ae])) {
 423                         pr_err("QAT: fail to batch init lmem for AE %d\n", ae);
 424                         return -EINVAL;
 425                 }
 426                 qat_uclo_cleanup_batch_init_list(handle,
 427                                                  &obj_handle->lm_init_tab[ae]);
 428                 qat_uclo_batch_wr_umem(handle, ae,
 429                                        obj_handle->umem_init_tab[ae]);
 430                 qat_uclo_cleanup_batch_init_list(handle,
 431                                                  &obj_handle->
 432                                                  umem_init_tab[ae]);
 433         }
 434         return 0;
 435 }
 436
 437 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr,
 438                                  char *chunk_id, void *cur)
 439 {
 440         int i;
 441         struct icp_qat_uof_chunkhdr *chunk_hdr =
 442             (struct icp_qat_uof_chunkhdr *)
 443             ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr));
 444
 445         for (i = 0; i < obj_hdr->num_chunks; i++) {
 446                 if ((cur < (void *)&chunk_hdr[i]) &&
 447                     !strncmp(chunk_hdr[i].chunk_id, chunk_id,
 448                              ICP_QAT_UOF_OBJID_LEN)) {
 449                         return &chunk_hdr[i];
 450                 }
 451         }
 452         return NULL;
 453 }
 454
 455 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch)
 456 {
 457         int i;
 458         unsigned int topbit = 1 << 0xF;
 459         unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch);
 460
 461         reg ^= inbyte << 0x8;
 462         for (i = 0; i < 0x8; i++) {
 463                 if (reg & topbit)
 464                         reg = (reg << 1) ^ 0x1021;
 465                 else
 466                         reg <<= 1;
 467         }
 468         return reg & 0xFFFF;
 469 }
 470
 471 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num)
 472 {
 473         unsigned int chksum = 0;
 474
 475         if (ptr)
 476                 while (num--)
 477                         chksum = qat_uclo_calc_checksum(chksum, *ptr++);
 478         return chksum;
 479 }
 480
 481 static struct icp_qat_uclo_objhdr *
 482 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr,
 483                    char *chunk_id)
 484 {
 485         struct icp_qat_uof_filechunkhdr *file_chunk;
 486         struct icp_qat_uclo_objhdr *obj_hdr;
 487         char *chunk;
 488         int i;
 489
 490         file_chunk = (struct icp_qat_uof_filechunkhdr *)
 491                 (buf + sizeof(struct icp_qat_uof_filehdr));
 492         for (i = 0; i < file_hdr->num_chunks; i++) {
 493                 if (!strncmp(file_chunk->chunk_id, chunk_id,
 494                              ICP_QAT_UOF_OBJID_LEN)) {
 495                         chunk = buf + file_chunk->offset;
 496                         if (file_chunk->checksum != qat_uclo_calc_str_checksum(
 497                                 chunk, file_chunk->size))
 498                                 break;
 499                         obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL);
 500                         if (!obj_hdr)
 501                                 break;
 502                         obj_hdr->file_buff = chunk;
 503                         obj_hdr->checksum = file_chunk->checksum;
 504                         obj_hdr->size = file_chunk->size;
 505                         return obj_hdr;
 506                 }
 507                 file_chunk++;
 508         }
 509         return NULL;
 510 }
 511
 512 static unsigned int
 513 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj,
 514                             struct icp_qat_uof_image *image)
 515 {
 516         struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab;
 517         struct icp_qat_uof_objtable *neigh_reg_tab;
 518         struct icp_qat_uof_code_page *code_page;
 519
 520         code_page = (struct icp_qat_uof_code_page *)
 521                         ((char *)image + sizeof(struct icp_qat_uof_image));
 522         uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
 523                      code_page->uc_var_tab_offset);
 524         imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
 525                       code_page->imp_var_tab_offset);
 526         imp_expr_tab = (struct icp_qat_uof_objtable *)
 527                        (encap_uof_obj->beg_uof +
 528                        code_page->imp_expr_tab_offset);
 529         if (uc_var_tab->entry_num || imp_var_tab->entry_num ||
 530             imp_expr_tab->entry_num) {
 531                 pr_err("QAT: UOF can't contain imported variable to be parsed\n");
 532                 return -EINVAL;
 533         }
 534         neigh_reg_tab = (struct icp_qat_uof_objtable *)
 535                         (encap_uof_obj->beg_uof +
 536                         code_page->neigh_reg_tab_offset);
 537         if (neigh_reg_tab->entry_num) {
 538                 pr_err("QAT: UOF can't contain shared control store feature\n");
 539                 return -EINVAL;
 540         }
 541         if (image->numpages > 1) {
 542                 pr_err("QAT: UOF can't contain multiple pages\n");
 543                 return -EINVAL;
 544         }
 545         if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) {
 546                 pr_err("QAT: UOF can't use shared control store feature\n");
 547                 return -EFAULT;
 548         }
 549         if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) {
 550                 pr_err("QAT: UOF can't use reloadable feature\n");
 551                 return -EFAULT;
 552         }
 553         return 0;
 554 }
 555
 556 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj
 557                                      *encap_uof_obj,
 558                                      struct icp_qat_uof_image *img,
 559                                      struct icp_qat_uclo_encap_page *page)
 560 {
 561         struct icp_qat_uof_code_page *code_page;
 562         struct icp_qat_uof_code_area *code_area;
 563         struct icp_qat_uof_objtable *uword_block_tab;
 564         struct icp_qat_uof_uword_block *uwblock;
 565         int i;
 566
 567         code_page = (struct icp_qat_uof_code_page *)
 568                         ((char *)img + sizeof(struct icp_qat_uof_image));
 569         page->def_page = code_page->def_page;
 570         page->page_region = code_page->page_region;
 571         page->beg_addr_v = code_page->beg_addr_v;
 572         page->beg_addr_p = code_page->beg_addr_p;
 573         code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof +
 574                                                 code_page->code_area_offset);
 575         page->micro_words_num = code_area->micro_words_num;
 576         uword_block_tab = (struct icp_qat_uof_objtable *)
 577                           (encap_uof_obj->beg_uof +
 578                           code_area->uword_block_tab);
 579         page->uwblock_num = uword_block_tab->entry_num;
 580         uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab +
 581                         sizeof(struct icp_qat_uof_objtable));
 582         page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock;
 583         for (i = 0; i < uword_block_tab->entry_num; i++)
 584                 page->uwblock[i].micro_words =
 585                 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset;
 586 }
 587
 588 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle,
 589                                struct icp_qat_uclo_encapme *ae_uimage,
 590                                int max_image)
 591 {
 592         int i, j;
 593         struct icp_qat_uof_chunkhdr *chunk_hdr = NULL;
 594         struct icp_qat_uof_image *image;
 595         struct icp_qat_uof_objtable *ae_regtab;
 596         struct icp_qat_uof_objtable *init_reg_sym_tab;
 597         struct icp_qat_uof_objtable *sbreak_tab;
 598         struct icp_qat_uof_encap_obj *encap_uof_obj =
 599                                         &obj_handle->encap_uof_obj;
 600
 601         for (j = 0; j < max_image; j++) {
 602                 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
 603                                                 ICP_QAT_UOF_IMAG, chunk_hdr);
 604                 if (!chunk_hdr)
 605                         break;
 606                 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof +
 607                                                      chunk_hdr->offset);
 608                 ae_regtab = (struct icp_qat_uof_objtable *)
 609                            (image->reg_tab_offset +
 610                            obj_handle->obj_hdr->file_buff);
 611                 ae_uimage[j].ae_reg_num = ae_regtab->entry_num;
 612                 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *)
 613                         (((char *)ae_regtab) +
 614                         sizeof(struct icp_qat_uof_objtable));
 615                 init_reg_sym_tab = (struct icp_qat_uof_objtable *)
 616                                    (image->init_reg_sym_tab +
 617                                    obj_handle->obj_hdr->file_buff);
 618                 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num;
 619                 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *)
 620                         (((char *)init_reg_sym_tab) +
 621                         sizeof(struct icp_qat_uof_objtable));
 622                 sbreak_tab = (struct icp_qat_uof_objtable *)
 623                         (image->sbreak_tab + obj_handle->obj_hdr->file_buff);
 624                 ae_uimage[j].sbreak_num = sbreak_tab->entry_num;
 625                 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *)
 626                                       (((char *)sbreak_tab) +
 627                                       sizeof(struct icp_qat_uof_objtable));
 628                 ae_uimage[j].img_ptr = image;
 629                 if (qat_uclo_check_image_compat(encap_uof_obj, image))
 630                         goto out_err;
 631                 ae_uimage[j].page =
 632                         kzalloc(sizeof(struct icp_qat_uclo_encap_page),
 633                                 GFP_KERNEL);
 634                 if (!ae_uimage[j].page)
 635                         goto out_err;
 636                 qat_uclo_map_image_page(encap_uof_obj, image,
 637                                         ae_uimage[j].page);
 638         }
 639         return j;
 640 out_err:
 641         for (i = 0; i < j; i++)
 642                 kfree(ae_uimage[i].page);
 643         return 0;
 644 }
 645
 646 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
 647 {
 648         int i, ae;
 649         int mflag = 0;
 650         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 651
 652         for (ae = 0; ae < max_ae; ae++) {
 653                 if (!test_bit(ae,
 654                               (unsigned long *)&handle->hal_handle->ae_mask))
 655                         continue;
 656                 for (i = 0; i < obj_handle->uimage_num; i++) {
 657                         if (!test_bit(ae, (unsigned long *)
 658                         &obj_handle->ae_uimage[i].img_ptr->ae_assigned))
 659                                 continue;
 660                         mflag = 1;
 661                         if (qat_uclo_init_ae_data(obj_handle, ae, i))
 662                                 return -EINVAL;
 663                 }
 664         }
 665         if (!mflag) {
 666                 pr_err("QAT: uimage uses AE not set\n");
 667                 return -EINVAL;
 668         }
 669         return 0;
 670 }
 671
 672 static struct icp_qat_uof_strtable *
 673 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr,
 674                        char *tab_name, struct icp_qat_uof_strtable *str_table)
 675 {
 676         struct icp_qat_uof_chunkhdr *chunk_hdr;
 677
 678         chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)
 679                                         obj_hdr->file_buff, tab_name, NULL);
 680         if (chunk_hdr) {
 681                 int hdr_size;
 682
 683                 memcpy(&str_table->table_len, obj_hdr->file_buff +
 684                        chunk_hdr->offset, sizeof(str_table->table_len));
 685                 hdr_size = (char *)&str_table->strings - (char *)str_table;
 686                 str_table->strings = (uintptr_t)obj_hdr->file_buff +
 687                                         chunk_hdr->offset + hdr_size;
 688                 return str_table;
 689         }
 690         return NULL;
 691 }
 692
 693 static void
 694 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj,
 695                            struct icp_qat_uclo_init_mem_table *init_mem_tab)
 696 {
 697         struct icp_qat_uof_chunkhdr *chunk_hdr;
 698
 699         chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
 700                                         ICP_QAT_UOF_IMEM, NULL);
 701         if (chunk_hdr) {
 702                 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof +
 703                         chunk_hdr->offset, sizeof(unsigned int));
 704                 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *)
 705                 (encap_uof_obj->beg_uof + chunk_hdr->offset +
 706                 sizeof(unsigned int));
 707         }
 708 }
 709
 710 static unsigned int
 711 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
 712 {
 713         switch (handle->pci_dev->device) {
 714         case ADF_DH895XCC_PCI_DEVICE_ID:
 715                 return ICP_QAT_AC_895XCC_DEV_TYPE;
 716         case ADF_C62X_PCI_DEVICE_ID:
 717                 return ICP_QAT_AC_C62X_DEV_TYPE;
 718         case ADF_C3XXX_PCI_DEVICE_ID:
 719                 return ICP_QAT_AC_C3XXX_DEV_TYPE;
 720         default:
 721                 pr_err("QAT: unsupported device 0x%x\n",
 722                        handle->pci_dev->device);
 723                 return 0;
 724         }
 725 }
 726
 727 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle)
 728 {
 729         unsigned int maj_ver, prod_type = obj_handle->prod_type;
 730
 731         if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) {
 732                 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n",
 733                        obj_handle->encap_uof_obj.obj_hdr->ac_dev_type,
 734                        prod_type);
 735                 return -EINVAL;
 736         }
 737         maj_ver = obj_handle->prod_rev & 0xff;
 738         if ((obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver) ||
 739             (obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver)) {
 740                 pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver);
 741                 return -EINVAL;
 742         }
 743         return 0;
 744 }
 745
 746 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle,
 747                              unsigned char ae, unsigned char ctx_mask,
 748                              enum icp_qat_uof_regtype reg_type,
 749                              unsigned short reg_addr, unsigned int value)
 750 {
 751         switch (reg_type) {
 752         case ICP_GPA_ABS:
 753         case ICP_GPB_ABS:
 754                 ctx_mask = 0;
 755                 fallthrough;
 756         case ICP_GPA_REL:
 757         case ICP_GPB_REL:
 758                 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type,
 759                                         reg_addr, value);
 760         case ICP_SR_ABS:
 761         case ICP_DR_ABS:
 762         case ICP_SR_RD_ABS:
 763         case ICP_DR_RD_ABS:
 764                 ctx_mask = 0;
 765                 fallthrough;
 766         case ICP_SR_REL:
 767         case ICP_DR_REL:
 768         case ICP_SR_RD_REL:
 769         case ICP_DR_RD_REL:
 770                 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type,
 771                                             reg_addr, value);
 772         case ICP_SR_WR_ABS:
 773         case ICP_DR_WR_ABS:
 774                 ctx_mask = 0;
 775                 fallthrough;
 776         case ICP_SR_WR_REL:
 777         case ICP_DR_WR_REL:
 778                 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type,
 779                                             reg_addr, value);
 780         case ICP_NEIGH_REL:
 781                 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value);
 782         default:
 783                 pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type);
 784                 return -EFAULT;
 785         }
 786         return 0;
 787 }
 788
 789 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle,
 790                                  unsigned int ae,
 791                                  struct icp_qat_uclo_encapme *encap_ae)
 792 {
 793         unsigned int i;
 794         unsigned char ctx_mask;
 795         struct icp_qat_uof_init_regsym *init_regsym;
 796
 797         if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) ==
 798             ICP_QAT_UCLO_MAX_CTX)
 799                 ctx_mask = 0xff;
 800         else
 801                 ctx_mask = 0x55;
 802
 803         for (i = 0; i < encap_ae->init_regsym_num; i++) {
 804                 unsigned int exp_res;
 805
 806                 init_regsym = &encap_ae->init_regsym[i];
 807                 exp_res = init_regsym->value;
 808                 switch (init_regsym->init_type) {
 809                 case ICP_QAT_UOF_INIT_REG:
 810                         qat_uclo_init_reg(handle, ae, ctx_mask,
 811                                           (enum icp_qat_uof_regtype)
 812                                           init_regsym->reg_type,
 813                                           (unsigned short)init_regsym->reg_addr,
 814                                           exp_res);
 815                         break;
 816                 case ICP_QAT_UOF_INIT_REG_CTX:
 817                         /* check if ctx is appropriate for the ctxMode */
 818                         if (!((1 << init_regsym->ctx) & ctx_mask)) {
 819                                 pr_err("QAT: invalid ctx num = 0x%x\n",
 820                                        init_regsym->ctx);
 821                                 return -EINVAL;
 822                         }
 823                         qat_uclo_init_reg(handle, ae,
 824                                           (unsigned char)
 825                                           (1 << init_regsym->ctx),
 826                                           (enum icp_qat_uof_regtype)
 827                                           init_regsym->reg_type,
 828                                           (unsigned short)init_regsym->reg_addr,
 829                                           exp_res);
 830                         break;
 831                 case ICP_QAT_UOF_INIT_EXPR:
 832                         pr_err("QAT: INIT_EXPR feature not supported\n");
 833                         return -EINVAL;
 834                 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP:
 835                         pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n");
 836                         return -EINVAL;
 837                 default:
 838                         break;
 839                 }
 840         }
 841         return 0;
 842 }
 843
 844 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle)
 845 {
 846         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 847         unsigned int s, ae;
 848
 849         if (obj_handle->global_inited)
 850                 return 0;
 851         if (obj_handle->init_mem_tab.entry_num) {
 852                 if (qat_uclo_init_memory(handle)) {
 853                         pr_err("QAT: initialize memory failed\n");
 854                         return -EINVAL;
 855                 }
 856         }
 857         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 858                 for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) {
 859                         if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
 860                                 continue;
 861                         if (qat_uclo_init_reg_sym(handle, ae,
 862                                                   obj_handle->ae_data[ae].
 863                                                   ae_slices[s].encap_image))
 864                                 return -EINVAL;
 865                 }
 866         }
 867         obj_handle->global_inited = 1;
 868         return 0;
 869 }
 870
 871 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle)
 872 {
 873         unsigned char ae, nn_mode, s;
 874         struct icp_qat_uof_image *uof_image;
 875         struct icp_qat_uclo_aedata *ae_data;
 876         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 877
 878         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 879                 if (!test_bit(ae,
 880                               (unsigned long *)&handle->hal_handle->ae_mask))
 881                         continue;
 882                 ae_data = &obj_handle->ae_data[ae];
 883                 for (s = 0; s < min_t(unsigned int, ae_data->slice_num,
 884                                       ICP_QAT_UCLO_MAX_CTX); s++) {
 885                         if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
 886                                 continue;
 887                         uof_image = ae_data->ae_slices[s].encap_image->img_ptr;
 888                         if (qat_hal_set_ae_ctx_mode(handle, ae,
 889                                                     (char)ICP_QAT_CTX_MODE
 890                                                     (uof_image->ae_mode))) {
 891                                 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n");
 892                                 return -EFAULT;
 893                         }
 894                         nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
 895                         if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) {
 896                                 pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
 897                                 return -EFAULT;
 898                         }
 899                         if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0,
 900                                                    (char)ICP_QAT_LOC_MEM0_MODE
 901                                                    (uof_image->ae_mode))) {
 902                                 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n");
 903                                 return -EFAULT;
 904                         }
 905                         if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1,
 906                                                    (char)ICP_QAT_LOC_MEM1_MODE
 907                                                    (uof_image->ae_mode))) {
 908                                 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n");
 909                                 return -EFAULT;
 910                         }
 911                 }
 912         }
 913         return 0;
 914 }
 915
 916 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle)
 917 {
 918         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 919         struct icp_qat_uclo_encapme *image;
 920         int a;
 921
 922         for (a = 0; a < obj_handle->uimage_num; a++) {
 923                 image = &obj_handle->ae_uimage[a];
 924                 image->uwords_num = image->page->beg_addr_p +
 925                                         image->page->micro_words_num;
 926         }
 927 }
 928
 929 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
 930 {
 931         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
 932         unsigned int ae;
 933
 934         obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
 935         obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)
 936                                              obj_handle->obj_hdr->file_buff;
 937         obj_handle->uword_in_bytes = 6;
 938         obj_handle->prod_type = qat_uclo_get_dev_type(handle);
 939         obj_handle->prod_rev = PID_MAJOR_REV |
 940                         (PID_MINOR_REV & handle->hal_handle->revision_id);
 941         if (qat_uclo_check_uof_compat(obj_handle)) {
 942                 pr_err("QAT: UOF incompatible\n");
 943                 return -EINVAL;
 944         }
 945         obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64),
 946                                         GFP_KERNEL);
 947         if (!obj_handle->uword_buf)
 948                 return -ENOMEM;
 949         obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE;
 950         if (!obj_handle->obj_hdr->file_buff ||
 951             !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT,
 952                                     &obj_handle->str_table)) {
 953                 pr_err("QAT: UOF doesn't have effective images\n");
 954                 goto out_err;
 955         }
 956         obj_handle->uimage_num =
 957                 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage,
 958                                     ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX);
 959         if (!obj_handle->uimage_num)
 960                 goto out_err;
 961         if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) {
 962                 pr_err("QAT: Bad object\n");
 963                 goto out_check_uof_aemask_err;
 964         }
 965         qat_uclo_init_uword_num(handle);
 966         qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj,
 967                                    &obj_handle->init_mem_tab);
 968         if (qat_uclo_set_ae_mode(handle))
 969                 goto out_check_uof_aemask_err;
 970         return 0;
 971 out_check_uof_aemask_err:
 972         for (ae = 0; ae < obj_handle->uimage_num; ae++)
 973                 kfree(obj_handle->ae_uimage[ae].page);
 974 out_err:
 975         kfree(obj_handle->uword_buf);
 976         return -EFAULT;
 977 }
 978
 979 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle,
 980                                       struct icp_qat_suof_filehdr *suof_ptr,
 981                                       int suof_size)
 982 {
 983         unsigned int check_sum = 0;
 984         unsigned int min_ver_offset = 0;
 985         struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
 986
 987         suof_handle->file_id = ICP_QAT_SUOF_FID;
 988         suof_handle->suof_buf = (char *)suof_ptr;
 989         suof_handle->suof_size = suof_size;
 990         min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr,
 991                                               min_ver);
 992         check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver,
 993                                                min_ver_offset);
 994         if (check_sum != suof_ptr->check_sum) {
 995                 pr_err("QAT: incorrect SUOF checksum\n");
 996                 return -EINVAL;
 997         }
 998         suof_handle->check_sum = suof_ptr->check_sum;
 999         suof_handle->min_ver = suof_ptr->min_ver;
1000         suof_handle->maj_ver = suof_ptr->maj_ver;
1001         suof_handle->fw_type = suof_ptr->fw_type;
1002         return 0;
1003 }
1004
1005 static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle,
1006                               struct icp_qat_suof_img_hdr *suof_img_hdr,
1007                               struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1008 {
1009         struct icp_qat_simg_ae_mode *ae_mode;
1010         struct icp_qat_suof_objhdr *suof_objhdr;
1011
1012         suof_img_hdr->simg_buf  = (suof_handle->suof_buf +
1013                                    suof_chunk_hdr->offset +
1014                                    sizeof(*suof_objhdr));
1015         suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t)
1016                                   (suof_handle->suof_buf +
1017                                    suof_chunk_hdr->offset))->img_length;
1018
1019         suof_img_hdr->css_header = suof_img_hdr->simg_buf;
1020         suof_img_hdr->css_key = (suof_img_hdr->css_header +
1021                                  sizeof(struct icp_qat_css_hdr));
1022         suof_img_hdr->css_signature = suof_img_hdr->css_key +
1023                                       ICP_QAT_CSS_FWSK_MODULUS_LEN +
1024                                       ICP_QAT_CSS_FWSK_EXPONENT_LEN;
1025         suof_img_hdr->css_simg = suof_img_hdr->css_signature +
1026                                  ICP_QAT_CSS_SIGNATURE_LEN;
1027
1028         ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
1029         suof_img_hdr->ae_mask = ae_mode->ae_mask;
1030         suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name;
1031         suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data;
1032         suof_img_hdr->fw_type = ae_mode->fw_type;
1033 }
1034
1035 static void
1036 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle,
1037                           struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1038 {
1039         char **sym_str = (char **)&suof_handle->sym_str;
1040         unsigned int *sym_size = &suof_handle->sym_size;
1041         struct icp_qat_suof_strtable *str_table_obj;
1042
1043         *sym_size = *(unsigned int *)(uintptr_t)
1044                    (suof_chunk_hdr->offset + suof_handle->suof_buf);
1045         *sym_str = (char *)(uintptr_t)
1046                    (suof_handle->suof_buf + suof_chunk_hdr->offset +
1047                    sizeof(str_table_obj->tab_length));
1048 }
1049
1050 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle,
1051                                       struct icp_qat_suof_img_hdr *img_hdr)
1052 {
1053         struct icp_qat_simg_ae_mode *img_ae_mode = NULL;
1054         unsigned int prod_rev, maj_ver, prod_type;
1055
1056         prod_type = qat_uclo_get_dev_type(handle);
1057         img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg;
1058         prod_rev = PID_MAJOR_REV |
1059                          (PID_MINOR_REV & handle->hal_handle->revision_id);
1060         if (img_ae_mode->dev_type != prod_type) {
1061                 pr_err("QAT: incompatible product type %x\n",
1062                        img_ae_mode->dev_type);
1063                 return -EINVAL;
1064         }
1065         maj_ver = prod_rev & 0xff;
1066         if ((maj_ver > img_ae_mode->devmax_ver) ||
1067             (maj_ver < img_ae_mode->devmin_ver)) {
1068                 pr_err("QAT: incompatible device majver 0x%x\n", maj_ver);
1069                 return -EINVAL;
1070         }
1071         return 0;
1072 }
1073
1074 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle)
1075 {
1076         struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1077
1078         kfree(sobj_handle->img_table.simg_hdr);
1079         sobj_handle->img_table.simg_hdr = NULL;
1080         kfree(handle->sobj_handle);
1081         handle->sobj_handle = NULL;
1082 }
1083
1084 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr,
1085                               unsigned int img_id, unsigned int num_simgs)
1086 {
1087         struct icp_qat_suof_img_hdr img_header;
1088
1089         if (img_id != num_simgs - 1) {
1090                 memcpy(&img_header, &suof_img_hdr[num_simgs - 1],
1091                        sizeof(*suof_img_hdr));
1092                 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id],
1093                        sizeof(*suof_img_hdr));
1094                 memcpy(&suof_img_hdr[img_id], &img_header,
1095                        sizeof(*suof_img_hdr));
1096         }
1097 }
1098
1099 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle,
1100                              struct icp_qat_suof_filehdr *suof_ptr,
1101                              int suof_size)
1102 {
1103         struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1104         struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
1105         struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
1106         int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE;
1107         unsigned int i = 0;
1108         struct icp_qat_suof_img_hdr img_header;
1109
1110         if (!suof_ptr || (suof_size == 0)) {
1111                 pr_err("QAT: input parameter SUOF pointer/size is NULL\n");
1112                 return -EINVAL;
1113         }
1114         if (qat_uclo_check_suof_format(suof_ptr))
1115                 return -EINVAL;
1116         ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size);
1117         if (ret)
1118                 return ret;
1119         suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)
1120                          ((uintptr_t)suof_ptr + sizeof(*suof_ptr));
1121
1122         qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr);
1123         suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1;
1124
1125         if (suof_handle->img_table.num_simgs != 0) {
1126                 suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs,
1127                                        sizeof(img_header),
1128                                        GFP_KERNEL);
1129                 if (!suof_img_hdr)
1130                         return -ENOMEM;
1131                 suof_handle->img_table.simg_hdr = suof_img_hdr;
1132         }
1133
1134         for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
1135                 qat_uclo_map_simg(handle->sobj_handle, &suof_img_hdr[i],
1136                                   &suof_chunk_hdr[1 + i]);
1137                 ret = qat_uclo_check_simg_compat(handle,
1138                                                  &suof_img_hdr[i]);
1139                 if (ret)
1140                         return ret;
1141                 if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
1142                         ae0_img = i;
1143         }
1144         qat_uclo_tail_img(suof_img_hdr, ae0_img,
1145                           suof_handle->img_table.num_simgs);
1146         return 0;
1147 }
1148
1149 #define ADD_ADDR(high, low)  ((((u64)high) << 32) + low)
1150 #define BITS_IN_DWORD 32
1151
1152 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
1153                             struct icp_qat_fw_auth_desc *desc)
1154 {
1155         unsigned int fcu_sts, retry = 0;
1156         u64 bus_addr;
1157
1158         bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low)
1159                            - sizeof(struct icp_qat_auth_chunk);
1160         SET_CAP_CSR(handle, FCU_DRAM_ADDR_HI, (bus_addr >> BITS_IN_DWORD));
1161         SET_CAP_CSR(handle, FCU_DRAM_ADDR_LO, bus_addr);
1162         SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_AUTH);
1163
1164         do {
1165                 msleep(FW_AUTH_WAIT_PERIOD);
1166                 fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
1167                 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL)
1168                         goto auth_fail;
1169                 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1))
1170                         if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE)
1171                                 return 0;
1172         } while (retry++ < FW_AUTH_MAX_RETRY);
1173 auth_fail:
1174         pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n",
1175                fcu_sts & FCU_AUTH_STS_MASK, retry);
1176         return -EINVAL;
1177 }
1178
1179 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
1180                                struct icp_firml_dram_desc *dram_desc,
1181                                unsigned int size)
1182 {
1183         void *vptr;
1184         dma_addr_t ptr;
1185
1186         vptr = dma_alloc_coherent(&handle->pci_dev->dev,
1187                                   size, &ptr, GFP_KERNEL);
1188         if (!vptr)
1189                 return -ENOMEM;
1190         dram_desc->dram_base_addr_v = vptr;
1191         dram_desc->dram_bus_addr = ptr;
1192         dram_desc->dram_size = size;
1193         return 0;
1194 }
1195
1196 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle,
1197                                struct icp_firml_dram_desc *dram_desc)
1198 {
1199         dma_free_coherent(&handle->pci_dev->dev,
1200                           (size_t)(dram_desc->dram_size),
1201                           (dram_desc->dram_base_addr_v),
1202                           dram_desc->dram_bus_addr);
1203         memset(dram_desc, 0, sizeof(*dram_desc));
1204 }
1205
1206 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle,
1207                                    struct icp_qat_fw_auth_desc **desc)
1208 {
1209         struct icp_firml_dram_desc dram_desc;
1210
1211         dram_desc.dram_base_addr_v = *desc;
1212         dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *)
1213                                    (*desc))->chunk_bus_addr;
1214         dram_desc.dram_size = ((struct icp_qat_auth_chunk *)
1215                                (*desc))->chunk_size;
1216         qat_uclo_simg_free(handle, &dram_desc);
1217 }
1218
1219 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
1220                                 char *image, unsigned int size,
1221                                 struct icp_qat_fw_auth_desc **desc)
1222 {
1223         struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image;
1224         struct icp_qat_fw_auth_desc *auth_desc;
1225         struct icp_qat_auth_chunk *auth_chunk;
1226         u64 virt_addr,  bus_addr, virt_base;
1227         unsigned int length, simg_offset = sizeof(*auth_chunk);
1228         struct icp_firml_dram_desc img_desc;
1229
1230         if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
1231                 pr_err("QAT: error, input image size overflow %d\n", size);
1232                 return -EINVAL;
1233         }
1234         length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ?
1235                  ICP_QAT_CSS_AE_SIMG_LEN + simg_offset :
1236                  size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset;
1237         if (qat_uclo_simg_alloc(handle, &img_desc, length)) {
1238                 pr_err("QAT: error, allocate continuous dram fail\n");
1239                 return -ENOMEM;
1240         }
1241
1242         auth_chunk = img_desc.dram_base_addr_v;
1243         auth_chunk->chunk_size = img_desc.dram_size;
1244         auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr;
1245         virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset;
1246         bus_addr  = img_desc.dram_bus_addr + simg_offset;
1247         auth_desc = img_desc.dram_base_addr_v;
1248         auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1249         auth_desc->css_hdr_low = (unsigned int)bus_addr;
1250         virt_addr = virt_base;
1251
1252         memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr));
1253         /* pub key */
1254         bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) +
1255                            sizeof(*css_hdr);
1256         virt_addr = virt_addr + sizeof(*css_hdr);
1257
1258         auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1259         auth_desc->fwsk_pub_low = (unsigned int)bus_addr;
1260
1261         memcpy((void *)(uintptr_t)virt_addr,
1262                (void *)(image + sizeof(*css_hdr)),
1263                ICP_QAT_CSS_FWSK_MODULUS_LEN);
1264         /* padding */
1265         memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN),
1266                0, ICP_QAT_CSS_FWSK_PAD_LEN);
1267
1268         /* exponent */
1269         memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN +
1270                ICP_QAT_CSS_FWSK_PAD_LEN),
1271                (void *)(image + sizeof(*css_hdr) +
1272                         ICP_QAT_CSS_FWSK_MODULUS_LEN),
1273                sizeof(unsigned int));
1274
1275         /* signature */
1276         bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high,
1277                             auth_desc->fwsk_pub_low) +
1278                    ICP_QAT_CSS_FWSK_PUB_LEN;
1279         virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN;
1280         auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1281         auth_desc->signature_low = (unsigned int)bus_addr;
1282
1283         memcpy((void *)(uintptr_t)virt_addr,
1284                (void *)(image + sizeof(*css_hdr) +
1285                ICP_QAT_CSS_FWSK_MODULUS_LEN +
1286                ICP_QAT_CSS_FWSK_EXPONENT_LEN),
1287                ICP_QAT_CSS_SIGNATURE_LEN);
1288
1289         bus_addr = ADD_ADDR(auth_desc->signature_high,
1290                             auth_desc->signature_low) +
1291                    ICP_QAT_CSS_SIGNATURE_LEN;
1292         virt_addr += ICP_QAT_CSS_SIGNATURE_LEN;
1293
1294         auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1295         auth_desc->img_low = (unsigned int)bus_addr;
1296         auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET;
1297         memcpy((void *)(uintptr_t)virt_addr,
1298                (void *)(image + ICP_QAT_AE_IMG_OFFSET),
1299                auth_desc->img_len);
1300         virt_addr = virt_base;
1301         /* AE firmware */
1302         if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type ==
1303             CSS_AE_FIRMWARE) {
1304                 auth_desc->img_ae_mode_data_high = auth_desc->img_high;
1305                 auth_desc->img_ae_mode_data_low = auth_desc->img_low;
1306                 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high,
1307                                     auth_desc->img_ae_mode_data_low) +
1308                            sizeof(struct icp_qat_simg_ae_mode);
1309
1310                 auth_desc->img_ae_init_data_high = (unsigned int)
1311                                                  (bus_addr >> BITS_IN_DWORD);
1312                 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr;
1313                 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
1314                 auth_desc->img_ae_insts_high = (unsigned int)
1315                                              (bus_addr >> BITS_IN_DWORD);
1316                 auth_desc->img_ae_insts_low = (unsigned int)bus_addr;
1317         } else {
1318                 auth_desc->img_ae_insts_high = auth_desc->img_high;
1319                 auth_desc->img_ae_insts_low = auth_desc->img_low;
1320         }
1321         *desc = auth_desc;
1322         return 0;
1323 }
1324
1325 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
1326                             struct icp_qat_fw_auth_desc *desc)
1327 {
1328         unsigned int i;
1329         unsigned int fcu_sts;
1330         struct icp_qat_simg_ae_mode *virt_addr;
1331         unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS;
1332
1333         virt_addr = (void *)((uintptr_t)desc +
1334                      sizeof(struct icp_qat_auth_chunk) +
1335                      sizeof(struct icp_qat_css_hdr) +
1336                      ICP_QAT_CSS_FWSK_PUB_LEN +
1337                      ICP_QAT_CSS_SIGNATURE_LEN);
1338         for (i = 0; i < handle->hal_handle->ae_max_num; i++) {
1339                 int retry = 0;
1340
1341                 if (!((virt_addr->ae_mask >> i) & 0x1))
1342                         continue;
1343                 if (qat_hal_check_ae_active(handle, i)) {
1344                         pr_err("QAT: AE %d is active\n", i);
1345                         return -EINVAL;
1346                 }
1347                 SET_CAP_CSR(handle, FCU_CONTROL,
1348                             (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS)));
1349
1350                 do {
1351                         msleep(FW_AUTH_WAIT_PERIOD);
1352                         fcu_sts = GET_CAP_CSR(handle, FCU_STATUS);
1353                         if (((fcu_sts & FCU_AUTH_STS_MASK) ==
1354                             FCU_STS_LOAD_DONE) &&
1355                             ((fcu_sts >> fcu_loaded_ae_pos) & (1 << i)))
1356                                 break;
1357                 } while (retry++ < FW_AUTH_MAX_RETRY);
1358                 if (retry > FW_AUTH_MAX_RETRY) {
1359                         pr_err("QAT: firmware load failed timeout %x\n", retry);
1360                         return -EINVAL;
1361                 }
1362         }
1363         return 0;
1364 }
1365
1366 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle,
1367                                  void *addr_ptr, int mem_size)
1368 {
1369         struct icp_qat_suof_handle *suof_handle;
1370
1371         suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL);
1372         if (!suof_handle)
1373                 return -ENOMEM;
1374         handle->sobj_handle = suof_handle;
1375         if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) {
1376                 qat_uclo_del_suof(handle);
1377                 pr_err("QAT: map SUOF failed\n");
1378                 return -EINVAL;
1379         }
1380         return 0;
1381 }
1382
1383 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
1384                        void *addr_ptr, int mem_size)
1385 {
1386         struct icp_qat_fw_auth_desc *desc = NULL;
1387         int status = 0;
1388
1389         if (handle->fw_auth) {
1390                 if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc))
1391                         status = qat_uclo_auth_fw(handle, desc);
1392                 qat_uclo_ummap_auth_fw(handle, &desc);
1393         } else {
1394                 if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID) {
1395                         pr_err("QAT: C3XXX doesn't support unsigned MMP\n");
1396                         return -EINVAL;
1397                 }
1398                 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size);
1399         }
1400         return status;
1401 }
1402
1403 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle,
1404                                 void *addr_ptr, int mem_size)
1405 {
1406         struct icp_qat_uof_filehdr *filehdr;
1407         struct icp_qat_uclo_objhandle *objhdl;
1408
1409         objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL);
1410         if (!objhdl)
1411                 return -ENOMEM;
1412         objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL);
1413         if (!objhdl->obj_buf)
1414                 goto out_objbuf_err;
1415         filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf;
1416         if (qat_uclo_check_uof_format(filehdr))
1417                 goto out_objhdr_err;
1418         objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr,
1419                                              ICP_QAT_UOF_OBJS);
1420         if (!objhdl->obj_hdr) {
1421                 pr_err("QAT: object file chunk is null\n");
1422                 goto out_objhdr_err;
1423         }
1424         handle->obj_handle = objhdl;
1425         if (qat_uclo_parse_uof_obj(handle))
1426                 goto out_overlay_obj_err;
1427         return 0;
1428
1429 out_overlay_obj_err:
1430         handle->obj_handle = NULL;
1431         kfree(objhdl->obj_hdr);
1432 out_objhdr_err:
1433         kfree(objhdl->obj_buf);
1434 out_objbuf_err:
1435         kfree(objhdl);
1436         return -ENOMEM;
1437 }
1438
1439 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
1440                      void *addr_ptr, int mem_size)
1441 {
1442         BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >=
1443                      (sizeof(handle->hal_handle->ae_mask) * 8));
1444
1445         if (!handle || !addr_ptr || mem_size < 24)
1446                 return -EINVAL;
1447
1448         return (handle->fw_auth) ?
1449                         qat_uclo_map_suof_obj(handle, addr_ptr, mem_size) :
1450                         qat_uclo_map_uof_obj(handle, addr_ptr, mem_size);
1451 }
1452
1453 void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle)
1454 {
1455         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1456         unsigned int a;
1457
1458         if (handle->sobj_handle)
1459                 qat_uclo_del_suof(handle);
1460         if (!obj_handle)
1461                 return;
1462
1463         kfree(obj_handle->uword_buf);
1464         for (a = 0; a < obj_handle->uimage_num; a++)
1465                 kfree(obj_handle->ae_uimage[a].page);
1466
1467         for (a = 0; a < handle->hal_handle->ae_max_num; a++)
1468                 qat_uclo_free_ae_data(&obj_handle->ae_data[a]);
1469
1470         kfree(obj_handle->obj_hdr);
1471         kfree(obj_handle->obj_buf);
1472         kfree(obj_handle);
1473         handle->obj_handle = NULL;
1474 }
1475
1476 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle,
1477                                  struct icp_qat_uclo_encap_page *encap_page,
1478                                  u64 *uword, unsigned int addr_p,
1479                                  unsigned int raddr, u64 fill)
1480 {
1481         u64 uwrd = 0;
1482         unsigned int i;
1483
1484         if (!encap_page) {
1485                 *uword = fill;
1486                 return;
1487         }
1488         for (i = 0; i < encap_page->uwblock_num; i++) {
1489                 if (raddr >= encap_page->uwblock[i].start_addr &&
1490                     raddr <= encap_page->uwblock[i].start_addr +
1491                     encap_page->uwblock[i].words_num - 1) {
1492                         raddr -= encap_page->uwblock[i].start_addr;
1493                         raddr *= obj_handle->uword_in_bytes;
1494                         memcpy(&uwrd, (void *)(((uintptr_t)
1495                                encap_page->uwblock[i].micro_words) + raddr),
1496                                obj_handle->uword_in_bytes);
1497                         uwrd = uwrd & 0xbffffffffffull;
1498                 }
1499         }
1500         *uword = uwrd;
1501         if (*uword == INVLD_UWORD)
1502                 *uword = fill;
1503 }
1504
1505 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle,
1506                                         struct icp_qat_uclo_encap_page
1507                                         *encap_page, unsigned int ae)
1508 {
1509         unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen;
1510         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1511         u64 fill_pat;
1512
1513         /* load the page starting at appropriate ustore address */
1514         /* get fill-pattern from an image -- they are all the same */
1515         memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern,
1516                sizeof(u64));
1517         uw_physical_addr = encap_page->beg_addr_p;
1518         uw_relative_addr = 0;
1519         words_num = encap_page->micro_words_num;
1520         while (words_num) {
1521                 if (words_num < UWORD_CPYBUF_SIZE)
1522                         cpylen = words_num;
1523                 else
1524                         cpylen = UWORD_CPYBUF_SIZE;
1525
1526                 /* load the buffer */
1527                 for (i = 0; i < cpylen; i++)
1528                         qat_uclo_fill_uwords(obj_handle, encap_page,
1529                                              &obj_handle->uword_buf[i],
1530                                              uw_physical_addr + i,
1531                                              uw_relative_addr + i, fill_pat);
1532
1533                 /* copy the buffer to ustore */
1534                 qat_hal_wr_uwords(handle, (unsigned char)ae,
1535                                   uw_physical_addr, cpylen,
1536                                   obj_handle->uword_buf);
1537
1538                 uw_physical_addr += cpylen;
1539                 uw_relative_addr += cpylen;
1540                 words_num -= cpylen;
1541         }
1542 }
1543
1544 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle,
1545                                     struct icp_qat_uof_image *image)
1546 {
1547         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1548         unsigned int ctx_mask, s;
1549         struct icp_qat_uclo_page *page;
1550         unsigned char ae;
1551         int ctx;
1552
1553         if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX)
1554                 ctx_mask = 0xff;
1555         else
1556                 ctx_mask = 0x55;
1557         /* load the default page and set assigned CTX PC
1558          * to the entrypoint address */
1559         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
1560                 if (!test_bit(ae, (unsigned long *)&image->ae_assigned))
1561                         continue;
1562                 /* find the slice to which this image is assigned */
1563                 for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) {
1564                         if (image->ctx_assigned & obj_handle->ae_data[ae].
1565                             ae_slices[s].ctx_mask_assigned)
1566                                 break;
1567                 }
1568                 if (s >= obj_handle->ae_data[ae].slice_num)
1569                         continue;
1570                 page = obj_handle->ae_data[ae].ae_slices[s].page;
1571                 if (!page->encap_page->def_page)
1572                         continue;
1573                 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae);
1574
1575                 page = obj_handle->ae_data[ae].ae_slices[s].page;
1576                 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++)
1577                         obj_handle->ae_data[ae].ae_slices[s].cur_page[ctx] =
1578                                         (ctx_mask & (1 << ctx)) ? page : NULL;
1579                 qat_hal_set_live_ctx(handle, (unsigned char)ae,
1580                                      image->ctx_assigned);
1581                 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned,
1582                                image->entry_address);
1583         }
1584 }
1585
1586 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
1587 {
1588         unsigned int i;
1589         struct icp_qat_fw_auth_desc *desc = NULL;
1590         struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1591         struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr;
1592
1593         for (i = 0; i < sobj_handle->img_table.num_simgs; i++) {
1594                 if (qat_uclo_map_auth_fw(handle,
1595                                          (char *)simg_hdr[i].simg_buf,
1596                                          (unsigned int)
1597                                          (simg_hdr[i].simg_len),
1598                                          &desc))
1599                         goto wr_err;
1600                 if (qat_uclo_auth_fw(handle, desc))
1601                         goto wr_err;
1602                 if (qat_uclo_load_fw(handle, desc))
1603                         goto wr_err;
1604                 qat_uclo_ummap_auth_fw(handle, &desc);
1605         }
1606         return 0;
1607 wr_err:
1608         qat_uclo_ummap_auth_fw(handle, &desc);
1609         return -EINVAL;
1610 }
1611
1612 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle)
1613 {
1614         struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1615         unsigned int i;
1616
1617         if (qat_uclo_init_globals(handle))
1618                 return -EINVAL;
1619         for (i = 0; i < obj_handle->uimage_num; i++) {
1620                 if (!obj_handle->ae_uimage[i].img_ptr)
1621                         return -EINVAL;
1622                 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i]))
1623                         return -EINVAL;
1624                 qat_uclo_wr_uimage_page(handle,
1625                                         obj_handle->ae_uimage[i].img_ptr);
1626         }
1627         return 0;
1628 }
1629
1630 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle)
1631 {
1632         return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) :
1633                                    qat_uclo_wr_uof_img(handle);
1634 }