drivers/scsi/mpi3mr/mpi3mr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for Broadcom MPI3 Storage Controllers
   4  *
   5  * Copyright (C) 2017-2021 Broadcom Inc.
   6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7  *
   8  */
   9
  10 #include "mpi3mr.h"
  11 #include <linux/io-64-nonatomic-lo-hi.h>
  12
  13 #if defined(writeq) && defined(CONFIG_64BIT)
  14 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  15 {
  16         writeq(b, addr);
  17 }
  18 #else
  19 static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  20 {
  21         __u64 data_out = b;
  22
  23         writel((u32)(data_out), addr);
  24         writel((u32)(data_out >> 32), (addr + 4));
  25 }
  26 #endif
  27
  28 static inline bool
  29 mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  30 {
  31         u16 pi, ci, max_entries;
  32         bool is_qfull = false;
  33
  34         pi = op_req_q->pi;
  35         ci = READ_ONCE(op_req_q->ci);
  36         max_entries = op_req_q->num_requests;
  37
  38         if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  39                 is_qfull = true;
  40
  41         return is_qfull;
  42 }
  43
  44 static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  45 {
  46         u16 i, max_vectors;
  47
  48         max_vectors = mrioc->intr_info_count;
  49
  50         for (i = 0; i < max_vectors; i++)
  51                 synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  52 }
  53
  54 void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  55 {
  56         mrioc->intr_enabled = 0;
  57         mpi3mr_sync_irqs(mrioc);
  58 }
  59
  60 void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  61 {
  62         mrioc->intr_enabled = 1;
  63 }
  64
  65 static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  66 {
  67         u16 i;
  68
  69         mpi3mr_ioc_disable_intr(mrioc);
  70
  71         if (!mrioc->intr_info)
  72                 return;
  73
  74         for (i = 0; i < mrioc->intr_info_count; i++)
  75                 free_irq(pci_irq_vector(mrioc->pdev, i),
  76                     (mrioc->intr_info + i));
  77
  78         kfree(mrioc->intr_info);
  79         mrioc->intr_info = NULL;
  80         mrioc->intr_info_count = 0;
  81         pci_free_irq_vectors(mrioc->pdev);
  82 }
  83
  84 void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  85         dma_addr_t dma_addr)
  86 {
  87         struct mpi3_sge_common *sgel = paddr;
  88
  89         sgel->flags = flags;
  90         sgel->length = cpu_to_le32(length);
  91         sgel->address = cpu_to_le64(dma_addr);
  92 }
  93
  94 void mpi3mr_build_zero_len_sge(void *paddr)
  95 {
  96         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
  97
  98         mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
  99 }
 100
 101 void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 102         dma_addr_t phys_addr)
 103 {
 104         if (!phys_addr)
 105                 return NULL;
 106
 107         if ((phys_addr < mrioc->reply_buf_dma) ||
 108             (phys_addr > mrioc->reply_buf_dma_max_address))
 109                 return NULL;
 110
 111         return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 112 }
 113
 114 void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 115         dma_addr_t phys_addr)
 116 {
 117         if (!phys_addr)
 118                 return NULL;
 119
 120         return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 121 }
 122
 123 static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 124         u64 reply_dma)
 125 {
 126         u32 old_idx = 0;
 127
 128         spin_lock(&mrioc->reply_free_queue_lock);
 129         old_idx  =  mrioc->reply_free_queue_host_index;
 130         mrioc->reply_free_queue_host_index = (
 131             (mrioc->reply_free_queue_host_index ==
 132             (mrioc->reply_free_qsz - 1)) ? 0 :
 133             (mrioc->reply_free_queue_host_index + 1));
 134         mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 135         writel(mrioc->reply_free_queue_host_index,
 136             &mrioc->sysif_regs->reply_free_host_index);
 137         spin_unlock(&mrioc->reply_free_queue_lock);
 138 }
 139
 140 void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 141         u64 sense_buf_dma)
 142 {
 143         u32 old_idx = 0;
 144
 145         spin_lock(&mrioc->sbq_lock);
 146         old_idx  =  mrioc->sbq_host_index;
 147         mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 148             (mrioc->sense_buf_q_sz - 1)) ? 0 :
 149             (mrioc->sbq_host_index + 1));
 150         mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 151         writel(mrioc->sbq_host_index,
 152             &mrioc->sysif_regs->sense_buffer_free_host_index);
 153         spin_unlock(&mrioc->sbq_lock);
 154 }
 155
 156 static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 157         struct mpi3_event_notification_reply *event_reply)
 158 {
 159         char *desc = NULL;
 160         u16 event;
 161
 162         event = event_reply->event;
 163
 164         switch (event) {
 165         case MPI3_EVENT_LOG_DATA:
 166                 desc = "Log Data";
 167                 break;
 168         case MPI3_EVENT_CHANGE:
 169                 desc = "Event Change";
 170                 break;
 171         case MPI3_EVENT_GPIO_INTERRUPT:
 172                 desc = "GPIO Interrupt";
 173                 break;
 174         case MPI3_EVENT_TEMP_THRESHOLD:
 175                 desc = "Temperature Threshold";
 176                 break;
 177         case MPI3_EVENT_CABLE_MGMT:
 178                 desc = "Cable Management";
 179                 break;
 180         case MPI3_EVENT_ENERGY_PACK_CHANGE:
 181                 desc = "Energy Pack Change";
 182                 break;
 183         case MPI3_EVENT_DEVICE_ADDED:
 184         {
 185                 struct mpi3_device_page0 *event_data =
 186                     (struct mpi3_device_page0 *)event_reply->event_data;
 187                 ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 188                     event_data->dev_handle, event_data->device_form);
 189                 return;
 190         }
 191         case MPI3_EVENT_DEVICE_INFO_CHANGED:
 192         {
 193                 struct mpi3_device_page0 *event_data =
 194                     (struct mpi3_device_page0 *)event_reply->event_data;
 195                 ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 196                     event_data->dev_handle, event_data->device_form);
 197                 return;
 198         }
 199         case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 200         {
 201                 struct mpi3_event_data_device_status_change *event_data =
 202                     (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 203                 ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 204                     event_data->dev_handle, event_data->reason_code);
 205                 return;
 206         }
 207         case MPI3_EVENT_SAS_DISCOVERY:
 208         {
 209                 struct mpi3_event_data_sas_discovery *event_data =
 210                     (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 211                 ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 212                     (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 213                     "start" : "stop",
 214                     le32_to_cpu(event_data->discovery_status));
 215                 return;
 216         }
 217         case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 218                 desc = "SAS Broadcast Primitive";
 219                 break;
 220         case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 221                 desc = "SAS Notify Primitive";
 222                 break;
 223         case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 224                 desc = "SAS Init Device Status Change";
 225                 break;
 226         case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 227                 desc = "SAS Init Table Overflow";
 228                 break;
 229         case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 230                 desc = "SAS Topology Change List";
 231                 break;
 232         case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 233                 desc = "Enclosure Device Status Change";
 234                 break;
 235         case MPI3_EVENT_HARD_RESET_RECEIVED:
 236                 desc = "Hard Reset Received";
 237                 break;
 238         case MPI3_EVENT_SAS_PHY_COUNTER:
 239                 desc = "SAS PHY Counter";
 240                 break;
 241         case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 242                 desc = "SAS Device Discovery Error";
 243                 break;
 244         case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 245                 desc = "PCIE Topology Change List";
 246                 break;
 247         case MPI3_EVENT_PCIE_ENUMERATION:
 248         {
 249                 struct mpi3_event_data_pcie_enumeration *event_data =
 250                     (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 251                 ioc_info(mrioc, "PCIE Enumeration: (%s)",
 252                     (event_data->reason_code ==
 253                     MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 254                 if (event_data->enumeration_status)
 255                         ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 256                             le32_to_cpu(event_data->enumeration_status));
 257                 return;
 258         }
 259         case MPI3_EVENT_PREPARE_FOR_RESET:
 260                 desc = "Prepare For Reset";
 261                 break;
 262         }
 263
 264         if (!desc)
 265                 return;
 266
 267         ioc_info(mrioc, "%s\n", desc);
 268 }
 269
 270 static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 271         struct mpi3_default_reply *def_reply)
 272 {
 273         struct mpi3_event_notification_reply *event_reply =
 274             (struct mpi3_event_notification_reply *)def_reply;
 275
 276         mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 277         mpi3mr_print_event_data(mrioc, event_reply);
 278         mpi3mr_os_handle_events(mrioc, event_reply);
 279 }
 280
 281 static struct mpi3mr_drv_cmd *
 282 mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 283         struct mpi3_default_reply *def_reply)
 284 {
 285         u16 idx;
 286
 287         switch (host_tag) {
 288         case MPI3MR_HOSTTAG_INITCMDS:
 289                 return &mrioc->init_cmds;
 290         case MPI3MR_HOSTTAG_BLK_TMS:
 291                 return &mrioc->host_tm_cmds;
 292         case MPI3MR_HOSTTAG_INVALID:
 293                 if (def_reply && def_reply->function ==
 294                     MPI3_FUNCTION_EVENT_NOTIFICATION)
 295                         mpi3mr_handle_events(mrioc, def_reply);
 296                 return NULL;
 297         default:
 298                 break;
 299         }
 300         if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 301             host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 302                 idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 303                 return &mrioc->dev_rmhs_cmds[idx];
 304         }
 305
 306         return NULL;
 307 }
 308
 309 static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 310         struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 311 {
 312         u16 reply_desc_type, host_tag = 0;
 313         u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 314         u32 ioc_loginfo = 0;
 315         struct mpi3_status_reply_descriptor *status_desc;
 316         struct mpi3_address_reply_descriptor *addr_desc;
 317         struct mpi3_success_reply_descriptor *success_desc;
 318         struct mpi3_default_reply *def_reply = NULL;
 319         struct mpi3mr_drv_cmd *cmdptr = NULL;
 320         struct mpi3_scsi_io_reply *scsi_reply;
 321         u8 *sense_buf = NULL;
 322
 323         *reply_dma = 0;
 324         reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 325             MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 326         switch (reply_desc_type) {
 327         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 328                 status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 329                 host_tag = le16_to_cpu(status_desc->host_tag);
 330                 ioc_status = le16_to_cpu(status_desc->ioc_status);
 331                 if (ioc_status &
 332                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 333                         ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 334                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 335                 break;
 336         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 337                 addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 338                 *reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 339                 def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 340                 if (!def_reply)
 341                         goto out;
 342                 host_tag = le16_to_cpu(def_reply->host_tag);
 343                 ioc_status = le16_to_cpu(def_reply->ioc_status);
 344                 if (ioc_status &
 345                     MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 346                         ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 347                 ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 348                 if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 349                         scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 350                         sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 351                             le64_to_cpu(scsi_reply->sense_data_buffer_address));
 352                 }
 353                 break;
 354         case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 355                 success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 356                 host_tag = le16_to_cpu(success_desc->host_tag);
 357                 break;
 358         default:
 359                 break;
 360         }
 361
 362         cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 363         if (cmdptr) {
 364                 if (cmdptr->state & MPI3MR_CMD_PENDING) {
 365                         cmdptr->state |= MPI3MR_CMD_COMPLETE;
 366                         cmdptr->ioc_loginfo = ioc_loginfo;
 367                         cmdptr->ioc_status = ioc_status;
 368                         cmdptr->state &= ~MPI3MR_CMD_PENDING;
 369                         if (def_reply) {
 370                                 cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 371                                 memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 372                                     mrioc->facts.reply_sz);
 373                         }
 374                         if (cmdptr->is_waiting) {
 375                                 complete(&cmdptr->done);
 376                                 cmdptr->is_waiting = 0;
 377                         } else if (cmdptr->callback)
 378                                 cmdptr->callback(mrioc, cmdptr);
 379                 }
 380         }
 381 out:
 382         if (sense_buf)
 383                 mpi3mr_repost_sense_buf(mrioc,
 384                     le64_to_cpu(scsi_reply->sense_data_buffer_address));
 385 }
 386
 387 static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 388 {
 389         u32 exp_phase = mrioc->admin_reply_ephase;
 390         u32 admin_reply_ci = mrioc->admin_reply_ci;
 391         u32 num_admin_replies = 0;
 392         u64 reply_dma = 0;
 393         struct mpi3_default_reply_descriptor *reply_desc;
 394
 395         reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 396             admin_reply_ci;
 397
 398         if ((le16_to_cpu(reply_desc->reply_flags) &
 399             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 400                 return 0;
 401
 402         do {
 403                 mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 404                 mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 405                 if (reply_dma)
 406                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 407                 num_admin_replies++;
 408                 if (++admin_reply_ci == mrioc->num_admin_replies) {
 409                         admin_reply_ci = 0;
 410                         exp_phase ^= 1;
 411                 }
 412                 reply_desc =
 413                     (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 414                     admin_reply_ci;
 415                 if ((le16_to_cpu(reply_desc->reply_flags) &
 416                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 417                         break;
 418         } while (1);
 419
 420         writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 421         mrioc->admin_reply_ci = admin_reply_ci;
 422         mrioc->admin_reply_ephase = exp_phase;
 423
 424         return num_admin_replies;
 425 }
 426
 427 /**
 428  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 429  *      queue's consumer index from operational reply descriptor queue.
 430  * @op_reply_q: op_reply_qinfo object
 431  * @reply_ci: operational reply descriptor's queue consumer index
 432  *
 433  * Returns reply descriptor frame address
 434  */
 435 static inline struct mpi3_default_reply_descriptor *
 436 mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 437 {
 438         void *segment_base_addr;
 439         struct segments *segments = op_reply_q->q_segments;
 440         struct mpi3_default_reply_descriptor *reply_desc = NULL;
 441
 442         segment_base_addr =
 443             segments[reply_ci / op_reply_q->segment_qd].segment;
 444         reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 445             (reply_ci % op_reply_q->segment_qd);
 446         return reply_desc;
 447 }
 448
 449 static int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 450         struct mpi3mr_intr_info *intr_info)
 451 {
 452         struct op_reply_qinfo *op_reply_q = intr_info->op_reply_q;
 453         struct op_req_qinfo *op_req_q;
 454         u32 exp_phase;
 455         u32 reply_ci;
 456         u32 num_op_reply = 0;
 457         u64 reply_dma = 0;
 458         struct mpi3_default_reply_descriptor *reply_desc;
 459         u16 req_q_idx = 0, reply_qidx;
 460
 461         reply_qidx = op_reply_q->qid - 1;
 462
 463         if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 464                 return 0;
 465
 466         exp_phase = op_reply_q->ephase;
 467         reply_ci = op_reply_q->ci;
 468
 469         reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 470         if ((le16_to_cpu(reply_desc->reply_flags) &
 471             MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 472                 atomic_dec(&op_reply_q->in_use);
 473                 return 0;
 474         }
 475
 476         do {
 477                 req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 478                 op_req_q = &mrioc->req_qinfo[req_q_idx];
 479
 480                 WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 481                 mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 482                     reply_qidx);
 483                 atomic_dec(&op_reply_q->pend_ios);
 484                 if (reply_dma)
 485                         mpi3mr_repost_reply_buf(mrioc, reply_dma);
 486                 num_op_reply++;
 487
 488                 if (++reply_ci == op_reply_q->num_replies) {
 489                         reply_ci = 0;
 490                         exp_phase ^= 1;
 491                 }
 492
 493                 reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 494
 495                 if ((le16_to_cpu(reply_desc->reply_flags) &
 496                     MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 497                         break;
 498                 /*
 499                  * Exit completion loop to avoid CPU lockup
 500                  * Ensure remaining completion happens from threaded ISR.
 501                  */
 502                 if (num_op_reply > mrioc->max_host_ios) {
 503                         intr_info->op_reply_q->enable_irq_poll = true;
 504                         break;
 505                 }
 506
 507         } while (1);
 508
 509         writel(reply_ci,
 510             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 511         op_reply_q->ci = reply_ci;
 512         op_reply_q->ephase = exp_phase;
 513
 514         atomic_dec(&op_reply_q->in_use);
 515         return num_op_reply;
 516 }
 517
 518 static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 519 {
 520         struct mpi3mr_intr_info *intr_info = privdata;
 521         struct mpi3mr_ioc *mrioc;
 522         u16 midx;
 523         u32 num_admin_replies = 0, num_op_reply = 0;
 524
 525         if (!intr_info)
 526                 return IRQ_NONE;
 527
 528         mrioc = intr_info->mrioc;
 529
 530         if (!mrioc->intr_enabled)
 531                 return IRQ_NONE;
 532
 533         midx = intr_info->msix_index;
 534
 535         if (!midx)
 536                 num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 537         if (intr_info->op_reply_q)
 538                 num_op_reply = mpi3mr_process_op_reply_q(mrioc, intr_info);
 539
 540         if (num_admin_replies || num_op_reply)
 541                 return IRQ_HANDLED;
 542         else
 543                 return IRQ_NONE;
 544 }
 545
 546 static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 547 {
 548         struct mpi3mr_intr_info *intr_info = privdata;
 549         struct mpi3mr_ioc *mrioc;
 550         u16 midx;
 551         int ret;
 552
 553         if (!intr_info)
 554                 return IRQ_NONE;
 555
 556         mrioc = intr_info->mrioc;
 557         midx = intr_info->msix_index;
 558         /* Call primary ISR routine */
 559         ret = mpi3mr_isr_primary(irq, privdata);
 560
 561         /*
 562          * If more IOs are expected, schedule IRQ polling thread.
 563          * Otherwise exit from ISR.
 564          */
 565         if (!intr_info->op_reply_q)
 566                 return ret;
 567
 568         if (!intr_info->op_reply_q->enable_irq_poll ||
 569             !atomic_read(&intr_info->op_reply_q->pend_ios))
 570                 return ret;
 571
 572         disable_irq_nosync(pci_irq_vector(mrioc->pdev, midx));
 573
 574         return IRQ_WAKE_THREAD;
 575 }
 576
 577 /**
 578  * mpi3mr_isr_poll - Reply queue polling routine
 579  * @irq: IRQ
 580  * @privdata: Interrupt info
 581  *
 582  * poll for pending I/O completions in a loop until pending I/Os
 583  * present or controller queue depth I/Os are processed.
 584  *
 585  * Return: IRQ_NONE or IRQ_HANDLED
 586  */
 587 static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 588 {
 589         struct mpi3mr_intr_info *intr_info = privdata;
 590         struct mpi3mr_ioc *mrioc;
 591         u16 midx;
 592         u32 num_op_reply = 0;
 593
 594         if (!intr_info || !intr_info->op_reply_q)
 595                 return IRQ_NONE;
 596
 597         mrioc = intr_info->mrioc;
 598         midx = intr_info->msix_index;
 599
 600         /* Poll for pending IOs completions */
 601         do {
 602                 if (!mrioc->intr_enabled)
 603                         break;
 604
 605                 if (!midx)
 606                         mpi3mr_process_admin_reply_q(mrioc);
 607                 if (intr_info->op_reply_q)
 608                         num_op_reply +=
 609                             mpi3mr_process_op_reply_q(mrioc, intr_info);
 610
 611                 usleep_range(mrioc->irqpoll_sleep, 10 * mrioc->irqpoll_sleep);
 612
 613         } while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 614             (num_op_reply < mrioc->max_host_ios));
 615
 616         intr_info->op_reply_q->enable_irq_poll = false;
 617         enable_irq(pci_irq_vector(mrioc->pdev, midx));
 618
 619         return IRQ_HANDLED;
 620 }
 621
 622 /**
 623  * mpi3mr_request_irq - Request IRQ and register ISR
 624  * @mrioc: Adapter instance reference
 625  * @index: IRQ vector index
 626  *
 627  * Request threaded ISR with primary ISR and secondary
 628  *
 629  * Return: 0 on success and non zero on failures.
 630  */
 631 static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 632 {
 633         struct pci_dev *pdev = mrioc->pdev;
 634         struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 635         int retval = 0;
 636
 637         intr_info->mrioc = mrioc;
 638         intr_info->msix_index = index;
 639         intr_info->op_reply_q = NULL;
 640
 641         snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 642             mrioc->driver_name, mrioc->id, index);
 643
 644         retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 645             mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 646         if (retval) {
 647                 ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 648                     intr_info->name, pci_irq_vector(pdev, index));
 649                 return retval;
 650         }
 651
 652         return retval;
 653 }
 654
 655 /**
 656  * mpi3mr_setup_isr - Setup ISR for the controller
 657  * @mrioc: Adapter instance reference
 658  * @setup_one: Request one IRQ or more
 659  *
 660  * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 661  *
 662  * Return: 0 on success and non zero on failures.
 663  */
 664 static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 665 {
 666         unsigned int irq_flags = PCI_IRQ_MSIX;
 667         int max_vectors;
 668         int retval;
 669         int i;
 670         struct irq_affinity desc = { .pre_vectors =  1};
 671
 672         mpi3mr_cleanup_isr(mrioc);
 673
 674         if (setup_one || reset_devices)
 675                 max_vectors = 1;
 676         else {
 677                 max_vectors =
 678                     min_t(int, mrioc->cpu_count + 1, mrioc->msix_count);
 679
 680                 ioc_info(mrioc,
 681                     "MSI-X vectors supported: %d, no of cores: %d,",
 682                     mrioc->msix_count, mrioc->cpu_count);
 683                 ioc_info(mrioc,
 684                     "MSI-x vectors requested: %d\n", max_vectors);
 685         }
 686
 687         irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 688
 689         mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 690         retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 691                                 1, max_vectors, irq_flags, &desc);
 692         if (retval < 0) {
 693                 ioc_err(mrioc, "Cannot alloc irq vectors\n");
 694                 goto out_failed;
 695         }
 696         if (retval != max_vectors) {
 697                 ioc_info(mrioc,
 698                     "allocated vectors (%d) are less than configured (%d)\n",
 699                     retval, max_vectors);
 700                 /*
 701                  * If only one MSI-x is allocated, then MSI-x 0 will be shared
 702                  * between Admin queue and operational queue
 703                  */
 704                 if (retval == 1)
 705                         mrioc->op_reply_q_offset = 0;
 706
 707                 max_vectors = retval;
 708         }
 709         mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 710             GFP_KERNEL);
 711         if (!mrioc->intr_info) {
 712                 retval = -ENOMEM;
 713                 pci_free_irq_vectors(mrioc->pdev);
 714                 goto out_failed;
 715         }
 716         for (i = 0; i < max_vectors; i++) {
 717                 retval = mpi3mr_request_irq(mrioc, i);
 718                 if (retval) {
 719                         mrioc->intr_info_count = i;
 720                         goto out_failed;
 721                 }
 722         }
 723         mrioc->intr_info_count = max_vectors;
 724         mpi3mr_ioc_enable_intr(mrioc);
 725         return 0;
 726
 727 out_failed:
 728         mpi3mr_cleanup_isr(mrioc);
 729
 730         return retval;
 731 }
 732
 733 static const struct {
 734         enum mpi3mr_iocstate value;
 735         char *name;
 736 } mrioc_states[] = {
 737         { MRIOC_STATE_READY, "ready" },
 738         { MRIOC_STATE_FAULT, "fault" },
 739         { MRIOC_STATE_RESET, "reset" },
 740         { MRIOC_STATE_BECOMING_READY, "becoming ready" },
 741         { MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 742         { MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 743 };
 744
 745 static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 746 {
 747         int i;
 748         char *name = NULL;
 749
 750         for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 751                 if (mrioc_states[i].value == mrioc_state) {
 752                         name = mrioc_states[i].name;
 753                         break;
 754                 }
 755         }
 756         return name;
 757 }
 758
 759 /* Reset reason to name mapper structure*/
 760 static const struct {
 761         enum mpi3mr_reset_reason value;
 762         char *name;
 763 } mpi3mr_reset_reason_codes[] = {
 764         { MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 765         { MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 766         { MPI3MR_RESET_FROM_IOCTL, "application invocation" },
 767         { MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 768         { MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 769         { MPI3MR_RESET_FROM_IOCTL_TIMEOUT, "IOCTL timeout" },
 770         { MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 771         { MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 772         { MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 773         { MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 774         { MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 775         { MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 776         { MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 777         {
 778                 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 779                 "create request queue timeout"
 780         },
 781         {
 782                 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 783                 "create reply queue timeout"
 784         },
 785         { MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 786         { MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 787         { MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 788         { MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 789         {
 790                 MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 791                 "component image activation timeout"
 792         },
 793         {
 794                 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 795                 "get package version timeout"
 796         },
 797         { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 798         { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 799 };
 800
 801 /**
 802  * mpi3mr_reset_rc_name - get reset reason code name
 803  * @reason_code: reset reason code value
 804  *
 805  * Map reset reason to an NULL terminated ASCII string
 806  *
 807  * Return: name corresponding to reset reason value or NULL.
 808  */
 809 static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 810 {
 811         int i;
 812         char *name = NULL;
 813
 814         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 815                 if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 816                         name = mpi3mr_reset_reason_codes[i].name;
 817                         break;
 818                 }
 819         }
 820         return name;
 821 }
 822
 823 /* Reset type to name mapper structure*/
 824 static const struct {
 825         u16 reset_type;
 826         char *name;
 827 } mpi3mr_reset_types[] = {
 828         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 829         { MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 830 };
 831
 832 /**
 833  * mpi3mr_reset_type_name - get reset type name
 834  * @reset_type: reset type value
 835  *
 836  * Map reset type to an NULL terminated ASCII string
 837  *
 838  * Return: name corresponding to reset type value or NULL.
 839  */
 840 static const char *mpi3mr_reset_type_name(u16 reset_type)
 841 {
 842         int i;
 843         char *name = NULL;
 844
 845         for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 846                 if (mpi3mr_reset_types[i].reset_type == reset_type) {
 847                         name = mpi3mr_reset_types[i].name;
 848                         break;
 849                 }
 850         }
 851         return name;
 852 }
 853
 854 /**
 855  * mpi3mr_print_fault_info - Display fault information
 856  * @mrioc: Adapter instance reference
 857  *
 858  * Display the controller fault information if there is a
 859  * controller fault.
 860  *
 861  * Return: Nothing.
 862  */
 863 static void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
 864 {
 865         u32 ioc_status, code, code1, code2, code3;
 866
 867         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 868
 869         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
 870                 code = readl(&mrioc->sysif_regs->fault);
 871                 code1 = readl(&mrioc->sysif_regs->fault_info[0]);
 872                 code2 = readl(&mrioc->sysif_regs->fault_info[1]);
 873                 code3 = readl(&mrioc->sysif_regs->fault_info[2]);
 874
 875                 ioc_info(mrioc,
 876                     "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
 877                     code, code1, code2, code3);
 878         }
 879 }
 880
 881 /**
 882  * mpi3mr_get_iocstate - Get IOC State
 883  * @mrioc: Adapter instance reference
 884  *
 885  * Return a proper IOC state enum based on the IOC status and
 886  * IOC configuration and unrcoverable state of the controller.
 887  *
 888  * Return: Current IOC state.
 889  */
 890 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
 891 {
 892         u32 ioc_status, ioc_config;
 893         u8 ready, enabled;
 894
 895         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 896         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 897
 898         if (mrioc->unrecoverable)
 899                 return MRIOC_STATE_UNRECOVERABLE;
 900         if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
 901                 return MRIOC_STATE_FAULT;
 902
 903         ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
 904         enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
 905
 906         if (ready && enabled)
 907                 return MRIOC_STATE_READY;
 908         if ((!ready) && (!enabled))
 909                 return MRIOC_STATE_RESET;
 910         if ((!ready) && (enabled))
 911                 return MRIOC_STATE_BECOMING_READY;
 912
 913         return MRIOC_STATE_RESET_REQUESTED;
 914 }
 915
 916 /**
 917  * mpi3mr_clear_reset_history - clear reset history
 918  * @mrioc: Adapter instance reference
 919  *
 920  * Write the reset history bit in IOC status to clear the bit,
 921  * if it is already set.
 922  *
 923  * Return: Nothing.
 924  */
 925 static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
 926 {
 927         u32 ioc_status;
 928
 929         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 930         if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
 931                 writel(ioc_status, &mrioc->sysif_regs->ioc_status);
 932 }
 933
 934 /**
 935  * mpi3mr_issue_and_process_mur - Message unit Reset handler
 936  * @mrioc: Adapter instance reference
 937  * @reset_reason: Reset reason code
 938  *
 939  * Issue Message unit Reset to the controller and wait for it to
 940  * be complete.
 941  *
 942  * Return: 0 on success, -1 on failure.
 943  */
 944 static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
 945         u32 reset_reason)
 946 {
 947         u32 ioc_config, timeout, ioc_status;
 948         int retval = -1;
 949
 950         ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
 951         if (mrioc->unrecoverable) {
 952                 ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
 953                 return retval;
 954         }
 955         mpi3mr_clear_reset_history(mrioc);
 956         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
 957         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 958         ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
 959         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
 960
 961         timeout = mrioc->ready_timeout * 10;
 962         do {
 963                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 964                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
 965                         mpi3mr_clear_reset_history(mrioc);
 966                         ioc_config =
 967                             readl(&mrioc->sysif_regs->ioc_configuration);
 968                         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
 969                               (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
 970                             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC))) {
 971                                 retval = 0;
 972                                 break;
 973                         }
 974                 }
 975                 msleep(100);
 976         } while (--timeout);
 977
 978         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
 979         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
 980
 981         ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
 982             (!retval) ? "successful" : "failed", ioc_status, ioc_config);
 983         return retval;
 984 }
 985
 986 /**
 987  * mpi3mr_bring_ioc_ready - Bring controller to ready state
 988  * @mrioc: Adapter instance reference
 989  *
 990  * Set Enable IOC bit in IOC configuration register and wait for
 991  * the controller to become ready.
 992  *
 993  * Return: 0 on success, -1 on failure.
 994  */
 995 static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
 996 {
 997         u32 ioc_config, timeout;
 998         enum mpi3mr_iocstate current_state;
 999
1000         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1001         ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1002         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1003
1004         timeout = mrioc->ready_timeout * 10;
1005         do {
1006                 current_state = mpi3mr_get_iocstate(mrioc);
1007                 if (current_state == MRIOC_STATE_READY)
1008                         return 0;
1009                 msleep(100);
1010         } while (--timeout);
1011
1012         return -1;
1013 }
1014
1015 /**
1016  * mpi3mr_soft_reset_success - Check softreset is success or not
1017  * @ioc_status: IOC status register value
1018  * @ioc_config: IOC config register value
1019  *
1020  * Check whether the soft reset is successful or not based on
1021  * IOC status and IOC config register values.
1022  *
1023  * Return: True when the soft reset is success, false otherwise.
1024  */
1025 static inline bool
1026 mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1027 {
1028         if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1029             (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1030             (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1031                 return true;
1032         return false;
1033 }
1034
1035 /**
1036  * mpi3mr_diagfault_success - Check diag fault is success or not
1037  * @mrioc: Adapter reference
1038  * @ioc_status: IOC status register value
1039  *
1040  * Check whether the controller hit diag reset fault code.
1041  *
1042  * Return: True when there is diag fault, false otherwise.
1043  */
1044 static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1045         u32 ioc_status)
1046 {
1047         u32 fault;
1048
1049         if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1050                 return false;
1051         fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1052         if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET)
1053                 return true;
1054         return false;
1055 }
1056
1057 /**
1058  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1059  * @mrioc: Adapter reference
1060  *
1061  * Set diag save bit in IOC configuration register to enable
1062  * snapdump.
1063  *
1064  * Return: Nothing.
1065  */
1066 static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1067 {
1068         u32 ioc_config;
1069
1070         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1071         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1072         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1073 }
1074
1075 /**
1076  * mpi3mr_issue_reset - Issue reset to the controller
1077  * @mrioc: Adapter reference
1078  * @reset_type: Reset type
1079  * @reset_reason: Reset reason code
1080  *
1081  * Unlock the host diagnostic registers and write the specific
1082  * reset type to that, wait for reset acknowledgment from the
1083  * controller, if the reset is not successful retry for the
1084  * predefined number of times.
1085  *
1086  * Return: 0 on success, non-zero on failure.
1087  */
1088 static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1089         u32 reset_reason)
1090 {
1091         int retval = -1;
1092         u8 unlock_retry_count, reset_retry_count = 0;
1093         u32 host_diagnostic, timeout, ioc_status, ioc_config;
1094
1095         pci_cfg_access_lock(mrioc->pdev);
1096         if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1097             (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1098                 goto out;
1099         if (mrioc->unrecoverable)
1100                 goto out;
1101 retry_reset:
1102         unlock_retry_count = 0;
1103         mpi3mr_clear_reset_history(mrioc);
1104         do {
1105                 ioc_info(mrioc,
1106                     "Write magic sequence to unlock host diag register (retry=%d)\n",
1107                     ++unlock_retry_count);
1108                 if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1109                         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1110                         mrioc->unrecoverable = 1;
1111                         goto out;
1112                 }
1113
1114                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1115                     &mrioc->sysif_regs->write_sequence);
1116                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1117                     &mrioc->sysif_regs->write_sequence);
1118                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1119                     &mrioc->sysif_regs->write_sequence);
1120                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1121                     &mrioc->sysif_regs->write_sequence);
1122                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1123                     &mrioc->sysif_regs->write_sequence);
1124                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1125                     &mrioc->sysif_regs->write_sequence);
1126                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1127                     &mrioc->sysif_regs->write_sequence);
1128                 usleep_range(1000, 1100);
1129                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1130                 ioc_info(mrioc,
1131                     "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1132                     unlock_retry_count, host_diagnostic);
1133         } while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1134
1135         writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1136         ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1137             mpi3mr_reset_type_name(reset_type),
1138             mpi3mr_reset_rc_name(reset_reason), reset_reason);
1139         writel(host_diagnostic | reset_type,
1140             &mrioc->sysif_regs->host_diagnostic);
1141         timeout = mrioc->ready_timeout * 10;
1142         if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) {
1143                 do {
1144                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1145                         if (ioc_status &
1146                             MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
1147                                 mpi3mr_clear_reset_history(mrioc);
1148                                 ioc_config =
1149                                     readl(&mrioc->sysif_regs->ioc_configuration);
1150                                 if (mpi3mr_soft_reset_success(ioc_status,
1151                                     ioc_config)) {
1152                                         retval = 0;
1153                                         break;
1154                                 }
1155                         }
1156                         msleep(100);
1157                 } while (--timeout);
1158                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1159                     &mrioc->sysif_regs->write_sequence);
1160         } else if (reset_type == MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT) {
1161                 do {
1162                         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1163                         if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1164                                 retval = 0;
1165                                 break;
1166                         }
1167                         msleep(100);
1168                 } while (--timeout);
1169                 mpi3mr_clear_reset_history(mrioc);
1170                 writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1171                     &mrioc->sysif_regs->write_sequence);
1172         }
1173         if (retval && ((++reset_retry_count) < MPI3MR_MAX_RESET_RETRY_COUNT)) {
1174                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1175                 ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1176                 ioc_info(mrioc,
1177                     "Base IOC Sts/Config after reset try %d is (0x%x)/(0x%x)\n",
1178                     reset_retry_count, ioc_status, ioc_config);
1179                 goto retry_reset;
1180         }
1181
1182 out:
1183         pci_cfg_access_unlock(mrioc->pdev);
1184         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1185         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1186
1187         ioc_info(mrioc,
1188             "Base IOC Sts/Config after %s reset is (0x%x)/(0x%x)\n",
1189             (!retval) ? "successful" : "failed", ioc_status,
1190             ioc_config);
1191         return retval;
1192 }
1193
1194 /**
1195  * mpi3mr_admin_request_post - Post request to admin queue
1196  * @mrioc: Adapter reference
1197  * @admin_req: MPI3 request
1198  * @admin_req_sz: Request size
1199  * @ignore_reset: Ignore reset in process
1200  *
1201  * Post the MPI3 request into admin request queue and
1202  * inform the controller, if the queue is full return
1203  * appropriate error.
1204  *
1205  * Return: 0 on success, non-zero on failure.
1206  */
1207 int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1208         u16 admin_req_sz, u8 ignore_reset)
1209 {
1210         u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1211         int retval = 0;
1212         unsigned long flags;
1213         u8 *areq_entry;
1214
1215         if (mrioc->unrecoverable) {
1216                 ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1217                 return -EFAULT;
1218         }
1219
1220         spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1221         areq_pi = mrioc->admin_req_pi;
1222         areq_ci = mrioc->admin_req_ci;
1223         max_entries = mrioc->num_admin_req;
1224         if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1225             (areq_pi == (max_entries - 1)))) {
1226                 ioc_err(mrioc, "AdminReqQ full condition detected\n");
1227                 retval = -EAGAIN;
1228                 goto out;
1229         }
1230         if (!ignore_reset && mrioc->reset_in_progress) {
1231                 ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1232                 retval = -EAGAIN;
1233                 goto out;
1234         }
1235         areq_entry = (u8 *)mrioc->admin_req_base +
1236             (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1237         memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1238         memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1239
1240         if (++areq_pi == max_entries)
1241                 areq_pi = 0;
1242         mrioc->admin_req_pi = areq_pi;
1243
1244         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1245
1246 out:
1247         spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1248
1249         return retval;
1250 }
1251
1252 /**
1253  * mpi3mr_free_op_req_q_segments - free request memory segments
1254  * @mrioc: Adapter instance reference
1255  * @q_idx: operational request queue index
1256  *
1257  * Free memory segments allocated for operational request queue
1258  *
1259  * Return: Nothing.
1260  */
1261 static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1262 {
1263         u16 j;
1264         int size;
1265         struct segments *segments;
1266
1267         segments = mrioc->req_qinfo[q_idx].q_segments;
1268         if (!segments)
1269                 return;
1270
1271         if (mrioc->enable_segqueue) {
1272                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1273                 if (mrioc->req_qinfo[q_idx].q_segment_list) {
1274                         dma_free_coherent(&mrioc->pdev->dev,
1275                             MPI3MR_MAX_SEG_LIST_SIZE,
1276                             mrioc->req_qinfo[q_idx].q_segment_list,
1277                             mrioc->req_qinfo[q_idx].q_segment_list_dma);
1278                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1279                 }
1280         } else
1281                 size = mrioc->req_qinfo[q_idx].num_requests *
1282                     mrioc->facts.op_req_sz;
1283
1284         for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1285                 if (!segments[j].segment)
1286                         continue;
1287                 dma_free_coherent(&mrioc->pdev->dev,
1288                     size, segments[j].segment, segments[j].segment_dma);
1289                 segments[j].segment = NULL;
1290         }
1291         kfree(mrioc->req_qinfo[q_idx].q_segments);
1292         mrioc->req_qinfo[q_idx].q_segments = NULL;
1293         mrioc->req_qinfo[q_idx].qid = 0;
1294 }
1295
1296 /**
1297  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1298  * @mrioc: Adapter instance reference
1299  * @q_idx: operational reply queue index
1300  *
1301  * Free memory segments allocated for operational reply queue
1302  *
1303  * Return: Nothing.
1304  */
1305 static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1306 {
1307         u16 j;
1308         int size;
1309         struct segments *segments;
1310
1311         segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1312         if (!segments)
1313                 return;
1314
1315         if (mrioc->enable_segqueue) {
1316                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1317                 if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1318                         dma_free_coherent(&mrioc->pdev->dev,
1319                             MPI3MR_MAX_SEG_LIST_SIZE,
1320                             mrioc->op_reply_qinfo[q_idx].q_segment_list,
1321                             mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1322                         mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1323                 }
1324         } else
1325                 size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1326                     mrioc->op_reply_desc_sz;
1327
1328         for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1329                 if (!segments[j].segment)
1330                         continue;
1331                 dma_free_coherent(&mrioc->pdev->dev,
1332                     size, segments[j].segment, segments[j].segment_dma);
1333                 segments[j].segment = NULL;
1334         }
1335
1336         kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1337         mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1338         mrioc->op_reply_qinfo[q_idx].qid = 0;
1339 }
1340
1341 /**
1342  * mpi3mr_delete_op_reply_q - delete operational reply queue
1343  * @mrioc: Adapter instance reference
1344  * @qidx: operational reply queue index
1345  *
1346  * Delete operatinal reply queue by issuing MPI request
1347  * through admin queue.
1348  *
1349  * Return:  0 on success, non-zero on failure.
1350  */
1351 static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1352 {
1353         struct mpi3_delete_reply_queue_request delq_req;
1354         int retval = 0;
1355         u16 reply_qid = 0, midx;
1356
1357         reply_qid = mrioc->op_reply_qinfo[qidx].qid;
1358
1359         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1360
1361         if (!reply_qid) {
1362                 retval = -1;
1363                 ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1364                 goto out;
1365         }
1366
1367         memset(&delq_req, 0, sizeof(delq_req));
1368         mutex_lock(&mrioc->init_cmds.mutex);
1369         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1370                 retval = -1;
1371                 ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1372                 mutex_unlock(&mrioc->init_cmds.mutex);
1373                 goto out;
1374         }
1375         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1376         mrioc->init_cmds.is_waiting = 1;
1377         mrioc->init_cmds.callback = NULL;
1378         delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1379         delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1380         delq_req.queue_id = cpu_to_le16(reply_qid);
1381
1382         init_completion(&mrioc->init_cmds.done);
1383         retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1384             1);
1385         if (retval) {
1386                 ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1387                 goto out_unlock;
1388         }
1389         wait_for_completion_timeout(&mrioc->init_cmds.done,
1390             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1391         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1392                 ioc_err(mrioc, "Issue DelRepQ: command timed out\n");
1393                 mpi3mr_set_diagsave(mrioc);
1394                 mpi3mr_issue_reset(mrioc,
1395                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1396                     MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1397                 mrioc->unrecoverable = 1;
1398
1399                 retval = -1;
1400                 goto out_unlock;
1401         }
1402         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1403             != MPI3_IOCSTATUS_SUCCESS) {
1404                 ioc_err(mrioc,
1405                     "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1406                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1407                     mrioc->init_cmds.ioc_loginfo);
1408                 retval = -1;
1409                 goto out_unlock;
1410         }
1411         mrioc->intr_info[midx].op_reply_q = NULL;
1412
1413         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1414 out_unlock:
1415         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1416         mutex_unlock(&mrioc->init_cmds.mutex);
1417 out:
1418
1419         return retval;
1420 }
1421
1422 /**
1423  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1424  * @mrioc: Adapter instance reference
1425  * @qidx: request queue index
1426  *
1427  * Allocate segmented memory pools for operational reply
1428  * queue.
1429  *
1430  * Return: 0 on success, non-zero on failure.
1431  */
1432 static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1433 {
1434         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1435         int i, size;
1436         u64 *q_segment_list_entry = NULL;
1437         struct segments *segments;
1438
1439         if (mrioc->enable_segqueue) {
1440                 op_reply_q->segment_qd =
1441                     MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1442
1443                 size = MPI3MR_OP_REP_Q_SEG_SIZE;
1444
1445                 op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1446                     MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1447                     GFP_KERNEL);
1448                 if (!op_reply_q->q_segment_list)
1449                         return -ENOMEM;
1450                 q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1451         } else {
1452                 op_reply_q->segment_qd = op_reply_q->num_replies;
1453                 size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1454         }
1455
1456         op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1457             op_reply_q->segment_qd);
1458
1459         op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1460             sizeof(struct segments), GFP_KERNEL);
1461         if (!op_reply_q->q_segments)
1462                 return -ENOMEM;
1463
1464         segments = op_reply_q->q_segments;
1465         for (i = 0; i < op_reply_q->num_segments; i++) {
1466                 segments[i].segment =
1467                     dma_alloc_coherent(&mrioc->pdev->dev,
1468                     size, &segments[i].segment_dma, GFP_KERNEL);
1469                 if (!segments[i].segment)
1470                         return -ENOMEM;
1471                 if (mrioc->enable_segqueue)
1472                         q_segment_list_entry[i] =
1473                             (unsigned long)segments[i].segment_dma;
1474         }
1475
1476         return 0;
1477 }
1478
1479 /**
1480  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1481  * @mrioc: Adapter instance reference
1482  * @qidx: request queue index
1483  *
1484  * Allocate segmented memory pools for operational request
1485  * queue.
1486  *
1487  * Return: 0 on success, non-zero on failure.
1488  */
1489 static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1490 {
1491         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1492         int i, size;
1493         u64 *q_segment_list_entry = NULL;
1494         struct segments *segments;
1495
1496         if (mrioc->enable_segqueue) {
1497                 op_req_q->segment_qd =
1498                     MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1499
1500                 size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1501
1502                 op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1503                     MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1504                     GFP_KERNEL);
1505                 if (!op_req_q->q_segment_list)
1506                         return -ENOMEM;
1507                 q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1508
1509         } else {
1510                 op_req_q->segment_qd = op_req_q->num_requests;
1511                 size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1512         }
1513
1514         op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1515             op_req_q->segment_qd);
1516
1517         op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1518             sizeof(struct segments), GFP_KERNEL);
1519         if (!op_req_q->q_segments)
1520                 return -ENOMEM;
1521
1522         segments = op_req_q->q_segments;
1523         for (i = 0; i < op_req_q->num_segments; i++) {
1524                 segments[i].segment =
1525                     dma_alloc_coherent(&mrioc->pdev->dev,
1526                     size, &segments[i].segment_dma, GFP_KERNEL);
1527                 if (!segments[i].segment)
1528                         return -ENOMEM;
1529                 if (mrioc->enable_segqueue)
1530                         q_segment_list_entry[i] =
1531                             (unsigned long)segments[i].segment_dma;
1532         }
1533
1534         return 0;
1535 }
1536
1537 /**
1538  * mpi3mr_create_op_reply_q - create operational reply queue
1539  * @mrioc: Adapter instance reference
1540  * @qidx: operational reply queue index
1541  *
1542  * Create operatinal reply queue by issuing MPI request
1543  * through admin queue.
1544  *
1545  * Return:  0 on success, non-zero on failure.
1546  */
1547 static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1548 {
1549         struct mpi3_create_reply_queue_request create_req;
1550         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1551         int retval = 0;
1552         u16 reply_qid = 0, midx;
1553
1554         reply_qid = op_reply_q->qid;
1555
1556         midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1557
1558         if (reply_qid) {
1559                 retval = -1;
1560                 ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1561                     reply_qid);
1562
1563                 return retval;
1564         }
1565
1566         reply_qid = qidx + 1;
1567         op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
1568         op_reply_q->ci = 0;
1569         op_reply_q->ephase = 1;
1570         atomic_set(&op_reply_q->pend_ios, 0);
1571         atomic_set(&op_reply_q->in_use, 0);
1572         op_reply_q->enable_irq_poll = false;
1573
1574         if (!op_reply_q->q_segments) {
1575                 retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
1576                 if (retval) {
1577                         mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1578                         goto out;
1579                 }
1580         }
1581
1582         memset(&create_req, 0, sizeof(create_req));
1583         mutex_lock(&mrioc->init_cmds.mutex);
1584         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1585                 retval = -1;
1586                 ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
1587                 goto out_unlock;
1588         }
1589         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1590         mrioc->init_cmds.is_waiting = 1;
1591         mrioc->init_cmds.callback = NULL;
1592         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1593         create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
1594         create_req.queue_id = cpu_to_le16(reply_qid);
1595         create_req.flags = MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
1596         create_req.msix_index = cpu_to_le16(mrioc->intr_info[midx].msix_index);
1597         if (mrioc->enable_segqueue) {
1598                 create_req.flags |=
1599                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1600                 create_req.base_address = cpu_to_le64(
1601                     op_reply_q->q_segment_list_dma);
1602         } else
1603                 create_req.base_address = cpu_to_le64(
1604                     op_reply_q->q_segments[0].segment_dma);
1605
1606         create_req.size = cpu_to_le16(op_reply_q->num_replies);
1607
1608         init_completion(&mrioc->init_cmds.done);
1609         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1610             sizeof(create_req), 1);
1611         if (retval) {
1612                 ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
1613                 goto out_unlock;
1614         }
1615         wait_for_completion_timeout(&mrioc->init_cmds.done,
1616             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1617         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1618                 ioc_err(mrioc, "CreateRepQ: command timed out\n");
1619                 mpi3mr_set_diagsave(mrioc);
1620                 mpi3mr_issue_reset(mrioc,
1621                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1622                     MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
1623                 mrioc->unrecoverable = 1;
1624                 retval = -1;
1625                 goto out_unlock;
1626         }
1627         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1628             != MPI3_IOCSTATUS_SUCCESS) {
1629                 ioc_err(mrioc,
1630                     "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1631                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1632                     mrioc->init_cmds.ioc_loginfo);
1633                 retval = -1;
1634                 goto out_unlock;
1635         }
1636         op_reply_q->qid = reply_qid;
1637         mrioc->intr_info[midx].op_reply_q = op_reply_q;
1638
1639 out_unlock:
1640         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1641         mutex_unlock(&mrioc->init_cmds.mutex);
1642 out:
1643
1644         return retval;
1645 }
1646
1647 /**
1648  * mpi3mr_create_op_req_q - create operational request queue
1649  * @mrioc: Adapter instance reference
1650  * @idx: operational request queue index
1651  * @reply_qid: Reply queue ID
1652  *
1653  * Create operatinal request queue by issuing MPI request
1654  * through admin queue.
1655  *
1656  * Return:  0 on success, non-zero on failure.
1657  */
1658 static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
1659         u16 reply_qid)
1660 {
1661         struct mpi3_create_request_queue_request create_req;
1662         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
1663         int retval = 0;
1664         u16 req_qid = 0;
1665
1666         req_qid = op_req_q->qid;
1667
1668         if (req_qid) {
1669                 retval = -1;
1670                 ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
1671                     req_qid);
1672
1673                 return retval;
1674         }
1675         req_qid = idx + 1;
1676
1677         op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
1678         op_req_q->ci = 0;
1679         op_req_q->pi = 0;
1680         op_req_q->reply_qid = reply_qid;
1681         spin_lock_init(&op_req_q->q_lock);
1682
1683         if (!op_req_q->q_segments) {
1684                 retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
1685                 if (retval) {
1686                         mpi3mr_free_op_req_q_segments(mrioc, idx);
1687                         goto out;
1688                 }
1689         }
1690
1691         memset(&create_req, 0, sizeof(create_req));
1692         mutex_lock(&mrioc->init_cmds.mutex);
1693         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1694                 retval = -1;
1695                 ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
1696                 goto out_unlock;
1697         }
1698         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1699         mrioc->init_cmds.is_waiting = 1;
1700         mrioc->init_cmds.callback = NULL;
1701         create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1702         create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
1703         create_req.queue_id = cpu_to_le16(req_qid);
1704         if (mrioc->enable_segqueue) {
1705                 create_req.flags =
1706                     MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
1707                 create_req.base_address = cpu_to_le64(
1708                     op_req_q->q_segment_list_dma);
1709         } else
1710                 create_req.base_address = cpu_to_le64(
1711                     op_req_q->q_segments[0].segment_dma);
1712         create_req.reply_queue_id = cpu_to_le16(reply_qid);
1713         create_req.size = cpu_to_le16(op_req_q->num_requests);
1714
1715         init_completion(&mrioc->init_cmds.done);
1716         retval = mpi3mr_admin_request_post(mrioc, &create_req,
1717             sizeof(create_req), 1);
1718         if (retval) {
1719                 ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
1720                 goto out_unlock;
1721         }
1722         wait_for_completion_timeout(&mrioc->init_cmds.done,
1723             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1724         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1725                 ioc_err(mrioc, "CreateReqQ: command timed out\n");
1726                 mpi3mr_set_diagsave(mrioc);
1727                 if (mpi3mr_issue_reset(mrioc,
1728                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
1729                     MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT))
1730                         mrioc->unrecoverable = 1;
1731                 retval = -1;
1732                 goto out_unlock;
1733         }
1734         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1735             != MPI3_IOCSTATUS_SUCCESS) {
1736                 ioc_err(mrioc,
1737                     "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1738                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1739                     mrioc->init_cmds.ioc_loginfo);
1740                 retval = -1;
1741                 goto out_unlock;
1742         }
1743         op_req_q->qid = req_qid;
1744
1745 out_unlock:
1746         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1747         mutex_unlock(&mrioc->init_cmds.mutex);
1748 out:
1749
1750         return retval;
1751 }
1752
1753 /**
1754  * mpi3mr_create_op_queues - create operational queue pairs
1755  * @mrioc: Adapter instance reference
1756  *
1757  * Allocate memory for operational queue meta data and call
1758  * create request and reply queue functions.
1759  *
1760  * Return: 0 on success, non-zero on failures.
1761  */
1762 static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
1763 {
1764         int retval = 0;
1765         u16 num_queues = 0, i = 0, msix_count_op_q = 1;
1766
1767         num_queues = min_t(int, mrioc->facts.max_op_reply_q,
1768             mrioc->facts.max_op_req_q);
1769
1770         msix_count_op_q =
1771             mrioc->intr_info_count - mrioc->op_reply_q_offset;
1772         if (!mrioc->num_queues)
1773                 mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
1774         num_queues = mrioc->num_queues;
1775         ioc_info(mrioc, "Trying to create %d Operational Q pairs\n",
1776             num_queues);
1777
1778         if (!mrioc->req_qinfo) {
1779                 mrioc->req_qinfo = kcalloc(num_queues,
1780                     sizeof(struct op_req_qinfo), GFP_KERNEL);
1781                 if (!mrioc->req_qinfo) {
1782                         retval = -1;
1783                         goto out_failed;
1784                 }
1785
1786                 mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
1787                     num_queues, GFP_KERNEL);
1788                 if (!mrioc->op_reply_qinfo) {
1789                         retval = -1;
1790                         goto out_failed;
1791                 }
1792         }
1793
1794         if (mrioc->enable_segqueue)
1795                 ioc_info(mrioc,
1796                     "allocating operational queues through segmented queues\n");
1797
1798         for (i = 0; i < num_queues; i++) {
1799                 if (mpi3mr_create_op_reply_q(mrioc, i)) {
1800                         ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
1801                         break;
1802                 }
1803                 if (mpi3mr_create_op_req_q(mrioc, i,
1804                     mrioc->op_reply_qinfo[i].qid)) {
1805                         ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
1806                         mpi3mr_delete_op_reply_q(mrioc, i);
1807                         break;
1808                 }
1809         }
1810
1811         if (i == 0) {
1812                 /* Not even one queue is created successfully*/
1813                 retval = -1;
1814                 goto out_failed;
1815         }
1816         mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
1817         ioc_info(mrioc, "Successfully created %d Operational Q pairs\n",
1818             mrioc->num_op_reply_q);
1819
1820         return retval;
1821 out_failed:
1822         kfree(mrioc->req_qinfo);
1823         mrioc->req_qinfo = NULL;
1824
1825         kfree(mrioc->op_reply_qinfo);
1826         mrioc->op_reply_qinfo = NULL;
1827
1828         return retval;
1829 }
1830
1831 /**
1832  * mpi3mr_op_request_post - Post request to operational queue
1833  * @mrioc: Adapter reference
1834  * @op_req_q: Operational request queue info
1835  * @req: MPI3 request
1836  *
1837  * Post the MPI3 request into operational request queue and
1838  * inform the controller, if the queue is full return
1839  * appropriate error.
1840  *
1841  * Return: 0 on success, non-zero on failure.
1842  */
1843 int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
1844         struct op_req_qinfo *op_req_q, u8 *req)
1845 {
1846         u16 pi = 0, max_entries, reply_qidx = 0, midx;
1847         int retval = 0;
1848         unsigned long flags;
1849         u8 *req_entry;
1850         void *segment_base_addr;
1851         u16 req_sz = mrioc->facts.op_req_sz;
1852         struct segments *segments = op_req_q->q_segments;
1853
1854         reply_qidx = op_req_q->reply_qid - 1;
1855
1856         if (mrioc->unrecoverable)
1857                 return -EFAULT;
1858
1859         spin_lock_irqsave(&op_req_q->q_lock, flags);
1860         pi = op_req_q->pi;
1861         max_entries = op_req_q->num_requests;
1862
1863         if (mpi3mr_check_req_qfull(op_req_q)) {
1864                 midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
1865                     reply_qidx, mrioc->op_reply_q_offset);
1866                 mpi3mr_process_op_reply_q(mrioc, &mrioc->intr_info[midx]);
1867
1868                 if (mpi3mr_check_req_qfull(op_req_q)) {
1869                         retval = -EAGAIN;
1870                         goto out;
1871                 }
1872         }
1873
1874         if (mrioc->reset_in_progress) {
1875                 ioc_err(mrioc, "OpReqQ submit reset in progress\n");
1876                 retval = -EAGAIN;
1877                 goto out;
1878         }
1879
1880         segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
1881         req_entry = (u8 *)segment_base_addr +
1882             ((pi % op_req_q->segment_qd) * req_sz);
1883
1884         memset(req_entry, 0, req_sz);
1885         memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
1886
1887         if (++pi == max_entries)
1888                 pi = 0;
1889         op_req_q->pi = pi;
1890
1891         if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
1892             > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
1893                 mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
1894
1895         writel(op_req_q->pi,
1896             &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
1897
1898 out:
1899         spin_unlock_irqrestore(&op_req_q->q_lock, flags);
1900         return retval;
1901 }
1902
1903 /**
1904  * mpi3mr_sync_timestamp - Issue time stamp sync request
1905  * @mrioc: Adapter reference
1906  *
1907  * Issue IO unit control MPI request to synchornize firmware
1908  * timestamp with host time.
1909  *
1910  * Return: 0 on success, non-zero on failure.
1911  */
1912 static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
1913 {
1914         ktime_t current_time;
1915         struct mpi3_iounit_control_request iou_ctrl;
1916         int retval = 0;
1917
1918         memset(&iou_ctrl, 0, sizeof(iou_ctrl));
1919         mutex_lock(&mrioc->init_cmds.mutex);
1920         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1921                 retval = -1;
1922                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
1923                 mutex_unlock(&mrioc->init_cmds.mutex);
1924                 goto out;
1925         }
1926         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1927         mrioc->init_cmds.is_waiting = 1;
1928         mrioc->init_cmds.callback = NULL;
1929         iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1930         iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
1931         iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
1932         current_time = ktime_get_real();
1933         iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
1934
1935         init_completion(&mrioc->init_cmds.done);
1936         retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
1937             sizeof(iou_ctrl), 0);
1938         if (retval) {
1939                 ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
1940                 goto out_unlock;
1941         }
1942
1943         wait_for_completion_timeout(&mrioc->init_cmds.done,
1944             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1945         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1946                 ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
1947                 mrioc->init_cmds.is_waiting = 0;
1948                 mpi3mr_soft_reset_handler(mrioc,
1949                     MPI3MR_RESET_FROM_TSU_TIMEOUT, 1);
1950                 retval = -1;
1951                 goto out_unlock;
1952         }
1953         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1954             != MPI3_IOCSTATUS_SUCCESS) {
1955                 ioc_err(mrioc,
1956                     "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1957                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1958                     mrioc->init_cmds.ioc_loginfo);
1959                 retval = -1;
1960                 goto out_unlock;
1961         }
1962
1963 out_unlock:
1964         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1965         mutex_unlock(&mrioc->init_cmds.mutex);
1966
1967 out:
1968         return retval;
1969 }
1970
1971 /**
1972  * mpi3mr_watchdog_work - watchdog thread to monitor faults
1973  * @work: work struct
1974  *
1975  * Watch dog work periodically executed (1 second interval) to
1976  * monitor firmware fault and to issue periodic timer sync to
1977  * the firmware.
1978  *
1979  * Return: Nothing.
1980  */
1981 static void mpi3mr_watchdog_work(struct work_struct *work)
1982 {
1983         struct mpi3mr_ioc *mrioc =
1984             container_of(work, struct mpi3mr_ioc, watchdog_work.work);
1985         unsigned long flags;
1986         enum mpi3mr_iocstate ioc_state;
1987         u32 fault, host_diagnostic;
1988
1989         if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
1990                 mrioc->ts_update_counter = 0;
1991                 mpi3mr_sync_timestamp(mrioc);
1992         }
1993
1994         /*Check for fault state every one second and issue Soft reset*/
1995         ioc_state = mpi3mr_get_iocstate(mrioc);
1996         if (ioc_state == MRIOC_STATE_FAULT) {
1997                 fault = readl(&mrioc->sysif_regs->fault) &
1998                     MPI3_SYSIF_FAULT_CODE_MASK;
1999                 host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2000                 if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2001                         if (!mrioc->diagsave_timeout) {
2002                                 mpi3mr_print_fault_info(mrioc);
2003                                 ioc_warn(mrioc, "Diag save in progress\n");
2004                         }
2005                         if ((mrioc->diagsave_timeout++) <=
2006                             MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2007                                 goto schedule_work;
2008                 } else
2009                         mpi3mr_print_fault_info(mrioc);
2010                 mrioc->diagsave_timeout = 0;
2011
2012                 if (fault == MPI3_SYSIF_FAULT_CODE_FACTORY_RESET) {
2013                         ioc_info(mrioc,
2014                             "Factory Reset fault occurred marking controller as unrecoverable"
2015                             );
2016                         mrioc->unrecoverable = 1;
2017                         goto out;
2018                 }
2019
2020                 if ((fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) ||
2021                     (fault == MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS) ||
2022                     (mrioc->reset_in_progress))
2023                         goto out;
2024                 if (fault == MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET)
2025                         mpi3mr_soft_reset_handler(mrioc,
2026                             MPI3MR_RESET_FROM_CIACTIV_FAULT, 0);
2027                 else
2028                         mpi3mr_soft_reset_handler(mrioc,
2029                             MPI3MR_RESET_FROM_FAULT_WATCH, 0);
2030         }
2031
2032 schedule_work:
2033         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2034         if (mrioc->watchdog_work_q)
2035                 queue_delayed_work(mrioc->watchdog_work_q,
2036                     &mrioc->watchdog_work,
2037                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2038         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2039 out:
2040         return;
2041 }
2042
2043 /**
2044  * mpi3mr_start_watchdog - Start watchdog
2045  * @mrioc: Adapter instance reference
2046  *
2047  * Create and start the watchdog thread to monitor controller
2048  * faults.
2049  *
2050  * Return: Nothing.
2051  */
2052 void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2053 {
2054         if (mrioc->watchdog_work_q)
2055                 return;
2056
2057         INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2058         snprintf(mrioc->watchdog_work_q_name,
2059             sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2060             mrioc->id);
2061         mrioc->watchdog_work_q =
2062             create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2063         if (!mrioc->watchdog_work_q) {
2064                 ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2065                 return;
2066         }
2067
2068         if (mrioc->watchdog_work_q)
2069                 queue_delayed_work(mrioc->watchdog_work_q,
2070                     &mrioc->watchdog_work,
2071                     msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2072 }
2073
2074 /**
2075  * mpi3mr_stop_watchdog - Stop watchdog
2076  * @mrioc: Adapter instance reference
2077  *
2078  * Stop the watchdog thread created to monitor controller
2079  * faults.
2080  *
2081  * Return: Nothing.
2082  */
2083 void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2084 {
2085         unsigned long flags;
2086         struct workqueue_struct *wq;
2087
2088         spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2089         wq = mrioc->watchdog_work_q;
2090         mrioc->watchdog_work_q = NULL;
2091         spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2092         if (wq) {
2093                 if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2094                         flush_workqueue(wq);
2095                 destroy_workqueue(wq);
2096         }
2097 }
2098
2099 /**
2100  * mpi3mr_kill_ioc - Kill the controller
2101  * @mrioc: Adapter instance reference
2102  * @reason: reason for the failure.
2103  *
2104  * If fault debug is enabled, display the fault info else issue
2105  * diag fault and freeze the system for controller debug
2106  * purpose.
2107  *
2108  * Return: Nothing.
2109  */
2110 static void mpi3mr_kill_ioc(struct mpi3mr_ioc *mrioc, u32 reason)
2111 {
2112         enum mpi3mr_iocstate ioc_state;
2113
2114         if (!mrioc->fault_dbg)
2115                 return;
2116
2117         dump_stack();
2118
2119         ioc_state = mpi3mr_get_iocstate(mrioc);
2120         if (ioc_state == MRIOC_STATE_FAULT)
2121                 mpi3mr_print_fault_info(mrioc);
2122         else {
2123                 ioc_err(mrioc, "Firmware is halted due to the reason %d\n",
2124                     reason);
2125                 mpi3mr_diagfault_reset_handler(mrioc, reason);
2126         }
2127         if (mrioc->fault_dbg == 2)
2128                 for (;;)
2129                         ;
2130         else
2131                 panic("panic in %s\n", __func__);
2132 }
2133
2134 /**
2135  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2136  * @mrioc: Adapter instance reference
2137  *
2138  * Allocate memory for admin queue pair if required and register
2139  * the admin queue with the controller.
2140  *
2141  * Return: 0 on success, non-zero on failures.
2142  */
2143 static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2144 {
2145         int retval = 0;
2146         u32 num_admin_entries = 0;
2147
2148         mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2149         mrioc->num_admin_req = mrioc->admin_req_q_sz /
2150             MPI3MR_ADMIN_REQ_FRAME_SZ;
2151         mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
2152         mrioc->admin_req_base = NULL;
2153
2154         mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2155         mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2156             MPI3MR_ADMIN_REPLY_FRAME_SZ;
2157         mrioc->admin_reply_ci = 0;
2158         mrioc->admin_reply_ephase = 1;
2159         mrioc->admin_reply_base = NULL;
2160
2161         if (!mrioc->admin_req_base) {
2162                 mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2163                     mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2164
2165                 if (!mrioc->admin_req_base) {
2166                         retval = -1;
2167                         goto out_failed;
2168                 }
2169
2170                 mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2171                     mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2172                     GFP_KERNEL);
2173
2174                 if (!mrioc->admin_reply_base) {
2175                         retval = -1;
2176                         goto out_failed;
2177                 }
2178         }
2179
2180         num_admin_entries = (mrioc->num_admin_replies << 16) |
2181             (mrioc->num_admin_req);
2182         writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2183         mpi3mr_writeq(mrioc->admin_req_dma,
2184             &mrioc->sysif_regs->admin_request_queue_address);
2185         mpi3mr_writeq(mrioc->admin_reply_dma,
2186             &mrioc->sysif_regs->admin_reply_queue_address);
2187         writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2188         writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2189         return retval;
2190
2191 out_failed:
2192
2193         if (mrioc->admin_reply_base) {
2194                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2195                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
2196                 mrioc->admin_reply_base = NULL;
2197         }
2198         if (mrioc->admin_req_base) {
2199                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2200                     mrioc->admin_req_base, mrioc->admin_req_dma);
2201                 mrioc->admin_req_base = NULL;
2202         }
2203         return retval;
2204 }
2205
2206 /**
2207  * mpi3mr_issue_iocfacts - Send IOC Facts
2208  * @mrioc: Adapter instance reference
2209  * @facts_data: Cached IOC facts data
2210  *
2211  * Issue IOC Facts MPI request through admin queue and wait for
2212  * the completion of it or time out.
2213  *
2214  * Return: 0 on success, non-zero on failures.
2215  */
2216 static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2217         struct mpi3_ioc_facts_data *facts_data)
2218 {
2219         struct mpi3_ioc_facts_request iocfacts_req;
2220         void *data = NULL;
2221         dma_addr_t data_dma;
2222         u32 data_len = sizeof(*facts_data);
2223         int retval = 0;
2224         u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2225
2226         data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2227             GFP_KERNEL);
2228
2229         if (!data) {
2230                 retval = -1;
2231                 goto out;
2232         }
2233
2234         memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2235         mutex_lock(&mrioc->init_cmds.mutex);
2236         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2237                 retval = -1;
2238                 ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2239                 mutex_unlock(&mrioc->init_cmds.mutex);
2240                 goto out;
2241         }
2242         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2243         mrioc->init_cmds.is_waiting = 1;
2244         mrioc->init_cmds.callback = NULL;
2245         iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2246         iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2247
2248         mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2249             data_dma);
2250
2251         init_completion(&mrioc->init_cmds.done);
2252         retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2253             sizeof(iocfacts_req), 1);
2254         if (retval) {
2255                 ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2256                 goto out_unlock;
2257         }
2258         wait_for_completion_timeout(&mrioc->init_cmds.done,
2259             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2260         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2261                 ioc_err(mrioc, "Issue IOCFacts: command timed out\n");
2262                 mpi3mr_set_diagsave(mrioc);
2263                 mpi3mr_issue_reset(mrioc,
2264                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2265                     MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2266                 mrioc->unrecoverable = 1;
2267                 retval = -1;
2268                 goto out_unlock;
2269         }
2270         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2271             != MPI3_IOCSTATUS_SUCCESS) {
2272                 ioc_err(mrioc,
2273                     "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2274                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2275                     mrioc->init_cmds.ioc_loginfo);
2276                 retval = -1;
2277                 goto out_unlock;
2278         }
2279         memcpy(facts_data, (u8 *)data, data_len);
2280 out_unlock:
2281         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2282         mutex_unlock(&mrioc->init_cmds.mutex);
2283
2284 out:
2285         if (data)
2286                 dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2287
2288         return retval;
2289 }
2290
2291 /**
2292  * mpi3mr_check_reset_dma_mask - Process IOC facts data
2293  * @mrioc: Adapter instance reference
2294  *
2295  * Check whether the new DMA mask requested through IOCFacts by
2296  * firmware needs to be set, if so set it .
2297  *
2298  * Return: 0 on success, non-zero on failure.
2299  */
2300 static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2301 {
2302         struct pci_dev *pdev = mrioc->pdev;
2303         int r;
2304         u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2305
2306         if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2307                 return 0;
2308
2309         ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2310             mrioc->dma_mask, facts_dma_mask);
2311
2312         r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2313         if (r) {
2314                 ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2315                     facts_dma_mask, r);
2316                 return r;
2317         }
2318         mrioc->dma_mask = facts_dma_mask;
2319         return r;
2320 }
2321
2322 /**
2323  * mpi3mr_process_factsdata - Process IOC facts data
2324  * @mrioc: Adapter instance reference
2325  * @facts_data: Cached IOC facts data
2326  *
2327  * Convert IOC facts data into cpu endianness and cache it in
2328  * the driver .
2329  *
2330  * Return: Nothing.
2331  */
2332 static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2333         struct mpi3_ioc_facts_data *facts_data)
2334 {
2335         u32 ioc_config, req_sz, facts_flags;
2336
2337         if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2338             (sizeof(*facts_data) / 4)) {
2339                 ioc_warn(mrioc,
2340                     "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2341                     sizeof(*facts_data),
2342                     le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2343         }
2344
2345         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2346         req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2347             MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2348         if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2349                 ioc_err(mrioc,
2350                     "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2351                     req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2352         }
2353
2354         memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2355
2356         facts_flags = le32_to_cpu(facts_data->flags);
2357         mrioc->facts.op_req_sz = req_sz;
2358         mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2359             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2360             MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2361
2362         mrioc->facts.ioc_num = facts_data->ioc_number;
2363         mrioc->facts.who_init = facts_data->who_init;
2364         mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2365         mrioc->facts.personality = (facts_flags &
2366             MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2367         mrioc->facts.dma_mask = (facts_flags &
2368             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2369             MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2370         mrioc->facts.protocol_flags = facts_data->protocol_flags;
2371         mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2372         mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_request);
2373         mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2374         mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2375         mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2376         mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2377         mrioc->facts.max_pds = le16_to_cpu(facts_data->max_pds);
2378         mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2379         mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2380         mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_advanced_host_pds);
2381         mrioc->facts.max_raidpds = le16_to_cpu(facts_data->max_raid_pds);
2382         mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2383         mrioc->facts.max_pcie_switches =
2384             le16_to_cpu(facts_data->max_pc_ie_switches);
2385         mrioc->facts.max_sasexpanders =
2386             le16_to_cpu(facts_data->max_sas_expanders);
2387         mrioc->facts.max_sasinitiators =
2388             le16_to_cpu(facts_data->max_sas_initiators);
2389         mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2390         mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2391         mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2392         mrioc->facts.max_op_req_q =
2393             le16_to_cpu(facts_data->max_operational_request_queues);
2394         mrioc->facts.max_op_reply_q =
2395             le16_to_cpu(facts_data->max_operational_reply_queues);
2396         mrioc->facts.ioc_capabilities =
2397             le32_to_cpu(facts_data->ioc_capabilities);
2398         mrioc->facts.fw_ver.build_num =
2399             le16_to_cpu(facts_data->fw_version.build_num);
2400         mrioc->facts.fw_ver.cust_id =
2401             le16_to_cpu(facts_data->fw_version.customer_id);
2402         mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2403         mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2404         mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2405         mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2406         mrioc->msix_count = min_t(int, mrioc->msix_count,
2407             mrioc->facts.max_msix_vectors);
2408         mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2409         mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2410         mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2411         mrioc->facts.shutdown_timeout =
2412             le16_to_cpu(facts_data->shutdown_timeout);
2413
2414         ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
2415             mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
2416             mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
2417         ioc_info(mrioc,
2418             "maxreqs(%d), mindh(%d) maxPDs(%d) maxvectors(%d) maxperids(%d)\n",
2419             mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
2420             mrioc->facts.max_pds, mrioc->facts.max_msix_vectors,
2421             mrioc->facts.max_perids);
2422         ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
2423             mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
2424             mrioc->facts.sge_mod_shift);
2425         ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x\n",
2426             mrioc->facts.dma_mask, (facts_flags &
2427             MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK));
2428
2429         mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
2430
2431         if (reset_devices)
2432                 mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
2433                     MPI3MR_HOST_IOS_KDUMP);
2434 }
2435
2436 /**
2437  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
2438  * @mrioc: Adapter instance reference
2439  *
2440  * Allocate and initialize the reply free buffers, sense
2441  * buffers, reply free queue and sense buffer queue.
2442  *
2443  * Return: 0 on success, non-zero on failures.
2444  */
2445 static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
2446 {
2447         int retval = 0;
2448         u32 sz, i;
2449         dma_addr_t phy_addr;
2450
2451         if (mrioc->init_cmds.reply)
2452                 goto post_reply_sbuf;
2453
2454         mrioc->init_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2455         if (!mrioc->init_cmds.reply)
2456                 goto out_failed;
2457
2458         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
2459                 mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->facts.reply_sz,
2460                     GFP_KERNEL);
2461                 if (!mrioc->dev_rmhs_cmds[i].reply)
2462                         goto out_failed;
2463         }
2464
2465         mrioc->host_tm_cmds.reply = kzalloc(mrioc->facts.reply_sz, GFP_KERNEL);
2466         if (!mrioc->host_tm_cmds.reply)
2467                 goto out_failed;
2468
2469         mrioc->dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8;
2470         if (mrioc->facts.max_devhandle % 8)
2471                 mrioc->dev_handle_bitmap_sz++;
2472         mrioc->removepend_bitmap = kzalloc(mrioc->dev_handle_bitmap_sz,
2473             GFP_KERNEL);
2474         if (!mrioc->removepend_bitmap)
2475                 goto out_failed;
2476
2477         mrioc->devrem_bitmap_sz = MPI3MR_NUM_DEVRMCMD / 8;
2478         if (MPI3MR_NUM_DEVRMCMD % 8)
2479                 mrioc->devrem_bitmap_sz++;
2480         mrioc->devrem_bitmap = kzalloc(mrioc->devrem_bitmap_sz,
2481             GFP_KERNEL);
2482         if (!mrioc->devrem_bitmap)
2483                 goto out_failed;
2484
2485         mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
2486         mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
2487         mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
2488         mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
2489
2490         /* reply buffer pool, 16 byte align */
2491         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2492         mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
2493             &mrioc->pdev->dev, sz, 16, 0);
2494         if (!mrioc->reply_buf_pool) {
2495                 ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
2496                 goto out_failed;
2497         }
2498
2499         mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
2500             &mrioc->reply_buf_dma);
2501         if (!mrioc->reply_buf)
2502                 goto out_failed;
2503
2504         mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
2505
2506         /* reply free queue, 8 byte align */
2507         sz = mrioc->reply_free_qsz * 8;
2508         mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
2509             &mrioc->pdev->dev, sz, 8, 0);
2510         if (!mrioc->reply_free_q_pool) {
2511                 ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
2512                 goto out_failed;
2513         }
2514         mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
2515             GFP_KERNEL, &mrioc->reply_free_q_dma);
2516         if (!mrioc->reply_free_q)
2517                 goto out_failed;
2518
2519         /* sense buffer pool,  4 byte align */
2520         sz = mrioc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;
2521         mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
2522             &mrioc->pdev->dev, sz, 4, 0);
2523         if (!mrioc->sense_buf_pool) {
2524                 ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
2525                 goto out_failed;
2526         }
2527         mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
2528             &mrioc->sense_buf_dma);
2529         if (!mrioc->sense_buf)
2530                 goto out_failed;
2531
2532         /* sense buffer queue, 8 byte align */
2533         sz = mrioc->sense_buf_q_sz * 8;
2534         mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
2535             &mrioc->pdev->dev, sz, 8, 0);
2536         if (!mrioc->sense_buf_q_pool) {
2537                 ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
2538                 goto out_failed;
2539         }
2540         mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
2541             GFP_KERNEL, &mrioc->sense_buf_q_dma);
2542         if (!mrioc->sense_buf_q)
2543                 goto out_failed;
2544
2545 post_reply_sbuf:
2546         sz = mrioc->num_reply_bufs * mrioc->facts.reply_sz;
2547         ioc_info(mrioc,
2548             "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2549             mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->facts.reply_sz,
2550             (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
2551         sz = mrioc->reply_free_qsz * 8;
2552         ioc_info(mrioc,
2553             "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
2554             mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
2555             (unsigned long long)mrioc->reply_free_q_dma);
2556         sz = mrioc->num_sense_bufs * MPI3MR_SENSEBUF_SZ;
2557         ioc_info(mrioc,
2558             "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2559             mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSEBUF_SZ,
2560             (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
2561         sz = mrioc->sense_buf_q_sz * 8;
2562         ioc_info(mrioc,
2563             "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
2564             mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
2565             (unsigned long long)mrioc->sense_buf_q_dma);
2566
2567         /* initialize Reply buffer Queue */
2568         for (i = 0, phy_addr = mrioc->reply_buf_dma;
2569             i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->facts.reply_sz)
2570                 mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
2571         mrioc->reply_free_q[i] = cpu_to_le64(0);
2572
2573         /* initialize Sense Buffer Queue */
2574         for (i = 0, phy_addr = mrioc->sense_buf_dma;
2575             i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSEBUF_SZ)
2576                 mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
2577         mrioc->sense_buf_q[i] = cpu_to_le64(0);
2578         return retval;
2579
2580 out_failed:
2581         retval = -1;
2582         return retval;
2583 }
2584
2585 /**
2586  * mpi3mr_issue_iocinit - Send IOC Init
2587  * @mrioc: Adapter instance reference
2588  *
2589  * Issue IOC Init MPI request through admin queue and wait for
2590  * the completion of it or time out.
2591  *
2592  * Return: 0 on success, non-zero on failures.
2593  */
2594 static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
2595 {
2596         struct mpi3_ioc_init_request iocinit_req;
2597         struct mpi3_driver_info_layout *drv_info;
2598         dma_addr_t data_dma;
2599         u32 data_len = sizeof(*drv_info);
2600         int retval = 0;
2601         ktime_t current_time;
2602
2603         drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2604             GFP_KERNEL);
2605         if (!drv_info) {
2606                 retval = -1;
2607                 goto out;
2608         }
2609         drv_info->information_length = cpu_to_le32(data_len);
2610         strncpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
2611         strncpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
2612         drv_info->os_name[sizeof(drv_info->os_name) - 1] = 0;
2613         strncpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
2614         drv_info->os_version[sizeof(drv_info->os_version) - 1] = 0;
2615         strncpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
2616         strncpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
2617         strncpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE, sizeof(drv_info->driver_release_date));
2618         drv_info->driver_capabilities = 0;
2619         memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
2620             sizeof(mrioc->driver_info));
2621
2622         memset(&iocinit_req, 0, sizeof(iocinit_req));
2623         mutex_lock(&mrioc->init_cmds.mutex);
2624         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2625                 retval = -1;
2626                 ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
2627                 mutex_unlock(&mrioc->init_cmds.mutex);
2628                 goto out;
2629         }
2630         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2631         mrioc->init_cmds.is_waiting = 1;
2632         mrioc->init_cmds.callback = NULL;
2633         iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2634         iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
2635         iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
2636         iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
2637         iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
2638         iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
2639         iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
2640         iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
2641         iocinit_req.reply_free_queue_address =
2642             cpu_to_le64(mrioc->reply_free_q_dma);
2643         iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSEBUF_SZ);
2644         iocinit_req.sense_buffer_free_queue_depth =
2645             cpu_to_le16(mrioc->sense_buf_q_sz);
2646         iocinit_req.sense_buffer_free_queue_address =
2647             cpu_to_le64(mrioc->sense_buf_q_dma);
2648         iocinit_req.driver_information_address = cpu_to_le64(data_dma);
2649
2650         current_time = ktime_get_real();
2651         iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
2652
2653         init_completion(&mrioc->init_cmds.done);
2654         retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
2655             sizeof(iocinit_req), 1);
2656         if (retval) {
2657                 ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
2658                 goto out_unlock;
2659         }
2660         wait_for_completion_timeout(&mrioc->init_cmds.done,
2661             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2662         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2663                 mpi3mr_set_diagsave(mrioc);
2664                 mpi3mr_issue_reset(mrioc,
2665                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2666                     MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
2667                 mrioc->unrecoverable = 1;
2668                 ioc_err(mrioc, "Issue IOCInit: command timed out\n");
2669                 retval = -1;
2670                 goto out_unlock;
2671         }
2672         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2673             != MPI3_IOCSTATUS_SUCCESS) {
2674                 ioc_err(mrioc,
2675                     "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2676                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2677                     mrioc->init_cmds.ioc_loginfo);
2678                 retval = -1;
2679                 goto out_unlock;
2680         }
2681
2682 out_unlock:
2683         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2684         mutex_unlock(&mrioc->init_cmds.mutex);
2685
2686 out:
2687         if (drv_info)
2688                 dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
2689                     data_dma);
2690
2691         return retval;
2692 }
2693
2694 /**
2695  * mpi3mr_unmask_events - Unmask events in event mask bitmap
2696  * @mrioc: Adapter instance reference
2697  * @event: MPI event ID
2698  *
2699  * Un mask the specific event by resetting the event_mask
2700  * bitmap.
2701  *
2702  * Return: 0 on success, non-zero on failures.
2703  */
2704 static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
2705 {
2706         u32 desired_event;
2707         u8 word;
2708
2709         if (event >= 128)
2710                 return;
2711
2712         desired_event = (1 << (event % 32));
2713         word = event / 32;
2714
2715         mrioc->event_masks[word] &= ~desired_event;
2716 }
2717
2718 /**
2719  * mpi3mr_issue_event_notification - Send event notification
2720  * @mrioc: Adapter instance reference
2721  *
2722  * Issue event notification MPI request through admin queue and
2723  * wait for the completion of it or time out.
2724  *
2725  * Return: 0 on success, non-zero on failures.
2726  */
2727 static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
2728 {
2729         struct mpi3_event_notification_request evtnotify_req;
2730         int retval = 0;
2731         u8 i;
2732
2733         memset(&evtnotify_req, 0, sizeof(evtnotify_req));
2734         mutex_lock(&mrioc->init_cmds.mutex);
2735         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2736                 retval = -1;
2737                 ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
2738                 mutex_unlock(&mrioc->init_cmds.mutex);
2739                 goto out;
2740         }
2741         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2742         mrioc->init_cmds.is_waiting = 1;
2743         mrioc->init_cmds.callback = NULL;
2744         evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2745         evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
2746         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
2747                 evtnotify_req.event_masks[i] =
2748                     cpu_to_le32(mrioc->event_masks[i]);
2749         init_completion(&mrioc->init_cmds.done);
2750         retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
2751             sizeof(evtnotify_req), 1);
2752         if (retval) {
2753                 ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
2754                 goto out_unlock;
2755         }
2756         wait_for_completion_timeout(&mrioc->init_cmds.done,
2757             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2758         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2759                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2760                 mpi3mr_set_diagsave(mrioc);
2761                 mpi3mr_issue_reset(mrioc,
2762                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2763                     MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
2764                 mrioc->unrecoverable = 1;
2765                 retval = -1;
2766                 goto out_unlock;
2767         }
2768         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2769             != MPI3_IOCSTATUS_SUCCESS) {
2770                 ioc_err(mrioc,
2771                     "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2772                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2773                     mrioc->init_cmds.ioc_loginfo);
2774                 retval = -1;
2775                 goto out_unlock;
2776         }
2777
2778 out_unlock:
2779         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2780         mutex_unlock(&mrioc->init_cmds.mutex);
2781 out:
2782         return retval;
2783 }
2784
2785 /**
2786  * mpi3mr_send_event_ack - Send event acknowledgment
2787  * @mrioc: Adapter instance reference
2788  * @event: MPI3 event ID
2789  * @event_ctx: Event context
2790  *
2791  * Send event acknowledgment through admin queue and wait for
2792  * it to complete.
2793  *
2794  * Return: 0 on success, non-zero on failures.
2795  */
2796 int mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
2797         u32 event_ctx)
2798 {
2799         struct mpi3_event_ack_request evtack_req;
2800         int retval = 0;
2801
2802         memset(&evtack_req, 0, sizeof(evtack_req));
2803         mutex_lock(&mrioc->init_cmds.mutex);
2804         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2805                 retval = -1;
2806                 ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
2807                 mutex_unlock(&mrioc->init_cmds.mutex);
2808                 goto out;
2809         }
2810         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2811         mrioc->init_cmds.is_waiting = 1;
2812         mrioc->init_cmds.callback = NULL;
2813         evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2814         evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
2815         evtack_req.event = event;
2816         evtack_req.event_context = cpu_to_le32(event_ctx);
2817
2818         init_completion(&mrioc->init_cmds.done);
2819         retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
2820             sizeof(evtack_req), 1);
2821         if (retval) {
2822                 ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
2823                 goto out_unlock;
2824         }
2825         wait_for_completion_timeout(&mrioc->init_cmds.done,
2826             (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2827         if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2828                 ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
2829                 mpi3mr_soft_reset_handler(mrioc,
2830                     MPI3MR_RESET_FROM_EVTACK_TIMEOUT, 1);
2831                 retval = -1;
2832                 goto out_unlock;
2833         }
2834         if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2835             != MPI3_IOCSTATUS_SUCCESS) {
2836                 ioc_err(mrioc,
2837                     "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2838                     (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2839                     mrioc->init_cmds.ioc_loginfo);
2840                 retval = -1;
2841                 goto out_unlock;
2842         }
2843
2844 out_unlock:
2845         mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2846         mutex_unlock(&mrioc->init_cmds.mutex);
2847 out:
2848         return retval;
2849 }
2850
2851 /**
2852  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
2853  * @mrioc: Adapter instance reference
2854  *
2855  * Allocate chain buffers and set a bitmap to indicate free
2856  * chain buffers. Chain buffers are used to pass the SGE
2857  * information along with MPI3 SCSI IO requests for host I/O.
2858  *
2859  * Return: 0 on success, non-zero on failure
2860  */
2861 static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
2862 {
2863         int retval = 0;
2864         u32 sz, i;
2865         u16 num_chains;
2866
2867         num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
2868
2869         if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
2870             | SHOST_DIX_TYPE1_PROTECTION
2871             | SHOST_DIX_TYPE2_PROTECTION
2872             | SHOST_DIX_TYPE3_PROTECTION))
2873                 num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
2874
2875         mrioc->chain_buf_count = num_chains;
2876         sz = sizeof(struct chain_element) * num_chains;
2877         mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
2878         if (!mrioc->chain_sgl_list)
2879                 goto out_failed;
2880
2881         sz = MPI3MR_PAGE_SIZE_4K;
2882         mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
2883             &mrioc->pdev->dev, sz, 16, 0);
2884         if (!mrioc->chain_buf_pool) {
2885                 ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
2886                 goto out_failed;
2887         }
2888
2889         for (i = 0; i < num_chains; i++) {
2890                 mrioc->chain_sgl_list[i].addr =
2891                     dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
2892                     &mrioc->chain_sgl_list[i].dma_addr);
2893
2894                 if (!mrioc->chain_sgl_list[i].addr)
2895                         goto out_failed;
2896         }
2897         mrioc->chain_bitmap_sz = num_chains / 8;
2898         if (num_chains % 8)
2899                 mrioc->chain_bitmap_sz++;
2900         mrioc->chain_bitmap = kzalloc(mrioc->chain_bitmap_sz, GFP_KERNEL);
2901         if (!mrioc->chain_bitmap)
2902                 goto out_failed;
2903         return retval;
2904 out_failed:
2905         retval = -1;
2906         return retval;
2907 }
2908
2909 /**
2910  * mpi3mr_port_enable_complete - Mark port enable complete
2911  * @mrioc: Adapter instance reference
2912  * @drv_cmd: Internal command tracker
2913  *
2914  * Call back for asynchronous port enable request sets the
2915  * driver command to indicate port enable request is complete.
2916  *
2917  * Return: Nothing
2918  */
2919 static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
2920         struct mpi3mr_drv_cmd *drv_cmd)
2921 {
2922         drv_cmd->state = MPI3MR_CMD_NOTUSED;
2923         drv_cmd->callback = NULL;
2924         mrioc->scan_failed = drv_cmd->ioc_status;
2925         mrioc->scan_started = 0;
2926 }
2927
2928 /**
2929  * mpi3mr_issue_port_enable - Issue Port Enable
2930  * @mrioc: Adapter instance reference
2931  * @async: Flag to wait for completion or not
2932  *
2933  * Issue Port Enable MPI request through admin queue and if the
2934  * async flag is not set wait for the completion of the port
2935  * enable or time out.
2936  *
2937  * Return: 0 on success, non-zero on failures.
2938  */
2939 int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
2940 {
2941         struct mpi3_port_enable_request pe_req;
2942         int retval = 0;
2943         u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
2944
2945         memset(&pe_req, 0, sizeof(pe_req));
2946         mutex_lock(&mrioc->init_cmds.mutex);
2947         if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2948                 retval = -1;
2949                 ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
2950                 mutex_unlock(&mrioc->init_cmds.mutex);
2951                 goto out;
2952         }
2953         mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2954         if (async) {
2955                 mrioc->init_cmds.is_waiting = 0;
2956                 mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
2957         } else {
2958                 mrioc->init_cmds.is_waiting = 1;
2959                 mrioc->init_cmds.callback = NULL;
2960                 init_completion(&mrioc->init_cmds.done);
2961         }
2962         pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2963         pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
2964
2965         retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
2966         if (retval) {
2967                 ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
2968                 goto out_unlock;
2969         }
2970         if (!async) {
2971                 wait_for_completion_timeout(&mrioc->init_cmds.done,
2972                     (pe_timeout * HZ));
2973                 if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2974                         ioc_err(mrioc, "Issue PortEnable: command timed out\n");
2975                         retval = -1;
2976                         mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
2977                         mpi3mr_set_diagsave(mrioc);
2978                         mpi3mr_issue_reset(mrioc,
2979                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2980                             MPI3MR_RESET_FROM_PE_TIMEOUT);
2981                         mrioc->unrecoverable = 1;
2982                         goto out_unlock;
2983                 }
2984                 mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
2985         }
2986 out_unlock:
2987         mutex_unlock(&mrioc->init_cmds.mutex);
2988 out:
2989         return retval;
2990 }
2991
2992 /* Protocol type to name mapper structure*/
2993 static const struct {
2994         u8 protocol;
2995         char *name;
2996 } mpi3mr_protocols[] = {
2997         { MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
2998         { MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
2999         { MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3000 };
3001
3002 /* Capability to name mapper structure*/
3003 static const struct {
3004         u32 capability;
3005         char *name;
3006 } mpi3mr_capabilities[] = {
3007         { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3008 };
3009
3010 /**
3011  * mpi3mr_print_ioc_info - Display controller information
3012  * @mrioc: Adapter instance reference
3013  *
3014  * Display controller personalit, capability, supported
3015  * protocols etc.
3016  *
3017  * Return: Nothing
3018  */
3019 static void
3020 mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3021 {
3022         int i = 0, bytes_wrote = 0;
3023         char personality[16];
3024         char protocol[50] = {0};
3025         char capabilities[100] = {0};
3026         bool is_string_nonempty = false;
3027         struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3028
3029         switch (mrioc->facts.personality) {
3030         case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3031                 strncpy(personality, "Enhanced HBA", sizeof(personality));
3032                 break;
3033         case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3034                 strncpy(personality, "RAID", sizeof(personality));
3035                 break;
3036         default:
3037                 strncpy(personality, "Unknown", sizeof(personality));
3038                 break;
3039         }
3040
3041         ioc_info(mrioc, "Running in %s Personality", personality);
3042
3043         ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3044             fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3045             fwver->ph_minor, fwver->cust_id, fwver->build_num);
3046
3047         for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3048                 if (mrioc->facts.protocol_flags &
3049                     mpi3mr_protocols[i].protocol) {
3050                         if (is_string_nonempty &&
3051                             (bytes_wrote < sizeof(protocol)))
3052                                 bytes_wrote += snprintf(protocol + bytes_wrote,
3053                                     (sizeof(protocol) - bytes_wrote), ",");
3054
3055                         if (bytes_wrote < sizeof(protocol))
3056                                 bytes_wrote += snprintf(protocol + bytes_wrote,
3057                                     (sizeof(protocol) - bytes_wrote), "%s",
3058                                     mpi3mr_protocols[i].name);
3059                         is_string_nonempty = true;
3060                 }
3061         }
3062
3063         bytes_wrote = 0;
3064         is_string_nonempty = false;
3065         for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3066                 if (mrioc->facts.protocol_flags &
3067                     mpi3mr_capabilities[i].capability) {
3068                         if (is_string_nonempty &&
3069                             (bytes_wrote < sizeof(capabilities)))
3070                                 bytes_wrote += snprintf(capabilities + bytes_wrote,
3071                                     (sizeof(capabilities) - bytes_wrote), ",");
3072
3073                         if (bytes_wrote < sizeof(capabilities))
3074                                 bytes_wrote += snprintf(capabilities + bytes_wrote,
3075                                     (sizeof(capabilities) - bytes_wrote), "%s",
3076                                     mpi3mr_capabilities[i].name);
3077                         is_string_nonempty = true;
3078                 }
3079         }
3080
3081         ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3082             protocol, capabilities);
3083 }
3084
3085 /**
3086  * mpi3mr_cleanup_resources - Free PCI resources
3087  * @mrioc: Adapter instance reference
3088  *
3089  * Unmap PCI device memory and disable PCI device.
3090  *
3091  * Return: 0 on success and non-zero on failure.
3092  */
3093 void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3094 {
3095         struct pci_dev *pdev = mrioc->pdev;
3096
3097         mpi3mr_cleanup_isr(mrioc);
3098
3099         if (mrioc->sysif_regs) {
3100                 iounmap((void __iomem *)mrioc->sysif_regs);
3101                 mrioc->sysif_regs = NULL;
3102         }
3103
3104         if (pci_is_enabled(pdev)) {
3105                 if (mrioc->bars)
3106                         pci_release_selected_regions(pdev, mrioc->bars);
3107                 pci_disable_device(pdev);
3108         }
3109 }
3110
3111 /**
3112  * mpi3mr_setup_resources - Enable PCI resources
3113  * @mrioc: Adapter instance reference
3114  *
3115  * Enable PCI device memory, MSI-x registers and set DMA mask.
3116  *
3117  * Return: 0 on success and non-zero on failure.
3118  */
3119 int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3120 {
3121         struct pci_dev *pdev = mrioc->pdev;
3122         u32 memap_sz = 0;
3123         int i, retval = 0, capb = 0;
3124         u16 message_control;
3125         u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3126             (((dma_get_required_mask(&pdev->dev) > DMA_BIT_MASK(32)) &&
3127             (sizeof(dma_addr_t) > 4)) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3128
3129         if (pci_enable_device_mem(pdev)) {
3130                 ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3131                 retval = -ENODEV;
3132                 goto out_failed;
3133         }
3134
3135         capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3136         if (!capb) {
3137                 ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3138                 retval = -ENODEV;
3139                 goto out_failed;
3140         }
3141         mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3142
3143         if (pci_request_selected_regions(pdev, mrioc->bars,
3144             mrioc->driver_name)) {
3145                 ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3146                 retval = -ENODEV;
3147                 goto out_failed;
3148         }
3149
3150         for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3151                 if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3152                         mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3153                         memap_sz = pci_resource_len(pdev, i);
3154                         mrioc->sysif_regs =
3155                             ioremap(mrioc->sysif_regs_phys, memap_sz);
3156                         break;
3157                 }
3158         }
3159
3160         pci_set_master(pdev);
3161
3162         retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3163         if (retval) {
3164                 if (dma_mask != DMA_BIT_MASK(32)) {
3165                         ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3166                         dma_mask = DMA_BIT_MASK(32);
3167                         retval = dma_set_mask_and_coherent(&pdev->dev,
3168                             dma_mask);
3169                 }
3170                 if (retval) {
3171                         mrioc->dma_mask = 0;
3172                         ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3173                         goto out_failed;
3174                 }
3175         }
3176         mrioc->dma_mask = dma_mask;
3177
3178         if (!mrioc->sysif_regs) {
3179                 ioc_err(mrioc,
3180                     "Unable to map adapter memory or resource not found\n");
3181                 retval = -EINVAL;
3182                 goto out_failed;
3183         }
3184
3185         pci_read_config_word(pdev, capb + 2, &message_control);
3186         mrioc->msix_count = (message_control & 0x3FF) + 1;
3187
3188         pci_save_state(pdev);
3189
3190         pci_set_drvdata(pdev, mrioc->shost);
3191
3192         mpi3mr_ioc_disable_intr(mrioc);
3193
3194         ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3195             (unsigned long long)mrioc->sysif_regs_phys,
3196             mrioc->sysif_regs, memap_sz);
3197         ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3198             mrioc->msix_count);
3199         return retval;
3200
3201 out_failed:
3202         mpi3mr_cleanup_resources(mrioc);
3203         return retval;
3204 }
3205
3206 /**
3207  * mpi3mr_init_ioc - Initialize the controller
3208  * @mrioc: Adapter instance reference
3209  * @re_init: Flag to indicate is this fresh init or re-init
3210  *
3211  * This the controller initialization routine, executed either
3212  * after soft reset or from pci probe callback.
3213  * Setup the required resources, memory map the controller
3214  * registers, create admin and operational reply queue pairs,
3215  * allocate required memory for reply pool, sense buffer pool,
3216  * issue IOC init request to the firmware, unmask the events and
3217  * issue port enable to discover SAS/SATA/NVMe devies and RAID
3218  * volumes.
3219  *
3220  * Return: 0 on success and non-zero on failure.
3221  */
3222 int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc, u8 re_init)
3223 {
3224         int retval = 0;
3225         enum mpi3mr_iocstate ioc_state;
3226         u64 base_info;
3227         u32 timeout;
3228         u32 ioc_status, ioc_config, i;
3229         struct mpi3_ioc_facts_data facts_data;
3230
3231         mrioc->irqpoll_sleep = MPI3MR_IRQ_POLL_SLEEP;
3232         mrioc->change_count = 0;
3233         if (!re_init) {
3234                 mrioc->cpu_count = num_online_cpus();
3235                 retval = mpi3mr_setup_resources(mrioc);
3236                 if (retval) {
3237                         ioc_err(mrioc, "Failed to setup resources:error %d\n",
3238                             retval);
3239                         goto out_nocleanup;
3240                 }
3241         }
3242
3243         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3244         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3245
3246         ioc_info(mrioc, "SOD status %x configuration %x\n",
3247             ioc_status, ioc_config);
3248
3249         base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
3250         ioc_info(mrioc, "SOD base_info %llx\n", base_info);
3251
3252         /*The timeout value is in 2sec unit, changing it to seconds*/
3253         mrioc->ready_timeout =
3254             ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
3255             MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
3256
3257         ioc_info(mrioc, "IOC ready timeout %d\n", mrioc->ready_timeout);
3258
3259         ioc_state = mpi3mr_get_iocstate(mrioc);
3260         ioc_info(mrioc, "IOC in %s state during detection\n",
3261             mpi3mr_iocstate_name(ioc_state));
3262
3263         if (ioc_state == MRIOC_STATE_BECOMING_READY ||
3264             ioc_state == MRIOC_STATE_RESET_REQUESTED) {
3265                 timeout = mrioc->ready_timeout * 10;
3266                 do {
3267                         msleep(100);
3268                 } while (--timeout);
3269
3270                 ioc_state = mpi3mr_get_iocstate(mrioc);
3271                 ioc_info(mrioc,
3272                     "IOC in %s state after waiting for reset time\n",
3273                     mpi3mr_iocstate_name(ioc_state));
3274         }
3275
3276         if (ioc_state == MRIOC_STATE_READY) {
3277                 retval = mpi3mr_issue_and_process_mur(mrioc,
3278                     MPI3MR_RESET_FROM_BRINGUP);
3279                 if (retval) {
3280                         ioc_err(mrioc, "Failed to MU reset IOC error %d\n",
3281                             retval);
3282                 }
3283                 ioc_state = mpi3mr_get_iocstate(mrioc);
3284         }
3285         if (ioc_state != MRIOC_STATE_RESET) {
3286                 mpi3mr_print_fault_info(mrioc);
3287                 retval = mpi3mr_issue_reset(mrioc,
3288                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3289                     MPI3MR_RESET_FROM_BRINGUP);
3290                 if (retval) {
3291                         ioc_err(mrioc,
3292                             "%s :Failed to soft reset IOC error %d\n",
3293                             __func__, retval);
3294                         goto out_failed;
3295                 }
3296         }
3297         ioc_state = mpi3mr_get_iocstate(mrioc);
3298         if (ioc_state != MRIOC_STATE_RESET) {
3299                 retval = -1;
3300                 ioc_err(mrioc, "Cannot bring IOC to reset state\n");
3301                 goto out_failed;
3302         }
3303
3304         retval = mpi3mr_setup_admin_qpair(mrioc);
3305         if (retval) {
3306                 ioc_err(mrioc, "Failed to setup admin Qs: error %d\n",
3307                     retval);
3308                 goto out_failed;
3309         }
3310
3311         retval = mpi3mr_bring_ioc_ready(mrioc);
3312         if (retval) {
3313                 ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3314                     retval);
3315                 goto out_failed;
3316         }
3317
3318         if (!re_init) {
3319                 retval = mpi3mr_setup_isr(mrioc, 1);
3320                 if (retval) {
3321                         ioc_err(mrioc, "Failed to setup ISR error %d\n",
3322                             retval);
3323                         goto out_failed;
3324                 }
3325         } else
3326                 mpi3mr_ioc_enable_intr(mrioc);
3327
3328         retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3329         if (retval) {
3330                 ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3331                     retval);
3332                 goto out_failed;
3333         }
3334
3335         mpi3mr_process_factsdata(mrioc, &facts_data);
3336         if (!re_init) {
3337                 retval = mpi3mr_check_reset_dma_mask(mrioc);
3338                 if (retval) {
3339                         ioc_err(mrioc, "Resetting dma mask failed %d\n",
3340                             retval);
3341                         goto out_failed;
3342                 }
3343         }
3344
3345         mpi3mr_print_ioc_info(mrioc);
3346
3347         retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3348         if (retval) {
3349                 ioc_err(mrioc,
3350                     "%s :Failed to allocated reply sense buffers %d\n",
3351                     __func__, retval);
3352                 goto out_failed;
3353         }
3354
3355         if (!re_init) {
3356                 retval = mpi3mr_alloc_chain_bufs(mrioc);
3357                 if (retval) {
3358                         ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
3359                             retval);
3360                         goto out_failed;
3361                 }
3362         }
3363
3364         retval = mpi3mr_issue_iocinit(mrioc);
3365         if (retval) {
3366                 ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
3367                     retval);
3368                 goto out_failed;
3369         }
3370         mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3371         writel(mrioc->reply_free_queue_host_index,
3372             &mrioc->sysif_regs->reply_free_host_index);
3373
3374         mrioc->sbq_host_index = mrioc->num_sense_bufs;
3375         writel(mrioc->sbq_host_index,
3376             &mrioc->sysif_regs->sense_buffer_free_host_index);
3377
3378         if (!re_init)  {
3379                 retval = mpi3mr_setup_isr(mrioc, 0);
3380                 if (retval) {
3381                         ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
3382                             retval);
3383                         goto out_failed;
3384                 }
3385         }
3386
3387         retval = mpi3mr_create_op_queues(mrioc);
3388         if (retval) {
3389                 ioc_err(mrioc, "Failed to create OpQueues error %d\n",
3390                     retval);
3391                 goto out_failed;
3392         }
3393
3394         if (re_init &&
3395             (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q)) {
3396                 retval = -1;
3397                 ioc_err(mrioc,
3398                     "Cannot create minimum number of OpQueues expected:%d created:%d\n",
3399                     mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
3400                 goto out_failed;
3401         }
3402
3403         for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3404                 mrioc->event_masks[i] = -1;
3405
3406         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3407         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3408         mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3409         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3410         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3411         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3412         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3413         mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3414         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3415         mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3416         mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3417         mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3418
3419         retval = mpi3mr_issue_event_notification(mrioc);
3420         if (retval) {
3421                 ioc_err(mrioc, "Failed to issue event notification %d\n",
3422                     retval);
3423                 goto out_failed;
3424         }
3425
3426         if (re_init) {
3427                 ioc_info(mrioc, "Issuing Port Enable\n");
3428                 retval = mpi3mr_issue_port_enable(mrioc, 0);
3429                 if (retval) {
3430                         ioc_err(mrioc, "Failed to issue port enable %d\n",
3431                             retval);
3432                         goto out_failed;
3433                 }
3434         }
3435         return retval;
3436
3437 out_failed:
3438         mpi3mr_cleanup_ioc(mrioc, re_init);
3439 out_nocleanup:
3440         return retval;
3441 }
3442
3443 /**
3444  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
3445  *                                      segments
3446  * @mrioc: Adapter instance reference
3447  * @qidx: Operational reply queue index
3448  *
3449  * Return: Nothing.
3450  */
3451 static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3452 {
3453         struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
3454         struct segments *segments;
3455         int i, size;
3456
3457         if (!op_reply_q->q_segments)
3458                 return;
3459
3460         size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
3461         segments = op_reply_q->q_segments;
3462         for (i = 0; i < op_reply_q->num_segments; i++)
3463                 memset(segments[i].segment, 0, size);
3464 }
3465
3466 /**
3467  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
3468  *                                      segments
3469  * @mrioc: Adapter instance reference
3470  * @qidx: Operational request queue index
3471  *
3472  * Return: Nothing.
3473  */
3474 static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
3475 {
3476         struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
3477         struct segments *segments;
3478         int i, size;
3479
3480         if (!op_req_q->q_segments)
3481                 return;
3482
3483         size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
3484         segments = op_req_q->q_segments;
3485         for (i = 0; i < op_req_q->num_segments; i++)
3486                 memset(segments[i].segment, 0, size);
3487 }
3488
3489 /**
3490  * mpi3mr_memset_buffers - memset memory for a controller
3491  * @mrioc: Adapter instance reference
3492  *
3493  * clear all the memory allocated for a controller, typically
3494  * called post reset to reuse the memory allocated during the
3495  * controller init.
3496  *
3497  * Return: Nothing.
3498  */
3499 static void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
3500 {
3501         u16 i;
3502
3503         memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
3504         memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
3505
3506         memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
3507         memset(mrioc->host_tm_cmds.reply, 0,
3508             sizeof(*mrioc->host_tm_cmds.reply));
3509         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
3510                 memset(mrioc->dev_rmhs_cmds[i].reply, 0,
3511                     sizeof(*mrioc->dev_rmhs_cmds[i].reply));
3512         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3513         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3514
3515         for (i = 0; i < mrioc->num_queues; i++) {
3516                 mrioc->op_reply_qinfo[i].qid = 0;
3517                 mrioc->op_reply_qinfo[i].ci = 0;
3518                 mrioc->op_reply_qinfo[i].num_replies = 0;
3519                 mrioc->op_reply_qinfo[i].ephase = 0;
3520                 atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
3521                 atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
3522                 mpi3mr_memset_op_reply_q_buffers(mrioc, i);
3523
3524                 mrioc->req_qinfo[i].ci = 0;
3525                 mrioc->req_qinfo[i].pi = 0;
3526                 mrioc->req_qinfo[i].num_requests = 0;
3527                 mrioc->req_qinfo[i].qid = 0;
3528                 mrioc->req_qinfo[i].reply_qid = 0;
3529                 spin_lock_init(&mrioc->req_qinfo[i].q_lock);
3530                 mpi3mr_memset_op_req_q_buffers(mrioc, i);
3531         }
3532 }
3533
3534 /**
3535  * mpi3mr_free_mem - Free memory allocated for a controller
3536  * @mrioc: Adapter instance reference
3537  *
3538  * Free all the memory allocated for a controller.
3539  *
3540  * Return: Nothing.
3541  */
3542 static void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
3543 {
3544         u16 i;
3545         struct mpi3mr_intr_info *intr_info;
3546
3547         if (mrioc->sense_buf_pool) {
3548                 if (mrioc->sense_buf)
3549                         dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
3550                             mrioc->sense_buf_dma);
3551                 dma_pool_destroy(mrioc->sense_buf_pool);
3552                 mrioc->sense_buf = NULL;
3553                 mrioc->sense_buf_pool = NULL;
3554         }
3555         if (mrioc->sense_buf_q_pool) {
3556                 if (mrioc->sense_buf_q)
3557                         dma_pool_free(mrioc->sense_buf_q_pool,
3558                             mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
3559                 dma_pool_destroy(mrioc->sense_buf_q_pool);
3560                 mrioc->sense_buf_q = NULL;
3561                 mrioc->sense_buf_q_pool = NULL;
3562         }
3563
3564         if (mrioc->reply_buf_pool) {
3565                 if (mrioc->reply_buf)
3566                         dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
3567                             mrioc->reply_buf_dma);
3568                 dma_pool_destroy(mrioc->reply_buf_pool);
3569                 mrioc->reply_buf = NULL;
3570                 mrioc->reply_buf_pool = NULL;
3571         }
3572         if (mrioc->reply_free_q_pool) {
3573                 if (mrioc->reply_free_q)
3574                         dma_pool_free(mrioc->reply_free_q_pool,
3575                             mrioc->reply_free_q, mrioc->reply_free_q_dma);
3576                 dma_pool_destroy(mrioc->reply_free_q_pool);
3577                 mrioc->reply_free_q = NULL;
3578                 mrioc->reply_free_q_pool = NULL;
3579         }
3580
3581         for (i = 0; i < mrioc->num_op_req_q; i++)
3582                 mpi3mr_free_op_req_q_segments(mrioc, i);
3583
3584         for (i = 0; i < mrioc->num_op_reply_q; i++)
3585                 mpi3mr_free_op_reply_q_segments(mrioc, i);
3586
3587         for (i = 0; i < mrioc->intr_info_count; i++) {
3588                 intr_info = mrioc->intr_info + i;
3589                 intr_info->op_reply_q = NULL;
3590         }
3591
3592         kfree(mrioc->req_qinfo);
3593         mrioc->req_qinfo = NULL;
3594         mrioc->num_op_req_q = 0;
3595
3596         kfree(mrioc->op_reply_qinfo);
3597         mrioc->op_reply_qinfo = NULL;
3598         mrioc->num_op_reply_q = 0;
3599
3600         kfree(mrioc->init_cmds.reply);
3601         mrioc->init_cmds.reply = NULL;
3602
3603         kfree(mrioc->host_tm_cmds.reply);
3604         mrioc->host_tm_cmds.reply = NULL;
3605
3606         kfree(mrioc->removepend_bitmap);
3607         mrioc->removepend_bitmap = NULL;
3608
3609         kfree(mrioc->devrem_bitmap);
3610         mrioc->devrem_bitmap = NULL;
3611
3612         kfree(mrioc->chain_bitmap);
3613         mrioc->chain_bitmap = NULL;
3614
3615         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3616                 kfree(mrioc->dev_rmhs_cmds[i].reply);
3617                 mrioc->dev_rmhs_cmds[i].reply = NULL;
3618         }
3619
3620         if (mrioc->chain_buf_pool) {
3621                 for (i = 0; i < mrioc->chain_buf_count; i++) {
3622                         if (mrioc->chain_sgl_list[i].addr) {
3623                                 dma_pool_free(mrioc->chain_buf_pool,
3624                                     mrioc->chain_sgl_list[i].addr,
3625                                     mrioc->chain_sgl_list[i].dma_addr);
3626                                 mrioc->chain_sgl_list[i].addr = NULL;
3627                         }
3628                 }
3629                 dma_pool_destroy(mrioc->chain_buf_pool);
3630                 mrioc->chain_buf_pool = NULL;
3631         }
3632
3633         kfree(mrioc->chain_sgl_list);
3634         mrioc->chain_sgl_list = NULL;
3635
3636         if (mrioc->admin_reply_base) {
3637                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
3638                     mrioc->admin_reply_base, mrioc->admin_reply_dma);
3639                 mrioc->admin_reply_base = NULL;
3640         }
3641         if (mrioc->admin_req_base) {
3642                 dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
3643                     mrioc->admin_req_base, mrioc->admin_req_dma);
3644                 mrioc->admin_req_base = NULL;
3645         }
3646 }
3647
3648 /**
3649  * mpi3mr_issue_ioc_shutdown - shutdown controller
3650  * @mrioc: Adapter instance reference
3651  *
3652  * Send shutodwn notification to the controller and wait for the
3653  * shutdown_timeout for it to be completed.
3654  *
3655  * Return: Nothing.
3656  */
3657 static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
3658 {
3659         u32 ioc_config, ioc_status;
3660         u8 retval = 1;
3661         u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
3662
3663         ioc_info(mrioc, "Issuing shutdown Notification\n");
3664         if (mrioc->unrecoverable) {
3665                 ioc_warn(mrioc,
3666                     "IOC is unrecoverable shutdown is not issued\n");
3667                 return;
3668         }
3669         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3670         if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3671             == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
3672                 ioc_info(mrioc, "shutdown already in progress\n");
3673                 return;
3674         }
3675
3676         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3677         ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
3678         ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN;
3679
3680         writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
3681
3682         if (mrioc->facts.shutdown_timeout)
3683                 timeout = mrioc->facts.shutdown_timeout * 10;
3684
3685         do {
3686                 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3687                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3688                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
3689                         retval = 0;
3690                         break;
3691                 }
3692                 msleep(100);
3693         } while (--timeout);
3694
3695         ioc_status = readl(&mrioc->sysif_regs->ioc_status);
3696         ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
3697
3698         if (retval) {
3699                 if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
3700                     == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
3701                         ioc_warn(mrioc,
3702                             "shutdown still in progress after timeout\n");
3703         }
3704
3705         ioc_info(mrioc,
3706             "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
3707             (!retval) ? "successful" : "failed", ioc_status,
3708             ioc_config);
3709 }
3710
3711 /**
3712  * mpi3mr_cleanup_ioc - Cleanup controller
3713  * @mrioc: Adapter instance reference
3714  * @re_init: Cleanup due to a reinit or not
3715  *
3716  * controller cleanup handler, Message unit reset or soft reset
3717  * and shutdown notification is issued to the controller and the
3718  * associated memory resources are freed.
3719  *
3720  * Return: Nothing.
3721  */
3722 void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc, u8 re_init)
3723 {
3724         enum mpi3mr_iocstate ioc_state;
3725
3726         if (!re_init)
3727                 mpi3mr_stop_watchdog(mrioc);
3728
3729         mpi3mr_ioc_disable_intr(mrioc);
3730
3731         ioc_state = mpi3mr_get_iocstate(mrioc);
3732
3733         if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
3734             (ioc_state == MRIOC_STATE_READY)) {
3735                 if (mpi3mr_issue_and_process_mur(mrioc,
3736                     MPI3MR_RESET_FROM_CTLR_CLEANUP))
3737                         mpi3mr_issue_reset(mrioc,
3738                             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
3739                             MPI3MR_RESET_FROM_MUR_FAILURE);
3740
3741                 if (!re_init)
3742                         mpi3mr_issue_ioc_shutdown(mrioc);
3743         }
3744
3745         if (!re_init) {
3746                 mpi3mr_free_mem(mrioc);
3747                 mpi3mr_cleanup_resources(mrioc);
3748         }
3749 }
3750
3751 /**
3752  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
3753  * @mrioc: Adapter instance reference
3754  * @cmdptr: Internal command tracker
3755  *
3756  * Complete an internal driver commands with state indicating it
3757  * is completed due to reset.
3758  *
3759  * Return: Nothing.
3760  */
3761 static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
3762         struct mpi3mr_drv_cmd *cmdptr)
3763 {
3764         if (cmdptr->state & MPI3MR_CMD_PENDING) {
3765                 cmdptr->state |= MPI3MR_CMD_RESET;
3766                 cmdptr->state &= ~MPI3MR_CMD_PENDING;
3767                 if (cmdptr->is_waiting) {
3768                         complete(&cmdptr->done);
3769                         cmdptr->is_waiting = 0;
3770                 } else if (cmdptr->callback)
3771                         cmdptr->callback(mrioc, cmdptr);
3772         }
3773 }
3774
3775 /**
3776  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
3777  * @mrioc: Adapter instance reference
3778  *
3779  * Flush all internal driver commands post reset
3780  *
3781  * Return: Nothing.
3782  */
3783 static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
3784 {
3785         struct mpi3mr_drv_cmd *cmdptr;
3786         u8 i;
3787
3788         cmdptr = &mrioc->init_cmds;
3789         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3790         cmdptr = &mrioc->host_tm_cmds;
3791         mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3792
3793         for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3794                 cmdptr = &mrioc->dev_rmhs_cmds[i];
3795                 mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
3796         }
3797 }
3798
3799 /**
3800  * mpi3mr_diagfault_reset_handler - Diag fault reset handler
3801  * @mrioc: Adapter instance reference
3802  * @reset_reason: Reset reason code
3803  *
3804  * This is an handler for issuing diag fault reset from the
3805  * applications through IOCTL path to stop the execution of the
3806  * controller
3807  *
3808  * Return: 0 on success, non-zero on failure.
3809  */
3810 int mpi3mr_diagfault_reset_handler(struct mpi3mr_ioc *mrioc,
3811         u32 reset_reason)
3812 {
3813         int retval = 0;
3814
3815         ioc_info(mrioc, "Entry: reason code: %s\n",
3816             mpi3mr_reset_rc_name(reset_reason));
3817         mrioc->reset_in_progress = 1;
3818
3819         mpi3mr_ioc_disable_intr(mrioc);
3820
3821         retval = mpi3mr_issue_reset(mrioc,
3822             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3823
3824         if (retval) {
3825                 ioc_err(mrioc, "The diag fault reset failed: reason %d\n",
3826                     reset_reason);
3827                 mpi3mr_ioc_enable_intr(mrioc);
3828         }
3829         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3830         mrioc->reset_in_progress = 0;
3831         return retval;
3832 }
3833
3834 /**
3835  * mpi3mr_soft_reset_handler - Reset the controller
3836  * @mrioc: Adapter instance reference
3837  * @reset_reason: Reset reason code
3838  * @snapdump: Flag to generate snapdump in firmware or not
3839  *
3840  * This is an handler for recovering controller by issuing soft
3841  * reset are diag fault reset.  This is a blocking function and
3842  * when one reset is executed if any other resets they will be
3843  * blocked. All IOCTLs/IO will be blocked during the reset. If
3844  * controller reset is successful then the controller will be
3845  * reinitalized, otherwise the controller will be marked as not
3846  * recoverable
3847  *
3848  * In snapdump bit is set, the controller is issued with diag
3849  * fault reset so that the firmware can create a snap dump and
3850  * post that the firmware will result in F000 fault and the
3851  * driver will issue soft reset to recover from that.
3852  *
3853  * Return: 0 on success, non-zero on failure.
3854  */
3855 int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
3856         u32 reset_reason, u8 snapdump)
3857 {
3858         int retval = 0, i;
3859         unsigned long flags;
3860         u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
3861
3862         if (mrioc->fault_dbg) {
3863                 if (snapdump)
3864                         mpi3mr_set_diagsave(mrioc);
3865                 mpi3mr_kill_ioc(mrioc, reset_reason);
3866         }
3867
3868         /*
3869          * Block new resets until the currently executing one is finished and
3870          * return the status of the existing reset for all blocked resets
3871          */
3872         if (!mutex_trylock(&mrioc->reset_mutex)) {
3873                 ioc_info(mrioc, "Another reset in progress\n");
3874                 return -1;
3875         }
3876         mrioc->reset_in_progress = 1;
3877
3878         if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
3879             (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
3880                 for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3881                         mrioc->event_masks[i] = -1;
3882
3883                 retval = mpi3mr_issue_event_notification(mrioc);
3884
3885                 if (retval) {
3886                         ioc_err(mrioc,
3887                             "Failed to turn off events prior to reset %d\n",
3888                             retval);
3889                 }
3890         }
3891
3892         mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
3893
3894         mpi3mr_ioc_disable_intr(mrioc);
3895
3896         if (snapdump) {
3897                 mpi3mr_set_diagsave(mrioc);
3898                 retval = mpi3mr_issue_reset(mrioc,
3899                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3900                 if (!retval) {
3901                         do {
3902                                 host_diagnostic =
3903                                     readl(&mrioc->sysif_regs->host_diagnostic);
3904                                 if (!(host_diagnostic &
3905                                     MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
3906                                         break;
3907                                 msleep(100);
3908                         } while (--timeout);
3909                 }
3910         }
3911
3912         retval = mpi3mr_issue_reset(mrioc,
3913             MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
3914         if (retval) {
3915                 ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
3916                 goto out;
3917         }
3918
3919         mpi3mr_flush_delayed_rmhs_list(mrioc);
3920         mpi3mr_flush_drv_cmds(mrioc);
3921         memset(mrioc->devrem_bitmap, 0, mrioc->devrem_bitmap_sz);
3922         memset(mrioc->removepend_bitmap, 0, mrioc->dev_handle_bitmap_sz);
3923         mpi3mr_cleanup_fwevt_list(mrioc);
3924         mpi3mr_flush_host_io(mrioc);
3925         mpi3mr_invalidate_devhandles(mrioc);
3926         mpi3mr_memset_buffers(mrioc);
3927         retval = mpi3mr_init_ioc(mrioc, 1);
3928         if (retval) {
3929                 pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
3930                     mrioc->name, reset_reason);
3931                 goto out;
3932         }
3933         ssleep(10);
3934
3935 out:
3936         if (!retval) {
3937                 mrioc->reset_in_progress = 0;
3938                 scsi_unblock_requests(mrioc->shost);
3939                 mpi3mr_rfresh_tgtdevs(mrioc);
3940                 mrioc->ts_update_counter = 0;
3941                 spin_lock_irqsave(&mrioc->watchdog_lock, flags);
3942                 if (mrioc->watchdog_work_q)
3943                         queue_delayed_work(mrioc->watchdog_work_q,
3944                             &mrioc->watchdog_work,
3945                             msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
3946                 spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
3947         } else {
3948                 mpi3mr_issue_reset(mrioc,
3949                     MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
3950                 mrioc->unrecoverable = 1;
3951                 mrioc->reset_in_progress = 0;
3952                 retval = -1;
3953         }
3954
3955         mutex_unlock(&mrioc->reset_mutex);
3956         ioc_info(mrioc, "%s\n", ((retval == 0) ? "SUCCESS" : "FAILED"));
3957         return retval;
3958 }