drivers/scsi/mpt3sas/mpt3sas_scsih.c

   1 /*
   2  * Scsi Host Layer for MPT (Message Passing Technology) based controllers
   3  *
   4  * This code is based on drivers/scsi/mpt3sas/mpt3sas_scsih.c
   5  * Copyright (C) 2012-2014  LSI Corporation
   6  * Copyright (C) 2013-2014 Avago Technologies
   7  *  (mailto: MPT-FusionLinux.pdl@avagotech.com)
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * as published by the Free Software Foundation; either version 2
  12  * of the License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * NO WARRANTY
  20  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
  21  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
  22  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
  23  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
  24  * solely responsible for determining the appropriateness of using and
  25  * distributing the Program and assumes all risks associated with its
  26  * exercise of rights under this Agreement, including but not limited to
  27  * the risks and costs of program errors, damage to or loss of data,
  28  * programs or equipment, and unavailability or interruption of operations.
  29
  30  * DISCLAIMER OF LIABILITY
  31  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
  32  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
  34  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  35  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  36  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
  37  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
  38
  39  * You should have received a copy of the GNU General Public License
  40  * along with this program; if not, write to the Free Software
  41  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
  42  * USA.
  43  */
  44
  45 #include <linux/module.h>
  46 #include <linux/kernel.h>
  47 #include <linux/init.h>
  48 #include <linux/errno.h>
  49 #include <linux/blkdev.h>
  50 #include <linux/sched.h>
  51 #include <linux/workqueue.h>
  52 #include <linux/delay.h>
  53 #include <linux/pci.h>
  54 #include <linux/interrupt.h>
  55 #include <linux/aer.h>
  56 #include <linux/raid_class.h>
  57 #include <linux/blk-mq-pci.h>
  58 #include <asm/unaligned.h>
  59
  60 #include "mpt3sas_base.h"
  61
  62 #define RAID_CHANNEL 1
  63
  64 #define PCIE_CHANNEL 2
  65
  66 /* forward proto's */
  67 static void _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
  68         struct _sas_node *sas_expander);
  69 static void _firmware_event_work(struct work_struct *work);
  70
  71 static void _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
  72         struct _sas_device *sas_device);
  73 static int _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
  74         u8 retry_count, u8 is_pd);
  75 static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
  76 static void _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
  77         struct _pcie_device *pcie_device);
  78 static void
  79 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
  80 static u8 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid);
  81 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc);
  82
  83 /* global parameters */
  84 LIST_HEAD(mpt3sas_ioc_list);
  85 /* global ioc lock for list operations */
  86 DEFINE_SPINLOCK(gioc_lock);
  87
  88 MODULE_AUTHOR(MPT3SAS_AUTHOR);
  89 MODULE_DESCRIPTION(MPT3SAS_DESCRIPTION);
  90 MODULE_LICENSE("GPL");
  91 MODULE_VERSION(MPT3SAS_DRIVER_VERSION);
  92 MODULE_ALIAS("mpt2sas");
  93
  94 /* local parameters */
  95 static u8 scsi_io_cb_idx = -1;
  96 static u8 tm_cb_idx = -1;
  97 static u8 ctl_cb_idx = -1;
  98 static u8 base_cb_idx = -1;
  99 static u8 port_enable_cb_idx = -1;
 100 static u8 transport_cb_idx = -1;
 101 static u8 scsih_cb_idx = -1;
 102 static u8 config_cb_idx = -1;
 103 static int mpt2_ids;
 104 static int mpt3_ids;
 105
 106 static u8 tm_tr_cb_idx = -1 ;
 107 static u8 tm_tr_volume_cb_idx = -1 ;
 108 static u8 tm_sas_control_cb_idx = -1;
 109
 110 /* command line options */
 111 static u32 logging_level;
 112 MODULE_PARM_DESC(logging_level,
 113         " bits for enabling additional logging info (default=0)");
 114
 115
 116 static ushort max_sectors = 0xFFFF;
 117 module_param(max_sectors, ushort, 0444);
 118 MODULE_PARM_DESC(max_sectors, "max sectors, range 64 to 32767  default=32767");
 119
 120
 121 static int missing_delay[2] = {-1, -1};
 122 module_param_array(missing_delay, int, NULL, 0444);
 123 MODULE_PARM_DESC(missing_delay, " device missing delay , io missing delay");
 124
 125 /* scsi-mid layer global parmeter is max_report_luns, which is 511 */
 126 #define MPT3SAS_MAX_LUN (16895)
 127 static u64 max_lun = MPT3SAS_MAX_LUN;
 128 module_param(max_lun, ullong, 0444);
 129 MODULE_PARM_DESC(max_lun, " max lun, default=16895 ");
 130
 131 static ushort hbas_to_enumerate;
 132 module_param(hbas_to_enumerate, ushort, 0444);
 133 MODULE_PARM_DESC(hbas_to_enumerate,
 134                 " 0 - enumerates both SAS 2.0 & SAS 3.0 generation HBAs\n \
 135                   1 - enumerates only SAS 2.0 generation HBAs\n \
 136                   2 - enumerates only SAS 3.0 generation HBAs (default=0)");
 137
 138 /* diag_buffer_enable is bitwise
 139  * bit 0 set = TRACE
 140  * bit 1 set = SNAPSHOT
 141  * bit 2 set = EXTENDED
 142  *
 143  * Either bit can be set, or both
 144  */
 145 static int diag_buffer_enable = -1;
 146 module_param(diag_buffer_enable, int, 0444);
 147 MODULE_PARM_DESC(diag_buffer_enable,
 148         " post diag buffers (TRACE=1/SNAPSHOT=2/EXTENDED=4/default=0)");
 149 static int disable_discovery = -1;
 150 module_param(disable_discovery, int, 0444);
 151 MODULE_PARM_DESC(disable_discovery, " disable discovery ");
 152
 153
 154 /* permit overriding the host protection capabilities mask (EEDP/T10 PI) */
 155 static int prot_mask = -1;
 156 module_param(prot_mask, int, 0444);
 157 MODULE_PARM_DESC(prot_mask, " host protection capabilities mask, def=7 ");
 158
 159 static bool enable_sdev_max_qd;
 160 module_param(enable_sdev_max_qd, bool, 0444);
 161 MODULE_PARM_DESC(enable_sdev_max_qd,
 162         "Enable sdev max qd as can_queue, def=disabled(0)");
 163
 164 static int multipath_on_hba = -1;
 165 module_param(multipath_on_hba, int, 0);
 166 MODULE_PARM_DESC(multipath_on_hba,
 167         "Multipath support to add same target device\n\t\t"
 168         "as many times as it is visible to HBA from various paths\n\t\t"
 169         "(by default:\n\t\t"
 170         "\t SAS 2.0 & SAS 3.0 HBA - This will be disabled,\n\t\t"
 171         "\t SAS 3.5 HBA - This will be enabled)");
 172
 173 static int host_tagset_enable = 1;
 174 module_param(host_tagset_enable, int, 0444);
 175 MODULE_PARM_DESC(host_tagset_enable,
 176         "Shared host tagset enable/disable Default: enable(1)");
 177
 178 /* raid transport support */
 179 static struct raid_template *mpt3sas_raid_template;
 180 static struct raid_template *mpt2sas_raid_template;
 181
 182
 183 /**
 184  * struct sense_info - common structure for obtaining sense keys
 185  * @skey: sense key
 186  * @asc: additional sense code
 187  * @ascq: additional sense code qualifier
 188  */
 189 struct sense_info {
 190         u8 skey;
 191         u8 asc;
 192         u8 ascq;
 193 };
 194
 195 #define MPT3SAS_PROCESS_TRIGGER_DIAG (0xFFFB)
 196 #define MPT3SAS_TURN_ON_PFA_LED (0xFFFC)
 197 #define MPT3SAS_PORT_ENABLE_COMPLETE (0xFFFD)
 198 #define MPT3SAS_ABRT_TASK_SET (0xFFFE)
 199 #define MPT3SAS_REMOVE_UNRESPONDING_DEVICES (0xFFFF)
 200 /**
 201  * struct fw_event_work - firmware event struct
 202  * @list: link list framework
 203  * @work: work object (ioc->fault_reset_work_q)
 204  * @ioc: per adapter object
 205  * @device_handle: device handle
 206  * @VF_ID: virtual function id
 207  * @VP_ID: virtual port id
 208  * @ignore: flag meaning this event has been marked to ignore
 209  * @event: firmware event MPI2_EVENT_XXX defined in mpi2_ioc.h
 210  * @refcount: kref for this event
 211  * @event_data: reply event data payload follows
 212  *
 213  * This object stored on ioc->fw_event_list.
 214  */
 215 struct fw_event_work {
 216         struct list_head        list;
 217         struct work_struct      work;
 218
 219         struct MPT3SAS_ADAPTER *ioc;
 220         u16                     device_handle;
 221         u8                      VF_ID;
 222         u8                      VP_ID;
 223         u8                      ignore;
 224         u16                     event;
 225         struct kref             refcount;
 226         char                    event_data[] __aligned(4);
 227 };
 228
 229 static void fw_event_work_free(struct kref *r)
 230 {
 231         kfree(container_of(r, struct fw_event_work, refcount));
 232 }
 233
 234 static void fw_event_work_get(struct fw_event_work *fw_work)
 235 {
 236         kref_get(&fw_work->refcount);
 237 }
 238
 239 static void fw_event_work_put(struct fw_event_work *fw_work)
 240 {
 241         kref_put(&fw_work->refcount, fw_event_work_free);
 242 }
 243
 244 static struct fw_event_work *alloc_fw_event_work(int len)
 245 {
 246         struct fw_event_work *fw_event;
 247
 248         fw_event = kzalloc(sizeof(*fw_event) + len, GFP_ATOMIC);
 249         if (!fw_event)
 250                 return NULL;
 251
 252         kref_init(&fw_event->refcount);
 253         return fw_event;
 254 }
 255
 256 /**
 257  * struct _scsi_io_transfer - scsi io transfer
 258  * @handle: sas device handle (assigned by firmware)
 259  * @is_raid: flag set for hidden raid components
 260  * @dir: DMA_TO_DEVICE, DMA_FROM_DEVICE,
 261  * @data_length: data transfer length
 262  * @data_dma: dma pointer to data
 263  * @sense: sense data
 264  * @lun: lun number
 265  * @cdb_length: cdb length
 266  * @cdb: cdb contents
 267  * @timeout: timeout for this command
 268  * @VF_ID: virtual function id
 269  * @VP_ID: virtual port id
 270  * @valid_reply: flag set for reply message
 271  * @sense_length: sense length
 272  * @ioc_status: ioc status
 273  * @scsi_state: scsi state
 274  * @scsi_status: scsi staus
 275  * @log_info: log information
 276  * @transfer_length: data length transfer when there is a reply message
 277  *
 278  * Used for sending internal scsi commands to devices within this module.
 279  * Refer to _scsi_send_scsi_io().
 280  */
 281 struct _scsi_io_transfer {
 282         u16     handle;
 283         u8      is_raid;
 284         enum dma_data_direction dir;
 285         u32     data_length;
 286         dma_addr_t data_dma;
 287         u8      sense[SCSI_SENSE_BUFFERSIZE];
 288         u32     lun;
 289         u8      cdb_length;
 290         u8      cdb[32];
 291         u8      timeout;
 292         u8      VF_ID;
 293         u8      VP_ID;
 294         u8      valid_reply;
 295   /* the following bits are only valid when 'valid_reply = 1' */
 296         u32     sense_length;
 297         u16     ioc_status;
 298         u8      scsi_state;
 299         u8      scsi_status;
 300         u32     log_info;
 301         u32     transfer_length;
 302 };
 303
 304 /**
 305  * _scsih_set_debug_level - global setting of ioc->logging_level.
 306  * @val: ?
 307  * @kp: ?
 308  *
 309  * Note: The logging levels are defined in mpt3sas_debug.h.
 310  */
 311 static int
 312 _scsih_set_debug_level(const char *val, const struct kernel_param *kp)
 313 {
 314         int ret = param_set_int(val, kp);
 315         struct MPT3SAS_ADAPTER *ioc;
 316
 317         if (ret)
 318                 return ret;
 319
 320         pr_info("setting logging_level(0x%08x)\n", logging_level);
 321         spin_lock(&gioc_lock);
 322         list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
 323                 ioc->logging_level = logging_level;
 324         spin_unlock(&gioc_lock);
 325         return 0;
 326 }
 327 module_param_call(logging_level, _scsih_set_debug_level, param_get_int,
 328         &logging_level, 0644);
 329
 330 /**
 331  * _scsih_srch_boot_sas_address - search based on sas_address
 332  * @sas_address: sas address
 333  * @boot_device: boot device object from bios page 2
 334  *
 335  * Return: 1 when there's a match, 0 means no match.
 336  */
 337 static inline int
 338 _scsih_srch_boot_sas_address(u64 sas_address,
 339         Mpi2BootDeviceSasWwid_t *boot_device)
 340 {
 341         return (sas_address == le64_to_cpu(boot_device->SASAddress)) ?  1 : 0;
 342 }
 343
 344 /**
 345  * _scsih_srch_boot_device_name - search based on device name
 346  * @device_name: device name specified in INDENTIFY fram
 347  * @boot_device: boot device object from bios page 2
 348  *
 349  * Return: 1 when there's a match, 0 means no match.
 350  */
 351 static inline int
 352 _scsih_srch_boot_device_name(u64 device_name,
 353         Mpi2BootDeviceDeviceName_t *boot_device)
 354 {
 355         return (device_name == le64_to_cpu(boot_device->DeviceName)) ? 1 : 0;
 356 }
 357
 358 /**
 359  * _scsih_srch_boot_encl_slot - search based on enclosure_logical_id/slot
 360  * @enclosure_logical_id: enclosure logical id
 361  * @slot_number: slot number
 362  * @boot_device: boot device object from bios page 2
 363  *
 364  * Return: 1 when there's a match, 0 means no match.
 365  */
 366 static inline int
 367 _scsih_srch_boot_encl_slot(u64 enclosure_logical_id, u16 slot_number,
 368         Mpi2BootDeviceEnclosureSlot_t *boot_device)
 369 {
 370         return (enclosure_logical_id == le64_to_cpu(boot_device->
 371             EnclosureLogicalID) && slot_number == le16_to_cpu(boot_device->
 372             SlotNumber)) ? 1 : 0;
 373 }
 374
 375 /**
 376  * mpt3sas_get_port_by_id - get hba port entry corresponding to provided
 377  *                        port number from port list
 378  * @ioc: per adapter object
 379  * @port_id: port number
 380  * @bypass_dirty_port_flag: when set look the matching hba port entry even
 381  *                      if hba port entry is marked as dirty.
 382  *
 383  * Search for hba port entry corresponding to provided port number,
 384  * if available return port object otherwise return NULL.
 385  */
 386 struct hba_port *
 387 mpt3sas_get_port_by_id(struct MPT3SAS_ADAPTER *ioc,
 388         u8 port_id, u8 bypass_dirty_port_flag)
 389 {
 390         struct hba_port *port, *port_next;
 391
 392         /*
 393          * When multipath_on_hba is disabled then
 394          * search the hba_port entry using default
 395          * port id i.e. 255
 396          */
 397         if (!ioc->multipath_on_hba)
 398                 port_id = MULTIPATH_DISABLED_PORT_ID;
 399
 400         list_for_each_entry_safe(port, port_next,
 401             &ioc->port_table_list, list) {
 402                 if (port->port_id != port_id)
 403                         continue;
 404                 if (bypass_dirty_port_flag)
 405                         return port;
 406                 if (port->flags & HBA_PORT_FLAG_DIRTY_PORT)
 407                         continue;
 408                 return port;
 409         }
 410
 411         /*
 412          * Allocate hba_port object for default port id (i.e. 255)
 413          * when multipath_on_hba is disabled for the HBA.
 414          * And add this object to port_table_list.
 415          */
 416         if (!ioc->multipath_on_hba) {
 417                 port = kzalloc(sizeof(struct hba_port), GFP_ATOMIC);
 418                 if (!port)
 419                         return NULL;
 420
 421                 port->port_id = port_id;
 422                 ioc_info(ioc,
 423                    "hba_port entry: %p, port: %d is added to hba_port list\n",
 424                    port, port->port_id);
 425                 list_add_tail(&port->list,
 426                     &ioc->port_table_list);
 427                 return port;
 428         }
 429         return NULL;
 430 }
 431
 432 /**
 433  * mpt3sas_get_vphy_by_phy - get virtual_phy object corresponding to phy number
 434  * @ioc: per adapter object
 435  * @port: hba_port object
 436  * @phy: phy number
 437  *
 438  * Return virtual_phy object corresponding to phy number.
 439  */
 440 struct virtual_phy *
 441 mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,
 442         struct hba_port *port, u32 phy)
 443 {
 444         struct virtual_phy *vphy, *vphy_next;
 445
 446         if (!port->vphys_mask)
 447                 return NULL;
 448
 449         list_for_each_entry_safe(vphy, vphy_next, &port->vphys_list, list) {
 450                 if (vphy->phy_mask & (1 << phy))
 451                         return vphy;
 452         }
 453         return NULL;
 454 }
 455
 456 /**
 457  * _scsih_is_boot_device - search for matching boot device.
 458  * @sas_address: sas address
 459  * @device_name: device name specified in INDENTIFY fram
 460  * @enclosure_logical_id: enclosure logical id
 461  * @slot: slot number
 462  * @form: specifies boot device form
 463  * @boot_device: boot device object from bios page 2
 464  *
 465  * Return: 1 when there's a match, 0 means no match.
 466  */
 467 static int
 468 _scsih_is_boot_device(u64 sas_address, u64 device_name,
 469         u64 enclosure_logical_id, u16 slot, u8 form,
 470         Mpi2BiosPage2BootDevice_t *boot_device)
 471 {
 472         int rc = 0;
 473
 474         switch (form) {
 475         case MPI2_BIOSPAGE2_FORM_SAS_WWID:
 476                 if (!sas_address)
 477                         break;
 478                 rc = _scsih_srch_boot_sas_address(
 479                     sas_address, &boot_device->SasWwid);
 480                 break;
 481         case MPI2_BIOSPAGE2_FORM_ENCLOSURE_SLOT:
 482                 if (!enclosure_logical_id)
 483                         break;
 484                 rc = _scsih_srch_boot_encl_slot(
 485                     enclosure_logical_id,
 486                     slot, &boot_device->EnclosureSlot);
 487                 break;
 488         case MPI2_BIOSPAGE2_FORM_DEVICE_NAME:
 489                 if (!device_name)
 490                         break;
 491                 rc = _scsih_srch_boot_device_name(
 492                     device_name, &boot_device->DeviceName);
 493                 break;
 494         case MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED:
 495                 break;
 496         }
 497
 498         return rc;
 499 }
 500
 501 /**
 502  * _scsih_get_sas_address - set the sas_address for given device handle
 503  * @ioc: ?
 504  * @handle: device handle
 505  * @sas_address: sas address
 506  *
 507  * Return: 0 success, non-zero when failure
 508  */
 509 static int
 510 _scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle,
 511         u64 *sas_address)
 512 {
 513         Mpi2SasDevicePage0_t sas_device_pg0;
 514         Mpi2ConfigReply_t mpi_reply;
 515         u32 ioc_status;
 516
 517         *sas_address = 0;
 518
 519         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
 520             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
 521                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
 522                         __FILE__, __LINE__, __func__);
 523                 return -ENXIO;
 524         }
 525
 526         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
 527         if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
 528                 /* For HBA, vSES doesn't return HBA SAS address. Instead return
 529                  * vSES's sas address.
 530                  */
 531                 if ((handle <= ioc->sas_hba.num_phys) &&
 532                    (!(le32_to_cpu(sas_device_pg0.DeviceInfo) &
 533                    MPI2_SAS_DEVICE_INFO_SEP)))
 534                         *sas_address = ioc->sas_hba.sas_address;
 535                 else
 536                         *sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
 537                 return 0;
 538         }
 539
 540         /* we hit this because the given parent handle doesn't exist */
 541         if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
 542                 return -ENXIO;
 543
 544         /* else error case */
 545         ioc_err(ioc, "handle(0x%04x), ioc_status(0x%04x), failure at %s:%d/%s()!\n",
 546                 handle, ioc_status, __FILE__, __LINE__, __func__);
 547         return -EIO;
 548 }
 549
 550 /**
 551  * _scsih_determine_boot_device - determine boot device.
 552  * @ioc: per adapter object
 553  * @device: sas_device or pcie_device object
 554  * @channel: SAS or PCIe channel
 555  *
 556  * Determines whether this device should be first reported device to
 557  * to scsi-ml or sas transport, this purpose is for persistent boot device.
 558  * There are primary, alternate, and current entries in bios page 2. The order
 559  * priority is primary, alternate, then current.  This routine saves
 560  * the corresponding device object.
 561  * The saved data to be used later in _scsih_probe_boot_devices().
 562  */
 563 static void
 564 _scsih_determine_boot_device(struct MPT3SAS_ADAPTER *ioc, void *device,
 565         u32 channel)
 566 {
 567         struct _sas_device *sas_device;
 568         struct _pcie_device *pcie_device;
 569         struct _raid_device *raid_device;
 570         u64 sas_address;
 571         u64 device_name;
 572         u64 enclosure_logical_id;
 573         u16 slot;
 574
 575          /* only process this function when driver loads */
 576         if (!ioc->is_driver_loading)
 577                 return;
 578
 579          /* no Bios, return immediately */
 580         if (!ioc->bios_pg3.BiosVersion)
 581                 return;
 582
 583         if (channel == RAID_CHANNEL) {
 584                 raid_device = device;
 585                 sas_address = raid_device->wwid;
 586                 device_name = 0;
 587                 enclosure_logical_id = 0;
 588                 slot = 0;
 589         } else if (channel == PCIE_CHANNEL) {
 590                 pcie_device = device;
 591                 sas_address = pcie_device->wwid;
 592                 device_name = 0;
 593                 enclosure_logical_id = 0;
 594                 slot = 0;
 595         } else {
 596                 sas_device = device;
 597                 sas_address = sas_device->sas_address;
 598                 device_name = sas_device->device_name;
 599                 enclosure_logical_id = sas_device->enclosure_logical_id;
 600                 slot = sas_device->slot;
 601         }
 602
 603         if (!ioc->req_boot_device.device) {
 604                 if (_scsih_is_boot_device(sas_address, device_name,
 605                     enclosure_logical_id, slot,
 606                     (ioc->bios_pg2.ReqBootDeviceForm &
 607                     MPI2_BIOSPAGE2_FORM_MASK),
 608                     &ioc->bios_pg2.RequestedBootDevice)) {
 609                         dinitprintk(ioc,
 610                                     ioc_info(ioc, "%s: req_boot_device(0x%016llx)\n",
 611                                              __func__, (u64)sas_address));
 612                         ioc->req_boot_device.device = device;
 613                         ioc->req_boot_device.channel = channel;
 614                 }
 615         }
 616
 617         if (!ioc->req_alt_boot_device.device) {
 618                 if (_scsih_is_boot_device(sas_address, device_name,
 619                     enclosure_logical_id, slot,
 620                     (ioc->bios_pg2.ReqAltBootDeviceForm &
 621                     MPI2_BIOSPAGE2_FORM_MASK),
 622                     &ioc->bios_pg2.RequestedAltBootDevice)) {
 623                         dinitprintk(ioc,
 624                                     ioc_info(ioc, "%s: req_alt_boot_device(0x%016llx)\n",
 625                                              __func__, (u64)sas_address));
 626                         ioc->req_alt_boot_device.device = device;
 627                         ioc->req_alt_boot_device.channel = channel;
 628                 }
 629         }
 630
 631         if (!ioc->current_boot_device.device) {
 632                 if (_scsih_is_boot_device(sas_address, device_name,
 633                     enclosure_logical_id, slot,
 634                     (ioc->bios_pg2.CurrentBootDeviceForm &
 635                     MPI2_BIOSPAGE2_FORM_MASK),
 636                     &ioc->bios_pg2.CurrentBootDevice)) {
 637                         dinitprintk(ioc,
 638                                     ioc_info(ioc, "%s: current_boot_device(0x%016llx)\n",
 639                                              __func__, (u64)sas_address));
 640                         ioc->current_boot_device.device = device;
 641                         ioc->current_boot_device.channel = channel;
 642                 }
 643         }
 644 }
 645
 646 static struct _sas_device *
 647 __mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 648                 struct MPT3SAS_TARGET *tgt_priv)
 649 {
 650         struct _sas_device *ret;
 651
 652         assert_spin_locked(&ioc->sas_device_lock);
 653
 654         ret = tgt_priv->sas_dev;
 655         if (ret)
 656                 sas_device_get(ret);
 657
 658         return ret;
 659 }
 660
 661 static struct _sas_device *
 662 mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 663                 struct MPT3SAS_TARGET *tgt_priv)
 664 {
 665         struct _sas_device *ret;
 666         unsigned long flags;
 667
 668         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 669         ret = __mpt3sas_get_sdev_from_target(ioc, tgt_priv);
 670         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 671
 672         return ret;
 673 }
 674
 675 static struct _pcie_device *
 676 __mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 677         struct MPT3SAS_TARGET *tgt_priv)
 678 {
 679         struct _pcie_device *ret;
 680
 681         assert_spin_locked(&ioc->pcie_device_lock);
 682
 683         ret = tgt_priv->pcie_dev;
 684         if (ret)
 685                 pcie_device_get(ret);
 686
 687         return ret;
 688 }
 689
 690 /**
 691  * mpt3sas_get_pdev_from_target - pcie device search
 692  * @ioc: per adapter object
 693  * @tgt_priv: starget private object
 694  *
 695  * Context: This function will acquire ioc->pcie_device_lock and will release
 696  * before returning the pcie_device object.
 697  *
 698  * This searches for pcie_device from target, then return pcie_device object.
 699  */
 700 static struct _pcie_device *
 701 mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,
 702         struct MPT3SAS_TARGET *tgt_priv)
 703 {
 704         struct _pcie_device *ret;
 705         unsigned long flags;
 706
 707         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
 708         ret = __mpt3sas_get_pdev_from_target(ioc, tgt_priv);
 709         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
 710
 711         return ret;
 712 }
 713
 714
 715 /**
 716  * __mpt3sas_get_sdev_by_rphy - sas device search
 717  * @ioc: per adapter object
 718  * @rphy: sas_rphy pointer
 719  *
 720  * Context: This function will acquire ioc->sas_device_lock and will release
 721  * before returning the sas_device object.
 722  *
 723  * This searches for sas_device from rphy object
 724  * then return sas_device object.
 725  */
 726 struct _sas_device *
 727 __mpt3sas_get_sdev_by_rphy(struct MPT3SAS_ADAPTER *ioc,
 728         struct sas_rphy *rphy)
 729 {
 730         struct _sas_device *sas_device;
 731
 732         assert_spin_locked(&ioc->sas_device_lock);
 733
 734         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
 735                 if (sas_device->rphy != rphy)
 736                         continue;
 737                 sas_device_get(sas_device);
 738                 return sas_device;
 739         }
 740
 741         sas_device = NULL;
 742         list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {
 743                 if (sas_device->rphy != rphy)
 744                         continue;
 745                 sas_device_get(sas_device);
 746                 return sas_device;
 747         }
 748
 749         return NULL;
 750 }
 751
 752 /**
 753  * __mpt3sas_get_sdev_by_addr - get _sas_device object corresponding to provided
 754  *                              sas address from sas_device_list list
 755  * @ioc: per adapter object
 756  * @sas_address: device sas address
 757  * @port: port number
 758  *
 759  * Search for _sas_device object corresponding to provided sas address,
 760  * if available return _sas_device object address otherwise return NULL.
 761  */
 762 struct _sas_device *
 763 __mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,
 764         u64 sas_address, struct hba_port *port)
 765 {
 766         struct _sas_device *sas_device;
 767
 768         if (!port)
 769                 return NULL;
 770
 771         assert_spin_locked(&ioc->sas_device_lock);
 772
 773         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
 774                 if (sas_device->sas_address != sas_address)
 775                         continue;
 776                 if (sas_device->port != port)
 777                         continue;
 778                 sas_device_get(sas_device);
 779                 return sas_device;
 780         }
 781
 782         list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {
 783                 if (sas_device->sas_address != sas_address)
 784                         continue;
 785                 if (sas_device->port != port)
 786                         continue;
 787                 sas_device_get(sas_device);
 788                 return sas_device;
 789         }
 790
 791         return NULL;
 792 }
 793
 794 /**
 795  * mpt3sas_get_sdev_by_addr - sas device search
 796  * @ioc: per adapter object
 797  * @sas_address: sas address
 798  * @port: hba port entry
 799  * Context: Calling function should acquire ioc->sas_device_lock
 800  *
 801  * This searches for sas_device based on sas_address & port number,
 802  * then return sas_device object.
 803  */
 804 struct _sas_device *
 805 mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,
 806         u64 sas_address, struct hba_port *port)
 807 {
 808         struct _sas_device *sas_device;
 809         unsigned long flags;
 810
 811         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 812         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
 813             sas_address, port);
 814         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 815
 816         return sas_device;
 817 }
 818
 819 static struct _sas_device *
 820 __mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 821 {
 822         struct _sas_device *sas_device;
 823
 824         assert_spin_locked(&ioc->sas_device_lock);
 825
 826         list_for_each_entry(sas_device, &ioc->sas_device_list, list)
 827                 if (sas_device->handle == handle)
 828                         goto found_device;
 829
 830         list_for_each_entry(sas_device, &ioc->sas_device_init_list, list)
 831                 if (sas_device->handle == handle)
 832                         goto found_device;
 833
 834         return NULL;
 835
 836 found_device:
 837         sas_device_get(sas_device);
 838         return sas_device;
 839 }
 840
 841 /**
 842  * mpt3sas_get_sdev_by_handle - sas device search
 843  * @ioc: per adapter object
 844  * @handle: sas device handle (assigned by firmware)
 845  * Context: Calling function should acquire ioc->sas_device_lock
 846  *
 847  * This searches for sas_device based on sas_address, then return sas_device
 848  * object.
 849  */
 850 struct _sas_device *
 851 mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 852 {
 853         struct _sas_device *sas_device;
 854         unsigned long flags;
 855
 856         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 857         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
 858         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 859
 860         return sas_device;
 861 }
 862
 863 /**
 864  * _scsih_display_enclosure_chassis_info - display device location info
 865  * @ioc: per adapter object
 866  * @sas_device: per sas device object
 867  * @sdev: scsi device struct
 868  * @starget: scsi target struct
 869  */
 870 static void
 871 _scsih_display_enclosure_chassis_info(struct MPT3SAS_ADAPTER *ioc,
 872         struct _sas_device *sas_device, struct scsi_device *sdev,
 873         struct scsi_target *starget)
 874 {
 875         if (sdev) {
 876                 if (sas_device->enclosure_handle != 0)
 877                         sdev_printk(KERN_INFO, sdev,
 878                             "enclosure logical id (0x%016llx), slot(%d) \n",
 879                             (unsigned long long)
 880                             sas_device->enclosure_logical_id,
 881                             sas_device->slot);
 882                 if (sas_device->connector_name[0] != '\0')
 883                         sdev_printk(KERN_INFO, sdev,
 884                             "enclosure level(0x%04x), connector name( %s)\n",
 885                             sas_device->enclosure_level,
 886                             sas_device->connector_name);
 887                 if (sas_device->is_chassis_slot_valid)
 888                         sdev_printk(KERN_INFO, sdev, "chassis slot(0x%04x)\n",
 889                             sas_device->chassis_slot);
 890         } else if (starget) {
 891                 if (sas_device->enclosure_handle != 0)
 892                         starget_printk(KERN_INFO, starget,
 893                             "enclosure logical id(0x%016llx), slot(%d) \n",
 894                             (unsigned long long)
 895                             sas_device->enclosure_logical_id,
 896                             sas_device->slot);
 897                 if (sas_device->connector_name[0] != '\0')
 898                         starget_printk(KERN_INFO, starget,
 899                             "enclosure level(0x%04x), connector name( %s)\n",
 900                             sas_device->enclosure_level,
 901                             sas_device->connector_name);
 902                 if (sas_device->is_chassis_slot_valid)
 903                         starget_printk(KERN_INFO, starget,
 904                             "chassis slot(0x%04x)\n",
 905                             sas_device->chassis_slot);
 906         } else {
 907                 if (sas_device->enclosure_handle != 0)
 908                         ioc_info(ioc, "enclosure logical id(0x%016llx), slot(%d)\n",
 909                                  (u64)sas_device->enclosure_logical_id,
 910                                  sas_device->slot);
 911                 if (sas_device->connector_name[0] != '\0')
 912                         ioc_info(ioc, "enclosure level(0x%04x), connector name( %s)\n",
 913                                  sas_device->enclosure_level,
 914                                  sas_device->connector_name);
 915                 if (sas_device->is_chassis_slot_valid)
 916                         ioc_info(ioc, "chassis slot(0x%04x)\n",
 917                                  sas_device->chassis_slot);
 918         }
 919 }
 920
 921 /**
 922  * _scsih_sas_device_remove - remove sas_device from list.
 923  * @ioc: per adapter object
 924  * @sas_device: the sas_device object
 925  * Context: This function will acquire ioc->sas_device_lock.
 926  *
 927  * If sas_device is on the list, remove it and decrement its reference count.
 928  */
 929 static void
 930 _scsih_sas_device_remove(struct MPT3SAS_ADAPTER *ioc,
 931         struct _sas_device *sas_device)
 932 {
 933         unsigned long flags;
 934
 935         if (!sas_device)
 936                 return;
 937         ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",
 938                  sas_device->handle, (u64)sas_device->sas_address);
 939
 940         _scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);
 941
 942         /*
 943          * The lock serializes access to the list, but we still need to verify
 944          * that nobody removed the entry while we were waiting on the lock.
 945          */
 946         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 947         if (!list_empty(&sas_device->list)) {
 948                 list_del_init(&sas_device->list);
 949                 sas_device_put(sas_device);
 950         }
 951         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 952 }
 953
 954 /**
 955  * _scsih_device_remove_by_handle - removing device object by handle
 956  * @ioc: per adapter object
 957  * @handle: device handle
 958  */
 959 static void
 960 _scsih_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
 961 {
 962         struct _sas_device *sas_device;
 963         unsigned long flags;
 964
 965         if (ioc->shost_recovery)
 966                 return;
 967
 968         spin_lock_irqsave(&ioc->sas_device_lock, flags);
 969         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
 970         if (sas_device) {
 971                 list_del_init(&sas_device->list);
 972                 sas_device_put(sas_device);
 973         }
 974         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
 975         if (sas_device) {
 976                 _scsih_remove_device(ioc, sas_device);
 977                 sas_device_put(sas_device);
 978         }
 979 }
 980
 981 /**
 982  * mpt3sas_device_remove_by_sas_address - removing device object by
 983  *                                      sas address & port number
 984  * @ioc: per adapter object
 985  * @sas_address: device sas_address
 986  * @port: hba port entry
 987  *
 988  * Return nothing.
 989  */
 990 void
 991 mpt3sas_device_remove_by_sas_address(struct MPT3SAS_ADAPTER *ioc,
 992         u64 sas_address, struct hba_port *port)
 993 {
 994         struct _sas_device *sas_device;
 995         unsigned long flags;
 996
 997         if (ioc->shost_recovery)
 998                 return;
 999
1000         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1001         sas_device = __mpt3sas_get_sdev_by_addr(ioc, sas_address, port);
1002         if (sas_device) {
1003                 list_del_init(&sas_device->list);
1004                 sas_device_put(sas_device);
1005         }
1006         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1007         if (sas_device) {
1008                 _scsih_remove_device(ioc, sas_device);
1009                 sas_device_put(sas_device);
1010         }
1011 }
1012
1013 /**
1014  * _scsih_sas_device_add - insert sas_device to the list.
1015  * @ioc: per adapter object
1016  * @sas_device: the sas_device object
1017  * Context: This function will acquire ioc->sas_device_lock.
1018  *
1019  * Adding new object to the ioc->sas_device_list.
1020  */
1021 static void
1022 _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,
1023         struct _sas_device *sas_device)
1024 {
1025         unsigned long flags;
1026
1027         dewtprintk(ioc,
1028                    ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",
1029                             __func__, sas_device->handle,
1030                             (u64)sas_device->sas_address));
1031
1032         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
1033             NULL, NULL));
1034
1035         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1036         sas_device_get(sas_device);
1037         list_add_tail(&sas_device->list, &ioc->sas_device_list);
1038         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1039
1040         if (ioc->hide_drives) {
1041                 clear_bit(sas_device->handle, ioc->pend_os_device_add);
1042                 return;
1043         }
1044
1045         if (!mpt3sas_transport_port_add(ioc, sas_device->handle,
1046              sas_device->sas_address_parent, sas_device->port)) {
1047                 _scsih_sas_device_remove(ioc, sas_device);
1048         } else if (!sas_device->starget) {
1049                 /*
1050                  * When asyn scanning is enabled, its not possible to remove
1051                  * devices while scanning is turned on due to an oops in
1052                  * scsi_sysfs_add_sdev()->add_device()->sysfs_addrm_start()
1053                  */
1054                 if (!ioc->is_driver_loading) {
1055                         mpt3sas_transport_port_remove(ioc,
1056                             sas_device->sas_address,
1057                             sas_device->sas_address_parent,
1058                             sas_device->port);
1059                         _scsih_sas_device_remove(ioc, sas_device);
1060                 }
1061         } else
1062                 clear_bit(sas_device->handle, ioc->pend_os_device_add);
1063 }
1064
1065 /**
1066  * _scsih_sas_device_init_add - insert sas_device to the list.
1067  * @ioc: per adapter object
1068  * @sas_device: the sas_device object
1069  * Context: This function will acquire ioc->sas_device_lock.
1070  *
1071  * Adding new object at driver load time to the ioc->sas_device_init_list.
1072  */
1073 static void
1074 _scsih_sas_device_init_add(struct MPT3SAS_ADAPTER *ioc,
1075         struct _sas_device *sas_device)
1076 {
1077         unsigned long flags;
1078
1079         dewtprintk(ioc,
1080                    ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",
1081                             __func__, sas_device->handle,
1082                             (u64)sas_device->sas_address));
1083
1084         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
1085             NULL, NULL));
1086
1087         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1088         sas_device_get(sas_device);
1089         list_add_tail(&sas_device->list, &ioc->sas_device_init_list);
1090         _scsih_determine_boot_device(ioc, sas_device, 0);
1091         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1092 }
1093
1094
1095 static struct _pcie_device *
1096 __mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1097 {
1098         struct _pcie_device *pcie_device;
1099
1100         assert_spin_locked(&ioc->pcie_device_lock);
1101
1102         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1103                 if (pcie_device->wwid == wwid)
1104                         goto found_device;
1105
1106         list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1107                 if (pcie_device->wwid == wwid)
1108                         goto found_device;
1109
1110         return NULL;
1111
1112 found_device:
1113         pcie_device_get(pcie_device);
1114         return pcie_device;
1115 }
1116
1117
1118 /**
1119  * mpt3sas_get_pdev_by_wwid - pcie device search
1120  * @ioc: per adapter object
1121  * @wwid: wwid
1122  *
1123  * Context: This function will acquire ioc->pcie_device_lock and will release
1124  * before returning the pcie_device object.
1125  *
1126  * This searches for pcie_device based on wwid, then return pcie_device object.
1127  */
1128 static struct _pcie_device *
1129 mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1130 {
1131         struct _pcie_device *pcie_device;
1132         unsigned long flags;
1133
1134         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1135         pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
1136         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1137
1138         return pcie_device;
1139 }
1140
1141
1142 static struct _pcie_device *
1143 __mpt3sas_get_pdev_by_idchannel(struct MPT3SAS_ADAPTER *ioc, int id,
1144         int channel)
1145 {
1146         struct _pcie_device *pcie_device;
1147
1148         assert_spin_locked(&ioc->pcie_device_lock);
1149
1150         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1151                 if (pcie_device->id == id && pcie_device->channel == channel)
1152                         goto found_device;
1153
1154         list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1155                 if (pcie_device->id == id && pcie_device->channel == channel)
1156                         goto found_device;
1157
1158         return NULL;
1159
1160 found_device:
1161         pcie_device_get(pcie_device);
1162         return pcie_device;
1163 }
1164
1165 static struct _pcie_device *
1166 __mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1167 {
1168         struct _pcie_device *pcie_device;
1169
1170         assert_spin_locked(&ioc->pcie_device_lock);
1171
1172         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)
1173                 if (pcie_device->handle == handle)
1174                         goto found_device;
1175
1176         list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)
1177                 if (pcie_device->handle == handle)
1178                         goto found_device;
1179
1180         return NULL;
1181
1182 found_device:
1183         pcie_device_get(pcie_device);
1184         return pcie_device;
1185 }
1186
1187
1188 /**
1189  * mpt3sas_get_pdev_by_handle - pcie device search
1190  * @ioc: per adapter object
1191  * @handle: Firmware device handle
1192  *
1193  * Context: This function will acquire ioc->pcie_device_lock and will release
1194  * before returning the pcie_device object.
1195  *
1196  * This searches for pcie_device based on handle, then return pcie_device
1197  * object.
1198  */
1199 struct _pcie_device *
1200 mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1201 {
1202         struct _pcie_device *pcie_device;
1203         unsigned long flags;
1204
1205         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1206         pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
1207         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1208
1209         return pcie_device;
1210 }
1211
1212 /**
1213  * _scsih_set_nvme_max_shutdown_latency - Update max_shutdown_latency.
1214  * @ioc: per adapter object
1215  * Context: This function will acquire ioc->pcie_device_lock
1216  *
1217  * Update ioc->max_shutdown_latency to that NVMe drives RTD3 Entry Latency
1218  * which has reported maximum among all available NVMe drives.
1219  * Minimum max_shutdown_latency will be six seconds.
1220  */
1221 static void
1222 _scsih_set_nvme_max_shutdown_latency(struct MPT3SAS_ADAPTER *ioc)
1223 {
1224         struct _pcie_device *pcie_device;
1225         unsigned long flags;
1226         u16 shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;
1227
1228         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1229         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
1230                 if (pcie_device->shutdown_latency) {
1231                         if (shutdown_latency < pcie_device->shutdown_latency)
1232                                 shutdown_latency =
1233                                         pcie_device->shutdown_latency;
1234                 }
1235         }
1236         ioc->max_shutdown_latency = shutdown_latency;
1237         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1238 }
1239
1240 /**
1241  * _scsih_pcie_device_remove - remove pcie_device from list.
1242  * @ioc: per adapter object
1243  * @pcie_device: the pcie_device object
1244  * Context: This function will acquire ioc->pcie_device_lock.
1245  *
1246  * If pcie_device is on the list, remove it and decrement its reference count.
1247  */
1248 static void
1249 _scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,
1250         struct _pcie_device *pcie_device)
1251 {
1252         unsigned long flags;
1253         int was_on_pcie_device_list = 0;
1254         u8 update_latency = 0;
1255
1256         if (!pcie_device)
1257                 return;
1258         ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
1259                  pcie_device->handle, (u64)pcie_device->wwid);
1260         if (pcie_device->enclosure_handle != 0)
1261                 ioc_info(ioc, "removing enclosure logical id(0x%016llx), slot(%d)\n",
1262                          (u64)pcie_device->enclosure_logical_id,
1263                          pcie_device->slot);
1264         if (pcie_device->connector_name[0] != '\0')
1265                 ioc_info(ioc, "removing enclosure level(0x%04x), connector name( %s)\n",
1266                          pcie_device->enclosure_level,
1267                          pcie_device->connector_name);
1268
1269         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1270         if (!list_empty(&pcie_device->list)) {
1271                 list_del_init(&pcie_device->list);
1272                 was_on_pcie_device_list = 1;
1273         }
1274         if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)
1275                 update_latency = 1;
1276         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1277         if (was_on_pcie_device_list) {
1278                 kfree(pcie_device->serial_number);
1279                 pcie_device_put(pcie_device);
1280         }
1281
1282         /*
1283          * This device's RTD3 Entry Latency matches IOC's
1284          * max_shutdown_latency. Recalculate IOC's max_shutdown_latency
1285          * from the available drives as current drive is getting removed.
1286          */
1287         if (update_latency)
1288                 _scsih_set_nvme_max_shutdown_latency(ioc);
1289 }
1290
1291
1292 /**
1293  * _scsih_pcie_device_remove_by_handle - removing pcie device object by handle
1294  * @ioc: per adapter object
1295  * @handle: device handle
1296  */
1297 static void
1298 _scsih_pcie_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1299 {
1300         struct _pcie_device *pcie_device;
1301         unsigned long flags;
1302         int was_on_pcie_device_list = 0;
1303         u8 update_latency = 0;
1304
1305         if (ioc->shost_recovery)
1306                 return;
1307
1308         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1309         pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
1310         if (pcie_device) {
1311                 if (!list_empty(&pcie_device->list)) {
1312                         list_del_init(&pcie_device->list);
1313                         was_on_pcie_device_list = 1;
1314                         pcie_device_put(pcie_device);
1315                 }
1316                 if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)
1317                         update_latency = 1;
1318         }
1319         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1320         if (was_on_pcie_device_list) {
1321                 _scsih_pcie_device_remove_from_sml(ioc, pcie_device);
1322                 pcie_device_put(pcie_device);
1323         }
1324
1325         /*
1326          * This device's RTD3 Entry Latency matches IOC's
1327          * max_shutdown_latency. Recalculate IOC's max_shutdown_latency
1328          * from the available drives as current drive is getting removed.
1329          */
1330         if (update_latency)
1331                 _scsih_set_nvme_max_shutdown_latency(ioc);
1332 }
1333
1334 /**
1335  * _scsih_pcie_device_add - add pcie_device object
1336  * @ioc: per adapter object
1337  * @pcie_device: pcie_device object
1338  *
1339  * This is added to the pcie_device_list link list.
1340  */
1341 static void
1342 _scsih_pcie_device_add(struct MPT3SAS_ADAPTER *ioc,
1343         struct _pcie_device *pcie_device)
1344 {
1345         unsigned long flags;
1346
1347         dewtprintk(ioc,
1348                    ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",
1349                             __func__,
1350                             pcie_device->handle, (u64)pcie_device->wwid));
1351         if (pcie_device->enclosure_handle != 0)
1352                 dewtprintk(ioc,
1353                            ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",
1354                                     __func__,
1355                                     (u64)pcie_device->enclosure_logical_id,
1356                                     pcie_device->slot));
1357         if (pcie_device->connector_name[0] != '\0')
1358                 dewtprintk(ioc,
1359                            ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",
1360                                     __func__, pcie_device->enclosure_level,
1361                                     pcie_device->connector_name));
1362
1363         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1364         pcie_device_get(pcie_device);
1365         list_add_tail(&pcie_device->list, &ioc->pcie_device_list);
1366         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1367
1368         if (pcie_device->access_status ==
1369             MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED) {
1370                 clear_bit(pcie_device->handle, ioc->pend_os_device_add);
1371                 return;
1372         }
1373         if (scsi_add_device(ioc->shost, PCIE_CHANNEL, pcie_device->id, 0)) {
1374                 _scsih_pcie_device_remove(ioc, pcie_device);
1375         } else if (!pcie_device->starget) {
1376                 if (!ioc->is_driver_loading) {
1377 /*TODO-- Need to find out whether this condition will occur or not*/
1378                         clear_bit(pcie_device->handle, ioc->pend_os_device_add);
1379                 }
1380         } else
1381                 clear_bit(pcie_device->handle, ioc->pend_os_device_add);
1382 }
1383
1384 /*
1385  * _scsih_pcie_device_init_add - insert pcie_device to the init list.
1386  * @ioc: per adapter object
1387  * @pcie_device: the pcie_device object
1388  * Context: This function will acquire ioc->pcie_device_lock.
1389  *
1390  * Adding new object at driver load time to the ioc->pcie_device_init_list.
1391  */
1392 static void
1393 _scsih_pcie_device_init_add(struct MPT3SAS_ADAPTER *ioc,
1394                                 struct _pcie_device *pcie_device)
1395 {
1396         unsigned long flags;
1397
1398         dewtprintk(ioc,
1399                    ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",
1400                             __func__,
1401                             pcie_device->handle, (u64)pcie_device->wwid));
1402         if (pcie_device->enclosure_handle != 0)
1403                 dewtprintk(ioc,
1404                            ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",
1405                                     __func__,
1406                                     (u64)pcie_device->enclosure_logical_id,
1407                                     pcie_device->slot));
1408         if (pcie_device->connector_name[0] != '\0')
1409                 dewtprintk(ioc,
1410                            ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",
1411                                     __func__, pcie_device->enclosure_level,
1412                                     pcie_device->connector_name));
1413
1414         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1415         pcie_device_get(pcie_device);
1416         list_add_tail(&pcie_device->list, &ioc->pcie_device_init_list);
1417         if (pcie_device->access_status !=
1418             MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED)
1419                 _scsih_determine_boot_device(ioc, pcie_device, PCIE_CHANNEL);
1420         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1421 }
1422 /**
1423  * _scsih_raid_device_find_by_id - raid device search
1424  * @ioc: per adapter object
1425  * @id: sas device target id
1426  * @channel: sas device channel
1427  * Context: Calling function should acquire ioc->raid_device_lock
1428  *
1429  * This searches for raid_device based on target id, then return raid_device
1430  * object.
1431  */
1432 static struct _raid_device *
1433 _scsih_raid_device_find_by_id(struct MPT3SAS_ADAPTER *ioc, int id, int channel)
1434 {
1435         struct _raid_device *raid_device, *r;
1436
1437         r = NULL;
1438         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1439                 if (raid_device->id == id && raid_device->channel == channel) {
1440                         r = raid_device;
1441                         goto out;
1442                 }
1443         }
1444
1445  out:
1446         return r;
1447 }
1448
1449 /**
1450  * mpt3sas_raid_device_find_by_handle - raid device search
1451  * @ioc: per adapter object
1452  * @handle: sas device handle (assigned by firmware)
1453  * Context: Calling function should acquire ioc->raid_device_lock
1454  *
1455  * This searches for raid_device based on handle, then return raid_device
1456  * object.
1457  */
1458 struct _raid_device *
1459 mpt3sas_raid_device_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1460 {
1461         struct _raid_device *raid_device, *r;
1462
1463         r = NULL;
1464         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1465                 if (raid_device->handle != handle)
1466                         continue;
1467                 r = raid_device;
1468                 goto out;
1469         }
1470
1471  out:
1472         return r;
1473 }
1474
1475 /**
1476  * _scsih_raid_device_find_by_wwid - raid device search
1477  * @ioc: per adapter object
1478  * @wwid: ?
1479  * Context: Calling function should acquire ioc->raid_device_lock
1480  *
1481  * This searches for raid_device based on wwid, then return raid_device
1482  * object.
1483  */
1484 static struct _raid_device *
1485 _scsih_raid_device_find_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)
1486 {
1487         struct _raid_device *raid_device, *r;
1488
1489         r = NULL;
1490         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
1491                 if (raid_device->wwid != wwid)
1492                         continue;
1493                 r = raid_device;
1494                 goto out;
1495         }
1496
1497  out:
1498         return r;
1499 }
1500
1501 /**
1502  * _scsih_raid_device_add - add raid_device object
1503  * @ioc: per adapter object
1504  * @raid_device: raid_device object
1505  *
1506  * This is added to the raid_device_list link list.
1507  */
1508 static void
1509 _scsih_raid_device_add(struct MPT3SAS_ADAPTER *ioc,
1510         struct _raid_device *raid_device)
1511 {
1512         unsigned long flags;
1513
1514         dewtprintk(ioc,
1515                    ioc_info(ioc, "%s: handle(0x%04x), wwid(0x%016llx)\n",
1516                             __func__,
1517                             raid_device->handle, (u64)raid_device->wwid));
1518
1519         spin_lock_irqsave(&ioc->raid_device_lock, flags);
1520         list_add_tail(&raid_device->list, &ioc->raid_device_list);
1521         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1522 }
1523
1524 /**
1525  * _scsih_raid_device_remove - delete raid_device object
1526  * @ioc: per adapter object
1527  * @raid_device: raid_device object
1528  *
1529  */
1530 static void
1531 _scsih_raid_device_remove(struct MPT3SAS_ADAPTER *ioc,
1532         struct _raid_device *raid_device)
1533 {
1534         unsigned long flags;
1535
1536         spin_lock_irqsave(&ioc->raid_device_lock, flags);
1537         list_del(&raid_device->list);
1538         kfree(raid_device);
1539         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1540 }
1541
1542 /**
1543  * mpt3sas_scsih_expander_find_by_handle - expander device search
1544  * @ioc: per adapter object
1545  * @handle: expander handle (assigned by firmware)
1546  * Context: Calling function should acquire ioc->sas_device_lock
1547  *
1548  * This searches for expander device based on handle, then returns the
1549  * sas_node object.
1550  */
1551 struct _sas_node *
1552 mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1553 {
1554         struct _sas_node *sas_expander, *r;
1555
1556         r = NULL;
1557         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
1558                 if (sas_expander->handle != handle)
1559                         continue;
1560                 r = sas_expander;
1561                 goto out;
1562         }
1563  out:
1564         return r;
1565 }
1566
1567 /**
1568  * mpt3sas_scsih_enclosure_find_by_handle - exclosure device search
1569  * @ioc: per adapter object
1570  * @handle: enclosure handle (assigned by firmware)
1571  * Context: Calling function should acquire ioc->sas_device_lock
1572  *
1573  * This searches for enclosure device based on handle, then returns the
1574  * enclosure object.
1575  */
1576 static struct _enclosure_node *
1577 mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)
1578 {
1579         struct _enclosure_node *enclosure_dev, *r;
1580
1581         r = NULL;
1582         list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {
1583                 if (le16_to_cpu(enclosure_dev->pg0.EnclosureHandle) != handle)
1584                         continue;
1585                 r = enclosure_dev;
1586                 goto out;
1587         }
1588 out:
1589         return r;
1590 }
1591 /**
1592  * mpt3sas_scsih_expander_find_by_sas_address - expander device search
1593  * @ioc: per adapter object
1594  * @sas_address: sas address
1595  * @port: hba port entry
1596  * Context: Calling function should acquire ioc->sas_node_lock.
1597  *
1598  * This searches for expander device based on sas_address & port number,
1599  * then returns the sas_node object.
1600  */
1601 struct _sas_node *
1602 mpt3sas_scsih_expander_find_by_sas_address(struct MPT3SAS_ADAPTER *ioc,
1603         u64 sas_address, struct hba_port *port)
1604 {
1605         struct _sas_node *sas_expander, *r = NULL;
1606
1607         if (!port)
1608                 return r;
1609
1610         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
1611                 if (sas_expander->sas_address != sas_address)
1612                         continue;
1613                 if (sas_expander->port != port)
1614                         continue;
1615                 r = sas_expander;
1616                 goto out;
1617         }
1618  out:
1619         return r;
1620 }
1621
1622 /**
1623  * _scsih_expander_node_add - insert expander device to the list.
1624  * @ioc: per adapter object
1625  * @sas_expander: the sas_device object
1626  * Context: This function will acquire ioc->sas_node_lock.
1627  *
1628  * Adding new object to the ioc->sas_expander_list.
1629  */
1630 static void
1631 _scsih_expander_node_add(struct MPT3SAS_ADAPTER *ioc,
1632         struct _sas_node *sas_expander)
1633 {
1634         unsigned long flags;
1635
1636         spin_lock_irqsave(&ioc->sas_node_lock, flags);
1637         list_add_tail(&sas_expander->list, &ioc->sas_expander_list);
1638         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
1639 }
1640
1641 /**
1642  * _scsih_is_end_device - determines if device is an end device
1643  * @device_info: bitfield providing information about the device.
1644  * Context: none
1645  *
1646  * Return: 1 if end device.
1647  */
1648 static int
1649 _scsih_is_end_device(u32 device_info)
1650 {
1651         if (device_info & MPI2_SAS_DEVICE_INFO_END_DEVICE &&
1652                 ((device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET) |
1653                 (device_info & MPI2_SAS_DEVICE_INFO_STP_TARGET) |
1654                 (device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)))
1655                 return 1;
1656         else
1657                 return 0;
1658 }
1659
1660 /**
1661  * _scsih_is_nvme_pciescsi_device - determines if
1662  *                      device is an pcie nvme/scsi device
1663  * @device_info: bitfield providing information about the device.
1664  * Context: none
1665  *
1666  * Returns 1 if device is pcie device type nvme/scsi.
1667  */
1668 static int
1669 _scsih_is_nvme_pciescsi_device(u32 device_info)
1670 {
1671         if (((device_info & MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE)
1672             == MPI26_PCIE_DEVINFO_NVME) ||
1673             ((device_info & MPI26_PCIE_DEVINFO_MASK_DEVICE_TYPE)
1674             == MPI26_PCIE_DEVINFO_SCSI))
1675                 return 1;
1676         else
1677                 return 0;
1678 }
1679
1680 /**
1681  * _scsih_scsi_lookup_find_by_target - search for matching channel:id
1682  * @ioc: per adapter object
1683  * @id: target id
1684  * @channel: channel
1685  * Context: This function will acquire ioc->scsi_lookup_lock.
1686  *
1687  * This will search for a matching channel:id in the scsi_lookup array,
1688  * returning 1 if found.
1689  */
1690 static u8
1691 _scsih_scsi_lookup_find_by_target(struct MPT3SAS_ADAPTER *ioc, int id,
1692         int channel)
1693 {
1694         int smid;
1695         struct scsi_cmnd *scmd;
1696
1697         for (smid = 1;
1698              smid <= ioc->shost->can_queue; smid++) {
1699                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1700                 if (!scmd)
1701                         continue;
1702                 if (scmd->device->id == id &&
1703                     scmd->device->channel == channel)
1704                         return 1;
1705         }
1706         return 0;
1707 }
1708
1709 /**
1710  * _scsih_scsi_lookup_find_by_lun - search for matching channel:id:lun
1711  * @ioc: per adapter object
1712  * @id: target id
1713  * @lun: lun number
1714  * @channel: channel
1715  * Context: This function will acquire ioc->scsi_lookup_lock.
1716  *
1717  * This will search for a matching channel:id:lun in the scsi_lookup array,
1718  * returning 1 if found.
1719  */
1720 static u8
1721 _scsih_scsi_lookup_find_by_lun(struct MPT3SAS_ADAPTER *ioc, int id,
1722         unsigned int lun, int channel)
1723 {
1724         int smid;
1725         struct scsi_cmnd *scmd;
1726
1727         for (smid = 1; smid <= ioc->shost->can_queue; smid++) {
1728
1729                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1730                 if (!scmd)
1731                         continue;
1732                 if (scmd->device->id == id &&
1733                     scmd->device->channel == channel &&
1734                     scmd->device->lun == lun)
1735                         return 1;
1736         }
1737         return 0;
1738 }
1739
1740 /**
1741  * mpt3sas_scsih_scsi_lookup_get - returns scmd entry
1742  * @ioc: per adapter object
1743  * @smid: system request message index
1744  *
1745  * Return: the smid stored scmd pointer.
1746  * Then will dereference the stored scmd pointer.
1747  */
1748 struct scsi_cmnd *
1749 mpt3sas_scsih_scsi_lookup_get(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1750 {
1751         struct scsi_cmnd *scmd = NULL;
1752         struct scsiio_tracker *st;
1753         Mpi25SCSIIORequest_t *mpi_request;
1754         u16 tag = smid - 1;
1755
1756         if (smid > 0  &&
1757             smid <= ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT) {
1758                 u32 unique_tag =
1759                     ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag;
1760
1761                 mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
1762
1763                 /*
1764                  * If SCSI IO request is outstanding at driver level then
1765                  * DevHandle filed must be non-zero. If DevHandle is zero
1766                  * then it means that this smid is free at driver level,
1767                  * so return NULL.
1768                  */
1769                 if (!mpi_request->DevHandle)
1770                         return scmd;
1771
1772                 scmd = scsi_host_find_tag(ioc->shost, unique_tag);
1773                 if (scmd) {
1774                         st = scsi_cmd_priv(scmd);
1775                         if (st->cb_idx == 0xFF || st->smid == 0)
1776                                 scmd = NULL;
1777                 }
1778         }
1779         return scmd;
1780 }
1781
1782 /**
1783  * scsih_change_queue_depth - setting device queue depth
1784  * @sdev: scsi device struct
1785  * @qdepth: requested queue depth
1786  *
1787  * Return: queue depth.
1788  */
1789 static int
1790 scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
1791 {
1792         struct Scsi_Host *shost = sdev->host;
1793         int max_depth;
1794         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1795         struct MPT3SAS_DEVICE *sas_device_priv_data;
1796         struct MPT3SAS_TARGET *sas_target_priv_data;
1797         struct _sas_device *sas_device;
1798         unsigned long flags;
1799
1800         max_depth = shost->can_queue;
1801
1802         /*
1803          * limit max device queue for SATA to 32 if enable_sdev_max_qd
1804          * is disabled.
1805          */
1806         if (ioc->enable_sdev_max_qd || ioc->is_gen35_ioc)
1807                 goto not_sata;
1808
1809         sas_device_priv_data = sdev->hostdata;
1810         if (!sas_device_priv_data)
1811                 goto not_sata;
1812         sas_target_priv_data = sas_device_priv_data->sas_target;
1813         if (!sas_target_priv_data)
1814                 goto not_sata;
1815         if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME))
1816                 goto not_sata;
1817
1818         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1819         sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);
1820         if (sas_device) {
1821                 if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
1822                         max_depth = MPT3SAS_SATA_QUEUE_DEPTH;
1823
1824                 sas_device_put(sas_device);
1825         }
1826         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1827
1828  not_sata:
1829
1830         if (!sdev->tagged_supported)
1831                 max_depth = 1;
1832         if (qdepth > max_depth)
1833                 qdepth = max_depth;
1834         scsi_change_queue_depth(sdev, qdepth);
1835         sdev_printk(KERN_INFO, sdev,
1836             "qdepth(%d), tagged(%d), scsi_level(%d), cmd_que(%d)\n",
1837             sdev->queue_depth, sdev->tagged_supported,
1838             sdev->scsi_level, ((sdev->inquiry[7] & 2) >> 1));
1839         return sdev->queue_depth;
1840 }
1841
1842 /**
1843  * mpt3sas_scsih_change_queue_depth - setting device queue depth
1844  * @sdev: scsi device struct
1845  * @qdepth: requested queue depth
1846  *
1847  * Returns nothing.
1848  */
1849 void
1850 mpt3sas_scsih_change_queue_depth(struct scsi_device *sdev, int qdepth)
1851 {
1852         struct Scsi_Host *shost = sdev->host;
1853         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1854
1855         if (ioc->enable_sdev_max_qd)
1856                 qdepth = shost->can_queue;
1857
1858         scsih_change_queue_depth(sdev, qdepth);
1859 }
1860
1861 /**
1862  * scsih_target_alloc - target add routine
1863  * @starget: scsi target struct
1864  *
1865  * Return: 0 if ok. Any other return is assumed to be an error and
1866  * the device is ignored.
1867  */
1868 static int
1869 scsih_target_alloc(struct scsi_target *starget)
1870 {
1871         struct Scsi_Host *shost = dev_to_shost(&starget->dev);
1872         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1873         struct MPT3SAS_TARGET *sas_target_priv_data;
1874         struct _sas_device *sas_device;
1875         struct _raid_device *raid_device;
1876         struct _pcie_device *pcie_device;
1877         unsigned long flags;
1878         struct sas_rphy *rphy;
1879
1880         sas_target_priv_data = kzalloc(sizeof(*sas_target_priv_data),
1881                                        GFP_KERNEL);
1882         if (!sas_target_priv_data)
1883                 return -ENOMEM;
1884
1885         starget->hostdata = sas_target_priv_data;
1886         sas_target_priv_data->starget = starget;
1887         sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
1888
1889         /* RAID volumes */
1890         if (starget->channel == RAID_CHANNEL) {
1891                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
1892                 raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,
1893                     starget->channel);
1894                 if (raid_device) {
1895                         sas_target_priv_data->handle = raid_device->handle;
1896                         sas_target_priv_data->sas_address = raid_device->wwid;
1897                         sas_target_priv_data->flags |= MPT_TARGET_FLAGS_VOLUME;
1898                         if (ioc->is_warpdrive)
1899                                 sas_target_priv_data->raid_device = raid_device;
1900                         raid_device->starget = starget;
1901                 }
1902                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1903                 return 0;
1904         }
1905
1906         /* PCIe devices */
1907         if (starget->channel == PCIE_CHANNEL) {
1908                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1909                 pcie_device = __mpt3sas_get_pdev_by_idchannel(ioc, starget->id,
1910                         starget->channel);
1911                 if (pcie_device) {
1912                         sas_target_priv_data->handle = pcie_device->handle;
1913                         sas_target_priv_data->sas_address = pcie_device->wwid;
1914                         sas_target_priv_data->port = NULL;
1915                         sas_target_priv_data->pcie_dev = pcie_device;
1916                         pcie_device->starget = starget;
1917                         pcie_device->id = starget->id;
1918                         pcie_device->channel = starget->channel;
1919                         sas_target_priv_data->flags |=
1920                                 MPT_TARGET_FLAGS_PCIE_DEVICE;
1921                         if (pcie_device->fast_path)
1922                                 sas_target_priv_data->flags |=
1923                                         MPT_TARGET_FASTPATH_IO;
1924                 }
1925                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
1926                 return 0;
1927         }
1928
1929         /* sas/sata devices */
1930         spin_lock_irqsave(&ioc->sas_device_lock, flags);
1931         rphy = dev_to_rphy(starget->dev.parent);
1932         sas_device = __mpt3sas_get_sdev_by_rphy(ioc, rphy);
1933
1934         if (sas_device) {
1935                 sas_target_priv_data->handle = sas_device->handle;
1936                 sas_target_priv_data->sas_address = sas_device->sas_address;
1937                 sas_target_priv_data->port = sas_device->port;
1938                 sas_target_priv_data->sas_dev = sas_device;
1939                 sas_device->starget = starget;
1940                 sas_device->id = starget->id;
1941                 sas_device->channel = starget->channel;
1942                 if (test_bit(sas_device->handle, ioc->pd_handles))
1943                         sas_target_priv_data->flags |=
1944                             MPT_TARGET_FLAGS_RAID_COMPONENT;
1945                 if (sas_device->fast_path)
1946                         sas_target_priv_data->flags |=
1947                                         MPT_TARGET_FASTPATH_IO;
1948         }
1949         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
1950
1951         return 0;
1952 }
1953
1954 /**
1955  * scsih_target_destroy - target destroy routine
1956  * @starget: scsi target struct
1957  */
1958 static void
1959 scsih_target_destroy(struct scsi_target *starget)
1960 {
1961         struct Scsi_Host *shost = dev_to_shost(&starget->dev);
1962         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
1963         struct MPT3SAS_TARGET *sas_target_priv_data;
1964         struct _sas_device *sas_device;
1965         struct _raid_device *raid_device;
1966         struct _pcie_device *pcie_device;
1967         unsigned long flags;
1968
1969         sas_target_priv_data = starget->hostdata;
1970         if (!sas_target_priv_data)
1971                 return;
1972
1973         if (starget->channel == RAID_CHANNEL) {
1974                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
1975                 raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,
1976                     starget->channel);
1977                 if (raid_device) {
1978                         raid_device->starget = NULL;
1979                         raid_device->sdev = NULL;
1980                 }
1981                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
1982                 goto out;
1983         }
1984
1985         if (starget->channel == PCIE_CHANNEL) {
1986                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
1987                 pcie_device = __mpt3sas_get_pdev_from_target(ioc,
1988                                                         sas_target_priv_data);
1989                 if (pcie_device && (pcie_device->starget == starget) &&
1990                         (pcie_device->id == starget->id) &&
1991                         (pcie_device->channel == starget->channel))
1992                         pcie_device->starget = NULL;
1993
1994                 if (pcie_device) {
1995                         /*
1996                          * Corresponding get() is in _scsih_target_alloc()
1997                          */
1998                         sas_target_priv_data->pcie_dev = NULL;
1999                         pcie_device_put(pcie_device);
2000                         pcie_device_put(pcie_device);
2001                 }
2002                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2003                 goto out;
2004         }
2005
2006         spin_lock_irqsave(&ioc->sas_device_lock, flags);
2007         sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);
2008         if (sas_device && (sas_device->starget == starget) &&
2009             (sas_device->id == starget->id) &&
2010             (sas_device->channel == starget->channel))
2011                 sas_device->starget = NULL;
2012
2013         if (sas_device) {
2014                 /*
2015                  * Corresponding get() is in _scsih_target_alloc()
2016                  */
2017                 sas_target_priv_data->sas_dev = NULL;
2018                 sas_device_put(sas_device);
2019
2020                 sas_device_put(sas_device);
2021         }
2022         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2023
2024  out:
2025         kfree(sas_target_priv_data);
2026         starget->hostdata = NULL;
2027 }
2028
2029 /**
2030  * scsih_slave_alloc - device add routine
2031  * @sdev: scsi device struct
2032  *
2033  * Return: 0 if ok. Any other return is assumed to be an error and
2034  * the device is ignored.
2035  */
2036 static int
2037 scsih_slave_alloc(struct scsi_device *sdev)
2038 {
2039         struct Scsi_Host *shost;
2040         struct MPT3SAS_ADAPTER *ioc;
2041         struct MPT3SAS_TARGET *sas_target_priv_data;
2042         struct MPT3SAS_DEVICE *sas_device_priv_data;
2043         struct scsi_target *starget;
2044         struct _raid_device *raid_device;
2045         struct _sas_device *sas_device;
2046         struct _pcie_device *pcie_device;
2047         unsigned long flags;
2048
2049         sas_device_priv_data = kzalloc(sizeof(*sas_device_priv_data),
2050                                        GFP_KERNEL);
2051         if (!sas_device_priv_data)
2052                 return -ENOMEM;
2053
2054         sas_device_priv_data->lun = sdev->lun;
2055         sas_device_priv_data->flags = MPT_DEVICE_FLAGS_INIT;
2056
2057         starget = scsi_target(sdev);
2058         sas_target_priv_data = starget->hostdata;
2059         sas_target_priv_data->num_luns++;
2060         sas_device_priv_data->sas_target = sas_target_priv_data;
2061         sdev->hostdata = sas_device_priv_data;
2062         if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT))
2063                 sdev->no_uld_attach = 1;
2064
2065         shost = dev_to_shost(&starget->dev);
2066         ioc = shost_priv(shost);
2067         if (starget->channel == RAID_CHANNEL) {
2068                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
2069                 raid_device = _scsih_raid_device_find_by_id(ioc,
2070                     starget->id, starget->channel);
2071                 if (raid_device)
2072                         raid_device->sdev = sdev; /* raid is single lun */
2073                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2074         }
2075         if (starget->channel == PCIE_CHANNEL) {
2076                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2077                 pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,
2078                                 sas_target_priv_data->sas_address);
2079                 if (pcie_device && (pcie_device->starget == NULL)) {
2080                         sdev_printk(KERN_INFO, sdev,
2081                             "%s : pcie_device->starget set to starget @ %d\n",
2082                             __func__, __LINE__);
2083                         pcie_device->starget = starget;
2084                 }
2085
2086                 if (pcie_device)
2087                         pcie_device_put(pcie_device);
2088                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2089
2090         } else  if (!(sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)) {
2091                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
2092                 sas_device = __mpt3sas_get_sdev_by_addr(ioc,
2093                     sas_target_priv_data->sas_address,
2094                     sas_target_priv_data->port);
2095                 if (sas_device && (sas_device->starget == NULL)) {
2096                         sdev_printk(KERN_INFO, sdev,
2097                         "%s : sas_device->starget set to starget @ %d\n",
2098                              __func__, __LINE__);
2099                         sas_device->starget = starget;
2100                 }
2101
2102                 if (sas_device)
2103                         sas_device_put(sas_device);
2104
2105                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2106         }
2107
2108         return 0;
2109 }
2110
2111 /**
2112  * scsih_slave_destroy - device destroy routine
2113  * @sdev: scsi device struct
2114  */
2115 static void
2116 scsih_slave_destroy(struct scsi_device *sdev)
2117 {
2118         struct MPT3SAS_TARGET *sas_target_priv_data;
2119         struct scsi_target *starget;
2120         struct Scsi_Host *shost;
2121         struct MPT3SAS_ADAPTER *ioc;
2122         struct _sas_device *sas_device;
2123         struct _pcie_device *pcie_device;
2124         unsigned long flags;
2125
2126         if (!sdev->hostdata)
2127                 return;
2128
2129         starget = scsi_target(sdev);
2130         sas_target_priv_data = starget->hostdata;
2131         sas_target_priv_data->num_luns--;
2132
2133         shost = dev_to_shost(&starget->dev);
2134         ioc = shost_priv(shost);
2135
2136         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
2137                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2138                 pcie_device = __mpt3sas_get_pdev_from_target(ioc,
2139                                 sas_target_priv_data);
2140                 if (pcie_device && !sas_target_priv_data->num_luns)
2141                         pcie_device->starget = NULL;
2142
2143                 if (pcie_device)
2144                         pcie_device_put(pcie_device);
2145
2146                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2147
2148         } else if (!(sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)) {
2149                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
2150                 sas_device = __mpt3sas_get_sdev_from_target(ioc,
2151                                 sas_target_priv_data);
2152                 if (sas_device && !sas_target_priv_data->num_luns)
2153                         sas_device->starget = NULL;
2154
2155                 if (sas_device)
2156                         sas_device_put(sas_device);
2157                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2158         }
2159
2160         kfree(sdev->hostdata);
2161         sdev->hostdata = NULL;
2162 }
2163
2164 /**
2165  * _scsih_display_sata_capabilities - sata capabilities
2166  * @ioc: per adapter object
2167  * @handle: device handle
2168  * @sdev: scsi device struct
2169  */
2170 static void
2171 _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,
2172         u16 handle, struct scsi_device *sdev)
2173 {
2174         Mpi2ConfigReply_t mpi_reply;
2175         Mpi2SasDevicePage0_t sas_device_pg0;
2176         u32 ioc_status;
2177         u16 flags;
2178         u32 device_info;
2179
2180         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
2181             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
2182                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2183                         __FILE__, __LINE__, __func__);
2184                 return;
2185         }
2186
2187         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
2188             MPI2_IOCSTATUS_MASK;
2189         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
2190                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2191                         __FILE__, __LINE__, __func__);
2192                 return;
2193         }
2194
2195         flags = le16_to_cpu(sas_device_pg0.Flags);
2196         device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
2197
2198         sdev_printk(KERN_INFO, sdev,
2199             "atapi(%s), ncq(%s), asyn_notify(%s), smart(%s), fua(%s), "
2200             "sw_preserve(%s)\n",
2201             (device_info & MPI2_SAS_DEVICE_INFO_ATAPI_DEVICE) ? "y" : "n",
2202             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_NCQ_SUPPORTED) ? "y" : "n",
2203             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_ASYNCHRONOUS_NOTIFY) ? "y" :
2204             "n",
2205             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_SMART_SUPPORTED) ? "y" : "n",
2206             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_FUA_SUPPORTED) ? "y" : "n",
2207             (flags & MPI2_SAS_DEVICE0_FLAGS_SATA_SW_PRESERVE) ? "y" : "n");
2208 }
2209
2210 /*
2211  * raid transport support -
2212  * Enabled for SLES11 and newer, in older kernels the driver will panic when
2213  * unloading the driver followed by a load - I believe that the subroutine
2214  * raid_class_release() is not cleaning up properly.
2215  */
2216
2217 /**
2218  * scsih_is_raid - return boolean indicating device is raid volume
2219  * @dev: the device struct object
2220  */
2221 static int
2222 scsih_is_raid(struct device *dev)
2223 {
2224         struct scsi_device *sdev = to_scsi_device(dev);
2225         struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);
2226
2227         if (ioc->is_warpdrive)
2228                 return 0;
2229         return (sdev->channel == RAID_CHANNEL) ? 1 : 0;
2230 }
2231
2232 static int
2233 scsih_is_nvme(struct device *dev)
2234 {
2235         struct scsi_device *sdev = to_scsi_device(dev);
2236
2237         return (sdev->channel == PCIE_CHANNEL) ? 1 : 0;
2238 }
2239
2240 /**
2241  * scsih_get_resync - get raid volume resync percent complete
2242  * @dev: the device struct object
2243  */
2244 static void
2245 scsih_get_resync(struct device *dev)
2246 {
2247         struct scsi_device *sdev = to_scsi_device(dev);
2248         struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);
2249         static struct _raid_device *raid_device;
2250         unsigned long flags;
2251         Mpi2RaidVolPage0_t vol_pg0;
2252         Mpi2ConfigReply_t mpi_reply;
2253         u32 volume_status_flags;
2254         u8 percent_complete;
2255         u16 handle;
2256
2257         percent_complete = 0;
2258         handle = 0;
2259         if (ioc->is_warpdrive)
2260                 goto out;
2261
2262         spin_lock_irqsave(&ioc->raid_device_lock, flags);
2263         raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,
2264             sdev->channel);
2265         if (raid_device) {
2266                 handle = raid_device->handle;
2267                 percent_complete = raid_device->percent_complete;
2268         }
2269         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2270
2271         if (!handle)
2272                 goto out;
2273
2274         if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,
2275              MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
2276              sizeof(Mpi2RaidVolPage0_t))) {
2277                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2278                         __FILE__, __LINE__, __func__);
2279                 percent_complete = 0;
2280                 goto out;
2281         }
2282
2283         volume_status_flags = le32_to_cpu(vol_pg0.VolumeStatusFlags);
2284         if (!(volume_status_flags &
2285             MPI2_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS))
2286                 percent_complete = 0;
2287
2288  out:
2289
2290         switch (ioc->hba_mpi_version_belonged) {
2291         case MPI2_VERSION:
2292                 raid_set_resync(mpt2sas_raid_template, dev, percent_complete);
2293                 break;
2294         case MPI25_VERSION:
2295         case MPI26_VERSION:
2296                 raid_set_resync(mpt3sas_raid_template, dev, percent_complete);
2297                 break;
2298         }
2299 }
2300
2301 /**
2302  * scsih_get_state - get raid volume level
2303  * @dev: the device struct object
2304  */
2305 static void
2306 scsih_get_state(struct device *dev)
2307 {
2308         struct scsi_device *sdev = to_scsi_device(dev);
2309         struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);
2310         static struct _raid_device *raid_device;
2311         unsigned long flags;
2312         Mpi2RaidVolPage0_t vol_pg0;
2313         Mpi2ConfigReply_t mpi_reply;
2314         u32 volstate;
2315         enum raid_state state = RAID_STATE_UNKNOWN;
2316         u16 handle = 0;
2317
2318         spin_lock_irqsave(&ioc->raid_device_lock, flags);
2319         raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,
2320             sdev->channel);
2321         if (raid_device)
2322                 handle = raid_device->handle;
2323         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2324
2325         if (!raid_device)
2326                 goto out;
2327
2328         if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,
2329              MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
2330              sizeof(Mpi2RaidVolPage0_t))) {
2331                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
2332                         __FILE__, __LINE__, __func__);
2333                 goto out;
2334         }
2335
2336         volstate = le32_to_cpu(vol_pg0.VolumeStatusFlags);
2337         if (volstate & MPI2_RAIDVOL0_STATUS_FLAG_RESYNC_IN_PROGRESS) {
2338                 state = RAID_STATE_RESYNCING;
2339                 goto out;
2340         }
2341
2342         switch (vol_pg0.VolumeState) {
2343         case MPI2_RAID_VOL_STATE_OPTIMAL:
2344         case MPI2_RAID_VOL_STATE_ONLINE:
2345                 state = RAID_STATE_ACTIVE;
2346                 break;
2347         case  MPI2_RAID_VOL_STATE_DEGRADED:
2348                 state = RAID_STATE_DEGRADED;
2349                 break;
2350         case MPI2_RAID_VOL_STATE_FAILED:
2351         case MPI2_RAID_VOL_STATE_MISSING:
2352                 state = RAID_STATE_OFFLINE;
2353                 break;
2354         }
2355  out:
2356         switch (ioc->hba_mpi_version_belonged) {
2357         case MPI2_VERSION:
2358                 raid_set_state(mpt2sas_raid_template, dev, state);
2359                 break;
2360         case MPI25_VERSION:
2361         case MPI26_VERSION:
2362                 raid_set_state(mpt3sas_raid_template, dev, state);
2363                 break;
2364         }
2365 }
2366
2367 /**
2368  * _scsih_set_level - set raid level
2369  * @ioc: ?
2370  * @sdev: scsi device struct
2371  * @volume_type: volume type
2372  */
2373 static void
2374 _scsih_set_level(struct MPT3SAS_ADAPTER *ioc,
2375         struct scsi_device *sdev, u8 volume_type)
2376 {
2377         enum raid_level level = RAID_LEVEL_UNKNOWN;
2378
2379         switch (volume_type) {
2380         case MPI2_RAID_VOL_TYPE_RAID0:
2381                 level = RAID_LEVEL_0;
2382                 break;
2383         case MPI2_RAID_VOL_TYPE_RAID10:
2384                 level = RAID_LEVEL_10;
2385                 break;
2386         case MPI2_RAID_VOL_TYPE_RAID1E:
2387                 level = RAID_LEVEL_1E;
2388                 break;
2389         case MPI2_RAID_VOL_TYPE_RAID1:
2390                 level = RAID_LEVEL_1;
2391                 break;
2392         }
2393
2394         switch (ioc->hba_mpi_version_belonged) {
2395         case MPI2_VERSION:
2396                 raid_set_level(mpt2sas_raid_template,
2397                         &sdev->sdev_gendev, level);
2398                 break;
2399         case MPI25_VERSION:
2400         case MPI26_VERSION:
2401                 raid_set_level(mpt3sas_raid_template,
2402                         &sdev->sdev_gendev, level);
2403                 break;
2404         }
2405 }
2406
2407
2408 /**
2409  * _scsih_get_volume_capabilities - volume capabilities
2410  * @ioc: per adapter object
2411  * @raid_device: the raid_device object
2412  *
2413  * Return: 0 for success, else 1
2414  */
2415 static int
2416 _scsih_get_volume_capabilities(struct MPT3SAS_ADAPTER *ioc,
2417         struct _raid_device *raid_device)
2418 {
2419         Mpi2RaidVolPage0_t *vol_pg0;
2420         Mpi2RaidPhysDiskPage0_t pd_pg0;
2421         Mpi2SasDevicePage0_t sas_device_pg0;
2422         Mpi2ConfigReply_t mpi_reply;
2423         u16 sz;
2424         u8 num_pds;
2425
2426         if ((mpt3sas_config_get_number_pds(ioc, raid_device->handle,
2427             &num_pds)) || !num_pds) {
2428                 dfailprintk(ioc,
2429                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2430                                      __FILE__, __LINE__, __func__));
2431                 return 1;
2432         }
2433
2434         raid_device->num_pds = num_pds;
2435         sz = offsetof(Mpi2RaidVolPage0_t, PhysDisk) + (num_pds *
2436             sizeof(Mpi2RaidVol0PhysDisk_t));
2437         vol_pg0 = kzalloc(sz, GFP_KERNEL);
2438         if (!vol_pg0) {
2439                 dfailprintk(ioc,
2440                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2441                                      __FILE__, __LINE__, __func__));
2442                 return 1;
2443         }
2444
2445         if ((mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, vol_pg0,
2446              MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, raid_device->handle, sz))) {
2447                 dfailprintk(ioc,
2448                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2449                                      __FILE__, __LINE__, __func__));
2450                 kfree(vol_pg0);
2451                 return 1;
2452         }
2453
2454         raid_device->volume_type = vol_pg0->VolumeType;
2455
2456         /* figure out what the underlying devices are by
2457          * obtaining the device_info bits for the 1st device
2458          */
2459         if (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
2460             &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_PHYSDISKNUM,
2461             vol_pg0->PhysDisk[0].PhysDiskNum))) {
2462                 if (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
2463                     &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
2464                     le16_to_cpu(pd_pg0.DevHandle)))) {
2465                         raid_device->device_info =
2466                             le32_to_cpu(sas_device_pg0.DeviceInfo);
2467                 }
2468         }
2469
2470         kfree(vol_pg0);
2471         return 0;
2472 }
2473
2474 /**
2475  * _scsih_enable_tlr - setting TLR flags
2476  * @ioc: per adapter object
2477  * @sdev: scsi device struct
2478  *
2479  * Enabling Transaction Layer Retries for tape devices when
2480  * vpd page 0x90 is present
2481  *
2482  */
2483 static void
2484 _scsih_enable_tlr(struct MPT3SAS_ADAPTER *ioc, struct scsi_device *sdev)
2485 {
2486
2487         /* only for TAPE */
2488         if (sdev->type != TYPE_TAPE)
2489                 return;
2490
2491         if (!(ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR))
2492                 return;
2493
2494         sas_enable_tlr(sdev);
2495         sdev_printk(KERN_INFO, sdev, "TLR %s\n",
2496             sas_is_tlr_enabled(sdev) ? "Enabled" : "Disabled");
2497         return;
2498
2499 }
2500
2501 /**
2502  * scsih_slave_configure - device configure routine.
2503  * @sdev: scsi device struct
2504  *
2505  * Return: 0 if ok. Any other return is assumed to be an error and
2506  * the device is ignored.
2507  */
2508 static int
2509 scsih_slave_configure(struct scsi_device *sdev)
2510 {
2511         struct Scsi_Host *shost = sdev->host;
2512         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
2513         struct MPT3SAS_DEVICE *sas_device_priv_data;
2514         struct MPT3SAS_TARGET *sas_target_priv_data;
2515         struct _sas_device *sas_device;
2516         struct _pcie_device *pcie_device;
2517         struct _raid_device *raid_device;
2518         unsigned long flags;
2519         int qdepth;
2520         u8 ssp_target = 0;
2521         char *ds = "";
2522         char *r_level = "";
2523         u16 handle, volume_handle = 0;
2524         u64 volume_wwid = 0;
2525
2526         qdepth = 1;
2527         sas_device_priv_data = sdev->hostdata;
2528         sas_device_priv_data->configured_lun = 1;
2529         sas_device_priv_data->flags &= ~MPT_DEVICE_FLAGS_INIT;
2530         sas_target_priv_data = sas_device_priv_data->sas_target;
2531         handle = sas_target_priv_data->handle;
2532
2533         /* raid volume handling */
2534         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME) {
2535
2536                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
2537                 raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
2538                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
2539                 if (!raid_device) {
2540                         dfailprintk(ioc,
2541                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2542                                              __FILE__, __LINE__, __func__));
2543                         return 1;
2544                 }
2545
2546                 if (_scsih_get_volume_capabilities(ioc, raid_device)) {
2547                         dfailprintk(ioc,
2548                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2549                                              __FILE__, __LINE__, __func__));
2550                         return 1;
2551                 }
2552
2553                 /*
2554                  * WARPDRIVE: Initialize the required data for Direct IO
2555                  */
2556                 mpt3sas_init_warpdrive_properties(ioc, raid_device);
2557
2558                 /* RAID Queue Depth Support
2559                  * IS volume = underlying qdepth of drive type, either
2560                  *    MPT3SAS_SAS_QUEUE_DEPTH or MPT3SAS_SATA_QUEUE_DEPTH
2561                  * IM/IME/R10 = 128 (MPT3SAS_RAID_QUEUE_DEPTH)
2562                  */
2563                 if (raid_device->device_info &
2564                     MPI2_SAS_DEVICE_INFO_SSP_TARGET) {
2565                         qdepth = MPT3SAS_SAS_QUEUE_DEPTH;
2566                         ds = "SSP";
2567                 } else {
2568                         qdepth = MPT3SAS_SATA_QUEUE_DEPTH;
2569                         if (raid_device->device_info &
2570                             MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
2571                                 ds = "SATA";
2572                         else
2573                                 ds = "STP";
2574                 }
2575
2576                 switch (raid_device->volume_type) {
2577                 case MPI2_RAID_VOL_TYPE_RAID0:
2578                         r_level = "RAID0";
2579                         break;
2580                 case MPI2_RAID_VOL_TYPE_RAID1E:
2581                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2582                         if (ioc->manu_pg10.OEMIdentifier &&
2583                             (le32_to_cpu(ioc->manu_pg10.GenericFlags0) &
2584                             MFG10_GF0_R10_DISPLAY) &&
2585                             !(raid_device->num_pds % 2))
2586                                 r_level = "RAID10";
2587                         else
2588                                 r_level = "RAID1E";
2589                         break;
2590                 case MPI2_RAID_VOL_TYPE_RAID1:
2591                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2592                         r_level = "RAID1";
2593                         break;
2594                 case MPI2_RAID_VOL_TYPE_RAID10:
2595                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2596                         r_level = "RAID10";
2597                         break;
2598                 case MPI2_RAID_VOL_TYPE_UNKNOWN:
2599                 default:
2600                         qdepth = MPT3SAS_RAID_QUEUE_DEPTH;
2601                         r_level = "RAIDX";
2602                         break;
2603                 }
2604
2605                 if (!ioc->hide_ir_msg)
2606                         sdev_printk(KERN_INFO, sdev,
2607                            "%s: handle(0x%04x), wwid(0x%016llx),"
2608                             " pd_count(%d), type(%s)\n",
2609                             r_level, raid_device->handle,
2610                             (unsigned long long)raid_device->wwid,
2611                             raid_device->num_pds, ds);
2612
2613                 if (shost->max_sectors > MPT3SAS_RAID_MAX_SECTORS) {
2614                         blk_queue_max_hw_sectors(sdev->request_queue,
2615                                                 MPT3SAS_RAID_MAX_SECTORS);
2616                         sdev_printk(KERN_INFO, sdev,
2617                                         "Set queue's max_sector to: %u\n",
2618                                                 MPT3SAS_RAID_MAX_SECTORS);
2619                 }
2620
2621                 mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2622
2623                 /* raid transport support */
2624                 if (!ioc->is_warpdrive)
2625                         _scsih_set_level(ioc, sdev, raid_device->volume_type);
2626                 return 0;
2627         }
2628
2629         /* non-raid handling */
2630         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT) {
2631                 if (mpt3sas_config_get_volume_handle(ioc, handle,
2632                     &volume_handle)) {
2633                         dfailprintk(ioc,
2634                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2635                                              __FILE__, __LINE__, __func__));
2636                         return 1;
2637                 }
2638                 if (volume_handle && mpt3sas_config_get_volume_wwid(ioc,
2639                     volume_handle, &volume_wwid)) {
2640                         dfailprintk(ioc,
2641                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2642                                              __FILE__, __LINE__, __func__));
2643                         return 1;
2644                 }
2645         }
2646
2647         /* PCIe handling */
2648         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
2649                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
2650                 pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,
2651                                 sas_device_priv_data->sas_target->sas_address);
2652                 if (!pcie_device) {
2653                         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2654                         dfailprintk(ioc,
2655                                     ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2656                                              __FILE__, __LINE__, __func__));
2657                         return 1;
2658                 }
2659
2660                 qdepth = ioc->max_nvme_qd;
2661                 ds = "NVMe";
2662                 sdev_printk(KERN_INFO, sdev,
2663                         "%s: handle(0x%04x), wwid(0x%016llx), port(%d)\n",
2664                         ds, handle, (unsigned long long)pcie_device->wwid,
2665                         pcie_device->port_num);
2666                 if (pcie_device->enclosure_handle != 0)
2667                         sdev_printk(KERN_INFO, sdev,
2668                         "%s: enclosure logical id(0x%016llx), slot(%d)\n",
2669                         ds,
2670                         (unsigned long long)pcie_device->enclosure_logical_id,
2671                         pcie_device->slot);
2672                 if (pcie_device->connector_name[0] != '\0')
2673                         sdev_printk(KERN_INFO, sdev,
2674                                 "%s: enclosure level(0x%04x),"
2675                                 "connector name( %s)\n", ds,
2676                                 pcie_device->enclosure_level,
2677                                 pcie_device->connector_name);
2678
2679                 if (pcie_device->nvme_mdts)
2680                         blk_queue_max_hw_sectors(sdev->request_queue,
2681                                         pcie_device->nvme_mdts/512);
2682
2683                 pcie_device_put(pcie_device);
2684                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
2685                 mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2686                 /* Enable QUEUE_FLAG_NOMERGES flag, so that IOs won't be
2687                  ** merged and can eliminate holes created during merging
2688                  ** operation.
2689                  **/
2690                 blk_queue_flag_set(QUEUE_FLAG_NOMERGES,
2691                                 sdev->request_queue);
2692                 blk_queue_virt_boundary(sdev->request_queue,
2693                                 ioc->page_size - 1);
2694                 return 0;
2695         }
2696
2697         spin_lock_irqsave(&ioc->sas_device_lock, flags);
2698         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
2699            sas_device_priv_data->sas_target->sas_address,
2700            sas_device_priv_data->sas_target->port);
2701         if (!sas_device) {
2702                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2703                 dfailprintk(ioc,
2704                             ioc_warn(ioc, "failure at %s:%d/%s()!\n",
2705                                      __FILE__, __LINE__, __func__));
2706                 return 1;
2707         }
2708
2709         sas_device->volume_handle = volume_handle;
2710         sas_device->volume_wwid = volume_wwid;
2711         if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET) {
2712                 qdepth = (sas_device->port_type > 1) ?
2713                         ioc->max_wideport_qd : ioc->max_narrowport_qd;
2714                 ssp_target = 1;
2715                 if (sas_device->device_info &
2716                                 MPI2_SAS_DEVICE_INFO_SEP) {
2717                         sdev_printk(KERN_WARNING, sdev,
2718                         "set ignore_delay_remove for handle(0x%04x)\n",
2719                         sas_device_priv_data->sas_target->handle);
2720                         sas_device_priv_data->ignore_delay_remove = 1;
2721                         ds = "SES";
2722                 } else
2723                         ds = "SSP";
2724         } else {
2725                 qdepth = ioc->max_sata_qd;
2726                 if (sas_device->device_info & MPI2_SAS_DEVICE_INFO_STP_TARGET)
2727                         ds = "STP";
2728                 else if (sas_device->device_info &
2729                     MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
2730                         ds = "SATA";
2731         }
2732
2733         sdev_printk(KERN_INFO, sdev, "%s: handle(0x%04x), " \
2734             "sas_addr(0x%016llx), phy(%d), device_name(0x%016llx)\n",
2735             ds, handle, (unsigned long long)sas_device->sas_address,
2736             sas_device->phy, (unsigned long long)sas_device->device_name);
2737
2738         _scsih_display_enclosure_chassis_info(NULL, sas_device, sdev, NULL);
2739
2740         sas_device_put(sas_device);
2741         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
2742
2743         if (!ssp_target)
2744                 _scsih_display_sata_capabilities(ioc, handle, sdev);
2745
2746
2747         mpt3sas_scsih_change_queue_depth(sdev, qdepth);
2748
2749         if (ssp_target) {
2750                 sas_read_port_mode_page(sdev);
2751                 _scsih_enable_tlr(ioc, sdev);
2752         }
2753
2754         return 0;
2755 }
2756
2757 /**
2758  * scsih_bios_param - fetch head, sector, cylinder info for a disk
2759  * @sdev: scsi device struct
2760  * @bdev: pointer to block device context
2761  * @capacity: device size (in 512 byte sectors)
2762  * @params: three element array to place output:
2763  *              params[0] number of heads (max 255)
2764  *              params[1] number of sectors (max 63)
2765  *              params[2] number of cylinders
2766  */
2767 static int
2768 scsih_bios_param(struct scsi_device *sdev, struct block_device *bdev,
2769         sector_t capacity, int params[])
2770 {
2771         int             heads;
2772         int             sectors;
2773         sector_t        cylinders;
2774         ulong           dummy;
2775
2776         heads = 64;
2777         sectors = 32;
2778
2779         dummy = heads * sectors;
2780         cylinders = capacity;
2781         sector_div(cylinders, dummy);
2782
2783         /*
2784          * Handle extended translation size for logical drives
2785          * > 1Gb
2786          */
2787         if ((ulong)capacity >= 0x200000) {
2788                 heads = 255;
2789                 sectors = 63;
2790                 dummy = heads * sectors;
2791                 cylinders = capacity;
2792                 sector_div(cylinders, dummy);
2793         }
2794
2795         /* return result */
2796         params[0] = heads;
2797         params[1] = sectors;
2798         params[2] = cylinders;
2799
2800         return 0;
2801 }
2802
2803 /**
2804  * _scsih_response_code - translation of device response code
2805  * @ioc: per adapter object
2806  * @response_code: response code returned by the device
2807  */
2808 static void
2809 _scsih_response_code(struct MPT3SAS_ADAPTER *ioc, u8 response_code)
2810 {
2811         char *desc;
2812
2813         switch (response_code) {
2814         case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
2815                 desc = "task management request completed";
2816                 break;
2817         case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
2818                 desc = "invalid frame";
2819                 break;
2820         case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
2821                 desc = "task management request not supported";
2822                 break;
2823         case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
2824                 desc = "task management request failed";
2825                 break;
2826         case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
2827                 desc = "task management request succeeded";
2828                 break;
2829         case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
2830                 desc = "invalid lun";
2831                 break;
2832         case 0xA:
2833                 desc = "overlapped tag attempted";
2834                 break;
2835         case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
2836                 desc = "task queued, however not sent to target";
2837                 break;
2838         default:
2839                 desc = "unknown";
2840                 break;
2841         }
2842         ioc_warn(ioc, "response_code(0x%01x): %s\n", response_code, desc);
2843 }
2844
2845 /**
2846  * _scsih_tm_done - tm completion routine
2847  * @ioc: per adapter object
2848  * @smid: system request message index
2849  * @msix_index: MSIX table index supplied by the OS
2850  * @reply: reply message frame(lower 32bit addr)
2851  * Context: none.
2852  *
2853  * The callback handler when using scsih_issue_tm.
2854  *
2855  * Return: 1 meaning mf should be freed from _base_interrupt
2856  *         0 means the mf is freed from this function.
2857  */
2858 static u8
2859 _scsih_tm_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
2860 {
2861         MPI2DefaultReply_t *mpi_reply;
2862
2863         if (ioc->tm_cmds.status == MPT3_CMD_NOT_USED)
2864                 return 1;
2865         if (ioc->tm_cmds.smid != smid)
2866                 return 1;
2867         ioc->tm_cmds.status |= MPT3_CMD_COMPLETE;
2868         mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);
2869         if (mpi_reply) {
2870                 memcpy(ioc->tm_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
2871                 ioc->tm_cmds.status |= MPT3_CMD_REPLY_VALID;
2872         }
2873         ioc->tm_cmds.status &= ~MPT3_CMD_PENDING;
2874         complete(&ioc->tm_cmds.done);
2875         return 1;
2876 }
2877
2878 /**
2879  * mpt3sas_scsih_set_tm_flag - set per target tm_busy
2880  * @ioc: per adapter object
2881  * @handle: device handle
2882  *
2883  * During taskmangement request, we need to freeze the device queue.
2884  */
2885 void
2886 mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
2887 {
2888         struct MPT3SAS_DEVICE *sas_device_priv_data;
2889         struct scsi_device *sdev;
2890         u8 skip = 0;
2891
2892         shost_for_each_device(sdev, ioc->shost) {
2893                 if (skip)
2894                         continue;
2895                 sas_device_priv_data = sdev->hostdata;
2896                 if (!sas_device_priv_data)
2897                         continue;
2898                 if (sas_device_priv_data->sas_target->handle == handle) {
2899                         sas_device_priv_data->sas_target->tm_busy = 1;
2900                         skip = 1;
2901                         ioc->ignore_loginfos = 1;
2902                 }
2903         }
2904 }
2905
2906 /**
2907  * mpt3sas_scsih_clear_tm_flag - clear per target tm_busy
2908  * @ioc: per adapter object
2909  * @handle: device handle
2910  *
2911  * During taskmangement request, we need to freeze the device queue.
2912  */
2913 void
2914 mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
2915 {
2916         struct MPT3SAS_DEVICE *sas_device_priv_data;
2917         struct scsi_device *sdev;
2918         u8 skip = 0;
2919
2920         shost_for_each_device(sdev, ioc->shost) {
2921                 if (skip)
2922                         continue;
2923                 sas_device_priv_data = sdev->hostdata;
2924                 if (!sas_device_priv_data)
2925                         continue;
2926                 if (sas_device_priv_data->sas_target->handle == handle) {
2927                         sas_device_priv_data->sas_target->tm_busy = 0;
2928                         skip = 1;
2929                         ioc->ignore_loginfos = 0;
2930                 }
2931         }
2932 }
2933
2934 /**
2935  * scsih_tm_cmd_map_status - map the target reset & LUN reset TM status
2936  * @ioc: per adapter object
2937  * @channel: the channel assigned by the OS
2938  * @id: the id assigned by the OS
2939  * @lun: lun number
2940  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
2941  * @smid_task: smid assigned to the task
2942  *
2943  * Look whether TM has aborted the timed out SCSI command, if
2944  * TM has aborted the IO then return SUCCESS else return FAILED.
2945  */
2946 static int
2947 scsih_tm_cmd_map_status(struct MPT3SAS_ADAPTER *ioc, uint channel,
2948         uint id, uint lun, u8 type, u16 smid_task)
2949 {
2950
2951         if (smid_task <= ioc->shost->can_queue) {
2952                 switch (type) {
2953                 case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
2954                         if (!(_scsih_scsi_lookup_find_by_target(ioc,
2955                             id, channel)))
2956                                 return SUCCESS;
2957                         break;
2958                 case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
2959                 case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
2960                         if (!(_scsih_scsi_lookup_find_by_lun(ioc, id,
2961                             lun, channel)))
2962                                 return SUCCESS;
2963                         break;
2964                 default:
2965                         return SUCCESS;
2966                 }
2967         } else if (smid_task == ioc->scsih_cmds.smid) {
2968                 if ((ioc->scsih_cmds.status & MPT3_CMD_COMPLETE) ||
2969                     (ioc->scsih_cmds.status & MPT3_CMD_NOT_USED))
2970                         return SUCCESS;
2971         } else if (smid_task == ioc->ctl_cmds.smid) {
2972                 if ((ioc->ctl_cmds.status & MPT3_CMD_COMPLETE) ||
2973                     (ioc->ctl_cmds.status & MPT3_CMD_NOT_USED))
2974                         return SUCCESS;
2975         }
2976
2977         return FAILED;
2978 }
2979
2980 /**
2981  * scsih_tm_post_processing - post processing of target & LUN reset
2982  * @ioc: per adapter object
2983  * @handle: device handle
2984  * @channel: the channel assigned by the OS
2985  * @id: the id assigned by the OS
2986  * @lun: lun number
2987  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
2988  * @smid_task: smid assigned to the task
2989  *
2990  * Post processing of target & LUN reset. Due to interrupt latency
2991  * issue it possible that interrupt for aborted IO might not be
2992  * received yet. So before returning failure status, poll the
2993  * reply descriptor pools for the reply of timed out SCSI command.
2994  * Return FAILED status if reply for timed out is not received
2995  * otherwise return SUCCESS.
2996  */
2997 static int
2998 scsih_tm_post_processing(struct MPT3SAS_ADAPTER *ioc, u16 handle,
2999         uint channel, uint id, uint lun, u8 type, u16 smid_task)
3000 {
3001         int rc;
3002
3003         rc = scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);
3004         if (rc == SUCCESS)
3005                 return rc;
3006
3007         ioc_info(ioc,
3008             "Poll ReplyDescriptor queues for completion of"
3009             " smid(%d), task_type(0x%02x), handle(0x%04x)\n",
3010             smid_task, type, handle);
3011
3012         /*
3013          * Due to interrupt latency issues, driver may receive interrupt for
3014          * TM first and then for aborted SCSI IO command. So, poll all the
3015          * ReplyDescriptor pools before returning the FAILED status to SML.
3016          */
3017         mpt3sas_base_mask_interrupts(ioc);
3018         mpt3sas_base_sync_reply_irqs(ioc, 1);
3019         mpt3sas_base_unmask_interrupts(ioc);
3020
3021         return scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);
3022 }
3023
3024 /**
3025  * mpt3sas_scsih_issue_tm - main routine for sending tm requests
3026  * @ioc: per adapter struct
3027  * @handle: device handle
3028  * @channel: the channel assigned by the OS
3029  * @id: the id assigned by the OS
3030  * @lun: lun number
3031  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in mpi2_init.h)
3032  * @smid_task: smid assigned to the task
3033  * @msix_task: MSIX table index supplied by the OS
3034  * @timeout: timeout in seconds
3035  * @tr_method: Target Reset Method
3036  * Context: user
3037  *
3038  * A generic API for sending task management requests to firmware.
3039  *
3040  * The callback index is set inside `ioc->tm_cb_idx`.
3041  * The caller is responsible to check for outstanding commands.
3042  *
3043  * Return: SUCCESS or FAILED.
3044  */
3045 int
3046 mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,
3047         uint id, u64 lun, u8 type, u16 smid_task, u16 msix_task,
3048         u8 timeout, u8 tr_method)
3049 {
3050         Mpi2SCSITaskManagementRequest_t *mpi_request;
3051         Mpi2SCSITaskManagementReply_t *mpi_reply;
3052         Mpi25SCSIIORequest_t *request;
3053         u16 smid = 0;
3054         u32 ioc_state;
3055         int rc;
3056         u8 issue_reset = 0;
3057
3058         lockdep_assert_held(&ioc->tm_cmds.mutex);
3059
3060         if (ioc->tm_cmds.status != MPT3_CMD_NOT_USED) {
3061                 ioc_info(ioc, "%s: tm_cmd busy!!!\n", __func__);
3062                 return FAILED;
3063         }
3064
3065         if (ioc->shost_recovery || ioc->remove_host ||
3066             ioc->pci_error_recovery) {
3067                 ioc_info(ioc, "%s: host reset in progress!\n", __func__);
3068                 return FAILED;
3069         }
3070
3071         ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
3072         if (ioc_state & MPI2_DOORBELL_USED) {
3073                 dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));
3074                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3075                 return (!rc) ? SUCCESS : FAILED;
3076         }
3077
3078         if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
3079                 mpt3sas_print_fault_code(ioc, ioc_state &
3080                     MPI2_DOORBELL_DATA_MASK);
3081                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3082                 return (!rc) ? SUCCESS : FAILED;
3083         } else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
3084             MPI2_IOC_STATE_COREDUMP) {
3085                 mpt3sas_print_coredump_info(ioc, ioc_state &
3086                     MPI2_DOORBELL_DATA_MASK);
3087                 rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3088                 return (!rc) ? SUCCESS : FAILED;
3089         }
3090
3091         smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_cb_idx);
3092         if (!smid) {
3093                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
3094                 return FAILED;
3095         }
3096
3097         dtmprintk(ioc,
3098                   ioc_info(ioc, "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%x)\n",
3099                            handle, type, smid_task, timeout, tr_method));
3100         ioc->tm_cmds.status = MPT3_CMD_PENDING;
3101         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
3102         ioc->tm_cmds.smid = smid;
3103         memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
3104         memset(ioc->tm_cmds.reply, 0, sizeof(Mpi2SCSITaskManagementReply_t));
3105         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3106         mpi_request->DevHandle = cpu_to_le16(handle);
3107         mpi_request->TaskType = type;
3108         if (type == MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK ||
3109             type == MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK)
3110                 mpi_request->MsgFlags = tr_method;
3111         mpi_request->TaskMID = cpu_to_le16(smid_task);
3112         int_to_scsilun(lun, (struct scsi_lun *)mpi_request->LUN);
3113         mpt3sas_scsih_set_tm_flag(ioc, handle);
3114         init_completion(&ioc->tm_cmds.done);
3115         ioc->put_smid_hi_priority(ioc, smid, msix_task);
3116         wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);
3117         if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {
3118                 mpt3sas_check_cmd_timeout(ioc,
3119                     ioc->tm_cmds.status, mpi_request,
3120                     sizeof(Mpi2SCSITaskManagementRequest_t)/4, issue_reset);
3121                 if (issue_reset) {
3122                         rc = mpt3sas_base_hard_reset_handler(ioc,
3123                                         FORCE_BIG_HAMMER);
3124                         rc = (!rc) ? SUCCESS : FAILED;
3125                         goto out;
3126                 }
3127         }
3128
3129         /* sync IRQs in case those were busy during flush. */
3130         mpt3sas_base_sync_reply_irqs(ioc, 0);
3131
3132         if (ioc->tm_cmds.status & MPT3_CMD_REPLY_VALID) {
3133                 mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);
3134                 mpi_reply = ioc->tm_cmds.reply;
3135                 dtmprintk(ioc,
3136                           ioc_info(ioc, "complete tm: ioc_status(0x%04x), loginfo(0x%08x), term_count(0x%08x)\n",
3137                                    le16_to_cpu(mpi_reply->IOCStatus),
3138                                    le32_to_cpu(mpi_reply->IOCLogInfo),
3139                                    le32_to_cpu(mpi_reply->TerminationCount)));
3140                 if (ioc->logging_level & MPT_DEBUG_TM) {
3141                         _scsih_response_code(ioc, mpi_reply->ResponseCode);
3142                         if (mpi_reply->IOCStatus)
3143                                 _debug_dump_mf(mpi_request,
3144                                     sizeof(Mpi2SCSITaskManagementRequest_t)/4);
3145                 }
3146         }
3147
3148         switch (type) {
3149         case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
3150                 rc = SUCCESS;
3151                 /*
3152                  * If DevHandle filed in smid_task's entry of request pool
3153                  * doesn't match with device handle on which this task abort
3154                  * TM is received then it means that TM has successfully
3155                  * aborted the timed out command. Since smid_task's entry in
3156                  * request pool will be memset to zero once the timed out
3157                  * command is returned to the SML. If the command is not
3158                  * aborted then smid_task’s entry won’t be cleared and it
3159                  * will have same DevHandle value on which this task abort TM
3160                  * is received and driver will return the TM status as FAILED.
3161                  */
3162                 request = mpt3sas_base_get_msg_frame(ioc, smid_task);
3163                 if (le16_to_cpu(request->DevHandle) != handle)
3164                         break;
3165
3166                 ioc_info(ioc, "Task abort tm failed: handle(0x%04x),"
3167                     "timeout(%d) tr_method(0x%x) smid(%d) msix_index(%d)\n",
3168                     handle, timeout, tr_method, smid_task, msix_task);
3169                 rc = FAILED;
3170                 break;
3171
3172         case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
3173         case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
3174         case MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
3175                 rc = scsih_tm_post_processing(ioc, handle, channel, id, lun,
3176                     type, smid_task);
3177                 break;
3178         case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
3179                 rc = SUCCESS;
3180                 break;
3181         default:
3182                 rc = FAILED;
3183                 break;
3184         }
3185
3186 out:
3187         mpt3sas_scsih_clear_tm_flag(ioc, handle);
3188         ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
3189         return rc;
3190 }
3191
3192 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,
3193                 uint channel, uint id, u64 lun, u8 type, u16 smid_task,
3194                 u16 msix_task, u8 timeout, u8 tr_method)
3195 {
3196         int ret;
3197
3198         mutex_lock(&ioc->tm_cmds.mutex);
3199         ret = mpt3sas_scsih_issue_tm(ioc, handle, channel, id, lun, type,
3200                         smid_task, msix_task, timeout, tr_method);
3201         mutex_unlock(&ioc->tm_cmds.mutex);
3202
3203         return ret;
3204 }
3205
3206 /**
3207  * _scsih_tm_display_info - displays info about the device
3208  * @ioc: per adapter struct
3209  * @scmd: pointer to scsi command object
3210  *
3211  * Called by task management callback handlers.
3212  */
3213 static void
3214 _scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd)
3215 {
3216         struct scsi_target *starget = scmd->device->sdev_target;
3217         struct MPT3SAS_TARGET *priv_target = starget->hostdata;
3218         struct _sas_device *sas_device = NULL;
3219         struct _pcie_device *pcie_device = NULL;
3220         unsigned long flags;
3221         char *device_str = NULL;
3222
3223         if (!priv_target)
3224                 return;
3225         if (ioc->hide_ir_msg)
3226                 device_str = "WarpDrive";
3227         else
3228                 device_str = "volume";
3229
3230         scsi_print_command(scmd);
3231         if (priv_target->flags & MPT_TARGET_FLAGS_VOLUME) {
3232                 starget_printk(KERN_INFO, starget,
3233                         "%s handle(0x%04x), %s wwid(0x%016llx)\n",
3234                         device_str, priv_target->handle,
3235                     device_str, (unsigned long long)priv_target->sas_address);
3236
3237         } else if (priv_target->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
3238                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
3239                 pcie_device = __mpt3sas_get_pdev_from_target(ioc, priv_target);
3240                 if (pcie_device) {
3241                         starget_printk(KERN_INFO, starget,
3242                                 "handle(0x%04x), wwid(0x%016llx), port(%d)\n",
3243                                 pcie_device->handle,
3244                                 (unsigned long long)pcie_device->wwid,
3245                                 pcie_device->port_num);
3246                         if (pcie_device->enclosure_handle != 0)
3247                                 starget_printk(KERN_INFO, starget,
3248                                         "enclosure logical id(0x%016llx), slot(%d)\n",
3249                                         (unsigned long long)
3250                                         pcie_device->enclosure_logical_id,
3251                                         pcie_device->slot);
3252                         if (pcie_device->connector_name[0] != '\0')
3253                                 starget_printk(KERN_INFO, starget,
3254                                         "enclosure level(0x%04x), connector name( %s)\n",
3255                                         pcie_device->enclosure_level,
3256                                         pcie_device->connector_name);
3257                         pcie_device_put(pcie_device);
3258                 }
3259                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
3260
3261         } else {
3262                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
3263                 sas_device = __mpt3sas_get_sdev_from_target(ioc, priv_target);
3264                 if (sas_device) {
3265                         if (priv_target->flags &
3266                             MPT_TARGET_FLAGS_RAID_COMPONENT) {
3267                                 starget_printk(KERN_INFO, starget,
3268                                     "volume handle(0x%04x), "
3269                                     "volume wwid(0x%016llx)\n",
3270                                     sas_device->volume_handle,
3271                                    (unsigned long long)sas_device->volume_wwid);
3272                         }
3273                         starget_printk(KERN_INFO, starget,
3274                             "handle(0x%04x), sas_address(0x%016llx), phy(%d)\n",
3275                             sas_device->handle,
3276                             (unsigned long long)sas_device->sas_address,
3277                             sas_device->phy);
3278
3279                         _scsih_display_enclosure_chassis_info(NULL, sas_device,
3280                             NULL, starget);
3281
3282                         sas_device_put(sas_device);
3283                 }
3284                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
3285         }
3286 }
3287
3288 /**
3289  * scsih_abort - eh threads main abort routine
3290  * @scmd: pointer to scsi command object
3291  *
3292  * Return: SUCCESS if command aborted else FAILED
3293  */
3294 static int
3295 scsih_abort(struct scsi_cmnd *scmd)
3296 {
3297         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3298         struct MPT3SAS_DEVICE *sas_device_priv_data;
3299         struct scsiio_tracker *st = scsi_cmd_priv(scmd);
3300         u16 handle;
3301         int r;
3302
3303         u8 timeout = 30;
3304         struct _pcie_device *pcie_device = NULL;
3305         sdev_printk(KERN_INFO, scmd->device, "attempting task abort!"
3306             "scmd(0x%p), outstanding for %u ms & timeout %u ms\n",
3307             scmd, jiffies_to_msecs(jiffies - scmd->jiffies_at_alloc),
3308             (scsi_cmd_to_rq(scmd)->timeout / HZ) * 1000);
3309         _scsih_tm_display_info(ioc, scmd);
3310
3311         sas_device_priv_data = scmd->device->hostdata;
3312         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3313             ioc->remove_host) {
3314                 sdev_printk(KERN_INFO, scmd->device,
3315                     "device been deleted! scmd(0x%p)\n", scmd);
3316                 scmd->result = DID_NO_CONNECT << 16;
3317                 scmd->scsi_done(scmd);
3318                 r = SUCCESS;
3319                 goto out;
3320         }
3321
3322         /* check for completed command */
3323         if (st == NULL || st->cb_idx == 0xFF) {
3324                 sdev_printk(KERN_INFO, scmd->device, "No reference found at "
3325                     "driver, assuming scmd(0x%p) might have completed\n", scmd);
3326                 scmd->result = DID_RESET << 16;
3327                 r = SUCCESS;
3328                 goto out;
3329         }
3330
3331         /* for hidden raid components and volumes this is not supported */
3332         if (sas_device_priv_data->sas_target->flags &
3333             MPT_TARGET_FLAGS_RAID_COMPONENT ||
3334             sas_device_priv_data->sas_target->flags & MPT_TARGET_FLAGS_VOLUME) {
3335                 scmd->result = DID_RESET << 16;
3336                 r = FAILED;
3337                 goto out;
3338         }
3339
3340         mpt3sas_halt_firmware(ioc);
3341
3342         handle = sas_device_priv_data->sas_target->handle;
3343         pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3344         if (pcie_device && (!ioc->tm_custom_handling) &&
3345             (!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))))
3346                 timeout = ioc->nvme_abort_timeout;
3347         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
3348                 scmd->device->id, scmd->device->lun,
3349                 MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
3350                 st->smid, st->msix_io, timeout, 0);
3351         /* Command must be cleared after abort */
3352         if (r == SUCCESS && st->cb_idx != 0xFF)
3353                 r = FAILED;
3354  out:
3355         sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(0x%p)\n",
3356             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3357         if (pcie_device)
3358                 pcie_device_put(pcie_device);
3359         return r;
3360 }
3361
3362 /**
3363  * scsih_dev_reset - eh threads main device reset routine
3364  * @scmd: pointer to scsi command object
3365  *
3366  * Return: SUCCESS if command aborted else FAILED
3367  */
3368 static int
3369 scsih_dev_reset(struct scsi_cmnd *scmd)
3370 {
3371         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3372         struct MPT3SAS_DEVICE *sas_device_priv_data;
3373         struct _sas_device *sas_device = NULL;
3374         struct _pcie_device *pcie_device = NULL;
3375         u16     handle;
3376         u8      tr_method = 0;
3377         u8      tr_timeout = 30;
3378         int r;
3379
3380         struct scsi_target *starget = scmd->device->sdev_target;
3381         struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
3382
3383         sdev_printk(KERN_INFO, scmd->device,
3384             "attempting device reset! scmd(0x%p)\n", scmd);
3385         _scsih_tm_display_info(ioc, scmd);
3386
3387         sas_device_priv_data = scmd->device->hostdata;
3388         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3389             ioc->remove_host) {
3390                 sdev_printk(KERN_INFO, scmd->device,
3391                     "device been deleted! scmd(0x%p)\n", scmd);
3392                 scmd->result = DID_NO_CONNECT << 16;
3393                 scmd->scsi_done(scmd);
3394                 r = SUCCESS;
3395                 goto out;
3396         }
3397
3398         /* for hidden raid components obtain the volume_handle */
3399         handle = 0;
3400         if (sas_device_priv_data->sas_target->flags &
3401             MPT_TARGET_FLAGS_RAID_COMPONENT) {
3402                 sas_device = mpt3sas_get_sdev_from_target(ioc,
3403                                 target_priv_data);
3404                 if (sas_device)
3405                         handle = sas_device->volume_handle;
3406         } else
3407                 handle = sas_device_priv_data->sas_target->handle;
3408
3409         if (!handle) {
3410                 scmd->result = DID_RESET << 16;
3411                 r = FAILED;
3412                 goto out;
3413         }
3414
3415         pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3416
3417         if (pcie_device && (!ioc->tm_custom_handling) &&
3418             (!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info)))) {
3419                 tr_timeout = pcie_device->reset_timeout;
3420                 tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
3421         } else
3422                 tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3423
3424         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
3425                 scmd->device->id, scmd->device->lun,
3426                 MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, 0, 0,
3427                 tr_timeout, tr_method);
3428         /* Check for busy commands after reset */
3429         if (r == SUCCESS && scsi_device_busy(scmd->device))
3430                 r = FAILED;
3431  out:
3432         sdev_printk(KERN_INFO, scmd->device, "device reset: %s scmd(0x%p)\n",
3433             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3434
3435         if (sas_device)
3436                 sas_device_put(sas_device);
3437         if (pcie_device)
3438                 pcie_device_put(pcie_device);
3439
3440         return r;
3441 }
3442
3443 /**
3444  * scsih_target_reset - eh threads main target reset routine
3445  * @scmd: pointer to scsi command object
3446  *
3447  * Return: SUCCESS if command aborted else FAILED
3448  */
3449 static int
3450 scsih_target_reset(struct scsi_cmnd *scmd)
3451 {
3452         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3453         struct MPT3SAS_DEVICE *sas_device_priv_data;
3454         struct _sas_device *sas_device = NULL;
3455         struct _pcie_device *pcie_device = NULL;
3456         u16     handle;
3457         u8      tr_method = 0;
3458         u8      tr_timeout = 30;
3459         int r;
3460         struct scsi_target *starget = scmd->device->sdev_target;
3461         struct MPT3SAS_TARGET *target_priv_data = starget->hostdata;
3462
3463         starget_printk(KERN_INFO, starget,
3464             "attempting target reset! scmd(0x%p)\n", scmd);
3465         _scsih_tm_display_info(ioc, scmd);
3466
3467         sas_device_priv_data = scmd->device->hostdata;
3468         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
3469             ioc->remove_host) {
3470                 starget_printk(KERN_INFO, starget,
3471                     "target been deleted! scmd(0x%p)\n", scmd);
3472                 scmd->result = DID_NO_CONNECT << 16;
3473                 scmd->scsi_done(scmd);
3474                 r = SUCCESS;
3475                 goto out;
3476         }
3477
3478         /* for hidden raid components obtain the volume_handle */
3479         handle = 0;
3480         if (sas_device_priv_data->sas_target->flags &
3481             MPT_TARGET_FLAGS_RAID_COMPONENT) {
3482                 sas_device = mpt3sas_get_sdev_from_target(ioc,
3483                                 target_priv_data);
3484                 if (sas_device)
3485                         handle = sas_device->volume_handle;
3486         } else
3487                 handle = sas_device_priv_data->sas_target->handle;
3488
3489         if (!handle) {
3490                 scmd->result = DID_RESET << 16;
3491                 r = FAILED;
3492                 goto out;
3493         }
3494
3495         pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);
3496
3497         if (pcie_device && (!ioc->tm_custom_handling) &&
3498             (!(mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info)))) {
3499                 tr_timeout = pcie_device->reset_timeout;
3500                 tr_method = MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
3501         } else
3502                 tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3503         r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,
3504                 scmd->device->id, 0,
3505                 MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, 0, 0,
3506             tr_timeout, tr_method);
3507         /* Check for busy commands after reset */
3508         if (r == SUCCESS && atomic_read(&starget->target_busy))
3509                 r = FAILED;
3510  out:
3511         starget_printk(KERN_INFO, starget, "target reset: %s scmd(0x%p)\n",
3512             ((r == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3513
3514         if (sas_device)
3515                 sas_device_put(sas_device);
3516         if (pcie_device)
3517                 pcie_device_put(pcie_device);
3518         return r;
3519 }
3520
3521
3522 /**
3523  * scsih_host_reset - eh threads main host reset routine
3524  * @scmd: pointer to scsi command object
3525  *
3526  * Return: SUCCESS if command aborted else FAILED
3527  */
3528 static int
3529 scsih_host_reset(struct scsi_cmnd *scmd)
3530 {
3531         struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);
3532         int r, retval;
3533
3534         ioc_info(ioc, "attempting host reset! scmd(0x%p)\n", scmd);
3535         scsi_print_command(scmd);
3536
3537         if (ioc->is_driver_loading || ioc->remove_host) {
3538                 ioc_info(ioc, "Blocking the host reset\n");
3539                 r = FAILED;
3540                 goto out;
3541         }
3542
3543         retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
3544         r = (retval < 0) ? FAILED : SUCCESS;
3545 out:
3546         ioc_info(ioc, "host reset: %s scmd(0x%p)\n",
3547                  r == SUCCESS ? "SUCCESS" : "FAILED", scmd);
3548
3549         return r;
3550 }
3551
3552 /**
3553  * _scsih_fw_event_add - insert and queue up fw_event
3554  * @ioc: per adapter object
3555  * @fw_event: object describing the event
3556  * Context: This function will acquire ioc->fw_event_lock.
3557  *
3558  * This adds the firmware event object into link list, then queues it up to
3559  * be processed from user context.
3560  */
3561 static void
3562 _scsih_fw_event_add(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
3563 {
3564         unsigned long flags;
3565
3566         if (ioc->firmware_event_thread == NULL)
3567                 return;
3568
3569         spin_lock_irqsave(&ioc->fw_event_lock, flags);
3570         fw_event_work_get(fw_event);
3571         INIT_LIST_HEAD(&fw_event->list);
3572         list_add_tail(&fw_event->list, &ioc->fw_event_list);
3573         INIT_WORK(&fw_event->work, _firmware_event_work);
3574         fw_event_work_get(fw_event);
3575         queue_work(ioc->firmware_event_thread, &fw_event->work);
3576         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
3577 }
3578
3579 /**
3580  * _scsih_fw_event_del_from_list - delete fw_event from the list
3581  * @ioc: per adapter object
3582  * @fw_event: object describing the event
3583  * Context: This function will acquire ioc->fw_event_lock.
3584  *
3585  * If the fw_event is on the fw_event_list, remove it and do a put.
3586  */
3587 static void
3588 _scsih_fw_event_del_from_list(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work
3589         *fw_event)
3590 {
3591         unsigned long flags;
3592
3593         spin_lock_irqsave(&ioc->fw_event_lock, flags);
3594         if (!list_empty(&fw_event->list)) {
3595                 list_del_init(&fw_event->list);
3596                 fw_event_work_put(fw_event);
3597         }
3598         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
3599 }
3600
3601
3602  /**
3603  * mpt3sas_send_trigger_data_event - send event for processing trigger data
3604  * @ioc: per adapter object
3605  * @event_data: trigger event data
3606  */
3607 void
3608 mpt3sas_send_trigger_data_event(struct MPT3SAS_ADAPTER *ioc,
3609         struct SL_WH_TRIGGERS_EVENT_DATA_T *event_data)
3610 {
3611         struct fw_event_work *fw_event;
3612         u16 sz;
3613
3614         if (ioc->is_driver_loading)
3615                 return;
3616         sz = sizeof(*event_data);
3617         fw_event = alloc_fw_event_work(sz);
3618         if (!fw_event)
3619                 return;
3620         fw_event->event = MPT3SAS_PROCESS_TRIGGER_DIAG;
3621         fw_event->ioc = ioc;
3622         memcpy(fw_event->event_data, event_data, sizeof(*event_data));
3623         _scsih_fw_event_add(ioc, fw_event);
3624         fw_event_work_put(fw_event);
3625 }
3626
3627 /**
3628  * _scsih_error_recovery_delete_devices - remove devices not responding
3629  * @ioc: per adapter object
3630  */
3631 static void
3632 _scsih_error_recovery_delete_devices(struct MPT3SAS_ADAPTER *ioc)
3633 {
3634         struct fw_event_work *fw_event;
3635
3636         fw_event = alloc_fw_event_work(0);
3637         if (!fw_event)
3638                 return;
3639         fw_event->event = MPT3SAS_REMOVE_UNRESPONDING_DEVICES;
3640         fw_event->ioc = ioc;
3641         _scsih_fw_event_add(ioc, fw_event);
3642         fw_event_work_put(fw_event);
3643 }
3644
3645 /**
3646  * mpt3sas_port_enable_complete - port enable completed (fake event)
3647  * @ioc: per adapter object
3648  */
3649 void
3650 mpt3sas_port_enable_complete(struct MPT3SAS_ADAPTER *ioc)
3651 {
3652         struct fw_event_work *fw_event;
3653
3654         fw_event = alloc_fw_event_work(0);
3655         if (!fw_event)
3656                 return;
3657         fw_event->event = MPT3SAS_PORT_ENABLE_COMPLETE;
3658         fw_event->ioc = ioc;
3659         _scsih_fw_event_add(ioc, fw_event);
3660         fw_event_work_put(fw_event);
3661 }
3662
3663 static struct fw_event_work *dequeue_next_fw_event(struct MPT3SAS_ADAPTER *ioc)
3664 {
3665         unsigned long flags;
3666         struct fw_event_work *fw_event = NULL;
3667
3668         spin_lock_irqsave(&ioc->fw_event_lock, flags);
3669         if (!list_empty(&ioc->fw_event_list)) {
3670                 fw_event = list_first_entry(&ioc->fw_event_list,
3671                                 struct fw_event_work, list);
3672                 list_del_init(&fw_event->list);
3673         }
3674         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
3675
3676         return fw_event;
3677 }
3678
3679 /**
3680  * _scsih_fw_event_cleanup_queue - cleanup event queue
3681  * @ioc: per adapter object
3682  *
3683  * Walk the firmware event queue, either killing timers, or waiting
3684  * for outstanding events to complete
3685  *
3686  * Context: task, can sleep
3687  */
3688 static void
3689 _scsih_fw_event_cleanup_queue(struct MPT3SAS_ADAPTER *ioc)
3690 {
3691         struct fw_event_work *fw_event;
3692
3693         if ((list_empty(&ioc->fw_event_list) && !ioc->current_event) ||
3694             !ioc->firmware_event_thread)
3695                 return;
3696         /*
3697          * Set current running event as ignore, so that
3698          * current running event will exit quickly.
3699          * As diag reset has occurred it is of no use
3700          * to process remaining stale event data entries.
3701          */
3702         if (ioc->shost_recovery && ioc->current_event)
3703                 ioc->current_event->ignore = 1;
3704
3705         ioc->fw_events_cleanup = 1;
3706         while ((fw_event = dequeue_next_fw_event(ioc)) ||
3707              (fw_event = ioc->current_event)) {
3708
3709                 /*
3710                  * Don't call cancel_work_sync() for current_event
3711                  * other than MPT3SAS_REMOVE_UNRESPONDING_DEVICES;
3712                  * otherwise we may observe deadlock if current
3713                  * hard reset issued as part of processing the current_event.
3714                  *
3715                  * Orginal logic of cleaning the current_event is added
3716                  * for handling the back to back host reset issued by the user.
3717                  * i.e. during back to back host reset, driver use to process
3718                  * the two instances of MPT3SAS_REMOVE_UNRESPONDING_DEVICES
3719                  * event back to back and this made the drives to unregister
3720                  * the devices from SML.
3721                  */
3722
3723                 if (fw_event == ioc->current_event &&
3724                     ioc->current_event->event !=
3725                     MPT3SAS_REMOVE_UNRESPONDING_DEVICES) {
3726                         ioc->current_event = NULL;
3727                         continue;
3728                 }
3729
3730                 /*
3731                  * Driver has to clear ioc->start_scan flag when
3732                  * it is cleaning up MPT3SAS_PORT_ENABLE_COMPLETE,
3733                  * otherwise scsi_scan_host() API waits for the
3734                  * 5 minute timer to expire. If we exit from
3735                  * scsi_scan_host() early then we can issue the
3736                  * new port enable request as part of current diag reset.
3737                  */
3738                 if (fw_event->event == MPT3SAS_PORT_ENABLE_COMPLETE) {
3739                         ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
3740                         ioc->start_scan = 0;
3741                 }
3742
3743                 /*
3744                  * Wait on the fw_event to complete. If this returns 1, then
3745                  * the event was never executed, and we need a put for the
3746                  * reference the work had on the fw_event.
3747                  *
3748                  * If it did execute, we wait for it to finish, and the put will
3749                  * happen from _firmware_event_work()
3750                  */
3751                 if (cancel_work_sync(&fw_event->work))
3752                         fw_event_work_put(fw_event);
3753
3754                 fw_event_work_put(fw_event);
3755         }
3756         ioc->fw_events_cleanup = 0;
3757 }
3758
3759 /**
3760  * _scsih_internal_device_block - block the sdev device
3761  * @sdev: per device object
3762  * @sas_device_priv_data : per device driver private data
3763  *
3764  * make sure device is blocked without error, if not
3765  * print an error
3766  */
3767 static void
3768 _scsih_internal_device_block(struct scsi_device *sdev,
3769                         struct MPT3SAS_DEVICE *sas_device_priv_data)
3770 {
3771         int r = 0;
3772
3773         sdev_printk(KERN_INFO, sdev, "device_block, handle(0x%04x)\n",
3774             sas_device_priv_data->sas_target->handle);
3775         sas_device_priv_data->block = 1;
3776
3777         r = scsi_internal_device_block_nowait(sdev);
3778         if (r == -EINVAL)
3779                 sdev_printk(KERN_WARNING, sdev,
3780                     "device_block failed with return(%d) for handle(0x%04x)\n",
3781                     r, sas_device_priv_data->sas_target->handle);
3782 }
3783
3784 /**
3785  * _scsih_internal_device_unblock - unblock the sdev device
3786  * @sdev: per device object
3787  * @sas_device_priv_data : per device driver private data
3788  * make sure device is unblocked without error, if not retry
3789  * by blocking and then unblocking
3790  */
3791
3792 static void
3793 _scsih_internal_device_unblock(struct scsi_device *sdev,
3794                         struct MPT3SAS_DEVICE *sas_device_priv_data)
3795 {
3796         int r = 0;
3797
3798         sdev_printk(KERN_WARNING, sdev, "device_unblock and setting to running, "
3799             "handle(0x%04x)\n", sas_device_priv_data->sas_target->handle);
3800         sas_device_priv_data->block = 0;
3801         r = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
3802         if (r == -EINVAL) {
3803                 /* The device has been set to SDEV_RUNNING by SD layer during
3804                  * device addition but the request queue is still stopped by
3805                  * our earlier block call. We need to perform a block again
3806                  * to get the device to SDEV_BLOCK and then to SDEV_RUNNING */
3807
3808                 sdev_printk(KERN_WARNING, sdev,
3809                     "device_unblock failed with return(%d) for handle(0x%04x) "
3810                     "performing a block followed by an unblock\n",
3811                     r, sas_device_priv_data->sas_target->handle);
3812                 sas_device_priv_data->block = 1;
3813                 r = scsi_internal_device_block_nowait(sdev);
3814                 if (r)
3815                         sdev_printk(KERN_WARNING, sdev, "retried device_block "
3816                             "failed with return(%d) for handle(0x%04x)\n",
3817                             r, sas_device_priv_data->sas_target->handle);
3818
3819                 sas_device_priv_data->block = 0;
3820                 r = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
3821                 if (r)
3822                         sdev_printk(KERN_WARNING, sdev, "retried device_unblock"
3823                             " failed with return(%d) for handle(0x%04x)\n",
3824                             r, sas_device_priv_data->sas_target->handle);
3825         }
3826 }
3827
3828 /**
3829  * _scsih_ublock_io_all_device - unblock every device
3830  * @ioc: per adapter object
3831  *
3832  * change the device state from block to running
3833  */
3834 static void
3835 _scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc)
3836 {
3837         struct MPT3SAS_DEVICE *sas_device_priv_data;
3838         struct scsi_device *sdev;
3839
3840         shost_for_each_device(sdev, ioc->shost) {
3841                 sas_device_priv_data = sdev->hostdata;
3842                 if (!sas_device_priv_data)
3843                         continue;
3844                 if (!sas_device_priv_data->block)
3845                         continue;
3846
3847                 dewtprintk(ioc, sdev_printk(KERN_INFO, sdev,
3848                         "device_running, handle(0x%04x)\n",
3849                     sas_device_priv_data->sas_target->handle));
3850                 _scsih_internal_device_unblock(sdev, sas_device_priv_data);
3851         }
3852 }
3853
3854
3855 /**
3856  * _scsih_ublock_io_device - prepare device to be deleted
3857  * @ioc: per adapter object
3858  * @sas_address: sas address
3859  * @port: hba port entry
3860  *
3861  * unblock then put device in offline state
3862  */
3863 static void
3864 _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc,
3865         u64 sas_address, struct hba_port *port)
3866 {
3867         struct MPT3SAS_DEVICE *sas_device_priv_data;
3868         struct scsi_device *sdev;
3869
3870         shost_for_each_device(sdev, ioc->shost) {
3871                 sas_device_priv_data = sdev->hostdata;
3872                 if (!sas_device_priv_data)
3873                         continue;
3874                 if (sas_device_priv_data->sas_target->sas_address
3875                     != sas_address)
3876                         continue;
3877                 if (sas_device_priv_data->sas_target->port != port)
3878                         continue;
3879                 if (sas_device_priv_data->block)
3880                         _scsih_internal_device_unblock(sdev,
3881                                 sas_device_priv_data);
3882         }
3883 }
3884
3885 /**
3886  * _scsih_block_io_all_device - set the device state to SDEV_BLOCK
3887  * @ioc: per adapter object
3888  *
3889  * During device pull we need to appropriately set the sdev state.
3890  */
3891 static void
3892 _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)
3893 {
3894         struct MPT3SAS_DEVICE *sas_device_priv_data;
3895         struct scsi_device *sdev;
3896
3897         shost_for_each_device(sdev, ioc->shost) {
3898                 sas_device_priv_data = sdev->hostdata;
3899                 if (!sas_device_priv_data)
3900                         continue;
3901                 if (sas_device_priv_data->block)
3902                         continue;
3903                 if (sas_device_priv_data->ignore_delay_remove) {
3904                         sdev_printk(KERN_INFO, sdev,
3905                         "%s skip device_block for SES handle(0x%04x)\n",
3906                         __func__, sas_device_priv_data->sas_target->handle);
3907                         continue;
3908                 }
3909                 _scsih_internal_device_block(sdev, sas_device_priv_data);
3910         }
3911 }
3912
3913 /**
3914  * _scsih_block_io_device - set the device state to SDEV_BLOCK
3915  * @ioc: per adapter object
3916  * @handle: device handle
3917  *
3918  * During device pull we need to appropriately set the sdev state.
3919  */
3920 static void
3921 _scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
3922 {
3923         struct MPT3SAS_DEVICE *sas_device_priv_data;
3924         struct scsi_device *sdev;
3925         struct _sas_device *sas_device;
3926
3927         sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
3928
3929         shost_for_each_device(sdev, ioc->shost) {
3930                 sas_device_priv_data = sdev->hostdata;
3931                 if (!sas_device_priv_data)
3932                         continue;
3933                 if (sas_device_priv_data->sas_target->handle != handle)
3934                         continue;
3935                 if (sas_device_priv_data->block)
3936                         continue;
3937                 if (sas_device && sas_device->pend_sas_rphy_add)
3938                         continue;
3939                 if (sas_device_priv_data->ignore_delay_remove) {
3940                         sdev_printk(KERN_INFO, sdev,
3941                         "%s skip device_block for SES handle(0x%04x)\n",
3942                         __func__, sas_device_priv_data->sas_target->handle);
3943                         continue;
3944                 }
3945                 _scsih_internal_device_block(sdev, sas_device_priv_data);
3946         }
3947
3948         if (sas_device)
3949                 sas_device_put(sas_device);
3950 }
3951
3952 /**
3953  * _scsih_block_io_to_children_attached_to_ex
3954  * @ioc: per adapter object
3955  * @sas_expander: the sas_device object
3956  *
3957  * This routine set sdev state to SDEV_BLOCK for all devices
3958  * attached to this expander. This function called when expander is
3959  * pulled.
3960  */
3961 static void
3962 _scsih_block_io_to_children_attached_to_ex(struct MPT3SAS_ADAPTER *ioc,
3963         struct _sas_node *sas_expander)
3964 {
3965         struct _sas_port *mpt3sas_port;
3966         struct _sas_device *sas_device;
3967         struct _sas_node *expander_sibling;
3968         unsigned long flags;
3969
3970         if (!sas_expander)
3971                 return;
3972
3973         list_for_each_entry(mpt3sas_port,
3974            &sas_expander->sas_port_list, port_list) {
3975                 if (mpt3sas_port->remote_identify.device_type ==
3976                     SAS_END_DEVICE) {
3977                         spin_lock_irqsave(&ioc->sas_device_lock, flags);
3978                         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
3979                             mpt3sas_port->remote_identify.sas_address,
3980                             mpt3sas_port->hba_port);
3981                         if (sas_device) {
3982                                 set_bit(sas_device->handle,
3983                                                 ioc->blocking_handles);
3984                                 sas_device_put(sas_device);
3985                         }
3986                         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
3987                 }
3988         }
3989
3990         list_for_each_entry(mpt3sas_port,
3991            &sas_expander->sas_port_list, port_list) {
3992
3993                 if (mpt3sas_port->remote_identify.device_type ==
3994                     SAS_EDGE_EXPANDER_DEVICE ||
3995                     mpt3sas_port->remote_identify.device_type ==
3996                     SAS_FANOUT_EXPANDER_DEVICE) {
3997                         expander_sibling =
3998                             mpt3sas_scsih_expander_find_by_sas_address(
3999                             ioc, mpt3sas_port->remote_identify.sas_address,
4000                             mpt3sas_port->hba_port);
4001                         _scsih_block_io_to_children_attached_to_ex(ioc,
4002                             expander_sibling);
4003                 }
4004         }
4005 }
4006
4007 /**
4008  * _scsih_block_io_to_children_attached_directly
4009  * @ioc: per adapter object
4010  * @event_data: topology change event data
4011  *
4012  * This routine set sdev state to SDEV_BLOCK for all devices
4013  * direct attached during device pull.
4014  */
4015 static void
4016 _scsih_block_io_to_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,
4017         Mpi2EventDataSasTopologyChangeList_t *event_data)
4018 {
4019         int i;
4020         u16 handle;
4021         u16 reason_code;
4022
4023         for (i = 0; i < event_data->NumEntries; i++) {
4024                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
4025                 if (!handle)
4026                         continue;
4027                 reason_code = event_data->PHY[i].PhyStatus &
4028                     MPI2_EVENT_SAS_TOPO_RC_MASK;
4029                 if (reason_code == MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING)
4030                         _scsih_block_io_device(ioc, handle);
4031         }
4032 }
4033
4034 /**
4035  * _scsih_block_io_to_pcie_children_attached_directly
4036  * @ioc: per adapter object
4037  * @event_data: topology change event data
4038  *
4039  * This routine set sdev state to SDEV_BLOCK for all devices
4040  * direct attached during device pull/reconnect.
4041  */
4042 static void
4043 _scsih_block_io_to_pcie_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,
4044                 Mpi26EventDataPCIeTopologyChangeList_t *event_data)
4045 {
4046         int i;
4047         u16 handle;
4048         u16 reason_code;
4049
4050         for (i = 0; i < event_data->NumEntries; i++) {
4051                 handle =
4052                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
4053                 if (!handle)
4054                         continue;
4055                 reason_code = event_data->PortEntry[i].PortStatus;
4056                 if (reason_code ==
4057                                 MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING)
4058                         _scsih_block_io_device(ioc, handle);
4059         }
4060 }
4061 /**
4062  * _scsih_tm_tr_send - send task management request
4063  * @ioc: per adapter object
4064  * @handle: device handle
4065  * Context: interrupt time.
4066  *
4067  * This code is to initiate the device removal handshake protocol
4068  * with controller firmware.  This function will issue target reset
4069  * using high priority request queue.  It will send a sas iounit
4070  * control request (MPI2_SAS_OP_REMOVE_DEVICE) from this completion.
4071  *
4072  * This is designed to send muliple task management request at the same
4073  * time to the fifo. If the fifo is full, we will append the request,
4074  * and process it in a future completion.
4075  */
4076 static void
4077 _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4078 {
4079         Mpi2SCSITaskManagementRequest_t *mpi_request;
4080         u16 smid;
4081         struct _sas_device *sas_device = NULL;
4082         struct _pcie_device *pcie_device = NULL;
4083         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
4084         u64 sas_address = 0;
4085         unsigned long flags;
4086         struct _tr_list *delayed_tr;
4087         u32 ioc_state;
4088         u8 tr_method = 0;
4089         struct hba_port *port = NULL;
4090
4091         if (ioc->pci_error_recovery) {
4092                 dewtprintk(ioc,
4093                            ioc_info(ioc, "%s: host in pci error recovery: handle(0x%04x)\n",
4094                                     __func__, handle));
4095                 return;
4096         }
4097         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
4098         if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4099                 dewtprintk(ioc,
4100                            ioc_info(ioc, "%s: host is not operational: handle(0x%04x)\n",
4101                                     __func__, handle));
4102                 return;
4103         }
4104
4105         /* if PD, then return */
4106         if (test_bit(handle, ioc->pd_handles))
4107                 return;
4108
4109         clear_bit(handle, ioc->pend_os_device_add);
4110
4111         spin_lock_irqsave(&ioc->sas_device_lock, flags);
4112         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
4113         if (sas_device && sas_device->starget &&
4114             sas_device->starget->hostdata) {
4115                 sas_target_priv_data = sas_device->starget->hostdata;
4116                 sas_target_priv_data->deleted = 1;
4117                 sas_address = sas_device->sas_address;
4118                 port = sas_device->port;
4119         }
4120         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
4121         if (!sas_device) {
4122                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
4123                 pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
4124                 if (pcie_device && pcie_device->starget &&
4125                         pcie_device->starget->hostdata) {
4126                         sas_target_priv_data = pcie_device->starget->hostdata;
4127                         sas_target_priv_data->deleted = 1;
4128                         sas_address = pcie_device->wwid;
4129                 }
4130                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
4131                 if (pcie_device && (!ioc->tm_custom_handling) &&
4132                     (!(mpt3sas_scsih_is_pcie_scsi_device(
4133                     pcie_device->device_info))))
4134                         tr_method =
4135                             MPI26_SCSITASKMGMT_MSGFLAGS_PROTOCOL_LVL_RST_PCIE;
4136                 else
4137                         tr_method = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
4138         }
4139         if (sas_target_priv_data) {
4140                 dewtprintk(ioc,
4141                            ioc_info(ioc, "setting delete flag: handle(0x%04x), sas_addr(0x%016llx)\n",
4142                                     handle, (u64)sas_address));
4143                 if (sas_device) {
4144                         if (sas_device->enclosure_handle != 0)
4145                                 dewtprintk(ioc,
4146                                            ioc_info(ioc, "setting delete flag:enclosure logical id(0x%016llx), slot(%d)\n",
4147                                                     (u64)sas_device->enclosure_logical_id,
4148                                                     sas_device->slot));
4149                         if (sas_device->connector_name[0] != '\0')
4150                                 dewtprintk(ioc,
4151                                            ioc_info(ioc, "setting delete flag: enclosure level(0x%04x), connector name( %s)\n",
4152                                                     sas_device->enclosure_level,
4153                                                     sas_device->connector_name));
4154                 } else if (pcie_device) {
4155                         if (pcie_device->enclosure_handle != 0)
4156                                 dewtprintk(ioc,
4157                                            ioc_info(ioc, "setting delete flag: logical id(0x%016llx), slot(%d)\n",
4158                                                     (u64)pcie_device->enclosure_logical_id,
4159                                                     pcie_device->slot));
4160                         if (pcie_device->connector_name[0] != '\0')
4161                                 dewtprintk(ioc,
4162                                            ioc_info(ioc, "setting delete flag:, enclosure level(0x%04x), connector name( %s)\n",
4163                                                     pcie_device->enclosure_level,
4164                                                     pcie_device->connector_name));
4165                 }
4166                 _scsih_ublock_io_device(ioc, sas_address, port);
4167                 sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
4168         }
4169
4170         smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_cb_idx);
4171         if (!smid) {
4172                 delayed_tr = kzalloc(sizeof(*delayed_tr), GFP_ATOMIC);
4173                 if (!delayed_tr)
4174                         goto out;
4175                 INIT_LIST_HEAD(&delayed_tr->list);
4176                 delayed_tr->handle = handle;
4177                 list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);
4178                 dewtprintk(ioc,
4179                            ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4180                                     handle));
4181                 goto out;
4182         }
4183
4184         dewtprintk(ioc,
4185                    ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4186                             handle, smid, ioc->tm_tr_cb_idx));
4187         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4188         memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
4189         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4190         mpi_request->DevHandle = cpu_to_le16(handle);
4191         mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
4192         mpi_request->MsgFlags = tr_method;
4193         set_bit(handle, ioc->device_remove_in_progress);
4194         ioc->put_smid_hi_priority(ioc, smid, 0);
4195         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);
4196
4197 out:
4198         if (sas_device)
4199                 sas_device_put(sas_device);
4200         if (pcie_device)
4201                 pcie_device_put(pcie_device);
4202 }
4203
4204 /**
4205  * _scsih_tm_tr_complete -
4206  * @ioc: per adapter object
4207  * @smid: system request message index
4208  * @msix_index: MSIX table index supplied by the OS
4209  * @reply: reply message frame(lower 32bit addr)
4210  * Context: interrupt time.
4211  *
4212  * This is the target reset completion routine.
4213  * This code is part of the code to initiate the device removal
4214  * handshake protocol with controller firmware.
4215  * It will send a sas iounit control request (MPI2_SAS_OP_REMOVE_DEVICE)
4216  *
4217  * Return: 1 meaning mf should be freed from _base_interrupt
4218  *         0 means the mf is freed from this function.
4219  */
4220 static u8
4221 _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
4222         u32 reply)
4223 {
4224         u16 handle;
4225         Mpi2SCSITaskManagementRequest_t *mpi_request_tm;
4226         Mpi2SCSITaskManagementReply_t *mpi_reply =
4227             mpt3sas_base_get_reply_virt_addr(ioc, reply);
4228         Mpi2SasIoUnitControlRequest_t *mpi_request;
4229         u16 smid_sas_ctrl;
4230         u32 ioc_state;
4231         struct _sc_list *delayed_sc;
4232
4233         if (ioc->pci_error_recovery) {
4234                 dewtprintk(ioc,
4235                            ioc_info(ioc, "%s: host in pci error recovery\n",
4236                                     __func__));
4237                 return 1;
4238         }
4239         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
4240         if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4241                 dewtprintk(ioc,
4242                            ioc_info(ioc, "%s: host is not operational\n",
4243                                     __func__));
4244                 return 1;
4245         }
4246         if (unlikely(!mpi_reply)) {
4247                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4248                         __FILE__, __LINE__, __func__);
4249                 return 1;
4250         }
4251         mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);
4252         handle = le16_to_cpu(mpi_request_tm->DevHandle);
4253         if (handle != le16_to_cpu(mpi_reply->DevHandle)) {
4254                 dewtprintk(ioc,
4255                            ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",
4256                                    handle,
4257                                    le16_to_cpu(mpi_reply->DevHandle), smid));
4258                 return 0;
4259         }
4260
4261         mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);
4262         dewtprintk(ioc,
4263                    ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), completed(%d)\n",
4264                             handle, smid, le16_to_cpu(mpi_reply->IOCStatus),
4265                             le32_to_cpu(mpi_reply->IOCLogInfo),
4266                             le32_to_cpu(mpi_reply->TerminationCount)));
4267
4268         smid_sas_ctrl = mpt3sas_base_get_smid(ioc, ioc->tm_sas_control_cb_idx);
4269         if (!smid_sas_ctrl) {
4270                 delayed_sc = kzalloc(sizeof(*delayed_sc), GFP_ATOMIC);
4271                 if (!delayed_sc)
4272                         return _scsih_check_for_pending_tm(ioc, smid);
4273                 INIT_LIST_HEAD(&delayed_sc->list);
4274                 delayed_sc->handle = le16_to_cpu(mpi_request_tm->DevHandle);
4275                 list_add_tail(&delayed_sc->list, &ioc->delayed_sc_list);
4276                 dewtprintk(ioc,
4277                            ioc_info(ioc, "DELAYED:sc:handle(0x%04x), (open)\n",
4278                                     handle));
4279                 return _scsih_check_for_pending_tm(ioc, smid);
4280         }
4281
4282         dewtprintk(ioc,
4283                    ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4284                             handle, smid_sas_ctrl, ioc->tm_sas_control_cb_idx));
4285         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid_sas_ctrl);
4286         memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
4287         mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
4288         mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
4289         mpi_request->DevHandle = mpi_request_tm->DevHandle;
4290         ioc->put_smid_default(ioc, smid_sas_ctrl);
4291
4292         return _scsih_check_for_pending_tm(ioc, smid);
4293 }
4294
4295 /** _scsih_allow_scmd_to_device - check whether scmd needs to
4296  *                               issue to IOC or not.
4297  * @ioc: per adapter object
4298  * @scmd: pointer to scsi command object
4299  *
4300  * Returns true if scmd can be issued to IOC otherwise returns false.
4301  */
4302 inline bool _scsih_allow_scmd_to_device(struct MPT3SAS_ADAPTER *ioc,
4303         struct scsi_cmnd *scmd)
4304 {
4305
4306         if (ioc->pci_error_recovery)
4307                 return false;
4308
4309         if (ioc->hba_mpi_version_belonged == MPI2_VERSION) {
4310                 if (ioc->remove_host)
4311                         return false;
4312
4313                 return true;
4314         }
4315
4316         if (ioc->remove_host) {
4317
4318                 switch (scmd->cmnd[0]) {
4319                 case SYNCHRONIZE_CACHE:
4320                 case START_STOP:
4321                         return true;
4322                 default:
4323                         return false;
4324                 }
4325         }
4326
4327         return true;
4328 }
4329
4330 /**
4331  * _scsih_sas_control_complete - completion routine
4332  * @ioc: per adapter object
4333  * @smid: system request message index
4334  * @msix_index: MSIX table index supplied by the OS
4335  * @reply: reply message frame(lower 32bit addr)
4336  * Context: interrupt time.
4337  *
4338  * This is the sas iounit control completion routine.
4339  * This code is part of the code to initiate the device removal
4340  * handshake protocol with controller firmware.
4341  *
4342  * Return: 1 meaning mf should be freed from _base_interrupt
4343  *         0 means the mf is freed from this function.
4344  */
4345 static u8
4346 _scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4347         u8 msix_index, u32 reply)
4348 {
4349         Mpi2SasIoUnitControlReply_t *mpi_reply =
4350             mpt3sas_base_get_reply_virt_addr(ioc, reply);
4351
4352         if (likely(mpi_reply)) {
4353                 dewtprintk(ioc,
4354                            ioc_info(ioc, "sc_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x)\n",
4355                                     le16_to_cpu(mpi_reply->DevHandle), smid,
4356                                     le16_to_cpu(mpi_reply->IOCStatus),
4357                                     le32_to_cpu(mpi_reply->IOCLogInfo)));
4358                 if (le16_to_cpu(mpi_reply->IOCStatus) ==
4359                      MPI2_IOCSTATUS_SUCCESS) {
4360                         clear_bit(le16_to_cpu(mpi_reply->DevHandle),
4361                             ioc->device_remove_in_progress);
4362                 }
4363         } else {
4364                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4365                         __FILE__, __LINE__, __func__);
4366         }
4367         return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
4368 }
4369
4370 /**
4371  * _scsih_tm_tr_volume_send - send target reset request for volumes
4372  * @ioc: per adapter object
4373  * @handle: device handle
4374  * Context: interrupt time.
4375  *
4376  * This is designed to send muliple task management request at the same
4377  * time to the fifo. If the fifo is full, we will append the request,
4378  * and process it in a future completion.
4379  */
4380 static void
4381 _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4382 {
4383         Mpi2SCSITaskManagementRequest_t *mpi_request;
4384         u16 smid;
4385         struct _tr_list *delayed_tr;
4386
4387         if (ioc->pci_error_recovery) {
4388                 dewtprintk(ioc,
4389                            ioc_info(ioc, "%s: host reset in progress!\n",
4390                                     __func__));
4391                 return;
4392         }
4393
4394         smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_volume_cb_idx);
4395         if (!smid) {
4396                 delayed_tr = kzalloc(sizeof(*delayed_tr), GFP_ATOMIC);
4397                 if (!delayed_tr)
4398                         return;
4399                 INIT_LIST_HEAD(&delayed_tr->list);
4400                 delayed_tr->handle = handle;
4401                 list_add_tail(&delayed_tr->list, &ioc->delayed_tr_volume_list);
4402                 dewtprintk(ioc,
4403                            ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4404                                     handle));
4405                 return;
4406         }
4407
4408         dewtprintk(ioc,
4409                    ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4410                             handle, smid, ioc->tm_tr_volume_cb_idx));
4411         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4412         memset(mpi_request, 0, sizeof(Mpi2SCSITaskManagementRequest_t));
4413         mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4414         mpi_request->DevHandle = cpu_to_le16(handle);
4415         mpi_request->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
4416         ioc->put_smid_hi_priority(ioc, smid, 0);
4417 }
4418
4419 /**
4420  * _scsih_tm_volume_tr_complete - target reset completion
4421  * @ioc: per adapter object
4422  * @smid: system request message index
4423  * @msix_index: MSIX table index supplied by the OS
4424  * @reply: reply message frame(lower 32bit addr)
4425  * Context: interrupt time.
4426  *
4427  * Return: 1 meaning mf should be freed from _base_interrupt
4428  *         0 means the mf is freed from this function.
4429  */
4430 static u8
4431 _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4432         u8 msix_index, u32 reply)
4433 {
4434         u16 handle;
4435         Mpi2SCSITaskManagementRequest_t *mpi_request_tm;
4436         Mpi2SCSITaskManagementReply_t *mpi_reply =
4437             mpt3sas_base_get_reply_virt_addr(ioc, reply);
4438
4439         if (ioc->shost_recovery || ioc->pci_error_recovery) {
4440                 dewtprintk(ioc,
4441                            ioc_info(ioc, "%s: host reset in progress!\n",
4442                                     __func__));
4443                 return 1;
4444         }
4445         if (unlikely(!mpi_reply)) {
4446                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
4447                         __FILE__, __LINE__, __func__);
4448                 return 1;
4449         }
4450
4451         mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);
4452         handle = le16_to_cpu(mpi_request_tm->DevHandle);
4453         if (handle != le16_to_cpu(mpi_reply->DevHandle)) {
4454                 dewtprintk(ioc,
4455                            ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",
4456                                    handle, le16_to_cpu(mpi_reply->DevHandle),
4457                                    smid));
4458                 return 0;
4459         }
4460
4461         dewtprintk(ioc,
4462                    ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), completed(%d)\n",
4463                             handle, smid, le16_to_cpu(mpi_reply->IOCStatus),
4464                             le32_to_cpu(mpi_reply->IOCLogInfo),
4465                             le32_to_cpu(mpi_reply->TerminationCount)));
4466
4467         return _scsih_check_for_pending_tm(ioc, smid);
4468 }
4469
4470 /**
4471  * _scsih_issue_delayed_event_ack - issue delayed Event ACK messages
4472  * @ioc: per adapter object
4473  * @smid: system request message index
4474  * @event: Event ID
4475  * @event_context: used to track events uniquely
4476  *
4477  * Context - processed in interrupt context.
4478  */
4479 static void
4480 _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,
4481                                 U32 event_context)
4482 {
4483         Mpi2EventAckRequest_t *ack_request;
4484         int i = smid - ioc->internal_smid;
4485         unsigned long flags;
4486
4487         /* Without releasing the smid just update the
4488          * call back index and reuse the same smid for
4489          * processing this delayed request
4490          */
4491         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4492         ioc->internal_lookup[i].cb_idx = ioc->base_cb_idx;
4493         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
4494
4495         dewtprintk(ioc,
4496                    ioc_info(ioc, "EVENT ACK: event(0x%04x), smid(%d), cb(%d)\n",
4497                             le16_to_cpu(event), smid, ioc->base_cb_idx));
4498         ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
4499         memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
4500         ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
4501         ack_request->Event = event;
4502         ack_request->EventContext = event_context;
4503         ack_request->VF_ID = 0;  /* TODO */
4504         ack_request->VP_ID = 0;
4505         ioc->put_smid_default(ioc, smid);
4506 }
4507
4508 /**
4509  * _scsih_issue_delayed_sas_io_unit_ctrl - issue delayed
4510  *                              sas_io_unit_ctrl messages
4511  * @ioc: per adapter object
4512  * @smid: system request message index
4513  * @handle: device handle
4514  *
4515  * Context - processed in interrupt context.
4516  */
4517 static void
4518 _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,
4519                                         u16 smid, u16 handle)
4520 {
4521         Mpi2SasIoUnitControlRequest_t *mpi_request;
4522         u32 ioc_state;
4523         int i = smid - ioc->internal_smid;
4524         unsigned long flags;
4525
4526         if (ioc->remove_host) {
4527                 dewtprintk(ioc,
4528                            ioc_info(ioc, "%s: host has been removed\n",
4529                                     __func__));
4530                 return;
4531         } else if (ioc->pci_error_recovery) {
4532                 dewtprintk(ioc,
4533                            ioc_info(ioc, "%s: host in pci error recovery\n",
4534                                     __func__));
4535                 return;
4536         }
4537         ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
4538         if (ioc_state != MPI2_IOC_STATE_OPERATIONAL) {
4539                 dewtprintk(ioc,
4540                            ioc_info(ioc, "%s: host is not operational\n",
4541                                     __func__));
4542                 return;
4543         }
4544
4545         /* Without releasing the smid just update the
4546          * call back index and reuse the same smid for
4547          * processing this delayed request
4548          */
4549         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
4550         ioc->internal_lookup[i].cb_idx = ioc->tm_sas_control_cb_idx;
4551         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
4552
4553         dewtprintk(ioc,
4554                    ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",
4555                             handle, smid, ioc->tm_sas_control_cb_idx));
4556         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4557         memset(mpi_request, 0, sizeof(Mpi2SasIoUnitControlRequest_t));
4558         mpi_request->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
4559         mpi_request->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
4560         mpi_request->DevHandle = cpu_to_le16(handle);
4561         ioc->put_smid_default(ioc, smid);
4562 }
4563
4564 /**
4565  * mpt3sas_check_for_pending_internal_cmds - check for pending internal messages
4566  * @ioc: per adapter object
4567  * @smid: system request message index
4568  *
4569  * Context: Executed in interrupt context
4570  *
4571  * This will check delayed internal messages list, and process the
4572  * next request.
4573  *
4574  * Return: 1 meaning mf should be freed from _base_interrupt
4575  *         0 means the mf is freed from this function.
4576  */
4577 u8
4578 mpt3sas_check_for_pending_internal_cmds(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4579 {
4580         struct _sc_list *delayed_sc;
4581         struct _event_ack_list *delayed_event_ack;
4582
4583         if (!list_empty(&ioc->delayed_event_ack_list)) {
4584                 delayed_event_ack = list_entry(ioc->delayed_event_ack_list.next,
4585                                                 struct _event_ack_list, list);
4586                 _scsih_issue_delayed_event_ack(ioc, smid,
4587                   delayed_event_ack->Event, delayed_event_ack->EventContext);
4588                 list_del(&delayed_event_ack->list);
4589                 kfree(delayed_event_ack);
4590                 return 0;
4591         }
4592
4593         if (!list_empty(&ioc->delayed_sc_list)) {
4594                 delayed_sc = list_entry(ioc->delayed_sc_list.next,
4595                                                 struct _sc_list, list);
4596                 _scsih_issue_delayed_sas_io_unit_ctrl(ioc, smid,
4597                                                  delayed_sc->handle);
4598                 list_del(&delayed_sc->list);
4599                 kfree(delayed_sc);
4600                 return 0;
4601         }
4602         return 1;
4603 }
4604
4605 /**
4606  * _scsih_check_for_pending_tm - check for pending task management
4607  * @ioc: per adapter object
4608  * @smid: system request message index
4609  *
4610  * This will check delayed target reset list, and feed the
4611  * next reqeust.
4612  *
4613  * Return: 1 meaning mf should be freed from _base_interrupt
4614  *         0 means the mf is freed from this function.
4615  */
4616 static u8
4617 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4618 {
4619         struct _tr_list *delayed_tr;
4620
4621         if (!list_empty(&ioc->delayed_tr_volume_list)) {
4622                 delayed_tr = list_entry(ioc->delayed_tr_volume_list.next,
4623                     struct _tr_list, list);
4624                 mpt3sas_base_free_smid(ioc, smid);
4625                 _scsih_tm_tr_volume_send(ioc, delayed_tr->handle);
4626                 list_del(&delayed_tr->list);
4627                 kfree(delayed_tr);
4628                 return 0;
4629         }
4630
4631         if (!list_empty(&ioc->delayed_tr_list)) {
4632                 delayed_tr = list_entry(ioc->delayed_tr_list.next,
4633                     struct _tr_list, list);
4634                 mpt3sas_base_free_smid(ioc, smid);
4635                 _scsih_tm_tr_send(ioc, delayed_tr->handle);
4636                 list_del(&delayed_tr->list);
4637                 kfree(delayed_tr);
4638                 return 0;
4639         }
4640
4641         return 1;
4642 }
4643
4644 /**
4645  * _scsih_check_topo_delete_events - sanity check on topo events
4646  * @ioc: per adapter object
4647  * @event_data: the event data payload
4648  *
4649  * This routine added to better handle cable breaker.
4650  *
4651  * This handles the case where driver receives multiple expander
4652  * add and delete events in a single shot.  When there is a delete event
4653  * the routine will void any pending add events waiting in the event queue.
4654  */
4655 static void
4656 _scsih_check_topo_delete_events(struct MPT3SAS_ADAPTER *ioc,
4657         Mpi2EventDataSasTopologyChangeList_t *event_data)
4658 {
4659         struct fw_event_work *fw_event;
4660         Mpi2EventDataSasTopologyChangeList_t *local_event_data;
4661         u16 expander_handle;
4662         struct _sas_node *sas_expander;
4663         unsigned long flags;
4664         int i, reason_code;
4665         u16 handle;
4666
4667         for (i = 0 ; i < event_data->NumEntries; i++) {
4668                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
4669                 if (!handle)
4670                         continue;
4671                 reason_code = event_data->PHY[i].PhyStatus &
4672                     MPI2_EVENT_SAS_TOPO_RC_MASK;
4673                 if (reason_code == MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING)
4674                         _scsih_tm_tr_send(ioc, handle);
4675         }
4676
4677         expander_handle = le16_to_cpu(event_data->ExpanderDevHandle);
4678         if (expander_handle < ioc->sas_hba.num_phys) {
4679                 _scsih_block_io_to_children_attached_directly(ioc, event_data);
4680                 return;
4681         }
4682         if (event_data->ExpStatus ==
4683             MPI2_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING) {
4684                 /* put expander attached devices into blocking state */
4685                 spin_lock_irqsave(&ioc->sas_node_lock, flags);
4686                 sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,
4687                     expander_handle);
4688                 _scsih_block_io_to_children_attached_to_ex(ioc, sas_expander);
4689                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
4690                 do {
4691                         handle = find_first_bit(ioc->blocking_handles,
4692                             ioc->facts.MaxDevHandle);
4693                         if (handle < ioc->facts.MaxDevHandle)
4694                                 _scsih_block_io_device(ioc, handle);
4695                 } while (test_and_clear_bit(handle, ioc->blocking_handles));
4696         } else if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_RESPONDING)
4697                 _scsih_block_io_to_children_attached_directly(ioc, event_data);
4698
4699         if (event_data->ExpStatus != MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
4700                 return;
4701
4702         /* mark ignore flag for pending events */
4703         spin_lock_irqsave(&ioc->fw_event_lock, flags);
4704         list_for_each_entry(fw_event, &ioc->fw_event_list, list) {
4705                 if (fw_event->event != MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST ||
4706                     fw_event->ignore)
4707                         continue;
4708                 local_event_data = (Mpi2EventDataSasTopologyChangeList_t *)
4709                                    fw_event->event_data;
4710                 if (local_event_data->ExpStatus ==
4711                     MPI2_EVENT_SAS_TOPO_ES_ADDED ||
4712                     local_event_data->ExpStatus ==
4713                     MPI2_EVENT_SAS_TOPO_ES_RESPONDING) {
4714                         if (le16_to_cpu(local_event_data->ExpanderDevHandle) ==
4715                             expander_handle) {
4716                                 dewtprintk(ioc,
4717                                            ioc_info(ioc, "setting ignoring flag\n"));
4718                                 fw_event->ignore = 1;
4719                         }
4720                 }
4721         }
4722         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
4723 }
4724
4725 /**
4726  * _scsih_check_pcie_topo_remove_events - sanity check on topo
4727  * events
4728  * @ioc: per adapter object
4729  * @event_data: the event data payload
4730  *
4731  * This handles the case where driver receives multiple switch
4732  * or device add and delete events in a single shot.  When there
4733  * is a delete event the routine will void any pending add
4734  * events waiting in the event queue.
4735  */
4736 static void
4737 _scsih_check_pcie_topo_remove_events(struct MPT3SAS_ADAPTER *ioc,
4738         Mpi26EventDataPCIeTopologyChangeList_t *event_data)
4739 {
4740         struct fw_event_work *fw_event;
4741         Mpi26EventDataPCIeTopologyChangeList_t *local_event_data;
4742         unsigned long flags;
4743         int i, reason_code;
4744         u16 handle, switch_handle;
4745
4746         for (i = 0; i < event_data->NumEntries; i++) {
4747                 handle =
4748                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
4749                 if (!handle)
4750                         continue;
4751                 reason_code = event_data->PortEntry[i].PortStatus;
4752                 if (reason_code == MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING)
4753                         _scsih_tm_tr_send(ioc, handle);
4754         }
4755
4756         switch_handle = le16_to_cpu(event_data->SwitchDevHandle);
4757         if (!switch_handle) {
4758                 _scsih_block_io_to_pcie_children_attached_directly(
4759                                                         ioc, event_data);
4760                 return;
4761         }
4762     /* TODO We are not supporting cascaded PCIe Switch removal yet*/
4763         if ((event_data->SwitchStatus
4764                 == MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING) ||
4765                 (event_data->SwitchStatus ==
4766                                         MPI26_EVENT_PCIE_TOPO_SS_RESPONDING))
4767                 _scsih_block_io_to_pcie_children_attached_directly(
4768                                                         ioc, event_data);
4769
4770         if (event_data->SwitchStatus != MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING)
4771                 return;
4772
4773         /* mark ignore flag for pending events */
4774         spin_lock_irqsave(&ioc->fw_event_lock, flags);
4775         list_for_each_entry(fw_event, &ioc->fw_event_list, list) {
4776                 if (fw_event->event != MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST ||
4777                         fw_event->ignore)
4778                         continue;
4779                 local_event_data =
4780                         (Mpi26EventDataPCIeTopologyChangeList_t *)
4781                         fw_event->event_data;
4782                 if (local_event_data->SwitchStatus ==
4783                     MPI2_EVENT_SAS_TOPO_ES_ADDED ||
4784                     local_event_data->SwitchStatus ==
4785                     MPI2_EVENT_SAS_TOPO_ES_RESPONDING) {
4786                         if (le16_to_cpu(local_event_data->SwitchDevHandle) ==
4787                                 switch_handle) {
4788                                 dewtprintk(ioc,
4789                                            ioc_info(ioc, "setting ignoring flag for switch event\n"));
4790                                 fw_event->ignore = 1;
4791                         }
4792                 }
4793         }
4794         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
4795 }
4796
4797 /**
4798  * _scsih_set_volume_delete_flag - setting volume delete flag
4799  * @ioc: per adapter object
4800  * @handle: device handle
4801  *
4802  * This returns nothing.
4803  */
4804 static void
4805 _scsih_set_volume_delete_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)
4806 {
4807         struct _raid_device *raid_device;
4808         struct MPT3SAS_TARGET *sas_target_priv_data;
4809         unsigned long flags;
4810
4811         spin_lock_irqsave(&ioc->raid_device_lock, flags);
4812         raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
4813         if (raid_device && raid_device->starget &&
4814             raid_device->starget->hostdata) {
4815                 sas_target_priv_data =
4816                     raid_device->starget->hostdata;
4817                 sas_target_priv_data->deleted = 1;
4818                 dewtprintk(ioc,
4819                            ioc_info(ioc, "setting delete flag: handle(0x%04x), wwid(0x%016llx)\n",
4820                                     handle, (u64)raid_device->wwid));
4821         }
4822         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
4823 }
4824
4825 /**
4826  * _scsih_set_volume_handle_for_tr - set handle for target reset to volume
4827  * @handle: input handle
4828  * @a: handle for volume a
4829  * @b: handle for volume b
4830  *
4831  * IR firmware only supports two raid volumes.  The purpose of this
4832  * routine is to set the volume handle in either a or b. When the given
4833  * input handle is non-zero, or when a and b have not been set before.
4834  */
4835 static void
4836 _scsih_set_volume_handle_for_tr(u16 handle, u16 *a, u16 *b)
4837 {
4838         if (!handle || handle == *a || handle == *b)
4839                 return;
4840         if (!*a)
4841                 *a = handle;
4842         else if (!*b)
4843                 *b = handle;
4844 }
4845
4846 /**
4847  * _scsih_check_ir_config_unhide_events - check for UNHIDE events
4848  * @ioc: per adapter object
4849  * @event_data: the event data payload
4850  * Context: interrupt time.
4851  *
4852  * This routine will send target reset to volume, followed by target
4853  * resets to the PDs. This is called when a PD has been removed, or
4854  * volume has been deleted or removed. When the target reset is sent
4855  * to volume, the PD target resets need to be queued to start upon
4856  * completion of the volume target reset.
4857  */
4858 static void
4859 _scsih_check_ir_config_unhide_events(struct MPT3SAS_ADAPTER *ioc,
4860         Mpi2EventDataIrConfigChangeList_t *event_data)
4861 {
4862         Mpi2EventIrConfigElement_t *element;
4863         int i;
4864         u16 handle, volume_handle, a, b;
4865         struct _tr_list *delayed_tr;
4866
4867         a = 0;
4868         b = 0;
4869
4870         if (ioc->is_warpdrive)
4871                 return;
4872
4873         /* Volume Resets for Deleted or Removed */
4874         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4875         for (i = 0; i < event_data->NumElements; i++, element++) {
4876                 if (le32_to_cpu(event_data->Flags) &
4877                     MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG)
4878                         continue;
4879                 if (element->ReasonCode ==
4880                     MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED ||
4881                     element->ReasonCode ==
4882                     MPI2_EVENT_IR_CHANGE_RC_REMOVED) {
4883                         volume_handle = le16_to_cpu(element->VolDevHandle);
4884                         _scsih_set_volume_delete_flag(ioc, volume_handle);
4885                         _scsih_set_volume_handle_for_tr(volume_handle, &a, &b);
4886                 }
4887         }
4888
4889         /* Volume Resets for UNHIDE events */
4890         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4891         for (i = 0; i < event_data->NumElements; i++, element++) {
4892                 if (le32_to_cpu(event_data->Flags) &
4893                     MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG)
4894                         continue;
4895                 if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_UNHIDE) {
4896                         volume_handle = le16_to_cpu(element->VolDevHandle);
4897                         _scsih_set_volume_handle_for_tr(volume_handle, &a, &b);
4898                 }
4899         }
4900
4901         if (a)
4902                 _scsih_tm_tr_volume_send(ioc, a);
4903         if (b)
4904                 _scsih_tm_tr_volume_send(ioc, b);
4905
4906         /* PD target resets */
4907         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
4908         for (i = 0; i < event_data->NumElements; i++, element++) {
4909                 if (element->ReasonCode != MPI2_EVENT_IR_CHANGE_RC_UNHIDE)
4910                         continue;
4911                 handle = le16_to_cpu(element->PhysDiskDevHandle);
4912                 volume_handle = le16_to_cpu(element->VolDevHandle);
4913                 clear_bit(handle, ioc->pd_handles);
4914                 if (!volume_handle)
4915                         _scsih_tm_tr_send(ioc, handle);
4916                 else if (volume_handle == a || volume_handle == b) {
4917                         delayed_tr = kzalloc(sizeof(*delayed_tr), GFP_ATOMIC);
4918                         BUG_ON(!delayed_tr);
4919                         INIT_LIST_HEAD(&delayed_tr->list);
4920                         delayed_tr->handle = handle;
4921                         list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);
4922                         dewtprintk(ioc,
4923                                    ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",
4924                                             handle));
4925                 } else
4926                         _scsih_tm_tr_send(ioc, handle);
4927         }
4928 }
4929
4930
4931 /**
4932  * _scsih_check_volume_delete_events - set delete flag for volumes
4933  * @ioc: per adapter object
4934  * @event_data: the event data payload
4935  * Context: interrupt time.
4936  *
4937  * This will handle the case when the cable connected to entire volume is
4938  * pulled. We will take care of setting the deleted flag so normal IO will
4939  * not be sent.
4940  */
4941 static void
4942 _scsih_check_volume_delete_events(struct MPT3SAS_ADAPTER *ioc,
4943         Mpi2EventDataIrVolume_t *event_data)
4944 {
4945         u32 state;
4946
4947         if (event_data->ReasonCode != MPI2_EVENT_IR_VOLUME_RC_STATE_CHANGED)
4948                 return;
4949         state = le32_to_cpu(event_data->NewValue);
4950         if (state == MPI2_RAID_VOL_STATE_MISSING || state ==
4951             MPI2_RAID_VOL_STATE_FAILED)
4952                 _scsih_set_volume_delete_flag(ioc,
4953                     le16_to_cpu(event_data->VolDevHandle));
4954 }
4955
4956 /**
4957  * _scsih_temp_threshold_events - display temperature threshold exceeded events
4958  * @ioc: per adapter object
4959  * @event_data: the temp threshold event data
4960  * Context: interrupt time.
4961  */
4962 static void
4963 _scsih_temp_threshold_events(struct MPT3SAS_ADAPTER *ioc,
4964         Mpi2EventDataTemperature_t *event_data)
4965 {
4966         u32 doorbell;
4967         if (ioc->temp_sensors_count >= event_data->SensorNum) {
4968                 ioc_err(ioc, "Temperature Threshold flags %s%s%s%s exceeded for Sensor: %d !!!\n",
4969                         le16_to_cpu(event_data->Status) & 0x1 ? "0 " : " ",
4970                         le16_to_cpu(event_data->Status) & 0x2 ? "1 " : " ",
4971                         le16_to_cpu(event_data->Status) & 0x4 ? "2 " : " ",
4972                         le16_to_cpu(event_data->Status) & 0x8 ? "3 " : " ",
4973                         event_data->SensorNum);
4974                 ioc_err(ioc, "Current Temp In Celsius: %d\n",
4975                         event_data->CurrentTemperature);
4976                 if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
4977                         doorbell = mpt3sas_base_get_iocstate(ioc, 0);
4978                         if ((doorbell & MPI2_IOC_STATE_MASK) ==
4979                             MPI2_IOC_STATE_FAULT) {
4980                                 mpt3sas_print_fault_code(ioc,
4981                                     doorbell & MPI2_DOORBELL_DATA_MASK);
4982                         } else if ((doorbell & MPI2_IOC_STATE_MASK) ==
4983                             MPI2_IOC_STATE_COREDUMP) {
4984                                 mpt3sas_print_coredump_info(ioc,
4985                                     doorbell & MPI2_DOORBELL_DATA_MASK);
4986                         }
4987                 }
4988         }
4989 }
4990
4991 static int _scsih_set_satl_pending(struct scsi_cmnd *scmd, bool pending)
4992 {
4993         struct MPT3SAS_DEVICE *priv = scmd->device->hostdata;
4994
4995         if (scmd->cmnd[0] != ATA_12 && scmd->cmnd[0] != ATA_16)
4996                 return 0;
4997
4998         if (pending)
4999                 return test_and_set_bit(0, &priv->ata_command_pending);
5000
5001         clear_bit(0, &priv->ata_command_pending);
5002         return 0;
5003 }
5004
5005 /**
5006  * _scsih_flush_running_cmds - completing outstanding commands.
5007  * @ioc: per adapter object
5008  *
5009  * The flushing out of all pending scmd commands following host reset,
5010  * where all IO is dropped to the floor.
5011  */
5012 static void
5013 _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc)
5014 {
5015         struct scsi_cmnd *scmd;
5016         struct scsiio_tracker *st;
5017         u16 smid;
5018         int count = 0;
5019
5020         for (smid = 1; smid <= ioc->scsiio_depth; smid++) {
5021                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
5022                 if (!scmd)
5023                         continue;
5024                 count++;
5025                 _scsih_set_satl_pending(scmd, false);
5026                 st = scsi_cmd_priv(scmd);
5027                 mpt3sas_base_clear_st(ioc, st);
5028                 scsi_dma_unmap(scmd);
5029                 if (ioc->pci_error_recovery || ioc->remove_host)
5030                         scmd->result = DID_NO_CONNECT << 16;
5031                 else
5032                         scmd->result = DID_RESET << 16;
5033                 scmd->scsi_done(scmd);
5034         }
5035         dtmprintk(ioc, ioc_info(ioc, "completing %d cmds\n", count));
5036 }
5037
5038 /**
5039  * _scsih_setup_eedp - setup MPI request for EEDP transfer
5040  * @ioc: per adapter object
5041  * @scmd: pointer to scsi command object
5042  * @mpi_request: pointer to the SCSI_IO request message frame
5043  *
5044  * Supporting protection 1 and 3.
5045  */
5046 static void
5047 _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
5048         Mpi25SCSIIORequest_t *mpi_request)
5049 {
5050         u16 eedp_flags;
5051         Mpi25SCSIIORequest_t *mpi_request_3v =
5052            (Mpi25SCSIIORequest_t *)mpi_request;
5053
5054         switch (scsi_get_prot_op(scmd)) {
5055         case SCSI_PROT_READ_STRIP:
5056                 eedp_flags = MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP;
5057                 break;
5058         case SCSI_PROT_WRITE_INSERT:
5059                 eedp_flags = MPI2_SCSIIO_EEDPFLAGS_INSERT_OP;
5060                 break;
5061         default:
5062                 return;
5063         }
5064
5065         if (scmd->prot_flags & SCSI_PROT_GUARD_CHECK)
5066                 eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD;
5067
5068         if (scmd->prot_flags & SCSI_PROT_REF_CHECK) {
5069                 eedp_flags |= MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
5070                         MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG;
5071                 mpi_request->CDB.EEDP32.PrimaryReferenceTag =
5072                         cpu_to_be32(scsi_prot_ref_tag(scmd));
5073         }
5074
5075         mpi_request_3v->EEDPBlockSize = cpu_to_le16(scsi_prot_interval(scmd));
5076
5077         if (ioc->is_gen35_ioc)
5078                 eedp_flags |= MPI25_SCSIIO_EEDPFLAGS_APPTAG_DISABLE_MODE;
5079         mpi_request->EEDPFlags = cpu_to_le16(eedp_flags);
5080 }
5081
5082 /**
5083  * _scsih_eedp_error_handling - return sense code for EEDP errors
5084  * @scmd: pointer to scsi command object
5085  * @ioc_status: ioc status
5086  */
5087 static void
5088 _scsih_eedp_error_handling(struct scsi_cmnd *scmd, u16 ioc_status)
5089 {
5090         u8 ascq;
5091
5092         switch (ioc_status) {
5093         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5094                 ascq = 0x01;
5095                 break;
5096         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5097                 ascq = 0x02;
5098                 break;
5099         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5100                 ascq = 0x03;
5101                 break;
5102         default:
5103                 ascq = 0x00;
5104                 break;
5105         }
5106         scsi_build_sense(scmd, 0, ILLEGAL_REQUEST, 0x10, ascq);
5107         set_host_byte(scmd, DID_ABORT);
5108 }
5109
5110 /**
5111  * scsih_qcmd - main scsi request entry point
5112  * @shost: SCSI host pointer
5113  * @scmd: pointer to scsi command object
5114  *
5115  * The callback index is set inside `ioc->scsi_io_cb_idx`.
5116  *
5117  * Return: 0 on success.  If there's a failure, return either:
5118  * SCSI_MLQUEUE_DEVICE_BUSY if the device queue is full, or
5119  * SCSI_MLQUEUE_HOST_BUSY if the entire host queue is full
5120  */
5121 static int
5122 scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd)
5123 {
5124         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
5125         struct MPT3SAS_DEVICE *sas_device_priv_data;
5126         struct MPT3SAS_TARGET *sas_target_priv_data;
5127         struct _raid_device *raid_device;
5128         struct request *rq = scsi_cmd_to_rq(scmd);
5129         int class;
5130         Mpi25SCSIIORequest_t *mpi_request;
5131         struct _pcie_device *pcie_device = NULL;
5132         u32 mpi_control;
5133         u16 smid;
5134         u16 handle;
5135
5136         if (ioc->logging_level & MPT_DEBUG_SCSI)
5137                 scsi_print_command(scmd);
5138
5139         sas_device_priv_data = scmd->device->hostdata;
5140         if (!sas_device_priv_data || !sas_device_priv_data->sas_target) {
5141                 scmd->result = DID_NO_CONNECT << 16;
5142                 scmd->scsi_done(scmd);
5143                 return 0;
5144         }
5145
5146         if (!(_scsih_allow_scmd_to_device(ioc, scmd))) {
5147                 scmd->result = DID_NO_CONNECT << 16;
5148                 scmd->scsi_done(scmd);
5149                 return 0;
5150         }
5151
5152         sas_target_priv_data = sas_device_priv_data->sas_target;
5153
5154         /* invalid device handle */
5155         handle = sas_target_priv_data->handle;
5156         if (handle == MPT3SAS_INVALID_DEVICE_HANDLE) {
5157                 scmd->result = DID_NO_CONNECT << 16;
5158                 scmd->scsi_done(scmd);
5159                 return 0;
5160         }
5161
5162
5163         if (ioc->shost_recovery || ioc->ioc_link_reset_in_progress) {
5164                 /* host recovery or link resets sent via IOCTLs */
5165                 return SCSI_MLQUEUE_HOST_BUSY;
5166         } else if (sas_target_priv_data->deleted) {
5167                 /* device has been deleted */
5168                 scmd->result = DID_NO_CONNECT << 16;
5169                 scmd->scsi_done(scmd);
5170                 return 0;
5171         } else if (sas_target_priv_data->tm_busy ||
5172                    sas_device_priv_data->block) {
5173                 /* device busy with task management */
5174                 return SCSI_MLQUEUE_DEVICE_BUSY;
5175         }
5176
5177         /*
5178          * Bug work around for firmware SATL handling.  The loop
5179          * is based on atomic operations and ensures consistency
5180          * since we're lockless at this point
5181          */
5182         do {
5183                 if (test_bit(0, &sas_device_priv_data->ata_command_pending))
5184                         return SCSI_MLQUEUE_DEVICE_BUSY;
5185         } while (_scsih_set_satl_pending(scmd, true));
5186
5187         if (scmd->sc_data_direction == DMA_FROM_DEVICE)
5188                 mpi_control = MPI2_SCSIIO_CONTROL_READ;
5189         else if (scmd->sc_data_direction == DMA_TO_DEVICE)
5190                 mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
5191         else
5192                 mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
5193
5194         /* set tags */
5195         mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
5196         /* NCQ Prio supported, make sure control indicated high priority */
5197         if (sas_device_priv_data->ncq_prio_enable) {
5198                 class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
5199                 if (class == IOPRIO_CLASS_RT)
5200                         mpi_control |= 1 << MPI2_SCSIIO_CONTROL_CMDPRI_SHIFT;
5201         }
5202         /* Make sure Device is not raid volume.
5203          * We do not expose raid functionality to upper layer for warpdrive.
5204          */
5205         if (((!ioc->is_warpdrive && !scsih_is_raid(&scmd->device->sdev_gendev))
5206                 && !scsih_is_nvme(&scmd->device->sdev_gendev))
5207                 && sas_is_tlr_enabled(scmd->device) && scmd->cmd_len != 32)
5208                 mpi_control |= MPI2_SCSIIO_CONTROL_TLR_ON;
5209
5210         smid = mpt3sas_base_get_smid_scsiio(ioc, ioc->scsi_io_cb_idx, scmd);
5211         if (!smid) {
5212                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
5213                 _scsih_set_satl_pending(scmd, false);
5214                 goto out;
5215         }
5216         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5217         memset(mpi_request, 0, ioc->request_sz);
5218         _scsih_setup_eedp(ioc, scmd, mpi_request);
5219
5220         if (scmd->cmd_len == 32)
5221                 mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
5222         mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
5223         if (sas_device_priv_data->sas_target->flags &
5224             MPT_TARGET_FLAGS_RAID_COMPONENT)
5225                 mpi_request->Function = MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH;
5226         else
5227                 mpi_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
5228         mpi_request->DevHandle = cpu_to_le16(handle);
5229         mpi_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
5230         mpi_request->Control = cpu_to_le32(mpi_control);
5231         mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len);
5232         mpi_request->MsgFlags = MPI2_SCSIIO_MSGFLAGS_SYSTEM_SENSE_ADDR;
5233         mpi_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
5234         mpi_request->SenseBufferLowAddress =
5235             mpt3sas_base_get_sense_buffer_dma(ioc, smid);
5236         mpi_request->SGLOffset0 = offsetof(Mpi25SCSIIORequest_t, SGL) / 4;
5237         int_to_scsilun(sas_device_priv_data->lun, (struct scsi_lun *)
5238             mpi_request->LUN);
5239         memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
5240
5241         if (mpi_request->DataLength) {
5242                 pcie_device = sas_target_priv_data->pcie_dev;
5243                 if (ioc->build_sg_scmd(ioc, scmd, smid, pcie_device)) {
5244                         mpt3sas_base_free_smid(ioc, smid);
5245                         _scsih_set_satl_pending(scmd, false);
5246                         goto out;
5247                 }
5248         } else
5249                 ioc->build_zero_len_sge(ioc, &mpi_request->SGL);
5250
5251         raid_device = sas_target_priv_data->raid_device;
5252         if (raid_device && raid_device->direct_io_enabled)
5253                 mpt3sas_setup_direct_io(ioc, scmd,
5254                         raid_device, mpi_request);
5255
5256         if (likely(mpi_request->Function == MPI2_FUNCTION_SCSI_IO_REQUEST)) {
5257                 if (sas_target_priv_data->flags & MPT_TARGET_FASTPATH_IO) {
5258                         mpi_request->IoFlags = cpu_to_le16(scmd->cmd_len |
5259                             MPI25_SCSIIO_IOFLAGS_FAST_PATH);
5260                         ioc->put_smid_fast_path(ioc, smid, handle);
5261                 } else
5262                         ioc->put_smid_scsi_io(ioc, smid,
5263                             le16_to_cpu(mpi_request->DevHandle));
5264         } else
5265                 ioc->put_smid_default(ioc, smid);
5266         return 0;
5267
5268  out:
5269         return SCSI_MLQUEUE_HOST_BUSY;
5270 }
5271
5272 /**
5273  * _scsih_normalize_sense - normalize descriptor and fixed format sense data
5274  * @sense_buffer: sense data returned by target
5275  * @data: normalized skey/asc/ascq
5276  */
5277 static void
5278 _scsih_normalize_sense(char *sense_buffer, struct sense_info *data)
5279 {
5280         if ((sense_buffer[0] & 0x7F) >= 0x72) {
5281                 /* descriptor format */
5282                 data->skey = sense_buffer[1] & 0x0F;
5283                 data->asc = sense_buffer[2];
5284                 data->ascq = sense_buffer[3];
5285         } else {
5286                 /* fixed format */
5287                 data->skey = sense_buffer[2] & 0x0F;
5288                 data->asc = sense_buffer[12];
5289                 data->ascq = sense_buffer[13];
5290         }
5291 }
5292
5293 /**
5294  * _scsih_scsi_ioc_info - translated non-succesfull SCSI_IO request
5295  * @ioc: per adapter object
5296  * @scmd: pointer to scsi command object
5297  * @mpi_reply: reply mf payload returned from firmware
5298  * @smid: ?
5299  *
5300  * scsi_status - SCSI Status code returned from target device
5301  * scsi_state - state info associated with SCSI_IO determined by ioc
5302  * ioc_status - ioc supplied status info
5303  */
5304 static void
5305 _scsih_scsi_ioc_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,
5306         Mpi2SCSIIOReply_t *mpi_reply, u16 smid)
5307 {
5308         u32 response_info;
5309         u8 *response_bytes;
5310         u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
5311             MPI2_IOCSTATUS_MASK;
5312         u8 scsi_state = mpi_reply->SCSIState;
5313         u8 scsi_status = mpi_reply->SCSIStatus;
5314         char *desc_ioc_state = NULL;
5315         char *desc_scsi_status = NULL;
5316         char *desc_scsi_state = ioc->tmp_string;
5317         u32 log_info = le32_to_cpu(mpi_reply->IOCLogInfo);
5318         struct _sas_device *sas_device = NULL;
5319         struct _pcie_device *pcie_device = NULL;
5320         struct scsi_target *starget = scmd->device->sdev_target;
5321         struct MPT3SAS_TARGET *priv_target = starget->hostdata;
5322         char *device_str = NULL;
5323
5324         if (!priv_target)
5325                 return;
5326         if (ioc->hide_ir_msg)
5327                 device_str = "WarpDrive";
5328         else
5329                 device_str = "volume";
5330
5331         if (log_info == 0x31170000)
5332                 return;
5333
5334         switch (ioc_status) {
5335         case MPI2_IOCSTATUS_SUCCESS:
5336                 desc_ioc_state = "success";
5337                 break;
5338         case MPI2_IOCSTATUS_INVALID_FUNCTION:
5339                 desc_ioc_state = "invalid function";
5340                 break;
5341         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
5342                 desc_ioc_state = "scsi recovered error";
5343                 break;
5344         case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
5345                 desc_ioc_state = "scsi invalid dev handle";
5346                 break;
5347         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5348                 desc_ioc_state = "scsi device not there";
5349                 break;
5350         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5351                 desc_ioc_state = "scsi data overrun";
5352                 break;
5353         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
5354                 desc_ioc_state = "scsi data underrun";
5355                 break;
5356         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5357                 desc_ioc_state = "scsi io data error";
5358                 break;
5359         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5360                 desc_ioc_state = "scsi protocol error";
5361                 break;
5362         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5363                 desc_ioc_state = "scsi task terminated";
5364                 break;
5365         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
5366                 desc_ioc_state = "scsi residual mismatch";
5367                 break;
5368         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
5369                 desc_ioc_state = "scsi task mgmt failed";
5370                 break;
5371         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5372                 desc_ioc_state = "scsi ioc terminated";
5373                 break;
5374         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5375                 desc_ioc_state = "scsi ext terminated";
5376                 break;
5377         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5378                 desc_ioc_state = "eedp guard error";
5379                 break;
5380         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5381                 desc_ioc_state = "eedp ref tag error";
5382                 break;
5383         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5384                 desc_ioc_state = "eedp app tag error";
5385                 break;
5386         case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
5387                 desc_ioc_state = "insufficient power";
5388                 break;
5389         default:
5390                 desc_ioc_state = "unknown";
5391                 break;
5392         }
5393
5394         switch (scsi_status) {
5395         case MPI2_SCSI_STATUS_GOOD:
5396                 desc_scsi_status = "good";
5397                 break;
5398         case MPI2_SCSI_STATUS_CHECK_CONDITION:
5399                 desc_scsi_status = "check condition";
5400                 break;
5401         case MPI2_SCSI_STATUS_CONDITION_MET:
5402                 desc_scsi_status = "condition met";
5403                 break;
5404         case MPI2_SCSI_STATUS_BUSY:
5405                 desc_scsi_status = "busy";
5406                 break;
5407         case MPI2_SCSI_STATUS_INTERMEDIATE:
5408                 desc_scsi_status = "intermediate";
5409                 break;
5410         case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
5411                 desc_scsi_status = "intermediate condmet";
5412                 break;
5413         case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
5414                 desc_scsi_status = "reservation conflict";
5415                 break;
5416         case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
5417                 desc_scsi_status = "command terminated";
5418                 break;
5419         case MPI2_SCSI_STATUS_TASK_SET_FULL:
5420                 desc_scsi_status = "task set full";
5421                 break;
5422         case MPI2_SCSI_STATUS_ACA_ACTIVE:
5423                 desc_scsi_status = "aca active";
5424                 break;
5425         case MPI2_SCSI_STATUS_TASK_ABORTED:
5426                 desc_scsi_status = "task aborted";
5427                 break;
5428         default:
5429                 desc_scsi_status = "unknown";
5430                 break;
5431         }
5432
5433         desc_scsi_state[0] = '\0';
5434         if (!scsi_state)
5435                 desc_scsi_state = " ";
5436         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
5437                 strcat(desc_scsi_state, "response info ");
5438         if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5439                 strcat(desc_scsi_state, "state terminated ");
5440         if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
5441                 strcat(desc_scsi_state, "no status ");
5442         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
5443                 strcat(desc_scsi_state, "autosense failed ");
5444         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
5445                 strcat(desc_scsi_state, "autosense valid ");
5446
5447         scsi_print_command(scmd);
5448
5449         if (priv_target->flags & MPT_TARGET_FLAGS_VOLUME) {
5450                 ioc_warn(ioc, "\t%s wwid(0x%016llx)\n",
5451                          device_str, (u64)priv_target->sas_address);
5452         } else if (priv_target->flags & MPT_TARGET_FLAGS_PCIE_DEVICE) {
5453                 pcie_device = mpt3sas_get_pdev_from_target(ioc, priv_target);
5454                 if (pcie_device) {
5455                         ioc_info(ioc, "\twwid(0x%016llx), port(%d)\n",
5456                                  (u64)pcie_device->wwid, pcie_device->port_num);
5457                         if (pcie_device->enclosure_handle != 0)
5458                                 ioc_info(ioc, "\tenclosure logical id(0x%016llx), slot(%d)\n",
5459                                          (u64)pcie_device->enclosure_logical_id,
5460                                          pcie_device->slot);
5461                         if (pcie_device->connector_name[0])
5462                                 ioc_info(ioc, "\tenclosure level(0x%04x), connector name( %s)\n",
5463                                          pcie_device->enclosure_level,
5464                                          pcie_device->connector_name);
5465                         pcie_device_put(pcie_device);
5466                 }
5467         } else {
5468                 sas_device = mpt3sas_get_sdev_from_target(ioc, priv_target);
5469                 if (sas_device) {
5470                         ioc_warn(ioc, "\tsas_address(0x%016llx), phy(%d)\n",
5471                                  (u64)sas_device->sas_address, sas_device->phy);
5472
5473                         _scsih_display_enclosure_chassis_info(ioc, sas_device,
5474                             NULL, NULL);
5475
5476                         sas_device_put(sas_device);
5477                 }
5478         }
5479
5480         ioc_warn(ioc, "\thandle(0x%04x), ioc_status(%s)(0x%04x), smid(%d)\n",
5481                  le16_to_cpu(mpi_reply->DevHandle),
5482                  desc_ioc_state, ioc_status, smid);
5483         ioc_warn(ioc, "\trequest_len(%d), underflow(%d), resid(%d)\n",
5484                  scsi_bufflen(scmd), scmd->underflow, scsi_get_resid(scmd));
5485         ioc_warn(ioc, "\ttag(%d), transfer_count(%d), sc->result(0x%08x)\n",
5486                  le16_to_cpu(mpi_reply->TaskTag),
5487                  le32_to_cpu(mpi_reply->TransferCount), scmd->result);
5488         ioc_warn(ioc, "\tscsi_status(%s)(0x%02x), scsi_state(%s)(0x%02x)\n",
5489                  desc_scsi_status, scsi_status, desc_scsi_state, scsi_state);
5490
5491         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5492                 struct sense_info data;
5493                 _scsih_normalize_sense(scmd->sense_buffer, &data);
5494                 ioc_warn(ioc, "\t[sense_key,asc,ascq]: [0x%02x,0x%02x,0x%02x], count(%d)\n",
5495                          data.skey, data.asc, data.ascq,
5496                          le32_to_cpu(mpi_reply->SenseCount));
5497         }
5498         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
5499                 response_info = le32_to_cpu(mpi_reply->ResponseInfo);
5500                 response_bytes = (u8 *)&response_info;
5501                 _scsih_response_code(ioc, response_bytes[0]);
5502         }
5503 }
5504
5505 /**
5506  * _scsih_turn_on_pfa_led - illuminate PFA LED
5507  * @ioc: per adapter object
5508  * @handle: device handle
5509  * Context: process
5510  */
5511 static void
5512 _scsih_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5513 {
5514         Mpi2SepReply_t mpi_reply;
5515         Mpi2SepRequest_t mpi_request;
5516         struct _sas_device *sas_device;
5517
5518         sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
5519         if (!sas_device)
5520                 return;
5521
5522         memset(&mpi_request, 0, sizeof(Mpi2SepRequest_t));
5523         mpi_request.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
5524         mpi_request.Action = MPI2_SEP_REQ_ACTION_WRITE_STATUS;
5525         mpi_request.SlotStatus =
5526             cpu_to_le32(MPI2_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT);
5527         mpi_request.DevHandle = cpu_to_le16(handle);
5528         mpi_request.Flags = MPI2_SEP_REQ_FLAGS_DEVHANDLE_ADDRESS;
5529         if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,
5530             &mpi_request)) != 0) {
5531                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5532                         __FILE__, __LINE__, __func__);
5533                 goto out;
5534         }
5535         sas_device->pfa_led_on = 1;
5536
5537         if (mpi_reply.IOCStatus || mpi_reply.IOCLogInfo) {
5538                 dewtprintk(ioc,
5539                            ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",
5540                                     le16_to_cpu(mpi_reply.IOCStatus),
5541                                     le32_to_cpu(mpi_reply.IOCLogInfo)));
5542                 goto out;
5543         }
5544 out:
5545         sas_device_put(sas_device);
5546 }
5547
5548 /**
5549  * _scsih_turn_off_pfa_led - turn off Fault LED
5550  * @ioc: per adapter object
5551  * @sas_device: sas device whose PFA LED has to turned off
5552  * Context: process
5553  */
5554 static void
5555 _scsih_turn_off_pfa_led(struct MPT3SAS_ADAPTER *ioc,
5556         struct _sas_device *sas_device)
5557 {
5558         Mpi2SepReply_t mpi_reply;
5559         Mpi2SepRequest_t mpi_request;
5560
5561         memset(&mpi_request, 0, sizeof(Mpi2SepRequest_t));
5562         mpi_request.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
5563         mpi_request.Action = MPI2_SEP_REQ_ACTION_WRITE_STATUS;
5564         mpi_request.SlotStatus = 0;
5565         mpi_request.Slot = cpu_to_le16(sas_device->slot);
5566         mpi_request.DevHandle = 0;
5567         mpi_request.EnclosureHandle = cpu_to_le16(sas_device->enclosure_handle);
5568         mpi_request.Flags = MPI2_SEP_REQ_FLAGS_ENCLOSURE_SLOT_ADDRESS;
5569         if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,
5570                 &mpi_request)) != 0) {
5571                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5572                         __FILE__, __LINE__, __func__);
5573                 return;
5574         }
5575
5576         if (mpi_reply.IOCStatus || mpi_reply.IOCLogInfo) {
5577                 dewtprintk(ioc,
5578                            ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",
5579                                     le16_to_cpu(mpi_reply.IOCStatus),
5580                                     le32_to_cpu(mpi_reply.IOCLogInfo)));
5581                 return;
5582         }
5583 }
5584
5585 /**
5586  * _scsih_send_event_to_turn_on_pfa_led - fire delayed event
5587  * @ioc: per adapter object
5588  * @handle: device handle
5589  * Context: interrupt.
5590  */
5591 static void
5592 _scsih_send_event_to_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5593 {
5594         struct fw_event_work *fw_event;
5595
5596         fw_event = alloc_fw_event_work(0);
5597         if (!fw_event)
5598                 return;
5599         fw_event->event = MPT3SAS_TURN_ON_PFA_LED;
5600         fw_event->device_handle = handle;
5601         fw_event->ioc = ioc;
5602         _scsih_fw_event_add(ioc, fw_event);
5603         fw_event_work_put(fw_event);
5604 }
5605
5606 /**
5607  * _scsih_smart_predicted_fault - process smart errors
5608  * @ioc: per adapter object
5609  * @handle: device handle
5610  * Context: interrupt.
5611  */
5612 static void
5613 _scsih_smart_predicted_fault(struct MPT3SAS_ADAPTER *ioc, u16 handle)
5614 {
5615         struct scsi_target *starget;
5616         struct MPT3SAS_TARGET *sas_target_priv_data;
5617         Mpi2EventNotificationReply_t *event_reply;
5618         Mpi2EventDataSasDeviceStatusChange_t *event_data;
5619         struct _sas_device *sas_device;
5620         ssize_t sz;
5621         unsigned long flags;
5622
5623         /* only handle non-raid devices */
5624         spin_lock_irqsave(&ioc->sas_device_lock, flags);
5625         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
5626         if (!sas_device)
5627                 goto out_unlock;
5628
5629         starget = sas_device->starget;
5630         sas_target_priv_data = starget->hostdata;
5631
5632         if ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_RAID_COMPONENT) ||
5633            ((sas_target_priv_data->flags & MPT_TARGET_FLAGS_VOLUME)))
5634                 goto out_unlock;
5635
5636         _scsih_display_enclosure_chassis_info(NULL, sas_device, NULL, starget);
5637
5638         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
5639
5640         if (ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM)
5641                 _scsih_send_event_to_turn_on_pfa_led(ioc, handle);
5642
5643         /* insert into event log */
5644         sz = offsetof(Mpi2EventNotificationReply_t, EventData) +
5645              sizeof(Mpi2EventDataSasDeviceStatusChange_t);
5646         event_reply = kzalloc(sz, GFP_ATOMIC);
5647         if (!event_reply) {
5648                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5649                         __FILE__, __LINE__, __func__);
5650                 goto out;
5651         }
5652
5653         event_reply->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
5654         event_reply->Event =
5655             cpu_to_le16(MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
5656         event_reply->MsgLength = sz/4;
5657         event_reply->EventDataLength =
5658             cpu_to_le16(sizeof(Mpi2EventDataSasDeviceStatusChange_t)/4);
5659         event_data = (Mpi2EventDataSasDeviceStatusChange_t *)
5660             event_reply->EventData;
5661         event_data->ReasonCode = MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA;
5662         event_data->ASC = 0x5D;
5663         event_data->DevHandle = cpu_to_le16(handle);
5664         event_data->SASAddress = cpu_to_le64(sas_target_priv_data->sas_address);
5665         mpt3sas_ctl_add_to_event_log(ioc, event_reply);
5666         kfree(event_reply);
5667 out:
5668         if (sas_device)
5669                 sas_device_put(sas_device);
5670         return;
5671
5672 out_unlock:
5673         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
5674         goto out;
5675 }
5676
5677 /**
5678  * _scsih_io_done - scsi request callback
5679  * @ioc: per adapter object
5680  * @smid: system request message index
5681  * @msix_index: MSIX table index supplied by the OS
5682  * @reply: reply message frame(lower 32bit addr)
5683  *
5684  * Callback handler when using _scsih_qcmd.
5685  *
5686  * Return: 1 meaning mf should be freed from _base_interrupt
5687  *         0 means the mf is freed from this function.
5688  */
5689 static u8
5690 _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
5691 {
5692         Mpi25SCSIIORequest_t *mpi_request;
5693         Mpi2SCSIIOReply_t *mpi_reply;
5694         struct scsi_cmnd *scmd;
5695         struct scsiio_tracker *st;
5696         u16 ioc_status;
5697         u32 xfer_cnt;
5698         u8 scsi_state;
5699         u8 scsi_status;
5700         u32 log_info;
5701         struct MPT3SAS_DEVICE *sas_device_priv_data;
5702         u32 response_code = 0;
5703
5704         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
5705
5706         scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
5707         if (scmd == NULL)
5708                 return 1;
5709
5710         _scsih_set_satl_pending(scmd, false);
5711
5712         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
5713
5714         if (mpi_reply == NULL) {
5715                 scmd->result = DID_OK << 16;
5716                 goto out;
5717         }
5718
5719         sas_device_priv_data = scmd->device->hostdata;
5720         if (!sas_device_priv_data || !sas_device_priv_data->sas_target ||
5721              sas_device_priv_data->sas_target->deleted) {
5722                 scmd->result = DID_NO_CONNECT << 16;
5723                 goto out;
5724         }
5725         ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
5726
5727         /*
5728          * WARPDRIVE: If direct_io is set then it is directIO,
5729          * the failed direct I/O should be redirected to volume
5730          */
5731         st = scsi_cmd_priv(scmd);
5732         if (st->direct_io &&
5733              ((ioc_status & MPI2_IOCSTATUS_MASK)
5734               != MPI2_IOCSTATUS_SCSI_TASK_TERMINATED)) {
5735                 st->direct_io = 0;
5736                 st->scmd = scmd;
5737                 memcpy(mpi_request->CDB.CDB32, scmd->cmnd, scmd->cmd_len);
5738                 mpi_request->DevHandle =
5739                     cpu_to_le16(sas_device_priv_data->sas_target->handle);
5740                 ioc->put_smid_scsi_io(ioc, smid,
5741                     sas_device_priv_data->sas_target->handle);
5742                 return 0;
5743         }
5744         /* turning off TLR */
5745         scsi_state = mpi_reply->SCSIState;
5746         if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
5747                 response_code =
5748                     le32_to_cpu(mpi_reply->ResponseInfo) & 0xFF;
5749         if (!sas_device_priv_data->tlr_snoop_check) {
5750                 sas_device_priv_data->tlr_snoop_check++;
5751                 if ((!ioc->is_warpdrive &&
5752                     !scsih_is_raid(&scmd->device->sdev_gendev) &&
5753                     !scsih_is_nvme(&scmd->device->sdev_gendev))
5754                     && sas_is_tlr_enabled(scmd->device) &&
5755                     response_code == MPI2_SCSITASKMGMT_RSP_INVALID_FRAME) {
5756                         sas_disable_tlr(scmd->device);
5757                         sdev_printk(KERN_INFO, scmd->device, "TLR disabled\n");
5758                 }
5759         }
5760
5761         xfer_cnt = le32_to_cpu(mpi_reply->TransferCount);
5762         scsi_set_resid(scmd, scsi_bufflen(scmd) - xfer_cnt);
5763         if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
5764                 log_info =  le32_to_cpu(mpi_reply->IOCLogInfo);
5765         else
5766                 log_info = 0;
5767         ioc_status &= MPI2_IOCSTATUS_MASK;
5768         scsi_status = mpi_reply->SCSIStatus;
5769
5770         if (ioc_status == MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN && xfer_cnt == 0 &&
5771             (scsi_status == MPI2_SCSI_STATUS_BUSY ||
5772              scsi_status == MPI2_SCSI_STATUS_RESERVATION_CONFLICT ||
5773              scsi_status == MPI2_SCSI_STATUS_TASK_SET_FULL)) {
5774                 ioc_status = MPI2_IOCSTATUS_SUCCESS;
5775         }
5776
5777         if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
5778                 struct sense_info data;
5779                 const void *sense_data = mpt3sas_base_get_sense_buffer(ioc,
5780                     smid);
5781                 u32 sz = min_t(u32, SCSI_SENSE_BUFFERSIZE,
5782                     le32_to_cpu(mpi_reply->SenseCount));
5783                 memcpy(scmd->sense_buffer, sense_data, sz);
5784                 _scsih_normalize_sense(scmd->sense_buffer, &data);
5785                 /* failure prediction threshold exceeded */
5786                 if (data.asc == 0x5D)
5787                         _scsih_smart_predicted_fault(ioc,
5788                             le16_to_cpu(mpi_reply->DevHandle));
5789                 mpt3sas_trigger_scsi(ioc, data.skey, data.asc, data.ascq);
5790
5791                 if ((ioc->logging_level & MPT_DEBUG_REPLY) &&
5792                      ((scmd->sense_buffer[2] == UNIT_ATTENTION) ||
5793                      (scmd->sense_buffer[2] == MEDIUM_ERROR) ||
5794                      (scmd->sense_buffer[2] == HARDWARE_ERROR)))
5795                         _scsih_scsi_ioc_info(ioc, scmd, mpi_reply, smid);
5796         }
5797         switch (ioc_status) {
5798         case MPI2_IOCSTATUS_BUSY:
5799         case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
5800                 scmd->result = SAM_STAT_BUSY;
5801                 break;
5802
5803         case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
5804                 scmd->result = DID_NO_CONNECT << 16;
5805                 break;
5806
5807         case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
5808                 if (sas_device_priv_data->block) {
5809                         scmd->result = DID_TRANSPORT_DISRUPTED << 16;
5810                         goto out;
5811                 }
5812                 if (log_info == 0x31110630) {
5813                         if (scmd->retries > 2) {
5814                                 scmd->result = DID_NO_CONNECT << 16;
5815                                 scsi_device_set_state(scmd->device,
5816                                     SDEV_OFFLINE);
5817                         } else {
5818                                 scmd->result = DID_SOFT_ERROR << 16;
5819                                 scmd->device->expecting_cc_ua = 1;
5820                         }
5821                         break;
5822                 } else if (log_info == VIRTUAL_IO_FAILED_RETRY) {
5823                         scmd->result = DID_RESET << 16;
5824                         break;
5825                 } else if ((scmd->device->channel == RAID_CHANNEL) &&
5826                    (scsi_state == (MPI2_SCSI_STATE_TERMINATED |
5827                    MPI2_SCSI_STATE_NO_SCSI_STATUS))) {
5828                         scmd->result = DID_RESET << 16;
5829                         break;
5830                 }
5831                 scmd->result = DID_SOFT_ERROR << 16;
5832                 break;
5833         case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
5834         case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
5835                 scmd->result = DID_RESET << 16;
5836                 break;
5837
5838         case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
5839                 if ((xfer_cnt == 0) || (scmd->underflow > xfer_cnt))
5840                         scmd->result = DID_SOFT_ERROR << 16;
5841                 else
5842                         scmd->result = (DID_OK << 16) | scsi_status;
5843                 break;
5844
5845         case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
5846                 scmd->result = (DID_OK << 16) | scsi_status;
5847
5848                 if ((scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID))
5849                         break;
5850
5851                 if (xfer_cnt < scmd->underflow) {
5852                         if (scsi_status == SAM_STAT_BUSY)
5853                                 scmd->result = SAM_STAT_BUSY;
5854                         else
5855                                 scmd->result = DID_SOFT_ERROR << 16;
5856                 } else if (scsi_state & (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
5857                      MPI2_SCSI_STATE_NO_SCSI_STATUS))
5858                         scmd->result = DID_SOFT_ERROR << 16;
5859                 else if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5860                         scmd->result = DID_RESET << 16;
5861                 else if (!xfer_cnt && scmd->cmnd[0] == REPORT_LUNS) {
5862                         mpi_reply->SCSIState = MPI2_SCSI_STATE_AUTOSENSE_VALID;
5863                         mpi_reply->SCSIStatus = SAM_STAT_CHECK_CONDITION;
5864                         scsi_build_sense(scmd, 0, ILLEGAL_REQUEST,
5865                                          0x20, 0);
5866                 }
5867                 break;
5868
5869         case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
5870                 scsi_set_resid(scmd, 0);
5871                 fallthrough;
5872         case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
5873         case MPI2_IOCSTATUS_SUCCESS:
5874                 scmd->result = (DID_OK << 16) | scsi_status;
5875                 if (response_code ==
5876                     MPI2_SCSITASKMGMT_RSP_INVALID_FRAME ||
5877                     (scsi_state & (MPI2_SCSI_STATE_AUTOSENSE_FAILED |
5878                      MPI2_SCSI_STATE_NO_SCSI_STATUS)))
5879                         scmd->result = DID_SOFT_ERROR << 16;
5880                 else if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
5881                         scmd->result = DID_RESET << 16;
5882                 break;
5883
5884         case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
5885         case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
5886         case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
5887                 _scsih_eedp_error_handling(scmd, ioc_status);
5888                 break;
5889
5890         case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
5891         case MPI2_IOCSTATUS_INVALID_FUNCTION:
5892         case MPI2_IOCSTATUS_INVALID_SGL:
5893         case MPI2_IOCSTATUS_INTERNAL_ERROR:
5894         case MPI2_IOCSTATUS_INVALID_FIELD:
5895         case MPI2_IOCSTATUS_INVALID_STATE:
5896         case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
5897         case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
5898         case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
5899         default:
5900                 scmd->result = DID_SOFT_ERROR << 16;
5901                 break;
5902
5903         }
5904
5905         if (scmd->result && (ioc->logging_level & MPT_DEBUG_REPLY))
5906                 _scsih_scsi_ioc_info(ioc , scmd, mpi_reply, smid);
5907
5908  out:
5909
5910         scsi_dma_unmap(scmd);
5911         mpt3sas_base_free_smid(ioc, smid);
5912         scmd->scsi_done(scmd);
5913         return 0;
5914 }
5915
5916 /**
5917  * _scsih_update_vphys_after_reset - update the Port's
5918  *                      vphys_list after reset
5919  * @ioc: per adapter object
5920  *
5921  * Returns nothing.
5922  */
5923 static void
5924 _scsih_update_vphys_after_reset(struct MPT3SAS_ADAPTER *ioc)
5925 {
5926         u16 sz, ioc_status;
5927         int i;
5928         Mpi2ConfigReply_t mpi_reply;
5929         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
5930         u16 attached_handle;
5931         u64 attached_sas_addr;
5932         u8 found = 0, port_id;
5933         Mpi2SasPhyPage0_t phy_pg0;
5934         struct hba_port *port, *port_next, *mport;
5935         struct virtual_phy *vphy, *vphy_next;
5936         struct _sas_device *sas_device;
5937
5938         /*
5939          * Mark all the vphys objects as dirty.
5940          */
5941         list_for_each_entry_safe(port, port_next,
5942             &ioc->port_table_list, list) {
5943                 if (!port->vphys_mask)
5944                         continue;
5945                 list_for_each_entry_safe(vphy, vphy_next,
5946                     &port->vphys_list, list) {
5947                         vphy->flags |= MPT_VPHY_FLAG_DIRTY_PHY;
5948                 }
5949         }
5950
5951         /*
5952          * Read SASIOUnitPage0 to get each HBA Phy's data.
5953          */
5954         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) +
5955             (ioc->sas_hba.num_phys * sizeof(Mpi2SasIOUnit0PhyData_t));
5956         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
5957         if (!sas_iounit_pg0) {
5958                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
5959                     __FILE__, __LINE__, __func__);
5960                 return;
5961         }
5962         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
5963             sas_iounit_pg0, sz)) != 0)
5964                 goto out;
5965         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
5966         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5967                 goto out;
5968         /*
5969          * Loop over each HBA Phy.
5970          */
5971         for (i = 0; i < ioc->sas_hba.num_phys; i++) {
5972                 /*
5973                  * Check whether Phy's Negotiation Link Rate is > 1.5G or not.
5974                  */
5975                 if ((sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4) <
5976                     MPI2_SAS_NEG_LINK_RATE_1_5)
5977                         continue;
5978                 /*
5979                  * Check whether Phy is connected to SEP device or not,
5980                  * if it is SEP device then read the Phy's SASPHYPage0 data to
5981                  * determine whether Phy is a virtual Phy or not. if it is
5982                  * virtual phy then it is conformed that the attached remote
5983                  * device is a HBA's vSES device.
5984                  */
5985                 if (!(le32_to_cpu(
5986                     sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
5987                     MPI2_SAS_DEVICE_INFO_SEP))
5988                         continue;
5989
5990                 if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,
5991                     i))) {
5992                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
5993                             __FILE__, __LINE__, __func__);
5994                         continue;
5995                 }
5996
5997                 if (!(le32_to_cpu(phy_pg0.PhyInfo) &
5998                     MPI2_SAS_PHYINFO_VIRTUAL_PHY))
5999                         continue;
6000                 /*
6001                  * Get the vSES device's SAS Address.
6002                  */
6003                 attached_handle = le16_to_cpu(
6004                     sas_iounit_pg0->PhyData[i].AttachedDevHandle);
6005                 if (_scsih_get_sas_address(ioc, attached_handle,
6006                     &attached_sas_addr) != 0) {
6007                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6008                             __FILE__, __LINE__, __func__);
6009                         continue;
6010                 }
6011
6012                 found = 0;
6013                 port = port_next = NULL;
6014                 /*
6015                  * Loop over each virtual_phy object from
6016                  * each port's vphys_list.
6017                  */
6018                 list_for_each_entry_safe(port,
6019                     port_next, &ioc->port_table_list, list) {
6020                         if (!port->vphys_mask)
6021                                 continue;
6022                         list_for_each_entry_safe(vphy, vphy_next,
6023                             &port->vphys_list, list) {
6024                                 /*
6025                                  * Continue with next virtual_phy object
6026                                  * if the object is not marked as dirty.
6027                                  */
6028                                 if (!(vphy->flags & MPT_VPHY_FLAG_DIRTY_PHY))
6029                                         continue;
6030
6031                                 /*
6032                                  * Continue with next virtual_phy object
6033                                  * if the object's SAS Address is not equals
6034                                  * to current Phy's vSES device SAS Address.
6035                                  */
6036                                 if (vphy->sas_address != attached_sas_addr)
6037                                         continue;
6038                                 /*
6039                                  * Enable current Phy number bit in object's
6040                                  * phy_mask field.
6041                                  */
6042                                 if (!(vphy->phy_mask & (1 << i)))
6043                                         vphy->phy_mask = (1 << i);
6044                                 /*
6045                                  * Get hba_port object from hba_port table
6046                                  * corresponding to current phy's Port ID.
6047                                  * if there is no hba_port object corresponding
6048                                  * to Phy's Port ID then create a new hba_port
6049                                  * object & add to hba_port table.
6050                                  */
6051                                 port_id = sas_iounit_pg0->PhyData[i].Port;
6052                                 mport = mpt3sas_get_port_by_id(ioc, port_id, 1);
6053                                 if (!mport) {
6054                                         mport = kzalloc(
6055                                             sizeof(struct hba_port), GFP_KERNEL);
6056                                         if (!mport)
6057                                                 break;
6058                                         mport->port_id = port_id;
6059                                         ioc_info(ioc,
6060                                             "%s: hba_port entry: %p, port: %d is added to hba_port list\n",
6061                                             __func__, mport, mport->port_id);
6062                                         list_add_tail(&mport->list,
6063                                                 &ioc->port_table_list);
6064                                 }
6065                                 /*
6066                                  * If mport & port pointers are not pointing to
6067                                  * same hba_port object then it means that vSES
6068                                  * device's Port ID got changed after reset and
6069                                  * hence move current virtual_phy object from
6070                                  * port's vphys_list to mport's vphys_list.
6071                                  */
6072                                 if (port != mport) {
6073                                         if (!mport->vphys_mask)
6074                                                 INIT_LIST_HEAD(
6075                                                     &mport->vphys_list);
6076                                         mport->vphys_mask |= (1 << i);
6077                                         port->vphys_mask &= ~(1 << i);
6078                                         list_move(&vphy->list,
6079                                             &mport->vphys_list);
6080                                         sas_device = mpt3sas_get_sdev_by_addr(
6081                                             ioc, attached_sas_addr, port);
6082                                         if (sas_device)
6083                                                 sas_device->port = mport;
6084                                 }
6085                                 /*
6086                                  * Earlier while updating the hba_port table,
6087                                  * it is determined that there is no other
6088                                  * direct attached device with mport's Port ID,
6089                                  * Hence mport was marked as dirty. Only vSES
6090                                  * device has this Port ID, so unmark the mport
6091                                  * as dirt.
6092                                  */
6093                                 if (mport->flags & HBA_PORT_FLAG_DIRTY_PORT) {
6094                                         mport->sas_address = 0;
6095                                         mport->phy_mask = 0;
6096                                         mport->flags &=
6097                                             ~HBA_PORT_FLAG_DIRTY_PORT;
6098                                 }
6099                                 /*
6100                                  * Unmark current virtual_phy object as dirty.
6101                                  */
6102                                 vphy->flags &= ~MPT_VPHY_FLAG_DIRTY_PHY;
6103                                 found = 1;
6104                                 break;
6105                         }
6106                         if (found)
6107                                 break;
6108                 }
6109         }
6110 out:
6111         kfree(sas_iounit_pg0);
6112 }
6113
6114 /**
6115  * _scsih_get_port_table_after_reset - Construct temporary port table
6116  * @ioc: per adapter object
6117  * @port_table: address where port table needs to be constructed
6118  *
6119  * return number of HBA port entries available after reset.
6120  */
6121 static int
6122 _scsih_get_port_table_after_reset(struct MPT3SAS_ADAPTER *ioc,
6123         struct hba_port *port_table)
6124 {
6125         u16 sz, ioc_status;
6126         int i, j;
6127         Mpi2ConfigReply_t mpi_reply;
6128         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6129         u16 attached_handle;
6130         u64 attached_sas_addr;
6131         u8 found = 0, port_count = 0, port_id;
6132
6133         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
6134             * sizeof(Mpi2SasIOUnit0PhyData_t));
6135         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
6136         if (!sas_iounit_pg0) {
6137                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6138                     __FILE__, __LINE__, __func__);
6139                 return port_count;
6140         }
6141
6142         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
6143             sas_iounit_pg0, sz)) != 0)
6144                 goto out;
6145         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6146         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6147                 goto out;
6148         for (i = 0; i < ioc->sas_hba.num_phys; i++) {
6149                 found = 0;
6150                 if ((sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4) <
6151                     MPI2_SAS_NEG_LINK_RATE_1_5)
6152                         continue;
6153                 attached_handle =
6154                     le16_to_cpu(sas_iounit_pg0->PhyData[i].AttachedDevHandle);
6155                 if (_scsih_get_sas_address(
6156                     ioc, attached_handle, &attached_sas_addr) != 0) {
6157                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6158                             __FILE__, __LINE__, __func__);
6159                         continue;
6160                 }
6161
6162                 for (j = 0; j < port_count; j++) {
6163                         port_id = sas_iounit_pg0->PhyData[i].Port;
6164                         if (port_table[j].port_id == port_id &&
6165                             port_table[j].sas_address == attached_sas_addr) {
6166                                 port_table[j].phy_mask |= (1 << i);
6167                                 found = 1;
6168                                 break;
6169                         }
6170                 }
6171
6172                 if (found)
6173                         continue;
6174
6175                 port_id = sas_iounit_pg0->PhyData[i].Port;
6176                 port_table[port_count].port_id = port_id;
6177                 port_table[port_count].phy_mask = (1 << i);
6178                 port_table[port_count].sas_address = attached_sas_addr;
6179                 port_count++;
6180         }
6181 out:
6182         kfree(sas_iounit_pg0);
6183         return port_count;
6184 }
6185
6186 enum hba_port_matched_codes {
6187         NOT_MATCHED = 0,
6188         MATCHED_WITH_ADDR_AND_PHYMASK,
6189         MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT,
6190         MATCHED_WITH_ADDR_AND_SUBPHYMASK,
6191         MATCHED_WITH_ADDR,
6192 };
6193
6194 /**
6195  * _scsih_look_and_get_matched_port_entry - Get matched hba port entry
6196  *                                      from HBA port table
6197  * @ioc: per adapter object
6198  * @port_entry: hba port entry from temporary port table which needs to be
6199  *              searched for matched entry in the HBA port table
6200  * @matched_port_entry: save matched hba port entry here
6201  * @count: count of matched entries
6202  *
6203  * return type of matched entry found.
6204  */
6205 static enum hba_port_matched_codes
6206 _scsih_look_and_get_matched_port_entry(struct MPT3SAS_ADAPTER *ioc,
6207         struct hba_port *port_entry,
6208         struct hba_port **matched_port_entry, int *count)
6209 {
6210         struct hba_port *port_table_entry, *matched_port = NULL;
6211         enum hba_port_matched_codes matched_code = NOT_MATCHED;
6212         int lcount = 0;
6213         *matched_port_entry = NULL;
6214
6215         list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {
6216                 if (!(port_table_entry->flags & HBA_PORT_FLAG_DIRTY_PORT))
6217                         continue;
6218
6219                 if ((port_table_entry->sas_address == port_entry->sas_address)
6220                     && (port_table_entry->phy_mask == port_entry->phy_mask)) {
6221                         matched_code = MATCHED_WITH_ADDR_AND_PHYMASK;
6222                         matched_port = port_table_entry;
6223                         break;
6224                 }
6225
6226                 if ((port_table_entry->sas_address == port_entry->sas_address)
6227                     && (port_table_entry->phy_mask & port_entry->phy_mask)
6228                     && (port_table_entry->port_id == port_entry->port_id)) {
6229                         matched_code = MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT;
6230                         matched_port = port_table_entry;
6231                         continue;
6232                 }
6233
6234                 if ((port_table_entry->sas_address == port_entry->sas_address)
6235                     && (port_table_entry->phy_mask & port_entry->phy_mask)) {
6236                         if (matched_code ==
6237                             MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT)
6238                                 continue;
6239                         matched_code = MATCHED_WITH_ADDR_AND_SUBPHYMASK;
6240                         matched_port = port_table_entry;
6241                         continue;
6242                 }
6243
6244                 if (port_table_entry->sas_address == port_entry->sas_address) {
6245                         if (matched_code ==
6246                             MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT)
6247                                 continue;
6248                         if (matched_code == MATCHED_WITH_ADDR_AND_SUBPHYMASK)
6249                                 continue;
6250                         matched_code = MATCHED_WITH_ADDR;
6251                         matched_port = port_table_entry;
6252                         lcount++;
6253                 }
6254         }
6255
6256         *matched_port_entry = matched_port;
6257         if (matched_code ==  MATCHED_WITH_ADDR)
6258                 *count = lcount;
6259         return matched_code;
6260 }
6261
6262 /**
6263  * _scsih_del_phy_part_of_anther_port - remove phy if it
6264  *                              is a part of anther port
6265  *@ioc: per adapter object
6266  *@port_table: port table after reset
6267  *@index: hba port entry index
6268  *@port_count: number of ports available after host reset
6269  *@offset: HBA phy bit offset
6270  *
6271  */
6272 static void
6273 _scsih_del_phy_part_of_anther_port(struct MPT3SAS_ADAPTER *ioc,
6274         struct hba_port *port_table,
6275         int index, u8 port_count, int offset)
6276 {
6277         struct _sas_node *sas_node = &ioc->sas_hba;
6278         u32 i, found = 0;
6279
6280         for (i = 0; i < port_count; i++) {
6281                 if (i == index)
6282                         continue;
6283
6284                 if (port_table[i].phy_mask & (1 << offset)) {
6285                         mpt3sas_transport_del_phy_from_an_existing_port(
6286                             ioc, sas_node, &sas_node->phy[offset]);
6287                         found = 1;
6288                         break;
6289                 }
6290         }
6291         if (!found)
6292                 port_table[index].phy_mask |= (1 << offset);
6293 }
6294
6295 /**
6296  * _scsih_add_or_del_phys_from_existing_port - add/remove phy to/from
6297  *                                              right port
6298  *@ioc: per adapter object
6299  *@hba_port_entry: hba port table entry
6300  *@port_table: temporary port table
6301  *@index: hba port entry index
6302  *@port_count: number of ports available after host reset
6303  *
6304  */
6305 static void
6306 _scsih_add_or_del_phys_from_existing_port(struct MPT3SAS_ADAPTER *ioc,
6307         struct hba_port *hba_port_entry, struct hba_port *port_table,
6308         int index, int port_count)
6309 {
6310         u32 phy_mask, offset = 0;
6311         struct _sas_node *sas_node = &ioc->sas_hba;
6312
6313         phy_mask = hba_port_entry->phy_mask ^ port_table[index].phy_mask;
6314
6315         for (offset = 0; offset < ioc->sas_hba.num_phys; offset++) {
6316                 if (phy_mask & (1 << offset)) {
6317                         if (!(port_table[index].phy_mask & (1 << offset))) {
6318                                 _scsih_del_phy_part_of_anther_port(
6319                                     ioc, port_table, index, port_count,
6320                                     offset);
6321                                 continue;
6322                         }
6323                         if (sas_node->phy[offset].phy_belongs_to_port)
6324                                 mpt3sas_transport_del_phy_from_an_existing_port(
6325                                     ioc, sas_node, &sas_node->phy[offset]);
6326                         mpt3sas_transport_add_phy_to_an_existing_port(
6327                             ioc, sas_node, &sas_node->phy[offset],
6328                             hba_port_entry->sas_address,
6329                             hba_port_entry);
6330                 }
6331         }
6332 }
6333
6334 /**
6335  * _scsih_del_dirty_vphy - delete virtual_phy objects marked as dirty.
6336  * @ioc: per adapter object
6337  *
6338  * Returns nothing.
6339  */
6340 static void
6341 _scsih_del_dirty_vphy(struct MPT3SAS_ADAPTER *ioc)
6342 {
6343         struct hba_port *port, *port_next;
6344         struct virtual_phy *vphy, *vphy_next;
6345
6346         list_for_each_entry_safe(port, port_next,
6347             &ioc->port_table_list, list) {
6348                 if (!port->vphys_mask)
6349                         continue;
6350                 list_for_each_entry_safe(vphy, vphy_next,
6351                     &port->vphys_list, list) {
6352                         if (vphy->flags & MPT_VPHY_FLAG_DIRTY_PHY) {
6353                                 drsprintk(ioc, ioc_info(ioc,
6354                                     "Deleting vphy %p entry from port id: %d\t, Phy_mask 0x%08x\n",
6355                                     vphy, port->port_id,
6356                                     vphy->phy_mask));
6357                                 port->vphys_mask &= ~vphy->phy_mask;
6358                                 list_del(&vphy->list);
6359                                 kfree(vphy);
6360                         }
6361                 }
6362                 if (!port->vphys_mask && !port->sas_address)
6363                         port->flags |= HBA_PORT_FLAG_DIRTY_PORT;
6364         }
6365 }
6366
6367 /**
6368  * _scsih_del_dirty_port_entries - delete dirty port entries from port list
6369  *                                      after host reset
6370  *@ioc: per adapter object
6371  *
6372  */
6373 static void
6374 _scsih_del_dirty_port_entries(struct MPT3SAS_ADAPTER *ioc)
6375 {
6376         struct hba_port *port, *port_next;
6377
6378         list_for_each_entry_safe(port, port_next,
6379             &ioc->port_table_list, list) {
6380                 if (!(port->flags & HBA_PORT_FLAG_DIRTY_PORT) ||
6381                     port->flags & HBA_PORT_FLAG_NEW_PORT)
6382                         continue;
6383
6384                 drsprintk(ioc, ioc_info(ioc,
6385                     "Deleting port table entry %p having Port: %d\t Phy_mask 0x%08x\n",
6386                     port, port->port_id, port->phy_mask));
6387                 list_del(&port->list);
6388                 kfree(port);
6389         }
6390 }
6391
6392 /**
6393  * _scsih_sas_port_refresh - Update HBA port table after host reset
6394  * @ioc: per adapter object
6395  */
6396 static void
6397 _scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)
6398 {
6399         u32 port_count = 0;
6400         struct hba_port *port_table;
6401         struct hba_port *port_table_entry;
6402         struct hba_port *port_entry = NULL;
6403         int i, j, count = 0, lcount = 0;
6404         int ret;
6405         u64 sas_addr;
6406
6407         drsprintk(ioc, ioc_info(ioc,
6408             "updating ports for sas_host(0x%016llx)\n",
6409             (unsigned long long)ioc->sas_hba.sas_address));
6410
6411         port_table = kcalloc(ioc->sas_hba.num_phys,
6412             sizeof(struct hba_port), GFP_KERNEL);
6413         if (!port_table)
6414                 return;
6415
6416         port_count = _scsih_get_port_table_after_reset(ioc, port_table);
6417         if (!port_count)
6418                 return;
6419
6420         drsprintk(ioc, ioc_info(ioc, "New Port table\n"));
6421         for (j = 0; j < port_count; j++)
6422                 drsprintk(ioc, ioc_info(ioc,
6423                     "Port: %d\t Phy_mask 0x%08x\t sas_addr(0x%016llx)\n",
6424                     port_table[j].port_id,
6425                     port_table[j].phy_mask, port_table[j].sas_address));
6426
6427         list_for_each_entry(port_table_entry, &ioc->port_table_list, list)
6428                 port_table_entry->flags |= HBA_PORT_FLAG_DIRTY_PORT;
6429
6430         drsprintk(ioc, ioc_info(ioc, "Old Port table\n"));
6431         port_table_entry = NULL;
6432         list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {
6433                 drsprintk(ioc, ioc_info(ioc,
6434                     "Port: %d\t Phy_mask 0x%08x\t sas_addr(0x%016llx)\n",
6435                     port_table_entry->port_id,
6436                     port_table_entry->phy_mask,
6437                     port_table_entry->sas_address));
6438         }
6439
6440         for (j = 0; j < port_count; j++) {
6441                 ret = _scsih_look_and_get_matched_port_entry(ioc,
6442                     &port_table[j], &port_entry, &count);
6443                 if (!port_entry) {
6444                         drsprintk(ioc, ioc_info(ioc,
6445                             "No Matched entry for sas_addr(0x%16llx), Port:%d\n",
6446                             port_table[j].sas_address,
6447                             port_table[j].port_id));
6448                         continue;
6449                 }
6450
6451                 switch (ret) {
6452                 case MATCHED_WITH_ADDR_SUBPHYMASK_AND_PORT:
6453                 case MATCHED_WITH_ADDR_AND_SUBPHYMASK:
6454                         _scsih_add_or_del_phys_from_existing_port(ioc,
6455                             port_entry, port_table, j, port_count);
6456                         break;
6457                 case MATCHED_WITH_ADDR:
6458                         sas_addr = port_table[j].sas_address;
6459                         for (i = 0; i < port_count; i++) {
6460                                 if (port_table[i].sas_address == sas_addr)
6461                                         lcount++;
6462                         }
6463
6464                         if (count > 1 || lcount > 1)
6465                                 port_entry = NULL;
6466                         else
6467                                 _scsih_add_or_del_phys_from_existing_port(ioc,
6468                                     port_entry, port_table, j, port_count);
6469                 }
6470
6471                 if (!port_entry)
6472                         continue;
6473
6474                 if (port_entry->port_id != port_table[j].port_id)
6475                         port_entry->port_id = port_table[j].port_id;
6476                 port_entry->flags &= ~HBA_PORT_FLAG_DIRTY_PORT;
6477                 port_entry->phy_mask = port_table[j].phy_mask;
6478         }
6479
6480         port_table_entry = NULL;
6481 }
6482
6483 /**
6484  * _scsih_alloc_vphy - allocate virtual_phy object
6485  * @ioc: per adapter object
6486  * @port_id: Port ID number
6487  * @phy_num: HBA Phy number
6488  *
6489  * Returns allocated virtual_phy object.
6490  */
6491 static struct virtual_phy *
6492 _scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num)
6493 {
6494         struct virtual_phy *vphy;
6495         struct hba_port *port;
6496
6497         port = mpt3sas_get_port_by_id(ioc, port_id, 0);
6498         if (!port)
6499                 return NULL;
6500
6501         vphy = mpt3sas_get_vphy_by_phy(ioc, port, phy_num);
6502         if (!vphy) {
6503                 vphy = kzalloc(sizeof(struct virtual_phy), GFP_KERNEL);
6504                 if (!vphy)
6505                         return NULL;
6506
6507                 if (!port->vphys_mask)
6508                         INIT_LIST_HEAD(&port->vphys_list);
6509
6510                 /*
6511                  * Enable bit corresponding to HBA phy number on its
6512                  * parent hba_port object's vphys_mask field.
6513                  */
6514                 port->vphys_mask |= (1 << phy_num);
6515                 vphy->phy_mask |= (1 << phy_num);
6516
6517                 list_add_tail(&vphy->list, &port->vphys_list);
6518
6519                 ioc_info(ioc,
6520                     "vphy entry: %p, port id: %d, phy:%d is added to port's vphys_list\n",
6521                     vphy, port->port_id, phy_num);
6522         }
6523         return vphy;
6524 }
6525
6526 /**
6527  * _scsih_sas_host_refresh - refreshing sas host object contents
6528  * @ioc: per adapter object
6529  * Context: user
6530  *
6531  * During port enable, fw will send topology events for every device. Its
6532  * possible that the handles may change from the previous setting, so this
6533  * code keeping handles updating if changed.
6534  */
6535 static void
6536 _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
6537 {
6538         u16 sz;
6539         u16 ioc_status;
6540         int i;
6541         Mpi2ConfigReply_t mpi_reply;
6542         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6543         u16 attached_handle;
6544         u8 link_rate, port_id;
6545         struct hba_port *port;
6546         Mpi2SasPhyPage0_t phy_pg0;
6547
6548         dtmprintk(ioc,
6549                   ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",
6550                            (u64)ioc->sas_hba.sas_address));
6551
6552         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
6553             * sizeof(Mpi2SasIOUnit0PhyData_t));
6554         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
6555         if (!sas_iounit_pg0) {
6556                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6557                         __FILE__, __LINE__, __func__);
6558                 return;
6559         }
6560
6561         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
6562             sas_iounit_pg0, sz)) != 0)
6563                 goto out;
6564         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
6565         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
6566                 goto out;
6567         for (i = 0; i < ioc->sas_hba.num_phys ; i++) {
6568                 link_rate = sas_iounit_pg0->PhyData[i].NegotiatedLinkRate >> 4;
6569                 if (i == 0)
6570                         ioc->sas_hba.handle = le16_to_cpu(
6571                             sas_iounit_pg0->PhyData[0].ControllerDevHandle);
6572                 port_id = sas_iounit_pg0->PhyData[i].Port;
6573                 if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {
6574                         port = kzalloc(sizeof(struct hba_port), GFP_KERNEL);
6575                         if (!port)
6576                                 goto out;
6577
6578                         port->port_id = port_id;
6579                         ioc_info(ioc,
6580                             "hba_port entry: %p, port: %d is added to hba_port list\n",
6581                             port, port->port_id);
6582                         if (ioc->shost_recovery)
6583                                 port->flags = HBA_PORT_FLAG_NEW_PORT;
6584                         list_add_tail(&port->list, &ioc->port_table_list);
6585                 }
6586                 /*
6587                  * Check whether current Phy belongs to HBA vSES device or not.
6588                  */
6589                 if (le32_to_cpu(sas_iounit_pg0->PhyData[i].ControllerPhyDeviceInfo) &
6590                     MPI2_SAS_DEVICE_INFO_SEP &&
6591                     (link_rate >=  MPI2_SAS_NEG_LINK_RATE_1_5)) {
6592                         if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,
6593                             &phy_pg0, i))) {
6594                                 ioc_err(ioc,
6595                                     "failure at %s:%d/%s()!\n",
6596                                      __FILE__, __LINE__, __func__);
6597                                 goto out;
6598                         }
6599                         if (!(le32_to_cpu(phy_pg0.PhyInfo) &
6600                             MPI2_SAS_PHYINFO_VIRTUAL_PHY))
6601                                 continue;
6602                         /*
6603                          * Allocate a virtual_phy object for vSES device, if
6604                          * this vSES device is hot added.
6605                          */
6606                         if (!_scsih_alloc_vphy(ioc, port_id, i))
6607                                 goto out;
6608                         ioc->sas_hba.phy[i].hba_vphy = 1;
6609                 }
6610
6611                 ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
6612                 attached_handle = le16_to_cpu(sas_iounit_pg0->PhyData[i].
6613                     AttachedDevHandle);
6614                 if (attached_handle && link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
6615                         link_rate = MPI2_SAS_NEG_LINK_RATE_1_5;
6616                 ioc->sas_hba.phy[i].port =
6617                     mpt3sas_get_port_by_id(ioc, port_id, 0);
6618                 mpt3sas_transport_update_links(ioc, ioc->sas_hba.sas_address,
6619                     attached_handle, i, link_rate,
6620                     ioc->sas_hba.phy[i].port);
6621         }
6622  out:
6623         kfree(sas_iounit_pg0);
6624 }
6625
6626 /**
6627  * _scsih_sas_host_add - create sas host object
6628  * @ioc: per adapter object
6629  *
6630  * Creating host side data object, stored in ioc->sas_hba
6631  */
6632 static void
6633 _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
6634 {
6635         int i;
6636         Mpi2ConfigReply_t mpi_reply;
6637         Mpi2SasIOUnitPage0_t *sas_iounit_pg0 = NULL;
6638         Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
6639         Mpi2SasPhyPage0_t phy_pg0;
6640         Mpi2SasDevicePage0_t sas_device_pg0;
6641         Mpi2SasEnclosurePage0_t enclosure_pg0;
6642         u16 ioc_status;
6643         u16 sz;
6644         u8 device_missing_delay;
6645         u8 num_phys, port_id;
6646         struct hba_port *port;
6647
6648         mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
6649         if (!num_phys) {
6650                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6651                         __FILE__, __LINE__, __func__);
6652                 return;
6653         }
6654         ioc->sas_hba.phy = kcalloc(num_phys,
6655             sizeof(struct _sas_phy), GFP_KERNEL);
6656         if (!ioc->sas_hba.phy) {
6657                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6658                         __FILE__, __LINE__, __func__);
6659                 goto out;
6660         }
6661         ioc->sas_hba.num_phys = num_phys;
6662
6663         /* sas_iounit page 0 */
6664         sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *
6665             sizeof(Mpi2SasIOUnit0PhyData_t));
6666         sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
6667         if (!sas_iounit_pg0) {
6668                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6669                         __FILE__, __LINE__, __func__);
6670                 return;
6671         }
6672         if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,
6673             sas_iounit_pg0, sz))) {
6674                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6675                         __FILE__, __LINE__, __func__);
6676                 goto out;
6677         }
6678         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6679             MPI2_IOCSTATUS_MASK;
6680         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6681                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6682                         __FILE__, __LINE__, __func__);
6683                 goto out;
6684         }
6685
6686         /* sas_iounit page 1 */
6687         sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *
6688             sizeof(Mpi2SasIOUnit1PhyData_t));
6689         sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
6690         if (!sas_iounit_pg1) {
6691                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6692                         __FILE__, __LINE__, __func__);
6693                 goto out;
6694         }
6695         if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
6696             sas_iounit_pg1, sz))) {
6697                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6698                         __FILE__, __LINE__, __func__);
6699                 goto out;
6700         }
6701         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6702             MPI2_IOCSTATUS_MASK;
6703         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6704                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6705                         __FILE__, __LINE__, __func__);
6706                 goto out;
6707         }
6708
6709         ioc->io_missing_delay =
6710             sas_iounit_pg1->IODeviceMissingDelay;
6711         device_missing_delay =
6712             sas_iounit_pg1->ReportDeviceMissingDelay;
6713         if (device_missing_delay & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
6714                 ioc->device_missing_delay = (device_missing_delay &
6715                     MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
6716         else
6717                 ioc->device_missing_delay = device_missing_delay &
6718                     MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
6719
6720         ioc->sas_hba.parent_dev = &ioc->shost->shost_gendev;
6721         for (i = 0; i < ioc->sas_hba.num_phys ; i++) {
6722                 if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,
6723                     i))) {
6724                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6725                                 __FILE__, __LINE__, __func__);
6726                         goto out;
6727                 }
6728                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6729                     MPI2_IOCSTATUS_MASK;
6730                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6731                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6732                                 __FILE__, __LINE__, __func__);
6733                         goto out;
6734                 }
6735
6736                 if (i == 0)
6737                         ioc->sas_hba.handle = le16_to_cpu(sas_iounit_pg0->
6738                             PhyData[0].ControllerDevHandle);
6739
6740                 port_id = sas_iounit_pg0->PhyData[i].Port;
6741                 if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {
6742                         port = kzalloc(sizeof(struct hba_port), GFP_KERNEL);
6743                         if (!port)
6744                                 goto out;
6745
6746                         port->port_id = port_id;
6747                         ioc_info(ioc,
6748                            "hba_port entry: %p, port: %d is added to hba_port list\n",
6749                            port, port->port_id);
6750                         list_add_tail(&port->list,
6751                             &ioc->port_table_list);
6752                 }
6753
6754                 /*
6755                  * Check whether current Phy belongs to HBA vSES device or not.
6756                  */
6757                 if ((le32_to_cpu(phy_pg0.PhyInfo) &
6758                     MPI2_SAS_PHYINFO_VIRTUAL_PHY) &&
6759                     (phy_pg0.NegotiatedLinkRate >> 4) >=
6760                     MPI2_SAS_NEG_LINK_RATE_1_5) {
6761                         /*
6762                          * Allocate a virtual_phy object for vSES device.
6763                          */
6764                         if (!_scsih_alloc_vphy(ioc, port_id, i))
6765                                 goto out;
6766                         ioc->sas_hba.phy[i].hba_vphy = 1;
6767                 }
6768
6769                 ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;
6770                 ioc->sas_hba.phy[i].phy_id = i;
6771                 ioc->sas_hba.phy[i].port =
6772                     mpt3sas_get_port_by_id(ioc, port_id, 0);
6773                 mpt3sas_transport_add_host_phy(ioc, &ioc->sas_hba.phy[i],
6774                     phy_pg0, ioc->sas_hba.parent_dev);
6775         }
6776         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
6777             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, ioc->sas_hba.handle))) {
6778                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6779                         __FILE__, __LINE__, __func__);
6780                 goto out;
6781         }
6782         ioc->sas_hba.enclosure_handle =
6783             le16_to_cpu(sas_device_pg0.EnclosureHandle);
6784         ioc->sas_hba.sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
6785         ioc_info(ioc, "host_add: handle(0x%04x), sas_addr(0x%016llx), phys(%d)\n",
6786                  ioc->sas_hba.handle,
6787                  (u64)ioc->sas_hba.sas_address,
6788                  ioc->sas_hba.num_phys);
6789
6790         if (ioc->sas_hba.enclosure_handle) {
6791                 if (!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
6792                     &enclosure_pg0, MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
6793                    ioc->sas_hba.enclosure_handle)))
6794                         ioc->sas_hba.enclosure_logical_id =
6795                             le64_to_cpu(enclosure_pg0.EnclosureLogicalID);
6796         }
6797
6798  out:
6799         kfree(sas_iounit_pg1);
6800         kfree(sas_iounit_pg0);
6801 }
6802
6803 /**
6804  * _scsih_expander_add -  creating expander object
6805  * @ioc: per adapter object
6806  * @handle: expander handle
6807  *
6808  * Creating expander object, stored in ioc->sas_expander_list.
6809  *
6810  * Return: 0 for success, else error.
6811  */
6812 static int
6813 _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)
6814 {
6815         struct _sas_node *sas_expander;
6816         struct _enclosure_node *enclosure_dev;
6817         Mpi2ConfigReply_t mpi_reply;
6818         Mpi2ExpanderPage0_t expander_pg0;
6819         Mpi2ExpanderPage1_t expander_pg1;
6820         u32 ioc_status;
6821         u16 parent_handle;
6822         u64 sas_address, sas_address_parent = 0;
6823         int i;
6824         unsigned long flags;
6825         struct _sas_port *mpt3sas_port = NULL;
6826         u8 port_id;
6827
6828         int rc = 0;
6829
6830         if (!handle)
6831                 return -1;
6832
6833         if (ioc->shost_recovery || ioc->pci_error_recovery)
6834                 return -1;
6835
6836         if ((mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
6837             MPI2_SAS_EXPAND_PGAD_FORM_HNDL, handle))) {
6838                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6839                         __FILE__, __LINE__, __func__);
6840                 return -1;
6841         }
6842
6843         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
6844             MPI2_IOCSTATUS_MASK;
6845         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
6846                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6847                         __FILE__, __LINE__, __func__);
6848                 return -1;
6849         }
6850
6851         /* handle out of order topology events */
6852         parent_handle = le16_to_cpu(expander_pg0.ParentDevHandle);
6853         if (_scsih_get_sas_address(ioc, parent_handle, &sas_address_parent)
6854             != 0) {
6855                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6856                         __FILE__, __LINE__, __func__);
6857                 return -1;
6858         }
6859
6860         port_id = expander_pg0.PhysicalPort;
6861         if (sas_address_parent != ioc->sas_hba.sas_address) {
6862                 spin_lock_irqsave(&ioc->sas_node_lock, flags);
6863                 sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6864                     sas_address_parent,
6865                     mpt3sas_get_port_by_id(ioc, port_id, 0));
6866                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
6867                 if (!sas_expander) {
6868                         rc = _scsih_expander_add(ioc, parent_handle);
6869                         if (rc != 0)
6870                                 return rc;
6871                 }
6872         }
6873
6874         spin_lock_irqsave(&ioc->sas_node_lock, flags);
6875         sas_address = le64_to_cpu(expander_pg0.SASAddress);
6876         sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6877             sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));
6878         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
6879
6880         if (sas_expander)
6881                 return 0;
6882
6883         sas_expander = kzalloc(sizeof(struct _sas_node),
6884             GFP_KERNEL);
6885         if (!sas_expander) {
6886                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6887                         __FILE__, __LINE__, __func__);
6888                 return -1;
6889         }
6890
6891         sas_expander->handle = handle;
6892         sas_expander->num_phys = expander_pg0.NumPhys;
6893         sas_expander->sas_address_parent = sas_address_parent;
6894         sas_expander->sas_address = sas_address;
6895         sas_expander->port = mpt3sas_get_port_by_id(ioc, port_id, 0);
6896         if (!sas_expander->port) {
6897                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6898                     __FILE__, __LINE__, __func__);
6899                 rc = -1;
6900                 goto out_fail;
6901         }
6902
6903         ioc_info(ioc, "expander_add: handle(0x%04x), parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n",
6904                  handle, parent_handle,
6905                  (u64)sas_expander->sas_address, sas_expander->num_phys);
6906
6907         if (!sas_expander->num_phys) {
6908                 rc = -1;
6909                 goto out_fail;
6910         }
6911         sas_expander->phy = kcalloc(sas_expander->num_phys,
6912             sizeof(struct _sas_phy), GFP_KERNEL);
6913         if (!sas_expander->phy) {
6914                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6915                         __FILE__, __LINE__, __func__);
6916                 rc = -1;
6917                 goto out_fail;
6918         }
6919
6920         INIT_LIST_HEAD(&sas_expander->sas_port_list);
6921         mpt3sas_port = mpt3sas_transport_port_add(ioc, handle,
6922             sas_address_parent, sas_expander->port);
6923         if (!mpt3sas_port) {
6924                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
6925                         __FILE__, __LINE__, __func__);
6926                 rc = -1;
6927                 goto out_fail;
6928         }
6929         sas_expander->parent_dev = &mpt3sas_port->rphy->dev;
6930         sas_expander->rphy = mpt3sas_port->rphy;
6931
6932         for (i = 0 ; i < sas_expander->num_phys ; i++) {
6933                 if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,
6934                     &expander_pg1, i, handle))) {
6935                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6936                                 __FILE__, __LINE__, __func__);
6937                         rc = -1;
6938                         goto out_fail;
6939                 }
6940                 sas_expander->phy[i].handle = handle;
6941                 sas_expander->phy[i].phy_id = i;
6942                 sas_expander->phy[i].port =
6943                     mpt3sas_get_port_by_id(ioc, port_id, 0);
6944
6945                 if ((mpt3sas_transport_add_expander_phy(ioc,
6946                     &sas_expander->phy[i], expander_pg1,
6947                     sas_expander->parent_dev))) {
6948                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
6949                                 __FILE__, __LINE__, __func__);
6950                         rc = -1;
6951                         goto out_fail;
6952                 }
6953         }
6954
6955         if (sas_expander->enclosure_handle) {
6956                 enclosure_dev =
6957                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
6958                                                 sas_expander->enclosure_handle);
6959                 if (enclosure_dev)
6960                         sas_expander->enclosure_logical_id =
6961                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
6962         }
6963
6964         _scsih_expander_node_add(ioc, sas_expander);
6965         return 0;
6966
6967  out_fail:
6968
6969         if (mpt3sas_port)
6970                 mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,
6971                     sas_address_parent, sas_expander->port);
6972         kfree(sas_expander);
6973         return rc;
6974 }
6975
6976 /**
6977  * mpt3sas_expander_remove - removing expander object
6978  * @ioc: per adapter object
6979  * @sas_address: expander sas_address
6980  * @port: hba port entry
6981  */
6982 void
6983 mpt3sas_expander_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
6984         struct hba_port *port)
6985 {
6986         struct _sas_node *sas_expander;
6987         unsigned long flags;
6988
6989         if (ioc->shost_recovery)
6990                 return;
6991
6992         if (!port)
6993                 return;
6994
6995         spin_lock_irqsave(&ioc->sas_node_lock, flags);
6996         sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,
6997             sas_address, port);
6998         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
6999         if (sas_expander)
7000                 _scsih_expander_node_remove(ioc, sas_expander);
7001 }
7002
7003 /**
7004  * _scsih_done -  internal SCSI_IO callback handler.
7005  * @ioc: per adapter object
7006  * @smid: system request message index
7007  * @msix_index: MSIX table index supplied by the OS
7008  * @reply: reply message frame(lower 32bit addr)
7009  *
7010  * Callback handler when sending internal generated SCSI_IO.
7011  * The callback index passed is `ioc->scsih_cb_idx`
7012  *
7013  * Return: 1 meaning mf should be freed from _base_interrupt
7014  *         0 means the mf is freed from this function.
7015  */
7016 static u8
7017 _scsih_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)
7018 {
7019         MPI2DefaultReply_t *mpi_reply;
7020
7021         mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);
7022         if (ioc->scsih_cmds.status == MPT3_CMD_NOT_USED)
7023                 return 1;
7024         if (ioc->scsih_cmds.smid != smid)
7025                 return 1;
7026         ioc->scsih_cmds.status |= MPT3_CMD_COMPLETE;
7027         if (mpi_reply) {
7028                 memcpy(ioc->scsih_cmds.reply, mpi_reply,
7029                     mpi_reply->MsgLength*4);
7030                 ioc->scsih_cmds.status |= MPT3_CMD_REPLY_VALID;
7031         }
7032         ioc->scsih_cmds.status &= ~MPT3_CMD_PENDING;
7033         complete(&ioc->scsih_cmds.done);
7034         return 1;
7035 }
7036
7037
7038
7039
7040 #define MPT3_MAX_LUNS (255)
7041
7042
7043 /**
7044  * _scsih_check_access_status - check access flags
7045  * @ioc: per adapter object
7046  * @sas_address: sas address
7047  * @handle: sas device handle
7048  * @access_status: errors returned during discovery of the device
7049  *
7050  * Return: 0 for success, else failure
7051  */
7052 static u8
7053 _scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,
7054         u16 handle, u8 access_status)
7055 {
7056         u8 rc = 1;
7057         char *desc = NULL;
7058
7059         switch (access_status) {
7060         case MPI2_SAS_DEVICE0_ASTATUS_NO_ERRORS:
7061         case MPI2_SAS_DEVICE0_ASTATUS_SATA_NEEDS_INITIALIZATION:
7062                 rc = 0;
7063                 break;
7064         case MPI2_SAS_DEVICE0_ASTATUS_SATA_CAPABILITY_FAILED:
7065                 desc = "sata capability failed";
7066                 break;
7067         case MPI2_SAS_DEVICE0_ASTATUS_SATA_AFFILIATION_CONFLICT:
7068                 desc = "sata affiliation conflict";
7069                 break;
7070         case MPI2_SAS_DEVICE0_ASTATUS_ROUTE_NOT_ADDRESSABLE:
7071                 desc = "route not addressable";
7072                 break;
7073         case MPI2_SAS_DEVICE0_ASTATUS_SMP_ERROR_NOT_ADDRESSABLE:
7074                 desc = "smp error not addressable";
7075                 break;
7076         case MPI2_SAS_DEVICE0_ASTATUS_DEVICE_BLOCKED:
7077                 desc = "device blocked";
7078                 break;
7079         case MPI2_SAS_DEVICE0_ASTATUS_SATA_INIT_FAILED:
7080         case MPI2_SAS_DEVICE0_ASTATUS_SIF_UNKNOWN:
7081         case MPI2_SAS_DEVICE0_ASTATUS_SIF_AFFILIATION_CONFLICT:
7082         case MPI2_SAS_DEVICE0_ASTATUS_SIF_DIAG:
7083         case MPI2_SAS_DEVICE0_ASTATUS_SIF_IDENTIFICATION:
7084         case MPI2_SAS_DEVICE0_ASTATUS_SIF_CHECK_POWER:
7085         case MPI2_SAS_DEVICE0_ASTATUS_SIF_PIO_SN:
7086         case MPI2_SAS_DEVICE0_ASTATUS_SIF_MDMA_SN:
7087         case MPI2_SAS_DEVICE0_ASTATUS_SIF_UDMA_SN:
7088         case MPI2_SAS_DEVICE0_ASTATUS_SIF_ZONING_VIOLATION:
7089         case MPI2_SAS_DEVICE0_ASTATUS_SIF_NOT_ADDRESSABLE:
7090         case MPI2_SAS_DEVICE0_ASTATUS_SIF_MAX:
7091                 desc = "sata initialization failed";
7092                 break;
7093         default:
7094                 desc = "unknown";
7095                 break;
7096         }
7097
7098         if (!rc)
7099                 return 0;
7100
7101         ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",
7102                 desc, (u64)sas_address, handle);
7103         return rc;
7104 }
7105
7106 /**
7107  * _scsih_check_device - checking device responsiveness
7108  * @ioc: per adapter object
7109  * @parent_sas_address: sas address of parent expander or sas host
7110  * @handle: attached device handle
7111  * @phy_number: phy number
7112  * @link_rate: new link rate
7113  */
7114 static void
7115 _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,
7116         u64 parent_sas_address, u16 handle, u8 phy_number, u8 link_rate)
7117 {
7118         Mpi2ConfigReply_t mpi_reply;
7119         Mpi2SasDevicePage0_t sas_device_pg0;
7120         struct _sas_device *sas_device = NULL;
7121         struct _enclosure_node *enclosure_dev = NULL;
7122         u32 ioc_status;
7123         unsigned long flags;
7124         u64 sas_address;
7125         struct scsi_target *starget;
7126         struct MPT3SAS_TARGET *sas_target_priv_data;
7127         u32 device_info;
7128         struct hba_port *port;
7129
7130         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
7131             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle)))
7132                 return;
7133
7134         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
7135         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
7136                 return;
7137
7138         /* wide port handling ~ we need only handle device once for the phy that
7139          * is matched in sas device page zero
7140          */
7141         if (phy_number != sas_device_pg0.PhyNum)
7142                 return;
7143
7144         /* check if this is end device */
7145         device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
7146         if (!(_scsih_is_end_device(device_info)))
7147                 return;
7148
7149         spin_lock_irqsave(&ioc->sas_device_lock, flags);
7150         sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
7151         port = mpt3sas_get_port_by_id(ioc, sas_device_pg0.PhysicalPort, 0);
7152         if (!port)
7153                 goto out_unlock;
7154         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
7155             sas_address, port);
7156
7157         if (!sas_device)
7158                 goto out_unlock;
7159
7160         if (unlikely(sas_device->handle != handle)) {
7161                 starget = sas_device->starget;
7162                 sas_target_priv_data = starget->hostdata;
7163                 starget_printk(KERN_INFO, starget,
7164                         "handle changed from(0x%04x) to (0x%04x)!!!\n",
7165                         sas_device->handle, handle);
7166                 sas_target_priv_data->handle = handle;
7167                 sas_device->handle = handle;
7168                 if (le16_to_cpu(sas_device_pg0.Flags) &
7169                      MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
7170                         sas_device->enclosure_level =
7171                                 sas_device_pg0.EnclosureLevel;
7172                         memcpy(sas_device->connector_name,
7173                                 sas_device_pg0.ConnectorName, 4);
7174                         sas_device->connector_name[4] = '\0';
7175                 } else {
7176                         sas_device->enclosure_level = 0;
7177                         sas_device->connector_name[0] = '\0';
7178                 }
7179
7180                 sas_device->enclosure_handle =
7181                                 le16_to_cpu(sas_device_pg0.EnclosureHandle);
7182                 sas_device->is_chassis_slot_valid = 0;
7183                 enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,
7184                                                 sas_device->enclosure_handle);
7185                 if (enclosure_dev) {
7186                         sas_device->enclosure_logical_id =
7187                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7188                         if (le16_to_cpu(enclosure_dev->pg0.Flags) &
7189                             MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
7190                                 sas_device->is_chassis_slot_valid = 1;
7191                                 sas_device->chassis_slot =
7192                                         enclosure_dev->pg0.ChassisSlot;
7193                         }
7194                 }
7195         }
7196
7197         /* check if device is present */
7198         if (!(le16_to_cpu(sas_device_pg0.Flags) &
7199             MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
7200                 ioc_err(ioc, "device is not present handle(0x%04x), flags!!!\n",
7201                         handle);
7202                 goto out_unlock;
7203         }
7204
7205         /* check if there were any issues with discovery */
7206         if (_scsih_check_access_status(ioc, sas_address, handle,
7207             sas_device_pg0.AccessStatus))
7208                 goto out_unlock;
7209
7210         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
7211         _scsih_ublock_io_device(ioc, sas_address, port);
7212
7213         if (sas_device)
7214                 sas_device_put(sas_device);
7215         return;
7216
7217 out_unlock:
7218         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
7219         if (sas_device)
7220                 sas_device_put(sas_device);
7221 }
7222
7223 /**
7224  * _scsih_add_device -  creating sas device object
7225  * @ioc: per adapter object
7226  * @handle: sas device handle
7227  * @phy_num: phy number end device attached to
7228  * @is_pd: is this hidden raid component
7229  *
7230  * Creating end device object, stored in ioc->sas_device_list.
7231  *
7232  * Return: 0 for success, non-zero for failure.
7233  */
7234 static int
7235 _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,
7236         u8 is_pd)
7237 {
7238         Mpi2ConfigReply_t mpi_reply;
7239         Mpi2SasDevicePage0_t sas_device_pg0;
7240         struct _sas_device *sas_device;
7241         struct _enclosure_node *enclosure_dev = NULL;
7242         u32 ioc_status;
7243         u64 sas_address;
7244         u32 device_info;
7245         u8 port_id;
7246
7247         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
7248             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
7249                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7250                         __FILE__, __LINE__, __func__);
7251                 return -1;
7252         }
7253
7254         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
7255             MPI2_IOCSTATUS_MASK;
7256         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
7257                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7258                         __FILE__, __LINE__, __func__);
7259                 return -1;
7260         }
7261
7262         /* check if this is end device */
7263         device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
7264         if (!(_scsih_is_end_device(device_info)))
7265                 return -1;
7266         set_bit(handle, ioc->pend_os_device_add);
7267         sas_address = le64_to_cpu(sas_device_pg0.SASAddress);
7268
7269         /* check if device is present */
7270         if (!(le16_to_cpu(sas_device_pg0.Flags) &
7271             MPI2_SAS_DEVICE0_FLAGS_DEVICE_PRESENT)) {
7272                 ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",
7273                         handle);
7274                 return -1;
7275         }
7276
7277         /* check if there were any issues with discovery */
7278         if (_scsih_check_access_status(ioc, sas_address, handle,
7279             sas_device_pg0.AccessStatus))
7280                 return -1;
7281
7282         port_id = sas_device_pg0.PhysicalPort;
7283         sas_device = mpt3sas_get_sdev_by_addr(ioc,
7284             sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));
7285         if (sas_device) {
7286                 clear_bit(handle, ioc->pend_os_device_add);
7287                 sas_device_put(sas_device);
7288                 return -1;
7289         }
7290
7291         if (sas_device_pg0.EnclosureHandle) {
7292                 enclosure_dev =
7293                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
7294                             le16_to_cpu(sas_device_pg0.EnclosureHandle));
7295                 if (enclosure_dev == NULL)
7296                         ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",
7297                                  sas_device_pg0.EnclosureHandle);
7298         }
7299
7300         sas_device = kzalloc(sizeof(struct _sas_device),
7301             GFP_KERNEL);
7302         if (!sas_device) {
7303                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7304                         __FILE__, __LINE__, __func__);
7305                 return 0;
7306         }
7307
7308         kref_init(&sas_device->refcount);
7309         sas_device->handle = handle;
7310         if (_scsih_get_sas_address(ioc,
7311             le16_to_cpu(sas_device_pg0.ParentDevHandle),
7312             &sas_device->sas_address_parent) != 0)
7313                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7314                         __FILE__, __LINE__, __func__);
7315         sas_device->enclosure_handle =
7316             le16_to_cpu(sas_device_pg0.EnclosureHandle);
7317         if (sas_device->enclosure_handle != 0)
7318                 sas_device->slot =
7319                     le16_to_cpu(sas_device_pg0.Slot);
7320         sas_device->device_info = device_info;
7321         sas_device->sas_address = sas_address;
7322         sas_device->phy = sas_device_pg0.PhyNum;
7323         sas_device->fast_path = (le16_to_cpu(sas_device_pg0.Flags) &
7324             MPI25_SAS_DEVICE0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
7325         sas_device->port = mpt3sas_get_port_by_id(ioc, port_id, 0);
7326         if (!sas_device->port) {
7327                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
7328                     __FILE__, __LINE__, __func__);
7329                 goto out;
7330         }
7331
7332         if (le16_to_cpu(sas_device_pg0.Flags)
7333                 & MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
7334                 sas_device->enclosure_level =
7335                         sas_device_pg0.EnclosureLevel;
7336                 memcpy(sas_device->connector_name,
7337                         sas_device_pg0.ConnectorName, 4);
7338                 sas_device->connector_name[4] = '\0';
7339         } else {
7340                 sas_device->enclosure_level = 0;
7341                 sas_device->connector_name[0] = '\0';
7342         }
7343         /* get enclosure_logical_id & chassis_slot*/
7344         sas_device->is_chassis_slot_valid = 0;
7345         if (enclosure_dev) {
7346                 sas_device->enclosure_logical_id =
7347                     le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
7348                 if (le16_to_cpu(enclosure_dev->pg0.Flags) &
7349                     MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
7350                         sas_device->is_chassis_slot_valid = 1;
7351                         sas_device->chassis_slot =
7352                                         enclosure_dev->pg0.ChassisSlot;
7353                 }
7354         }
7355
7356         /* get device name */
7357         sas_device->device_name = le64_to_cpu(sas_device_pg0.DeviceName);
7358         sas_device->port_type = sas_device_pg0.MaxPortConnections;
7359         ioc_info(ioc,
7360             "handle(0x%0x) sas_address(0x%016llx) port_type(0x%0x)\n",
7361             handle, sas_device->sas_address, sas_device->port_type);
7362
7363         if (ioc->wait_for_discovery_to_complete)
7364                 _scsih_sas_device_init_add(ioc, sas_device);
7365         else
7366                 _scsih_sas_device_add(ioc, sas_device);
7367
7368 out:
7369         sas_device_put(sas_device);
7370         return 0;
7371 }
7372
7373 /**
7374  * _scsih_remove_device -  removing sas device object
7375  * @ioc: per adapter object
7376  * @sas_device: the sas_device object
7377  */
7378 static void
7379 _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
7380         struct _sas_device *sas_device)
7381 {
7382         struct MPT3SAS_TARGET *sas_target_priv_data;
7383
7384         if ((ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM) &&
7385              (sas_device->pfa_led_on)) {
7386                 _scsih_turn_off_pfa_led(ioc, sas_device);
7387                 sas_device->pfa_led_on = 0;
7388         }
7389
7390         dewtprintk(ioc,
7391                    ioc_info(ioc, "%s: enter: handle(0x%04x), sas_addr(0x%016llx)\n",
7392                             __func__,
7393                             sas_device->handle, (u64)sas_device->sas_address));
7394
7395         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
7396             NULL, NULL));
7397
7398         if (sas_device->starget && sas_device->starget->hostdata) {
7399                 sas_target_priv_data = sas_device->starget->hostdata;
7400                 sas_target_priv_data->deleted = 1;
7401                 _scsih_ublock_io_device(ioc, sas_device->sas_address,
7402                     sas_device->port);
7403                 sas_target_priv_data->handle =
7404                      MPT3SAS_INVALID_DEVICE_HANDLE;
7405         }
7406
7407         if (!ioc->hide_drives)
7408                 mpt3sas_transport_port_remove(ioc,
7409                     sas_device->sas_address,
7410                     sas_device->sas_address_parent,
7411                     sas_device->port);
7412
7413         ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",
7414                  sas_device->handle, (u64)sas_device->sas_address);
7415
7416         _scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);
7417
7418         dewtprintk(ioc,
7419                    ioc_info(ioc, "%s: exit: handle(0x%04x), sas_addr(0x%016llx)\n",
7420                             __func__,
7421                             sas_device->handle, (u64)sas_device->sas_address));
7422         dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,
7423             NULL, NULL));
7424 }
7425
7426 /**
7427  * _scsih_sas_topology_change_event_debug - debug for topology event
7428  * @ioc: per adapter object
7429  * @event_data: event data payload
7430  * Context: user.
7431  */
7432 static void
7433 _scsih_sas_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
7434         Mpi2EventDataSasTopologyChangeList_t *event_data)
7435 {
7436         int i;
7437         u16 handle;
7438         u16 reason_code;
7439         u8 phy_number;
7440         char *status_str = NULL;
7441         u8 link_rate, prev_link_rate;
7442
7443         switch (event_data->ExpStatus) {
7444         case MPI2_EVENT_SAS_TOPO_ES_ADDED:
7445                 status_str = "add";
7446                 break;
7447         case MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING:
7448                 status_str = "remove";
7449                 break;
7450         case MPI2_EVENT_SAS_TOPO_ES_RESPONDING:
7451         case 0:
7452                 status_str =  "responding";
7453                 break;
7454         case MPI2_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING:
7455                 status_str = "remove delay";
7456                 break;
7457         default:
7458                 status_str = "unknown status";
7459                 break;
7460         }
7461         ioc_info(ioc, "sas topology change: (%s)\n", status_str);
7462         pr_info("\thandle(0x%04x), enclosure_handle(0x%04x) " \
7463             "start_phy(%02d), count(%d)\n",
7464             le16_to_cpu(event_data->ExpanderDevHandle),
7465             le16_to_cpu(event_data->EnclosureHandle),
7466             event_data->StartPhyNum, event_data->NumEntries);
7467         for (i = 0; i < event_data->NumEntries; i++) {
7468                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
7469                 if (!handle)
7470                         continue;
7471                 phy_number = event_data->StartPhyNum + i;
7472                 reason_code = event_data->PHY[i].PhyStatus &
7473                     MPI2_EVENT_SAS_TOPO_RC_MASK;
7474                 switch (reason_code) {
7475                 case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
7476                         status_str = "target add";
7477                         break;
7478                 case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
7479                         status_str = "target remove";
7480                         break;
7481                 case MPI2_EVENT_SAS_TOPO_RC_DELAY_NOT_RESPONDING:
7482                         status_str = "delay target remove";
7483                         break;
7484                 case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
7485                         status_str = "link rate change";
7486                         break;
7487                 case MPI2_EVENT_SAS_TOPO_RC_NO_CHANGE:
7488                         status_str = "target responding";
7489                         break;
7490                 default:
7491                         status_str = "unknown";
7492                         break;
7493                 }
7494                 link_rate = event_data->PHY[i].LinkRate >> 4;
7495                 prev_link_rate = event_data->PHY[i].LinkRate & 0xF;
7496                 pr_info("\tphy(%02d), attached_handle(0x%04x): %s:" \
7497                     " link rate: new(0x%02x), old(0x%02x)\n", phy_number,
7498                     handle, status_str, link_rate, prev_link_rate);
7499
7500         }
7501 }
7502
7503 /**
7504  * _scsih_sas_topology_change_event - handle topology changes
7505  * @ioc: per adapter object
7506  * @fw_event: The fw_event_work object
7507  * Context: user.
7508  *
7509  */
7510 static int
7511 _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
7512         struct fw_event_work *fw_event)
7513 {
7514         int i;
7515         u16 parent_handle, handle;
7516         u16 reason_code;
7517         u8 phy_number, max_phys;
7518         struct _sas_node *sas_expander;
7519         u64 sas_address;
7520         unsigned long flags;
7521         u8 link_rate, prev_link_rate;
7522         struct hba_port *port;
7523         Mpi2EventDataSasTopologyChangeList_t *event_data =
7524                 (Mpi2EventDataSasTopologyChangeList_t *)
7525                 fw_event->event_data;
7526
7527         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
7528                 _scsih_sas_topology_change_event_debug(ioc, event_data);
7529
7530         if (ioc->shost_recovery || ioc->remove_host || ioc->pci_error_recovery)
7531                 return 0;
7532
7533         if (!ioc->sas_hba.num_phys)
7534                 _scsih_sas_host_add(ioc);
7535         else
7536                 _scsih_sas_host_refresh(ioc);
7537
7538         if (fw_event->ignore) {
7539                 dewtprintk(ioc, ioc_info(ioc, "ignoring expander event\n"));
7540                 return 0;
7541         }
7542
7543         parent_handle = le16_to_cpu(event_data->ExpanderDevHandle);
7544         port = mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0);
7545
7546         /* handle expander add */
7547         if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_ADDED)
7548                 if (_scsih_expander_add(ioc, parent_handle) != 0)
7549                         return 0;
7550
7551         spin_lock_irqsave(&ioc->sas_node_lock, flags);
7552         sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,
7553             parent_handle);
7554         if (sas_expander) {
7555                 sas_address = sas_expander->sas_address;
7556                 max_phys = sas_expander->num_phys;
7557                 port = sas_expander->port;
7558         } else if (parent_handle < ioc->sas_hba.num_phys) {
7559                 sas_address = ioc->sas_hba.sas_address;
7560                 max_phys = ioc->sas_hba.num_phys;
7561         } else {
7562                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
7563                 return 0;
7564         }
7565         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
7566
7567         /* handle siblings events */
7568         for (i = 0; i < event_data->NumEntries; i++) {
7569                 if (fw_event->ignore) {
7570                         dewtprintk(ioc,
7571                                    ioc_info(ioc, "ignoring expander event\n"));
7572                         return 0;
7573                 }
7574                 if (ioc->remove_host || ioc->pci_error_recovery)
7575                         return 0;
7576                 phy_number = event_data->StartPhyNum + i;
7577                 if (phy_number >= max_phys)
7578                         continue;
7579                 reason_code = event_data->PHY[i].PhyStatus &
7580                     MPI2_EVENT_SAS_TOPO_RC_MASK;
7581                 if ((event_data->PHY[i].PhyStatus &
7582                     MPI2_EVENT_SAS_TOPO_PHYSTATUS_VACANT) && (reason_code !=
7583                     MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING))
7584                                 continue;
7585                 handle = le16_to_cpu(event_data->PHY[i].AttachedDevHandle);
7586                 if (!handle)
7587                         continue;
7588                 link_rate = event_data->PHY[i].LinkRate >> 4;
7589                 prev_link_rate = event_data->PHY[i].LinkRate & 0xF;
7590                 switch (reason_code) {
7591                 case MPI2_EVENT_SAS_TOPO_RC_PHY_CHANGED:
7592
7593                         if (ioc->shost_recovery)
7594                                 break;
7595
7596                         if (link_rate == prev_link_rate)
7597                                 break;
7598
7599                         mpt3sas_transport_update_links(ioc, sas_address,
7600                             handle, phy_number, link_rate, port);
7601
7602                         if (link_rate < MPI2_SAS_NEG_LINK_RATE_1_5)
7603                                 break;
7604
7605                         _scsih_check_device(ioc, sas_address, handle,
7606                             phy_number, link_rate);
7607
7608                         if (!test_bit(handle, ioc->pend_os_device_add))
7609                                 break;
7610
7611                         fallthrough;
7612
7613                 case MPI2_EVENT_SAS_TOPO_RC_TARG_ADDED:
7614
7615                         if (ioc->shost_recovery)
7616                                 break;
7617
7618                         mpt3sas_transport_update_links(ioc, sas_address,
7619                             handle, phy_number, link_rate, port);
7620
7621                         _scsih_add_device(ioc, handle, phy_number, 0);
7622
7623                         break;
7624                 case MPI2_EVENT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
7625
7626                         _scsih_device_remove_by_handle(ioc, handle);
7627                         break;
7628                 }
7629         }
7630
7631         /* handle expander removal */
7632         if (event_data->ExpStatus == MPI2_EVENT_SAS_TOPO_ES_NOT_RESPONDING &&
7633             sas_expander)
7634                 mpt3sas_expander_remove(ioc, sas_address, port);
7635
7636         return 0;
7637 }
7638
7639 /**
7640  * _scsih_sas_device_status_change_event_debug - debug for device event
7641  * @ioc: ?
7642  * @event_data: event data payload
7643  * Context: user.
7644  */
7645 static void
7646 _scsih_sas_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
7647         Mpi2EventDataSasDeviceStatusChange_t *event_data)
7648 {
7649         char *reason_str = NULL;
7650
7651         switch (event_data->ReasonCode) {
7652         case MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA:
7653                 reason_str = "smart data";
7654                 break;
7655         case MPI2_EVENT_SAS_DEV_STAT_RC_UNSUPPORTED:
7656                 reason_str = "unsupported device discovered";
7657                 break;
7658         case MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET:
7659                 reason_str = "internal device reset";
7660                 break;
7661         case MPI2_EVENT_SAS_DEV_STAT_RC_TASK_ABORT_INTERNAL:
7662                 reason_str = "internal task abort";
7663                 break;
7664         case MPI2_EVENT_SAS_DEV_STAT_RC_ABORT_TASK_SET_INTERNAL:
7665                 reason_str = "internal task abort set";
7666                 break;
7667         case MPI2_EVENT_SAS_DEV_STAT_RC_CLEAR_TASK_SET_INTERNAL:
7668                 reason_str = "internal clear task set";
7669                 break;
7670         case MPI2_EVENT_SAS_DEV_STAT_RC_QUERY_TASK_INTERNAL:
7671                 reason_str = "internal query task";
7672                 break;
7673         case MPI2_EVENT_SAS_DEV_STAT_RC_SATA_INIT_FAILURE:
7674                 reason_str = "sata init failure";
7675                 break;
7676         case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
7677                 reason_str = "internal device reset complete";
7678                 break;
7679         case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_TASK_ABORT_INTERNAL:
7680                 reason_str = "internal task abort complete";
7681                 break;
7682         case MPI2_EVENT_SAS_DEV_STAT_RC_ASYNC_NOTIFICATION:
7683                 reason_str = "internal async notification";
7684                 break;
7685         case MPI2_EVENT_SAS_DEV_STAT_RC_EXPANDER_REDUCED_FUNCTIONALITY:
7686                 reason_str = "expander reduced functionality";
7687                 break;
7688         case MPI2_EVENT_SAS_DEV_STAT_RC_CMP_EXPANDER_REDUCED_FUNCTIONALITY:
7689                 reason_str = "expander reduced functionality complete";
7690                 break;
7691         default:
7692                 reason_str = "unknown reason";
7693                 break;
7694         }
7695         ioc_info(ioc, "device status change: (%s)\thandle(0x%04x), sas address(0x%016llx), tag(%d)",
7696                  reason_str, le16_to_cpu(event_data->DevHandle),
7697                  (u64)le64_to_cpu(event_data->SASAddress),
7698                  le16_to_cpu(event_data->TaskTag));
7699         if (event_data->ReasonCode == MPI2_EVENT_SAS_DEV_STAT_RC_SMART_DATA)
7700                 pr_cont(", ASC(0x%x), ASCQ(0x%x)\n",
7701                         event_data->ASC, event_data->ASCQ);
7702         pr_cont("\n");
7703 }
7704
7705 /**
7706  * _scsih_sas_device_status_change_event - handle device status change
7707  * @ioc: per adapter object
7708  * @event_data: The fw event
7709  * Context: user.
7710  */
7711 static void
7712 _scsih_sas_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,
7713         Mpi2EventDataSasDeviceStatusChange_t *event_data)
7714 {
7715         struct MPT3SAS_TARGET *target_priv_data;
7716         struct _sas_device *sas_device;
7717         u64 sas_address;
7718         unsigned long flags;
7719
7720         /* In MPI Revision K (0xC), the internal device reset complete was
7721          * implemented, so avoid setting tm_busy flag for older firmware.
7722          */
7723         if ((ioc->facts.HeaderVersion >> 8) < 0xC)
7724                 return;
7725
7726         if (event_data->ReasonCode !=
7727             MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET &&
7728            event_data->ReasonCode !=
7729             MPI2_EVENT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET)
7730                 return;
7731
7732         spin_lock_irqsave(&ioc->sas_device_lock, flags);
7733         sas_address = le64_to_cpu(event_data->SASAddress);
7734         sas_device = __mpt3sas_get_sdev_by_addr(ioc,
7735             sas_address,
7736             mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0));
7737
7738         if (!sas_device || !sas_device->starget)
7739                 goto out;
7740
7741         target_priv_data = sas_device->starget->hostdata;
7742         if (!target_priv_data)
7743                 goto out;
7744
7745         if (event_data->ReasonCode ==
7746             MPI2_EVENT_SAS_DEV_STAT_RC_INTERNAL_DEVICE_RESET)
7747                 target_priv_data->tm_busy = 1;
7748         else
7749                 target_priv_data->tm_busy = 0;
7750
7751         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
7752                 ioc_info(ioc,
7753                     "%s tm_busy flag for handle(0x%04x)\n",
7754                     (target_priv_data->tm_busy == 1) ? "Enable" : "Disable",
7755                     target_priv_data->handle);
7756
7757 out:
7758         if (sas_device)
7759                 sas_device_put(sas_device);
7760
7761         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
7762 }
7763
7764
7765 /**
7766  * _scsih_check_pcie_access_status - check access flags
7767  * @ioc: per adapter object
7768  * @wwid: wwid
7769  * @handle: sas device handle
7770  * @access_status: errors returned during discovery of the device
7771  *
7772  * Return: 0 for success, else failure
7773  */
7774 static u8
7775 _scsih_check_pcie_access_status(struct MPT3SAS_ADAPTER *ioc, u64 wwid,
7776         u16 handle, u8 access_status)
7777 {
7778         u8 rc = 1;
7779         char *desc = NULL;
7780
7781         switch (access_status) {
7782         case MPI26_PCIEDEV0_ASTATUS_NO_ERRORS:
7783         case MPI26_PCIEDEV0_ASTATUS_NEEDS_INITIALIZATION:
7784                 rc = 0;
7785                 break;
7786         case MPI26_PCIEDEV0_ASTATUS_CAPABILITY_FAILED:
7787                 desc = "PCIe device capability failed";
7788                 break;
7789         case MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED:
7790                 desc = "PCIe device blocked";
7791                 ioc_info(ioc,
7792                     "Device with Access Status (%s): wwid(0x%016llx), "
7793                     "handle(0x%04x)\n ll only be added to the internal list",
7794                     desc, (u64)wwid, handle);
7795                 rc = 0;
7796                 break;
7797         case MPI26_PCIEDEV0_ASTATUS_MEMORY_SPACE_ACCESS_FAILED:
7798                 desc = "PCIe device mem space access failed";
7799                 break;
7800         case MPI26_PCIEDEV0_ASTATUS_UNSUPPORTED_DEVICE:
7801                 desc = "PCIe device unsupported";
7802                 break;
7803         case MPI26_PCIEDEV0_ASTATUS_MSIX_REQUIRED:
7804                 desc = "PCIe device MSIx Required";
7805                 break;
7806         case MPI26_PCIEDEV0_ASTATUS_INIT_FAIL_MAX:
7807                 desc = "PCIe device init fail max";
7808                 break;
7809         case MPI26_PCIEDEV0_ASTATUS_UNKNOWN:
7810                 desc = "PCIe device status unknown";
7811                 break;
7812         case MPI26_PCIEDEV0_ASTATUS_NVME_READY_TIMEOUT:
7813                 desc = "nvme ready timeout";
7814                 break;
7815         case MPI26_PCIEDEV0_ASTATUS_NVME_DEVCFG_UNSUPPORTED:
7816                 desc = "nvme device configuration unsupported";
7817                 break;
7818         case MPI26_PCIEDEV0_ASTATUS_NVME_IDENTIFY_FAILED:
7819                 desc = "nvme identify failed";
7820                 break;
7821         case MPI26_PCIEDEV0_ASTATUS_NVME_QCONFIG_FAILED:
7822                 desc = "nvme qconfig failed";
7823                 break;
7824         case MPI26_PCIEDEV0_ASTATUS_NVME_QCREATION_FAILED:
7825                 desc = "nvme qcreation failed";
7826                 break;
7827         case MPI26_PCIEDEV0_ASTATUS_NVME_EVENTCFG_FAILED:
7828                 desc = "nvme eventcfg failed";
7829                 break;
7830         case MPI26_PCIEDEV0_ASTATUS_NVME_GET_FEATURE_STAT_FAILED:
7831                 desc = "nvme get feature stat failed";
7832                 break;
7833         case MPI26_PCIEDEV0_ASTATUS_NVME_IDLE_TIMEOUT:
7834                 desc = "nvme idle timeout";
7835                 break;
7836         case MPI26_PCIEDEV0_ASTATUS_NVME_FAILURE_STATUS:
7837                 desc = "nvme failure status";
7838                 break;
7839         default:
7840                 ioc_err(ioc, "NVMe discovery error(0x%02x): wwid(0x%016llx), handle(0x%04x)\n",
7841                         access_status, (u64)wwid, handle);
7842                 return rc;
7843         }
7844
7845         if (!rc)
7846                 return rc;
7847
7848         ioc_info(ioc, "NVMe discovery error(%s): wwid(0x%016llx), handle(0x%04x)\n",
7849                  desc, (u64)wwid, handle);
7850         return rc;
7851 }
7852
7853 /**
7854  * _scsih_pcie_device_remove_from_sml -  removing pcie device
7855  * from SML and free up associated memory
7856  * @ioc: per adapter object
7857  * @pcie_device: the pcie_device object
7858  */
7859 static void
7860 _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
7861         struct _pcie_device *pcie_device)
7862 {
7863         struct MPT3SAS_TARGET *sas_target_priv_data;
7864
7865         dewtprintk(ioc,
7866                    ioc_info(ioc, "%s: enter: handle(0x%04x), wwid(0x%016llx)\n",
7867                             __func__,
7868                             pcie_device->handle, (u64)pcie_device->wwid));
7869         if (pcie_device->enclosure_handle != 0)
7870                 dewtprintk(ioc,
7871                            ioc_info(ioc, "%s: enter: enclosure logical id(0x%016llx), slot(%d)\n",
7872                                     __func__,
7873                                     (u64)pcie_device->enclosure_logical_id,
7874                                     pcie_device->slot));
7875         if (pcie_device->connector_name[0] != '\0')
7876                 dewtprintk(ioc,
7877                            ioc_info(ioc, "%s: enter: enclosure level(0x%04x), connector name(%s)\n",
7878                                     __func__,
7879                                     pcie_device->enclosure_level,
7880                                     pcie_device->connector_name));
7881
7882         if (pcie_device->starget && pcie_device->starget->hostdata) {
7883                 sas_target_priv_data = pcie_device->starget->hostdata;
7884                 sas_target_priv_data->deleted = 1;
7885                 _scsih_ublock_io_device(ioc, pcie_device->wwid, NULL);
7886                 sas_target_priv_data->handle = MPT3SAS_INVALID_DEVICE_HANDLE;
7887         }
7888
7889         ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
7890                  pcie_device->handle, (u64)pcie_device->wwid);
7891         if (pcie_device->enclosure_handle != 0)
7892                 ioc_info(ioc, "removing : enclosure logical id(0x%016llx), slot(%d)\n",
7893                          (u64)pcie_device->enclosure_logical_id,
7894                          pcie_device->slot);
7895         if (pcie_device->connector_name[0] != '\0')
7896                 ioc_info(ioc, "removing: enclosure level(0x%04x), connector name( %s)\n",
7897                          pcie_device->enclosure_level,
7898                          pcie_device->connector_name);
7899
7900         if (pcie_device->starget && (pcie_device->access_status !=
7901                                 MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED))
7902                 scsi_remove_target(&pcie_device->starget->dev);
7903         dewtprintk(ioc,
7904                    ioc_info(ioc, "%s: exit: handle(0x%04x), wwid(0x%016llx)\n",
7905                             __func__,
7906                             pcie_device->handle, (u64)pcie_device->wwid));
7907         if (pcie_device->enclosure_handle != 0)
7908                 dewtprintk(ioc,
7909                            ioc_info(ioc, "%s: exit: enclosure logical id(0x%016llx), slot(%d)\n",
7910                                     __func__,
7911                                     (u64)pcie_device->enclosure_logical_id,
7912                                     pcie_device->slot));
7913         if (pcie_device->connector_name[0] != '\0')
7914                 dewtprintk(ioc,
7915                            ioc_info(ioc, "%s: exit: enclosure level(0x%04x), connector name( %s)\n",
7916                                     __func__,
7917                                     pcie_device->enclosure_level,
7918                                     pcie_device->connector_name));
7919
7920         kfree(pcie_device->serial_number);
7921 }
7922
7923
7924 /**
7925  * _scsih_pcie_check_device - checking device responsiveness
7926  * @ioc: per adapter object
7927  * @handle: attached device handle
7928  */
7929 static void
7930 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
7931 {
7932         Mpi2ConfigReply_t mpi_reply;
7933         Mpi26PCIeDevicePage0_t pcie_device_pg0;
7934         u32 ioc_status;
7935         struct _pcie_device *pcie_device;
7936         u64 wwid;
7937         unsigned long flags;
7938         struct scsi_target *starget;
7939         struct MPT3SAS_TARGET *sas_target_priv_data;
7940         u32 device_info;
7941
7942         if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
7943                 &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle)))
7944                 return;
7945
7946         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
7947         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
7948                 return;
7949
7950         /* check if this is end device */
7951         device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
7952         if (!(_scsih_is_nvme_pciescsi_device(device_info)))
7953                 return;
7954
7955         wwid = le64_to_cpu(pcie_device_pg0.WWID);
7956         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
7957         pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
7958
7959         if (!pcie_device) {
7960                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
7961                 return;
7962         }
7963
7964         if (unlikely(pcie_device->handle != handle)) {
7965                 starget = pcie_device->starget;
7966                 sas_target_priv_data = starget->hostdata;
7967                 pcie_device->access_status = pcie_device_pg0.AccessStatus;
7968                 starget_printk(KERN_INFO, starget,
7969                     "handle changed from(0x%04x) to (0x%04x)!!!\n",
7970                     pcie_device->handle, handle);
7971                 sas_target_priv_data->handle = handle;
7972                 pcie_device->handle = handle;
7973
7974                 if (le32_to_cpu(pcie_device_pg0.Flags) &
7975                     MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
7976                         pcie_device->enclosure_level =
7977                             pcie_device_pg0.EnclosureLevel;
7978                         memcpy(&pcie_device->connector_name[0],
7979                             &pcie_device_pg0.ConnectorName[0], 4);
7980                 } else {
7981                         pcie_device->enclosure_level = 0;
7982                         pcie_device->connector_name[0] = '\0';
7983                 }
7984         }
7985
7986         /* check if device is present */
7987         if (!(le32_to_cpu(pcie_device_pg0.Flags) &
7988             MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT)) {
7989                 ioc_info(ioc, "device is not present handle(0x%04x), flags!!!\n",
7990                          handle);
7991                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
7992                 pcie_device_put(pcie_device);
7993                 return;
7994         }
7995
7996         /* check if there were any issues with discovery */
7997         if (_scsih_check_pcie_access_status(ioc, wwid, handle,
7998             pcie_device_pg0.AccessStatus)) {
7999                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8000                 pcie_device_put(pcie_device);
8001                 return;
8002         }
8003
8004         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8005         pcie_device_put(pcie_device);
8006
8007         _scsih_ublock_io_device(ioc, wwid, NULL);
8008
8009         return;
8010 }
8011
8012 /**
8013  * _scsih_pcie_add_device -  creating pcie device object
8014  * @ioc: per adapter object
8015  * @handle: pcie device handle
8016  *
8017  * Creating end device object, stored in ioc->pcie_device_list.
8018  *
8019  * Return: 1 means queue the event later, 0 means complete the event
8020  */
8021 static int
8022 _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)
8023 {
8024         Mpi26PCIeDevicePage0_t pcie_device_pg0;
8025         Mpi26PCIeDevicePage2_t pcie_device_pg2;
8026         Mpi2ConfigReply_t mpi_reply;
8027         struct _pcie_device *pcie_device;
8028         struct _enclosure_node *enclosure_dev;
8029         u32 ioc_status;
8030         u64 wwid;
8031
8032         if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
8033             &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_HANDLE, handle))) {
8034                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8035                         __FILE__, __LINE__, __func__);
8036                 return 0;
8037         }
8038         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
8039             MPI2_IOCSTATUS_MASK;
8040         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8041                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8042                         __FILE__, __LINE__, __func__);
8043                 return 0;
8044         }
8045
8046         set_bit(handle, ioc->pend_os_device_add);
8047         wwid = le64_to_cpu(pcie_device_pg0.WWID);
8048
8049         /* check if device is present */
8050         if (!(le32_to_cpu(pcie_device_pg0.Flags) &
8051                 MPI26_PCIEDEV0_FLAGS_DEVICE_PRESENT)) {
8052                 ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",
8053                         handle);
8054                 return 0;
8055         }
8056
8057         /* check if there were any issues with discovery */
8058         if (_scsih_check_pcie_access_status(ioc, wwid, handle,
8059             pcie_device_pg0.AccessStatus))
8060                 return 0;
8061
8062         if (!(_scsih_is_nvme_pciescsi_device(le32_to_cpu
8063             (pcie_device_pg0.DeviceInfo))))
8064                 return 0;
8065
8066         pcie_device = mpt3sas_get_pdev_by_wwid(ioc, wwid);
8067         if (pcie_device) {
8068                 clear_bit(handle, ioc->pend_os_device_add);
8069                 pcie_device_put(pcie_device);
8070                 return 0;
8071         }
8072
8073         /* PCIe Device Page 2 contains read-only information about a
8074          * specific NVMe device; therefore, this page is only
8075          * valid for NVMe devices and skip for pcie devices of type scsi.
8076          */
8077         if (!(mpt3sas_scsih_is_pcie_scsi_device(
8078                 le32_to_cpu(pcie_device_pg0.DeviceInfo)))) {
8079                 if (mpt3sas_config_get_pcie_device_pg2(ioc, &mpi_reply,
8080                     &pcie_device_pg2, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
8081                     handle)) {
8082                         ioc_err(ioc,
8083                             "failure at %s:%d/%s()!\n", __FILE__,
8084                             __LINE__, __func__);
8085                         return 0;
8086                 }
8087
8088                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
8089                                         MPI2_IOCSTATUS_MASK;
8090                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8091                         ioc_err(ioc,
8092                             "failure at %s:%d/%s()!\n", __FILE__,
8093                             __LINE__, __func__);
8094                         return 0;
8095                 }
8096         }
8097
8098         pcie_device = kzalloc(sizeof(struct _pcie_device), GFP_KERNEL);
8099         if (!pcie_device) {
8100                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8101                         __FILE__, __LINE__, __func__);
8102                 return 0;
8103         }
8104
8105         kref_init(&pcie_device->refcount);
8106         pcie_device->id = ioc->pcie_target_id++;
8107         pcie_device->channel = PCIE_CHANNEL;
8108         pcie_device->handle = handle;
8109         pcie_device->access_status = pcie_device_pg0.AccessStatus;
8110         pcie_device->device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
8111         pcie_device->wwid = wwid;
8112         pcie_device->port_num = pcie_device_pg0.PortNum;
8113         pcie_device->fast_path = (le32_to_cpu(pcie_device_pg0.Flags) &
8114             MPI26_PCIEDEV0_FLAGS_FAST_PATH_CAPABLE) ? 1 : 0;
8115
8116         pcie_device->enclosure_handle =
8117             le16_to_cpu(pcie_device_pg0.EnclosureHandle);
8118         if (pcie_device->enclosure_handle != 0)
8119                 pcie_device->slot = le16_to_cpu(pcie_device_pg0.Slot);
8120
8121         if (le32_to_cpu(pcie_device_pg0.Flags) &
8122             MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) {
8123                 pcie_device->enclosure_level = pcie_device_pg0.EnclosureLevel;
8124                 memcpy(&pcie_device->connector_name[0],
8125                     &pcie_device_pg0.ConnectorName[0], 4);
8126         } else {
8127                 pcie_device->enclosure_level = 0;
8128                 pcie_device->connector_name[0] = '\0';
8129         }
8130
8131         /* get enclosure_logical_id */
8132         if (pcie_device->enclosure_handle) {
8133                 enclosure_dev =
8134                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
8135                                                 pcie_device->enclosure_handle);
8136                 if (enclosure_dev)
8137                         pcie_device->enclosure_logical_id =
8138                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
8139         }
8140         /* TODO -- Add device name once FW supports it */
8141         if (!(mpt3sas_scsih_is_pcie_scsi_device(
8142             le32_to_cpu(pcie_device_pg0.DeviceInfo)))) {
8143                 pcie_device->nvme_mdts =
8144                     le32_to_cpu(pcie_device_pg2.MaximumDataTransferSize);
8145                 pcie_device->shutdown_latency =
8146                         le16_to_cpu(pcie_device_pg2.ShutdownLatency);
8147                 /*
8148                  * Set IOC's max_shutdown_latency to drive's RTD3 Entry Latency
8149                  * if drive's RTD3 Entry Latency is greater then IOC's
8150                  * max_shutdown_latency.
8151                  */
8152                 if (pcie_device->shutdown_latency > ioc->max_shutdown_latency)
8153                         ioc->max_shutdown_latency =
8154                                 pcie_device->shutdown_latency;
8155                 if (pcie_device_pg2.ControllerResetTO)
8156                         pcie_device->reset_timeout =
8157                             pcie_device_pg2.ControllerResetTO;
8158                 else
8159                         pcie_device->reset_timeout = 30;
8160         } else
8161                 pcie_device->reset_timeout = 30;
8162
8163         if (ioc->wait_for_discovery_to_complete)
8164                 _scsih_pcie_device_init_add(ioc, pcie_device);
8165         else
8166                 _scsih_pcie_device_add(ioc, pcie_device);
8167
8168         pcie_device_put(pcie_device);
8169         return 0;
8170 }
8171
8172 /**
8173  * _scsih_pcie_topology_change_event_debug - debug for topology
8174  * event
8175  * @ioc: per adapter object
8176  * @event_data: event data payload
8177  * Context: user.
8178  */
8179 static void
8180 _scsih_pcie_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8181         Mpi26EventDataPCIeTopologyChangeList_t *event_data)
8182 {
8183         int i;
8184         u16 handle;
8185         u16 reason_code;
8186         u8 port_number;
8187         char *status_str = NULL;
8188         u8 link_rate, prev_link_rate;
8189
8190         switch (event_data->SwitchStatus) {
8191         case MPI26_EVENT_PCIE_TOPO_SS_ADDED:
8192                 status_str = "add";
8193                 break;
8194         case MPI26_EVENT_PCIE_TOPO_SS_NOT_RESPONDING:
8195                 status_str = "remove";
8196                 break;
8197         case MPI26_EVENT_PCIE_TOPO_SS_RESPONDING:
8198         case 0:
8199                 status_str =  "responding";
8200                 break;
8201         case MPI26_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING:
8202                 status_str = "remove delay";
8203                 break;
8204         default:
8205                 status_str = "unknown status";
8206                 break;
8207         }
8208         ioc_info(ioc, "pcie topology change: (%s)\n", status_str);
8209         pr_info("\tswitch_handle(0x%04x), enclosure_handle(0x%04x)"
8210                 "start_port(%02d), count(%d)\n",
8211                 le16_to_cpu(event_data->SwitchDevHandle),
8212                 le16_to_cpu(event_data->EnclosureHandle),
8213                 event_data->StartPortNum, event_data->NumEntries);
8214         for (i = 0; i < event_data->NumEntries; i++) {
8215                 handle =
8216                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
8217                 if (!handle)
8218                         continue;
8219                 port_number = event_data->StartPortNum + i;
8220                 reason_code = event_data->PortEntry[i].PortStatus;
8221                 switch (reason_code) {
8222                 case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
8223                         status_str = "target add";
8224                         break;
8225                 case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
8226                         status_str = "target remove";
8227                         break;
8228                 case MPI26_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
8229                         status_str = "delay target remove";
8230                         break;
8231                 case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
8232                         status_str = "link rate change";
8233                         break;
8234                 case MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE:
8235                         status_str = "target responding";
8236                         break;
8237                 default:
8238                         status_str = "unknown";
8239                         break;
8240                 }
8241                 link_rate = event_data->PortEntry[i].CurrentPortInfo &
8242                         MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8243                 prev_link_rate = event_data->PortEntry[i].PreviousPortInfo &
8244                         MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8245                 pr_info("\tport(%02d), attached_handle(0x%04x): %s:"
8246                         " link rate: new(0x%02x), old(0x%02x)\n", port_number,
8247                         handle, status_str, link_rate, prev_link_rate);
8248         }
8249 }
8250
8251 /**
8252  * _scsih_pcie_topology_change_event - handle PCIe topology
8253  *  changes
8254  * @ioc: per adapter object
8255  * @fw_event: The fw_event_work object
8256  * Context: user.
8257  *
8258  */
8259 static void
8260 _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,
8261         struct fw_event_work *fw_event)
8262 {
8263         int i;
8264         u16 handle;
8265         u16 reason_code;
8266         u8 link_rate, prev_link_rate;
8267         unsigned long flags;
8268         int rc;
8269         Mpi26EventDataPCIeTopologyChangeList_t *event_data =
8270                 (Mpi26EventDataPCIeTopologyChangeList_t *) fw_event->event_data;
8271         struct _pcie_device *pcie_device;
8272
8273         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8274                 _scsih_pcie_topology_change_event_debug(ioc, event_data);
8275
8276         if (ioc->shost_recovery || ioc->remove_host ||
8277                 ioc->pci_error_recovery)
8278                 return;
8279
8280         if (fw_event->ignore) {
8281                 dewtprintk(ioc, ioc_info(ioc, "ignoring switch event\n"));
8282                 return;
8283         }
8284
8285         /* handle siblings events */
8286         for (i = 0; i < event_data->NumEntries; i++) {
8287                 if (fw_event->ignore) {
8288                         dewtprintk(ioc,
8289                                    ioc_info(ioc, "ignoring switch event\n"));
8290                         return;
8291                 }
8292                 if (ioc->remove_host || ioc->pci_error_recovery)
8293                         return;
8294                 reason_code = event_data->PortEntry[i].PortStatus;
8295                 handle =
8296                         le16_to_cpu(event_data->PortEntry[i].AttachedDevHandle);
8297                 if (!handle)
8298                         continue;
8299
8300                 link_rate = event_data->PortEntry[i].CurrentPortInfo
8301                         & MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8302                 prev_link_rate = event_data->PortEntry[i].PreviousPortInfo
8303                         & MPI26_EVENT_PCIE_TOPO_PI_RATE_MASK;
8304
8305                 switch (reason_code) {
8306                 case MPI26_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
8307                         if (ioc->shost_recovery)
8308                                 break;
8309                         if (link_rate == prev_link_rate)
8310                                 break;
8311                         if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
8312                                 break;
8313
8314                         _scsih_pcie_check_device(ioc, handle);
8315
8316                         /* This code after this point handles the test case
8317                          * where a device has been added, however its returning
8318                          * BUSY for sometime.  Then before the Device Missing
8319                          * Delay expires and the device becomes READY, the
8320                          * device is removed and added back.
8321                          */
8322                         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8323                         pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);
8324                         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8325
8326                         if (pcie_device) {
8327                                 pcie_device_put(pcie_device);
8328                                 break;
8329                         }
8330
8331                         if (!test_bit(handle, ioc->pend_os_device_add))
8332                                 break;
8333
8334                         dewtprintk(ioc,
8335                                    ioc_info(ioc, "handle(0x%04x) device not found: convert event to a device add\n",
8336                                             handle));
8337                         event_data->PortEntry[i].PortStatus &= 0xF0;
8338                         event_data->PortEntry[i].PortStatus |=
8339                                 MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED;
8340                         fallthrough;
8341                 case MPI26_EVENT_PCIE_TOPO_PS_DEV_ADDED:
8342                         if (ioc->shost_recovery)
8343                                 break;
8344                         if (link_rate < MPI26_EVENT_PCIE_TOPO_PI_RATE_2_5)
8345                                 break;
8346
8347                         rc = _scsih_pcie_add_device(ioc, handle);
8348                         if (!rc) {
8349                                 /* mark entry vacant */
8350                                 /* TODO This needs to be reviewed and fixed,
8351                                  * we dont have an entry
8352                                  * to make an event void like vacant
8353                                  */
8354                                 event_data->PortEntry[i].PortStatus |=
8355                                         MPI26_EVENT_PCIE_TOPO_PS_NO_CHANGE;
8356                         }
8357                         break;
8358                 case MPI26_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
8359                         _scsih_pcie_device_remove_by_handle(ioc, handle);
8360                         break;
8361                 }
8362         }
8363 }
8364
8365 /**
8366  * _scsih_pcie_device_status_change_event_debug - debug for device event
8367  * @ioc: ?
8368  * @event_data: event data payload
8369  * Context: user.
8370  */
8371 static void
8372 _scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8373         Mpi26EventDataPCIeDeviceStatusChange_t *event_data)
8374 {
8375         char *reason_str = NULL;
8376
8377         switch (event_data->ReasonCode) {
8378         case MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA:
8379                 reason_str = "smart data";
8380                 break;
8381         case MPI26_EVENT_PCIDEV_STAT_RC_UNSUPPORTED:
8382                 reason_str = "unsupported device discovered";
8383                 break;
8384         case MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET:
8385                 reason_str = "internal device reset";
8386                 break;
8387         case MPI26_EVENT_PCIDEV_STAT_RC_TASK_ABORT_INTERNAL:
8388                 reason_str = "internal task abort";
8389                 break;
8390         case MPI26_EVENT_PCIDEV_STAT_RC_ABORT_TASK_SET_INTERNAL:
8391                 reason_str = "internal task abort set";
8392                 break;
8393         case MPI26_EVENT_PCIDEV_STAT_RC_CLEAR_TASK_SET_INTERNAL:
8394                 reason_str = "internal clear task set";
8395                 break;
8396         case MPI26_EVENT_PCIDEV_STAT_RC_QUERY_TASK_INTERNAL:
8397                 reason_str = "internal query task";
8398                 break;
8399         case MPI26_EVENT_PCIDEV_STAT_RC_DEV_INIT_FAILURE:
8400                 reason_str = "device init failure";
8401                 break;
8402         case MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
8403                 reason_str = "internal device reset complete";
8404                 break;
8405         case MPI26_EVENT_PCIDEV_STAT_RC_CMP_TASK_ABORT_INTERNAL:
8406                 reason_str = "internal task abort complete";
8407                 break;
8408         case MPI26_EVENT_PCIDEV_STAT_RC_ASYNC_NOTIFICATION:
8409                 reason_str = "internal async notification";
8410                 break;
8411         case MPI26_EVENT_PCIDEV_STAT_RC_PCIE_HOT_RESET_FAILED:
8412                 reason_str = "pcie hot reset failed";
8413                 break;
8414         default:
8415                 reason_str = "unknown reason";
8416                 break;
8417         }
8418
8419         ioc_info(ioc, "PCIE device status change: (%s)\n"
8420                  "\thandle(0x%04x), WWID(0x%016llx), tag(%d)",
8421                  reason_str, le16_to_cpu(event_data->DevHandle),
8422                  (u64)le64_to_cpu(event_data->WWID),
8423                  le16_to_cpu(event_data->TaskTag));
8424         if (event_data->ReasonCode == MPI26_EVENT_PCIDEV_STAT_RC_SMART_DATA)
8425                 pr_cont(", ASC(0x%x), ASCQ(0x%x)\n",
8426                         event_data->ASC, event_data->ASCQ);
8427         pr_cont("\n");
8428 }
8429
8430 /**
8431  * _scsih_pcie_device_status_change_event - handle device status
8432  * change
8433  * @ioc: per adapter object
8434  * @fw_event: The fw_event_work object
8435  * Context: user.
8436  */
8437 static void
8438 _scsih_pcie_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,
8439         struct fw_event_work *fw_event)
8440 {
8441         struct MPT3SAS_TARGET *target_priv_data;
8442         struct _pcie_device *pcie_device;
8443         u64 wwid;
8444         unsigned long flags;
8445         Mpi26EventDataPCIeDeviceStatusChange_t *event_data =
8446                 (Mpi26EventDataPCIeDeviceStatusChange_t *)fw_event->event_data;
8447         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8448                 _scsih_pcie_device_status_change_event_debug(ioc,
8449                         event_data);
8450
8451         if (event_data->ReasonCode !=
8452                 MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET &&
8453                 event_data->ReasonCode !=
8454                 MPI26_EVENT_PCIDEV_STAT_RC_CMP_INTERNAL_DEV_RESET)
8455                 return;
8456
8457         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
8458         wwid = le64_to_cpu(event_data->WWID);
8459         pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);
8460
8461         if (!pcie_device || !pcie_device->starget)
8462                 goto out;
8463
8464         target_priv_data = pcie_device->starget->hostdata;
8465         if (!target_priv_data)
8466                 goto out;
8467
8468         if (event_data->ReasonCode ==
8469                 MPI26_EVENT_PCIDEV_STAT_RC_INTERNAL_DEVICE_RESET)
8470                 target_priv_data->tm_busy = 1;
8471         else
8472                 target_priv_data->tm_busy = 0;
8473 out:
8474         if (pcie_device)
8475                 pcie_device_put(pcie_device);
8476
8477         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
8478 }
8479
8480 /**
8481  * _scsih_sas_enclosure_dev_status_change_event_debug - debug for enclosure
8482  * event
8483  * @ioc: per adapter object
8484  * @event_data: event data payload
8485  * Context: user.
8486  */
8487 static void
8488 _scsih_sas_enclosure_dev_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
8489         Mpi2EventDataSasEnclDevStatusChange_t *event_data)
8490 {
8491         char *reason_str = NULL;
8492
8493         switch (event_data->ReasonCode) {
8494         case MPI2_EVENT_SAS_ENCL_RC_ADDED:
8495                 reason_str = "enclosure add";
8496                 break;
8497         case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
8498                 reason_str = "enclosure remove";
8499                 break;
8500         default:
8501                 reason_str = "unknown reason";
8502                 break;
8503         }
8504
8505         ioc_info(ioc, "enclosure status change: (%s)\n"
8506                  "\thandle(0x%04x), enclosure logical id(0x%016llx) number slots(%d)\n",
8507                  reason_str,
8508                  le16_to_cpu(event_data->EnclosureHandle),
8509                  (u64)le64_to_cpu(event_data->EnclosureLogicalID),
8510                  le16_to_cpu(event_data->StartSlot));
8511 }
8512
8513 /**
8514  * _scsih_sas_enclosure_dev_status_change_event - handle enclosure events
8515  * @ioc: per adapter object
8516  * @fw_event: The fw_event_work object
8517  * Context: user.
8518  */
8519 static void
8520 _scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,
8521         struct fw_event_work *fw_event)
8522 {
8523         Mpi2ConfigReply_t mpi_reply;
8524         struct _enclosure_node *enclosure_dev = NULL;
8525         Mpi2EventDataSasEnclDevStatusChange_t *event_data =
8526                 (Mpi2EventDataSasEnclDevStatusChange_t *)fw_event->event_data;
8527         int rc;
8528         u16 enclosure_handle = le16_to_cpu(event_data->EnclosureHandle);
8529
8530         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
8531                 _scsih_sas_enclosure_dev_status_change_event_debug(ioc,
8532                      (Mpi2EventDataSasEnclDevStatusChange_t *)
8533                      fw_event->event_data);
8534         if (ioc->shost_recovery)
8535                 return;
8536
8537         if (enclosure_handle)
8538                 enclosure_dev =
8539                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
8540                                                 enclosure_handle);
8541         switch (event_data->ReasonCode) {
8542         case MPI2_EVENT_SAS_ENCL_RC_ADDED:
8543                 if (!enclosure_dev) {
8544                         enclosure_dev =
8545                                 kzalloc(sizeof(struct _enclosure_node),
8546                                         GFP_KERNEL);
8547                         if (!enclosure_dev) {
8548                                 ioc_info(ioc, "failure at %s:%d/%s()!\n",
8549                                          __FILE__, __LINE__, __func__);
8550                                 return;
8551                         }
8552                         rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
8553                                 &enclosure_dev->pg0,
8554                                 MPI2_SAS_ENCLOS_PGAD_FORM_HANDLE,
8555                                 enclosure_handle);
8556
8557                         if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
8558                                                 MPI2_IOCSTATUS_MASK)) {
8559                                 kfree(enclosure_dev);
8560                                 return;
8561                         }
8562
8563                         list_add_tail(&enclosure_dev->list,
8564                                                         &ioc->enclosure_list);
8565                 }
8566                 break;
8567         case MPI2_EVENT_SAS_ENCL_RC_NOT_RESPONDING:
8568                 if (enclosure_dev) {
8569                         list_del(&enclosure_dev->list);
8570                         kfree(enclosure_dev);
8571                 }
8572                 break;
8573         default:
8574                 break;
8575         }
8576 }
8577
8578 /**
8579  * _scsih_sas_broadcast_primitive_event - handle broadcast events
8580  * @ioc: per adapter object
8581  * @fw_event: The fw_event_work object
8582  * Context: user.
8583  */
8584 static void
8585 _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,
8586         struct fw_event_work *fw_event)
8587 {
8588         struct scsi_cmnd *scmd;
8589         struct scsi_device *sdev;
8590         struct scsiio_tracker *st;
8591         u16 smid, handle;
8592         u32 lun;
8593         struct MPT3SAS_DEVICE *sas_device_priv_data;
8594         u32 termination_count;
8595         u32 query_count;
8596         Mpi2SCSITaskManagementReply_t *mpi_reply;
8597         Mpi2EventDataSasBroadcastPrimitive_t *event_data =
8598                 (Mpi2EventDataSasBroadcastPrimitive_t *)
8599                 fw_event->event_data;
8600         u16 ioc_status;
8601         unsigned long flags;
8602         int r;
8603         u8 max_retries = 0;
8604         u8 task_abort_retries;
8605
8606         mutex_lock(&ioc->tm_cmds.mutex);
8607         ioc_info(ioc, "%s: enter: phy number(%d), width(%d)\n",
8608                  __func__, event_data->PhyNum, event_data->PortWidth);
8609
8610         _scsih_block_io_all_device(ioc);
8611
8612         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8613         mpi_reply = ioc->tm_cmds.reply;
8614  broadcast_aen_retry:
8615
8616         /* sanity checks for retrying this loop */
8617         if (max_retries++ == 5) {
8618                 dewtprintk(ioc, ioc_info(ioc, "%s: giving up\n", __func__));
8619                 goto out;
8620         } else if (max_retries > 1)
8621                 dewtprintk(ioc,
8622                            ioc_info(ioc, "%s: %d retry\n",
8623                                     __func__, max_retries - 1));
8624
8625         termination_count = 0;
8626         query_count = 0;
8627         for (smid = 1; smid <= ioc->scsiio_depth; smid++) {
8628                 if (ioc->shost_recovery)
8629                         goto out;
8630                 scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
8631                 if (!scmd)
8632                         continue;
8633                 st = scsi_cmd_priv(scmd);
8634                 sdev = scmd->device;
8635                 sas_device_priv_data = sdev->hostdata;
8636                 if (!sas_device_priv_data || !sas_device_priv_data->sas_target)
8637                         continue;
8638                  /* skip hidden raid components */
8639                 if (sas_device_priv_data->sas_target->flags &
8640                     MPT_TARGET_FLAGS_RAID_COMPONENT)
8641                         continue;
8642                  /* skip volumes */
8643                 if (sas_device_priv_data->sas_target->flags &
8644                     MPT_TARGET_FLAGS_VOLUME)
8645                         continue;
8646                  /* skip PCIe devices */
8647                 if (sas_device_priv_data->sas_target->flags &
8648                     MPT_TARGET_FLAGS_PCIE_DEVICE)
8649                         continue;
8650
8651                 handle = sas_device_priv_data->sas_target->handle;
8652                 lun = sas_device_priv_data->lun;
8653                 query_count++;
8654
8655                 if (ioc->shost_recovery)
8656                         goto out;
8657
8658                 spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
8659                 r = mpt3sas_scsih_issue_tm(ioc, handle, 0, 0, lun,
8660                         MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK, st->smid,
8661                         st->msix_io, 30, 0);
8662                 if (r == FAILED) {
8663                         sdev_printk(KERN_WARNING, sdev,
8664                             "mpt3sas_scsih_issue_tm: FAILED when sending "
8665                             "QUERY_TASK: scmd(%p)\n", scmd);
8666                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8667                         goto broadcast_aen_retry;
8668                 }
8669                 ioc_status = le16_to_cpu(mpi_reply->IOCStatus)
8670                     & MPI2_IOCSTATUS_MASK;
8671                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8672                         sdev_printk(KERN_WARNING, sdev,
8673                                 "query task: FAILED with IOCSTATUS(0x%04x), scmd(%p)\n",
8674                                 ioc_status, scmd);
8675                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8676                         goto broadcast_aen_retry;
8677                 }
8678
8679                 /* see if IO is still owned by IOC and target */
8680                 if (mpi_reply->ResponseCode ==
8681                      MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED ||
8682                      mpi_reply->ResponseCode ==
8683                      MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC) {
8684                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8685                         continue;
8686                 }
8687                 task_abort_retries = 0;
8688  tm_retry:
8689                 if (task_abort_retries++ == 60) {
8690                         dewtprintk(ioc,
8691                                    ioc_info(ioc, "%s: ABORT_TASK: giving up\n",
8692                                             __func__));
8693                         spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8694                         goto broadcast_aen_retry;
8695                 }
8696
8697                 if (ioc->shost_recovery)
8698                         goto out_no_lock;
8699
8700                 r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->channel, sdev->id,
8701                         sdev->lun, MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
8702                         st->smid, st->msix_io, 30, 0);
8703                 if (r == FAILED || st->cb_idx != 0xFF) {
8704                         sdev_printk(KERN_WARNING, sdev,
8705                             "mpt3sas_scsih_issue_tm: ABORT_TASK: FAILED : "
8706                             "scmd(%p)\n", scmd);
8707                         goto tm_retry;
8708                 }
8709
8710                 if (task_abort_retries > 1)
8711                         sdev_printk(KERN_WARNING, sdev,
8712                             "mpt3sas_scsih_issue_tm: ABORT_TASK: RETRIES (%d):"
8713                             " scmd(%p)\n",
8714                             task_abort_retries - 1, scmd);
8715
8716                 termination_count += le32_to_cpu(mpi_reply->TerminationCount);
8717                 spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
8718         }
8719
8720         if (ioc->broadcast_aen_pending) {
8721                 dewtprintk(ioc,
8722                            ioc_info(ioc,
8723                                     "%s: loop back due to pending AEN\n",
8724                                     __func__));
8725                  ioc->broadcast_aen_pending = 0;
8726                  goto broadcast_aen_retry;
8727         }
8728
8729  out:
8730         spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
8731  out_no_lock:
8732
8733         dewtprintk(ioc,
8734                    ioc_info(ioc, "%s - exit, query_count = %d termination_count = %d\n",
8735                             __func__, query_count, termination_count));
8736
8737         ioc->broadcast_aen_busy = 0;
8738         if (!ioc->shost_recovery)
8739                 _scsih_ublock_io_all_device(ioc);
8740         mutex_unlock(&ioc->tm_cmds.mutex);
8741 }
8742
8743 /**
8744  * _scsih_sas_discovery_event - handle discovery events
8745  * @ioc: per adapter object
8746  * @fw_event: The fw_event_work object
8747  * Context: user.
8748  */
8749 static void
8750 _scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,
8751         struct fw_event_work *fw_event)
8752 {
8753         Mpi2EventDataSasDiscovery_t *event_data =
8754                 (Mpi2EventDataSasDiscovery_t *) fw_event->event_data;
8755
8756         if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) {
8757                 ioc_info(ioc, "discovery event: (%s)",
8758                          event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ?
8759                          "start" : "stop");
8760                 if (event_data->DiscoveryStatus)
8761                         pr_cont("discovery_status(0x%08x)",
8762                                 le32_to_cpu(event_data->DiscoveryStatus));
8763                 pr_cont("\n");
8764         }
8765
8766         if (event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED &&
8767             !ioc->sas_hba.num_phys) {
8768                 if (disable_discovery > 0 && ioc->shost_recovery) {
8769                         /* Wait for the reset to complete */
8770                         while (ioc->shost_recovery)
8771                                 ssleep(1);
8772                 }
8773                 _scsih_sas_host_add(ioc);
8774         }
8775 }
8776
8777 /**
8778  * _scsih_sas_device_discovery_error_event - display SAS device discovery error
8779  *                                              events
8780  * @ioc: per adapter object
8781  * @fw_event: The fw_event_work object
8782  * Context: user.
8783  */
8784 static void
8785 _scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,
8786         struct fw_event_work *fw_event)
8787 {
8788         Mpi25EventDataSasDeviceDiscoveryError_t *event_data =
8789                 (Mpi25EventDataSasDeviceDiscoveryError_t *)fw_event->event_data;
8790
8791         switch (event_data->ReasonCode) {
8792         case MPI25_EVENT_SAS_DISC_ERR_SMP_FAILED:
8793                 ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_port:0x%02x) has failed\n",
8794                          le16_to_cpu(event_data->DevHandle),
8795                          (u64)le64_to_cpu(event_data->SASAddress),
8796                          event_data->PhysicalPort);
8797                 break;
8798         case MPI25_EVENT_SAS_DISC_ERR_SMP_TIMEOUT:
8799                 ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_port:0x%02x) has timed out\n",
8800                          le16_to_cpu(event_data->DevHandle),
8801                          (u64)le64_to_cpu(event_data->SASAddress),
8802                          event_data->PhysicalPort);
8803                 break;
8804         default:
8805                 break;
8806         }
8807 }
8808
8809 /**
8810  * _scsih_pcie_enumeration_event - handle enumeration events
8811  * @ioc: per adapter object
8812  * @fw_event: The fw_event_work object
8813  * Context: user.
8814  */
8815 static void
8816 _scsih_pcie_enumeration_event(struct MPT3SAS_ADAPTER *ioc,
8817         struct fw_event_work *fw_event)
8818 {
8819         Mpi26EventDataPCIeEnumeration_t *event_data =
8820                 (Mpi26EventDataPCIeEnumeration_t *)fw_event->event_data;
8821
8822         if (!(ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK))
8823                 return;
8824
8825         ioc_info(ioc, "pcie enumeration event: (%s) Flag 0x%02x",
8826                  (event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED) ?
8827                  "started" : "completed",
8828                  event_data->Flags);
8829         if (event_data->EnumerationStatus)
8830                 pr_cont("enumeration_status(0x%08x)",
8831                         le32_to_cpu(event_data->EnumerationStatus));
8832         pr_cont("\n");
8833 }
8834
8835 /**
8836  * _scsih_ir_fastpath - turn on fastpath for IR physdisk
8837  * @ioc: per adapter object
8838  * @handle: device handle for physical disk
8839  * @phys_disk_num: physical disk number
8840  *
8841  * Return: 0 for success, else failure.
8842  */
8843 static int
8844 _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)
8845 {
8846         Mpi2RaidActionRequest_t *mpi_request;
8847         Mpi2RaidActionReply_t *mpi_reply;
8848         u16 smid;
8849         u8 issue_reset = 0;
8850         int rc = 0;
8851         u16 ioc_status;
8852         u32 log_info;
8853
8854         if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
8855                 return rc;
8856
8857         mutex_lock(&ioc->scsih_cmds.mutex);
8858
8859         if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
8860                 ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
8861                 rc = -EAGAIN;
8862                 goto out;
8863         }
8864         ioc->scsih_cmds.status = MPT3_CMD_PENDING;
8865
8866         smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
8867         if (!smid) {
8868                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
8869                 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8870                 rc = -EAGAIN;
8871                 goto out;
8872         }
8873
8874         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
8875         ioc->scsih_cmds.smid = smid;
8876         memset(mpi_request, 0, sizeof(Mpi2RaidActionRequest_t));
8877
8878         mpi_request->Function = MPI2_FUNCTION_RAID_ACTION;
8879         mpi_request->Action = MPI2_RAID_ACTION_PHYSDISK_HIDDEN;
8880         mpi_request->PhysDiskNum = phys_disk_num;
8881
8882         dewtprintk(ioc,
8883                    ioc_info(ioc, "IR RAID_ACTION: turning fast path on for handle(0x%04x), phys_disk_num (0x%02x)\n",
8884                             handle, phys_disk_num));
8885
8886         init_completion(&ioc->scsih_cmds.done);
8887         ioc->put_smid_default(ioc, smid);
8888         wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);
8889
8890         if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
8891                 mpt3sas_check_cmd_timeout(ioc,
8892                     ioc->scsih_cmds.status, mpi_request,
8893                     sizeof(Mpi2RaidActionRequest_t)/4, issue_reset);
8894                 rc = -EFAULT;
8895                 goto out;
8896         }
8897
8898         if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
8899
8900                 mpi_reply = ioc->scsih_cmds.reply;
8901                 ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
8902                 if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE)
8903                         log_info =  le32_to_cpu(mpi_reply->IOCLogInfo);
8904                 else
8905                         log_info = 0;
8906                 ioc_status &= MPI2_IOCSTATUS_MASK;
8907                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
8908                         dewtprintk(ioc,
8909                                    ioc_info(ioc, "IR RAID_ACTION: failed: ioc_status(0x%04x), loginfo(0x%08x)!!!\n",
8910                                             ioc_status, log_info));
8911                         rc = -EFAULT;
8912                 } else
8913                         dewtprintk(ioc,
8914                                    ioc_info(ioc, "IR RAID_ACTION: completed successfully\n"));
8915         }
8916
8917  out:
8918         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8919         mutex_unlock(&ioc->scsih_cmds.mutex);
8920
8921         if (issue_reset)
8922                 mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
8923         return rc;
8924 }
8925
8926 /**
8927  * _scsih_reprobe_lun - reprobing lun
8928  * @sdev: scsi device struct
8929  * @no_uld_attach: sdev->no_uld_attach flag setting
8930  *
8931  **/
8932 static void
8933 _scsih_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
8934 {
8935         sdev->no_uld_attach = no_uld_attach ? 1 : 0;
8936         sdev_printk(KERN_INFO, sdev, "%s raid component\n",
8937             sdev->no_uld_attach ? "hiding" : "exposing");
8938         WARN_ON(scsi_device_reprobe(sdev));
8939 }
8940
8941 /**
8942  * _scsih_sas_volume_add - add new volume
8943  * @ioc: per adapter object
8944  * @element: IR config element data
8945  * Context: user.
8946  */
8947 static void
8948 _scsih_sas_volume_add(struct MPT3SAS_ADAPTER *ioc,
8949         Mpi2EventIrConfigElement_t *element)
8950 {
8951         struct _raid_device *raid_device;
8952         unsigned long flags;
8953         u64 wwid;
8954         u16 handle = le16_to_cpu(element->VolDevHandle);
8955         int rc;
8956
8957         mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);
8958         if (!wwid) {
8959                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8960                         __FILE__, __LINE__, __func__);
8961                 return;
8962         }
8963
8964         spin_lock_irqsave(&ioc->raid_device_lock, flags);
8965         raid_device = _scsih_raid_device_find_by_wwid(ioc, wwid);
8966         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
8967
8968         if (raid_device)
8969                 return;
8970
8971         raid_device = kzalloc(sizeof(struct _raid_device), GFP_KERNEL);
8972         if (!raid_device) {
8973                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
8974                         __FILE__, __LINE__, __func__);
8975                 return;
8976         }
8977
8978         raid_device->id = ioc->sas_id++;
8979         raid_device->channel = RAID_CHANNEL;
8980         raid_device->handle = handle;
8981         raid_device->wwid = wwid;
8982         _scsih_raid_device_add(ioc, raid_device);
8983         if (!ioc->wait_for_discovery_to_complete) {
8984                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
8985                     raid_device->id, 0);
8986                 if (rc)
8987                         _scsih_raid_device_remove(ioc, raid_device);
8988         } else {
8989                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
8990                 _scsih_determine_boot_device(ioc, raid_device, 1);
8991                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
8992         }
8993 }
8994
8995 /**
8996  * _scsih_sas_volume_delete - delete volume
8997  * @ioc: per adapter object
8998  * @handle: volume device handle
8999  * Context: user.
9000  */
9001 static void
9002 _scsih_sas_volume_delete(struct MPT3SAS_ADAPTER *ioc, u16 handle)
9003 {
9004         struct _raid_device *raid_device;
9005         unsigned long flags;
9006         struct MPT3SAS_TARGET *sas_target_priv_data;
9007         struct scsi_target *starget = NULL;
9008
9009         spin_lock_irqsave(&ioc->raid_device_lock, flags);
9010         raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9011         if (raid_device) {
9012                 if (raid_device->starget) {
9013                         starget = raid_device->starget;
9014                         sas_target_priv_data = starget->hostdata;
9015                         sas_target_priv_data->deleted = 1;
9016                 }
9017                 ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
9018                          raid_device->handle, (u64)raid_device->wwid);
9019                 list_del(&raid_device->list);
9020                 kfree(raid_device);
9021         }
9022         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9023         if (starget)
9024                 scsi_remove_target(&starget->dev);
9025 }
9026
9027 /**
9028  * _scsih_sas_pd_expose - expose pd component to /dev/sdX
9029  * @ioc: per adapter object
9030  * @element: IR config element data
9031  * Context: user.
9032  */
9033 static void
9034 _scsih_sas_pd_expose(struct MPT3SAS_ADAPTER *ioc,
9035         Mpi2EventIrConfigElement_t *element)
9036 {
9037         struct _sas_device *sas_device;
9038         struct scsi_target *starget = NULL;
9039         struct MPT3SAS_TARGET *sas_target_priv_data;
9040         unsigned long flags;
9041         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9042
9043         spin_lock_irqsave(&ioc->sas_device_lock, flags);
9044         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
9045         if (sas_device) {
9046                 sas_device->volume_handle = 0;
9047                 sas_device->volume_wwid = 0;
9048                 clear_bit(handle, ioc->pd_handles);
9049                 if (sas_device->starget && sas_device->starget->hostdata) {
9050                         starget = sas_device->starget;
9051                         sas_target_priv_data = starget->hostdata;
9052                         sas_target_priv_data->flags &=
9053                             ~MPT_TARGET_FLAGS_RAID_COMPONENT;
9054                 }
9055         }
9056         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
9057         if (!sas_device)
9058                 return;
9059
9060         /* exposing raid component */
9061         if (starget)
9062                 starget_for_each_device(starget, NULL, _scsih_reprobe_lun);
9063
9064         sas_device_put(sas_device);
9065 }
9066
9067 /**
9068  * _scsih_sas_pd_hide - hide pd component from /dev/sdX
9069  * @ioc: per adapter object
9070  * @element: IR config element data
9071  * Context: user.
9072  */
9073 static void
9074 _scsih_sas_pd_hide(struct MPT3SAS_ADAPTER *ioc,
9075         Mpi2EventIrConfigElement_t *element)
9076 {
9077         struct _sas_device *sas_device;
9078         struct scsi_target *starget = NULL;
9079         struct MPT3SAS_TARGET *sas_target_priv_data;
9080         unsigned long flags;
9081         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9082         u16 volume_handle = 0;
9083         u64 volume_wwid = 0;
9084
9085         mpt3sas_config_get_volume_handle(ioc, handle, &volume_handle);
9086         if (volume_handle)
9087                 mpt3sas_config_get_volume_wwid(ioc, volume_handle,
9088                     &volume_wwid);
9089
9090         spin_lock_irqsave(&ioc->sas_device_lock, flags);
9091         sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);
9092         if (sas_device) {
9093                 set_bit(handle, ioc->pd_handles);
9094                 if (sas_device->starget && sas_device->starget->hostdata) {
9095                         starget = sas_device->starget;
9096                         sas_target_priv_data = starget->hostdata;
9097                         sas_target_priv_data->flags |=
9098                             MPT_TARGET_FLAGS_RAID_COMPONENT;
9099                         sas_device->volume_handle = volume_handle;
9100                         sas_device->volume_wwid = volume_wwid;
9101                 }
9102         }
9103         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
9104         if (!sas_device)
9105                 return;
9106
9107         /* hiding raid component */
9108         _scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);
9109
9110         if (starget)
9111                 starget_for_each_device(starget, (void *)1, _scsih_reprobe_lun);
9112
9113         sas_device_put(sas_device);
9114 }
9115
9116 /**
9117  * _scsih_sas_pd_delete - delete pd component
9118  * @ioc: per adapter object
9119  * @element: IR config element data
9120  * Context: user.
9121  */
9122 static void
9123 _scsih_sas_pd_delete(struct MPT3SAS_ADAPTER *ioc,
9124         Mpi2EventIrConfigElement_t *element)
9125 {
9126         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9127
9128         _scsih_device_remove_by_handle(ioc, handle);
9129 }
9130
9131 /**
9132  * _scsih_sas_pd_add - remove pd component
9133  * @ioc: per adapter object
9134  * @element: IR config element data
9135  * Context: user.
9136  */
9137 static void
9138 _scsih_sas_pd_add(struct MPT3SAS_ADAPTER *ioc,
9139         Mpi2EventIrConfigElement_t *element)
9140 {
9141         struct _sas_device *sas_device;
9142         u16 handle = le16_to_cpu(element->PhysDiskDevHandle);
9143         Mpi2ConfigReply_t mpi_reply;
9144         Mpi2SasDevicePage0_t sas_device_pg0;
9145         u32 ioc_status;
9146         u64 sas_address;
9147         u16 parent_handle;
9148
9149         set_bit(handle, ioc->pd_handles);
9150
9151         sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
9152         if (sas_device) {
9153                 _scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);
9154                 sas_device_put(sas_device);
9155                 return;
9156         }
9157
9158         if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,
9159             MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, handle))) {
9160                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
9161                         __FILE__, __LINE__, __func__);
9162                 return;
9163         }
9164
9165         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9166             MPI2_IOCSTATUS_MASK;
9167         if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9168                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
9169                         __FILE__, __LINE__, __func__);
9170                 return;
9171         }
9172
9173         parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
9174         if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))
9175                 mpt3sas_transport_update_links(ioc, sas_address, handle,
9176                     sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
9177                     mpt3sas_get_port_by_id(ioc,
9178                     sas_device_pg0.PhysicalPort, 0));
9179
9180         _scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);
9181         _scsih_add_device(ioc, handle, 0, 1);
9182 }
9183
9184 /**
9185  * _scsih_sas_ir_config_change_event_debug - debug for IR Config Change events
9186  * @ioc: per adapter object
9187  * @event_data: event data payload
9188  * Context: user.
9189  */
9190 static void
9191 _scsih_sas_ir_config_change_event_debug(struct MPT3SAS_ADAPTER *ioc,
9192         Mpi2EventDataIrConfigChangeList_t *event_data)
9193 {
9194         Mpi2EventIrConfigElement_t *element;
9195         u8 element_type;
9196         int i;
9197         char *reason_str = NULL, *element_str = NULL;
9198
9199         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
9200
9201         ioc_info(ioc, "raid config change: (%s), elements(%d)\n",
9202                  le32_to_cpu(event_data->Flags) & MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG ?
9203                  "foreign" : "native",
9204                  event_data->NumElements);
9205         for (i = 0; i < event_data->NumElements; i++, element++) {
9206                 switch (element->ReasonCode) {
9207                 case MPI2_EVENT_IR_CHANGE_RC_ADDED:
9208                         reason_str = "add";
9209                         break;
9210                 case MPI2_EVENT_IR_CHANGE_RC_REMOVED:
9211                         reason_str = "remove";
9212                         break;
9213                 case MPI2_EVENT_IR_CHANGE_RC_NO_CHANGE:
9214                         reason_str = "no change";
9215                         break;
9216                 case MPI2_EVENT_IR_CHANGE_RC_HIDE:
9217                         reason_str = "hide";
9218                         break;
9219                 case MPI2_EVENT_IR_CHANGE_RC_UNHIDE:
9220                         reason_str = "unhide";
9221                         break;
9222                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED:
9223                         reason_str = "volume_created";
9224                         break;
9225                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED:
9226                         reason_str = "volume_deleted";
9227                         break;
9228                 case MPI2_EVENT_IR_CHANGE_RC_PD_CREATED:
9229                         reason_str = "pd_created";
9230                         break;
9231                 case MPI2_EVENT_IR_CHANGE_RC_PD_DELETED:
9232                         reason_str = "pd_deleted";
9233                         break;
9234                 default:
9235                         reason_str = "unknown reason";
9236                         break;
9237                 }
9238                 element_type = le16_to_cpu(element->ElementFlags) &
9239                     MPI2_EVENT_IR_CHANGE_EFLAGS_ELEMENT_TYPE_MASK;
9240                 switch (element_type) {
9241                 case MPI2_EVENT_IR_CHANGE_EFLAGS_VOLUME_ELEMENT:
9242                         element_str = "volume";
9243                         break;
9244                 case MPI2_EVENT_IR_CHANGE_EFLAGS_VOLPHYSDISK_ELEMENT:
9245                         element_str = "phys disk";
9246                         break;
9247                 case MPI2_EVENT_IR_CHANGE_EFLAGS_HOTSPARE_ELEMENT:
9248                         element_str = "hot spare";
9249                         break;
9250                 default:
9251                         element_str = "unknown element";
9252                         break;
9253                 }
9254                 pr_info("\t(%s:%s), vol handle(0x%04x), " \
9255                     "pd handle(0x%04x), pd num(0x%02x)\n", element_str,
9256                     reason_str, le16_to_cpu(element->VolDevHandle),
9257                     le16_to_cpu(element->PhysDiskDevHandle),
9258                     element->PhysDiskNum);
9259         }
9260 }
9261
9262 /**
9263  * _scsih_sas_ir_config_change_event - handle ir configuration change events
9264  * @ioc: per adapter object
9265  * @fw_event: The fw_event_work object
9266  * Context: user.
9267  */
9268 static void
9269 _scsih_sas_ir_config_change_event(struct MPT3SAS_ADAPTER *ioc,
9270         struct fw_event_work *fw_event)
9271 {
9272         Mpi2EventIrConfigElement_t *element;
9273         int i;
9274         u8 foreign_config;
9275         Mpi2EventDataIrConfigChangeList_t *event_data =
9276                 (Mpi2EventDataIrConfigChangeList_t *)
9277                 fw_event->event_data;
9278
9279         if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&
9280              (!ioc->hide_ir_msg))
9281                 _scsih_sas_ir_config_change_event_debug(ioc, event_data);
9282
9283         foreign_config = (le32_to_cpu(event_data->Flags) &
9284             MPI2_EVENT_IR_CHANGE_FLAGS_FOREIGN_CONFIG) ? 1 : 0;
9285
9286         element = (Mpi2EventIrConfigElement_t *)&event_data->ConfigElement[0];
9287         if (ioc->shost_recovery &&
9288             ioc->hba_mpi_version_belonged != MPI2_VERSION) {
9289                 for (i = 0; i < event_data->NumElements; i++, element++) {
9290                         if (element->ReasonCode == MPI2_EVENT_IR_CHANGE_RC_HIDE)
9291                                 _scsih_ir_fastpath(ioc,
9292                                         le16_to_cpu(element->PhysDiskDevHandle),
9293                                         element->PhysDiskNum);
9294                 }
9295                 return;
9296         }
9297
9298         for (i = 0; i < event_data->NumElements; i++, element++) {
9299
9300                 switch (element->ReasonCode) {
9301                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_CREATED:
9302                 case MPI2_EVENT_IR_CHANGE_RC_ADDED:
9303                         if (!foreign_config)
9304                                 _scsih_sas_volume_add(ioc, element);
9305                         break;
9306                 case MPI2_EVENT_IR_CHANGE_RC_VOLUME_DELETED:
9307                 case MPI2_EVENT_IR_CHANGE_RC_REMOVED:
9308                         if (!foreign_config)
9309                                 _scsih_sas_volume_delete(ioc,
9310                                     le16_to_cpu(element->VolDevHandle));
9311                         break;
9312                 case MPI2_EVENT_IR_CHANGE_RC_PD_CREATED:
9313                         if (!ioc->is_warpdrive)
9314                                 _scsih_sas_pd_hide(ioc, element);
9315                         break;
9316                 case MPI2_EVENT_IR_CHANGE_RC_PD_DELETED:
9317                         if (!ioc->is_warpdrive)
9318                                 _scsih_sas_pd_expose(ioc, element);
9319                         break;
9320                 case MPI2_EVENT_IR_CHANGE_RC_HIDE:
9321                         if (!ioc->is_warpdrive)
9322                                 _scsih_sas_pd_add(ioc, element);
9323                         break;
9324                 case MPI2_EVENT_IR_CHANGE_RC_UNHIDE:
9325                         if (!ioc->is_warpdrive)
9326                                 _scsih_sas_pd_delete(ioc, element);
9327                         break;
9328                 }
9329         }
9330 }
9331
9332 /**
9333  * _scsih_sas_ir_volume_event - IR volume event
9334  * @ioc: per adapter object
9335  * @fw_event: The fw_event_work object
9336  * Context: user.
9337  */
9338 static void
9339 _scsih_sas_ir_volume_event(struct MPT3SAS_ADAPTER *ioc,
9340         struct fw_event_work *fw_event)
9341 {
9342         u64 wwid;
9343         unsigned long flags;
9344         struct _raid_device *raid_device;
9345         u16 handle;
9346         u32 state;
9347         int rc;
9348         Mpi2EventDataIrVolume_t *event_data =
9349                 (Mpi2EventDataIrVolume_t *) fw_event->event_data;
9350
9351         if (ioc->shost_recovery)
9352                 return;
9353
9354         if (event_data->ReasonCode != MPI2_EVENT_IR_VOLUME_RC_STATE_CHANGED)
9355                 return;
9356
9357         handle = le16_to_cpu(event_data->VolDevHandle);
9358         state = le32_to_cpu(event_data->NewValue);
9359         if (!ioc->hide_ir_msg)
9360                 dewtprintk(ioc,
9361                            ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",
9362                                     __func__, handle,
9363                                     le32_to_cpu(event_data->PreviousValue),
9364                                     state));
9365         switch (state) {
9366         case MPI2_RAID_VOL_STATE_MISSING:
9367         case MPI2_RAID_VOL_STATE_FAILED:
9368                 _scsih_sas_volume_delete(ioc, handle);
9369                 break;
9370
9371         case MPI2_RAID_VOL_STATE_ONLINE:
9372         case MPI2_RAID_VOL_STATE_DEGRADED:
9373         case MPI2_RAID_VOL_STATE_OPTIMAL:
9374
9375                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
9376                 raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9377                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9378
9379                 if (raid_device)
9380                         break;
9381
9382                 mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);
9383                 if (!wwid) {
9384                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9385                                 __FILE__, __LINE__, __func__);
9386                         break;
9387                 }
9388
9389                 raid_device = kzalloc(sizeof(struct _raid_device), GFP_KERNEL);
9390                 if (!raid_device) {
9391                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9392                                 __FILE__, __LINE__, __func__);
9393                         break;
9394                 }
9395
9396                 raid_device->id = ioc->sas_id++;
9397                 raid_device->channel = RAID_CHANNEL;
9398                 raid_device->handle = handle;
9399                 raid_device->wwid = wwid;
9400                 _scsih_raid_device_add(ioc, raid_device);
9401                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
9402                     raid_device->id, 0);
9403                 if (rc)
9404                         _scsih_raid_device_remove(ioc, raid_device);
9405                 break;
9406
9407         case MPI2_RAID_VOL_STATE_INITIALIZING:
9408         default:
9409                 break;
9410         }
9411 }
9412
9413 /**
9414  * _scsih_sas_ir_physical_disk_event - PD event
9415  * @ioc: per adapter object
9416  * @fw_event: The fw_event_work object
9417  * Context: user.
9418  */
9419 static void
9420 _scsih_sas_ir_physical_disk_event(struct MPT3SAS_ADAPTER *ioc,
9421         struct fw_event_work *fw_event)
9422 {
9423         u16 handle, parent_handle;
9424         u32 state;
9425         struct _sas_device *sas_device;
9426         Mpi2ConfigReply_t mpi_reply;
9427         Mpi2SasDevicePage0_t sas_device_pg0;
9428         u32 ioc_status;
9429         Mpi2EventDataIrPhysicalDisk_t *event_data =
9430                 (Mpi2EventDataIrPhysicalDisk_t *) fw_event->event_data;
9431         u64 sas_address;
9432
9433         if (ioc->shost_recovery)
9434                 return;
9435
9436         if (event_data->ReasonCode != MPI2_EVENT_IR_PHYSDISK_RC_STATE_CHANGED)
9437                 return;
9438
9439         handle = le16_to_cpu(event_data->PhysDiskDevHandle);
9440         state = le32_to_cpu(event_data->NewValue);
9441
9442         if (!ioc->hide_ir_msg)
9443                 dewtprintk(ioc,
9444                            ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",
9445                                     __func__, handle,
9446                                     le32_to_cpu(event_data->PreviousValue),
9447                                     state));
9448
9449         switch (state) {
9450         case MPI2_RAID_PD_STATE_ONLINE:
9451         case MPI2_RAID_PD_STATE_DEGRADED:
9452         case MPI2_RAID_PD_STATE_REBUILDING:
9453         case MPI2_RAID_PD_STATE_OPTIMAL:
9454         case MPI2_RAID_PD_STATE_HOT_SPARE:
9455
9456                 if (!ioc->is_warpdrive)
9457                         set_bit(handle, ioc->pd_handles);
9458
9459                 sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
9460                 if (sas_device) {
9461                         sas_device_put(sas_device);
9462                         return;
9463                 }
9464
9465                 if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
9466                     &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
9467                     handle))) {
9468                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9469                                 __FILE__, __LINE__, __func__);
9470                         return;
9471                 }
9472
9473                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9474                     MPI2_IOCSTATUS_MASK;
9475                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9476                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9477                                 __FILE__, __LINE__, __func__);
9478                         return;
9479                 }
9480
9481                 parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
9482                 if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))
9483                         mpt3sas_transport_update_links(ioc, sas_address, handle,
9484                             sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
9485                             mpt3sas_get_port_by_id(ioc,
9486                             sas_device_pg0.PhysicalPort, 0));
9487
9488                 _scsih_add_device(ioc, handle, 0, 1);
9489
9490                 break;
9491
9492         case MPI2_RAID_PD_STATE_OFFLINE:
9493         case MPI2_RAID_PD_STATE_NOT_CONFIGURED:
9494         case MPI2_RAID_PD_STATE_NOT_COMPATIBLE:
9495         default:
9496                 break;
9497         }
9498 }
9499
9500 /**
9501  * _scsih_sas_ir_operation_status_event_debug - debug for IR op event
9502  * @ioc: per adapter object
9503  * @event_data: event data payload
9504  * Context: user.
9505  */
9506 static void
9507 _scsih_sas_ir_operation_status_event_debug(struct MPT3SAS_ADAPTER *ioc,
9508         Mpi2EventDataIrOperationStatus_t *event_data)
9509 {
9510         char *reason_str = NULL;
9511
9512         switch (event_data->RAIDOperation) {
9513         case MPI2_EVENT_IR_RAIDOP_RESYNC:
9514                 reason_str = "resync";
9515                 break;
9516         case MPI2_EVENT_IR_RAIDOP_ONLINE_CAP_EXPANSION:
9517                 reason_str = "online capacity expansion";
9518                 break;
9519         case MPI2_EVENT_IR_RAIDOP_CONSISTENCY_CHECK:
9520                 reason_str = "consistency check";
9521                 break;
9522         case MPI2_EVENT_IR_RAIDOP_BACKGROUND_INIT:
9523                 reason_str = "background init";
9524                 break;
9525         case MPI2_EVENT_IR_RAIDOP_MAKE_DATA_CONSISTENT:
9526                 reason_str = "make data consistent";
9527                 break;
9528         }
9529
9530         if (!reason_str)
9531                 return;
9532
9533         ioc_info(ioc, "raid operational status: (%s)\thandle(0x%04x), percent complete(%d)\n",
9534                  reason_str,
9535                  le16_to_cpu(event_data->VolDevHandle),
9536                  event_data->PercentComplete);
9537 }
9538
9539 /**
9540  * _scsih_sas_ir_operation_status_event - handle RAID operation events
9541  * @ioc: per adapter object
9542  * @fw_event: The fw_event_work object
9543  * Context: user.
9544  */
9545 static void
9546 _scsih_sas_ir_operation_status_event(struct MPT3SAS_ADAPTER *ioc,
9547         struct fw_event_work *fw_event)
9548 {
9549         Mpi2EventDataIrOperationStatus_t *event_data =
9550                 (Mpi2EventDataIrOperationStatus_t *)
9551                 fw_event->event_data;
9552         static struct _raid_device *raid_device;
9553         unsigned long flags;
9554         u16 handle;
9555
9556         if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&
9557             (!ioc->hide_ir_msg))
9558                 _scsih_sas_ir_operation_status_event_debug(ioc,
9559                      event_data);
9560
9561         /* code added for raid transport support */
9562         if (event_data->RAIDOperation == MPI2_EVENT_IR_RAIDOP_RESYNC) {
9563
9564                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
9565                 handle = le16_to_cpu(event_data->VolDevHandle);
9566                 raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);
9567                 if (raid_device)
9568                         raid_device->percent_complete =
9569                             event_data->PercentComplete;
9570                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9571         }
9572 }
9573
9574 /**
9575  * _scsih_prep_device_scan - initialize parameters prior to device scan
9576  * @ioc: per adapter object
9577  *
9578  * Set the deleted flag prior to device scan.  If the device is found during
9579  * the scan, then we clear the deleted flag.
9580  */
9581 static void
9582 _scsih_prep_device_scan(struct MPT3SAS_ADAPTER *ioc)
9583 {
9584         struct MPT3SAS_DEVICE *sas_device_priv_data;
9585         struct scsi_device *sdev;
9586
9587         shost_for_each_device(sdev, ioc->shost) {
9588                 sas_device_priv_data = sdev->hostdata;
9589                 if (sas_device_priv_data && sas_device_priv_data->sas_target)
9590                         sas_device_priv_data->sas_target->deleted = 1;
9591         }
9592 }
9593
9594 /**
9595  * _scsih_update_device_qdepth - Update QD during Reset.
9596  * @ioc: per adapter object
9597  *
9598  */
9599 static void
9600 _scsih_update_device_qdepth(struct MPT3SAS_ADAPTER *ioc)
9601 {
9602         struct MPT3SAS_DEVICE *sas_device_priv_data;
9603         struct MPT3SAS_TARGET *sas_target_priv_data;
9604         struct _sas_device *sas_device;
9605         struct scsi_device *sdev;
9606         u16 qdepth;
9607
9608         ioc_info(ioc, "Update devices with firmware reported queue depth\n");
9609         shost_for_each_device(sdev, ioc->shost) {
9610                 sas_device_priv_data = sdev->hostdata;
9611                 if (sas_device_priv_data && sas_device_priv_data->sas_target) {
9612                         sas_target_priv_data = sas_device_priv_data->sas_target;
9613                         sas_device = sas_device_priv_data->sas_target->sas_dev;
9614                         if (sas_target_priv_data->flags & MPT_TARGET_FLAGS_PCIE_DEVICE)
9615                                 qdepth = ioc->max_nvme_qd;
9616                         else if (sas_device &&
9617                             sas_device->device_info & MPI2_SAS_DEVICE_INFO_SSP_TARGET)
9618                                 qdepth = (sas_device->port_type > 1) ?
9619                                     ioc->max_wideport_qd : ioc->max_narrowport_qd;
9620                         else if (sas_device &&
9621                             sas_device->device_info & MPI2_SAS_DEVICE_INFO_SATA_DEVICE)
9622                                 qdepth = ioc->max_sata_qd;
9623                         else
9624                                 continue;
9625                         mpt3sas_scsih_change_queue_depth(sdev, qdepth);
9626                 }
9627         }
9628 }
9629
9630 /**
9631  * _scsih_mark_responding_sas_device - mark a sas_devices as responding
9632  * @ioc: per adapter object
9633  * @sas_device_pg0: SAS Device page 0
9634  *
9635  * After host reset, find out whether devices are still responding.
9636  * Used in _scsih_remove_unresponsive_sas_devices.
9637  */
9638 static void
9639 _scsih_mark_responding_sas_device(struct MPT3SAS_ADAPTER *ioc,
9640 Mpi2SasDevicePage0_t *sas_device_pg0)
9641 {
9642         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9643         struct scsi_target *starget;
9644         struct _sas_device *sas_device = NULL;
9645         struct _enclosure_node *enclosure_dev = NULL;
9646         unsigned long flags;
9647         struct hba_port *port = mpt3sas_get_port_by_id(
9648             ioc, sas_device_pg0->PhysicalPort, 0);
9649
9650         if (sas_device_pg0->EnclosureHandle) {
9651                 enclosure_dev =
9652                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
9653                                 le16_to_cpu(sas_device_pg0->EnclosureHandle));
9654                 if (enclosure_dev == NULL)
9655                         ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",
9656                                  sas_device_pg0->EnclosureHandle);
9657         }
9658         spin_lock_irqsave(&ioc->sas_device_lock, flags);
9659         list_for_each_entry(sas_device, &ioc->sas_device_list, list) {
9660                 if (sas_device->sas_address != le64_to_cpu(
9661                     sas_device_pg0->SASAddress))
9662                         continue;
9663                 if (sas_device->slot != le16_to_cpu(sas_device_pg0->Slot))
9664                         continue;
9665                 if (sas_device->port != port)
9666                         continue;
9667                 sas_device->responding = 1;
9668                 starget = sas_device->starget;
9669                 if (starget && starget->hostdata) {
9670                         sas_target_priv_data = starget->hostdata;
9671                         sas_target_priv_data->tm_busy = 0;
9672                         sas_target_priv_data->deleted = 0;
9673                 } else
9674                         sas_target_priv_data = NULL;
9675                 if (starget) {
9676                         starget_printk(KERN_INFO, starget,
9677                             "handle(0x%04x), sas_addr(0x%016llx)\n",
9678                             le16_to_cpu(sas_device_pg0->DevHandle),
9679                             (unsigned long long)
9680                             sas_device->sas_address);
9681
9682                         if (sas_device->enclosure_handle != 0)
9683                                 starget_printk(KERN_INFO, starget,
9684                                  "enclosure logical id(0x%016llx), slot(%d)\n",
9685                                  (unsigned long long)
9686                                  sas_device->enclosure_logical_id,
9687                                  sas_device->slot);
9688                 }
9689                 if (le16_to_cpu(sas_device_pg0->Flags) &
9690                       MPI2_SAS_DEVICE0_FLAGS_ENCL_LEVEL_VALID) {
9691                         sas_device->enclosure_level =
9692                            sas_device_pg0->EnclosureLevel;
9693                         memcpy(&sas_device->connector_name[0],
9694                                 &sas_device_pg0->ConnectorName[0], 4);
9695                 } else {
9696                         sas_device->enclosure_level = 0;
9697                         sas_device->connector_name[0] = '\0';
9698                 }
9699
9700                 sas_device->enclosure_handle =
9701                         le16_to_cpu(sas_device_pg0->EnclosureHandle);
9702                 sas_device->is_chassis_slot_valid = 0;
9703                 if (enclosure_dev) {
9704                         sas_device->enclosure_logical_id = le64_to_cpu(
9705                                 enclosure_dev->pg0.EnclosureLogicalID);
9706                         if (le16_to_cpu(enclosure_dev->pg0.Flags) &
9707                             MPI2_SAS_ENCLS0_FLAGS_CHASSIS_SLOT_VALID) {
9708                                 sas_device->is_chassis_slot_valid = 1;
9709                                 sas_device->chassis_slot =
9710                                         enclosure_dev->pg0.ChassisSlot;
9711                         }
9712                 }
9713
9714                 if (sas_device->handle == le16_to_cpu(
9715                     sas_device_pg0->DevHandle))
9716                         goto out;
9717                 pr_info("\thandle changed from(0x%04x)!!!\n",
9718                     sas_device->handle);
9719                 sas_device->handle = le16_to_cpu(
9720                     sas_device_pg0->DevHandle);
9721                 if (sas_target_priv_data)
9722                         sas_target_priv_data->handle =
9723                             le16_to_cpu(sas_device_pg0->DevHandle);
9724                 goto out;
9725         }
9726  out:
9727         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
9728 }
9729
9730 /**
9731  * _scsih_create_enclosure_list_after_reset - Free Existing list,
9732  *      And create enclosure list by scanning all Enclosure Page(0)s
9733  * @ioc: per adapter object
9734  */
9735 static void
9736 _scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)
9737 {
9738         struct _enclosure_node *enclosure_dev;
9739         Mpi2ConfigReply_t mpi_reply;
9740         u16 enclosure_handle;
9741         int rc;
9742
9743         /* Free existing enclosure list */
9744         mpt3sas_free_enclosure_list(ioc);
9745
9746         /* Re constructing enclosure list after reset*/
9747         enclosure_handle = 0xFFFF;
9748         do {
9749                 enclosure_dev =
9750                         kzalloc(sizeof(struct _enclosure_node), GFP_KERNEL);
9751                 if (!enclosure_dev) {
9752                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
9753                                 __FILE__, __LINE__, __func__);
9754                         return;
9755                 }
9756                 rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,
9757                                 &enclosure_dev->pg0,
9758                                 MPI2_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE,
9759                                 enclosure_handle);
9760
9761                 if (rc || (le16_to_cpu(mpi_reply.IOCStatus) &
9762                                                 MPI2_IOCSTATUS_MASK)) {
9763                         kfree(enclosure_dev);
9764                         return;
9765                 }
9766                 list_add_tail(&enclosure_dev->list,
9767                                                 &ioc->enclosure_list);
9768                 enclosure_handle =
9769                         le16_to_cpu(enclosure_dev->pg0.EnclosureHandle);
9770         } while (1);
9771 }
9772
9773 /**
9774  * _scsih_search_responding_sas_devices -
9775  * @ioc: per adapter object
9776  *
9777  * After host reset, find out whether devices are still responding.
9778  * If not remove.
9779  */
9780 static void
9781 _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)
9782 {
9783         Mpi2SasDevicePage0_t sas_device_pg0;
9784         Mpi2ConfigReply_t mpi_reply;
9785         u16 ioc_status;
9786         u16 handle;
9787         u32 device_info;
9788
9789         ioc_info(ioc, "search for end-devices: start\n");
9790
9791         if (list_empty(&ioc->sas_device_list))
9792                 goto out;
9793
9794         handle = 0xFFFF;
9795         while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
9796             &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
9797             handle))) {
9798                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9799                     MPI2_IOCSTATUS_MASK;
9800                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
9801                         break;
9802                 handle = le16_to_cpu(sas_device_pg0.DevHandle);
9803                 device_info = le32_to_cpu(sas_device_pg0.DeviceInfo);
9804                 if (!(_scsih_is_end_device(device_info)))
9805                         continue;
9806                 _scsih_mark_responding_sas_device(ioc, &sas_device_pg0);
9807         }
9808
9809  out:
9810         ioc_info(ioc, "search for end-devices: complete\n");
9811 }
9812
9813 /**
9814  * _scsih_mark_responding_pcie_device - mark a pcie_device as responding
9815  * @ioc: per adapter object
9816  * @pcie_device_pg0: PCIe Device page 0
9817  *
9818  * After host reset, find out whether devices are still responding.
9819  * Used in _scsih_remove_unresponding_devices.
9820  */
9821 static void
9822 _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,
9823         Mpi26PCIeDevicePage0_t *pcie_device_pg0)
9824 {
9825         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9826         struct scsi_target *starget;
9827         struct _pcie_device *pcie_device;
9828         unsigned long flags;
9829
9830         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
9831         list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {
9832                 if ((pcie_device->wwid == le64_to_cpu(pcie_device_pg0->WWID))
9833                     && (pcie_device->slot == le16_to_cpu(
9834                     pcie_device_pg0->Slot))) {
9835                         pcie_device->access_status =
9836                                         pcie_device_pg0->AccessStatus;
9837                         pcie_device->responding = 1;
9838                         starget = pcie_device->starget;
9839                         if (starget && starget->hostdata) {
9840                                 sas_target_priv_data = starget->hostdata;
9841                                 sas_target_priv_data->tm_busy = 0;
9842                                 sas_target_priv_data->deleted = 0;
9843                         } else
9844                                 sas_target_priv_data = NULL;
9845                         if (starget) {
9846                                 starget_printk(KERN_INFO, starget,
9847                                     "handle(0x%04x), wwid(0x%016llx) ",
9848                                     pcie_device->handle,
9849                                     (unsigned long long)pcie_device->wwid);
9850                                 if (pcie_device->enclosure_handle != 0)
9851                                         starget_printk(KERN_INFO, starget,
9852                                             "enclosure logical id(0x%016llx), "
9853                                             "slot(%d)\n",
9854                                             (unsigned long long)
9855                                             pcie_device->enclosure_logical_id,
9856                                             pcie_device->slot);
9857                         }
9858
9859                         if (((le32_to_cpu(pcie_device_pg0->Flags)) &
9860                             MPI26_PCIEDEV0_FLAGS_ENCL_LEVEL_VALID) &&
9861                             (ioc->hba_mpi_version_belonged != MPI2_VERSION)) {
9862                                 pcie_device->enclosure_level =
9863                                     pcie_device_pg0->EnclosureLevel;
9864                                 memcpy(&pcie_device->connector_name[0],
9865                                     &pcie_device_pg0->ConnectorName[0], 4);
9866                         } else {
9867                                 pcie_device->enclosure_level = 0;
9868                                 pcie_device->connector_name[0] = '\0';
9869                         }
9870
9871                         if (pcie_device->handle == le16_to_cpu(
9872                             pcie_device_pg0->DevHandle))
9873                                 goto out;
9874                         pr_info("\thandle changed from(0x%04x)!!!\n",
9875                             pcie_device->handle);
9876                         pcie_device->handle = le16_to_cpu(
9877                             pcie_device_pg0->DevHandle);
9878                         if (sas_target_priv_data)
9879                                 sas_target_priv_data->handle =
9880                                     le16_to_cpu(pcie_device_pg0->DevHandle);
9881                         goto out;
9882                 }
9883         }
9884
9885  out:
9886         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
9887 }
9888
9889 /**
9890  * _scsih_search_responding_pcie_devices -
9891  * @ioc: per adapter object
9892  *
9893  * After host reset, find out whether devices are still responding.
9894  * If not remove.
9895  */
9896 static void
9897 _scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)
9898 {
9899         Mpi26PCIeDevicePage0_t pcie_device_pg0;
9900         Mpi2ConfigReply_t mpi_reply;
9901         u16 ioc_status;
9902         u16 handle;
9903         u32 device_info;
9904
9905         ioc_info(ioc, "search for end-devices: start\n");
9906
9907         if (list_empty(&ioc->pcie_device_list))
9908                 goto out;
9909
9910         handle = 0xFFFF;
9911         while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
9912                 &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
9913                 handle))) {
9914                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
9915                     MPI2_IOCSTATUS_MASK;
9916                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
9917                         ioc_info(ioc, "\tbreak from %s: ioc_status(0x%04x), loginfo(0x%08x)\n",
9918                                  __func__, ioc_status,
9919                                  le32_to_cpu(mpi_reply.IOCLogInfo));
9920                         break;
9921                 }
9922                 handle = le16_to_cpu(pcie_device_pg0.DevHandle);
9923                 device_info = le32_to_cpu(pcie_device_pg0.DeviceInfo);
9924                 if (!(_scsih_is_nvme_pciescsi_device(device_info)))
9925                         continue;
9926                 _scsih_mark_responding_pcie_device(ioc, &pcie_device_pg0);
9927         }
9928 out:
9929         ioc_info(ioc, "search for PCIe end-devices: complete\n");
9930 }
9931
9932 /**
9933  * _scsih_mark_responding_raid_device - mark a raid_device as responding
9934  * @ioc: per adapter object
9935  * @wwid: world wide identifier for raid volume
9936  * @handle: device handle
9937  *
9938  * After host reset, find out whether devices are still responding.
9939  * Used in _scsih_remove_unresponsive_raid_devices.
9940  */
9941 static void
9942 _scsih_mark_responding_raid_device(struct MPT3SAS_ADAPTER *ioc, u64 wwid,
9943         u16 handle)
9944 {
9945         struct MPT3SAS_TARGET *sas_target_priv_data = NULL;
9946         struct scsi_target *starget;
9947         struct _raid_device *raid_device;
9948         unsigned long flags;
9949
9950         spin_lock_irqsave(&ioc->raid_device_lock, flags);
9951         list_for_each_entry(raid_device, &ioc->raid_device_list, list) {
9952                 if (raid_device->wwid == wwid && raid_device->starget) {
9953                         starget = raid_device->starget;
9954                         if (starget && starget->hostdata) {
9955                                 sas_target_priv_data = starget->hostdata;
9956                                 sas_target_priv_data->deleted = 0;
9957                         } else
9958                                 sas_target_priv_data = NULL;
9959                         raid_device->responding = 1;
9960                         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9961                         starget_printk(KERN_INFO, raid_device->starget,
9962                             "handle(0x%04x), wwid(0x%016llx)\n", handle,
9963                             (unsigned long long)raid_device->wwid);
9964
9965                         /*
9966                          * WARPDRIVE: The handles of the PDs might have changed
9967                          * across the host reset so re-initialize the
9968                          * required data for Direct IO
9969                          */
9970                         mpt3sas_init_warpdrive_properties(ioc, raid_device);
9971                         spin_lock_irqsave(&ioc->raid_device_lock, flags);
9972                         if (raid_device->handle == handle) {
9973                                 spin_unlock_irqrestore(&ioc->raid_device_lock,
9974                                     flags);
9975                                 return;
9976                         }
9977                         pr_info("\thandle changed from(0x%04x)!!!\n",
9978                             raid_device->handle);
9979                         raid_device->handle = handle;
9980                         if (sas_target_priv_data)
9981                                 sas_target_priv_data->handle = handle;
9982                         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9983                         return;
9984                 }
9985         }
9986         spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
9987 }
9988
9989 /**
9990  * _scsih_search_responding_raid_devices -
9991  * @ioc: per adapter object
9992  *
9993  * After host reset, find out whether devices are still responding.
9994  * If not remove.
9995  */
9996 static void
9997 _scsih_search_responding_raid_devices(struct MPT3SAS_ADAPTER *ioc)
9998 {
9999         Mpi2RaidVolPage1_t volume_pg1;
10000         Mpi2RaidVolPage0_t volume_pg0;
10001         Mpi2RaidPhysDiskPage0_t pd_pg0;
10002         Mpi2ConfigReply_t mpi_reply;
10003         u16 ioc_status;
10004         u16 handle;
10005         u8 phys_disk_num;
10006
10007         if (!ioc->ir_firmware)
10008                 return;
10009
10010         ioc_info(ioc, "search for raid volumes: start\n");
10011
10012         if (list_empty(&ioc->raid_device_list))
10013                 goto out;
10014
10015         handle = 0xFFFF;
10016         while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,
10017             &volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
10018                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10019                     MPI2_IOCSTATUS_MASK;
10020                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10021                         break;
10022                 handle = le16_to_cpu(volume_pg1.DevHandle);
10023
10024                 if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,
10025                     &volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
10026                      sizeof(Mpi2RaidVolPage0_t)))
10027                         continue;
10028
10029                 if (volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
10030                     volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
10031                     volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_DEGRADED)
10032                         _scsih_mark_responding_raid_device(ioc,
10033                             le64_to_cpu(volume_pg1.WWID), handle);
10034         }
10035
10036         /* refresh the pd_handles */
10037         if (!ioc->is_warpdrive) {
10038                 phys_disk_num = 0xFF;
10039                 memset(ioc->pd_handles, 0, ioc->pd_handles_sz);
10040                 while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
10041                     &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_GET_NEXT_PHYSDISKNUM,
10042                     phys_disk_num))) {
10043                         ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10044                             MPI2_IOCSTATUS_MASK;
10045                         if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10046                                 break;
10047                         phys_disk_num = pd_pg0.PhysDiskNum;
10048                         handle = le16_to_cpu(pd_pg0.DevHandle);
10049                         set_bit(handle, ioc->pd_handles);
10050                 }
10051         }
10052  out:
10053         ioc_info(ioc, "search for responding raid volumes: complete\n");
10054 }
10055
10056 /**
10057  * _scsih_mark_responding_expander - mark a expander as responding
10058  * @ioc: per adapter object
10059  * @expander_pg0:SAS Expander Config Page0
10060  *
10061  * After host reset, find out whether devices are still responding.
10062  * Used in _scsih_remove_unresponsive_expanders.
10063  */
10064 static void
10065 _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,
10066         Mpi2ExpanderPage0_t *expander_pg0)
10067 {
10068         struct _sas_node *sas_expander = NULL;
10069         unsigned long flags;
10070         int i;
10071         struct _enclosure_node *enclosure_dev = NULL;
10072         u16 handle = le16_to_cpu(expander_pg0->DevHandle);
10073         u16 enclosure_handle = le16_to_cpu(expander_pg0->EnclosureHandle);
10074         u64 sas_address = le64_to_cpu(expander_pg0->SASAddress);
10075         struct hba_port *port = mpt3sas_get_port_by_id(
10076             ioc, expander_pg0->PhysicalPort, 0);
10077
10078         if (enclosure_handle)
10079                 enclosure_dev =
10080                         mpt3sas_scsih_enclosure_find_by_handle(ioc,
10081                                                         enclosure_handle);
10082
10083         spin_lock_irqsave(&ioc->sas_node_lock, flags);
10084         list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {
10085                 if (sas_expander->sas_address != sas_address)
10086                         continue;
10087                 if (sas_expander->port != port)
10088                         continue;
10089                 sas_expander->responding = 1;
10090
10091                 if (enclosure_dev) {
10092                         sas_expander->enclosure_logical_id =
10093                             le64_to_cpu(enclosure_dev->pg0.EnclosureLogicalID);
10094                         sas_expander->enclosure_handle =
10095                             le16_to_cpu(expander_pg0->EnclosureHandle);
10096                 }
10097
10098                 if (sas_expander->handle == handle)
10099                         goto out;
10100                 pr_info("\texpander(0x%016llx): handle changed" \
10101                     " from(0x%04x) to (0x%04x)!!!\n",
10102                     (unsigned long long)sas_expander->sas_address,
10103                     sas_expander->handle, handle);
10104                 sas_expander->handle = handle;
10105                 for (i = 0 ; i < sas_expander->num_phys ; i++)
10106                         sas_expander->phy[i].handle = handle;
10107                 goto out;
10108         }
10109  out:
10110         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
10111 }
10112
10113 /**
10114  * _scsih_search_responding_expanders -
10115  * @ioc: per adapter object
10116  *
10117  * After host reset, find out whether devices are still responding.
10118  * If not remove.
10119  */
10120 static void
10121 _scsih_search_responding_expanders(struct MPT3SAS_ADAPTER *ioc)
10122 {
10123         Mpi2ExpanderPage0_t expander_pg0;
10124         Mpi2ConfigReply_t mpi_reply;
10125         u16 ioc_status;
10126         u64 sas_address;
10127         u16 handle;
10128         u8 port;
10129
10130         ioc_info(ioc, "search for expanders: start\n");
10131
10132         if (list_empty(&ioc->sas_expander_list))
10133                 goto out;
10134
10135         handle = 0xFFFF;
10136         while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
10137             MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL, handle))) {
10138
10139                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10140                     MPI2_IOCSTATUS_MASK;
10141                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
10142                         break;
10143
10144                 handle = le16_to_cpu(expander_pg0.DevHandle);
10145                 sas_address = le64_to_cpu(expander_pg0.SASAddress);
10146                 port = expander_pg0.PhysicalPort;
10147                 pr_info(
10148                     "\texpander present: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
10149                     handle, (unsigned long long)sas_address,
10150                     (ioc->multipath_on_hba ?
10151                     port : MULTIPATH_DISABLED_PORT_ID));
10152                 _scsih_mark_responding_expander(ioc, &expander_pg0);
10153         }
10154
10155  out:
10156         ioc_info(ioc, "search for expanders: complete\n");
10157 }
10158
10159 /**
10160  * _scsih_remove_unresponding_devices - removing unresponding devices
10161  * @ioc: per adapter object
10162  */
10163 static void
10164 _scsih_remove_unresponding_devices(struct MPT3SAS_ADAPTER *ioc)
10165 {
10166         struct _sas_device *sas_device, *sas_device_next;
10167         struct _sas_node *sas_expander, *sas_expander_next;
10168         struct _raid_device *raid_device, *raid_device_next;
10169         struct _pcie_device *pcie_device, *pcie_device_next;
10170         struct list_head tmp_list;
10171         unsigned long flags;
10172         LIST_HEAD(head);
10173
10174         ioc_info(ioc, "removing unresponding devices: start\n");
10175
10176         /* removing unresponding end devices */
10177         ioc_info(ioc, "removing unresponding devices: end-devices\n");
10178         /*
10179          * Iterate, pulling off devices marked as non-responding. We become the
10180          * owner for the reference the list had on any object we prune.
10181          */
10182         spin_lock_irqsave(&ioc->sas_device_lock, flags);
10183
10184         /*
10185          * Clean up the sas_device_init_list list as
10186          * driver goes for fresh scan as part of diag reset.
10187          */
10188         list_for_each_entry_safe(sas_device, sas_device_next,
10189             &ioc->sas_device_init_list, list) {
10190                 list_del_init(&sas_device->list);
10191                 sas_device_put(sas_device);
10192         }
10193
10194         list_for_each_entry_safe(sas_device, sas_device_next,
10195             &ioc->sas_device_list, list) {
10196                 if (!sas_device->responding)
10197                         list_move_tail(&sas_device->list, &head);
10198                 else
10199                         sas_device->responding = 0;
10200         }
10201         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
10202
10203         /*
10204          * Now, uninitialize and remove the unresponding devices we pruned.
10205          */
10206         list_for_each_entry_safe(sas_device, sas_device_next, &head, list) {
10207                 _scsih_remove_device(ioc, sas_device);
10208                 list_del_init(&sas_device->list);
10209                 sas_device_put(sas_device);
10210         }
10211
10212         ioc_info(ioc, "Removing unresponding devices: pcie end-devices\n");
10213         INIT_LIST_HEAD(&head);
10214         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
10215         /*
10216          * Clean up the pcie_device_init_list list as
10217          * driver goes for fresh scan as part of diag reset.
10218          */
10219         list_for_each_entry_safe(pcie_device, pcie_device_next,
10220             &ioc->pcie_device_init_list, list) {
10221                 list_del_init(&pcie_device->list);
10222                 pcie_device_put(pcie_device);
10223         }
10224
10225         list_for_each_entry_safe(pcie_device, pcie_device_next,
10226             &ioc->pcie_device_list, list) {
10227                 if (!pcie_device->responding)
10228                         list_move_tail(&pcie_device->list, &head);
10229                 else
10230                         pcie_device->responding = 0;
10231         }
10232         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
10233
10234         list_for_each_entry_safe(pcie_device, pcie_device_next, &head, list) {
10235                 _scsih_pcie_device_remove_from_sml(ioc, pcie_device);
10236                 list_del_init(&pcie_device->list);
10237                 pcie_device_put(pcie_device);
10238         }
10239
10240         /* removing unresponding volumes */
10241         if (ioc->ir_firmware) {
10242                 ioc_info(ioc, "removing unresponding devices: volumes\n");
10243                 list_for_each_entry_safe(raid_device, raid_device_next,
10244                     &ioc->raid_device_list, list) {
10245                         if (!raid_device->responding)
10246                                 _scsih_sas_volume_delete(ioc,
10247                                     raid_device->handle);
10248                         else
10249                                 raid_device->responding = 0;
10250                 }
10251         }
10252
10253         /* removing unresponding expanders */
10254         ioc_info(ioc, "removing unresponding devices: expanders\n");
10255         spin_lock_irqsave(&ioc->sas_node_lock, flags);
10256         INIT_LIST_HEAD(&tmp_list);
10257         list_for_each_entry_safe(sas_expander, sas_expander_next,
10258             &ioc->sas_expander_list, list) {
10259                 if (!sas_expander->responding)
10260                         list_move_tail(&sas_expander->list, &tmp_list);
10261                 else
10262                         sas_expander->responding = 0;
10263         }
10264         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
10265         list_for_each_entry_safe(sas_expander, sas_expander_next, &tmp_list,
10266             list) {
10267                 _scsih_expander_node_remove(ioc, sas_expander);
10268         }
10269
10270         ioc_info(ioc, "removing unresponding devices: complete\n");
10271
10272         /* unblock devices */
10273         _scsih_ublock_io_all_device(ioc);
10274 }
10275
10276 static void
10277 _scsih_refresh_expander_links(struct MPT3SAS_ADAPTER *ioc,
10278         struct _sas_node *sas_expander, u16 handle)
10279 {
10280         Mpi2ExpanderPage1_t expander_pg1;
10281         Mpi2ConfigReply_t mpi_reply;
10282         int i;
10283
10284         for (i = 0 ; i < sas_expander->num_phys ; i++) {
10285                 if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,
10286                     &expander_pg1, i, handle))) {
10287                         ioc_err(ioc, "failure at %s:%d/%s()!\n",
10288                                 __FILE__, __LINE__, __func__);
10289                         return;
10290                 }
10291
10292                 mpt3sas_transport_update_links(ioc, sas_expander->sas_address,
10293                     le16_to_cpu(expander_pg1.AttachedDevHandle), i,
10294                     expander_pg1.NegotiatedLinkRate >> 4,
10295                     sas_expander->port);
10296         }
10297 }
10298
10299 /**
10300  * _scsih_scan_for_devices_after_reset - scan for devices after host reset
10301  * @ioc: per adapter object
10302  */
10303 static void
10304 _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
10305 {
10306         Mpi2ExpanderPage0_t expander_pg0;
10307         Mpi2SasDevicePage0_t sas_device_pg0;
10308         Mpi26PCIeDevicePage0_t pcie_device_pg0;
10309         Mpi2RaidVolPage1_t *volume_pg1;
10310         Mpi2RaidVolPage0_t *volume_pg0;
10311         Mpi2RaidPhysDiskPage0_t pd_pg0;
10312         Mpi2EventIrConfigElement_t element;
10313         Mpi2ConfigReply_t mpi_reply;
10314         u8 phys_disk_num, port_id;
10315         u16 ioc_status;
10316         u16 handle, parent_handle;
10317         u64 sas_address;
10318         struct _sas_device *sas_device;
10319         struct _pcie_device *pcie_device;
10320         struct _sas_node *expander_device;
10321         static struct _raid_device *raid_device;
10322         u8 retry_count;
10323         unsigned long flags;
10324
10325         volume_pg0 = kzalloc(sizeof(*volume_pg0), GFP_KERNEL);
10326         if (!volume_pg0)
10327                 return;
10328
10329         volume_pg1 = kzalloc(sizeof(*volume_pg1), GFP_KERNEL);
10330         if (!volume_pg1) {
10331                 kfree(volume_pg0);
10332                 return;
10333         }
10334
10335         ioc_info(ioc, "scan devices: start\n");
10336
10337         _scsih_sas_host_refresh(ioc);
10338
10339         ioc_info(ioc, "\tscan devices: expanders start\n");
10340
10341         /* expanders */
10342         handle = 0xFFFF;
10343         while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,
10344             MPI2_SAS_EXPAND_PGAD_FORM_GET_NEXT_HNDL, handle))) {
10345                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10346                     MPI2_IOCSTATUS_MASK;
10347                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10348                         ioc_info(ioc, "\tbreak from expander scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10349                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10350                         break;
10351                 }
10352                 handle = le16_to_cpu(expander_pg0.DevHandle);
10353                 spin_lock_irqsave(&ioc->sas_node_lock, flags);
10354                 port_id = expander_pg0.PhysicalPort;
10355                 expander_device = mpt3sas_scsih_expander_find_by_sas_address(
10356                     ioc, le64_to_cpu(expander_pg0.SASAddress),
10357                     mpt3sas_get_port_by_id(ioc, port_id, 0));
10358                 spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
10359                 if (expander_device)
10360                         _scsih_refresh_expander_links(ioc, expander_device,
10361                             handle);
10362                 else {
10363                         ioc_info(ioc, "\tBEFORE adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",
10364                                  handle,
10365                                  (u64)le64_to_cpu(expander_pg0.SASAddress));
10366                         _scsih_expander_add(ioc, handle);
10367                         ioc_info(ioc, "\tAFTER adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",
10368                                  handle,
10369                                  (u64)le64_to_cpu(expander_pg0.SASAddress));
10370                 }
10371         }
10372
10373         ioc_info(ioc, "\tscan devices: expanders complete\n");
10374
10375         if (!ioc->ir_firmware)
10376                 goto skip_to_sas;
10377
10378         ioc_info(ioc, "\tscan devices: phys disk start\n");
10379
10380         /* phys disk */
10381         phys_disk_num = 0xFF;
10382         while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,
10383             &pd_pg0, MPI2_PHYSDISK_PGAD_FORM_GET_NEXT_PHYSDISKNUM,
10384             phys_disk_num))) {
10385                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10386                     MPI2_IOCSTATUS_MASK;
10387                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10388                         ioc_info(ioc, "\tbreak from phys disk scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10389                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10390                         break;
10391                 }
10392                 phys_disk_num = pd_pg0.PhysDiskNum;
10393                 handle = le16_to_cpu(pd_pg0.DevHandle);
10394                 sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);
10395                 if (sas_device) {
10396                         sas_device_put(sas_device);
10397                         continue;
10398                 }
10399                 if (mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
10400                     &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_HANDLE,
10401                     handle) != 0)
10402                         continue;
10403                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10404                     MPI2_IOCSTATUS_MASK;
10405                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10406                         ioc_info(ioc, "\tbreak from phys disk scan ioc_status(0x%04x), loginfo(0x%08x)\n",
10407                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10408                         break;
10409                 }
10410                 parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
10411                 if (!_scsih_get_sas_address(ioc, parent_handle,
10412                     &sas_address)) {
10413                         ioc_info(ioc, "\tBEFORE adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",
10414                                  handle,
10415                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10416                         port_id = sas_device_pg0.PhysicalPort;
10417                         mpt3sas_transport_update_links(ioc, sas_address,
10418                             handle, sas_device_pg0.PhyNum,
10419                             MPI2_SAS_NEG_LINK_RATE_1_5,
10420                             mpt3sas_get_port_by_id(ioc, port_id, 0));
10421                         set_bit(handle, ioc->pd_handles);
10422                         retry_count = 0;
10423                         /* This will retry adding the end device.
10424                          * _scsih_add_device() will decide on retries and
10425                          * return "1" when it should be retried
10426                          */
10427                         while (_scsih_add_device(ioc, handle, retry_count++,
10428                             1)) {
10429                                 ssleep(1);
10430                         }
10431                         ioc_info(ioc, "\tAFTER adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",
10432                                  handle,
10433                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10434                 }
10435         }
10436
10437         ioc_info(ioc, "\tscan devices: phys disk complete\n");
10438
10439         ioc_info(ioc, "\tscan devices: volumes start\n");
10440
10441         /* volumes */
10442         handle = 0xFFFF;
10443         while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,
10444             volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
10445                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10446                     MPI2_IOCSTATUS_MASK;
10447                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10448                         ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10449                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10450                         break;
10451                 }
10452                 handle = le16_to_cpu(volume_pg1->DevHandle);
10453                 spin_lock_irqsave(&ioc->raid_device_lock, flags);
10454                 raid_device = _scsih_raid_device_find_by_wwid(ioc,
10455                     le64_to_cpu(volume_pg1->WWID));
10456                 spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
10457                 if (raid_device)
10458                         continue;
10459                 if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,
10460                     volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
10461                      sizeof(Mpi2RaidVolPage0_t)))
10462                         continue;
10463                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10464                     MPI2_IOCSTATUS_MASK;
10465                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10466                         ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10467                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10468                         break;
10469                 }
10470                 if (volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
10471                     volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
10472                     volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_DEGRADED) {
10473                         memset(&element, 0, sizeof(Mpi2EventIrConfigElement_t));
10474                         element.ReasonCode = MPI2_EVENT_IR_CHANGE_RC_ADDED;
10475                         element.VolDevHandle = volume_pg1->DevHandle;
10476                         ioc_info(ioc, "\tBEFORE adding volume: handle (0x%04x)\n",
10477                                  volume_pg1->DevHandle);
10478                         _scsih_sas_volume_add(ioc, &element);
10479                         ioc_info(ioc, "\tAFTER adding volume: handle (0x%04x)\n",
10480                                  volume_pg1->DevHandle);
10481                 }
10482         }
10483
10484         ioc_info(ioc, "\tscan devices: volumes complete\n");
10485
10486  skip_to_sas:
10487
10488         ioc_info(ioc, "\tscan devices: end devices start\n");
10489
10490         /* sas devices */
10491         handle = 0xFFFF;
10492         while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,
10493             &sas_device_pg0, MPI2_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
10494             handle))) {
10495                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
10496                     MPI2_IOCSTATUS_MASK;
10497                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10498                         ioc_info(ioc, "\tbreak from end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10499                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10500                         break;
10501                 }
10502                 handle = le16_to_cpu(sas_device_pg0.DevHandle);
10503                 if (!(_scsih_is_end_device(
10504                     le32_to_cpu(sas_device_pg0.DeviceInfo))))
10505                         continue;
10506                 port_id = sas_device_pg0.PhysicalPort;
10507                 sas_device = mpt3sas_get_sdev_by_addr(ioc,
10508                     le64_to_cpu(sas_device_pg0.SASAddress),
10509                     mpt3sas_get_port_by_id(ioc, port_id, 0));
10510                 if (sas_device) {
10511                         sas_device_put(sas_device);
10512                         continue;
10513                 }
10514                 parent_handle = le16_to_cpu(sas_device_pg0.ParentDevHandle);
10515                 if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address)) {
10516                         ioc_info(ioc, "\tBEFORE adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",
10517                                  handle,
10518                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10519                         mpt3sas_transport_update_links(ioc, sas_address, handle,
10520                             sas_device_pg0.PhyNum, MPI2_SAS_NEG_LINK_RATE_1_5,
10521                             mpt3sas_get_port_by_id(ioc, port_id, 0));
10522                         retry_count = 0;
10523                         /* This will retry adding the end device.
10524                          * _scsih_add_device() will decide on retries and
10525                          * return "1" when it should be retried
10526                          */
10527                         while (_scsih_add_device(ioc, handle, retry_count++,
10528                             0)) {
10529                                 ssleep(1);
10530                         }
10531                         ioc_info(ioc, "\tAFTER adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",
10532                                  handle,
10533                                  (u64)le64_to_cpu(sas_device_pg0.SASAddress));
10534                 }
10535         }
10536         ioc_info(ioc, "\tscan devices: end devices complete\n");
10537         ioc_info(ioc, "\tscan devices: pcie end devices start\n");
10538
10539         /* pcie devices */
10540         handle = 0xFFFF;
10541         while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,
10542                 &pcie_device_pg0, MPI26_PCIE_DEVICE_PGAD_FORM_GET_NEXT_HANDLE,
10543                 handle))) {
10544                 ioc_status = le16_to_cpu(mpi_reply.IOCStatus)
10545                                 & MPI2_IOCSTATUS_MASK;
10546                 if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
10547                         ioc_info(ioc, "\tbreak from pcie end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",
10548                                  ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
10549                         break;
10550                 }
10551                 handle = le16_to_cpu(pcie_device_pg0.DevHandle);
10552                 if (!(_scsih_is_nvme_pciescsi_device(
10553                         le32_to_cpu(pcie_device_pg0.DeviceInfo))))
10554                         continue;
10555                 pcie_device = mpt3sas_get_pdev_by_wwid(ioc,
10556                                 le64_to_cpu(pcie_device_pg0.WWID));
10557                 if (pcie_device) {
10558                         pcie_device_put(pcie_device);
10559                         continue;
10560                 }
10561                 retry_count = 0;
10562                 parent_handle = le16_to_cpu(pcie_device_pg0.ParentDevHandle);
10563                 _scsih_pcie_add_device(ioc, handle);
10564
10565                 ioc_info(ioc, "\tAFTER adding pcie end device: handle (0x%04x), wwid(0x%016llx)\n",
10566                          handle, (u64)le64_to_cpu(pcie_device_pg0.WWID));
10567         }
10568
10569         kfree(volume_pg0);
10570         kfree(volume_pg1);
10571
10572         ioc_info(ioc, "\tpcie devices: pcie end devices complete\n");
10573         ioc_info(ioc, "scan devices: complete\n");
10574 }
10575
10576 /**
10577  * mpt3sas_scsih_pre_reset_handler - reset callback handler (for scsih)
10578  * @ioc: per adapter object
10579  *
10580  * The handler for doing any required cleanup or initialization.
10581  */
10582 void mpt3sas_scsih_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)
10583 {
10584         dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));
10585 }
10586
10587 /**
10588  * mpt3sas_scsih_clear_outstanding_scsi_tm_commands - clears outstanding
10589  *                                                      scsi & tm cmds.
10590  * @ioc: per adapter object
10591  *
10592  * The handler for doing any required cleanup or initialization.
10593  */
10594 void
10595 mpt3sas_scsih_clear_outstanding_scsi_tm_commands(struct MPT3SAS_ADAPTER *ioc)
10596 {
10597         dtmprintk(ioc,
10598             ioc_info(ioc, "%s: clear outstanding scsi & tm cmds\n", __func__));
10599         if (ioc->scsih_cmds.status & MPT3_CMD_PENDING) {
10600                 ioc->scsih_cmds.status |= MPT3_CMD_RESET;
10601                 mpt3sas_base_free_smid(ioc, ioc->scsih_cmds.smid);
10602                 complete(&ioc->scsih_cmds.done);
10603         }
10604         if (ioc->tm_cmds.status & MPT3_CMD_PENDING) {
10605                 ioc->tm_cmds.status |= MPT3_CMD_RESET;
10606                 mpt3sas_base_free_smid(ioc, ioc->tm_cmds.smid);
10607                 complete(&ioc->tm_cmds.done);
10608         }
10609
10610         memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz);
10611         memset(ioc->device_remove_in_progress, 0,
10612                ioc->device_remove_in_progress_sz);
10613         _scsih_fw_event_cleanup_queue(ioc);
10614         _scsih_flush_running_cmds(ioc);
10615 }
10616
10617 /**
10618  * mpt3sas_scsih_reset_done_handler - reset callback handler (for scsih)
10619  * @ioc: per adapter object
10620  *
10621  * The handler for doing any required cleanup or initialization.
10622  */
10623 void
10624 mpt3sas_scsih_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)
10625 {
10626         dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));
10627         if (!(disable_discovery > 0 && !ioc->sas_hba.num_phys)) {
10628                 if (ioc->multipath_on_hba) {
10629                         _scsih_sas_port_refresh(ioc);
10630                         _scsih_update_vphys_after_reset(ioc);
10631                 }
10632                 _scsih_prep_device_scan(ioc);
10633                 _scsih_create_enclosure_list_after_reset(ioc);
10634                 _scsih_search_responding_sas_devices(ioc);
10635                 _scsih_search_responding_pcie_devices(ioc);
10636                 _scsih_search_responding_raid_devices(ioc);
10637                 _scsih_search_responding_expanders(ioc);
10638                 _scsih_error_recovery_delete_devices(ioc);
10639         }
10640 }
10641
10642 /**
10643  * _mpt3sas_fw_work - delayed task for processing firmware events
10644  * @ioc: per adapter object
10645  * @fw_event: The fw_event_work object
10646  * Context: user.
10647  */
10648 static void
10649 _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)
10650 {
10651         ioc->current_event = fw_event;
10652         _scsih_fw_event_del_from_list(ioc, fw_event);
10653
10654         /* the queue is being flushed so ignore this event */
10655         if (ioc->remove_host || ioc->pci_error_recovery) {
10656                 fw_event_work_put(fw_event);
10657                 ioc->current_event = NULL;
10658                 return;
10659         }
10660
10661         switch (fw_event->event) {
10662         case MPT3SAS_PROCESS_TRIGGER_DIAG:
10663                 mpt3sas_process_trigger_data(ioc,
10664                         (struct SL_WH_TRIGGERS_EVENT_DATA_T *)
10665                         fw_event->event_data);
10666                 break;
10667         case MPT3SAS_REMOVE_UNRESPONDING_DEVICES:
10668                 while (scsi_host_in_recovery(ioc->shost) ||
10669                                          ioc->shost_recovery) {
10670                         /*
10671                          * If we're unloading or cancelling the work, bail.
10672                          * Otherwise, this can become an infinite loop.
10673                          */
10674                         if (ioc->remove_host || ioc->fw_events_cleanup)
10675                                 goto out;
10676                         ssleep(1);
10677                 }
10678                 _scsih_remove_unresponding_devices(ioc);
10679                 _scsih_del_dirty_vphy(ioc);
10680                 _scsih_del_dirty_port_entries(ioc);
10681                 if (ioc->is_gen35_ioc)
10682                         _scsih_update_device_qdepth(ioc);
10683                 _scsih_scan_for_devices_after_reset(ioc);
10684                 /*
10685                  * If diag reset has occurred during the driver load
10686                  * then driver has to complete the driver load operation
10687                  * by executing the following items:
10688                  *- Register the devices from sas_device_init_list to SML
10689                  *- clear is_driver_loading flag,
10690                  *- start the watchdog thread.
10691                  * In happy driver load path, above things are taken care of when
10692                  * driver executes scsih_scan_finished().
10693                  */
10694                 if (ioc->is_driver_loading)
10695                         _scsih_complete_devices_scanning(ioc);
10696                 _scsih_set_nvme_max_shutdown_latency(ioc);
10697                 break;
10698         case MPT3SAS_PORT_ENABLE_COMPLETE:
10699                 ioc->start_scan = 0;
10700                 if (missing_delay[0] != -1 && missing_delay[1] != -1)
10701                         mpt3sas_base_update_missing_delay(ioc, missing_delay[0],
10702                             missing_delay[1]);
10703                 dewtprintk(ioc,
10704                            ioc_info(ioc, "port enable: complete from worker thread\n"));
10705                 break;
10706         case MPT3SAS_TURN_ON_PFA_LED:
10707                 _scsih_turn_on_pfa_led(ioc, fw_event->device_handle);
10708                 break;
10709         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
10710                 _scsih_sas_topology_change_event(ioc, fw_event);
10711                 break;
10712         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
10713                 if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)
10714                         _scsih_sas_device_status_change_event_debug(ioc,
10715                             (Mpi2EventDataSasDeviceStatusChange_t *)
10716                             fw_event->event_data);
10717                 break;
10718         case MPI2_EVENT_SAS_DISCOVERY:
10719                 _scsih_sas_discovery_event(ioc, fw_event);
10720                 break;
10721         case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
10722                 _scsih_sas_device_discovery_error_event(ioc, fw_event);
10723                 break;
10724         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
10725                 _scsih_sas_broadcast_primitive_event(ioc, fw_event);
10726                 break;
10727         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
10728                 _scsih_sas_enclosure_dev_status_change_event(ioc,
10729                     fw_event);
10730                 break;
10731         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
10732                 _scsih_sas_ir_config_change_event(ioc, fw_event);
10733                 break;
10734         case MPI2_EVENT_IR_VOLUME:
10735                 _scsih_sas_ir_volume_event(ioc, fw_event);
10736                 break;
10737         case MPI2_EVENT_IR_PHYSICAL_DISK:
10738                 _scsih_sas_ir_physical_disk_event(ioc, fw_event);
10739                 break;
10740         case MPI2_EVENT_IR_OPERATION_STATUS:
10741                 _scsih_sas_ir_operation_status_event(ioc, fw_event);
10742                 break;
10743         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
10744                 _scsih_pcie_device_status_change_event(ioc, fw_event);
10745                 break;
10746         case MPI2_EVENT_PCIE_ENUMERATION:
10747                 _scsih_pcie_enumeration_event(ioc, fw_event);
10748                 break;
10749         case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
10750                 _scsih_pcie_topology_change_event(ioc, fw_event);
10751                 ioc->current_event = NULL;
10752                         return;
10753         break;
10754         }
10755 out:
10756         fw_event_work_put(fw_event);
10757         ioc->current_event = NULL;
10758 }
10759
10760 /**
10761  * _firmware_event_work
10762  * @work: The fw_event_work object
10763  * Context: user.
10764  *
10765  * wrappers for the work thread handling firmware events
10766  */
10767
10768 static void
10769 _firmware_event_work(struct work_struct *work)
10770 {
10771         struct fw_event_work *fw_event = container_of(work,
10772             struct fw_event_work, work);
10773
10774         _mpt3sas_fw_work(fw_event->ioc, fw_event);
10775 }
10776
10777 /**
10778  * mpt3sas_scsih_event_callback - firmware event handler (called at ISR time)
10779  * @ioc: per adapter object
10780  * @msix_index: MSIX table index supplied by the OS
10781  * @reply: reply message frame(lower 32bit addr)
10782  * Context: interrupt.
10783  *
10784  * This function merely adds a new work task into ioc->firmware_event_thread.
10785  * The tasks are worked from _firmware_event_work in user context.
10786  *
10787  * Return: 1 meaning mf should be freed from _base_interrupt
10788  *         0 means the mf is freed from this function.
10789  */
10790 u8
10791 mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,
10792         u32 reply)
10793 {
10794         struct fw_event_work *fw_event;
10795         Mpi2EventNotificationReply_t *mpi_reply;
10796         u16 event;
10797         u16 sz;
10798         Mpi26EventDataActiveCableExcept_t *ActiveCableEventData;
10799
10800         /* events turned off due to host reset */
10801         if (ioc->pci_error_recovery)
10802                 return 1;
10803
10804         mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
10805
10806         if (unlikely(!mpi_reply)) {
10807                 ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
10808                         __FILE__, __LINE__, __func__);
10809                 return 1;
10810         }
10811
10812         event = le16_to_cpu(mpi_reply->Event);
10813
10814         if (event != MPI2_EVENT_LOG_ENTRY_ADDED)
10815                 mpt3sas_trigger_event(ioc, event, 0);
10816
10817         switch (event) {
10818         /* handle these */
10819         case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
10820         {
10821                 Mpi2EventDataSasBroadcastPrimitive_t *baen_data =
10822                     (Mpi2EventDataSasBroadcastPrimitive_t *)
10823                     mpi_reply->EventData;
10824
10825                 if (baen_data->Primitive !=
10826                     MPI2_EVENT_PRIMITIVE_ASYNCHRONOUS_EVENT)
10827                         return 1;
10828
10829                 if (ioc->broadcast_aen_busy) {
10830                         ioc->broadcast_aen_pending++;
10831                         return 1;
10832                 } else
10833                         ioc->broadcast_aen_busy = 1;
10834                 break;
10835         }
10836
10837         case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
10838                 _scsih_check_topo_delete_events(ioc,
10839                     (Mpi2EventDataSasTopologyChangeList_t *)
10840                     mpi_reply->EventData);
10841                 /*
10842                  * No need to add the topology change list
10843                  * event to fw event work queue when
10844                  * diag reset is going on. Since during diag
10845                  * reset driver scan the devices by reading
10846                  * sas device page0's not by processing the
10847                  * events.
10848                  */
10849                 if (ioc->shost_recovery)
10850                         return 1;
10851                 break;
10852         case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
10853         _scsih_check_pcie_topo_remove_events(ioc,
10854                     (Mpi26EventDataPCIeTopologyChangeList_t *)
10855                     mpi_reply->EventData);
10856                 if (ioc->shost_recovery)
10857                         return 1;
10858                 break;
10859         case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
10860                 _scsih_check_ir_config_unhide_events(ioc,
10861                     (Mpi2EventDataIrConfigChangeList_t *)
10862                     mpi_reply->EventData);
10863                 break;
10864         case MPI2_EVENT_IR_VOLUME:
10865                 _scsih_check_volume_delete_events(ioc,
10866                     (Mpi2EventDataIrVolume_t *)
10867                     mpi_reply->EventData);
10868                 break;
10869         case MPI2_EVENT_LOG_ENTRY_ADDED:
10870         {
10871                 Mpi2EventDataLogEntryAdded_t *log_entry;
10872                 u32 *log_code;
10873
10874                 if (!ioc->is_warpdrive)
10875                         break;
10876
10877                 log_entry = (Mpi2EventDataLogEntryAdded_t *)
10878                     mpi_reply->EventData;
10879                 log_code = (u32 *)log_entry->LogData;
10880
10881                 if (le16_to_cpu(log_entry->LogEntryQualifier)
10882                     != MPT2_WARPDRIVE_LOGENTRY)
10883                         break;
10884
10885                 switch (le32_to_cpu(*log_code)) {
10886                 case MPT2_WARPDRIVE_LC_SSDT:
10887                         ioc_warn(ioc, "WarpDrive Warning: IO Throttling has occurred in the WarpDrive subsystem. Check WarpDrive documentation for additional details.\n");
10888                         break;
10889                 case MPT2_WARPDRIVE_LC_SSDLW:
10890                         ioc_warn(ioc, "WarpDrive Warning: Program/Erase Cycles for the WarpDrive subsystem in degraded range. Check WarpDrive documentation for additional details.\n");
10891                         break;
10892                 case MPT2_WARPDRIVE_LC_SSDLF:
10893                         ioc_err(ioc, "WarpDrive Fatal Error: There are no Program/Erase Cycles for the WarpDrive subsystem. The storage device will be in read-only mode. Check WarpDrive documentation for additional details.\n");
10894                         break;
10895                 case MPT2_WARPDRIVE_LC_BRMF:
10896                         ioc_err(ioc, "WarpDrive Fatal Error: The Backup Rail Monitor has failed on the WarpDrive subsystem. Check WarpDrive documentation for additional details.\n");
10897                         break;
10898                 }
10899
10900                 break;
10901         }
10902         case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
10903                 _scsih_sas_device_status_change_event(ioc,
10904                     (Mpi2EventDataSasDeviceStatusChange_t *)
10905                     mpi_reply->EventData);
10906                 break;
10907         case MPI2_EVENT_IR_OPERATION_STATUS:
10908         case MPI2_EVENT_SAS_DISCOVERY:
10909         case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
10910         case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
10911         case MPI2_EVENT_IR_PHYSICAL_DISK:
10912         case MPI2_EVENT_PCIE_ENUMERATION:
10913         case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
10914                 break;
10915
10916         case MPI2_EVENT_TEMP_THRESHOLD:
10917                 _scsih_temp_threshold_events(ioc,
10918                         (Mpi2EventDataTemperature_t *)
10919                         mpi_reply->EventData);
10920                 break;
10921         case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
10922                 ActiveCableEventData =
10923                     (Mpi26EventDataActiveCableExcept_t *) mpi_reply->EventData;
10924                 switch (ActiveCableEventData->ReasonCode) {
10925                 case MPI26_EVENT_ACTIVE_CABLE_INSUFFICIENT_POWER:
10926                         ioc_notice(ioc, "Currently an active cable with ReceptacleID %d\n",
10927                                    ActiveCableEventData->ReceptacleID);
10928                         pr_notice("cannot be powered and devices connected\n");
10929                         pr_notice("to this active cable will not be seen\n");
10930                         pr_notice("This active cable requires %d mW of power\n",
10931                             le32_to_cpu(
10932                             ActiveCableEventData->ActiveCablePowerRequirement));
10933                         break;
10934
10935                 case MPI26_EVENT_ACTIVE_CABLE_DEGRADED:
10936                         ioc_notice(ioc, "Currently a cable with ReceptacleID %d\n",
10937                                    ActiveCableEventData->ReceptacleID);
10938                         pr_notice(
10939                             "is not running at optimal speed(12 Gb/s rate)\n");
10940                         break;
10941                 }
10942
10943                 break;
10944
10945         default: /* ignore the rest */
10946                 return 1;
10947         }
10948
10949         sz = le16_to_cpu(mpi_reply->EventDataLength) * 4;
10950         fw_event = alloc_fw_event_work(sz);
10951         if (!fw_event) {
10952                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
10953                         __FILE__, __LINE__, __func__);
10954                 return 1;
10955         }
10956
10957         memcpy(fw_event->event_data, mpi_reply->EventData, sz);
10958         fw_event->ioc = ioc;
10959         fw_event->VF_ID = mpi_reply->VF_ID;
10960         fw_event->VP_ID = mpi_reply->VP_ID;
10961         fw_event->event = event;
10962         _scsih_fw_event_add(ioc, fw_event);
10963         fw_event_work_put(fw_event);
10964         return 1;
10965 }
10966
10967 /**
10968  * _scsih_expander_node_remove - removing expander device from list.
10969  * @ioc: per adapter object
10970  * @sas_expander: the sas_device object
10971  *
10972  * Removing object and freeing associated memory from the
10973  * ioc->sas_expander_list.
10974  */
10975 static void
10976 _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
10977         struct _sas_node *sas_expander)
10978 {
10979         struct _sas_port *mpt3sas_port, *next;
10980         unsigned long flags;
10981
10982         /* remove sibling ports attached to this expander */
10983         list_for_each_entry_safe(mpt3sas_port, next,
10984            &sas_expander->sas_port_list, port_list) {
10985                 if (ioc->shost_recovery)
10986                         return;
10987                 if (mpt3sas_port->remote_identify.device_type ==
10988                     SAS_END_DEVICE)
10989                         mpt3sas_device_remove_by_sas_address(ioc,
10990                             mpt3sas_port->remote_identify.sas_address,
10991                             mpt3sas_port->hba_port);
10992                 else if (mpt3sas_port->remote_identify.device_type ==
10993                     SAS_EDGE_EXPANDER_DEVICE ||
10994                     mpt3sas_port->remote_identify.device_type ==
10995                     SAS_FANOUT_EXPANDER_DEVICE)
10996                         mpt3sas_expander_remove(ioc,
10997                             mpt3sas_port->remote_identify.sas_address,
10998                             mpt3sas_port->hba_port);
10999         }
11000
11001         mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,
11002             sas_expander->sas_address_parent, sas_expander->port);
11003
11004         ioc_info(ioc,
11005             "expander_remove: handle(0x%04x), sas_addr(0x%016llx), port:%d\n",
11006             sas_expander->handle, (unsigned long long)
11007             sas_expander->sas_address,
11008             sas_expander->port->port_id);
11009
11010         spin_lock_irqsave(&ioc->sas_node_lock, flags);
11011         list_del(&sas_expander->list);
11012         spin_unlock_irqrestore(&ioc->sas_node_lock, flags);
11013
11014         kfree(sas_expander->phy);
11015         kfree(sas_expander);
11016 }
11017
11018 /**
11019  * _scsih_nvme_shutdown - NVMe shutdown notification
11020  * @ioc: per adapter object
11021  *
11022  * Sending IoUnitControl request with shutdown operation code to alert IOC that
11023  * the host system is shutting down so that IOC can issue NVMe shutdown to
11024  * NVMe drives attached to it.
11025  */
11026 static void
11027 _scsih_nvme_shutdown(struct MPT3SAS_ADAPTER *ioc)
11028 {
11029         Mpi26IoUnitControlRequest_t *mpi_request;
11030         Mpi26IoUnitControlReply_t *mpi_reply;
11031         u16 smid;
11032
11033         /* are there any NVMe devices ? */
11034         if (list_empty(&ioc->pcie_device_list))
11035                 return;
11036
11037         mutex_lock(&ioc->scsih_cmds.mutex);
11038
11039         if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
11040                 ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
11041                 goto out;
11042         }
11043
11044         ioc->scsih_cmds.status = MPT3_CMD_PENDING;
11045
11046         smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
11047         if (!smid) {
11048                 ioc_err(ioc,
11049                     "%s: failed obtaining a smid\n", __func__);
11050                 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11051                 goto out;
11052         }
11053
11054         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
11055         ioc->scsih_cmds.smid = smid;
11056         memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t));
11057         mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL;
11058         mpi_request->Operation = MPI26_CTRL_OP_SHUTDOWN;
11059
11060         init_completion(&ioc->scsih_cmds.done);
11061         ioc->put_smid_default(ioc, smid);
11062         /* Wait for max_shutdown_latency seconds */
11063         ioc_info(ioc,
11064                 "Io Unit Control shutdown (sending), Shutdown latency %d sec\n",
11065                 ioc->max_shutdown_latency);
11066         wait_for_completion_timeout(&ioc->scsih_cmds.done,
11067                         ioc->max_shutdown_latency*HZ);
11068
11069         if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
11070                 ioc_err(ioc, "%s: timeout\n", __func__);
11071                 goto out;
11072         }
11073
11074         if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
11075                 mpi_reply = ioc->scsih_cmds.reply;
11076                 ioc_info(ioc, "Io Unit Control shutdown (complete):"
11077                         "ioc_status(0x%04x), loginfo(0x%08x)\n",
11078                         le16_to_cpu(mpi_reply->IOCStatus),
11079                         le32_to_cpu(mpi_reply->IOCLogInfo));
11080         }
11081  out:
11082         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11083         mutex_unlock(&ioc->scsih_cmds.mutex);
11084 }
11085
11086
11087 /**
11088  * _scsih_ir_shutdown - IR shutdown notification
11089  * @ioc: per adapter object
11090  *
11091  * Sending RAID Action to alert the Integrated RAID subsystem of the IOC that
11092  * the host system is shutting down.
11093  */
11094 static void
11095 _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)
11096 {
11097         Mpi2RaidActionRequest_t *mpi_request;
11098         Mpi2RaidActionReply_t *mpi_reply;
11099         u16 smid;
11100
11101         /* is IR firmware build loaded ? */
11102         if (!ioc->ir_firmware)
11103                 return;
11104
11105         /* are there any volumes ? */
11106         if (list_empty(&ioc->raid_device_list))
11107                 return;
11108
11109         mutex_lock(&ioc->scsih_cmds.mutex);
11110
11111         if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
11112                 ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);
11113                 goto out;
11114         }
11115         ioc->scsih_cmds.status = MPT3_CMD_PENDING;
11116
11117         smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
11118         if (!smid) {
11119                 ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
11120                 ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11121                 goto out;
11122         }
11123
11124         mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
11125         ioc->scsih_cmds.smid = smid;
11126         memset(mpi_request, 0, sizeof(Mpi2RaidActionRequest_t));
11127
11128         mpi_request->Function = MPI2_FUNCTION_RAID_ACTION;
11129         mpi_request->Action = MPI2_RAID_ACTION_SYSTEM_SHUTDOWN_INITIATED;
11130
11131         if (!ioc->hide_ir_msg)
11132                 ioc_info(ioc, "IR shutdown (sending)\n");
11133         init_completion(&ioc->scsih_cmds.done);
11134         ioc->put_smid_default(ioc, smid);
11135         wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);
11136
11137         if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
11138                 ioc_err(ioc, "%s: timeout\n", __func__);
11139                 goto out;
11140         }
11141
11142         if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
11143                 mpi_reply = ioc->scsih_cmds.reply;
11144                 if (!ioc->hide_ir_msg)
11145                         ioc_info(ioc, "IR shutdown (complete): ioc_status(0x%04x), loginfo(0x%08x)\n",
11146                                  le16_to_cpu(mpi_reply->IOCStatus),
11147                                  le32_to_cpu(mpi_reply->IOCLogInfo));
11148         }
11149
11150  out:
11151         ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
11152         mutex_unlock(&ioc->scsih_cmds.mutex);
11153 }
11154
11155 /**
11156  * _scsih_get_shost_and_ioc - get shost and ioc
11157  *                      and verify whether they are NULL or not
11158  * @pdev: PCI device struct
11159  * @shost: address of scsi host pointer
11160  * @ioc: address of HBA adapter pointer
11161  *
11162  * Return zero if *shost and *ioc are not NULL otherwise return error number.
11163  */
11164 static int
11165 _scsih_get_shost_and_ioc(struct pci_dev *pdev,
11166         struct Scsi_Host **shost, struct MPT3SAS_ADAPTER **ioc)
11167 {
11168         *shost = pci_get_drvdata(pdev);
11169         if (*shost == NULL) {
11170                 dev_err(&pdev->dev, "pdev's driver data is null\n");
11171                 return -ENXIO;
11172         }
11173
11174         *ioc = shost_priv(*shost);
11175         if (*ioc == NULL) {
11176                 dev_err(&pdev->dev, "shost's private data is null\n");
11177                 return -ENXIO;
11178         }
11179
11180         return 0;
11181 }
11182
11183 /**
11184  * scsih_remove - detach and remove add host
11185  * @pdev: PCI device struct
11186  *
11187  * Routine called when unloading the driver.
11188  */
11189 static void scsih_remove(struct pci_dev *pdev)
11190 {
11191         struct Scsi_Host *shost;
11192         struct MPT3SAS_ADAPTER *ioc;
11193         struct _sas_port *mpt3sas_port, *next_port;
11194         struct _raid_device *raid_device, *next;
11195         struct MPT3SAS_TARGET *sas_target_priv_data;
11196         struct _pcie_device *pcie_device, *pcienext;
11197         struct workqueue_struct *wq;
11198         unsigned long flags;
11199         Mpi2ConfigReply_t mpi_reply;
11200         struct hba_port *port, *port_next;
11201
11202         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
11203                 return;
11204
11205         ioc->remove_host = 1;
11206
11207         if (!pci_device_is_present(pdev)) {
11208                 mpt3sas_base_pause_mq_polling(ioc);
11209                 _scsih_flush_running_cmds(ioc);
11210         }
11211
11212         _scsih_fw_event_cleanup_queue(ioc);
11213
11214         spin_lock_irqsave(&ioc->fw_event_lock, flags);
11215         wq = ioc->firmware_event_thread;
11216         ioc->firmware_event_thread = NULL;
11217         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
11218         if (wq)
11219                 destroy_workqueue(wq);
11220         /*
11221          * Copy back the unmodified ioc page1. so that on next driver load,
11222          * current modified changes on ioc page1 won't take effect.
11223          */
11224         if (ioc->is_aero_ioc)
11225                 mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,
11226                                 &ioc->ioc_pg1_copy);
11227         /* release all the volumes */
11228         _scsih_ir_shutdown(ioc);
11229         mpt3sas_destroy_debugfs(ioc);
11230         sas_remove_host(shost);
11231         list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,
11232             list) {
11233                 if (raid_device->starget) {
11234                         sas_target_priv_data =
11235                             raid_device->starget->hostdata;
11236                         sas_target_priv_data->deleted = 1;
11237                         scsi_remove_target(&raid_device->starget->dev);
11238                 }
11239                 ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",
11240                          raid_device->handle, (u64)raid_device->wwid);
11241                 _scsih_raid_device_remove(ioc, raid_device);
11242         }
11243         list_for_each_entry_safe(pcie_device, pcienext, &ioc->pcie_device_list,
11244                 list) {
11245                 _scsih_pcie_device_remove_from_sml(ioc, pcie_device);
11246                 list_del_init(&pcie_device->list);
11247                 pcie_device_put(pcie_device);
11248         }
11249
11250         /* free ports attached to the sas_host */
11251         list_for_each_entry_safe(mpt3sas_port, next_port,
11252            &ioc->sas_hba.sas_port_list, port_list) {
11253                 if (mpt3sas_port->remote_identify.device_type ==
11254                     SAS_END_DEVICE)
11255                         mpt3sas_device_remove_by_sas_address(ioc,
11256                             mpt3sas_port->remote_identify.sas_address,
11257                             mpt3sas_port->hba_port);
11258                 else if (mpt3sas_port->remote_identify.device_type ==
11259                     SAS_EDGE_EXPANDER_DEVICE ||
11260                     mpt3sas_port->remote_identify.device_type ==
11261                     SAS_FANOUT_EXPANDER_DEVICE)
11262                         mpt3sas_expander_remove(ioc,
11263                             mpt3sas_port->remote_identify.sas_address,
11264                             mpt3sas_port->hba_port);
11265         }
11266
11267         list_for_each_entry_safe(port, port_next,
11268             &ioc->port_table_list, list) {
11269                 list_del(&port->list);
11270                 kfree(port);
11271         }
11272
11273         /* free phys attached to the sas_host */
11274         if (ioc->sas_hba.num_phys) {
11275                 kfree(ioc->sas_hba.phy);
11276                 ioc->sas_hba.phy = NULL;
11277                 ioc->sas_hba.num_phys = 0;
11278         }
11279
11280         mpt3sas_base_detach(ioc);
11281         spin_lock(&gioc_lock);
11282         list_del(&ioc->list);
11283         spin_unlock(&gioc_lock);
11284         scsi_host_put(shost);
11285 }
11286
11287 /**
11288  * scsih_shutdown - routine call during system shutdown
11289  * @pdev: PCI device struct
11290  */
11291 static void
11292 scsih_shutdown(struct pci_dev *pdev)
11293 {
11294         struct Scsi_Host *shost;
11295         struct MPT3SAS_ADAPTER *ioc;
11296         struct workqueue_struct *wq;
11297         unsigned long flags;
11298         Mpi2ConfigReply_t mpi_reply;
11299
11300         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
11301                 return;
11302
11303         ioc->remove_host = 1;
11304
11305         if (!pci_device_is_present(pdev)) {
11306                 mpt3sas_base_pause_mq_polling(ioc);
11307                 _scsih_flush_running_cmds(ioc);
11308         }
11309
11310         _scsih_fw_event_cleanup_queue(ioc);
11311
11312         spin_lock_irqsave(&ioc->fw_event_lock, flags);
11313         wq = ioc->firmware_event_thread;
11314         ioc->firmware_event_thread = NULL;
11315         spin_unlock_irqrestore(&ioc->fw_event_lock, flags);
11316         if (wq)
11317                 destroy_workqueue(wq);
11318         /*
11319          * Copy back the unmodified ioc page1 so that on next driver load,
11320          * current modified changes on ioc page1 won't take effect.
11321          */
11322         if (ioc->is_aero_ioc)
11323                 mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,
11324                                 &ioc->ioc_pg1_copy);
11325
11326         _scsih_ir_shutdown(ioc);
11327         _scsih_nvme_shutdown(ioc);
11328         mpt3sas_base_mask_interrupts(ioc);
11329         ioc->shost_recovery = 1;
11330         mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);
11331         ioc->shost_recovery = 0;
11332         mpt3sas_base_free_irq(ioc);
11333         mpt3sas_base_disable_msix(ioc);
11334 }
11335
11336
11337 /**
11338  * _scsih_probe_boot_devices - reports 1st device
11339  * @ioc: per adapter object
11340  *
11341  * If specified in bios page 2, this routine reports the 1st
11342  * device scsi-ml or sas transport for persistent boot device
11343  * purposes.  Please refer to function _scsih_determine_boot_device()
11344  */
11345 static void
11346 _scsih_probe_boot_devices(struct MPT3SAS_ADAPTER *ioc)
11347 {
11348         u32 channel;
11349         void *device;
11350         struct _sas_device *sas_device;
11351         struct _raid_device *raid_device;
11352         struct _pcie_device *pcie_device;
11353         u16 handle;
11354         u64 sas_address_parent;
11355         u64 sas_address;
11356         unsigned long flags;
11357         int rc;
11358         int tid;
11359         struct hba_port *port;
11360
11361          /* no Bios, return immediately */
11362         if (!ioc->bios_pg3.BiosVersion)
11363                 return;
11364
11365         device = NULL;
11366         if (ioc->req_boot_device.device) {
11367                 device =  ioc->req_boot_device.device;
11368                 channel = ioc->req_boot_device.channel;
11369         } else if (ioc->req_alt_boot_device.device) {
11370                 device =  ioc->req_alt_boot_device.device;
11371                 channel = ioc->req_alt_boot_device.channel;
11372         } else if (ioc->current_boot_device.device) {
11373                 device =  ioc->current_boot_device.device;
11374                 channel = ioc->current_boot_device.channel;
11375         }
11376
11377         if (!device)
11378                 return;
11379
11380         if (channel == RAID_CHANNEL) {
11381                 raid_device = device;
11382                 /*
11383                  * If this boot vd is already registered with SML then
11384                  * no need to register it again as part of device scanning
11385                  * after diag reset during driver load operation.
11386                  */
11387                 if (raid_device->starget)
11388                         return;
11389                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
11390                     raid_device->id, 0);
11391                 if (rc)
11392                         _scsih_raid_device_remove(ioc, raid_device);
11393         } else if (channel == PCIE_CHANNEL) {
11394                 pcie_device = device;
11395                 /*
11396                  * If this boot NVMe device is already registered with SML then
11397                  * no need to register it again as part of device scanning
11398                  * after diag reset during driver load operation.
11399                  */
11400                 if (pcie_device->starget)
11401                         return;
11402                 spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11403                 tid = pcie_device->id;
11404                 list_move_tail(&pcie_device->list, &ioc->pcie_device_list);
11405                 spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
11406                 rc = scsi_add_device(ioc->shost, PCIE_CHANNEL, tid, 0);
11407                 if (rc)
11408                         _scsih_pcie_device_remove(ioc, pcie_device);
11409         } else {
11410                 sas_device = device;
11411                 /*
11412                  * If this boot sas/sata device is already registered with SML
11413                  * then no need to register it again as part of device scanning
11414                  * after diag reset during driver load operation.
11415                  */
11416                 if (sas_device->starget)
11417                         return;
11418                 spin_lock_irqsave(&ioc->sas_device_lock, flags);
11419                 handle = sas_device->handle;
11420                 sas_address_parent = sas_device->sas_address_parent;
11421                 sas_address = sas_device->sas_address;
11422                 port = sas_device->port;
11423                 list_move_tail(&sas_device->list, &ioc->sas_device_list);
11424                 spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
11425
11426                 if (ioc->hide_drives)
11427                         return;
11428
11429                 if (!port)
11430                         return;
11431
11432                 if (!mpt3sas_transport_port_add(ioc, handle,
11433                     sas_address_parent, port)) {
11434                         _scsih_sas_device_remove(ioc, sas_device);
11435                 } else if (!sas_device->starget) {
11436                         if (!ioc->is_driver_loading) {
11437                                 mpt3sas_transport_port_remove(ioc,
11438                                     sas_address,
11439                                     sas_address_parent, port);
11440                                 _scsih_sas_device_remove(ioc, sas_device);
11441                         }
11442                 }
11443         }
11444 }
11445
11446 /**
11447  * _scsih_probe_raid - reporting raid volumes to scsi-ml
11448  * @ioc: per adapter object
11449  *
11450  * Called during initial loading of the driver.
11451  */
11452 static void
11453 _scsih_probe_raid(struct MPT3SAS_ADAPTER *ioc)
11454 {
11455         struct _raid_device *raid_device, *raid_next;
11456         int rc;
11457
11458         list_for_each_entry_safe(raid_device, raid_next,
11459             &ioc->raid_device_list, list) {
11460                 if (raid_device->starget)
11461                         continue;
11462                 rc = scsi_add_device(ioc->shost, RAID_CHANNEL,
11463                     raid_device->id, 0);
11464                 if (rc)
11465                         _scsih_raid_device_remove(ioc, raid_device);
11466         }
11467 }
11468
11469 static struct _sas_device *get_next_sas_device(struct MPT3SAS_ADAPTER *ioc)
11470 {
11471         struct _sas_device *sas_device = NULL;
11472         unsigned long flags;
11473
11474         spin_lock_irqsave(&ioc->sas_device_lock, flags);
11475         if (!list_empty(&ioc->sas_device_init_list)) {
11476                 sas_device = list_first_entry(&ioc->sas_device_init_list,
11477                                 struct _sas_device, list);
11478                 sas_device_get(sas_device);
11479         }
11480         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
11481
11482         return sas_device;
11483 }
11484
11485 static void sas_device_make_active(struct MPT3SAS_ADAPTER *ioc,
11486                 struct _sas_device *sas_device)
11487 {
11488         unsigned long flags;
11489
11490         spin_lock_irqsave(&ioc->sas_device_lock, flags);
11491
11492         /*
11493          * Since we dropped the lock during the call to port_add(), we need to
11494          * be careful here that somebody else didn't move or delete this item
11495          * while we were busy with other things.
11496          *
11497          * If it was on the list, we need a put() for the reference the list
11498          * had. Either way, we need a get() for the destination list.
11499          */
11500         if (!list_empty(&sas_device->list)) {
11501                 list_del_init(&sas_device->list);
11502                 sas_device_put(sas_device);
11503         }
11504
11505         sas_device_get(sas_device);
11506         list_add_tail(&sas_device->list, &ioc->sas_device_list);
11507
11508         spin_unlock_irqrestore(&ioc->sas_device_lock, flags);
11509 }
11510
11511 /**
11512  * _scsih_probe_sas - reporting sas devices to sas transport
11513  * @ioc: per adapter object
11514  *
11515  * Called during initial loading of the driver.
11516  */
11517 static void
11518 _scsih_probe_sas(struct MPT3SAS_ADAPTER *ioc)
11519 {
11520         struct _sas_device *sas_device;
11521
11522         if (ioc->hide_drives)
11523                 return;
11524
11525         while ((sas_device = get_next_sas_device(ioc))) {
11526                 if (!mpt3sas_transport_port_add(ioc, sas_device->handle,
11527                     sas_device->sas_address_parent, sas_device->port)) {
11528                         _scsih_sas_device_remove(ioc, sas_device);
11529                         sas_device_put(sas_device);
11530                         continue;
11531                 } else if (!sas_device->starget) {
11532                         /*
11533                          * When asyn scanning is enabled, its not possible to
11534                          * remove devices while scanning is turned on due to an
11535                          * oops in scsi_sysfs_add_sdev()->add_device()->
11536                          * sysfs_addrm_start()
11537                          */
11538                         if (!ioc->is_driver_loading) {
11539                                 mpt3sas_transport_port_remove(ioc,
11540                                     sas_device->sas_address,
11541                                     sas_device->sas_address_parent,
11542                                     sas_device->port);
11543                                 _scsih_sas_device_remove(ioc, sas_device);
11544                                 sas_device_put(sas_device);
11545                                 continue;
11546                         }
11547                 }
11548                 sas_device_make_active(ioc, sas_device);
11549                 sas_device_put(sas_device);
11550         }
11551 }
11552
11553 /**
11554  * get_next_pcie_device - Get the next pcie device
11555  * @ioc: per adapter object
11556  *
11557  * Get the next pcie device from pcie_device_init_list list.
11558  *
11559  * Return: pcie device structure if pcie_device_init_list list is not empty
11560  * otherwise returns NULL
11561  */
11562 static struct _pcie_device *get_next_pcie_device(struct MPT3SAS_ADAPTER *ioc)
11563 {
11564         struct _pcie_device *pcie_device = NULL;
11565         unsigned long flags;
11566
11567         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11568         if (!list_empty(&ioc->pcie_device_init_list)) {
11569                 pcie_device = list_first_entry(&ioc->pcie_device_init_list,
11570                                 struct _pcie_device, list);
11571                 pcie_device_get(pcie_device);
11572         }
11573         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
11574
11575         return pcie_device;
11576 }
11577
11578 /**
11579  * pcie_device_make_active - Add pcie device to pcie_device_list list
11580  * @ioc: per adapter object
11581  * @pcie_device: pcie device object
11582  *
11583  * Add the pcie device which has registered with SCSI Transport Later to
11584  * pcie_device_list list
11585  */
11586 static void pcie_device_make_active(struct MPT3SAS_ADAPTER *ioc,
11587                 struct _pcie_device *pcie_device)
11588 {
11589         unsigned long flags;
11590
11591         spin_lock_irqsave(&ioc->pcie_device_lock, flags);
11592
11593         if (!list_empty(&pcie_device->list)) {
11594                 list_del_init(&pcie_device->list);
11595                 pcie_device_put(pcie_device);
11596         }
11597         pcie_device_get(pcie_device);
11598         list_add_tail(&pcie_device->list, &ioc->pcie_device_list);
11599
11600         spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);
11601 }
11602
11603 /**
11604  * _scsih_probe_pcie - reporting PCIe devices to scsi-ml
11605  * @ioc: per adapter object
11606  *
11607  * Called during initial loading of the driver.
11608  */
11609 static void
11610 _scsih_probe_pcie(struct MPT3SAS_ADAPTER *ioc)
11611 {
11612         struct _pcie_device *pcie_device;
11613         int rc;
11614
11615         /* PCIe Device List */
11616         while ((pcie_device = get_next_pcie_device(ioc))) {
11617                 if (pcie_device->starget) {
11618                         pcie_device_put(pcie_device);
11619                         continue;
11620                 }
11621                 if (pcie_device->access_status ==
11622                     MPI26_PCIEDEV0_ASTATUS_DEVICE_BLOCKED) {
11623                         pcie_device_make_active(ioc, pcie_device);
11624                         pcie_device_put(pcie_device);
11625                         continue;
11626                 }
11627                 rc = scsi_add_device(ioc->shost, PCIE_CHANNEL,
11628                         pcie_device->id, 0);
11629                 if (rc) {
11630                         _scsih_pcie_device_remove(ioc, pcie_device);
11631                         pcie_device_put(pcie_device);
11632                         continue;
11633                 } else if (!pcie_device->starget) {
11634                         /*
11635                          * When async scanning is enabled, its not possible to
11636                          * remove devices while scanning is turned on due to an
11637                          * oops in scsi_sysfs_add_sdev()->add_device()->
11638                          * sysfs_addrm_start()
11639                          */
11640                         if (!ioc->is_driver_loading) {
11641                         /* TODO-- Need to find out whether this condition will
11642                          * occur or not
11643                          */
11644                                 _scsih_pcie_device_remove(ioc, pcie_device);
11645                                 pcie_device_put(pcie_device);
11646                                 continue;
11647                         }
11648                 }
11649                 pcie_device_make_active(ioc, pcie_device);
11650                 pcie_device_put(pcie_device);
11651         }
11652 }
11653
11654 /**
11655  * _scsih_probe_devices - probing for devices
11656  * @ioc: per adapter object
11657  *
11658  * Called during initial loading of the driver.
11659  */
11660 static void
11661 _scsih_probe_devices(struct MPT3SAS_ADAPTER *ioc)
11662 {
11663         u16 volume_mapping_flags;
11664
11665         if (!(ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR))
11666                 return;  /* return when IOC doesn't support initiator mode */
11667
11668         _scsih_probe_boot_devices(ioc);
11669
11670         if (ioc->ir_firmware) {
11671                 volume_mapping_flags =
11672                     le16_to_cpu(ioc->ioc_pg8.IRVolumeMappingFlags) &
11673                     MPI2_IOCPAGE8_IRFLAGS_MASK_VOLUME_MAPPING_MODE;
11674                 if (volume_mapping_flags ==
11675                     MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING) {
11676                         _scsih_probe_raid(ioc);
11677                         _scsih_probe_sas(ioc);
11678                 } else {
11679                         _scsih_probe_sas(ioc);
11680                         _scsih_probe_raid(ioc);
11681                 }
11682         } else {
11683                 _scsih_probe_sas(ioc);
11684                 _scsih_probe_pcie(ioc);
11685         }
11686 }
11687
11688 /**
11689  * scsih_scan_start - scsi lld callback for .scan_start
11690  * @shost: SCSI host pointer
11691  *
11692  * The shost has the ability to discover targets on its own instead
11693  * of scanning the entire bus.  In our implemention, we will kick off
11694  * firmware discovery.
11695  */
11696 static void
11697 scsih_scan_start(struct Scsi_Host *shost)
11698 {
11699         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
11700         int rc;
11701         if (diag_buffer_enable != -1 && diag_buffer_enable != 0)
11702                 mpt3sas_enable_diag_buffer(ioc, diag_buffer_enable);
11703         else if (ioc->manu_pg11.HostTraceBufferMaxSizeKB != 0)
11704                 mpt3sas_enable_diag_buffer(ioc, 1);
11705
11706         if (disable_discovery > 0)
11707                 return;
11708
11709         ioc->start_scan = 1;
11710         rc = mpt3sas_port_enable(ioc);
11711
11712         if (rc != 0)
11713                 ioc_info(ioc, "port enable: FAILED\n");
11714 }
11715
11716 /**
11717  * _scsih_complete_devices_scanning - add the devices to sml and
11718  * complete ioc initialization.
11719  * @ioc: per adapter object
11720  *
11721  * Return nothing.
11722  */
11723 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc)
11724 {
11725
11726         if (ioc->wait_for_discovery_to_complete) {
11727                 ioc->wait_for_discovery_to_complete = 0;
11728                 _scsih_probe_devices(ioc);
11729         }
11730
11731         mpt3sas_base_start_watchdog(ioc);
11732         ioc->is_driver_loading = 0;
11733 }
11734
11735 /**
11736  * scsih_scan_finished - scsi lld callback for .scan_finished
11737  * @shost: SCSI host pointer
11738  * @time: elapsed time of the scan in jiffies
11739  *
11740  * This function will be called periodicallyn until it returns 1 with the
11741  * scsi_host and the elapsed time of the scan in jiffies. In our implemention,
11742  * we wait for firmware discovery to complete, then return 1.
11743  */
11744 static int
11745 scsih_scan_finished(struct Scsi_Host *shost, unsigned long time)
11746 {
11747         struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);
11748         u32 ioc_state;
11749         int issue_hard_reset = 0;
11750
11751         if (disable_discovery > 0) {
11752                 ioc->is_driver_loading = 0;
11753                 ioc->wait_for_discovery_to_complete = 0;
11754                 return 1;
11755         }
11756
11757         if (time >= (300 * HZ)) {
11758                 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11759                 ioc_info(ioc, "port enable: FAILED with timeout (timeout=300s)\n");
11760                 ioc->is_driver_loading = 0;
11761                 return 1;
11762         }
11763
11764         if (ioc->start_scan) {
11765                 ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
11766                 if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
11767                         mpt3sas_print_fault_code(ioc, ioc_state &
11768                             MPI2_DOORBELL_DATA_MASK);
11769                         issue_hard_reset = 1;
11770                         goto out;
11771                 } else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
11772                                 MPI2_IOC_STATE_COREDUMP) {
11773                         mpt3sas_base_coredump_info(ioc, ioc_state &
11774                             MPI2_DOORBELL_DATA_MASK);
11775                         mpt3sas_base_wait_for_coredump_completion(ioc, __func__);
11776                         issue_hard_reset = 1;
11777                         goto out;
11778                 }
11779                 return 0;
11780         }
11781
11782         if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) {
11783                 ioc_info(ioc,
11784                     "port enable: aborted due to diag reset\n");
11785                 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11786                 goto out;
11787         }
11788         if (ioc->start_scan_failed) {
11789                 ioc_info(ioc, "port enable: FAILED with (ioc_status=0x%08x)\n",
11790                          ioc->start_scan_failed);
11791                 ioc->is_driver_loading = 0;
11792                 ioc->wait_for_discovery_to_complete = 0;
11793                 ioc->remove_host = 1;
11794                 return 1;
11795         }
11796
11797         ioc_info(ioc, "port enable: SUCCESS\n");
11798         ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11799         _scsih_complete_devices_scanning(ioc);
11800
11801 out:
11802         if (issue_hard_reset) {
11803                 ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
11804                 if (mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET))
11805                         ioc->is_driver_loading = 0;
11806         }
11807         return 1;
11808 }
11809
11810 /**
11811  * scsih_map_queues - map reply queues with request queues
11812  * @shost: SCSI host pointer
11813  */
11814 static int scsih_map_queues(struct Scsi_Host *shost)
11815 {
11816         struct MPT3SAS_ADAPTER *ioc =
11817             (struct MPT3SAS_ADAPTER *)shost->hostdata;
11818         struct blk_mq_queue_map *map;
11819         int i, qoff, offset;
11820         int nr_msix_vectors = ioc->iopoll_q_start_index;
11821         int iopoll_q_count = ioc->reply_queue_count - nr_msix_vectors;
11822
11823         if (shost->nr_hw_queues == 1)
11824                 return 0;
11825
11826         for (i = 0, qoff = 0; i < shost->nr_maps; i++) {
11827                 map = &shost->tag_set.map[i];
11828                 map->nr_queues = 0;
11829                 offset = 0;
11830                 if (i == HCTX_TYPE_DEFAULT) {
11831                         map->nr_queues =
11832                             nr_msix_vectors - ioc->high_iops_queues;
11833                         offset = ioc->high_iops_queues;
11834                 } else if (i == HCTX_TYPE_POLL)
11835                         map->nr_queues = iopoll_q_count;
11836
11837                 if (!map->nr_queues)
11838                         BUG_ON(i == HCTX_TYPE_DEFAULT);
11839
11840                 /*
11841                  * The poll queue(s) doesn't have an IRQ (and hence IRQ
11842                  * affinity), so use the regular blk-mq cpu mapping
11843                  */
11844                 map->queue_offset = qoff;
11845                 if (i != HCTX_TYPE_POLL)
11846                         blk_mq_pci_map_queues(map, ioc->pdev, offset);
11847                 else
11848                         blk_mq_map_queues(map);
11849
11850                 qoff += map->nr_queues;
11851         }
11852         return 0;
11853 }
11854
11855 /* shost template for SAS 2.0 HBA devices */
11856 static struct scsi_host_template mpt2sas_driver_template = {
11857         .module                         = THIS_MODULE,
11858         .name                           = "Fusion MPT SAS Host",
11859         .proc_name                      = MPT2SAS_DRIVER_NAME,
11860         .queuecommand                   = scsih_qcmd,
11861         .target_alloc                   = scsih_target_alloc,
11862         .slave_alloc                    = scsih_slave_alloc,
11863         .slave_configure                = scsih_slave_configure,
11864         .target_destroy                 = scsih_target_destroy,
11865         .slave_destroy                  = scsih_slave_destroy,
11866         .scan_finished                  = scsih_scan_finished,
11867         .scan_start                     = scsih_scan_start,
11868         .change_queue_depth             = scsih_change_queue_depth,
11869         .eh_abort_handler               = scsih_abort,
11870         .eh_device_reset_handler        = scsih_dev_reset,
11871         .eh_target_reset_handler        = scsih_target_reset,
11872         .eh_host_reset_handler          = scsih_host_reset,
11873         .bios_param                     = scsih_bios_param,
11874         .can_queue                      = 1,
11875         .this_id                        = -1,
11876         .sg_tablesize                   = MPT2SAS_SG_DEPTH,
11877         .max_sectors                    = 32767,
11878         .cmd_per_lun                    = 7,
11879         .shost_attrs                    = mpt3sas_host_attrs,
11880         .sdev_attrs                     = mpt3sas_dev_attrs,
11881         .track_queue_depth              = 1,
11882         .cmd_size                       = sizeof(struct scsiio_tracker),
11883 };
11884
11885 /* raid transport support for SAS 2.0 HBA devices */
11886 static struct raid_function_template mpt2sas_raid_functions = {
11887         .cookie         = &mpt2sas_driver_template,
11888         .is_raid        = scsih_is_raid,
11889         .get_resync     = scsih_get_resync,
11890         .get_state      = scsih_get_state,
11891 };
11892
11893 /* shost template for SAS 3.0 HBA devices */
11894 static struct scsi_host_template mpt3sas_driver_template = {
11895         .module                         = THIS_MODULE,
11896         .name                           = "Fusion MPT SAS Host",
11897         .proc_name                      = MPT3SAS_DRIVER_NAME,
11898         .queuecommand                   = scsih_qcmd,
11899         .target_alloc                   = scsih_target_alloc,
11900         .slave_alloc                    = scsih_slave_alloc,
11901         .slave_configure                = scsih_slave_configure,
11902         .target_destroy                 = scsih_target_destroy,
11903         .slave_destroy                  = scsih_slave_destroy,
11904         .scan_finished                  = scsih_scan_finished,
11905         .scan_start                     = scsih_scan_start,
11906         .change_queue_depth             = scsih_change_queue_depth,
11907         .eh_abort_handler               = scsih_abort,
11908         .eh_device_reset_handler        = scsih_dev_reset,
11909         .eh_target_reset_handler        = scsih_target_reset,
11910         .eh_host_reset_handler          = scsih_host_reset,
11911         .bios_param                     = scsih_bios_param,
11912         .can_queue                      = 1,
11913         .this_id                        = -1,
11914         .sg_tablesize                   = MPT3SAS_SG_DEPTH,
11915         .max_sectors                    = 32767,
11916         .max_segment_size               = 0xffffffff,
11917         .cmd_per_lun                    = 7,
11918         .shost_attrs                    = mpt3sas_host_attrs,
11919         .sdev_attrs                     = mpt3sas_dev_attrs,
11920         .track_queue_depth              = 1,
11921         .cmd_size                       = sizeof(struct scsiio_tracker),
11922         .map_queues                     = scsih_map_queues,
11923         .mq_poll                        = mpt3sas_blk_mq_poll,
11924 };
11925
11926 /* raid transport support for SAS 3.0 HBA devices */
11927 static struct raid_function_template mpt3sas_raid_functions = {
11928         .cookie         = &mpt3sas_driver_template,
11929         .is_raid        = scsih_is_raid,
11930         .get_resync     = scsih_get_resync,
11931         .get_state      = scsih_get_state,
11932 };
11933
11934 /**
11935  * _scsih_determine_hba_mpi_version - determine in which MPI version class
11936  *                                      this device belongs to.
11937  * @pdev: PCI device struct
11938  *
11939  * return MPI2_VERSION for SAS 2.0 HBA devices,
11940  *      MPI25_VERSION for SAS 3.0 HBA devices, and
11941  *      MPI26 VERSION for Cutlass & Invader SAS 3.0 HBA devices
11942  */
11943 static u16
11944 _scsih_determine_hba_mpi_version(struct pci_dev *pdev)
11945 {
11946
11947         switch (pdev->device) {
11948         case MPI2_MFGPAGE_DEVID_SSS6200:
11949         case MPI2_MFGPAGE_DEVID_SAS2004:
11950         case MPI2_MFGPAGE_DEVID_SAS2008:
11951         case MPI2_MFGPAGE_DEVID_SAS2108_1:
11952         case MPI2_MFGPAGE_DEVID_SAS2108_2:
11953         case MPI2_MFGPAGE_DEVID_SAS2108_3:
11954         case MPI2_MFGPAGE_DEVID_SAS2116_1:
11955         case MPI2_MFGPAGE_DEVID_SAS2116_2:
11956         case MPI2_MFGPAGE_DEVID_SAS2208_1:
11957         case MPI2_MFGPAGE_DEVID_SAS2208_2:
11958         case MPI2_MFGPAGE_DEVID_SAS2208_3:
11959         case MPI2_MFGPAGE_DEVID_SAS2208_4:
11960         case MPI2_MFGPAGE_DEVID_SAS2208_5:
11961         case MPI2_MFGPAGE_DEVID_SAS2208_6:
11962         case MPI2_MFGPAGE_DEVID_SAS2308_1:
11963         case MPI2_MFGPAGE_DEVID_SAS2308_2:
11964         case MPI2_MFGPAGE_DEVID_SAS2308_3:
11965         case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP:
11966         case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1:
11967                 return MPI2_VERSION;
11968         case MPI25_MFGPAGE_DEVID_SAS3004:
11969         case MPI25_MFGPAGE_DEVID_SAS3008:
11970         case MPI25_MFGPAGE_DEVID_SAS3108_1:
11971         case MPI25_MFGPAGE_DEVID_SAS3108_2:
11972         case MPI25_MFGPAGE_DEVID_SAS3108_5:
11973         case MPI25_MFGPAGE_DEVID_SAS3108_6:
11974                 return MPI25_VERSION;
11975         case MPI26_MFGPAGE_DEVID_SAS3216:
11976         case MPI26_MFGPAGE_DEVID_SAS3224:
11977         case MPI26_MFGPAGE_DEVID_SAS3316_1:
11978         case MPI26_MFGPAGE_DEVID_SAS3316_2:
11979         case MPI26_MFGPAGE_DEVID_SAS3316_3:
11980         case MPI26_MFGPAGE_DEVID_SAS3316_4:
11981         case MPI26_MFGPAGE_DEVID_SAS3324_1:
11982         case MPI26_MFGPAGE_DEVID_SAS3324_2:
11983         case MPI26_MFGPAGE_DEVID_SAS3324_3:
11984         case MPI26_MFGPAGE_DEVID_SAS3324_4:
11985         case MPI26_MFGPAGE_DEVID_SAS3508:
11986         case MPI26_MFGPAGE_DEVID_SAS3508_1:
11987         case MPI26_MFGPAGE_DEVID_SAS3408:
11988         case MPI26_MFGPAGE_DEVID_SAS3516:
11989         case MPI26_MFGPAGE_DEVID_SAS3516_1:
11990         case MPI26_MFGPAGE_DEVID_SAS3416:
11991         case MPI26_MFGPAGE_DEVID_SAS3616:
11992         case MPI26_ATLAS_PCIe_SWITCH_DEVID:
11993         case MPI26_MFGPAGE_DEVID_CFG_SEC_3916:
11994         case MPI26_MFGPAGE_DEVID_HARD_SEC_3916:
11995         case MPI26_MFGPAGE_DEVID_CFG_SEC_3816:
11996         case MPI26_MFGPAGE_DEVID_HARD_SEC_3816:
11997         case MPI26_MFGPAGE_DEVID_INVALID0_3916:
11998         case MPI26_MFGPAGE_DEVID_INVALID1_3916:
11999         case MPI26_MFGPAGE_DEVID_INVALID0_3816:
12000         case MPI26_MFGPAGE_DEVID_INVALID1_3816:
12001                 return MPI26_VERSION;
12002         }
12003         return 0;
12004 }
12005
12006 /**
12007  * _scsih_probe - attach and add scsi host
12008  * @pdev: PCI device struct
12009  * @id: pci device id
12010  *
12011  * Return: 0 success, anything else error.
12012  */
12013 static int
12014 _scsih_probe(struct pci_dev *pdev, const struct pci_device_id *id)
12015 {
12016         struct MPT3SAS_ADAPTER *ioc;
12017         struct Scsi_Host *shost = NULL;
12018         int rv;
12019         u16 hba_mpi_version;
12020         int iopoll_q_count = 0;
12021
12022         /* Determine in which MPI version class this pci device belongs */
12023         hba_mpi_version = _scsih_determine_hba_mpi_version(pdev);
12024         if (hba_mpi_version == 0)
12025                 return -ENODEV;
12026
12027         /* Enumerate only SAS 2.0 HBA's if hbas_to_enumerate is one,
12028          * for other generation HBA's return with -ENODEV
12029          */
12030         if ((hbas_to_enumerate == 1) && (hba_mpi_version !=  MPI2_VERSION))
12031                 return -ENODEV;
12032
12033         /* Enumerate only SAS 3.0 HBA's if hbas_to_enumerate is two,
12034          * for other generation HBA's return with -ENODEV
12035          */
12036         if ((hbas_to_enumerate == 2) && (!(hba_mpi_version ==  MPI25_VERSION
12037                 || hba_mpi_version ==  MPI26_VERSION)))
12038                 return -ENODEV;
12039
12040         switch (hba_mpi_version) {
12041         case MPI2_VERSION:
12042                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
12043                         PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_CLKPM);
12044                 /* Use mpt2sas driver host template for SAS 2.0 HBA's */
12045                 shost = scsi_host_alloc(&mpt2sas_driver_template,
12046                   sizeof(struct MPT3SAS_ADAPTER));
12047                 if (!shost)
12048                         return -ENODEV;
12049                 ioc = shost_priv(shost);
12050                 memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));
12051                 ioc->hba_mpi_version_belonged = hba_mpi_version;
12052                 ioc->id = mpt2_ids++;
12053                 sprintf(ioc->driver_name, "%s", MPT2SAS_DRIVER_NAME);
12054                 switch (pdev->device) {
12055                 case MPI2_MFGPAGE_DEVID_SSS6200:
12056                         ioc->is_warpdrive = 1;
12057                         ioc->hide_ir_msg = 1;
12058                         break;
12059                 case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP:
12060                 case MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1:
12061                         ioc->is_mcpu_endpoint = 1;
12062                         break;
12063                 default:
12064                         ioc->mfg_pg10_hide_flag = MFG_PAGE10_EXPOSE_ALL_DISKS;
12065                         break;
12066                 }
12067
12068                 if (multipath_on_hba == -1 || multipath_on_hba == 0)
12069                         ioc->multipath_on_hba = 0;
12070                 else
12071                         ioc->multipath_on_hba = 1;
12072
12073                 break;
12074         case MPI25_VERSION:
12075         case MPI26_VERSION:
12076                 /* Use mpt3sas driver host template for SAS 3.0 HBA's */
12077                 shost = scsi_host_alloc(&mpt3sas_driver_template,
12078                   sizeof(struct MPT3SAS_ADAPTER));
12079                 if (!shost)
12080                         return -ENODEV;
12081                 ioc = shost_priv(shost);
12082                 memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));
12083                 ioc->hba_mpi_version_belonged = hba_mpi_version;
12084                 ioc->id = mpt3_ids++;
12085                 sprintf(ioc->driver_name, "%s", MPT3SAS_DRIVER_NAME);
12086                 switch (pdev->device) {
12087                 case MPI26_MFGPAGE_DEVID_SAS3508:
12088                 case MPI26_MFGPAGE_DEVID_SAS3508_1:
12089                 case MPI26_MFGPAGE_DEVID_SAS3408:
12090                 case MPI26_MFGPAGE_DEVID_SAS3516:
12091                 case MPI26_MFGPAGE_DEVID_SAS3516_1:
12092                 case MPI26_MFGPAGE_DEVID_SAS3416:
12093                 case MPI26_MFGPAGE_DEVID_SAS3616:
12094                 case MPI26_ATLAS_PCIe_SWITCH_DEVID:
12095                         ioc->is_gen35_ioc = 1;
12096                         break;
12097                 case MPI26_MFGPAGE_DEVID_INVALID0_3816:
12098                 case MPI26_MFGPAGE_DEVID_INVALID0_3916:
12099                         dev_err(&pdev->dev,
12100                             "HBA with DeviceId 0x%04x, sub VendorId 0x%04x, sub DeviceId 0x%04x is Invalid",
12101                             pdev->device, pdev->subsystem_vendor,
12102                             pdev->subsystem_device);
12103                         return 1;
12104                 case MPI26_MFGPAGE_DEVID_INVALID1_3816:
12105                 case MPI26_MFGPAGE_DEVID_INVALID1_3916:
12106                         dev_err(&pdev->dev,
12107                             "HBA with DeviceId 0x%04x, sub VendorId 0x%04x, sub DeviceId 0x%04x is Tampered",
12108                             pdev->device, pdev->subsystem_vendor,
12109                             pdev->subsystem_device);
12110                         return 1;
12111                 case MPI26_MFGPAGE_DEVID_CFG_SEC_3816:
12112                 case MPI26_MFGPAGE_DEVID_CFG_SEC_3916:
12113                         dev_info(&pdev->dev,
12114                             "HBA is in Configurable Secure mode\n");
12115                         fallthrough;
12116                 case MPI26_MFGPAGE_DEVID_HARD_SEC_3816:
12117                 case MPI26_MFGPAGE_DEVID_HARD_SEC_3916:
12118                         ioc->is_aero_ioc = ioc->is_gen35_ioc = 1;
12119                         break;
12120                 default:
12121                         ioc->is_gen35_ioc = ioc->is_aero_ioc = 0;
12122                 }
12123                 if ((ioc->hba_mpi_version_belonged == MPI25_VERSION &&
12124                         pdev->revision >= SAS3_PCI_DEVICE_C0_REVISION) ||
12125                         (ioc->hba_mpi_version_belonged == MPI26_VERSION)) {
12126                         ioc->combined_reply_queue = 1;
12127                         if (ioc->is_gen35_ioc)
12128                                 ioc->combined_reply_index_count =
12129                                  MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G35;
12130                         else
12131                                 ioc->combined_reply_index_count =
12132                                  MPT3_SUP_REPLY_POST_HOST_INDEX_REG_COUNT_G3;
12133                 }
12134
12135                 switch (ioc->is_gen35_ioc) {
12136                 case 0:
12137                         if (multipath_on_hba == -1 || multipath_on_hba == 0)
12138                                 ioc->multipath_on_hba = 0;
12139                         else
12140                                 ioc->multipath_on_hba = 1;
12141                         break;
12142                 case 1:
12143                         if (multipath_on_hba == -1 || multipath_on_hba > 0)
12144                                 ioc->multipath_on_hba = 1;
12145                         else
12146                                 ioc->multipath_on_hba = 0;
12147                         break;
12148                 default:
12149                         break;
12150                 }
12151
12152                 break;
12153         default:
12154                 return -ENODEV;
12155         }
12156
12157         INIT_LIST_HEAD(&ioc->list);
12158         spin_lock(&gioc_lock);
12159         list_add_tail(&ioc->list, &mpt3sas_ioc_list);
12160         spin_unlock(&gioc_lock);
12161         ioc->shost = shost;
12162         ioc->pdev = pdev;
12163         ioc->scsi_io_cb_idx = scsi_io_cb_idx;
12164         ioc->tm_cb_idx = tm_cb_idx;
12165         ioc->ctl_cb_idx = ctl_cb_idx;
12166         ioc->base_cb_idx = base_cb_idx;
12167         ioc->port_enable_cb_idx = port_enable_cb_idx;
12168         ioc->transport_cb_idx = transport_cb_idx;
12169         ioc->scsih_cb_idx = scsih_cb_idx;
12170         ioc->config_cb_idx = config_cb_idx;
12171         ioc->tm_tr_cb_idx = tm_tr_cb_idx;
12172         ioc->tm_tr_volume_cb_idx = tm_tr_volume_cb_idx;
12173         ioc->tm_sas_control_cb_idx = tm_sas_control_cb_idx;
12174         ioc->logging_level = logging_level;
12175         ioc->schedule_dead_ioc_flush_running_cmds = &_scsih_flush_running_cmds;
12176         /* Host waits for minimum of six seconds */
12177         ioc->max_shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;
12178         /*
12179          * Enable MEMORY MOVE support flag.
12180          */
12181         ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_MEMMOVE;
12182         /* Enable ADDITIONAL QUERY support flag. */
12183         ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY;
12184
12185         ioc->enable_sdev_max_qd = enable_sdev_max_qd;
12186
12187         /* misc semaphores and spin locks */
12188         mutex_init(&ioc->reset_in_progress_mutex);
12189         /* initializing pci_access_mutex lock */
12190         mutex_init(&ioc->pci_access_mutex);
12191         spin_lock_init(&ioc->ioc_reset_in_progress_lock);
12192         spin_lock_init(&ioc->scsi_lookup_lock);
12193         spin_lock_init(&ioc->sas_device_lock);
12194         spin_lock_init(&ioc->sas_node_lock);
12195         spin_lock_init(&ioc->fw_event_lock);
12196         spin_lock_init(&ioc->raid_device_lock);
12197         spin_lock_init(&ioc->pcie_device_lock);
12198         spin_lock_init(&ioc->diag_trigger_lock);
12199
12200         INIT_LIST_HEAD(&ioc->sas_device_list);
12201         INIT_LIST_HEAD(&ioc->sas_device_init_list);
12202         INIT_LIST_HEAD(&ioc->sas_expander_list);
12203         INIT_LIST_HEAD(&ioc->enclosure_list);
12204         INIT_LIST_HEAD(&ioc->pcie_device_list);
12205         INIT_LIST_HEAD(&ioc->pcie_device_init_list);
12206         INIT_LIST_HEAD(&ioc->fw_event_list);
12207         INIT_LIST_HEAD(&ioc->raid_device_list);
12208         INIT_LIST_HEAD(&ioc->sas_hba.sas_port_list);
12209         INIT_LIST_HEAD(&ioc->delayed_tr_list);
12210         INIT_LIST_HEAD(&ioc->delayed_sc_list);
12211         INIT_LIST_HEAD(&ioc->delayed_event_ack_list);
12212         INIT_LIST_HEAD(&ioc->delayed_tr_volume_list);
12213         INIT_LIST_HEAD(&ioc->reply_queue_list);
12214         INIT_LIST_HEAD(&ioc->port_table_list);
12215
12216         sprintf(ioc->name, "%s_cm%d", ioc->driver_name, ioc->id);
12217
12218         /* init shost parameters */
12219         shost->max_cmd_len = 32;
12220         shost->max_lun = max_lun;
12221         shost->transportt = mpt3sas_transport_template;
12222         shost->unique_id = ioc->id;
12223
12224         if (ioc->is_mcpu_endpoint) {
12225                 /* mCPU MPI support 64K max IO */
12226                 shost->max_sectors = 128;
12227                 ioc_info(ioc, "The max_sectors value is set to %d\n",
12228                          shost->max_sectors);
12229         } else {
12230                 if (max_sectors != 0xFFFF) {
12231                         if (max_sectors < 64) {
12232                                 shost->max_sectors = 64;
12233                                 ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767. Assigning value of 64.\n",
12234                                          max_sectors);
12235                         } else if (max_sectors > 32767) {
12236                                 shost->max_sectors = 32767;
12237                                 ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767.Assigning default value of 32767.\n",
12238                                          max_sectors);
12239                         } else {
12240                                 shost->max_sectors = max_sectors & 0xFFFE;
12241                                 ioc_info(ioc, "The max_sectors value is set to %d\n",
12242                                          shost->max_sectors);
12243                         }
12244                 }
12245         }
12246         /* register EEDP capabilities with SCSI layer */
12247         if (prot_mask >= 0)
12248                 scsi_host_set_prot(shost, (prot_mask & 0x07));
12249         else
12250                 scsi_host_set_prot(shost, SHOST_DIF_TYPE1_PROTECTION
12251                                    | SHOST_DIF_TYPE2_PROTECTION
12252                                    | SHOST_DIF_TYPE3_PROTECTION);
12253
12254         scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
12255
12256         /* event thread */
12257         snprintf(ioc->firmware_event_name, sizeof(ioc->firmware_event_name),
12258             "fw_event_%s%d", ioc->driver_name, ioc->id);
12259         ioc->firmware_event_thread = alloc_ordered_workqueue(
12260             ioc->firmware_event_name, 0);
12261         if (!ioc->firmware_event_thread) {
12262                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
12263                         __FILE__, __LINE__, __func__);
12264                 rv = -ENODEV;
12265                 goto out_thread_fail;
12266         }
12267
12268         shost->host_tagset = 0;
12269
12270         if (ioc->is_gen35_ioc && host_tagset_enable)
12271                 shost->host_tagset = 1;
12272
12273         ioc->is_driver_loading = 1;
12274         if ((mpt3sas_base_attach(ioc))) {
12275                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
12276                         __FILE__, __LINE__, __func__);
12277                 rv = -ENODEV;
12278                 goto out_attach_fail;
12279         }
12280
12281         if (ioc->is_warpdrive) {
12282                 if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_EXPOSE_ALL_DISKS)
12283                         ioc->hide_drives = 0;
12284                 else if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_HIDE_ALL_DISKS)
12285                         ioc->hide_drives = 1;
12286                 else {
12287                         if (mpt3sas_get_num_volumes(ioc))
12288                                 ioc->hide_drives = 1;
12289                         else
12290                                 ioc->hide_drives = 0;
12291                 }
12292         } else
12293                 ioc->hide_drives = 0;
12294
12295         shost->nr_hw_queues = 1;
12296
12297         if (shost->host_tagset) {
12298                 shost->nr_hw_queues =
12299                     ioc->reply_queue_count - ioc->high_iops_queues;
12300
12301                 iopoll_q_count =
12302                     ioc->reply_queue_count - ioc->iopoll_q_start_index;
12303
12304                 shost->nr_maps = iopoll_q_count ? 3 : 1;
12305
12306                 dev_info(&ioc->pdev->dev,
12307                     "Max SCSIIO MPT commands: %d shared with nr_hw_queues = %d\n",
12308                     shost->can_queue, shost->nr_hw_queues);
12309         }
12310
12311         rv = scsi_add_host(shost, &pdev->dev);
12312         if (rv) {
12313                 ioc_err(ioc, "failure at %s:%d/%s()!\n",
12314                         __FILE__, __LINE__, __func__);
12315                 goto out_add_shost_fail;
12316         }
12317
12318         scsi_scan_host(shost);
12319         mpt3sas_setup_debugfs(ioc);
12320         return 0;
12321 out_add_shost_fail:
12322         mpt3sas_base_detach(ioc);
12323  out_attach_fail:
12324         destroy_workqueue(ioc->firmware_event_thread);
12325  out_thread_fail:
12326         spin_lock(&gioc_lock);
12327         list_del(&ioc->list);
12328         spin_unlock(&gioc_lock);
12329         scsi_host_put(shost);
12330         return rv;
12331 }
12332
12333 /**
12334  * scsih_suspend - power management suspend main entry point
12335  * @dev: Device struct
12336  *
12337  * Return: 0 success, anything else error.
12338  */
12339 static int __maybe_unused
12340 scsih_suspend(struct device *dev)
12341 {
12342         struct pci_dev *pdev = to_pci_dev(dev);
12343         struct Scsi_Host *shost;
12344         struct MPT3SAS_ADAPTER *ioc;
12345         int rc;
12346
12347         rc = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);
12348         if (rc)
12349                 return rc;
12350
12351         mpt3sas_base_stop_watchdog(ioc);
12352         flush_scheduled_work();
12353         scsi_block_requests(shost);
12354         _scsih_nvme_shutdown(ioc);
12355         ioc_info(ioc, "pdev=0x%p, slot=%s, entering operating state\n",
12356                  pdev, pci_name(pdev));
12357
12358         mpt3sas_base_free_resources(ioc);
12359         return 0;
12360 }
12361
12362 /**
12363  * scsih_resume - power management resume main entry point
12364  * @dev: Device struct
12365  *
12366  * Return: 0 success, anything else error.
12367  */
12368 static int __maybe_unused
12369 scsih_resume(struct device *dev)
12370 {
12371         struct pci_dev *pdev = to_pci_dev(dev);
12372         struct Scsi_Host *shost;
12373         struct MPT3SAS_ADAPTER *ioc;
12374         pci_power_t device_state = pdev->current_state;
12375         int r;
12376
12377         r = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);
12378         if (r)
12379                 return r;
12380
12381         ioc_info(ioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",
12382                  pdev, pci_name(pdev), device_state);
12383
12384         ioc->pdev = pdev;
12385         r = mpt3sas_base_map_resources(ioc);
12386         if (r)
12387                 return r;
12388         ioc_info(ioc, "Issuing Hard Reset as part of OS Resume\n");
12389         mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET);
12390         scsi_unblock_requests(shost);
12391         mpt3sas_base_start_watchdog(ioc);
12392         return 0;
12393 }
12394
12395 /**
12396  * scsih_pci_error_detected - Called when a PCI error is detected.
12397  * @pdev: PCI device struct
12398  * @state: PCI channel state
12399  *
12400  * Description: Called when a PCI error is detected.
12401  *
12402  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
12403  */
12404 static pci_ers_result_t
12405 scsih_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
12406 {
12407         struct Scsi_Host *shost;
12408         struct MPT3SAS_ADAPTER *ioc;
12409
12410         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12411                 return PCI_ERS_RESULT_DISCONNECT;
12412
12413         ioc_info(ioc, "PCI error: detected callback, state(%d)!!\n", state);
12414
12415         switch (state) {
12416         case pci_channel_io_normal:
12417                 return PCI_ERS_RESULT_CAN_RECOVER;
12418         case pci_channel_io_frozen:
12419                 /* Fatal error, prepare for slot reset */
12420                 ioc->pci_error_recovery = 1;
12421                 scsi_block_requests(ioc->shost);
12422                 mpt3sas_base_stop_watchdog(ioc);
12423                 mpt3sas_base_free_resources(ioc);
12424                 return PCI_ERS_RESULT_NEED_RESET;
12425         case pci_channel_io_perm_failure:
12426                 /* Permanent error, prepare for device removal */
12427                 ioc->pci_error_recovery = 1;
12428                 mpt3sas_base_stop_watchdog(ioc);
12429                 mpt3sas_base_pause_mq_polling(ioc);
12430                 _scsih_flush_running_cmds(ioc);
12431                 return PCI_ERS_RESULT_DISCONNECT;
12432         }
12433         return PCI_ERS_RESULT_NEED_RESET;
12434 }
12435
12436 /**
12437  * scsih_pci_slot_reset - Called when PCI slot has been reset.
12438  * @pdev: PCI device struct
12439  *
12440  * Description: This routine is called by the pci error recovery
12441  * code after the PCI slot has been reset, just before we
12442  * should resume normal operations.
12443  */
12444 static pci_ers_result_t
12445 scsih_pci_slot_reset(struct pci_dev *pdev)
12446 {
12447         struct Scsi_Host *shost;
12448         struct MPT3SAS_ADAPTER *ioc;
12449         int rc;
12450
12451         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12452                 return PCI_ERS_RESULT_DISCONNECT;
12453
12454         ioc_info(ioc, "PCI error: slot reset callback!!\n");
12455
12456         ioc->pci_error_recovery = 0;
12457         ioc->pdev = pdev;
12458         pci_restore_state(pdev);
12459         rc = mpt3sas_base_map_resources(ioc);
12460         if (rc)
12461                 return PCI_ERS_RESULT_DISCONNECT;
12462
12463         ioc_info(ioc, "Issuing Hard Reset as part of PCI Slot Reset\n");
12464         rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
12465
12466         ioc_warn(ioc, "hard reset: %s\n",
12467                  (rc == 0) ? "success" : "failed");
12468
12469         if (!rc)
12470                 return PCI_ERS_RESULT_RECOVERED;
12471         else
12472                 return PCI_ERS_RESULT_DISCONNECT;
12473 }
12474
12475 /**
12476  * scsih_pci_resume() - resume normal ops after PCI reset
12477  * @pdev: pointer to PCI device
12478  *
12479  * Called when the error recovery driver tells us that its
12480  * OK to resume normal operation. Use completion to allow
12481  * halted scsi ops to resume.
12482  */
12483 static void
12484 scsih_pci_resume(struct pci_dev *pdev)
12485 {
12486         struct Scsi_Host *shost;
12487         struct MPT3SAS_ADAPTER *ioc;
12488
12489         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12490                 return;
12491
12492         ioc_info(ioc, "PCI error: resume callback!!\n");
12493
12494         mpt3sas_base_start_watchdog(ioc);
12495         scsi_unblock_requests(ioc->shost);
12496 }
12497
12498 /**
12499  * scsih_pci_mmio_enabled - Enable MMIO and dump debug registers
12500  * @pdev: pointer to PCI device
12501  */
12502 static pci_ers_result_t
12503 scsih_pci_mmio_enabled(struct pci_dev *pdev)
12504 {
12505         struct Scsi_Host *shost;
12506         struct MPT3SAS_ADAPTER *ioc;
12507
12508         if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))
12509                 return PCI_ERS_RESULT_DISCONNECT;
12510
12511         ioc_info(ioc, "PCI error: mmio enabled callback!!\n");
12512
12513         /* TODO - dump whatever for debugging purposes */
12514
12515         /* This called only if scsih_pci_error_detected returns
12516          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
12517          * works, no need to reset slot.
12518          */
12519         return PCI_ERS_RESULT_RECOVERED;
12520 }
12521
12522 /**
12523  * scsih_ncq_prio_supp - Check for NCQ command priority support
12524  * @sdev: scsi device struct
12525  *
12526  * This is called when a user indicates they would like to enable
12527  * ncq command priorities. This works only on SATA devices.
12528  */
12529 bool scsih_ncq_prio_supp(struct scsi_device *sdev)
12530 {
12531         unsigned char *buf;
12532         bool ncq_prio_supp = false;
12533
12534         if (!scsi_device_supports_vpd(sdev))
12535                 return ncq_prio_supp;
12536
12537         buf = kmalloc(SCSI_VPD_PG_LEN, GFP_KERNEL);
12538         if (!buf)
12539                 return ncq_prio_supp;
12540
12541         if (!scsi_get_vpd_page(sdev, 0x89, buf, SCSI_VPD_PG_LEN))
12542                 ncq_prio_supp = (buf[213] >> 4) & 1;
12543
12544         kfree(buf);
12545         return ncq_prio_supp;
12546 }
12547 /*
12548  * The pci device ids are defined in mpi/mpi2_cnfg.h.
12549  */
12550 static const struct pci_device_id mpt3sas_pci_table[] = {
12551         /* Spitfire ~ 2004 */
12552         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2004,
12553                 PCI_ANY_ID, PCI_ANY_ID },
12554         /* Falcon ~ 2008 */
12555         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2008,
12556                 PCI_ANY_ID, PCI_ANY_ID },
12557         /* Liberator ~ 2108 */
12558         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_1,
12559                 PCI_ANY_ID, PCI_ANY_ID },
12560         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_2,
12561                 PCI_ANY_ID, PCI_ANY_ID },
12562         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2108_3,
12563                 PCI_ANY_ID, PCI_ANY_ID },
12564         /* Meteor ~ 2116 */
12565         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2116_1,
12566                 PCI_ANY_ID, PCI_ANY_ID },
12567         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2116_2,
12568                 PCI_ANY_ID, PCI_ANY_ID },
12569         /* Thunderbolt ~ 2208 */
12570         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_1,
12571                 PCI_ANY_ID, PCI_ANY_ID },
12572         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_2,
12573                 PCI_ANY_ID, PCI_ANY_ID },
12574         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_3,
12575                 PCI_ANY_ID, PCI_ANY_ID },
12576         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_4,
12577                 PCI_ANY_ID, PCI_ANY_ID },
12578         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_5,
12579                 PCI_ANY_ID, PCI_ANY_ID },
12580         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2208_6,
12581                 PCI_ANY_ID, PCI_ANY_ID },
12582         /* Mustang ~ 2308 */
12583         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_1,
12584                 PCI_ANY_ID, PCI_ANY_ID },
12585         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_2,
12586                 PCI_ANY_ID, PCI_ANY_ID },
12587         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SAS2308_3,
12588                 PCI_ANY_ID, PCI_ANY_ID },
12589         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP,
12590                 PCI_ANY_ID, PCI_ANY_ID },
12591         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SWITCH_MPI_EP_1,
12592                 PCI_ANY_ID, PCI_ANY_ID },
12593         /* SSS6200 */
12594         { MPI2_MFGPAGE_VENDORID_LSI, MPI2_MFGPAGE_DEVID_SSS6200,
12595                 PCI_ANY_ID, PCI_ANY_ID },
12596         /* Fury ~ 3004 and 3008 */
12597         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3004,
12598                 PCI_ANY_ID, PCI_ANY_ID },
12599         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3008,
12600                 PCI_ANY_ID, PCI_ANY_ID },
12601         /* Invader ~ 3108 */
12602         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_1,
12603                 PCI_ANY_ID, PCI_ANY_ID },
12604         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_2,
12605                 PCI_ANY_ID, PCI_ANY_ID },
12606         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_5,
12607                 PCI_ANY_ID, PCI_ANY_ID },
12608         { MPI2_MFGPAGE_VENDORID_LSI, MPI25_MFGPAGE_DEVID_SAS3108_6,
12609                 PCI_ANY_ID, PCI_ANY_ID },
12610         /* Cutlass ~ 3216 and 3224 */
12611         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3216,
12612                 PCI_ANY_ID, PCI_ANY_ID },
12613         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3224,
12614                 PCI_ANY_ID, PCI_ANY_ID },
12615         /* Intruder ~ 3316 and 3324 */
12616         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_1,
12617                 PCI_ANY_ID, PCI_ANY_ID },
12618         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_2,
12619                 PCI_ANY_ID, PCI_ANY_ID },
12620         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_3,
12621                 PCI_ANY_ID, PCI_ANY_ID },
12622         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3316_4,
12623                 PCI_ANY_ID, PCI_ANY_ID },
12624         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_1,
12625                 PCI_ANY_ID, PCI_ANY_ID },
12626         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_2,
12627                 PCI_ANY_ID, PCI_ANY_ID },
12628         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_3,
12629                 PCI_ANY_ID, PCI_ANY_ID },
12630         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3324_4,
12631                 PCI_ANY_ID, PCI_ANY_ID },
12632         /* Ventura, Crusader, Harpoon & Tomcat ~ 3516, 3416, 3508 & 3408*/
12633         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508,
12634                 PCI_ANY_ID, PCI_ANY_ID },
12635         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3508_1,
12636                 PCI_ANY_ID, PCI_ANY_ID },
12637         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3408,
12638                 PCI_ANY_ID, PCI_ANY_ID },
12639         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516,
12640                 PCI_ANY_ID, PCI_ANY_ID },
12641         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3516_1,
12642                 PCI_ANY_ID, PCI_ANY_ID },
12643         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3416,
12644                 PCI_ANY_ID, PCI_ANY_ID },
12645         /* Mercator ~ 3616*/
12646         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_SAS3616,
12647                 PCI_ANY_ID, PCI_ANY_ID },
12648
12649         /* Aero SI 0x00E1 Configurable Secure
12650          * 0x00E2 Hard Secure
12651          */
12652         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_CFG_SEC_3916,
12653                 PCI_ANY_ID, PCI_ANY_ID },
12654         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3916,
12655                 PCI_ANY_ID, PCI_ANY_ID },
12656
12657         /*
12658          *  Aero SI –> 0x00E0 Invalid, 0x00E3 Tampered
12659          */
12660         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID0_3916,
12661                 PCI_ANY_ID, PCI_ANY_ID },
12662         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID1_3916,
12663                 PCI_ANY_ID, PCI_ANY_ID },
12664
12665         /* Atlas PCIe Switch Management Port */
12666         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_ATLAS_PCIe_SWITCH_DEVID,
12667                 PCI_ANY_ID, PCI_ANY_ID },
12668
12669         /* Sea SI 0x00E5 Configurable Secure
12670          * 0x00E6 Hard Secure
12671          */
12672         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_CFG_SEC_3816,
12673                 PCI_ANY_ID, PCI_ANY_ID },
12674         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3816,
12675                 PCI_ANY_ID, PCI_ANY_ID },
12676
12677         /*
12678          *  Sea SI –> 0x00E4 Invalid, 0x00E7 Tampered
12679          */
12680         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID0_3816,
12681                 PCI_ANY_ID, PCI_ANY_ID },
12682         { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_INVALID1_3816,
12683                 PCI_ANY_ID, PCI_ANY_ID },
12684
12685         {0}     /* Terminating entry */
12686 };
12687 MODULE_DEVICE_TABLE(pci, mpt3sas_pci_table);
12688
12689 static struct pci_error_handlers _mpt3sas_err_handler = {
12690         .error_detected = scsih_pci_error_detected,
12691         .mmio_enabled   = scsih_pci_mmio_enabled,
12692         .slot_reset     = scsih_pci_slot_reset,
12693         .resume         = scsih_pci_resume,
12694 };
12695
12696 static SIMPLE_DEV_PM_OPS(scsih_pm_ops, scsih_suspend, scsih_resume);
12697
12698 static struct pci_driver mpt3sas_driver = {
12699         .name           = MPT3SAS_DRIVER_NAME,
12700         .id_table       = mpt3sas_pci_table,
12701         .probe          = _scsih_probe,
12702         .remove         = scsih_remove,
12703         .shutdown       = scsih_shutdown,
12704         .err_handler    = &_mpt3sas_err_handler,
12705         .driver.pm      = &scsih_pm_ops,
12706 };
12707
12708 /**
12709  * scsih_init - main entry point for this driver.
12710  *
12711  * Return: 0 success, anything else error.
12712  */
12713 static int
12714 scsih_init(void)
12715 {
12716         mpt2_ids = 0;
12717         mpt3_ids = 0;
12718
12719         mpt3sas_base_initialize_callback_handler();
12720
12721          /* queuecommand callback hander */
12722         scsi_io_cb_idx = mpt3sas_base_register_callback_handler(_scsih_io_done);
12723
12724         /* task management callback handler */
12725         tm_cb_idx = mpt3sas_base_register_callback_handler(_scsih_tm_done);
12726
12727         /* base internal commands callback handler */
12728         base_cb_idx = mpt3sas_base_register_callback_handler(mpt3sas_base_done);
12729         port_enable_cb_idx = mpt3sas_base_register_callback_handler(
12730             mpt3sas_port_enable_done);
12731
12732         /* transport internal commands callback handler */
12733         transport_cb_idx = mpt3sas_base_register_callback_handler(
12734             mpt3sas_transport_done);
12735
12736         /* scsih internal commands callback handler */
12737         scsih_cb_idx = mpt3sas_base_register_callback_handler(_scsih_done);
12738
12739         /* configuration page API internal commands callback handler */
12740         config_cb_idx = mpt3sas_base_register_callback_handler(
12741             mpt3sas_config_done);
12742
12743         /* ctl module callback handler */
12744         ctl_cb_idx = mpt3sas_base_register_callback_handler(mpt3sas_ctl_done);
12745
12746         tm_tr_cb_idx = mpt3sas_base_register_callback_handler(
12747             _scsih_tm_tr_complete);
12748
12749         tm_tr_volume_cb_idx = mpt3sas_base_register_callback_handler(
12750             _scsih_tm_volume_tr_complete);
12751
12752         tm_sas_control_cb_idx = mpt3sas_base_register_callback_handler(
12753             _scsih_sas_control_complete);
12754
12755         mpt3sas_init_debugfs();
12756         return 0;
12757 }
12758
12759 /**
12760  * scsih_exit - exit point for this driver (when it is a module).
12761  *
12762  * Return: 0 success, anything else error.
12763  */
12764 static void
12765 scsih_exit(void)
12766 {
12767
12768         mpt3sas_base_release_callback_handler(scsi_io_cb_idx);
12769         mpt3sas_base_release_callback_handler(tm_cb_idx);
12770         mpt3sas_base_release_callback_handler(base_cb_idx);
12771         mpt3sas_base_release_callback_handler(port_enable_cb_idx);
12772         mpt3sas_base_release_callback_handler(transport_cb_idx);
12773         mpt3sas_base_release_callback_handler(scsih_cb_idx);
12774         mpt3sas_base_release_callback_handler(config_cb_idx);
12775         mpt3sas_base_release_callback_handler(ctl_cb_idx);
12776
12777         mpt3sas_base_release_callback_handler(tm_tr_cb_idx);
12778         mpt3sas_base_release_callback_handler(tm_tr_volume_cb_idx);
12779         mpt3sas_base_release_callback_handler(tm_sas_control_cb_idx);
12780
12781 /* raid transport support */
12782         if (hbas_to_enumerate != 1)
12783                 raid_class_release(mpt3sas_raid_template);
12784         if (hbas_to_enumerate != 2)
12785                 raid_class_release(mpt2sas_raid_template);
12786         sas_release_transport(mpt3sas_transport_template);
12787         mpt3sas_exit_debugfs();
12788 }
12789
12790 /**
12791  * _mpt3sas_init - main entry point for this driver.
12792  *
12793  * Return: 0 success, anything else error.
12794  */
12795 static int __init
12796 _mpt3sas_init(void)
12797 {
12798         int error;
12799
12800         pr_info("%s version %s loaded\n", MPT3SAS_DRIVER_NAME,
12801                                         MPT3SAS_DRIVER_VERSION);
12802
12803         mpt3sas_transport_template =
12804             sas_attach_transport(&mpt3sas_transport_functions);
12805         if (!mpt3sas_transport_template)
12806                 return -ENODEV;
12807
12808         /* No need attach mpt3sas raid functions template
12809          * if hbas_to_enumarate value is one.
12810          */
12811         if (hbas_to_enumerate != 1) {
12812                 mpt3sas_raid_template =
12813                                 raid_class_attach(&mpt3sas_raid_functions);
12814                 if (!mpt3sas_raid_template) {
12815                         sas_release_transport(mpt3sas_transport_template);
12816                         return -ENODEV;
12817                 }
12818         }
12819
12820         /* No need to attach mpt2sas raid functions template
12821          * if hbas_to_enumarate value is two
12822          */
12823         if (hbas_to_enumerate != 2) {
12824                 mpt2sas_raid_template =
12825                                 raid_class_attach(&mpt2sas_raid_functions);
12826                 if (!mpt2sas_raid_template) {
12827                         sas_release_transport(mpt3sas_transport_template);
12828                         return -ENODEV;
12829                 }
12830         }
12831
12832         error = scsih_init();
12833         if (error) {
12834                 scsih_exit();
12835                 return error;
12836         }
12837
12838         mpt3sas_ctl_init(hbas_to_enumerate);
12839
12840         error = pci_register_driver(&mpt3sas_driver);
12841         if (error)
12842                 scsih_exit();
12843
12844         return error;
12845 }
12846
12847 /**
12848  * _mpt3sas_exit - exit point for this driver (when it is a module).
12849  *
12850  */
12851 static void __exit
12852 _mpt3sas_exit(void)
12853 {
12854         pr_info("mpt3sas version %s unloading\n",
12855                                 MPT3SAS_DRIVER_VERSION);
12856
12857         mpt3sas_ctl_exit(hbas_to_enumerate);
12858
12859         pci_unregister_driver(&mpt3sas_driver);
12860
12861         scsih_exit();
12862 }
12863
12864 module_init(_mpt3sas_init);
12865 module_exit(_mpt3sas_exit);