drivers/scsi/ufs/ufshcd.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Universal Flash Storage Host controller driver Core
   4  * Copyright (C) 2011-2013 Samsung India Software Operations
   5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   6  *
   7  * Authors:
   8  *      Santosh Yaraganavi <santosh.sy@samsung.com>
   9  *      Vinayak Holikatti <h.vinayak@samsung.com>
  10  */
  11
  12 #include <linux/async.h>
  13 #include <linux/devfreq.h>
  14 #include <linux/nls.h>
  15 #include <linux/of.h>
  16 #include <linux/bitfield.h>
  17 #include <linux/blk-pm.h>
  18 #include <linux/blkdev.h>
  19 #include <scsi/scsi_driver.h>
  20 #include "ufshcd.h"
  21 #include "ufs_quirks.h"
  22 #include "unipro.h"
  23 #include "ufs-sysfs.h"
  24 #include "ufs-debugfs.h"
  25 #include "ufs_bsg.h"
  26 #include "ufshcd-crypto.h"
  27 #include <asm/unaligned.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/ufs.h>
  31
  32 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  33                                  UTP_TASK_REQ_COMPL |\
  34                                  UFSHCD_ERROR_MASK)
  35 /* UIC command timeout, unit: ms */
  36 #define UIC_CMD_TIMEOUT 500
  37
  38 /* NOP OUT retries waiting for NOP IN response */
  39 #define NOP_OUT_RETRIES    10
  40 /* Timeout after 50 msecs if NOP OUT hangs without response */
  41 #define NOP_OUT_TIMEOUT    50 /* msecs */
  42
  43 /* Query request retries */
  44 #define QUERY_REQ_RETRIES 3
  45 /* Query request timeout */
  46 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  47
  48 /* Task management command timeout */
  49 #define TM_CMD_TIMEOUT  100 /* msecs */
  50
  51 /* maximum number of retries for a general UIC command  */
  52 #define UFS_UIC_COMMAND_RETRIES 3
  53
  54 /* maximum number of link-startup retries */
  55 #define DME_LINKSTARTUP_RETRIES 3
  56
  57 /* Maximum retries for Hibern8 enter */
  58 #define UIC_HIBERN8_ENTER_RETRIES 3
  59
  60 /* maximum number of reset retries before giving up */
  61 #define MAX_HOST_RESET_RETRIES 5
  62
  63 /* Expose the flag value from utp_upiu_query.value */
  64 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  65
  66 /* Interrupt aggregation default timeout, unit: 40us */
  67 #define INT_AGGR_DEF_TO 0x02
  68
  69 /* default delay of autosuspend: 2000 ms */
  70 #define RPM_AUTOSUSPEND_DELAY_MS 2000
  71
  72 /* Default delay of RPM device flush delayed work */
  73 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
  74
  75 /* Default value of wait time before gating device ref clock */
  76 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
  77
  78 /* Polling time to wait for fDeviceInit */
  79 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
  80
  81 #define wlun_dev_to_hba(dv) shost_priv(to_scsi_device(dv)->host)
  82
  83 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
  84         ({                                                              \
  85                 int _ret;                                               \
  86                 if (_on)                                                \
  87                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
  88                 else                                                    \
  89                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
  90                 _ret;                                                   \
  91         })
  92
  93 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
  94         size_t __len = (len);                                            \
  95         print_hex_dump(KERN_ERR, prefix_str,                             \
  96                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
  97                        16, 4, buf, __len, false);                        \
  98 } while (0)
  99
 100 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 101                      const char *prefix)
 102 {
 103         u32 *regs;
 104         size_t pos;
 105
 106         if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 107                 return -EINVAL;
 108
 109         regs = kzalloc(len, GFP_ATOMIC);
 110         if (!regs)
 111                 return -ENOMEM;
 112
 113         for (pos = 0; pos < len; pos += 4)
 114                 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 115
 116         ufshcd_hex_dump(prefix, regs, len);
 117         kfree(regs);
 118
 119         return 0;
 120 }
 121 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 122
 123 enum {
 124         UFSHCD_MAX_CHANNEL      = 0,
 125         UFSHCD_MAX_ID           = 1,
 126         UFSHCD_CMD_PER_LUN      = 32,
 127         UFSHCD_CAN_QUEUE        = 32,
 128 };
 129
 130 /* UFSHCD states */
 131 enum {
 132         UFSHCD_STATE_RESET,
 133         UFSHCD_STATE_ERROR,
 134         UFSHCD_STATE_OPERATIONAL,
 135         UFSHCD_STATE_EH_SCHEDULED_FATAL,
 136         UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
 137 };
 138
 139 /* UFSHCD error handling flags */
 140 enum {
 141         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 142 };
 143
 144 /* UFSHCD UIC layer error flags */
 145 enum {
 146         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 147         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 148         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 149         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 150         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 151         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 152         UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
 153 };
 154
 155 #define ufshcd_set_eh_in_progress(h) \
 156         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 157 #define ufshcd_eh_in_progress(h) \
 158         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 159 #define ufshcd_clear_eh_in_progress(h) \
 160         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 161
 162 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 163         [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 164         [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 165         [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 166         [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 167         [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 168         [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 169         /*
 170          * For DeepSleep, the link is first put in hibern8 and then off.
 171          * Leaving the link in hibern8 is not supported.
 172          */
 173         [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
 174 };
 175
 176 static inline enum ufs_dev_pwr_mode
 177 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 178 {
 179         return ufs_pm_lvl_states[lvl].dev_state;
 180 }
 181
 182 static inline enum uic_link_state
 183 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 184 {
 185         return ufs_pm_lvl_states[lvl].link_state;
 186 }
 187
 188 static inline enum ufs_pm_level
 189 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 190                                         enum uic_link_state link_state)
 191 {
 192         enum ufs_pm_level lvl;
 193
 194         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 195                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 196                         (ufs_pm_lvl_states[lvl].link_state == link_state))
 197                         return lvl;
 198         }
 199
 200         /* if no match found, return the level 0 */
 201         return UFS_PM_LVL_0;
 202 }
 203
 204 static struct ufs_dev_fix ufs_fixups[] = {
 205         /* UFS cards deviations table */
 206         UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
 207                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 208         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 209                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 210                 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
 211                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
 212         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
 213                 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
 214         UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
 215                 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
 216         UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
 217                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 218         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
 219                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
 220         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
 221                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
 222         END_FIX
 223 };
 224
 225 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
 226 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 227 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 228 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 229 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 230 static void ufshcd_hba_exit(struct ufs_hba *hba);
 231 static int ufshcd_clear_ua_wluns(struct ufs_hba *hba);
 232 static int ufshcd_probe_hba(struct ufs_hba *hba, bool async);
 233 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 234 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
 235 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 236 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 237 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 238 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 239 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 240 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 241 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 242 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 243                              struct ufs_pa_layer_attr *pwr_mode);
 244 static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
 245 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
 246 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
 247 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
 248                                          struct ufs_vreg *vreg);
 249 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
 250 static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set);
 251 static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable);
 252 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
 253 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
 254
 255 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
 256 {
 257         return tag >= 0 && tag < hba->nutrs;
 258 }
 259
 260 static inline void ufshcd_enable_irq(struct ufs_hba *hba)
 261 {
 262         if (!hba->is_irq_enabled) {
 263                 enable_irq(hba->irq);
 264                 hba->is_irq_enabled = true;
 265         }
 266 }
 267
 268 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 269 {
 270         if (hba->is_irq_enabled) {
 271                 disable_irq(hba->irq);
 272                 hba->is_irq_enabled = false;
 273         }
 274 }
 275
 276 static inline void ufshcd_wb_config(struct ufs_hba *hba)
 277 {
 278         if (!ufshcd_is_wb_allowed(hba))
 279                 return;
 280
 281         ufshcd_wb_toggle(hba, true);
 282
 283         ufshcd_wb_toggle_flush_during_h8(hba, true);
 284         if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL))
 285                 ufshcd_wb_toggle_flush(hba, true);
 286 }
 287
 288 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 289 {
 290         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
 291                 scsi_unblock_requests(hba->host);
 292 }
 293
 294 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
 295 {
 296         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
 297                 scsi_block_requests(hba->host);
 298 }
 299
 300 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 301                                       enum ufs_trace_str_t str_t)
 302 {
 303         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 304         struct utp_upiu_header *header;
 305
 306         if (!trace_ufshcd_upiu_enabled())
 307                 return;
 308
 309         if (str_t == UFS_CMD_SEND)
 310                 header = &rq->header;
 311         else
 312                 header = &hba->lrb[tag].ucd_rsp_ptr->header;
 313
 314         trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
 315                           UFS_TSF_CDB);
 316 }
 317
 318 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
 319                                         enum ufs_trace_str_t str_t,
 320                                         struct utp_upiu_req *rq_rsp)
 321 {
 322         if (!trace_ufshcd_upiu_enabled())
 323                 return;
 324
 325         trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
 326                           &rq_rsp->qr, UFS_TSF_OSF);
 327 }
 328
 329 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 330                                      enum ufs_trace_str_t str_t)
 331 {
 332         int off = (int)tag - hba->nutrs;
 333         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[off];
 334
 335         if (!trace_ufshcd_upiu_enabled())
 336                 return;
 337
 338         if (str_t == UFS_TM_SEND)
 339                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 340                                   &descp->upiu_req.req_header,
 341                                   &descp->upiu_req.input_param1,
 342                                   UFS_TSF_TM_INPUT);
 343         else
 344                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 345                                   &descp->upiu_rsp.rsp_header,
 346                                   &descp->upiu_rsp.output_param1,
 347                                   UFS_TSF_TM_OUTPUT);
 348 }
 349
 350 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
 351                                          struct uic_command *ucmd,
 352                                          enum ufs_trace_str_t str_t)
 353 {
 354         u32 cmd;
 355
 356         if (!trace_ufshcd_uic_command_enabled())
 357                 return;
 358
 359         if (str_t == UFS_CMD_SEND)
 360                 cmd = ucmd->command;
 361         else
 362                 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
 363
 364         trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
 365                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
 366                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
 367                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
 368 }
 369
 370 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 371                                      enum ufs_trace_str_t str_t)
 372 {
 373         u64 lba = -1;
 374         u8 opcode = 0, group_id = 0;
 375         u32 intr, doorbell;
 376         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 377         struct scsi_cmnd *cmd = lrbp->cmd;
 378         int transfer_len = -1;
 379
 380         if (!cmd)
 381                 return;
 382
 383         if (!trace_ufshcd_command_enabled()) {
 384                 /* trace UPIU W/O tracing command */
 385                 ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
 386                 return;
 387         }
 388
 389         /* trace UPIU also */
 390         ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
 391         opcode = cmd->cmnd[0];
 392         lba = scsi_get_lba(cmd);
 393
 394         if (opcode == READ_10 || opcode == WRITE_10) {
 395                 /*
 396                  * Currently we only fully trace read(10) and write(10) commands
 397                  */
 398                 transfer_len =
 399                        be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 400                 if (opcode == WRITE_10)
 401                         group_id = lrbp->cmd->cmnd[6];
 402         } else if (opcode == UNMAP) {
 403                 /*
 404                  * The number of Bytes to be unmapped beginning with the lba.
 405                  */
 406                 transfer_len = blk_rq_bytes(cmd->request);
 407         }
 408
 409         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 410         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 411         trace_ufshcd_command(dev_name(hba->dev), str_t, tag,
 412                         doorbell, transfer_len, intr, lba, opcode, group_id);
 413 }
 414
 415 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 416 {
 417         struct ufs_clk_info *clki;
 418         struct list_head *head = &hba->clk_list_head;
 419
 420         if (list_empty(head))
 421                 return;
 422
 423         list_for_each_entry(clki, head, list) {
 424                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 425                                 clki->max_freq)
 426                         dev_err(hba->dev, "clk: %s, rate: %u\n",
 427                                         clki->name, clki->curr_freq);
 428         }
 429 }
 430
 431 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
 432                              char *err_name)
 433 {
 434         int i;
 435         bool found = false;
 436         struct ufs_event_hist *e;
 437
 438         if (id >= UFS_EVT_CNT)
 439                 return;
 440
 441         e = &hba->ufs_stats.event[id];
 442
 443         for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
 444                 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
 445
 446                 if (e->tstamp[p] == 0)
 447                         continue;
 448                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
 449                         e->val[p], ktime_to_us(e->tstamp[p]));
 450                 found = true;
 451         }
 452
 453         if (!found)
 454                 dev_err(hba->dev, "No record of %s\n", err_name);
 455         else
 456                 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
 457 }
 458
 459 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
 460 {
 461         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 462
 463         ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
 464         ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
 465         ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
 466         ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
 467         ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
 468         ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
 469                          "auto_hibern8_err");
 470         ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
 471         ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
 472                          "link_startup_fail");
 473         ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
 474         ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
 475                          "suspend_fail");
 476         ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
 477         ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
 478         ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
 479
 480         ufshcd_vops_dbg_register_dump(hba);
 481 }
 482
 483 static
 484 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
 485 {
 486         struct ufshcd_lrb *lrbp;
 487         int prdt_length;
 488         int tag;
 489
 490         for_each_set_bit(tag, &bitmap, hba->nutrs) {
 491                 lrbp = &hba->lrb[tag];
 492
 493                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 494                                 tag, ktime_to_us(lrbp->issue_time_stamp));
 495                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 496                                 tag, ktime_to_us(lrbp->compl_time_stamp));
 497                 dev_err(hba->dev,
 498                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 499                         tag, (u64)lrbp->utrd_dma_addr);
 500
 501                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 502                                 sizeof(struct utp_transfer_req_desc));
 503                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 504                         (u64)lrbp->ucd_req_dma_addr);
 505                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 506                                 sizeof(struct utp_upiu_req));
 507                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 508                         (u64)lrbp->ucd_rsp_dma_addr);
 509                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 510                                 sizeof(struct utp_upiu_rsp));
 511
 512                 prdt_length = le16_to_cpu(
 513                         lrbp->utr_descriptor_ptr->prd_table_length);
 514                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
 515                         prdt_length /= sizeof(struct ufshcd_sg_entry);
 516
 517                 dev_err(hba->dev,
 518                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 519                         tag, prdt_length,
 520                         (u64)lrbp->ucd_prdt_dma_addr);
 521
 522                 if (pr_prdt)
 523                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 524                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
 525         }
 526 }
 527
 528 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 529 {
 530         int tag;
 531
 532         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 533                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
 534
 535                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 536                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 537         }
 538 }
 539
 540 static void ufshcd_print_host_state(struct ufs_hba *hba)
 541 {
 542         struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
 543
 544         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 545         dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
 546                 hba->outstanding_reqs, hba->outstanding_tasks);
 547         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 548                 hba->saved_err, hba->saved_uic_err);
 549         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 550                 hba->curr_dev_pwr_mode, hba->uic_link_state);
 551         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 552                 hba->pm_op_in_progress, hba->is_sys_suspended);
 553         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 554                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
 555         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 556         dev_err(hba->dev,
 557                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
 558                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
 559                 hba->ufs_stats.hibern8_exit_cnt);
 560         dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
 561                 ktime_to_us(hba->ufs_stats.last_intr_ts),
 562                 hba->ufs_stats.last_intr_status);
 563         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 564                 hba->eh_flags, hba->req_abort_count);
 565         dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
 566                 hba->ufs_version, hba->capabilities, hba->caps);
 567         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 568                 hba->dev_quirks);
 569         if (sdev_ufs)
 570                 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
 571                         sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
 572
 573         ufshcd_print_clk_freqs(hba);
 574 }
 575
 576 /**
 577  * ufshcd_print_pwr_info - print power params as saved in hba
 578  * power info
 579  * @hba: per-adapter instance
 580  */
 581 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 582 {
 583         static const char * const names[] = {
 584                 "INVALID MODE",
 585                 "FAST MODE",
 586                 "SLOW_MODE",
 587                 "INVALID MODE",
 588                 "FASTAUTO_MODE",
 589                 "SLOWAUTO_MODE",
 590                 "INVALID MODE",
 591         };
 592
 593         dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 594                  __func__,
 595                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 596                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 597                  names[hba->pwr_info.pwr_rx],
 598                  names[hba->pwr_info.pwr_tx],
 599                  hba->pwr_info.hs_rate);
 600 }
 601
 602 static void ufshcd_device_reset(struct ufs_hba *hba)
 603 {
 604         int err;
 605
 606         err = ufshcd_vops_device_reset(hba);
 607
 608         if (!err) {
 609                 ufshcd_set_ufs_dev_active(hba);
 610                 if (ufshcd_is_wb_allowed(hba)) {
 611                         hba->dev_info.wb_enabled = false;
 612                         hba->dev_info.wb_buf_flush_enabled = false;
 613                 }
 614         }
 615         if (err != -EOPNOTSUPP)
 616                 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
 617 }
 618
 619 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
 620 {
 621         if (!us)
 622                 return;
 623
 624         if (us < 10)
 625                 udelay(us);
 626         else
 627                 usleep_range(us, us + tolerance);
 628 }
 629 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
 630
 631 /**
 632  * ufshcd_wait_for_register - wait for register value to change
 633  * @hba: per-adapter interface
 634  * @reg: mmio register offset
 635  * @mask: mask to apply to the read register value
 636  * @val: value to wait for
 637  * @interval_us: polling interval in microseconds
 638  * @timeout_ms: timeout in milliseconds
 639  *
 640  * Return:
 641  * -ETIMEDOUT on error, zero on success.
 642  */
 643 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 644                                 u32 val, unsigned long interval_us,
 645                                 unsigned long timeout_ms)
 646 {
 647         int err = 0;
 648         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 649
 650         /* ignore bits that we don't intend to wait on */
 651         val = val & mask;
 652
 653         while ((ufshcd_readl(hba, reg) & mask) != val) {
 654                 usleep_range(interval_us, interval_us + 50);
 655                 if (time_after(jiffies, timeout)) {
 656                         if ((ufshcd_readl(hba, reg) & mask) != val)
 657                                 err = -ETIMEDOUT;
 658                         break;
 659                 }
 660         }
 661
 662         return err;
 663 }
 664
 665 /**
 666  * ufshcd_get_intr_mask - Get the interrupt bit mask
 667  * @hba: Pointer to adapter instance
 668  *
 669  * Returns interrupt bit mask per version
 670  */
 671 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 672 {
 673         if (hba->ufs_version == ufshci_version(1, 0))
 674                 return INTERRUPT_MASK_ALL_VER_10;
 675         if (hba->ufs_version <= ufshci_version(2, 0))
 676                 return INTERRUPT_MASK_ALL_VER_11;
 677
 678         return INTERRUPT_MASK_ALL_VER_21;
 679 }
 680
 681 /**
 682  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 683  * @hba: Pointer to adapter instance
 684  *
 685  * Returns UFSHCI version supported by the controller
 686  */
 687 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 688 {
 689         u32 ufshci_ver;
 690
 691         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 692                 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
 693         else
 694                 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
 695
 696         /*
 697          * UFSHCI v1.x uses a different version scheme, in order
 698          * to allow the use of comparisons with the ufshci_version
 699          * function, we convert it to the same scheme as ufs 2.0+.
 700          */
 701         if (ufshci_ver & 0x00010000)
 702                 return ufshci_version(1, ufshci_ver & 0x00000100);
 703
 704         return ufshci_ver;
 705 }
 706
 707 /**
 708  * ufshcd_is_device_present - Check if any device connected to
 709  *                            the host controller
 710  * @hba: pointer to adapter instance
 711  *
 712  * Returns true if device present, false if no device detected
 713  */
 714 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 715 {
 716         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 717                                                 DEVICE_PRESENT) ? true : false;
 718 }
 719
 720 /**
 721  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 722  * @lrbp: pointer to local command reference block
 723  *
 724  * This function is used to get the OCS field from UTRD
 725  * Returns the OCS field in the UTRD
 726  */
 727 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 728 {
 729         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
 730 }
 731
 732 /**
 733  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 734  * @hba: per adapter instance
 735  * @pos: position of the bit to be cleared
 736  */
 737 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 738 {
 739         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 740                 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 741         else
 742                 ufshcd_writel(hba, ~(1 << pos),
 743                                 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 744 }
 745
 746 /**
 747  * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
 748  * @hba: per adapter instance
 749  * @pos: position of the bit to be cleared
 750  */
 751 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 752 {
 753         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 754                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 755         else
 756                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 757 }
 758
 759 /**
 760  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
 761  * @hba: per adapter instance
 762  * @tag: position of the bit to be cleared
 763  */
 764 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
 765 {
 766         clear_bit(tag, &hba->outstanding_reqs);
 767 }
 768
 769 /**
 770  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 771  * @reg: Register value of host controller status
 772  *
 773  * Returns integer, 0 on Success and positive value if failed
 774  */
 775 static inline int ufshcd_get_lists_status(u32 reg)
 776 {
 777         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 778 }
 779
 780 /**
 781  * ufshcd_get_uic_cmd_result - Get the UIC command result
 782  * @hba: Pointer to adapter instance
 783  *
 784  * This function gets the result of UIC command completion
 785  * Returns 0 on success, non zero value on error
 786  */
 787 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 788 {
 789         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 790                MASK_UIC_COMMAND_RESULT;
 791 }
 792
 793 /**
 794  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 795  * @hba: Pointer to adapter instance
 796  *
 797  * This function gets UIC command argument3
 798  * Returns 0 on success, non zero value on error
 799  */
 800 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 801 {
 802         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 803 }
 804
 805 /**
 806  * ufshcd_get_req_rsp - returns the TR response transaction type
 807  * @ucd_rsp_ptr: pointer to response UPIU
 808  */
 809 static inline int
 810 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 811 {
 812         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 813 }
 814
 815 /**
 816  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 817  * @ucd_rsp_ptr: pointer to response UPIU
 818  *
 819  * This function gets the response status and scsi_status from response UPIU
 820  * Returns the response result code.
 821  */
 822 static inline int
 823 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 824 {
 825         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 826 }
 827
 828 /*
 829  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
 830  *                              from response UPIU
 831  * @ucd_rsp_ptr: pointer to response UPIU
 832  *
 833  * Return the data segment length.
 834  */
 835 static inline unsigned int
 836 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
 837 {
 838         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 839                 MASK_RSP_UPIU_DATA_SEG_LEN;
 840 }
 841
 842 /**
 843  * ufshcd_is_exception_event - Check if the device raised an exception event
 844  * @ucd_rsp_ptr: pointer to response UPIU
 845  *
 846  * The function checks if the device raised an exception event indicated in
 847  * the Device Information field of response UPIU.
 848  *
 849  * Returns true if exception is raised, false otherwise.
 850  */
 851 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 852 {
 853         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 854                         MASK_RSP_EXCEPTION_EVENT ? true : false;
 855 }
 856
 857 /**
 858  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 859  * @hba: per adapter instance
 860  */
 861 static inline void
 862 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 863 {
 864         ufshcd_writel(hba, INT_AGGR_ENABLE |
 865                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 866                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 867 }
 868
 869 /**
 870  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 871  * @hba: per adapter instance
 872  * @cnt: Interrupt aggregation counter threshold
 873  * @tmout: Interrupt aggregation timeout value
 874  */
 875 static inline void
 876 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 877 {
 878         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 879                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 880                       INT_AGGR_TIMEOUT_VAL(tmout),
 881                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 882 }
 883
 884 /**
 885  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 886  * @hba: per adapter instance
 887  */
 888 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 889 {
 890         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 891 }
 892
 893 /**
 894  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 895  *                      When run-stop registers are set to 1, it indicates the
 896  *                      host controller that it can process the requests
 897  * @hba: per adapter instance
 898  */
 899 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 900 {
 901         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 902                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 903         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 904                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 905 }
 906
 907 /**
 908  * ufshcd_hba_start - Start controller initialization sequence
 909  * @hba: per adapter instance
 910  */
 911 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 912 {
 913         u32 val = CONTROLLER_ENABLE;
 914
 915         if (ufshcd_crypto_enable(hba))
 916                 val |= CRYPTO_GENERAL_ENABLE;
 917
 918         ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
 919 }
 920
 921 /**
 922  * ufshcd_is_hba_active - Get controller state
 923  * @hba: per adapter instance
 924  *
 925  * Returns false if controller is active, true otherwise
 926  */
 927 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 928 {
 929         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
 930                 ? false : true;
 931 }
 932
 933 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
 934 {
 935         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
 936         if (hba->ufs_version <= ufshci_version(1, 1))
 937                 return UFS_UNIPRO_VER_1_41;
 938         else
 939                 return UFS_UNIPRO_VER_1_6;
 940 }
 941 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
 942
 943 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
 944 {
 945         /*
 946          * If both host and device support UniPro ver1.6 or later, PA layer
 947          * parameters tuning happens during link startup itself.
 948          *
 949          * We can manually tune PA layer parameters if either host or device
 950          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
 951          * logic simple, we will only do manual tuning if local unipro version
 952          * doesn't support ver1.6 or later.
 953          */
 954         if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
 955                 return true;
 956         else
 957                 return false;
 958 }
 959
 960 /**
 961  * ufshcd_set_clk_freq - set UFS controller clock frequencies
 962  * @hba: per adapter instance
 963  * @scale_up: If True, set max possible frequency othewise set low frequency
 964  *
 965  * Returns 0 if successful
 966  * Returns < 0 for any other errors
 967  */
 968 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
 969 {
 970         int ret = 0;
 971         struct ufs_clk_info *clki;
 972         struct list_head *head = &hba->clk_list_head;
 973
 974         if (list_empty(head))
 975                 goto out;
 976
 977         list_for_each_entry(clki, head, list) {
 978                 if (!IS_ERR_OR_NULL(clki->clk)) {
 979                         if (scale_up && clki->max_freq) {
 980                                 if (clki->curr_freq == clki->max_freq)
 981                                         continue;
 982
 983                                 ret = clk_set_rate(clki->clk, clki->max_freq);
 984                                 if (ret) {
 985                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 986                                                 __func__, clki->name,
 987                                                 clki->max_freq, ret);
 988                                         break;
 989                                 }
 990                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
 991                                                 "scaled up", clki->name,
 992                                                 clki->curr_freq,
 993                                                 clki->max_freq);
 994
 995                                 clki->curr_freq = clki->max_freq;
 996
 997                         } else if (!scale_up && clki->min_freq) {
 998                                 if (clki->curr_freq == clki->min_freq)
 999                                         continue;
1000
1001                                 ret = clk_set_rate(clki->clk, clki->min_freq);
1002                                 if (ret) {
1003                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1004                                                 __func__, clki->name,
1005                                                 clki->min_freq, ret);
1006                                         break;
1007                                 }
1008                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1009                                                 "scaled down", clki->name,
1010                                                 clki->curr_freq,
1011                                                 clki->min_freq);
1012                                 clki->curr_freq = clki->min_freq;
1013                         }
1014                 }
1015                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1016                                 clki->name, clk_get_rate(clki->clk));
1017         }
1018
1019 out:
1020         return ret;
1021 }
1022
1023 /**
1024  * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1025  * @hba: per adapter instance
1026  * @scale_up: True if scaling up and false if scaling down
1027  *
1028  * Returns 0 if successful
1029  * Returns < 0 for any other errors
1030  */
1031 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
1032 {
1033         int ret = 0;
1034         ktime_t start = ktime_get();
1035
1036         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1037         if (ret)
1038                 goto out;
1039
1040         ret = ufshcd_set_clk_freq(hba, scale_up);
1041         if (ret)
1042                 goto out;
1043
1044         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1045         if (ret)
1046                 ufshcd_set_clk_freq(hba, !scale_up);
1047
1048 out:
1049         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1050                         (scale_up ? "up" : "down"),
1051                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1052         return ret;
1053 }
1054
1055 /**
1056  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1057  * @hba: per adapter instance
1058  * @scale_up: True if scaling up and false if scaling down
1059  *
1060  * Returns true if scaling is required, false otherwise.
1061  */
1062 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1063                                                bool scale_up)
1064 {
1065         struct ufs_clk_info *clki;
1066         struct list_head *head = &hba->clk_list_head;
1067
1068         if (list_empty(head))
1069                 return false;
1070
1071         list_for_each_entry(clki, head, list) {
1072                 if (!IS_ERR_OR_NULL(clki->clk)) {
1073                         if (scale_up && clki->max_freq) {
1074                                 if (clki->curr_freq == clki->max_freq)
1075                                         continue;
1076                                 return true;
1077                         } else if (!scale_up && clki->min_freq) {
1078                                 if (clki->curr_freq == clki->min_freq)
1079                                         continue;
1080                                 return true;
1081                         }
1082                 }
1083         }
1084
1085         return false;
1086 }
1087
1088 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1089                                         u64 wait_timeout_us)
1090 {
1091         unsigned long flags;
1092         int ret = 0;
1093         u32 tm_doorbell;
1094         u32 tr_doorbell;
1095         bool timeout = false, do_last_check = false;
1096         ktime_t start;
1097
1098         ufshcd_hold(hba, false);
1099         spin_lock_irqsave(hba->host->host_lock, flags);
1100         /*
1101          * Wait for all the outstanding tasks/transfer requests.
1102          * Verify by checking the doorbell registers are clear.
1103          */
1104         start = ktime_get();
1105         do {
1106                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1107                         ret = -EBUSY;
1108                         goto out;
1109                 }
1110
1111                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1112                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1113                 if (!tm_doorbell && !tr_doorbell) {
1114                         timeout = false;
1115                         break;
1116                 } else if (do_last_check) {
1117                         break;
1118                 }
1119
1120                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1121                 schedule();
1122                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1123                     wait_timeout_us) {
1124                         timeout = true;
1125                         /*
1126                          * We might have scheduled out for long time so make
1127                          * sure to check if doorbells are cleared by this time
1128                          * or not.
1129                          */
1130                         do_last_check = true;
1131                 }
1132                 spin_lock_irqsave(hba->host->host_lock, flags);
1133         } while (tm_doorbell || tr_doorbell);
1134
1135         if (timeout) {
1136                 dev_err(hba->dev,
1137                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1138                         __func__, tm_doorbell, tr_doorbell);
1139                 ret = -EBUSY;
1140         }
1141 out:
1142         spin_unlock_irqrestore(hba->host->host_lock, flags);
1143         ufshcd_release(hba);
1144         return ret;
1145 }
1146
1147 /**
1148  * ufshcd_scale_gear - scale up/down UFS gear
1149  * @hba: per adapter instance
1150  * @scale_up: True for scaling up gear and false for scaling down
1151  *
1152  * Returns 0 for success,
1153  * Returns -EBUSY if scaling can't happen at this time
1154  * Returns non-zero for any other errors
1155  */
1156 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1157 {
1158         int ret = 0;
1159         struct ufs_pa_layer_attr new_pwr_info;
1160
1161         if (scale_up) {
1162                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1163                        sizeof(struct ufs_pa_layer_attr));
1164         } else {
1165                 memcpy(&new_pwr_info, &hba->pwr_info,
1166                        sizeof(struct ufs_pa_layer_attr));
1167
1168                 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1169                     hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1170                         /* save the current power mode */
1171                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1172                                 &hba->pwr_info,
1173                                 sizeof(struct ufs_pa_layer_attr));
1174
1175                         /* scale down gear */
1176                         new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1177                         new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1178                 }
1179         }
1180
1181         /* check if the power mode needs to be changed or not? */
1182         ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1183         if (ret)
1184                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1185                         __func__, ret,
1186                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1187                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1188
1189         return ret;
1190 }
1191
1192 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1193 {
1194         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1195         int ret = 0;
1196         /*
1197          * make sure that there are no outstanding requests when
1198          * clock scaling is in progress
1199          */
1200         ufshcd_scsi_block_requests(hba);
1201         down_write(&hba->clk_scaling_lock);
1202
1203         if (!hba->clk_scaling.is_allowed ||
1204             ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1205                 ret = -EBUSY;
1206                 up_write(&hba->clk_scaling_lock);
1207                 ufshcd_scsi_unblock_requests(hba);
1208                 goto out;
1209         }
1210
1211         /* let's not get into low power until clock scaling is completed */
1212         ufshcd_hold(hba, false);
1213
1214 out:
1215         return ret;
1216 }
1217
1218 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, bool writelock)
1219 {
1220         if (writelock)
1221                 up_write(&hba->clk_scaling_lock);
1222         else
1223                 up_read(&hba->clk_scaling_lock);
1224         ufshcd_scsi_unblock_requests(hba);
1225         ufshcd_release(hba);
1226 }
1227
1228 /**
1229  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1230  * @hba: per adapter instance
1231  * @scale_up: True for scaling up and false for scalin down
1232  *
1233  * Returns 0 for success,
1234  * Returns -EBUSY if scaling can't happen at this time
1235  * Returns non-zero for any other errors
1236  */
1237 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1238 {
1239         int ret = 0;
1240         bool is_writelock = true;
1241
1242         ret = ufshcd_clock_scaling_prepare(hba);
1243         if (ret)
1244                 return ret;
1245
1246         /* scale down the gear before scaling down clocks */
1247         if (!scale_up) {
1248                 ret = ufshcd_scale_gear(hba, false);
1249                 if (ret)
1250                         goto out_unprepare;
1251         }
1252
1253         ret = ufshcd_scale_clks(hba, scale_up);
1254         if (ret) {
1255                 if (!scale_up)
1256                         ufshcd_scale_gear(hba, true);
1257                 goto out_unprepare;
1258         }
1259
1260         /* scale up the gear after scaling up clocks */
1261         if (scale_up) {
1262                 ret = ufshcd_scale_gear(hba, true);
1263                 if (ret) {
1264                         ufshcd_scale_clks(hba, false);
1265                         goto out_unprepare;
1266                 }
1267         }
1268
1269         /* Enable Write Booster if we have scaled up else disable it */
1270         downgrade_write(&hba->clk_scaling_lock);
1271         is_writelock = false;
1272         ufshcd_wb_toggle(hba, scale_up);
1273
1274 out_unprepare:
1275         ufshcd_clock_scaling_unprepare(hba, is_writelock);
1276         return ret;
1277 }
1278
1279 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1280 {
1281         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1282                                            clk_scaling.suspend_work);
1283         unsigned long irq_flags;
1284
1285         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1286         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1287                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1288                 return;
1289         }
1290         hba->clk_scaling.is_suspended = true;
1291         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1292
1293         __ufshcd_suspend_clkscaling(hba);
1294 }
1295
1296 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1297 {
1298         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1299                                            clk_scaling.resume_work);
1300         unsigned long irq_flags;
1301
1302         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1303         if (!hba->clk_scaling.is_suspended) {
1304                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1305                 return;
1306         }
1307         hba->clk_scaling.is_suspended = false;
1308         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1309
1310         devfreq_resume_device(hba->devfreq);
1311 }
1312
1313 static int ufshcd_devfreq_target(struct device *dev,
1314                                 unsigned long *freq, u32 flags)
1315 {
1316         int ret = 0;
1317         struct ufs_hba *hba = dev_get_drvdata(dev);
1318         ktime_t start;
1319         bool scale_up, sched_clk_scaling_suspend_work = false;
1320         struct list_head *clk_list = &hba->clk_list_head;
1321         struct ufs_clk_info *clki;
1322         unsigned long irq_flags;
1323
1324         if (!ufshcd_is_clkscaling_supported(hba))
1325                 return -EINVAL;
1326
1327         clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1328         /* Override with the closest supported frequency */
1329         *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1330         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1331         if (ufshcd_eh_in_progress(hba)) {
1332                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1333                 return 0;
1334         }
1335
1336         if (!hba->clk_scaling.active_reqs)
1337                 sched_clk_scaling_suspend_work = true;
1338
1339         if (list_empty(clk_list)) {
1340                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1341                 goto out;
1342         }
1343
1344         /* Decide based on the rounded-off frequency and update */
1345         scale_up = (*freq == clki->max_freq) ? true : false;
1346         if (!scale_up)
1347                 *freq = clki->min_freq;
1348         /* Update the frequency */
1349         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1350                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1351                 ret = 0;
1352                 goto out; /* no state change required */
1353         }
1354         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1355
1356         start = ktime_get();
1357         ret = ufshcd_devfreq_scale(hba, scale_up);
1358
1359         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1360                 (scale_up ? "up" : "down"),
1361                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1362
1363 out:
1364         if (sched_clk_scaling_suspend_work)
1365                 queue_work(hba->clk_scaling.workq,
1366                            &hba->clk_scaling.suspend_work);
1367
1368         return ret;
1369 }
1370
1371 static bool ufshcd_is_busy(struct request *req, void *priv, bool reserved)
1372 {
1373         int *busy = priv;
1374
1375         WARN_ON_ONCE(reserved);
1376         (*busy)++;
1377         return false;
1378 }
1379
1380 /* Whether or not any tag is in use by a request that is in progress. */
1381 static bool ufshcd_any_tag_in_use(struct ufs_hba *hba)
1382 {
1383         struct request_queue *q = hba->cmd_queue;
1384         int busy = 0;
1385
1386         blk_mq_tagset_busy_iter(q->tag_set, ufshcd_is_busy, &busy);
1387         return busy;
1388 }
1389
1390 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1391                 struct devfreq_dev_status *stat)
1392 {
1393         struct ufs_hba *hba = dev_get_drvdata(dev);
1394         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1395         unsigned long flags;
1396         struct list_head *clk_list = &hba->clk_list_head;
1397         struct ufs_clk_info *clki;
1398         ktime_t curr_t;
1399
1400         if (!ufshcd_is_clkscaling_supported(hba))
1401                 return -EINVAL;
1402
1403         memset(stat, 0, sizeof(*stat));
1404
1405         spin_lock_irqsave(hba->host->host_lock, flags);
1406         curr_t = ktime_get();
1407         if (!scaling->window_start_t)
1408                 goto start_window;
1409
1410         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1411         /*
1412          * If current frequency is 0, then the ondemand governor considers
1413          * there's no initial frequency set. And it always requests to set
1414          * to max. frequency.
1415          */
1416         stat->current_frequency = clki->curr_freq;
1417         if (scaling->is_busy_started)
1418                 scaling->tot_busy_t += ktime_us_delta(curr_t,
1419                                 scaling->busy_start_t);
1420
1421         stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1422         stat->busy_time = scaling->tot_busy_t;
1423 start_window:
1424         scaling->window_start_t = curr_t;
1425         scaling->tot_busy_t = 0;
1426
1427         if (hba->outstanding_reqs) {
1428                 scaling->busy_start_t = curr_t;
1429                 scaling->is_busy_started = true;
1430         } else {
1431                 scaling->busy_start_t = 0;
1432                 scaling->is_busy_started = false;
1433         }
1434         spin_unlock_irqrestore(hba->host->host_lock, flags);
1435         return 0;
1436 }
1437
1438 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1439 {
1440         struct list_head *clk_list = &hba->clk_list_head;
1441         struct ufs_clk_info *clki;
1442         struct devfreq *devfreq;
1443         int ret;
1444
1445         /* Skip devfreq if we don't have any clocks in the list */
1446         if (list_empty(clk_list))
1447                 return 0;
1448
1449         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1450         dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1451         dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1452
1453         ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1454                                          &hba->vps->ondemand_data);
1455         devfreq = devfreq_add_device(hba->dev,
1456                         &hba->vps->devfreq_profile,
1457                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1458                         &hba->vps->ondemand_data);
1459         if (IS_ERR(devfreq)) {
1460                 ret = PTR_ERR(devfreq);
1461                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1462
1463                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1464                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1465                 return ret;
1466         }
1467
1468         hba->devfreq = devfreq;
1469
1470         return 0;
1471 }
1472
1473 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1474 {
1475         struct list_head *clk_list = &hba->clk_list_head;
1476         struct ufs_clk_info *clki;
1477
1478         if (!hba->devfreq)
1479                 return;
1480
1481         devfreq_remove_device(hba->devfreq);
1482         hba->devfreq = NULL;
1483
1484         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1485         dev_pm_opp_remove(hba->dev, clki->min_freq);
1486         dev_pm_opp_remove(hba->dev, clki->max_freq);
1487 }
1488
1489 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1490 {
1491         unsigned long flags;
1492
1493         devfreq_suspend_device(hba->devfreq);
1494         spin_lock_irqsave(hba->host->host_lock, flags);
1495         hba->clk_scaling.window_start_t = 0;
1496         spin_unlock_irqrestore(hba->host->host_lock, flags);
1497 }
1498
1499 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1500 {
1501         unsigned long flags;
1502         bool suspend = false;
1503
1504         cancel_work_sync(&hba->clk_scaling.suspend_work);
1505         cancel_work_sync(&hba->clk_scaling.resume_work);
1506
1507         spin_lock_irqsave(hba->host->host_lock, flags);
1508         if (!hba->clk_scaling.is_suspended) {
1509                 suspend = true;
1510                 hba->clk_scaling.is_suspended = true;
1511         }
1512         spin_unlock_irqrestore(hba->host->host_lock, flags);
1513
1514         if (suspend)
1515                 __ufshcd_suspend_clkscaling(hba);
1516 }
1517
1518 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1519 {
1520         unsigned long flags;
1521         bool resume = false;
1522
1523         spin_lock_irqsave(hba->host->host_lock, flags);
1524         if (hba->clk_scaling.is_suspended) {
1525                 resume = true;
1526                 hba->clk_scaling.is_suspended = false;
1527         }
1528         spin_unlock_irqrestore(hba->host->host_lock, flags);
1529
1530         if (resume)
1531                 devfreq_resume_device(hba->devfreq);
1532 }
1533
1534 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1535                 struct device_attribute *attr, char *buf)
1536 {
1537         struct ufs_hba *hba = dev_get_drvdata(dev);
1538
1539         return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1540 }
1541
1542 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1543                 struct device_attribute *attr, const char *buf, size_t count)
1544 {
1545         struct ufs_hba *hba = dev_get_drvdata(dev);
1546         u32 value;
1547         int err = 0;
1548
1549         if (kstrtou32(buf, 0, &value))
1550                 return -EINVAL;
1551
1552         down(&hba->host_sem);
1553         if (!ufshcd_is_user_access_allowed(hba)) {
1554                 err = -EBUSY;
1555                 goto out;
1556         }
1557
1558         value = !!value;
1559         if (value == hba->clk_scaling.is_enabled)
1560                 goto out;
1561
1562         ufshcd_rpm_get_sync(hba);
1563         ufshcd_hold(hba, false);
1564
1565         hba->clk_scaling.is_enabled = value;
1566
1567         if (value) {
1568                 ufshcd_resume_clkscaling(hba);
1569         } else {
1570                 ufshcd_suspend_clkscaling(hba);
1571                 err = ufshcd_devfreq_scale(hba, true);
1572                 if (err)
1573                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1574                                         __func__, err);
1575         }
1576
1577         ufshcd_release(hba);
1578         ufshcd_rpm_put_sync(hba);
1579 out:
1580         up(&hba->host_sem);
1581         return err ? err : count;
1582 }
1583
1584 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1585 {
1586         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1587         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1588         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1589         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1590         hba->clk_scaling.enable_attr.attr.mode = 0644;
1591         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1592                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1593 }
1594
1595 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1596 {
1597         if (hba->clk_scaling.enable_attr.attr.name)
1598                 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1599 }
1600
1601 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1602 {
1603         char wq_name[sizeof("ufs_clkscaling_00")];
1604
1605         if (!ufshcd_is_clkscaling_supported(hba))
1606                 return;
1607
1608         if (!hba->clk_scaling.min_gear)
1609                 hba->clk_scaling.min_gear = UFS_HS_G1;
1610
1611         INIT_WORK(&hba->clk_scaling.suspend_work,
1612                   ufshcd_clk_scaling_suspend_work);
1613         INIT_WORK(&hba->clk_scaling.resume_work,
1614                   ufshcd_clk_scaling_resume_work);
1615
1616         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1617                  hba->host->host_no);
1618         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1619
1620         hba->clk_scaling.is_initialized = true;
1621 }
1622
1623 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1624 {
1625         if (!hba->clk_scaling.is_initialized)
1626                 return;
1627
1628         ufshcd_remove_clk_scaling_sysfs(hba);
1629         destroy_workqueue(hba->clk_scaling.workq);
1630         ufshcd_devfreq_remove(hba);
1631         hba->clk_scaling.is_initialized = false;
1632 }
1633
1634 static void ufshcd_ungate_work(struct work_struct *work)
1635 {
1636         int ret;
1637         unsigned long flags;
1638         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1639                         clk_gating.ungate_work);
1640
1641         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1642
1643         spin_lock_irqsave(hba->host->host_lock, flags);
1644         if (hba->clk_gating.state == CLKS_ON) {
1645                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1646                 goto unblock_reqs;
1647         }
1648
1649         spin_unlock_irqrestore(hba->host->host_lock, flags);
1650         ufshcd_hba_vreg_set_hpm(hba);
1651         ufshcd_setup_clocks(hba, true);
1652
1653         ufshcd_enable_irq(hba);
1654
1655         /* Exit from hibern8 */
1656         if (ufshcd_can_hibern8_during_gating(hba)) {
1657                 /* Prevent gating in this path */
1658                 hba->clk_gating.is_suspended = true;
1659                 if (ufshcd_is_link_hibern8(hba)) {
1660                         ret = ufshcd_uic_hibern8_exit(hba);
1661                         if (ret)
1662                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1663                                         __func__, ret);
1664                         else
1665                                 ufshcd_set_link_active(hba);
1666                 }
1667                 hba->clk_gating.is_suspended = false;
1668         }
1669 unblock_reqs:
1670         ufshcd_scsi_unblock_requests(hba);
1671 }
1672
1673 /**
1674  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1675  * Also, exit from hibern8 mode and set the link as active.
1676  * @hba: per adapter instance
1677  * @async: This indicates whether caller should ungate clocks asynchronously.
1678  */
1679 int ufshcd_hold(struct ufs_hba *hba, bool async)
1680 {
1681         int rc = 0;
1682         bool flush_result;
1683         unsigned long flags;
1684
1685         if (!ufshcd_is_clkgating_allowed(hba))
1686                 goto out;
1687         spin_lock_irqsave(hba->host->host_lock, flags);
1688         hba->clk_gating.active_reqs++;
1689
1690 start:
1691         switch (hba->clk_gating.state) {
1692         case CLKS_ON:
1693                 /*
1694                  * Wait for the ungate work to complete if in progress.
1695                  * Though the clocks may be in ON state, the link could
1696                  * still be in hibner8 state if hibern8 is allowed
1697                  * during clock gating.
1698                  * Make sure we exit hibern8 state also in addition to
1699                  * clocks being ON.
1700                  */
1701                 if (ufshcd_can_hibern8_during_gating(hba) &&
1702                     ufshcd_is_link_hibern8(hba)) {
1703                         if (async) {
1704                                 rc = -EAGAIN;
1705                                 hba->clk_gating.active_reqs--;
1706                                 break;
1707                         }
1708                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1709                         flush_result = flush_work(&hba->clk_gating.ungate_work);
1710                         if (hba->clk_gating.is_suspended && !flush_result)
1711                                 goto out;
1712                         spin_lock_irqsave(hba->host->host_lock, flags);
1713                         goto start;
1714                 }
1715                 break;
1716         case REQ_CLKS_OFF:
1717                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1718                         hba->clk_gating.state = CLKS_ON;
1719                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1720                                                 hba->clk_gating.state);
1721                         break;
1722                 }
1723                 /*
1724                  * If we are here, it means gating work is either done or
1725                  * currently running. Hence, fall through to cancel gating
1726                  * work and to enable clocks.
1727                  */
1728                 fallthrough;
1729         case CLKS_OFF:
1730                 hba->clk_gating.state = REQ_CLKS_ON;
1731                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1732                                         hba->clk_gating.state);
1733                 if (queue_work(hba->clk_gating.clk_gating_workq,
1734                                &hba->clk_gating.ungate_work))
1735                         ufshcd_scsi_block_requests(hba);
1736                 /*
1737                  * fall through to check if we should wait for this
1738                  * work to be done or not.
1739                  */
1740                 fallthrough;
1741         case REQ_CLKS_ON:
1742                 if (async) {
1743                         rc = -EAGAIN;
1744                         hba->clk_gating.active_reqs--;
1745                         break;
1746                 }
1747
1748                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1749                 flush_work(&hba->clk_gating.ungate_work);
1750                 /* Make sure state is CLKS_ON before returning */
1751                 spin_lock_irqsave(hba->host->host_lock, flags);
1752                 goto start;
1753         default:
1754                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1755                                 __func__, hba->clk_gating.state);
1756                 break;
1757         }
1758         spin_unlock_irqrestore(hba->host->host_lock, flags);
1759 out:
1760         return rc;
1761 }
1762 EXPORT_SYMBOL_GPL(ufshcd_hold);
1763
1764 static void ufshcd_gate_work(struct work_struct *work)
1765 {
1766         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1767                         clk_gating.gate_work.work);
1768         unsigned long flags;
1769         int ret;
1770
1771         spin_lock_irqsave(hba->host->host_lock, flags);
1772         /*
1773          * In case you are here to cancel this work the gating state
1774          * would be marked as REQ_CLKS_ON. In this case save time by
1775          * skipping the gating work and exit after changing the clock
1776          * state to CLKS_ON.
1777          */
1778         if (hba->clk_gating.is_suspended ||
1779                 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1780                 hba->clk_gating.state = CLKS_ON;
1781                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1782                                         hba->clk_gating.state);
1783                 goto rel_lock;
1784         }
1785
1786         if (hba->clk_gating.active_reqs
1787                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1788                 || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
1789                 || hba->active_uic_cmd || hba->uic_async_done)
1790                 goto rel_lock;
1791
1792         spin_unlock_irqrestore(hba->host->host_lock, flags);
1793
1794         /* put the link into hibern8 mode before turning off clocks */
1795         if (ufshcd_can_hibern8_during_gating(hba)) {
1796                 ret = ufshcd_uic_hibern8_enter(hba);
1797                 if (ret) {
1798                         hba->clk_gating.state = CLKS_ON;
1799                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1800                                         __func__, ret);
1801                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1802                                                 hba->clk_gating.state);
1803                         goto out;
1804                 }
1805                 ufshcd_set_link_hibern8(hba);
1806         }
1807
1808         ufshcd_disable_irq(hba);
1809
1810         ufshcd_setup_clocks(hba, false);
1811
1812         /* Put the host controller in low power mode if possible */
1813         ufshcd_hba_vreg_set_lpm(hba);
1814         /*
1815          * In case you are here to cancel this work the gating state
1816          * would be marked as REQ_CLKS_ON. In this case keep the state
1817          * as REQ_CLKS_ON which would anyway imply that clocks are off
1818          * and a request to turn them on is pending. By doing this way,
1819          * we keep the state machine in tact and this would ultimately
1820          * prevent from doing cancel work multiple times when there are
1821          * new requests arriving before the current cancel work is done.
1822          */
1823         spin_lock_irqsave(hba->host->host_lock, flags);
1824         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1825                 hba->clk_gating.state = CLKS_OFF;
1826                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1827                                         hba->clk_gating.state);
1828         }
1829 rel_lock:
1830         spin_unlock_irqrestore(hba->host->host_lock, flags);
1831 out:
1832         return;
1833 }
1834
1835 /* host lock must be held before calling this variant */
1836 static void __ufshcd_release(struct ufs_hba *hba)
1837 {
1838         if (!ufshcd_is_clkgating_allowed(hba))
1839                 return;
1840
1841         hba->clk_gating.active_reqs--;
1842
1843         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
1844             hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
1845             hba->outstanding_tasks ||
1846             hba->active_uic_cmd || hba->uic_async_done ||
1847             hba->clk_gating.state == CLKS_OFF)
1848                 return;
1849
1850         hba->clk_gating.state = REQ_CLKS_OFF;
1851         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1852         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1853                            &hba->clk_gating.gate_work,
1854                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1855 }
1856
1857 void ufshcd_release(struct ufs_hba *hba)
1858 {
1859         unsigned long flags;
1860
1861         spin_lock_irqsave(hba->host->host_lock, flags);
1862         __ufshcd_release(hba);
1863         spin_unlock_irqrestore(hba->host->host_lock, flags);
1864 }
1865 EXPORT_SYMBOL_GPL(ufshcd_release);
1866
1867 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1868                 struct device_attribute *attr, char *buf)
1869 {
1870         struct ufs_hba *hba = dev_get_drvdata(dev);
1871
1872         return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
1873 }
1874
1875 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1876                 struct device_attribute *attr, const char *buf, size_t count)
1877 {
1878         struct ufs_hba *hba = dev_get_drvdata(dev);
1879         unsigned long flags, value;
1880
1881         if (kstrtoul(buf, 0, &value))
1882                 return -EINVAL;
1883
1884         spin_lock_irqsave(hba->host->host_lock, flags);
1885         hba->clk_gating.delay_ms = value;
1886         spin_unlock_irqrestore(hba->host->host_lock, flags);
1887         return count;
1888 }
1889
1890 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1891                 struct device_attribute *attr, char *buf)
1892 {
1893         struct ufs_hba *hba = dev_get_drvdata(dev);
1894
1895         return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
1896 }
1897
1898 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1899                 struct device_attribute *attr, const char *buf, size_t count)
1900 {
1901         struct ufs_hba *hba = dev_get_drvdata(dev);
1902         unsigned long flags;
1903         u32 value;
1904
1905         if (kstrtou32(buf, 0, &value))
1906                 return -EINVAL;
1907
1908         value = !!value;
1909
1910         spin_lock_irqsave(hba->host->host_lock, flags);
1911         if (value == hba->clk_gating.is_enabled)
1912                 goto out;
1913
1914         if (value)
1915                 __ufshcd_release(hba);
1916         else
1917                 hba->clk_gating.active_reqs++;
1918
1919         hba->clk_gating.is_enabled = value;
1920 out:
1921         spin_unlock_irqrestore(hba->host->host_lock, flags);
1922         return count;
1923 }
1924
1925 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
1926 {
1927         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1928         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1929         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1930         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1931         hba->clk_gating.delay_attr.attr.mode = 0644;
1932         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1933                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1934
1935         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1936         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1937         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1938         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1939         hba->clk_gating.enable_attr.attr.mode = 0644;
1940         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1941                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1942 }
1943
1944 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
1945 {
1946         if (hba->clk_gating.delay_attr.attr.name)
1947                 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1948         if (hba->clk_gating.enable_attr.attr.name)
1949                 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1950 }
1951
1952 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1953 {
1954         char wq_name[sizeof("ufs_clk_gating_00")];
1955
1956         if (!ufshcd_is_clkgating_allowed(hba))
1957                 return;
1958
1959         hba->clk_gating.state = CLKS_ON;
1960
1961         hba->clk_gating.delay_ms = 150;
1962         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1963         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1964
1965         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1966                  hba->host->host_no);
1967         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1968                                         WQ_MEM_RECLAIM | WQ_HIGHPRI);
1969
1970         ufshcd_init_clk_gating_sysfs(hba);
1971
1972         hba->clk_gating.is_enabled = true;
1973         hba->clk_gating.is_initialized = true;
1974 }
1975
1976 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1977 {
1978         if (!hba->clk_gating.is_initialized)
1979                 return;
1980         ufshcd_remove_clk_gating_sysfs(hba);
1981         cancel_work_sync(&hba->clk_gating.ungate_work);
1982         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1983         destroy_workqueue(hba->clk_gating.clk_gating_workq);
1984         hba->clk_gating.is_initialized = false;
1985 }
1986
1987 /* Must be called with host lock acquired */
1988 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1989 {
1990         bool queue_resume_work = false;
1991         ktime_t curr_t = ktime_get();
1992         unsigned long flags;
1993
1994         if (!ufshcd_is_clkscaling_supported(hba))
1995                 return;
1996
1997         spin_lock_irqsave(hba->host->host_lock, flags);
1998         if (!hba->clk_scaling.active_reqs++)
1999                 queue_resume_work = true;
2000
2001         if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2002                 spin_unlock_irqrestore(hba->host->host_lock, flags);
2003                 return;
2004         }
2005
2006         if (queue_resume_work)
2007                 queue_work(hba->clk_scaling.workq,
2008                            &hba->clk_scaling.resume_work);
2009
2010         if (!hba->clk_scaling.window_start_t) {
2011                 hba->clk_scaling.window_start_t = curr_t;
2012                 hba->clk_scaling.tot_busy_t = 0;
2013                 hba->clk_scaling.is_busy_started = false;
2014         }
2015
2016         if (!hba->clk_scaling.is_busy_started) {
2017                 hba->clk_scaling.busy_start_t = curr_t;
2018                 hba->clk_scaling.is_busy_started = true;
2019         }
2020         spin_unlock_irqrestore(hba->host->host_lock, flags);
2021 }
2022
2023 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2024 {
2025         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2026         unsigned long flags;
2027
2028         if (!ufshcd_is_clkscaling_supported(hba))
2029                 return;
2030
2031         spin_lock_irqsave(hba->host->host_lock, flags);
2032         hba->clk_scaling.active_reqs--;
2033         if (!hba->outstanding_reqs && scaling->is_busy_started) {
2034                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2035                                         scaling->busy_start_t));
2036                 scaling->busy_start_t = 0;
2037                 scaling->is_busy_started = false;
2038         }
2039         spin_unlock_irqrestore(hba->host->host_lock, flags);
2040 }
2041
2042 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2043 {
2044         if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2045                 return READ;
2046         else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2047                 return WRITE;
2048         else
2049                 return -EINVAL;
2050 }
2051
2052 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2053                                                 struct ufshcd_lrb *lrbp)
2054 {
2055         struct ufs_hba_monitor *m = &hba->monitor;
2056
2057         return (m->enabled && lrbp && lrbp->cmd &&
2058                 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2059                 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2060 }
2061
2062 static void ufshcd_start_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2063 {
2064         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2065         unsigned long flags;
2066
2067         spin_lock_irqsave(hba->host->host_lock, flags);
2068         if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2069                 hba->monitor.busy_start_ts[dir] = ktime_get();
2070         spin_unlock_irqrestore(hba->host->host_lock, flags);
2071 }
2072
2073 static void ufshcd_update_monitor(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2074 {
2075         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2076         unsigned long flags;
2077
2078         spin_lock_irqsave(hba->host->host_lock, flags);
2079         if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2080                 struct request *req = lrbp->cmd->request;
2081                 struct ufs_hba_monitor *m = &hba->monitor;
2082                 ktime_t now, inc, lat;
2083
2084                 now = lrbp->compl_time_stamp;
2085                 inc = ktime_sub(now, m->busy_start_ts[dir]);
2086                 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2087                 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2088
2089                 /* Update latencies */
2090                 m->nr_req[dir]++;
2091                 lat = ktime_sub(now, lrbp->issue_time_stamp);
2092                 m->lat_sum[dir] += lat;
2093                 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2094                         m->lat_max[dir] = lat;
2095                 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2096                         m->lat_min[dir] = lat;
2097
2098                 m->nr_queued[dir]--;
2099                 /* Push forward the busy start of monitor */
2100                 m->busy_start_ts[dir] = now;
2101         }
2102         spin_unlock_irqrestore(hba->host->host_lock, flags);
2103 }
2104
2105 /**
2106  * ufshcd_send_command - Send SCSI or device management commands
2107  * @hba: per adapter instance
2108  * @task_tag: Task tag of the command
2109  */
2110 static inline
2111 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
2112 {
2113         struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2114
2115         lrbp->issue_time_stamp = ktime_get();
2116         lrbp->compl_time_stamp = ktime_set(0, 0);
2117         ufshcd_vops_setup_xfer_req(hba, task_tag, (lrbp->cmd ? true : false));
2118         ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2119         ufshcd_clk_scaling_start_busy(hba);
2120         if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2121                 ufshcd_start_monitor(hba, lrbp);
2122         if (ufshcd_has_utrlcnr(hba)) {
2123                 set_bit(task_tag, &hba->outstanding_reqs);
2124                 ufshcd_writel(hba, 1 << task_tag,
2125                               REG_UTP_TRANSFER_REQ_DOOR_BELL);
2126         } else {
2127                 unsigned long flags;
2128
2129                 spin_lock_irqsave(hba->host->host_lock, flags);
2130                 set_bit(task_tag, &hba->outstanding_reqs);
2131                 ufshcd_writel(hba, 1 << task_tag,
2132                               REG_UTP_TRANSFER_REQ_DOOR_BELL);
2133                 spin_unlock_irqrestore(hba->host->host_lock, flags);
2134         }
2135         /* Make sure that doorbell is committed immediately */
2136         wmb();
2137 }
2138
2139 /**
2140  * ufshcd_copy_sense_data - Copy sense data in case of check condition
2141  * @lrbp: pointer to local reference block
2142  */
2143 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2144 {
2145         int len;
2146         if (lrbp->sense_buffer &&
2147             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
2148                 int len_to_copy;
2149
2150                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2151                 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2152
2153                 memcpy(lrbp->sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2154                        len_to_copy);
2155         }
2156 }
2157
2158 /**
2159  * ufshcd_copy_query_response() - Copy the Query Response and the data
2160  * descriptor
2161  * @hba: per adapter instance
2162  * @lrbp: pointer to local reference block
2163  */
2164 static
2165 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2166 {
2167         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2168
2169         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2170
2171         /* Get the descriptor */
2172         if (hba->dev_cmd.query.descriptor &&
2173             lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2174                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2175                                 GENERAL_UPIU_REQUEST_SIZE;
2176                 u16 resp_len;
2177                 u16 buf_len;
2178
2179                 /* data segment length */
2180                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
2181                                                 MASK_QUERY_DATA_SEG_LEN;
2182                 buf_len = be16_to_cpu(
2183                                 hba->dev_cmd.query.request.upiu_req.length);
2184                 if (likely(buf_len >= resp_len)) {
2185                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2186                 } else {
2187                         dev_warn(hba->dev,
2188                                  "%s: rsp size %d is bigger than buffer size %d",
2189                                  __func__, resp_len, buf_len);
2190                         return -EINVAL;
2191                 }
2192         }
2193
2194         return 0;
2195 }
2196
2197 /**
2198  * ufshcd_hba_capabilities - Read controller capabilities
2199  * @hba: per adapter instance
2200  *
2201  * Return: 0 on success, negative on error.
2202  */
2203 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2204 {
2205         int err;
2206
2207         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2208
2209         /* nutrs and nutmrs are 0 based values */
2210         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2211         hba->nutmrs =
2212         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2213
2214         /* Read crypto capabilities */
2215         err = ufshcd_hba_init_crypto_capabilities(hba);
2216         if (err)
2217                 dev_err(hba->dev, "crypto setup failed\n");
2218
2219         return err;
2220 }
2221
2222 /**
2223  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2224  *                            to accept UIC commands
2225  * @hba: per adapter instance
2226  * Return true on success, else false
2227  */
2228 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2229 {
2230         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2231                 return true;
2232         else
2233                 return false;
2234 }
2235
2236 /**
2237  * ufshcd_get_upmcrs - Get the power mode change request status
2238  * @hba: Pointer to adapter instance
2239  *
2240  * This function gets the UPMCRS field of HCS register
2241  * Returns value of UPMCRS field
2242  */
2243 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2244 {
2245         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2246 }
2247
2248 /**
2249  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2250  * @hba: per adapter instance
2251  * @uic_cmd: UIC command
2252  *
2253  * Mutex must be held.
2254  */
2255 static inline void
2256 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2257 {
2258         WARN_ON(hba->active_uic_cmd);
2259
2260         hba->active_uic_cmd = uic_cmd;
2261
2262         /* Write Args */
2263         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2264         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2265         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2266
2267         ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2268
2269         /* Write UIC Cmd */
2270         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2271                       REG_UIC_COMMAND);
2272 }
2273
2274 /**
2275  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2276  * @hba: per adapter instance
2277  * @uic_cmd: UIC command
2278  *
2279  * Must be called with mutex held.
2280  * Returns 0 only if success.
2281  */
2282 static int
2283 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2284 {
2285         int ret;
2286         unsigned long flags;
2287
2288         if (wait_for_completion_timeout(&uic_cmd->done,
2289                                         msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
2290                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2291         } else {
2292                 ret = -ETIMEDOUT;
2293                 dev_err(hba->dev,
2294                         "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2295                         uic_cmd->command, uic_cmd->argument3);
2296
2297                 if (!uic_cmd->cmd_active) {
2298                         dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2299                                 __func__);
2300                         ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2301                 }
2302         }
2303
2304         spin_lock_irqsave(hba->host->host_lock, flags);
2305         hba->active_uic_cmd = NULL;
2306         spin_unlock_irqrestore(hba->host->host_lock, flags);
2307
2308         return ret;
2309 }
2310
2311 /**
2312  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2313  * @hba: per adapter instance
2314  * @uic_cmd: UIC command
2315  * @completion: initialize the completion only if this is set to true
2316  *
2317  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2318  * with mutex held and host_lock locked.
2319  * Returns 0 only if success.
2320  */
2321 static int
2322 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2323                       bool completion)
2324 {
2325         if (!ufshcd_ready_for_uic_cmd(hba)) {
2326                 dev_err(hba->dev,
2327                         "Controller not ready to accept UIC commands\n");
2328                 return -EIO;
2329         }
2330
2331         if (completion)
2332                 init_completion(&uic_cmd->done);
2333
2334         uic_cmd->cmd_active = 1;
2335         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2336
2337         return 0;
2338 }
2339
2340 /**
2341  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2342  * @hba: per adapter instance
2343  * @uic_cmd: UIC command
2344  *
2345  * Returns 0 only if success.
2346  */
2347 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2348 {
2349         int ret;
2350         unsigned long flags;
2351
2352         ufshcd_hold(hba, false);
2353         mutex_lock(&hba->uic_cmd_mutex);
2354         ufshcd_add_delay_before_dme_cmd(hba);
2355
2356         spin_lock_irqsave(hba->host->host_lock, flags);
2357         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2358         spin_unlock_irqrestore(hba->host->host_lock, flags);
2359         if (!ret)
2360                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2361
2362         mutex_unlock(&hba->uic_cmd_mutex);
2363
2364         ufshcd_release(hba);
2365         return ret;
2366 }
2367
2368 /**
2369  * ufshcd_map_sg - Map scatter-gather list to prdt
2370  * @hba: per adapter instance
2371  * @lrbp: pointer to local reference block
2372  *
2373  * Returns 0 in case of success, non-zero value in case of failure
2374  */
2375 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2376 {
2377         struct ufshcd_sg_entry *prd_table;
2378         struct scatterlist *sg;
2379         struct scsi_cmnd *cmd;
2380         int sg_segments;
2381         int i;
2382
2383         cmd = lrbp->cmd;
2384         sg_segments = scsi_dma_map(cmd);
2385         if (sg_segments < 0)
2386                 return sg_segments;
2387
2388         if (sg_segments) {
2389
2390                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2391                         lrbp->utr_descriptor_ptr->prd_table_length =
2392                                 cpu_to_le16((sg_segments *
2393                                         sizeof(struct ufshcd_sg_entry)));
2394                 else
2395                         lrbp->utr_descriptor_ptr->prd_table_length =
2396                                 cpu_to_le16((u16) (sg_segments));
2397
2398                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2399
2400                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2401                         prd_table[i].size  =
2402                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2403                         prd_table[i].base_addr =
2404                                 cpu_to_le32(lower_32_bits(sg->dma_address));
2405                         prd_table[i].upper_addr =
2406                                 cpu_to_le32(upper_32_bits(sg->dma_address));
2407                         prd_table[i].reserved = 0;
2408                 }
2409         } else {
2410                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2411         }
2412
2413         return 0;
2414 }
2415
2416 /**
2417  * ufshcd_enable_intr - enable interrupts
2418  * @hba: per adapter instance
2419  * @intrs: interrupt bits
2420  */
2421 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2422 {
2423         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2424
2425         if (hba->ufs_version == ufshci_version(1, 0)) {
2426                 u32 rw;
2427                 rw = set & INTERRUPT_MASK_RW_VER_10;
2428                 set = rw | ((set ^ intrs) & intrs);
2429         } else {
2430                 set |= intrs;
2431         }
2432
2433         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2434 }
2435
2436 /**
2437  * ufshcd_disable_intr - disable interrupts
2438  * @hba: per adapter instance
2439  * @intrs: interrupt bits
2440  */
2441 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2442 {
2443         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2444
2445         if (hba->ufs_version == ufshci_version(1, 0)) {
2446                 u32 rw;
2447                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2448                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2449                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2450
2451         } else {
2452                 set &= ~intrs;
2453         }
2454
2455         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2456 }
2457
2458 /**
2459  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2460  * descriptor according to request
2461  * @lrbp: pointer to local reference block
2462  * @upiu_flags: flags required in the header
2463  * @cmd_dir: requests data direction
2464  */
2465 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2466                         u8 *upiu_flags, enum dma_data_direction cmd_dir)
2467 {
2468         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2469         u32 data_direction;
2470         u32 dword_0;
2471         u32 dword_1 = 0;
2472         u32 dword_3 = 0;
2473
2474         if (cmd_dir == DMA_FROM_DEVICE) {
2475                 data_direction = UTP_DEVICE_TO_HOST;
2476                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2477         } else if (cmd_dir == DMA_TO_DEVICE) {
2478                 data_direction = UTP_HOST_TO_DEVICE;
2479                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2480         } else {
2481                 data_direction = UTP_NO_DATA_TRANSFER;
2482                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2483         }
2484
2485         dword_0 = data_direction | (lrbp->command_type
2486                                 << UPIU_COMMAND_TYPE_OFFSET);
2487         if (lrbp->intr_cmd)
2488                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2489
2490         /* Prepare crypto related dwords */
2491         ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3);
2492
2493         /* Transfer request descriptor header fields */
2494         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2495         req_desc->header.dword_1 = cpu_to_le32(dword_1);
2496         /*
2497          * assigning invalid value for command status. Controller
2498          * updates OCS on command completion, with the command
2499          * status
2500          */
2501         req_desc->header.dword_2 =
2502                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2503         req_desc->header.dword_3 = cpu_to_le32(dword_3);
2504
2505         req_desc->prd_table_length = 0;
2506 }
2507
2508 /**
2509  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2510  * for scsi commands
2511  * @lrbp: local reference block pointer
2512  * @upiu_flags: flags
2513  */
2514 static
2515 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2516 {
2517         struct scsi_cmnd *cmd = lrbp->cmd;
2518         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2519         unsigned short cdb_len;
2520
2521         /* command descriptor fields */
2522         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2523                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2524                                 lrbp->lun, lrbp->task_tag);
2525         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2526                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2527
2528         /* Total EHS length and Data segment length will be zero */
2529         ucd_req_ptr->header.dword_2 = 0;
2530
2531         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2532
2533         cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2534         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
2535         memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2536
2537         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2538 }
2539
2540 /**
2541  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2542  * for query requsts
2543  * @hba: UFS hba
2544  * @lrbp: local reference block pointer
2545  * @upiu_flags: flags
2546  */
2547 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2548                                 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2549 {
2550         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2551         struct ufs_query *query = &hba->dev_cmd.query;
2552         u16 len = be16_to_cpu(query->request.upiu_req.length);
2553
2554         /* Query request header */
2555         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2556                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2557                         lrbp->lun, lrbp->task_tag);
2558         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2559                         0, query->request.query_func, 0, 0);
2560
2561         /* Data segment length only need for WRITE_DESC */
2562         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2563                 ucd_req_ptr->header.dword_2 =
2564                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2565         else
2566                 ucd_req_ptr->header.dword_2 = 0;
2567
2568         /* Copy the Query Request buffer as is */
2569         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2570                         QUERY_OSF_SIZE);
2571
2572         /* Copy the Descriptor */
2573         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2574                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2575
2576         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2577 }
2578
2579 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2580 {
2581         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2582
2583         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2584
2585         /* command descriptor fields */
2586         ucd_req_ptr->header.dword_0 =
2587                 UPIU_HEADER_DWORD(
2588                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2589         /* clear rest of the fields of basic header */
2590         ucd_req_ptr->header.dword_1 = 0;
2591         ucd_req_ptr->header.dword_2 = 0;
2592
2593         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2594 }
2595
2596 /**
2597  * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2598  *                           for Device Management Purposes
2599  * @hba: per adapter instance
2600  * @lrbp: pointer to local reference block
2601  */
2602 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2603                                       struct ufshcd_lrb *lrbp)
2604 {
2605         u8 upiu_flags;
2606         int ret = 0;
2607
2608         if (hba->ufs_version <= ufshci_version(1, 1))
2609                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2610         else
2611                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2612
2613         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2614         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2615                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2616         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2617                 ufshcd_prepare_utp_nop_upiu(lrbp);
2618         else
2619                 ret = -EINVAL;
2620
2621         return ret;
2622 }
2623
2624 /**
2625  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2626  *                         for SCSI Purposes
2627  * @hba: per adapter instance
2628  * @lrbp: pointer to local reference block
2629  */
2630 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2631 {
2632         u8 upiu_flags;
2633         int ret = 0;
2634
2635         if (hba->ufs_version <= ufshci_version(1, 1))
2636                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2637         else
2638                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2639
2640         if (likely(lrbp->cmd)) {
2641                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2642                                                 lrbp->cmd->sc_data_direction);
2643                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2644         } else {
2645                 ret = -EINVAL;
2646         }
2647
2648         return ret;
2649 }
2650
2651 /**
2652  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2653  * @upiu_wlun_id: UPIU W-LUN id
2654  *
2655  * Returns SCSI W-LUN id
2656  */
2657 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2658 {
2659         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2660 }
2661
2662 static inline bool is_rpmb_wlun(struct scsi_device *sdev)
2663 {
2664         return sdev->lun == ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN);
2665 }
2666
2667 static inline bool is_device_wlun(struct scsi_device *sdev)
2668 {
2669         return sdev->lun ==
2670                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2671 }
2672
2673 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2674 {
2675         struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
2676         struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2677         dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2678                 i * sizeof(struct utp_transfer_cmd_desc);
2679         u16 response_offset = offsetof(struct utp_transfer_cmd_desc,
2680                                        response_upiu);
2681         u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
2682
2683         lrb->utr_descriptor_ptr = utrdlp + i;
2684         lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2685                 i * sizeof(struct utp_transfer_req_desc);
2686         lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i);
2687         lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2688         lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
2689         lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2690         lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
2691         lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2692 }
2693
2694 /**
2695  * ufshcd_queuecommand - main entry point for SCSI requests
2696  * @host: SCSI host pointer
2697  * @cmd: command from SCSI Midlayer
2698  *
2699  * Returns 0 for success, non-zero in case of failure
2700  */
2701 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2702 {
2703         struct ufshcd_lrb *lrbp;
2704         struct ufs_hba *hba;
2705         int tag;
2706         int err = 0;
2707
2708         hba = shost_priv(host);
2709
2710         tag = cmd->request->tag;
2711         if (!ufshcd_valid_tag(hba, tag)) {
2712                 dev_err(hba->dev,
2713                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2714                         __func__, tag, cmd, cmd->request);
2715                 BUG();
2716         }
2717
2718         if (!down_read_trylock(&hba->clk_scaling_lock))
2719                 return SCSI_MLQUEUE_HOST_BUSY;
2720
2721         switch (hba->ufshcd_state) {
2722         case UFSHCD_STATE_OPERATIONAL:
2723         case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2724                 break;
2725         case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2726                 /*
2727                  * pm_runtime_get_sync() is used at error handling preparation
2728                  * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2729                  * PM ops, it can never be finished if we let SCSI layer keep
2730                  * retrying it, which gets err handler stuck forever. Neither
2731                  * can we let the scsi cmd pass through, because UFS is in bad
2732                  * state, the scsi cmd may eventually time out, which will get
2733                  * err handler blocked for too long. So, just fail the scsi cmd
2734                  * sent from PM ops, err handler can recover PM error anyways.
2735                  */
2736                 if (hba->pm_op_in_progress) {
2737                         hba->force_reset = true;
2738                         set_host_byte(cmd, DID_BAD_TARGET);
2739                         cmd->scsi_done(cmd);
2740                         goto out;
2741                 }
2742                 fallthrough;
2743         case UFSHCD_STATE_RESET:
2744                 err = SCSI_MLQUEUE_HOST_BUSY;
2745                 goto out;
2746         case UFSHCD_STATE_ERROR:
2747                 set_host_byte(cmd, DID_ERROR);
2748                 cmd->scsi_done(cmd);
2749                 goto out;
2750         default:
2751                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2752                                 __func__, hba->ufshcd_state);
2753                 set_host_byte(cmd, DID_BAD_TARGET);
2754                 cmd->scsi_done(cmd);
2755                 goto out;
2756         }
2757
2758         hba->req_abort_count = 0;
2759
2760         err = ufshcd_hold(hba, true);
2761         if (err) {
2762                 err = SCSI_MLQUEUE_HOST_BUSY;
2763                 goto out;
2764         }
2765         WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
2766                 (hba->clk_gating.state != CLKS_ON));
2767
2768         if (unlikely(test_bit(tag, &hba->outstanding_reqs))) {
2769                 if (hba->pm_op_in_progress)
2770                         set_host_byte(cmd, DID_BAD_TARGET);
2771                 else
2772                         err = SCSI_MLQUEUE_HOST_BUSY;
2773                 ufshcd_release(hba);
2774                 goto out;
2775         }
2776
2777         lrbp = &hba->lrb[tag];
2778         WARN_ON(lrbp->cmd);
2779         lrbp->cmd = cmd;
2780         lrbp->sense_bufflen = UFS_SENSE_SIZE;
2781         lrbp->sense_buffer = cmd->sense_buffer;
2782         lrbp->task_tag = tag;
2783         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2784         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2785
2786         ufshcd_prepare_lrbp_crypto(cmd->request, lrbp);
2787
2788         lrbp->req_abort_skip = false;
2789
2790         ufshcd_comp_scsi_upiu(hba, lrbp);
2791
2792         err = ufshcd_map_sg(hba, lrbp);
2793         if (err) {
2794                 lrbp->cmd = NULL;
2795                 ufshcd_release(hba);
2796                 goto out;
2797         }
2798         /* Make sure descriptors are ready before ringing the doorbell */
2799         wmb();
2800
2801         ufshcd_send_command(hba, tag);
2802 out:
2803         up_read(&hba->clk_scaling_lock);
2804         return err;
2805 }
2806
2807 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2808                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2809 {
2810         lrbp->cmd = NULL;
2811         lrbp->sense_bufflen = 0;
2812         lrbp->sense_buffer = NULL;
2813         lrbp->task_tag = tag;
2814         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2815         lrbp->intr_cmd = true; /* No interrupt aggregation */
2816         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
2817         hba->dev_cmd.type = cmd_type;
2818
2819         return ufshcd_compose_devman_upiu(hba, lrbp);
2820 }
2821
2822 static int
2823 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2824 {
2825         int err = 0;
2826         unsigned long flags;
2827         u32 mask = 1 << tag;
2828
2829         /* clear outstanding transaction before retry */
2830         spin_lock_irqsave(hba->host->host_lock, flags);
2831         ufshcd_utrl_clear(hba, tag);
2832         spin_unlock_irqrestore(hba->host->host_lock, flags);
2833
2834         /*
2835          * wait for h/w to clear corresponding bit in door-bell.
2836          * max. wait is 1 sec.
2837          */
2838         err = ufshcd_wait_for_register(hba,
2839                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
2840                         mask, ~mask, 1000, 1000);
2841
2842         return err;
2843 }
2844
2845 static int
2846 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2847 {
2848         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2849
2850         /* Get the UPIU response */
2851         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2852                                 UPIU_RSP_CODE_OFFSET;
2853         return query_res->response;
2854 }
2855
2856 /**
2857  * ufshcd_dev_cmd_completion() - handles device management command responses
2858  * @hba: per adapter instance
2859  * @lrbp: pointer to local reference block
2860  */
2861 static int
2862 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2863 {
2864         int resp;
2865         int err = 0;
2866
2867         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2868         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2869
2870         switch (resp) {
2871         case UPIU_TRANSACTION_NOP_IN:
2872                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2873                         err = -EINVAL;
2874                         dev_err(hba->dev, "%s: unexpected response %x\n",
2875                                         __func__, resp);
2876                 }
2877                 break;
2878         case UPIU_TRANSACTION_QUERY_RSP:
2879                 err = ufshcd_check_query_response(hba, lrbp);
2880                 if (!err)
2881                         err = ufshcd_copy_query_response(hba, lrbp);
2882                 break;
2883         case UPIU_TRANSACTION_REJECT_UPIU:
2884                 /* TODO: handle Reject UPIU Response */
2885                 err = -EPERM;
2886                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2887                                 __func__);
2888                 break;
2889         default:
2890                 err = -EINVAL;
2891                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2892                                 __func__, resp);
2893                 break;
2894         }
2895
2896         return err;
2897 }
2898
2899 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2900                 struct ufshcd_lrb *lrbp, int max_timeout)
2901 {
2902         int err = 0;
2903         unsigned long time_left;
2904         unsigned long flags;
2905
2906         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2907                         msecs_to_jiffies(max_timeout));
2908
2909         /* Make sure descriptors are ready before ringing the doorbell */
2910         wmb();
2911         spin_lock_irqsave(hba->host->host_lock, flags);
2912         hba->dev_cmd.complete = NULL;
2913         if (likely(time_left)) {
2914                 err = ufshcd_get_tr_ocs(lrbp);
2915                 if (!err)
2916                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2917         }
2918         spin_unlock_irqrestore(hba->host->host_lock, flags);
2919
2920         if (!time_left) {
2921                 err = -ETIMEDOUT;
2922                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2923                         __func__, lrbp->task_tag);
2924                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2925                         /* successfully cleared the command, retry if needed */
2926                         err = -EAGAIN;
2927                 /*
2928                  * in case of an error, after clearing the doorbell,
2929                  * we also need to clear the outstanding_request
2930                  * field in hba
2931                  */
2932                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2933         }
2934
2935         return err;
2936 }
2937
2938 /**
2939  * ufshcd_exec_dev_cmd - API for sending device management requests
2940  * @hba: UFS hba
2941  * @cmd_type: specifies the type (NOP, Query...)
2942  * @timeout: timeout in milliseconds
2943  *
2944  * NOTE: Since there is only one available tag for device management commands,
2945  * it is expected you hold the hba->dev_cmd.lock mutex.
2946  */
2947 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2948                 enum dev_cmd_type cmd_type, int timeout)
2949 {
2950         struct request_queue *q = hba->cmd_queue;
2951         struct request *req;
2952         struct ufshcd_lrb *lrbp;
2953         int err;
2954         int tag;
2955         struct completion wait;
2956
2957         down_read(&hba->clk_scaling_lock);
2958
2959         /*
2960          * Get free slot, sleep if slots are unavailable.
2961          * Even though we use wait_event() which sleeps indefinitely,
2962          * the maximum wait time is bounded by SCSI request timeout.
2963          */
2964         req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
2965         if (IS_ERR(req)) {
2966                 err = PTR_ERR(req);
2967                 goto out_unlock;
2968         }
2969         tag = req->tag;
2970         WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
2971         /* Set the timeout such that the SCSI error handler is not activated. */
2972         req->timeout = msecs_to_jiffies(2 * timeout);
2973         blk_mq_start_request(req);
2974
2975         if (unlikely(test_bit(tag, &hba->outstanding_reqs))) {
2976                 err = -EBUSY;
2977                 goto out;
2978         }
2979
2980         init_completion(&wait);
2981         lrbp = &hba->lrb[tag];
2982         WARN_ON(lrbp->cmd);
2983         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2984         if (unlikely(err))
2985                 goto out;
2986
2987         hba->dev_cmd.complete = &wait;
2988
2989         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
2990         /* Make sure descriptors are ready before ringing the doorbell */
2991         wmb();
2992
2993         ufshcd_send_command(hba, tag);
2994         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2995         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
2996                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
2997
2998 out:
2999         blk_put_request(req);
3000 out_unlock:
3001         up_read(&hba->clk_scaling_lock);
3002         return err;
3003 }
3004
3005 /**
3006  * ufshcd_init_query() - init the query response and request parameters
3007  * @hba: per-adapter instance
3008  * @request: address of the request pointer to be initialized
3009  * @response: address of the response pointer to be initialized
3010  * @opcode: operation to perform
3011  * @idn: flag idn to access
3012  * @index: LU number to access
3013  * @selector: query/flag/descriptor further identification
3014  */
3015 static inline void ufshcd_init_query(struct ufs_hba *hba,
3016                 struct ufs_query_req **request, struct ufs_query_res **response,
3017                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3018 {
3019         *request = &hba->dev_cmd.query.request;
3020         *response = &hba->dev_cmd.query.response;
3021         memset(*request, 0, sizeof(struct ufs_query_req));
3022         memset(*response, 0, sizeof(struct ufs_query_res));
3023         (*request)->upiu_req.opcode = opcode;
3024         (*request)->upiu_req.idn = idn;
3025         (*request)->upiu_req.index = index;
3026         (*request)->upiu_req.selector = selector;
3027 }
3028
3029 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3030         enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3031 {
3032         int ret;
3033         int retries;
3034
3035         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3036                 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3037                 if (ret)
3038                         dev_dbg(hba->dev,
3039                                 "%s: failed with error %d, retries %d\n",
3040                                 __func__, ret, retries);
3041                 else
3042                         break;
3043         }
3044
3045         if (ret)
3046                 dev_err(hba->dev,
3047                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
3048                         __func__, opcode, idn, ret, retries);
3049         return ret;
3050 }
3051
3052 /**
3053  * ufshcd_query_flag() - API function for sending flag query requests
3054  * @hba: per-adapter instance
3055  * @opcode: flag query to perform
3056  * @idn: flag idn to access
3057  * @index: flag index to access
3058  * @flag_res: the flag value after the query request completes
3059  *
3060  * Returns 0 for success, non-zero in case of failure
3061  */
3062 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3063                         enum flag_idn idn, u8 index, bool *flag_res)
3064 {
3065         struct ufs_query_req *request = NULL;
3066         struct ufs_query_res *response = NULL;
3067         int err, selector = 0;
3068         int timeout = QUERY_REQ_TIMEOUT;
3069
3070         BUG_ON(!hba);
3071
3072         ufshcd_hold(hba, false);
3073         mutex_lock(&hba->dev_cmd.lock);
3074         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3075                         selector);
3076
3077         switch (opcode) {
3078         case UPIU_QUERY_OPCODE_SET_FLAG:
3079         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3080         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3081                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3082                 break;
3083         case UPIU_QUERY_OPCODE_READ_FLAG:
3084                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3085                 if (!flag_res) {
3086                         /* No dummy reads */
3087                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
3088                                         __func__);
3089                         err = -EINVAL;
3090                         goto out_unlock;
3091                 }
3092                 break;
3093         default:
3094                 dev_err(hba->dev,
3095                         "%s: Expected query flag opcode but got = %d\n",
3096                         __func__, opcode);
3097                 err = -EINVAL;
3098                 goto out_unlock;
3099         }
3100
3101         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3102
3103         if (err) {
3104                 dev_err(hba->dev,
3105                         "%s: Sending flag query for idn %d failed, err = %d\n",
3106                         __func__, idn, err);
3107                 goto out_unlock;
3108         }
3109
3110         if (flag_res)
3111                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3112                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3113
3114 out_unlock:
3115         mutex_unlock(&hba->dev_cmd.lock);
3116         ufshcd_release(hba);
3117         return err;
3118 }
3119
3120 /**
3121  * ufshcd_query_attr - API function for sending attribute requests
3122  * @hba: per-adapter instance
3123  * @opcode: attribute opcode
3124  * @idn: attribute idn to access
3125  * @index: index field
3126  * @selector: selector field
3127  * @attr_val: the attribute value after the query request completes
3128  *
3129  * Returns 0 for success, non-zero in case of failure
3130 */
3131 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3132                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3133 {
3134         struct ufs_query_req *request = NULL;
3135         struct ufs_query_res *response = NULL;
3136         int err;
3137
3138         BUG_ON(!hba);
3139
3140         if (!attr_val) {
3141                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3142                                 __func__, opcode);
3143                 return -EINVAL;
3144         }
3145
3146         ufshcd_hold(hba, false);
3147
3148         mutex_lock(&hba->dev_cmd.lock);
3149         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3150                         selector);
3151
3152         switch (opcode) {
3153         case UPIU_QUERY_OPCODE_WRITE_ATTR:
3154                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3155                 request->upiu_req.value = cpu_to_be32(*attr_val);
3156                 break;
3157         case UPIU_QUERY_OPCODE_READ_ATTR:
3158                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3159                 break;
3160         default:
3161                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3162                                 __func__, opcode);
3163                 err = -EINVAL;
3164                 goto out_unlock;
3165         }
3166
3167         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3168
3169         if (err) {
3170                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3171                                 __func__, opcode, idn, index, err);
3172                 goto out_unlock;
3173         }
3174
3175         *attr_val = be32_to_cpu(response->upiu_res.value);
3176
3177 out_unlock:
3178         mutex_unlock(&hba->dev_cmd.lock);
3179         ufshcd_release(hba);
3180         return err;
3181 }
3182
3183 /**
3184  * ufshcd_query_attr_retry() - API function for sending query
3185  * attribute with retries
3186  * @hba: per-adapter instance
3187  * @opcode: attribute opcode
3188  * @idn: attribute idn to access
3189  * @index: index field
3190  * @selector: selector field
3191  * @attr_val: the attribute value after the query request
3192  * completes
3193  *
3194  * Returns 0 for success, non-zero in case of failure
3195 */
3196 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
3197         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3198         u32 *attr_val)
3199 {
3200         int ret = 0;
3201         u32 retries;
3202
3203         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3204                 ret = ufshcd_query_attr(hba, opcode, idn, index,
3205                                                 selector, attr_val);
3206                 if (ret)
3207                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3208                                 __func__, ret, retries);
3209                 else
3210                         break;
3211         }
3212
3213         if (ret)
3214                 dev_err(hba->dev,
3215                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
3216                         __func__, idn, ret, QUERY_REQ_RETRIES);
3217         return ret;
3218 }
3219
3220 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3221                         enum query_opcode opcode, enum desc_idn idn, u8 index,
3222                         u8 selector, u8 *desc_buf, int *buf_len)
3223 {
3224         struct ufs_query_req *request = NULL;
3225         struct ufs_query_res *response = NULL;
3226         int err;
3227
3228         BUG_ON(!hba);
3229
3230         if (!desc_buf) {
3231                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3232                                 __func__, opcode);
3233                 return -EINVAL;
3234         }
3235
3236         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3237                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3238                                 __func__, *buf_len);
3239                 return -EINVAL;
3240         }
3241
3242         ufshcd_hold(hba, false);
3243
3244         mutex_lock(&hba->dev_cmd.lock);
3245         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3246                         selector);
3247         hba->dev_cmd.query.descriptor = desc_buf;
3248         request->upiu_req.length = cpu_to_be16(*buf_len);
3249
3250         switch (opcode) {
3251         case UPIU_QUERY_OPCODE_WRITE_DESC:
3252                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3253                 break;
3254         case UPIU_QUERY_OPCODE_READ_DESC:
3255                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3256                 break;
3257         default:
3258                 dev_err(hba->dev,
3259                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3260                                 __func__, opcode);
3261                 err = -EINVAL;
3262                 goto out_unlock;
3263         }
3264
3265         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3266
3267         if (err) {
3268                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3269                                 __func__, opcode, idn, index, err);
3270                 goto out_unlock;
3271         }
3272
3273         *buf_len = be16_to_cpu(response->upiu_res.length);
3274
3275 out_unlock:
3276         hba->dev_cmd.query.descriptor = NULL;
3277         mutex_unlock(&hba->dev_cmd.lock);
3278         ufshcd_release(hba);
3279         return err;
3280 }
3281
3282 /**
3283  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3284  * @hba: per-adapter instance
3285  * @opcode: attribute opcode
3286  * @idn: attribute idn to access
3287  * @index: index field
3288  * @selector: selector field
3289  * @desc_buf: the buffer that contains the descriptor
3290  * @buf_len: length parameter passed to the device
3291  *
3292  * Returns 0 for success, non-zero in case of failure.
3293  * The buf_len parameter will contain, on return, the length parameter
3294  * received on the response.
3295  */
3296 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3297                                   enum query_opcode opcode,
3298                                   enum desc_idn idn, u8 index,
3299                                   u8 selector,
3300                                   u8 *desc_buf, int *buf_len)
3301 {
3302         int err;
3303         int retries;
3304
3305         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3306                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3307                                                 selector, desc_buf, buf_len);
3308                 if (!err || err == -EINVAL)
3309                         break;
3310         }
3311
3312         return err;
3313 }
3314
3315 /**
3316  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3317  * @hba: Pointer to adapter instance
3318  * @desc_id: descriptor idn value
3319  * @desc_len: mapped desc length (out)
3320  */
3321 void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
3322                                   int *desc_len)
3323 {
3324         if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 ||
3325             desc_id == QUERY_DESC_IDN_RFU_1)
3326                 *desc_len = 0;
3327         else
3328                 *desc_len = hba->desc_size[desc_id];
3329 }
3330 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3331
3332 static void ufshcd_update_desc_length(struct ufs_hba *hba,
3333                                       enum desc_idn desc_id, int desc_index,
3334                                       unsigned char desc_len)
3335 {
3336         if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE &&
3337             desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT)
3338                 /* For UFS 3.1, the normal unit descriptor is 10 bytes larger
3339                  * than the RPMB unit, however, both descriptors share the same
3340                  * desc_idn, to cover both unit descriptors with one length, we
3341                  * choose the normal unit descriptor length by desc_index.
3342                  */
3343                 hba->desc_size[desc_id] = desc_len;
3344 }
3345
3346 /**
3347  * ufshcd_read_desc_param - read the specified descriptor parameter
3348  * @hba: Pointer to adapter instance
3349  * @desc_id: descriptor idn value
3350  * @desc_index: descriptor index
3351  * @param_offset: offset of the parameter to read
3352  * @param_read_buf: pointer to buffer where parameter would be read
3353  * @param_size: sizeof(param_read_buf)
3354  *
3355  * Return 0 in case of success, non-zero otherwise
3356  */
3357 int ufshcd_read_desc_param(struct ufs_hba *hba,
3358                            enum desc_idn desc_id,
3359                            int desc_index,
3360                            u8 param_offset,
3361                            u8 *param_read_buf,
3362                            u8 param_size)
3363 {
3364         int ret;
3365         u8 *desc_buf;
3366         int buff_len;
3367         bool is_kmalloc = true;
3368
3369         /* Safety check */
3370         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3371                 return -EINVAL;
3372
3373         /* Get the length of descriptor */
3374         ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3375         if (!buff_len) {
3376                 dev_err(hba->dev, "%s: Failed to get desc length\n", __func__);
3377                 return -EINVAL;
3378         }
3379
3380         if (param_offset >= buff_len) {
3381                 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3382                         __func__, param_offset, desc_id, buff_len);
3383                 return -EINVAL;
3384         }
3385
3386         /* Check whether we need temp memory */
3387         if (param_offset != 0 || param_size < buff_len) {
3388                 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3389                 if (!desc_buf)
3390                         return -ENOMEM;
3391         } else {
3392                 desc_buf = param_read_buf;
3393                 is_kmalloc = false;
3394         }
3395
3396         /* Request for full descriptor */
3397         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3398                                         desc_id, desc_index, 0,
3399                                         desc_buf, &buff_len);
3400
3401         if (ret) {
3402                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3403                         __func__, desc_id, desc_index, param_offset, ret);
3404                 goto out;
3405         }
3406
3407         /* Sanity check */
3408         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3409                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3410                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3411                 ret = -EINVAL;
3412                 goto out;
3413         }
3414
3415         /* Update descriptor length */
3416         buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3417         ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len);
3418
3419         if (is_kmalloc) {
3420                 /* Make sure we don't copy more data than available */
3421                 if (param_offset + param_size > buff_len)
3422                         param_size = buff_len - param_offset;
3423                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3424         }
3425 out:
3426         if (is_kmalloc)
3427                 kfree(desc_buf);
3428         return ret;
3429 }
3430
3431 /**
3432  * struct uc_string_id - unicode string
3433  *
3434  * @len: size of this descriptor inclusive
3435  * @type: descriptor type
3436  * @uc: unicode string character
3437  */
3438 struct uc_string_id {
3439         u8 len;
3440         u8 type;
3441         wchar_t uc[];
3442 } __packed;
3443
3444 /* replace non-printable or non-ASCII characters with spaces */
3445 static inline char ufshcd_remove_non_printable(u8 ch)
3446 {
3447         return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3448 }
3449
3450 /**
3451  * ufshcd_read_string_desc - read string descriptor
3452  * @hba: pointer to adapter instance
3453  * @desc_index: descriptor index
3454  * @buf: pointer to buffer where descriptor would be read,
3455  *       the caller should free the memory.
3456  * @ascii: if true convert from unicode to ascii characters
3457  *         null terminated string.
3458  *
3459  * Return:
3460  * *      string size on success.
3461  * *      -ENOMEM: on allocation failure
3462  * *      -EINVAL: on a wrong parameter
3463  */
3464 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3465                             u8 **buf, bool ascii)
3466 {
3467         struct uc_string_id *uc_str;
3468         u8 *str;
3469         int ret;
3470
3471         if (!buf)
3472                 return -EINVAL;
3473
3474         uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3475         if (!uc_str)
3476                 return -ENOMEM;
3477
3478         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3479                                      (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3480         if (ret < 0) {
3481                 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3482                         QUERY_REQ_RETRIES, ret);
3483                 str = NULL;
3484                 goto out;
3485         }
3486
3487         if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3488                 dev_dbg(hba->dev, "String Desc is of zero length\n");
3489                 str = NULL;
3490                 ret = 0;
3491                 goto out;
3492         }
3493
3494         if (ascii) {
3495                 ssize_t ascii_len;
3496                 int i;
3497                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3498                 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3499                 str = kzalloc(ascii_len, GFP_KERNEL);
3500                 if (!str) {
3501                         ret = -ENOMEM;
3502                         goto out;
3503                 }
3504
3505                 /*
3506                  * the descriptor contains string in UTF16 format
3507                  * we need to convert to utf-8 so it can be displayed
3508                  */
3509                 ret = utf16s_to_utf8s(uc_str->uc,
3510                                       uc_str->len - QUERY_DESC_HDR_SIZE,
3511                                       UTF16_BIG_ENDIAN, str, ascii_len);
3512
3513                 /* replace non-printable or non-ASCII characters with spaces */
3514                 for (i = 0; i < ret; i++)
3515                         str[i] = ufshcd_remove_non_printable(str[i]);
3516
3517                 str[ret++] = '\0';
3518
3519         } else {
3520                 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3521                 if (!str) {
3522                         ret = -ENOMEM;
3523                         goto out;
3524                 }
3525                 ret = uc_str->len;
3526         }
3527 out:
3528         *buf = str;
3529         kfree(uc_str);
3530         return ret;
3531 }
3532
3533 /**
3534  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3535  * @hba: Pointer to adapter instance
3536  * @lun: lun id
3537  * @param_offset: offset of the parameter to read
3538  * @param_read_buf: pointer to buffer where parameter would be read
3539  * @param_size: sizeof(param_read_buf)
3540  *
3541  * Return 0 in case of success, non-zero otherwise
3542  */
3543 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3544                                               int lun,
3545                                               enum unit_desc_param param_offset,
3546                                               u8 *param_read_buf,
3547                                               u32 param_size)
3548 {
3549         /*
3550          * Unit descriptors are only available for general purpose LUs (LUN id
3551          * from 0 to 7) and RPMB Well known LU.
3552          */
3553         if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun, param_offset))
3554                 return -EOPNOTSUPP;
3555
3556         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3557                                       param_offset, param_read_buf, param_size);
3558 }
3559
3560 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3561 {
3562         int err = 0;
3563         u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3564
3565         if (hba->dev_info.wspecversion >= 0x300) {
3566                 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3567                                 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3568                                 &gating_wait);
3569                 if (err)
3570                         dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3571                                          err, gating_wait);
3572
3573                 if (gating_wait == 0) {
3574                         gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3575                         dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3576                                          gating_wait);
3577                 }
3578
3579                 hba->dev_info.clk_gating_wait_us = gating_wait;
3580         }
3581
3582         return err;
3583 }
3584
3585 /**
3586  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3587  * @hba: per adapter instance
3588  *
3589  * 1. Allocate DMA memory for Command Descriptor array
3590  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3591  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3592  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3593  *      (UTMRDL)
3594  * 4. Allocate memory for local reference block(lrb).
3595  *
3596  * Returns 0 for success, non-zero in case of failure
3597  */
3598 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3599 {
3600         size_t utmrdl_size, utrdl_size, ucdl_size;
3601
3602         /* Allocate memory for UTP command descriptors */
3603         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3604         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3605                                                   ucdl_size,
3606                                                   &hba->ucdl_dma_addr,
3607                                                   GFP_KERNEL);
3608
3609         /*
3610          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3611          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3612          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3613          * be aligned to 128 bytes as well
3614          */
3615         if (!hba->ucdl_base_addr ||
3616             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3617                 dev_err(hba->dev,
3618                         "Command Descriptor Memory allocation failed\n");
3619                 goto out;
3620         }
3621
3622         /*
3623          * Allocate memory for UTP Transfer descriptors
3624          * UFSHCI requires 1024 byte alignment of UTRD
3625          */
3626         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3627         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3628                                                    utrdl_size,
3629                                                    &hba->utrdl_dma_addr,
3630                                                    GFP_KERNEL);
3631         if (!hba->utrdl_base_addr ||
3632             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3633                 dev_err(hba->dev,
3634                         "Transfer Descriptor Memory allocation failed\n");
3635                 goto out;
3636         }
3637
3638         /*
3639          * Allocate memory for UTP Task Management descriptors
3640          * UFSHCI requires 1024 byte alignment of UTMRD
3641          */
3642         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3643         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3644                                                     utmrdl_size,
3645                                                     &hba->utmrdl_dma_addr,
3646                                                     GFP_KERNEL);
3647         if (!hba->utmrdl_base_addr ||
3648             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3649                 dev_err(hba->dev,
3650                 "Task Management Descriptor Memory allocation failed\n");
3651                 goto out;
3652         }
3653
3654         /* Allocate memory for local reference block */
3655         hba->lrb = devm_kcalloc(hba->dev,
3656                                 hba->nutrs, sizeof(struct ufshcd_lrb),
3657                                 GFP_KERNEL);
3658         if (!hba->lrb) {
3659                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3660                 goto out;
3661         }
3662         return 0;
3663 out:
3664         return -ENOMEM;
3665 }
3666
3667 /**
3668  * ufshcd_host_memory_configure - configure local reference block with
3669  *                              memory offsets
3670  * @hba: per adapter instance
3671  *
3672  * Configure Host memory space
3673  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3674  * address.
3675  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3676  * and PRDT offset.
3677  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3678  * into local reference block.
3679  */
3680 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3681 {
3682         struct utp_transfer_req_desc *utrdlp;
3683         dma_addr_t cmd_desc_dma_addr;
3684         dma_addr_t cmd_desc_element_addr;
3685         u16 response_offset;
3686         u16 prdt_offset;
3687         int cmd_desc_size;
3688         int i;
3689
3690         utrdlp = hba->utrdl_base_addr;
3691
3692         response_offset =
3693                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3694         prdt_offset =
3695                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3696
3697         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3698         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3699
3700         for (i = 0; i < hba->nutrs; i++) {
3701                 /* Configure UTRD with command descriptor base address */
3702                 cmd_desc_element_addr =
3703                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3704                 utrdlp[i].command_desc_base_addr_lo =
3705                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3706                 utrdlp[i].command_desc_base_addr_hi =
3707                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3708
3709                 /* Response upiu and prdt offset should be in double words */
3710                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3711                         utrdlp[i].response_upiu_offset =
3712                                 cpu_to_le16(response_offset);
3713                         utrdlp[i].prd_table_offset =
3714                                 cpu_to_le16(prdt_offset);
3715                         utrdlp[i].response_upiu_length =
3716                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3717                 } else {
3718                         utrdlp[i].response_upiu_offset =
3719                                 cpu_to_le16(response_offset >> 2);
3720                         utrdlp[i].prd_table_offset =
3721                                 cpu_to_le16(prdt_offset >> 2);
3722                         utrdlp[i].response_upiu_length =
3723                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3724                 }
3725
3726                 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3727         }
3728 }
3729
3730 /**
3731  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3732  * @hba: per adapter instance
3733  *
3734  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3735  * in order to initialize the Unipro link startup procedure.
3736  * Once the Unipro links are up, the device connected to the controller
3737  * is detected.
3738  *
3739  * Returns 0 on success, non-zero value on failure
3740  */
3741 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3742 {
3743         struct uic_command uic_cmd = {0};
3744         int ret;
3745
3746         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3747
3748         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3749         if (ret)
3750                 dev_dbg(hba->dev,
3751                         "dme-link-startup: error code %d\n", ret);
3752         return ret;
3753 }
3754 /**
3755  * ufshcd_dme_reset - UIC command for DME_RESET
3756  * @hba: per adapter instance
3757  *
3758  * DME_RESET command is issued in order to reset UniPro stack.
3759  * This function now deals with cold reset.
3760  *
3761  * Returns 0 on success, non-zero value on failure
3762  */
3763 static int ufshcd_dme_reset(struct ufs_hba *hba)
3764 {
3765         struct uic_command uic_cmd = {0};
3766         int ret;
3767
3768         uic_cmd.command = UIC_CMD_DME_RESET;
3769
3770         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3771         if (ret)
3772                 dev_err(hba->dev,
3773                         "dme-reset: error code %d\n", ret);
3774
3775         return ret;
3776 }
3777
3778 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
3779                                int agreed_gear,
3780                                int adapt_val)
3781 {
3782         int ret;
3783
3784         if (agreed_gear != UFS_HS_G4)
3785                 adapt_val = PA_NO_ADAPT;
3786
3787         ret = ufshcd_dme_set(hba,
3788                              UIC_ARG_MIB(PA_TXHSADAPTTYPE),
3789                              adapt_val);
3790         return ret;
3791 }
3792 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
3793
3794 /**
3795  * ufshcd_dme_enable - UIC command for DME_ENABLE
3796  * @hba: per adapter instance
3797  *
3798  * DME_ENABLE command is issued in order to enable UniPro stack.
3799  *
3800  * Returns 0 on success, non-zero value on failure
3801  */
3802 static int ufshcd_dme_enable(struct ufs_hba *hba)
3803 {
3804         struct uic_command uic_cmd = {0};
3805         int ret;
3806
3807         uic_cmd.command = UIC_CMD_DME_ENABLE;
3808
3809         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3810         if (ret)
3811                 dev_err(hba->dev,
3812                         "dme-enable: error code %d\n", ret);
3813
3814         return ret;
3815 }
3816
3817 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3818 {
3819         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
3820         unsigned long min_sleep_time_us;
3821
3822         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3823                 return;
3824
3825         /*
3826          * last_dme_cmd_tstamp will be 0 only for 1st call to
3827          * this function
3828          */
3829         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3830                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3831         } else {
3832                 unsigned long delta =
3833                         (unsigned long) ktime_to_us(
3834                                 ktime_sub(ktime_get(),
3835                                 hba->last_dme_cmd_tstamp));
3836
3837                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3838                         min_sleep_time_us =
3839                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3840                 else
3841                         return; /* no more delay required */
3842         }
3843
3844         /* allow sleep for extra 50us if needed */
3845         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3846 }
3847
3848 /**
3849  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3850  * @hba: per adapter instance
3851  * @attr_sel: uic command argument1
3852  * @attr_set: attribute set type as uic command argument2
3853  * @mib_val: setting value as uic command argument3
3854  * @peer: indicate whether peer or local
3855  *
3856  * Returns 0 on success, non-zero value on failure
3857  */
3858 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3859                         u8 attr_set, u32 mib_val, u8 peer)
3860 {
3861         struct uic_command uic_cmd = {0};
3862         static const char *const action[] = {
3863                 "dme-set",
3864                 "dme-peer-set"
3865         };
3866         const char *set = action[!!peer];
3867         int ret;
3868         int retries = UFS_UIC_COMMAND_RETRIES;
3869
3870         uic_cmd.command = peer ?
3871                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3872         uic_cmd.argument1 = attr_sel;
3873         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3874         uic_cmd.argument3 = mib_val;
3875
3876         do {
3877                 /* for peer attributes we retry upon failure */
3878                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3879                 if (ret)
3880                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3881                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3882         } while (ret && peer && --retries);
3883
3884         if (ret)
3885                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3886                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3887                         UFS_UIC_COMMAND_RETRIES - retries);
3888
3889         return ret;
3890 }
3891 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3892
3893 /**
3894  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3895  * @hba: per adapter instance
3896  * @attr_sel: uic command argument1
3897  * @mib_val: the value of the attribute as returned by the UIC command
3898  * @peer: indicate whether peer or local
3899  *
3900  * Returns 0 on success, non-zero value on failure
3901  */
3902 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3903                         u32 *mib_val, u8 peer)
3904 {
3905         struct uic_command uic_cmd = {0};
3906         static const char *const action[] = {
3907                 "dme-get",
3908                 "dme-peer-get"
3909         };
3910         const char *get = action[!!peer];
3911         int ret;
3912         int retries = UFS_UIC_COMMAND_RETRIES;
3913         struct ufs_pa_layer_attr orig_pwr_info;
3914         struct ufs_pa_layer_attr temp_pwr_info;
3915         bool pwr_mode_change = false;
3916
3917         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3918                 orig_pwr_info = hba->pwr_info;
3919                 temp_pwr_info = orig_pwr_info;
3920
3921                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3922                     orig_pwr_info.pwr_rx == FAST_MODE) {
3923                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3924                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3925                         pwr_mode_change = true;
3926                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3927                     orig_pwr_info.pwr_rx == SLOW_MODE) {
3928                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3929                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3930                         pwr_mode_change = true;
3931                 }
3932                 if (pwr_mode_change) {
3933                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3934                         if (ret)
3935                                 goto out;
3936                 }
3937         }
3938
3939         uic_cmd.command = peer ?
3940                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3941         uic_cmd.argument1 = attr_sel;
3942
3943         do {
3944                 /* for peer attributes we retry upon failure */
3945                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3946                 if (ret)
3947                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3948                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
3949         } while (ret && peer && --retries);
3950
3951         if (ret)
3952                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
3953                         get, UIC_GET_ATTR_ID(attr_sel),
3954                         UFS_UIC_COMMAND_RETRIES - retries);
3955
3956         if (mib_val && !ret)
3957                 *mib_val = uic_cmd.argument3;
3958
3959         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3960             && pwr_mode_change)
3961                 ufshcd_change_power_mode(hba, &orig_pwr_info);
3962 out:
3963         return ret;
3964 }
3965 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3966
3967 /**
3968  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3969  * state) and waits for it to take effect.
3970  *
3971  * @hba: per adapter instance
3972  * @cmd: UIC command to execute
3973  *
3974  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3975  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3976  * and device UniPro link and hence it's final completion would be indicated by
3977  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3978  * addition to normal UIC command completion Status (UCCS). This function only
3979  * returns after the relevant status bits indicate the completion.
3980  *
3981  * Returns 0 on success, non-zero value on failure
3982  */
3983 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
3984 {
3985         struct completion uic_async_done;
3986         unsigned long flags;
3987         u8 status;
3988         int ret;
3989         bool reenable_intr = false;
3990
3991         mutex_lock(&hba->uic_cmd_mutex);
3992         init_completion(&uic_async_done);
3993         ufshcd_add_delay_before_dme_cmd(hba);
3994
3995         spin_lock_irqsave(hba->host->host_lock, flags);
3996         if (ufshcd_is_link_broken(hba)) {
3997                 ret = -ENOLINK;
3998                 goto out_unlock;
3999         }
4000         hba->uic_async_done = &uic_async_done;
4001         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4002                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4003                 /*
4004                  * Make sure UIC command completion interrupt is disabled before
4005                  * issuing UIC command.
4006                  */
4007                 wmb();
4008                 reenable_intr = true;
4009         }
4010         ret = __ufshcd_send_uic_cmd(hba, cmd, false);
4011         spin_unlock_irqrestore(hba->host->host_lock, flags);
4012         if (ret) {
4013                 dev_err(hba->dev,
4014                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4015                         cmd->command, cmd->argument3, ret);
4016                 goto out;
4017         }
4018
4019         if (!wait_for_completion_timeout(hba->uic_async_done,
4020                                          msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
4021                 dev_err(hba->dev,
4022                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4023                         cmd->command, cmd->argument3);
4024
4025                 if (!cmd->cmd_active) {
4026                         dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4027                                 __func__);
4028                         goto check_upmcrs;
4029                 }
4030
4031                 ret = -ETIMEDOUT;
4032                 goto out;
4033         }
4034
4035 check_upmcrs:
4036         status = ufshcd_get_upmcrs(hba);
4037         if (status != PWR_LOCAL) {
4038                 dev_err(hba->dev,
4039                         "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4040                         cmd->command, status);
4041                 ret = (status != PWR_OK) ? status : -1;
4042         }
4043 out:
4044         if (ret) {
4045                 ufshcd_print_host_state(hba);
4046                 ufshcd_print_pwr_info(hba);
4047                 ufshcd_print_evt_hist(hba);
4048         }
4049
4050         spin_lock_irqsave(hba->host->host_lock, flags);
4051         hba->active_uic_cmd = NULL;
4052         hba->uic_async_done = NULL;
4053         if (reenable_intr)
4054                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4055         if (ret) {
4056                 ufshcd_set_link_broken(hba);
4057                 ufshcd_schedule_eh_work(hba);
4058         }
4059 out_unlock:
4060         spin_unlock_irqrestore(hba->host->host_lock, flags);
4061         mutex_unlock(&hba->uic_cmd_mutex);
4062
4063         return ret;
4064 }
4065
4066 /**
4067  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4068  *                              using DME_SET primitives.
4069  * @hba: per adapter instance
4070  * @mode: powr mode value
4071  *
4072  * Returns 0 on success, non-zero value on failure
4073  */
4074 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4075 {
4076         struct uic_command uic_cmd = {0};
4077         int ret;
4078
4079         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4080                 ret = ufshcd_dme_set(hba,
4081                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4082                 if (ret) {
4083                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4084                                                 __func__, ret);
4085                         goto out;
4086                 }
4087         }
4088
4089         uic_cmd.command = UIC_CMD_DME_SET;
4090         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
4091         uic_cmd.argument3 = mode;
4092         ufshcd_hold(hba, false);
4093         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4094         ufshcd_release(hba);
4095
4096 out:
4097         return ret;
4098 }
4099
4100 int ufshcd_link_recovery(struct ufs_hba *hba)
4101 {
4102         int ret;
4103         unsigned long flags;
4104
4105         spin_lock_irqsave(hba->host->host_lock, flags);
4106         hba->ufshcd_state = UFSHCD_STATE_RESET;
4107         ufshcd_set_eh_in_progress(hba);
4108         spin_unlock_irqrestore(hba->host->host_lock, flags);
4109
4110         /* Reset the attached device */
4111         ufshcd_device_reset(hba);
4112
4113         ret = ufshcd_host_reset_and_restore(hba);
4114
4115         spin_lock_irqsave(hba->host->host_lock, flags);
4116         if (ret)
4117                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4118         ufshcd_clear_eh_in_progress(hba);
4119         spin_unlock_irqrestore(hba->host->host_lock, flags);
4120
4121         if (ret)
4122                 dev_err(hba->dev, "%s: link recovery failed, err %d",
4123                         __func__, ret);
4124         else
4125                 ufshcd_clear_ua_wluns(hba);
4126
4127         return ret;
4128 }
4129 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4130
4131 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4132 {
4133         int ret;
4134         struct uic_command uic_cmd = {0};
4135         ktime_t start = ktime_get();
4136
4137         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4138
4139         uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
4140         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4141         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4142                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4143
4144         if (ret)
4145                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4146                         __func__, ret);
4147         else
4148                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4149                                                                 POST_CHANGE);
4150
4151         return ret;
4152 }
4153
4154 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4155 {
4156         struct uic_command uic_cmd = {0};
4157         int ret;
4158         ktime_t start = ktime_get();
4159
4160         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4161
4162         uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
4163         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4164         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4165                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4166
4167         if (ret) {
4168                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4169                         __func__, ret);
4170         } else {
4171                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4172                                                                 POST_CHANGE);
4173                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
4174                 hba->ufs_stats.hibern8_exit_cnt++;
4175         }
4176
4177         return ret;
4178 }
4179 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4180
4181 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4182 {
4183         unsigned long flags;
4184         bool update = false;
4185
4186         if (!ufshcd_is_auto_hibern8_supported(hba))
4187                 return;
4188
4189         spin_lock_irqsave(hba->host->host_lock, flags);
4190         if (hba->ahit != ahit) {
4191                 hba->ahit = ahit;
4192                 update = true;
4193         }
4194         spin_unlock_irqrestore(hba->host->host_lock, flags);
4195
4196         if (update &&
4197             !pm_runtime_suspended(&hba->sdev_ufs_device->sdev_gendev)) {
4198                 ufshcd_rpm_get_sync(hba);
4199                 ufshcd_hold(hba, false);
4200                 ufshcd_auto_hibern8_enable(hba);
4201                 ufshcd_release(hba);
4202                 ufshcd_rpm_put_sync(hba);
4203         }
4204 }
4205 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4206
4207 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
4208 {
4209         unsigned long flags;
4210
4211         if (!ufshcd_is_auto_hibern8_supported(hba))
4212                 return;
4213
4214         spin_lock_irqsave(hba->host->host_lock, flags);
4215         ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4216         spin_unlock_irqrestore(hba->host->host_lock, flags);
4217 }
4218
4219  /**
4220  * ufshcd_init_pwr_info - setting the POR (power on reset)
4221  * values in hba power info
4222  * @hba: per-adapter instance
4223  */
4224 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4225 {
4226         hba->pwr_info.gear_rx = UFS_PWM_G1;
4227         hba->pwr_info.gear_tx = UFS_PWM_G1;
4228         hba->pwr_info.lane_rx = 1;
4229         hba->pwr_info.lane_tx = 1;
4230         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4231         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4232         hba->pwr_info.hs_rate = 0;
4233 }
4234
4235 /**
4236  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4237  * @hba: per-adapter instance
4238  */
4239 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4240 {
4241         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4242
4243         if (hba->max_pwr_info.is_valid)
4244                 return 0;
4245
4246         pwr_info->pwr_tx = FAST_MODE;
4247         pwr_info->pwr_rx = FAST_MODE;
4248         pwr_info->hs_rate = PA_HS_MODE_B;
4249
4250         /* Get the connected lane count */
4251         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4252                         &pwr_info->lane_rx);
4253         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4254                         &pwr_info->lane_tx);
4255
4256         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4257                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4258                                 __func__,
4259                                 pwr_info->lane_rx,
4260                                 pwr_info->lane_tx);
4261                 return -EINVAL;
4262         }
4263
4264         /*
4265          * First, get the maximum gears of HS speed.
4266          * If a zero value, it means there is no HSGEAR capability.
4267          * Then, get the maximum gears of PWM speed.
4268          */
4269         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4270         if (!pwr_info->gear_rx) {
4271                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4272                                 &pwr_info->gear_rx);
4273                 if (!pwr_info->gear_rx) {
4274                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4275                                 __func__, pwr_info->gear_rx);
4276                         return -EINVAL;
4277                 }
4278                 pwr_info->pwr_rx = SLOW_MODE;
4279         }
4280
4281         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4282                         &pwr_info->gear_tx);
4283         if (!pwr_info->gear_tx) {
4284                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4285                                 &pwr_info->gear_tx);
4286                 if (!pwr_info->gear_tx) {
4287                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4288                                 __func__, pwr_info->gear_tx);
4289                         return -EINVAL;
4290                 }
4291                 pwr_info->pwr_tx = SLOW_MODE;
4292         }
4293
4294         hba->max_pwr_info.is_valid = true;
4295         return 0;
4296 }
4297
4298 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4299                              struct ufs_pa_layer_attr *pwr_mode)
4300 {
4301         int ret;
4302
4303         /* if already configured to the requested pwr_mode */
4304         if (!hba->force_pmc &&
4305             pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4306             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4307             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4308             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4309             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4310             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4311             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4312                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4313                 return 0;
4314         }
4315
4316         /*
4317          * Configure attributes for power mode change with below.
4318          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4319          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4320          * - PA_HSSERIES
4321          */
4322         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4323         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4324                         pwr_mode->lane_rx);
4325         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4326                         pwr_mode->pwr_rx == FAST_MODE)
4327                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
4328         else
4329                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
4330
4331         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4332         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4333                         pwr_mode->lane_tx);
4334         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4335                         pwr_mode->pwr_tx == FAST_MODE)
4336                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
4337         else
4338                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
4339
4340         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4341             pwr_mode->pwr_tx == FASTAUTO_MODE ||
4342             pwr_mode->pwr_rx == FAST_MODE ||
4343             pwr_mode->pwr_tx == FAST_MODE)
4344                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4345                                                 pwr_mode->hs_rate);
4346
4347         if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4348                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4349                                 DL_FC0ProtectionTimeOutVal_Default);
4350                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4351                                 DL_TC0ReplayTimeOutVal_Default);
4352                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4353                                 DL_AFC0ReqTimeOutVal_Default);
4354                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4355                                 DL_FC1ProtectionTimeOutVal_Default);
4356                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4357                                 DL_TC1ReplayTimeOutVal_Default);
4358                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4359                                 DL_AFC1ReqTimeOutVal_Default);
4360
4361                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4362                                 DL_FC0ProtectionTimeOutVal_Default);
4363                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4364                                 DL_TC0ReplayTimeOutVal_Default);
4365                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4366                                 DL_AFC0ReqTimeOutVal_Default);
4367         }
4368
4369         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4370                         | pwr_mode->pwr_tx);
4371
4372         if (ret) {
4373                 dev_err(hba->dev,
4374                         "%s: power mode change failed %d\n", __func__, ret);
4375         } else {
4376                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4377                                                                 pwr_mode);
4378
4379                 memcpy(&hba->pwr_info, pwr_mode,
4380                         sizeof(struct ufs_pa_layer_attr));
4381         }
4382
4383         return ret;
4384 }
4385
4386 /**
4387  * ufshcd_config_pwr_mode - configure a new power mode
4388  * @hba: per-adapter instance
4389  * @desired_pwr_mode: desired power configuration
4390  */
4391 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4392                 struct ufs_pa_layer_attr *desired_pwr_mode)
4393 {
4394         struct ufs_pa_layer_attr final_params = { 0 };
4395         int ret;
4396
4397         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4398                                         desired_pwr_mode, &final_params);
4399
4400         if (ret)
4401                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4402
4403         ret = ufshcd_change_power_mode(hba, &final_params);
4404
4405         return ret;
4406 }
4407 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4408
4409 /**
4410  * ufshcd_complete_dev_init() - checks device readiness
4411  * @hba: per-adapter instance
4412  *
4413  * Set fDeviceInit flag and poll until device toggles it.
4414  */
4415 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4416 {
4417         int err;
4418         bool flag_res = true;
4419         ktime_t timeout;
4420
4421         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4422                 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4423         if (err) {
4424                 dev_err(hba->dev,
4425                         "%s setting fDeviceInit flag failed with error %d\n",
4426                         __func__, err);
4427                 goto out;
4428         }
4429
4430         /* Poll fDeviceInit flag to be cleared */
4431         timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4432         do {
4433                 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4434                                         QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4435                 if (!flag_res)
4436                         break;
4437                 usleep_range(5000, 10000);
4438         } while (ktime_before(ktime_get(), timeout));
4439
4440         if (err) {
4441                 dev_err(hba->dev,
4442                                 "%s reading fDeviceInit flag failed with error %d\n",
4443                                 __func__, err);
4444         } else if (flag_res) {
4445                 dev_err(hba->dev,
4446                                 "%s fDeviceInit was not cleared by the device\n",
4447                                 __func__);
4448                 err = -EBUSY;
4449         }
4450 out:
4451         return err;
4452 }
4453
4454 /**
4455  * ufshcd_make_hba_operational - Make UFS controller operational
4456  * @hba: per adapter instance
4457  *
4458  * To bring UFS host controller to operational state,
4459  * 1. Enable required interrupts
4460  * 2. Configure interrupt aggregation
4461  * 3. Program UTRL and UTMRL base address
4462  * 4. Configure run-stop-registers
4463  *
4464  * Returns 0 on success, non-zero value on failure
4465  */
4466 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4467 {
4468         int err = 0;
4469         u32 reg;
4470
4471         /* Enable required interrupts */
4472         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4473
4474         /* Configure interrupt aggregation */
4475         if (ufshcd_is_intr_aggr_allowed(hba))
4476                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4477         else
4478                 ufshcd_disable_intr_aggr(hba);
4479
4480         /* Configure UTRL and UTMRL base address registers */
4481         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4482                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4483         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4484                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4485         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4486                         REG_UTP_TASK_REQ_LIST_BASE_L);
4487         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4488                         REG_UTP_TASK_REQ_LIST_BASE_H);
4489
4490         /*
4491          * Make sure base address and interrupt setup are updated before
4492          * enabling the run/stop registers below.
4493          */
4494         wmb();
4495
4496         /*
4497          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4498          */
4499         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4500         if (!(ufshcd_get_lists_status(reg))) {
4501                 ufshcd_enable_run_stop_reg(hba);
4502         } else {
4503                 dev_err(hba->dev,
4504                         "Host controller not ready to process requests");
4505                 err = -EIO;
4506         }
4507
4508         return err;
4509 }
4510 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4511
4512 /**
4513  * ufshcd_hba_stop - Send controller to reset state
4514  * @hba: per adapter instance
4515  */
4516 void ufshcd_hba_stop(struct ufs_hba *hba)
4517 {
4518         unsigned long flags;
4519         int err;
4520
4521         /*
4522          * Obtain the host lock to prevent that the controller is disabled
4523          * while the UFS interrupt handler is active on another CPU.
4524          */
4525         spin_lock_irqsave(hba->host->host_lock, flags);
4526         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4527         spin_unlock_irqrestore(hba->host->host_lock, flags);
4528
4529         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4530                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4531                                         10, 1);
4532         if (err)
4533                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4534 }
4535 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4536
4537 /**
4538  * ufshcd_hba_execute_hce - initialize the controller
4539  * @hba: per adapter instance
4540  *
4541  * The controller resets itself and controller firmware initialization
4542  * sequence kicks off. When controller is ready it will set
4543  * the Host Controller Enable bit to 1.
4544  *
4545  * Returns 0 on success, non-zero value on failure
4546  */
4547 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4548 {
4549         int retry_outer = 3;
4550         int retry_inner;
4551
4552 start:
4553         if (!ufshcd_is_hba_active(hba))
4554                 /* change controller state to "reset state" */
4555                 ufshcd_hba_stop(hba);
4556
4557         /* UniPro link is disabled at this point */
4558         ufshcd_set_link_off(hba);
4559
4560         ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4561
4562         /* start controller initialization sequence */
4563         ufshcd_hba_start(hba);
4564
4565         /*
4566          * To initialize a UFS host controller HCE bit must be set to 1.
4567          * During initialization the HCE bit value changes from 1->0->1.
4568          * When the host controller completes initialization sequence
4569          * it sets the value of HCE bit to 1. The same HCE bit is read back
4570          * to check if the controller has completed initialization sequence.
4571          * So without this delay the value HCE = 1, set in the previous
4572          * instruction might be read back.
4573          * This delay can be changed based on the controller.
4574          */
4575         ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4576
4577         /* wait for the host controller to complete initialization */
4578         retry_inner = 50;
4579         while (ufshcd_is_hba_active(hba)) {
4580                 if (retry_inner) {
4581                         retry_inner--;
4582                 } else {
4583                         dev_err(hba->dev,
4584                                 "Controller enable failed\n");
4585                         if (retry_outer) {
4586                                 retry_outer--;
4587                                 goto start;
4588                         }
4589                         return -EIO;
4590                 }
4591                 usleep_range(1000, 1100);
4592         }
4593
4594         /* enable UIC related interrupts */
4595         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4596
4597         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4598
4599         return 0;
4600 }
4601
4602 int ufshcd_hba_enable(struct ufs_hba *hba)
4603 {
4604         int ret;
4605
4606         if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4607                 ufshcd_set_link_off(hba);
4608                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4609
4610                 /* enable UIC related interrupts */
4611                 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4612                 ret = ufshcd_dme_reset(hba);
4613                 if (!ret) {
4614                         ret = ufshcd_dme_enable(hba);
4615                         if (!ret)
4616                                 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4617                         if (ret)
4618                                 dev_err(hba->dev,
4619                                         "Host controller enable failed with non-hce\n");
4620                 }
4621         } else {
4622                 ret = ufshcd_hba_execute_hce(hba);
4623         }
4624
4625         return ret;
4626 }
4627 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4628
4629 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4630 {
4631         int tx_lanes = 0, i, err = 0;
4632
4633         if (!peer)
4634                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4635                                &tx_lanes);
4636         else
4637                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4638                                     &tx_lanes);
4639         for (i = 0; i < tx_lanes; i++) {
4640                 if (!peer)
4641                         err = ufshcd_dme_set(hba,
4642                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4643                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4644                                         0);
4645                 else
4646                         err = ufshcd_dme_peer_set(hba,
4647                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4648                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4649                                         0);
4650                 if (err) {
4651                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4652                                 __func__, peer, i, err);
4653                         break;
4654                 }
4655         }
4656
4657         return err;
4658 }
4659
4660 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4661 {
4662         return ufshcd_disable_tx_lcc(hba, true);
4663 }
4664
4665 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4666 {
4667         struct ufs_event_hist *e;
4668
4669         if (id >= UFS_EVT_CNT)
4670                 return;
4671
4672         e = &hba->ufs_stats.event[id];
4673         e->val[e->pos] = val;
4674         e->tstamp[e->pos] = ktime_get();
4675         e->cnt += 1;
4676         e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4677
4678         ufshcd_vops_event_notify(hba, id, &val);
4679 }
4680 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4681
4682 /**
4683  * ufshcd_link_startup - Initialize unipro link startup
4684  * @hba: per adapter instance
4685  *
4686  * Returns 0 for success, non-zero in case of failure
4687  */
4688 static int ufshcd_link_startup(struct ufs_hba *hba)
4689 {
4690         int ret;
4691         int retries = DME_LINKSTARTUP_RETRIES;
4692         bool link_startup_again = false;
4693
4694         /*
4695          * If UFS device isn't active then we will have to issue link startup
4696          * 2 times to make sure the device state move to active.
4697          */
4698         if (!ufshcd_is_ufs_dev_active(hba))
4699                 link_startup_again = true;
4700
4701 link_startup:
4702         do {
4703                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4704
4705                 ret = ufshcd_dme_link_startup(hba);
4706
4707                 /* check if device is detected by inter-connect layer */
4708                 if (!ret && !ufshcd_is_device_present(hba)) {
4709                         ufshcd_update_evt_hist(hba,
4710                                                UFS_EVT_LINK_STARTUP_FAIL,
4711                                                0);
4712                         dev_err(hba->dev, "%s: Device not present\n", __func__);
4713                         ret = -ENXIO;
4714                         goto out;
4715                 }
4716
4717                 /*
4718                  * DME link lost indication is only received when link is up,
4719                  * but we can't be sure if the link is up until link startup
4720                  * succeeds. So reset the local Uni-Pro and try again.
4721                  */
4722                 if (ret && ufshcd_hba_enable(hba)) {
4723                         ufshcd_update_evt_hist(hba,
4724                                                UFS_EVT_LINK_STARTUP_FAIL,
4725                                                (u32)ret);
4726                         goto out;
4727                 }
4728         } while (ret && retries--);
4729
4730         if (ret) {
4731                 /* failed to get the link up... retire */
4732                 ufshcd_update_evt_hist(hba,
4733                                        UFS_EVT_LINK_STARTUP_FAIL,
4734                                        (u32)ret);
4735                 goto out;
4736         }
4737
4738         if (link_startup_again) {
4739                 link_startup_again = false;
4740                 retries = DME_LINKSTARTUP_RETRIES;
4741                 goto link_startup;
4742         }
4743
4744         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4745         ufshcd_init_pwr_info(hba);
4746         ufshcd_print_pwr_info(hba);
4747
4748         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4749                 ret = ufshcd_disable_device_tx_lcc(hba);
4750                 if (ret)
4751                         goto out;
4752         }
4753
4754         /* Include any host controller configuration via UIC commands */
4755         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4756         if (ret)
4757                 goto out;
4758
4759         /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
4760         ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
4761         ret = ufshcd_make_hba_operational(hba);
4762 out:
4763         if (ret) {
4764                 dev_err(hba->dev, "link startup failed %d\n", ret);
4765                 ufshcd_print_host_state(hba);
4766                 ufshcd_print_pwr_info(hba);
4767                 ufshcd_print_evt_hist(hba);
4768         }
4769         return ret;
4770 }
4771
4772 /**
4773  * ufshcd_verify_dev_init() - Verify device initialization
4774  * @hba: per-adapter instance
4775  *
4776  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4777  * device Transport Protocol (UTP) layer is ready after a reset.
4778  * If the UTP layer at the device side is not initialized, it may
4779  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4780  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4781  */
4782 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4783 {
4784         int err = 0;
4785         int retries;
4786
4787         ufshcd_hold(hba, false);
4788         mutex_lock(&hba->dev_cmd.lock);
4789         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4790                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4791                                                NOP_OUT_TIMEOUT);
4792
4793                 if (!err || err == -ETIMEDOUT)
4794                         break;
4795
4796                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4797         }
4798         mutex_unlock(&hba->dev_cmd.lock);
4799         ufshcd_release(hba);
4800
4801         if (err)
4802                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4803         return err;
4804 }
4805
4806 /**
4807  * ufshcd_set_queue_depth - set lun queue depth
4808  * @sdev: pointer to SCSI device
4809  *
4810  * Read bLUQueueDepth value and activate scsi tagged command
4811  * queueing. For WLUN, queue depth is set to 1. For best-effort
4812  * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4813  * value that host can queue.
4814  */
4815 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4816 {
4817         int ret = 0;
4818         u8 lun_qdepth;
4819         struct ufs_hba *hba;
4820
4821         hba = shost_priv(sdev->host);
4822
4823         lun_qdepth = hba->nutrs;
4824         ret = ufshcd_read_unit_desc_param(hba,
4825                                           ufshcd_scsi_to_upiu_lun(sdev->lun),
4826                                           UNIT_DESC_PARAM_LU_Q_DEPTH,
4827                                           &lun_qdepth,
4828                                           sizeof(lun_qdepth));
4829
4830         /* Some WLUN doesn't support unit descriptor */
4831         if (ret == -EOPNOTSUPP)
4832                 lun_qdepth = 1;
4833         else if (!lun_qdepth)
4834                 /* eventually, we can figure out the real queue depth */
4835                 lun_qdepth = hba->nutrs;
4836         else
4837                 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4838
4839         dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4840                         __func__, lun_qdepth);
4841         scsi_change_queue_depth(sdev, lun_qdepth);
4842 }
4843
4844 /*
4845  * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4846  * @hba: per-adapter instance
4847  * @lun: UFS device lun id
4848  * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4849  *
4850  * Returns 0 in case of success and b_lu_write_protect status would be returned
4851  * @b_lu_write_protect parameter.
4852  * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4853  * Returns -EINVAL in case of invalid parameters passed to this function.
4854  */
4855 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4856                             u8 lun,
4857                             u8 *b_lu_write_protect)
4858 {
4859         int ret;
4860
4861         if (!b_lu_write_protect)
4862                 ret = -EINVAL;
4863         /*
4864          * According to UFS device spec, RPMB LU can't be write
4865          * protected so skip reading bLUWriteProtect parameter for
4866          * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4867          */
4868         else if (lun >= hba->dev_info.max_lu_supported)
4869                 ret = -ENOTSUPP;
4870         else
4871                 ret = ufshcd_read_unit_desc_param(hba,
4872                                           lun,
4873                                           UNIT_DESC_PARAM_LU_WR_PROTECT,
4874                                           b_lu_write_protect,
4875                                           sizeof(*b_lu_write_protect));
4876         return ret;
4877 }
4878
4879 /**
4880  * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4881  * status
4882  * @hba: per-adapter instance
4883  * @sdev: pointer to SCSI device
4884  *
4885  */
4886 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4887                                                     struct scsi_device *sdev)
4888 {
4889         if (hba->dev_info.f_power_on_wp_en &&
4890             !hba->dev_info.is_lu_power_on_wp) {
4891                 u8 b_lu_write_protect;
4892
4893                 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4894                                       &b_lu_write_protect) &&
4895                     (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4896                         hba->dev_info.is_lu_power_on_wp = true;
4897         }
4898 }
4899
4900 /**
4901  * ufshcd_setup_links - associate link b/w device wlun and other luns
4902  * @sdev: pointer to SCSI device
4903  * @hba: pointer to ufs hba
4904  */
4905 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
4906 {
4907         struct device_link *link;
4908
4909         /*
4910          * Device wlun is the supplier & rest of the luns are consumers.
4911          * This ensures that device wlun suspends after all other luns.
4912          */
4913         if (hba->sdev_ufs_device) {
4914                 link = device_link_add(&sdev->sdev_gendev,
4915                                        &hba->sdev_ufs_device->sdev_gendev,
4916                                        DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
4917                 if (!link) {
4918                         dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
4919                                 dev_name(&hba->sdev_ufs_device->sdev_gendev));
4920                         return;
4921                 }
4922                 hba->luns_avail--;
4923                 /* Ignore REPORT_LUN wlun probing */
4924                 if (hba->luns_avail == 1) {
4925                         ufshcd_rpm_put(hba);
4926                         return;
4927                 }
4928         } else {
4929                 /*
4930                  * Device wlun is probed. The assumption is that WLUNs are
4931                  * scanned before other LUNs.
4932                  */
4933                 hba->luns_avail--;
4934         }
4935 }
4936
4937 /**
4938  * ufshcd_slave_alloc - handle initial SCSI device configurations
4939  * @sdev: pointer to SCSI device
4940  *
4941  * Returns success
4942  */
4943 static int ufshcd_slave_alloc(struct scsi_device *sdev)
4944 {
4945         struct ufs_hba *hba;
4946
4947         hba = shost_priv(sdev->host);
4948
4949         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4950         sdev->use_10_for_ms = 1;
4951
4952         /* DBD field should be set to 1 in mode sense(10) */
4953         sdev->set_dbd_for_ms = 1;
4954
4955         /* allow SCSI layer to restart the device in case of errors */
4956         sdev->allow_restart = 1;
4957
4958         /* REPORT SUPPORTED OPERATION CODES is not supported */
4959         sdev->no_report_opcodes = 1;
4960
4961         /* WRITE_SAME command is not supported */
4962         sdev->no_write_same = 1;
4963
4964         ufshcd_set_queue_depth(sdev);
4965
4966         ufshcd_get_lu_power_on_wp_status(hba, sdev);
4967
4968         ufshcd_setup_links(hba, sdev);
4969
4970         return 0;
4971 }
4972
4973 /**
4974  * ufshcd_change_queue_depth - change queue depth
4975  * @sdev: pointer to SCSI device
4976  * @depth: required depth to set
4977  *
4978  * Change queue depth and make sure the max. limits are not crossed.
4979  */
4980 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4981 {
4982         struct ufs_hba *hba = shost_priv(sdev->host);
4983
4984         if (depth > hba->nutrs)
4985                 depth = hba->nutrs;
4986         return scsi_change_queue_depth(sdev, depth);
4987 }
4988
4989 /**
4990  * ufshcd_slave_configure - adjust SCSI device configurations
4991  * @sdev: pointer to SCSI device
4992  */
4993 static int ufshcd_slave_configure(struct scsi_device *sdev)
4994 {
4995         struct ufs_hba *hba = shost_priv(sdev->host);
4996         struct request_queue *q = sdev->request_queue;
4997
4998         blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4999         if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE)
5000                 blk_queue_update_dma_alignment(q, PAGE_SIZE - 1);
5001         /*
5002          * Block runtime-pm until all consumers are added.
5003          * Refer ufshcd_setup_links().
5004          */
5005         if (is_device_wlun(sdev))
5006                 pm_runtime_get_noresume(&sdev->sdev_gendev);
5007         else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5008                 sdev->rpm_autosuspend = 1;
5009
5010         ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
5011
5012         return 0;
5013 }
5014
5015 /**
5016  * ufshcd_slave_destroy - remove SCSI device configurations
5017  * @sdev: pointer to SCSI device
5018  */
5019 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5020 {
5021         struct ufs_hba *hba;
5022
5023         hba = shost_priv(sdev->host);
5024         /* Drop the reference as it won't be needed anymore */
5025         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5026                 unsigned long flags;
5027
5028                 spin_lock_irqsave(hba->host->host_lock, flags);
5029                 hba->sdev_ufs_device = NULL;
5030                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5031         }
5032 }
5033
5034 /**
5035  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5036  * @lrbp: pointer to local reference block of completed command
5037  * @scsi_status: SCSI command status
5038  *
5039  * Returns value base on SCSI command status
5040  */
5041 static inline int
5042 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5043 {
5044         int result = 0;
5045
5046         switch (scsi_status) {
5047         case SAM_STAT_CHECK_CONDITION:
5048                 ufshcd_copy_sense_data(lrbp);
5049                 fallthrough;
5050         case SAM_STAT_GOOD:
5051                 result |= DID_OK << 16 | scsi_status;
5052                 break;
5053         case SAM_STAT_TASK_SET_FULL:
5054         case SAM_STAT_BUSY:
5055         case SAM_STAT_TASK_ABORTED:
5056                 ufshcd_copy_sense_data(lrbp);
5057                 result |= scsi_status;
5058                 break;
5059         default:
5060                 result |= DID_ERROR << 16;
5061                 break;
5062         } /* end of switch */
5063
5064         return result;
5065 }
5066
5067 /**
5068  * ufshcd_transfer_rsp_status - Get overall status of the response
5069  * @hba: per adapter instance
5070  * @lrbp: pointer to local reference block of completed command
5071  *
5072  * Returns result of the command to notify SCSI midlayer
5073  */
5074 static inline int
5075 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
5076 {
5077         int result = 0;
5078         int scsi_status;
5079         int ocs;
5080
5081         /* overall command status of utrd */
5082         ocs = ufshcd_get_tr_ocs(lrbp);
5083
5084         if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5085                 if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) &
5086                                         MASK_RSP_UPIU_RESULT)
5087                         ocs = OCS_SUCCESS;
5088         }
5089
5090         switch (ocs) {
5091         case OCS_SUCCESS:
5092                 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
5093                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5094                 switch (result) {
5095                 case UPIU_TRANSACTION_RESPONSE:
5096                         /*
5097                          * get the response UPIU result to extract
5098                          * the SCSI command status
5099                          */
5100                         result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
5101
5102                         /*
5103                          * get the result based on SCSI status response
5104                          * to notify the SCSI midlayer of the command status
5105                          */
5106                         scsi_status = result & MASK_SCSI_STATUS;
5107                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5108
5109                         /*
5110                          * Currently we are only supporting BKOPs exception
5111                          * events hence we can ignore BKOPs exception event
5112                          * during power management callbacks. BKOPs exception
5113                          * event is not expected to be raised in runtime suspend
5114                          * callback as it allows the urgent bkops.
5115                          * During system suspend, we are anyway forcefully
5116                          * disabling the bkops and if urgent bkops is needed
5117                          * it will be enabled on system resume. Long term
5118                          * solution could be to abort the system suspend if
5119                          * UFS device needs urgent BKOPs.
5120                          */
5121                         if (!hba->pm_op_in_progress &&
5122                             !ufshcd_eh_in_progress(hba) &&
5123                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5124                                 /* Flushed in suspend */
5125                                 schedule_work(&hba->eeh_work);
5126                         break;
5127                 case UPIU_TRANSACTION_REJECT_UPIU:
5128                         /* TODO: handle Reject UPIU Response */
5129                         result = DID_ERROR << 16;
5130                         dev_err(hba->dev,
5131                                 "Reject UPIU not fully implemented\n");
5132                         break;
5133                 default:
5134                         dev_err(hba->dev,
5135                                 "Unexpected request response code = %x\n",
5136                                 result);
5137                         result = DID_ERROR << 16;
5138                         break;
5139                 }
5140                 break;
5141         case OCS_ABORTED:
5142                 result |= DID_ABORT << 16;
5143                 break;
5144         case OCS_INVALID_COMMAND_STATUS:
5145                 result |= DID_REQUEUE << 16;
5146                 break;
5147         case OCS_INVALID_CMD_TABLE_ATTR:
5148         case OCS_INVALID_PRDT_ATTR:
5149         case OCS_MISMATCH_DATA_BUF_SIZE:
5150         case OCS_MISMATCH_RESP_UPIU_SIZE:
5151         case OCS_PEER_COMM_FAILURE:
5152         case OCS_FATAL_ERROR:
5153         case OCS_DEVICE_FATAL_ERROR:
5154         case OCS_INVALID_CRYPTO_CONFIG:
5155         case OCS_GENERAL_CRYPTO_ERROR:
5156         default:
5157                 result |= DID_ERROR << 16;
5158                 dev_err(hba->dev,
5159                                 "OCS error from controller = %x for tag %d\n",
5160                                 ocs, lrbp->task_tag);
5161                 ufshcd_print_evt_hist(hba);
5162                 ufshcd_print_host_state(hba);
5163                 break;
5164         } /* end of switch */
5165
5166         if ((host_byte(result) != DID_OK) &&
5167             (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5168                 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
5169         return result;
5170 }
5171
5172 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5173                                          u32 intr_mask)
5174 {
5175         if (!ufshcd_is_auto_hibern8_supported(hba) ||
5176             !ufshcd_is_auto_hibern8_enabled(hba))
5177                 return false;
5178
5179         if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5180                 return false;
5181
5182         if (hba->active_uic_cmd &&
5183             (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5184             hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5185                 return false;
5186
5187         return true;
5188 }
5189
5190 /**
5191  * ufshcd_uic_cmd_compl - handle completion of uic command
5192  * @hba: per adapter instance
5193  * @intr_status: interrupt status generated by the controller
5194  *
5195  * Returns
5196  *  IRQ_HANDLED - If interrupt is valid
5197  *  IRQ_NONE    - If invalid interrupt
5198  */
5199 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5200 {
5201         irqreturn_t retval = IRQ_NONE;
5202
5203         spin_lock(hba->host->host_lock);
5204         if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5205                 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5206
5207         if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
5208                 hba->active_uic_cmd->argument2 |=
5209                         ufshcd_get_uic_cmd_result(hba);
5210                 hba->active_uic_cmd->argument3 =
5211                         ufshcd_get_dme_attr_val(hba);
5212                 if (!hba->uic_async_done)
5213                         hba->active_uic_cmd->cmd_active = 0;
5214                 complete(&hba->active_uic_cmd->done);
5215                 retval = IRQ_HANDLED;
5216         }
5217
5218         if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) {
5219                 hba->active_uic_cmd->cmd_active = 0;
5220                 complete(hba->uic_async_done);
5221                 retval = IRQ_HANDLED;
5222         }
5223
5224         if (retval == IRQ_HANDLED)
5225                 ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd,
5226                                              UFS_CMD_COMP);
5227         spin_unlock(hba->host->host_lock);
5228         return retval;
5229 }
5230
5231 /**
5232  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5233  * @hba: per adapter instance
5234  * @completed_reqs: requests to complete
5235  */
5236 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5237                                         unsigned long completed_reqs)
5238 {
5239         struct ufshcd_lrb *lrbp;
5240         struct scsi_cmnd *cmd;
5241         int result;
5242         int index;
5243         bool update_scaling = false;
5244
5245         for_each_set_bit(index, &completed_reqs, hba->nutrs) {
5246                 if (!test_and_clear_bit(index, &hba->outstanding_reqs))
5247                         continue;
5248                 lrbp = &hba->lrb[index];
5249                 lrbp->compl_time_stamp = ktime_get();
5250                 cmd = lrbp->cmd;
5251                 if (cmd) {
5252                         if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5253                                 ufshcd_update_monitor(hba, lrbp);
5254                         ufshcd_add_command_trace(hba, index, UFS_CMD_COMP);
5255                         result = ufshcd_transfer_rsp_status(hba, lrbp);
5256                         scsi_dma_unmap(cmd);
5257                         cmd->result = result;
5258                         /* Mark completed command as NULL in LRB */
5259                         lrbp->cmd = NULL;
5260                         /* Do not touch lrbp after scsi done */
5261                         cmd->scsi_done(cmd);
5262                         ufshcd_release(hba);
5263                         update_scaling = true;
5264                 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
5265                         lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
5266                         if (hba->dev_cmd.complete) {
5267                                 ufshcd_add_command_trace(hba, index,
5268                                                          UFS_DEV_COMP);
5269                                 complete(hba->dev_cmd.complete);
5270                                 update_scaling = true;
5271                         }
5272                 }
5273                 if (update_scaling)
5274                         ufshcd_clk_scaling_update_busy(hba);
5275         }
5276 }
5277
5278 /**
5279  * ufshcd_trc_handler - handle transfer requests completion
5280  * @hba: per adapter instance
5281  * @use_utrlcnr: get completed requests from UTRLCNR
5282  *
5283  * Returns
5284  *  IRQ_HANDLED - If interrupt is valid
5285  *  IRQ_NONE    - If invalid interrupt
5286  */
5287 static irqreturn_t ufshcd_trc_handler(struct ufs_hba *hba, bool use_utrlcnr)
5288 {
5289         unsigned long completed_reqs = 0;
5290
5291         /* Resetting interrupt aggregation counters first and reading the
5292          * DOOR_BELL afterward allows us to handle all the completed requests.
5293          * In order to prevent other interrupts starvation the DB is read once
5294          * after reset. The down side of this solution is the possibility of
5295          * false interrupt if device completes another request after resetting
5296          * aggregation and before reading the DB.
5297          */
5298         if (ufshcd_is_intr_aggr_allowed(hba) &&
5299             !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5300                 ufshcd_reset_intr_aggr(hba);
5301
5302         if (use_utrlcnr) {
5303                 u32 utrlcnr;
5304
5305                 utrlcnr = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_LIST_COMPL);
5306                 if (utrlcnr) {
5307                         ufshcd_writel(hba, utrlcnr,
5308                                       REG_UTP_TRANSFER_REQ_LIST_COMPL);
5309                         completed_reqs = utrlcnr;
5310                 }
5311         } else {
5312                 unsigned long flags;
5313                 u32 tr_doorbell;
5314
5315                 spin_lock_irqsave(hba->host->host_lock, flags);
5316                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5317                 completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
5318                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5319         }
5320
5321         if (completed_reqs) {
5322                 __ufshcd_transfer_req_compl(hba, completed_reqs);
5323                 return IRQ_HANDLED;
5324         } else {
5325                 return IRQ_NONE;
5326         }
5327 }
5328
5329 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5330 {
5331         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5332                                        QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5333                                        &ee_ctrl_mask);
5334 }
5335
5336 int ufshcd_write_ee_control(struct ufs_hba *hba)
5337 {
5338         int err;
5339
5340         mutex_lock(&hba->ee_ctrl_mutex);
5341         err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5342         mutex_unlock(&hba->ee_ctrl_mutex);
5343         if (err)
5344                 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5345                         __func__, err);
5346         return err;
5347 }
5348
5349 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, u16 *other_mask,
5350                              u16 set, u16 clr)
5351 {
5352         u16 new_mask, ee_ctrl_mask;
5353         int err = 0;
5354
5355         mutex_lock(&hba->ee_ctrl_mutex);
5356         new_mask = (*mask & ~clr) | set;
5357         ee_ctrl_mask = new_mask | *other_mask;
5358         if (ee_ctrl_mask != hba->ee_ctrl_mask)
5359                 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5360         /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5361         if (!err) {
5362                 hba->ee_ctrl_mask = ee_ctrl_mask;
5363                 *mask = new_mask;
5364         }
5365         mutex_unlock(&hba->ee_ctrl_mutex);
5366         return err;
5367 }
5368
5369 /**
5370  * ufshcd_disable_ee - disable exception event
5371  * @hba: per-adapter instance
5372  * @mask: exception event to disable
5373  *
5374  * Disables exception event in the device so that the EVENT_ALERT
5375  * bit is not set.
5376  *
5377  * Returns zero on success, non-zero error value on failure.
5378  */
5379 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5380 {
5381         return ufshcd_update_ee_drv_mask(hba, 0, mask);
5382 }
5383
5384 /**
5385  * ufshcd_enable_ee - enable exception event
5386  * @hba: per-adapter instance
5387  * @mask: exception event to enable
5388  *
5389  * Enable corresponding exception event in the device to allow
5390  * device to alert host in critical scenarios.
5391  *
5392  * Returns zero on success, non-zero error value on failure.
5393  */
5394 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5395 {
5396         return ufshcd_update_ee_drv_mask(hba, mask, 0);
5397 }
5398
5399 /**
5400  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5401  * @hba: per-adapter instance
5402  *
5403  * Allow device to manage background operations on its own. Enabling
5404  * this might lead to inconsistent latencies during normal data transfers
5405  * as the device is allowed to manage its own way of handling background
5406  * operations.
5407  *
5408  * Returns zero on success, non-zero on failure.
5409  */
5410 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5411 {
5412         int err = 0;
5413
5414         if (hba->auto_bkops_enabled)
5415                 goto out;
5416
5417         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5418                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5419         if (err) {
5420                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5421                                 __func__, err);
5422                 goto out;
5423         }
5424
5425         hba->auto_bkops_enabled = true;
5426         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5427
5428         /* No need of URGENT_BKOPS exception from the device */
5429         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5430         if (err)
5431                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5432                                 __func__, err);
5433 out:
5434         return err;
5435 }
5436
5437 /**
5438  * ufshcd_disable_auto_bkops - block device in doing background operations
5439  * @hba: per-adapter instance
5440  *
5441  * Disabling background operations improves command response latency but
5442  * has drawback of device moving into critical state where the device is
5443  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5444  * host is idle so that BKOPS are managed effectively without any negative
5445  * impacts.
5446  *
5447  * Returns zero on success, non-zero on failure.
5448  */
5449 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5450 {
5451         int err = 0;
5452
5453         if (!hba->auto_bkops_enabled)
5454                 goto out;
5455
5456         /*
5457          * If host assisted BKOPs is to be enabled, make sure
5458          * urgent bkops exception is allowed.
5459          */
5460         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5461         if (err) {
5462                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5463                                 __func__, err);
5464                 goto out;
5465         }
5466
5467         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5468                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5469         if (err) {
5470                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5471                                 __func__, err);
5472                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5473                 goto out;
5474         }
5475
5476         hba->auto_bkops_enabled = false;
5477         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5478         hba->is_urgent_bkops_lvl_checked = false;
5479 out:
5480         return err;
5481 }
5482
5483 /**
5484  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5485  * @hba: per adapter instance
5486  *
5487  * After a device reset the device may toggle the BKOPS_EN flag
5488  * to default value. The s/w tracking variables should be updated
5489  * as well. This function would change the auto-bkops state based on
5490  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5491  */
5492 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5493 {
5494         if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5495                 hba->auto_bkops_enabled = false;
5496                 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5497                 ufshcd_enable_auto_bkops(hba);
5498         } else {
5499                 hba->auto_bkops_enabled = true;
5500                 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5501                 ufshcd_disable_auto_bkops(hba);
5502         }
5503         hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5504         hba->is_urgent_bkops_lvl_checked = false;
5505 }
5506
5507 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5508 {
5509         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5510                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5511 }
5512
5513 /**
5514  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5515  * @hba: per-adapter instance
5516  * @status: bkops_status value
5517  *
5518  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5519  * flag in the device to permit background operations if the device
5520  * bkops_status is greater than or equal to "status" argument passed to
5521  * this function, disable otherwise.
5522  *
5523  * Returns 0 for success, non-zero in case of failure.
5524  *
5525  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5526  * to know whether auto bkops is enabled or disabled after this function
5527  * returns control to it.
5528  */
5529 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5530                              enum bkops_status status)
5531 {
5532         int err;
5533         u32 curr_status = 0;
5534
5535         err = ufshcd_get_bkops_status(hba, &curr_status);
5536         if (err) {
5537                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5538                                 __func__, err);
5539                 goto out;
5540         } else if (curr_status > BKOPS_STATUS_MAX) {
5541                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5542                                 __func__, curr_status);
5543                 err = -EINVAL;
5544                 goto out;
5545         }
5546
5547         if (curr_status >= status)
5548                 err = ufshcd_enable_auto_bkops(hba);
5549         else
5550                 err = ufshcd_disable_auto_bkops(hba);
5551 out:
5552         return err;
5553 }
5554
5555 /**
5556  * ufshcd_urgent_bkops - handle urgent bkops exception event
5557  * @hba: per-adapter instance
5558  *
5559  * Enable fBackgroundOpsEn flag in the device to permit background
5560  * operations.
5561  *
5562  * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5563  * and negative error value for any other failure.
5564  */
5565 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5566 {
5567         return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5568 }
5569
5570 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5571 {
5572         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5573                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5574 }
5575
5576 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5577 {
5578         int err;
5579         u32 curr_status = 0;
5580
5581         if (hba->is_urgent_bkops_lvl_checked)
5582                 goto enable_auto_bkops;
5583
5584         err = ufshcd_get_bkops_status(hba, &curr_status);
5585         if (err) {
5586                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5587                                 __func__, err);
5588                 goto out;
5589         }
5590
5591         /*
5592          * We are seeing that some devices are raising the urgent bkops
5593          * exception events even when BKOPS status doesn't indicate performace
5594          * impacted or critical. Handle these device by determining their urgent
5595          * bkops status at runtime.
5596          */
5597         if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5598                 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5599                                 __func__, curr_status);
5600                 /* update the current status as the urgent bkops level */
5601                 hba->urgent_bkops_lvl = curr_status;
5602                 hba->is_urgent_bkops_lvl_checked = true;
5603         }
5604
5605 enable_auto_bkops:
5606         err = ufshcd_enable_auto_bkops(hba);
5607 out:
5608         if (err < 0)
5609                 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5610                                 __func__, err);
5611 }
5612
5613 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
5614 {
5615         u8 index;
5616         enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
5617                                    UPIU_QUERY_OPCODE_CLEAR_FLAG;
5618
5619         index = ufshcd_wb_get_query_index(hba);
5620         return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
5621 }
5622
5623 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
5624 {
5625         int ret;
5626
5627         if (!ufshcd_is_wb_allowed(hba))
5628                 return 0;
5629
5630         if (!(enable ^ hba->dev_info.wb_enabled))
5631                 return 0;
5632
5633         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
5634         if (ret) {
5635                 dev_err(hba->dev, "%s Write Booster %s failed %d\n",
5636                         __func__, enable ? "enable" : "disable", ret);
5637                 return ret;
5638         }
5639
5640         hba->dev_info.wb_enabled = enable;
5641         dev_info(hba->dev, "%s Write Booster %s\n",
5642                         __func__, enable ? "enabled" : "disabled");
5643
5644         return ret;
5645 }
5646
5647 static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
5648 {
5649         int ret;
5650
5651         ret = __ufshcd_wb_toggle(hba, set,
5652                         QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
5653         if (ret) {
5654                 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed: %d\n",
5655                         __func__, set ? "enable" : "disable", ret);
5656                 return;
5657         }
5658         dev_dbg(hba->dev, "%s WB-Buf Flush during H8 %s\n",
5659                         __func__, set ? "enabled" : "disabled");
5660 }
5661
5662 static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
5663 {
5664         int ret;
5665
5666         if (!ufshcd_is_wb_allowed(hba) ||
5667             hba->dev_info.wb_buf_flush_enabled == enable)
5668                 return;
5669
5670         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
5671         if (ret) {
5672                 dev_err(hba->dev, "%s WB-Buf Flush %s failed %d\n", __func__,
5673                         enable ? "enable" : "disable", ret);
5674                 return;
5675         }
5676
5677         hba->dev_info.wb_buf_flush_enabled = enable;
5678
5679         dev_dbg(hba->dev, "%s WB-Buf Flush %s\n",
5680                         __func__, enable ? "enabled" : "disabled");
5681 }
5682
5683 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
5684                                                 u32 avail_buf)
5685 {
5686         u32 cur_buf;
5687         int ret;
5688         u8 index;
5689
5690         index = ufshcd_wb_get_query_index(hba);
5691         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5692                                               QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
5693                                               index, 0, &cur_buf);
5694         if (ret) {
5695                 dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n",
5696                         __func__, ret);
5697                 return false;
5698         }
5699
5700         if (!cur_buf) {
5701                 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
5702                          cur_buf);
5703                 return false;
5704         }
5705         /* Let it continue to flush when available buffer exceeds threshold */
5706         if (avail_buf < hba->vps->wb_flush_threshold)
5707                 return true;
5708
5709         return false;
5710 }
5711
5712 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
5713 {
5714         int ret;
5715         u32 avail_buf;
5716         u8 index;
5717
5718         if (!ufshcd_is_wb_allowed(hba))
5719                 return false;
5720         /*
5721          * The ufs device needs the vcc to be ON to flush.
5722          * With user-space reduction enabled, it's enough to enable flush
5723          * by checking only the available buffer. The threshold
5724          * defined here is > 90% full.
5725          * With user-space preserved enabled, the current-buffer
5726          * should be checked too because the wb buffer size can reduce
5727          * when disk tends to be full. This info is provided by current
5728          * buffer (dCurrentWriteBoosterBufferSize). There's no point in
5729          * keeping vcc on when current buffer is empty.
5730          */
5731         index = ufshcd_wb_get_query_index(hba);
5732         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5733                                       QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
5734                                       index, 0, &avail_buf);
5735         if (ret) {
5736                 dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n",
5737                          __func__, ret);
5738                 return false;
5739         }
5740
5741         if (!hba->dev_info.b_presrv_uspc_en) {
5742                 if (avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10))
5743                         return true;
5744                 return false;
5745         }
5746
5747         return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
5748 }
5749
5750 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
5751 {
5752         struct ufs_hba *hba = container_of(to_delayed_work(work),
5753                                            struct ufs_hba,
5754                                            rpm_dev_flush_recheck_work);
5755         /*
5756          * To prevent unnecessary VCC power drain after device finishes
5757          * WriteBooster buffer flush or Auto BKOPs, force runtime resume
5758          * after a certain delay to recheck the threshold by next runtime
5759          * suspend.
5760          */
5761         ufshcd_rpm_get_sync(hba);
5762         ufshcd_rpm_put_sync(hba);
5763 }
5764
5765 /**
5766  * ufshcd_exception_event_handler - handle exceptions raised by device
5767  * @work: pointer to work data
5768  *
5769  * Read bExceptionEventStatus attribute from the device and handle the
5770  * exception event accordingly.
5771  */
5772 static void ufshcd_exception_event_handler(struct work_struct *work)
5773 {
5774         struct ufs_hba *hba;
5775         int err;
5776         u32 status = 0;
5777         hba = container_of(work, struct ufs_hba, eeh_work);
5778
5779         ufshcd_scsi_block_requests(hba);
5780         err = ufshcd_get_ee_status(hba, &status);
5781         if (err) {
5782                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5783                                 __func__, err);
5784                 goto out;
5785         }
5786
5787         trace_ufshcd_exception_event(dev_name(hba->dev), status);
5788
5789         if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
5790                 ufshcd_bkops_exception_event_handler(hba);
5791
5792         ufs_debugfs_exception_event(hba, status);
5793 out:
5794         ufshcd_scsi_unblock_requests(hba);
5795         return;
5796 }
5797
5798 /* Complete requests that have door-bell cleared */
5799 static void ufshcd_complete_requests(struct ufs_hba *hba)
5800 {
5801         ufshcd_trc_handler(hba, false);
5802         ufshcd_tmc_handler(hba);
5803 }
5804
5805 /**
5806  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5807  *                              to recover from the DL NAC errors or not.
5808  * @hba: per-adapter instance
5809  *
5810  * Returns true if error handling is required, false otherwise
5811  */
5812 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5813 {
5814         unsigned long flags;
5815         bool err_handling = true;
5816
5817         spin_lock_irqsave(hba->host->host_lock, flags);
5818         /*
5819          * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5820          * device fatal error and/or DL NAC & REPLAY timeout errors.
5821          */
5822         if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5823                 goto out;
5824
5825         if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5826             ((hba->saved_err & UIC_ERROR) &&
5827              (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5828                 goto out;
5829
5830         if ((hba->saved_err & UIC_ERROR) &&
5831             (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5832                 int err;
5833                 /*
5834                  * wait for 50ms to see if we can get any other errors or not.
5835                  */
5836                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5837                 msleep(50);
5838                 spin_lock_irqsave(hba->host->host_lock, flags);
5839
5840                 /*
5841                  * now check if we have got any other severe errors other than
5842                  * DL NAC error?
5843                  */
5844                 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5845                     ((hba->saved_err & UIC_ERROR) &&
5846                     (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5847                         goto out;
5848
5849                 /*
5850                  * As DL NAC is the only error received so far, send out NOP
5851                  * command to confirm if link is still active or not.
5852                  *   - If we don't get any response then do error recovery.
5853                  *   - If we get response then clear the DL NAC error bit.
5854                  */
5855
5856                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5857                 err = ufshcd_verify_dev_init(hba);
5858                 spin_lock_irqsave(hba->host->host_lock, flags);
5859
5860                 if (err)
5861                         goto out;
5862
5863                 /* Link seems to be alive hence ignore the DL NAC errors */
5864                 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5865                         hba->saved_err &= ~UIC_ERROR;
5866                 /* clear NAC error */
5867                 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5868                 if (!hba->saved_uic_err)
5869                         err_handling = false;
5870         }
5871 out:
5872         spin_unlock_irqrestore(hba->host->host_lock, flags);
5873         return err_handling;
5874 }
5875
5876 /* host lock must be held before calling this func */
5877 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
5878 {
5879         return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
5880                (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
5881 }
5882
5883 /* host lock must be held before calling this func */
5884 static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
5885 {
5886         /* handle fatal errors only when link is not in error state */
5887         if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
5888                 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
5889                     ufshcd_is_saved_err_fatal(hba))
5890                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
5891                 else
5892                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
5893                 queue_work(hba->eh_wq, &hba->eh_work);
5894         }
5895 }
5896
5897 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
5898 {
5899         down_write(&hba->clk_scaling_lock);
5900         hba->clk_scaling.is_allowed = allow;
5901         up_write(&hba->clk_scaling_lock);
5902 }
5903
5904 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
5905 {
5906         if (suspend) {
5907                 if (hba->clk_scaling.is_enabled)
5908                         ufshcd_suspend_clkscaling(hba);
5909                 ufshcd_clk_scaling_allow(hba, false);
5910         } else {
5911                 ufshcd_clk_scaling_allow(hba, true);
5912                 if (hba->clk_scaling.is_enabled)
5913                         ufshcd_resume_clkscaling(hba);
5914         }
5915 }
5916
5917 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
5918 {
5919         ufshcd_rpm_get_sync(hba);
5920         if (pm_runtime_status_suspended(&hba->sdev_ufs_device->sdev_gendev) ||
5921             hba->is_sys_suspended) {
5922                 enum ufs_pm_op pm_op;
5923
5924                 /*
5925                  * Don't assume anything of resume, if
5926                  * resume fails, irq and clocks can be OFF, and powers
5927                  * can be OFF or in LPM.
5928                  */
5929                 ufshcd_setup_hba_vreg(hba, true);
5930                 ufshcd_enable_irq(hba);
5931                 ufshcd_setup_vreg(hba, true);
5932                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
5933                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
5934                 ufshcd_hold(hba, false);
5935                 if (!ufshcd_is_clkgating_allowed(hba))
5936                         ufshcd_setup_clocks(hba, true);
5937                 ufshcd_release(hba);
5938                 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
5939                 ufshcd_vops_resume(hba, pm_op);
5940         } else {
5941                 ufshcd_hold(hba, false);
5942                 if (ufshcd_is_clkscaling_supported(hba) &&
5943                     hba->clk_scaling.is_enabled)
5944                         ufshcd_suspend_clkscaling(hba);
5945                 ufshcd_clk_scaling_allow(hba, false);
5946         }
5947         ufshcd_scsi_block_requests(hba);
5948         /* Drain ufshcd_queuecommand() */
5949         down_write(&hba->clk_scaling_lock);
5950         up_write(&hba->clk_scaling_lock);
5951         cancel_work_sync(&hba->eeh_work);
5952 }
5953
5954 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
5955 {
5956         ufshcd_scsi_unblock_requests(hba);
5957         ufshcd_release(hba);
5958         if (ufshcd_is_clkscaling_supported(hba))
5959                 ufshcd_clk_scaling_suspend(hba, false);
5960         ufshcd_clear_ua_wluns(hba);
5961         ufshcd_rpm_put(hba);
5962 }
5963
5964 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
5965 {
5966         return (!hba->is_powered || hba->shutting_down ||
5967                 !hba->sdev_ufs_device ||
5968                 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
5969                 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
5970                    ufshcd_is_link_broken(hba))));
5971 }
5972
5973 #ifdef CONFIG_PM
5974 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
5975 {
5976         struct Scsi_Host *shost = hba->host;
5977         struct scsi_device *sdev;
5978         struct request_queue *q;
5979         int ret;
5980
5981         hba->is_sys_suspended = false;
5982         /*
5983          * Set RPM status of wlun device to RPM_ACTIVE,
5984          * this also clears its runtime error.
5985          */
5986         ret = pm_runtime_set_active(&hba->sdev_ufs_device->sdev_gendev);
5987
5988         /* hba device might have a runtime error otherwise */
5989         if (ret)
5990                 ret = pm_runtime_set_active(hba->dev);
5991         /*
5992          * If wlun device had runtime error, we also need to resume those
5993          * consumer scsi devices in case any of them has failed to be
5994          * resumed due to supplier runtime resume failure. This is to unblock
5995          * blk_queue_enter in case there are bios waiting inside it.
5996          */
5997         if (!ret) {
5998                 shost_for_each_device(sdev, shost) {
5999                         q = sdev->request_queue;
6000                         if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6001                                        q->rpm_status == RPM_SUSPENDING))
6002                                 pm_request_resume(q->dev);
6003                 }
6004         }
6005 }
6006 #else
6007 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6008 {
6009 }
6010 #endif
6011
6012 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6013 {
6014         struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6015         u32 mode;
6016
6017         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6018
6019         if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6020                 return true;
6021
6022         if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6023                 return true;
6024
6025         return false;
6026 }
6027
6028 /**
6029  * ufshcd_err_handler - handle UFS errors that require s/w attention
6030  * @work: pointer to work structure
6031  */
6032 static void ufshcd_err_handler(struct work_struct *work)
6033 {
6034         struct ufs_hba *hba;
6035         unsigned long flags;
6036         bool err_xfer = false;
6037         bool err_tm = false;
6038         int err = 0, pmc_err;
6039         int tag;
6040         bool needs_reset = false, needs_restore = false;
6041
6042         hba = container_of(work, struct ufs_hba, eh_work);
6043
6044         down(&hba->host_sem);
6045         spin_lock_irqsave(hba->host->host_lock, flags);
6046         if (ufshcd_err_handling_should_stop(hba)) {
6047                 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6048                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6049                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6050                 up(&hba->host_sem);
6051                 return;
6052         }
6053         ufshcd_set_eh_in_progress(hba);
6054         spin_unlock_irqrestore(hba->host->host_lock, flags);
6055         ufshcd_err_handling_prepare(hba);
6056         /* Complete requests that have door-bell cleared by h/w */
6057         ufshcd_complete_requests(hba);
6058         spin_lock_irqsave(hba->host->host_lock, flags);
6059         if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6060                 hba->ufshcd_state = UFSHCD_STATE_RESET;
6061         /*
6062          * A full reset and restore might have happened after preparation
6063          * is finished, double check whether we should stop.
6064          */
6065         if (ufshcd_err_handling_should_stop(hba))
6066                 goto skip_err_handling;
6067
6068         if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6069                 bool ret;
6070
6071                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6072                 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6073                 ret = ufshcd_quirk_dl_nac_errors(hba);
6074                 spin_lock_irqsave(hba->host->host_lock, flags);
6075                 if (!ret && ufshcd_err_handling_should_stop(hba))
6076                         goto skip_err_handling;
6077         }
6078
6079         if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6080             (hba->saved_uic_err &&
6081              (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6082                 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6083
6084                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6085                 ufshcd_print_host_state(hba);
6086                 ufshcd_print_pwr_info(hba);
6087                 ufshcd_print_evt_hist(hba);
6088                 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6089                 ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
6090                 spin_lock_irqsave(hba->host->host_lock, flags);
6091         }
6092
6093         /*
6094          * if host reset is required then skip clearing the pending
6095          * transfers forcefully because they will get cleared during
6096          * host reset and restore
6097          */
6098         if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6099             ufshcd_is_saved_err_fatal(hba) ||
6100             ((hba->saved_err & UIC_ERROR) &&
6101              (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6102                                     UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6103                 needs_reset = true;
6104                 goto do_reset;
6105         }
6106
6107         /*
6108          * If LINERESET was caught, UFS might have been put to PWM mode,
6109          * check if power mode restore is needed.
6110          */
6111         if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6112                 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6113                 if (!hba->saved_uic_err)
6114                         hba->saved_err &= ~UIC_ERROR;
6115                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6116                 if (ufshcd_is_pwr_mode_restore_needed(hba))
6117                         needs_restore = true;
6118                 spin_lock_irqsave(hba->host->host_lock, flags);
6119                 if (!hba->saved_err && !needs_restore)
6120                         goto skip_err_handling;
6121         }
6122
6123         hba->silence_err_logs = true;
6124         /* release lock as clear command might sleep */
6125         spin_unlock_irqrestore(hba->host->host_lock, flags);
6126         /* Clear pending transfer requests */
6127         for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
6128                 if (ufshcd_try_to_abort_task(hba, tag)) {
6129                         err_xfer = true;
6130                         goto lock_skip_pending_xfer_clear;
6131                 }
6132         }
6133
6134         /* Clear pending task management requests */
6135         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6136                 if (ufshcd_clear_tm_cmd(hba, tag)) {
6137                         err_tm = true;
6138                         goto lock_skip_pending_xfer_clear;
6139                 }
6140         }
6141
6142 lock_skip_pending_xfer_clear:
6143         /* Complete the requests that are cleared by s/w */
6144         ufshcd_complete_requests(hba);
6145
6146         spin_lock_irqsave(hba->host->host_lock, flags);
6147         hba->silence_err_logs = false;
6148         if (err_xfer || err_tm) {
6149                 needs_reset = true;
6150                 goto do_reset;
6151         }
6152
6153         /*
6154          * After all reqs and tasks are cleared from doorbell,
6155          * now it is safe to retore power mode.
6156          */
6157         if (needs_restore) {
6158                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6159                 /*
6160                  * Hold the scaling lock just in case dev cmds
6161                  * are sent via bsg and/or sysfs.
6162                  */
6163                 down_write(&hba->clk_scaling_lock);
6164                 hba->force_pmc = true;
6165                 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6166                 if (pmc_err) {
6167                         needs_reset = true;
6168                         dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6169                                         __func__, pmc_err);
6170                 }
6171                 hba->force_pmc = false;
6172                 ufshcd_print_pwr_info(hba);
6173                 up_write(&hba->clk_scaling_lock);
6174                 spin_lock_irqsave(hba->host->host_lock, flags);
6175         }
6176
6177 do_reset:
6178         /* Fatal errors need reset */
6179         if (needs_reset) {
6180                 hba->force_reset = false;
6181                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6182                 err = ufshcd_reset_and_restore(hba);
6183                 if (err)
6184                         dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6185                                         __func__, err);
6186                 else
6187                         ufshcd_recover_pm_error(hba);
6188                 spin_lock_irqsave(hba->host->host_lock, flags);
6189         }
6190
6191 skip_err_handling:
6192         if (!needs_reset) {
6193                 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6194                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6195                 if (hba->saved_err || hba->saved_uic_err)
6196                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6197                             __func__, hba->saved_err, hba->saved_uic_err);
6198         }
6199         ufshcd_clear_eh_in_progress(hba);
6200         spin_unlock_irqrestore(hba->host->host_lock, flags);
6201         ufshcd_err_handling_unprepare(hba);
6202         up(&hba->host_sem);
6203 }
6204
6205 /**
6206  * ufshcd_update_uic_error - check and set fatal UIC error flags.
6207  * @hba: per-adapter instance
6208  *
6209  * Returns
6210  *  IRQ_HANDLED - If interrupt is valid
6211  *  IRQ_NONE    - If invalid interrupt
6212  */
6213 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6214 {
6215         u32 reg;
6216         irqreturn_t retval = IRQ_NONE;
6217
6218         /* PHY layer error */
6219         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6220         if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6221             (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6222                 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6223                 /*
6224                  * To know whether this error is fatal or not, DB timeout
6225                  * must be checked but this error is handled separately.
6226                  */
6227                 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6228                         dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6229                                         __func__);
6230
6231                 /* Got a LINERESET indication. */
6232                 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6233                         struct uic_command *cmd = NULL;
6234
6235                         hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6236                         if (hba->uic_async_done && hba->active_uic_cmd)
6237                                 cmd = hba->active_uic_cmd;
6238                         /*
6239                          * Ignore the LINERESET during power mode change
6240                          * operation via DME_SET command.
6241                          */
6242                         if (cmd && (cmd->command == UIC_CMD_DME_SET))
6243                                 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6244                 }
6245                 retval |= IRQ_HANDLED;
6246         }
6247
6248         /* PA_INIT_ERROR is fatal and needs UIC reset */
6249         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6250         if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6251             (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6252                 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6253
6254                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6255                         hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6256                 else if (hba->dev_quirks &
6257                                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6258                         if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6259                                 hba->uic_error |=
6260                                         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6261                         else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6262                                 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6263                 }
6264                 retval |= IRQ_HANDLED;
6265         }
6266
6267         /* UIC NL/TL/DME errors needs software retry */
6268         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6269         if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6270             (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6271                 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6272                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6273                 retval |= IRQ_HANDLED;
6274         }
6275
6276         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6277         if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6278             (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6279                 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6280                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6281                 retval |= IRQ_HANDLED;
6282         }
6283
6284         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6285         if ((reg & UIC_DME_ERROR) &&
6286             (reg & UIC_DME_ERROR_CODE_MASK)) {
6287                 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6288                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6289                 retval |= IRQ_HANDLED;
6290         }
6291
6292         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6293                         __func__, hba->uic_error);
6294         return retval;
6295 }
6296
6297 /**
6298  * ufshcd_check_errors - Check for errors that need s/w attention
6299  * @hba: per-adapter instance
6300  * @intr_status: interrupt status generated by the controller
6301  *
6302  * Returns
6303  *  IRQ_HANDLED - If interrupt is valid
6304  *  IRQ_NONE    - If invalid interrupt
6305  */
6306 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6307 {
6308         bool queue_eh_work = false;
6309         irqreturn_t retval = IRQ_NONE;
6310
6311         spin_lock(hba->host->host_lock);
6312         hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6313
6314         if (hba->errors & INT_FATAL_ERRORS) {
6315                 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6316                                        hba->errors);
6317                 queue_eh_work = true;
6318         }
6319
6320         if (hba->errors & UIC_ERROR) {
6321                 hba->uic_error = 0;
6322                 retval = ufshcd_update_uic_error(hba);
6323                 if (hba->uic_error)
6324                         queue_eh_work = true;
6325         }
6326
6327         if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6328                 dev_err(hba->dev,
6329                         "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6330                         __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6331                         "Enter" : "Exit",
6332                         hba->errors, ufshcd_get_upmcrs(hba));
6333                 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6334                                        hba->errors);
6335                 ufshcd_set_link_broken(hba);
6336                 queue_eh_work = true;
6337         }
6338
6339         if (queue_eh_work) {
6340                 /*
6341                  * update the transfer error masks to sticky bits, let's do this
6342                  * irrespective of current ufshcd_state.
6343                  */
6344                 hba->saved_err |= hba->errors;
6345                 hba->saved_uic_err |= hba->uic_error;
6346
6347                 /* dump controller state before resetting */
6348                 if ((hba->saved_err &
6349                      (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6350                     (hba->saved_uic_err &&
6351                      (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6352                         dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6353                                         __func__, hba->saved_err,
6354                                         hba->saved_uic_err);
6355                         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6356                                          "host_regs: ");
6357                         ufshcd_print_pwr_info(hba);
6358                 }
6359                 ufshcd_schedule_eh_work(hba);
6360                 retval |= IRQ_HANDLED;
6361         }
6362         /*
6363          * if (!queue_eh_work) -
6364          * Other errors are either non-fatal where host recovers
6365          * itself without s/w intervention or errors that will be
6366          * handled by the SCSI core layer.
6367          */
6368         hba->errors = 0;
6369         hba->uic_error = 0;
6370         spin_unlock(hba->host->host_lock);
6371         return retval;
6372 }
6373
6374 struct ctm_info {
6375         struct ufs_hba  *hba;
6376         unsigned long   pending;
6377         unsigned int    ncpl;
6378 };
6379
6380 static bool ufshcd_compl_tm(struct request *req, void *priv, bool reserved)
6381 {
6382         struct ctm_info *const ci = priv;
6383         struct completion *c;
6384
6385         WARN_ON_ONCE(reserved);
6386         if (test_bit(req->tag, &ci->pending))
6387                 return true;
6388         ci->ncpl++;
6389         c = req->end_io_data;
6390         if (c)
6391                 complete(c);
6392         return true;
6393 }
6394
6395 /**
6396  * ufshcd_tmc_handler - handle task management function completion
6397  * @hba: per adapter instance
6398  *
6399  * Returns
6400  *  IRQ_HANDLED - If interrupt is valid
6401  *  IRQ_NONE    - If invalid interrupt
6402  */
6403 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6404 {
6405         unsigned long flags;
6406         struct request_queue *q = hba->tmf_queue;
6407         struct ctm_info ci = {
6408                 .hba     = hba,
6409         };
6410
6411         spin_lock_irqsave(hba->host->host_lock, flags);
6412         ci.pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6413         blk_mq_tagset_busy_iter(q->tag_set, ufshcd_compl_tm, &ci);
6414         spin_unlock_irqrestore(hba->host->host_lock, flags);
6415
6416         return ci.ncpl ? IRQ_HANDLED : IRQ_NONE;
6417 }
6418
6419 /**
6420  * ufshcd_sl_intr - Interrupt service routine
6421  * @hba: per adapter instance
6422  * @intr_status: contains interrupts generated by the controller
6423  *
6424  * Returns
6425  *  IRQ_HANDLED - If interrupt is valid
6426  *  IRQ_NONE    - If invalid interrupt
6427  */
6428 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6429 {
6430         irqreturn_t retval = IRQ_NONE;
6431
6432         if (intr_status & UFSHCD_UIC_MASK)
6433                 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6434
6435         if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6436                 retval |= ufshcd_check_errors(hba, intr_status);
6437
6438         if (intr_status & UTP_TASK_REQ_COMPL)
6439                 retval |= ufshcd_tmc_handler(hba);
6440
6441         if (intr_status & UTP_TRANSFER_REQ_COMPL)
6442                 retval |= ufshcd_trc_handler(hba, ufshcd_has_utrlcnr(hba));
6443
6444         return retval;
6445 }
6446
6447 /**
6448  * ufshcd_intr - Main interrupt service routine
6449  * @irq: irq number
6450  * @__hba: pointer to adapter instance
6451  *
6452  * Returns
6453  *  IRQ_HANDLED - If interrupt is valid
6454  *  IRQ_NONE    - If invalid interrupt
6455  */
6456 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6457 {
6458         u32 intr_status, enabled_intr_status = 0;
6459         irqreturn_t retval = IRQ_NONE;
6460         struct ufs_hba *hba = __hba;
6461         int retries = hba->nutrs;
6462
6463         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6464         hba->ufs_stats.last_intr_status = intr_status;
6465         hba->ufs_stats.last_intr_ts = ktime_get();
6466
6467         /*
6468          * There could be max of hba->nutrs reqs in flight and in worst case
6469          * if the reqs get finished 1 by 1 after the interrupt status is
6470          * read, make sure we handle them by checking the interrupt status
6471          * again in a loop until we process all of the reqs before returning.
6472          */
6473         while (intr_status && retries--) {
6474                 enabled_intr_status =
6475                         intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6476                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
6477                 if (enabled_intr_status)
6478                         retval |= ufshcd_sl_intr(hba, enabled_intr_status);
6479
6480                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6481         }
6482
6483         if (enabled_intr_status && retval == IRQ_NONE &&
6484             (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
6485              hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
6486                 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
6487                                         __func__,
6488                                         intr_status,
6489                                         hba->ufs_stats.last_intr_status,
6490                                         enabled_intr_status);
6491                 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
6492         }
6493
6494         return retval;
6495 }
6496
6497 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
6498 {
6499         int err = 0;
6500         u32 mask = 1 << tag;
6501         unsigned long flags;
6502
6503         if (!test_bit(tag, &hba->outstanding_tasks))
6504                 goto out;
6505
6506         spin_lock_irqsave(hba->host->host_lock, flags);
6507         ufshcd_utmrl_clear(hba, tag);
6508         spin_unlock_irqrestore(hba->host->host_lock, flags);
6509
6510         /* poll for max. 1 sec to clear door bell register by h/w */
6511         err = ufshcd_wait_for_register(hba,
6512                         REG_UTP_TASK_REQ_DOOR_BELL,
6513                         mask, 0, 1000, 1000);
6514 out:
6515         return err;
6516 }
6517
6518 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
6519                 struct utp_task_req_desc *treq, u8 tm_function)
6520 {
6521         struct request_queue *q = hba->tmf_queue;
6522         struct Scsi_Host *host = hba->host;
6523         DECLARE_COMPLETION_ONSTACK(wait);
6524         struct request *req;
6525         unsigned long flags;
6526         int task_tag, err;
6527
6528         /*
6529          * blk_get_request() is used here only to get a free tag.
6530          */
6531         req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
6532         if (IS_ERR(req))
6533                 return PTR_ERR(req);
6534
6535         req->end_io_data = &wait;
6536         ufshcd_hold(hba, false);
6537
6538         spin_lock_irqsave(host->host_lock, flags);
6539         blk_mq_start_request(req);
6540
6541         task_tag = req->tag;
6542         treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag);
6543
6544         memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
6545         ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
6546
6547         /* send command to the controller */
6548         __set_bit(task_tag, &hba->outstanding_tasks);
6549
6550         /* Make sure descriptors are ready before ringing the task doorbell */
6551         wmb();
6552
6553         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
6554         /* Make sure that doorbell is committed immediately */
6555         wmb();
6556
6557         spin_unlock_irqrestore(host->host_lock, flags);
6558
6559         ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
6560
6561         /* wait until the task management command is completed */
6562         err = wait_for_completion_io_timeout(&wait,
6563                         msecs_to_jiffies(TM_CMD_TIMEOUT));
6564         if (!err) {
6565                 /*
6566                  * Make sure that ufshcd_compl_tm() does not trigger a
6567                  * use-after-free.
6568                  */
6569                 req->end_io_data = NULL;
6570                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
6571                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
6572                                 __func__, tm_function);
6573                 if (ufshcd_clear_tm_cmd(hba, task_tag))
6574                         dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
6575                                         __func__, task_tag);
6576                 err = -ETIMEDOUT;
6577         } else {
6578                 err = 0;
6579                 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
6580
6581                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
6582         }
6583
6584         spin_lock_irqsave(hba->host->host_lock, flags);
6585         __clear_bit(task_tag, &hba->outstanding_tasks);
6586         spin_unlock_irqrestore(hba->host->host_lock, flags);
6587
6588         ufshcd_release(hba);
6589         blk_put_request(req);
6590
6591         return err;
6592 }
6593
6594 /**
6595  * ufshcd_issue_tm_cmd - issues task management commands to controller
6596  * @hba: per adapter instance
6597  * @lun_id: LUN ID to which TM command is sent
6598  * @task_id: task ID to which the TM command is applicable
6599  * @tm_function: task management function opcode
6600  * @tm_response: task management service response return value
6601  *
6602  * Returns non-zero value on error, zero on success.
6603  */
6604 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
6605                 u8 tm_function, u8 *tm_response)
6606 {
6607         struct utp_task_req_desc treq = { { 0 }, };
6608         int ocs_value, err;
6609
6610         /* Configure task request descriptor */
6611         treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6612         treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6613
6614         /* Configure task request UPIU */
6615         treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) |
6616                                   cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24);
6617         treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16);
6618
6619         /*
6620          * The host shall provide the same value for LUN field in the basic
6621          * header and for Input Parameter.
6622          */
6623         treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
6624         treq.upiu_req.input_param2 = cpu_to_be32(task_id);
6625
6626         err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
6627         if (err == -ETIMEDOUT)
6628                 return err;
6629
6630         ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6631         if (ocs_value != OCS_SUCCESS)
6632                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
6633                                 __func__, ocs_value);
6634         else if (tm_response)
6635                 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
6636                                 MASK_TM_SERVICE_RESP;
6637         return err;
6638 }
6639
6640 /**
6641  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
6642  * @hba:        per-adapter instance
6643  * @req_upiu:   upiu request
6644  * @rsp_upiu:   upiu reply
6645  * @desc_buff:  pointer to descriptor buffer, NULL if NA
6646  * @buff_len:   descriptor size, 0 if NA
6647  * @cmd_type:   specifies the type (NOP, Query...)
6648  * @desc_op:    descriptor operation
6649  *
6650  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
6651  * Therefore, it "rides" the device management infrastructure: uses its tag and
6652  * tasks work queues.
6653  *
6654  * Since there is only one available tag for device management commands,
6655  * the caller is expected to hold the hba->dev_cmd.lock mutex.
6656  */
6657 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
6658                                         struct utp_upiu_req *req_upiu,
6659                                         struct utp_upiu_req *rsp_upiu,
6660                                         u8 *desc_buff, int *buff_len,
6661                                         enum dev_cmd_type cmd_type,
6662                                         enum query_opcode desc_op)
6663 {
6664         struct request_queue *q = hba->cmd_queue;
6665         struct request *req;
6666         struct ufshcd_lrb *lrbp;
6667         int err = 0;
6668         int tag;
6669         struct completion wait;
6670         u8 upiu_flags;
6671
6672         down_read(&hba->clk_scaling_lock);
6673
6674         req = blk_get_request(q, REQ_OP_DRV_OUT, 0);
6675         if (IS_ERR(req)) {
6676                 err = PTR_ERR(req);
6677                 goto out_unlock;
6678         }
6679         tag = req->tag;
6680         WARN_ON_ONCE(!ufshcd_valid_tag(hba, tag));
6681
6682         if (unlikely(test_bit(tag, &hba->outstanding_reqs))) {
6683                 err = -EBUSY;
6684                 goto out;
6685         }
6686
6687         init_completion(&wait);
6688         lrbp = &hba->lrb[tag];
6689         WARN_ON(lrbp->cmd);
6690         lrbp->cmd = NULL;
6691         lrbp->sense_bufflen = 0;
6692         lrbp->sense_buffer = NULL;
6693         lrbp->task_tag = tag;
6694         lrbp->lun = 0;
6695         lrbp->intr_cmd = true;
6696         ufshcd_prepare_lrbp_crypto(NULL, lrbp);
6697         hba->dev_cmd.type = cmd_type;
6698
6699         if (hba->ufs_version <= ufshci_version(1, 1))
6700                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
6701         else
6702                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
6703
6704         /* update the task tag in the request upiu */
6705         req_upiu->header.dword_0 |= cpu_to_be32(tag);
6706
6707         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
6708
6709         /* just copy the upiu request as it is */
6710         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
6711         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
6712                 /* The Data Segment Area is optional depending upon the query
6713                  * function value. for WRITE DESCRIPTOR, the data segment
6714                  * follows right after the tsf.
6715                  */
6716                 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
6717                 *buff_len = 0;
6718         }
6719
6720         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
6721
6722         hba->dev_cmd.complete = &wait;
6723
6724         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
6725         /* Make sure descriptors are ready before ringing the doorbell */
6726         wmb();
6727
6728         ufshcd_send_command(hba, tag);
6729         /*
6730          * ignore the returning value here - ufshcd_check_query_response is
6731          * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
6732          * read the response directly ignoring all errors.
6733          */
6734         ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT);
6735
6736         /* just copy the upiu response as it is */
6737         memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
6738         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
6739                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
6740                 u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
6741                                MASK_QUERY_DATA_SEG_LEN;
6742
6743                 if (*buff_len >= resp_len) {
6744                         memcpy(desc_buff, descp, resp_len);
6745                         *buff_len = resp_len;
6746                 } else {
6747                         dev_warn(hba->dev,
6748                                  "%s: rsp size %d is bigger than buffer size %d",
6749                                  __func__, resp_len, *buff_len);
6750                         *buff_len = 0;
6751                         err = -EINVAL;
6752                 }
6753         }
6754         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
6755                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
6756
6757 out:
6758         blk_put_request(req);
6759 out_unlock:
6760         up_read(&hba->clk_scaling_lock);
6761         return err;
6762 }
6763
6764 /**
6765  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
6766  * @hba:        per-adapter instance
6767  * @req_upiu:   upiu request
6768  * @rsp_upiu:   upiu reply - only 8 DW as we do not support scsi commands
6769  * @msgcode:    message code, one of UPIU Transaction Codes Initiator to Target
6770  * @desc_buff:  pointer to descriptor buffer, NULL if NA
6771  * @buff_len:   descriptor size, 0 if NA
6772  * @desc_op:    descriptor operation
6773  *
6774  * Supports UTP Transfer requests (nop and query), and UTP Task
6775  * Management requests.
6776  * It is up to the caller to fill the upiu conent properly, as it will
6777  * be copied without any further input validations.
6778  */
6779 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
6780                              struct utp_upiu_req *req_upiu,
6781                              struct utp_upiu_req *rsp_upiu,
6782                              int msgcode,
6783                              u8 *desc_buff, int *buff_len,
6784                              enum query_opcode desc_op)
6785 {
6786         int err;
6787         enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
6788         struct utp_task_req_desc treq = { { 0 }, };
6789         int ocs_value;
6790         u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC;
6791
6792         switch (msgcode) {
6793         case UPIU_TRANSACTION_NOP_OUT:
6794                 cmd_type = DEV_CMD_TYPE_NOP;
6795                 fallthrough;
6796         case UPIU_TRANSACTION_QUERY_REQ:
6797                 ufshcd_hold(hba, false);
6798                 mutex_lock(&hba->dev_cmd.lock);
6799                 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
6800                                                    desc_buff, buff_len,
6801                                                    cmd_type, desc_op);
6802                 mutex_unlock(&hba->dev_cmd.lock);
6803                 ufshcd_release(hba);
6804
6805                 break;
6806         case UPIU_TRANSACTION_TASK_REQ:
6807                 treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
6808                 treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
6809
6810                 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
6811
6812                 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
6813                 if (err == -ETIMEDOUT)
6814                         break;
6815
6816                 ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS;
6817                 if (ocs_value != OCS_SUCCESS) {
6818                         dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
6819                                 ocs_value);
6820                         break;
6821                 }
6822
6823                 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
6824
6825                 break;
6826         default:
6827                 err = -EINVAL;
6828
6829                 break;
6830         }
6831
6832         return err;
6833 }
6834
6835 /**
6836  * ufshcd_eh_device_reset_handler - device reset handler registered to
6837  *                                    scsi layer.
6838  * @cmd: SCSI command pointer
6839  *
6840  * Returns SUCCESS/FAILED
6841  */
6842 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
6843 {
6844         struct Scsi_Host *host;
6845         struct ufs_hba *hba;
6846         u32 pos;
6847         int err;
6848         u8 resp = 0xF, lun;
6849
6850         host = cmd->device->host;
6851         hba = shost_priv(host);
6852
6853         lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
6854         err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
6855         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6856                 if (!err)
6857                         err = resp;
6858                 goto out;
6859         }
6860
6861         /* clear the commands that were pending for corresponding LUN */
6862         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
6863                 if (hba->lrb[pos].lun == lun) {
6864                         err = ufshcd_clear_cmd(hba, pos);
6865                         if (err)
6866                                 break;
6867                         __ufshcd_transfer_req_compl(hba, pos);
6868                 }
6869         }
6870
6871 out:
6872         hba->req_abort_count = 0;
6873         ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
6874         if (!err) {
6875                 err = SUCCESS;
6876         } else {
6877                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
6878                 err = FAILED;
6879         }
6880         return err;
6881 }
6882
6883 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
6884 {
6885         struct ufshcd_lrb *lrbp;
6886         int tag;
6887
6888         for_each_set_bit(tag, &bitmap, hba->nutrs) {
6889                 lrbp = &hba->lrb[tag];
6890                 lrbp->req_abort_skip = true;
6891         }
6892 }
6893
6894 /**
6895  * ufshcd_try_to_abort_task - abort a specific task
6896  * @hba: Pointer to adapter instance
6897  * @tag: Task tag/index to be aborted
6898  *
6899  * Abort the pending command in device by sending UFS_ABORT_TASK task management
6900  * command, and in host controller by clearing the door-bell register. There can
6901  * be race between controller sending the command to the device while abort is
6902  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6903  * really issued and then try to abort it.
6904  *
6905  * Returns zero on success, non-zero on failure
6906  */
6907 static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
6908 {
6909         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
6910         int err = 0;
6911         int poll_cnt;
6912         u8 resp = 0xF;
6913         u32 reg;
6914
6915         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
6916                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6917                                 UFS_QUERY_TASK, &resp);
6918                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
6919                         /* cmd pending in the device */
6920                         dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
6921                                 __func__, tag);
6922                         break;
6923                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6924                         /*
6925                          * cmd not pending in the device, check if it is
6926                          * in transition.
6927                          */
6928                         dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
6929                                 __func__, tag);
6930                         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
6931                         if (reg & (1 << tag)) {
6932                                 /* sleep for max. 200us to stabilize */
6933                                 usleep_range(100, 200);
6934                                 continue;
6935                         }
6936                         /* command completed already */
6937                         dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
6938                                 __func__, tag);
6939                         goto out;
6940                 } else {
6941                         dev_err(hba->dev,
6942                                 "%s: no response from device. tag = %d, err %d\n",
6943                                 __func__, tag, err);
6944                         if (!err)
6945                                 err = resp; /* service response error */
6946                         goto out;
6947                 }
6948         }
6949
6950         if (!poll_cnt) {
6951                 err = -EBUSY;
6952                 goto out;
6953         }
6954
6955         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
6956                         UFS_ABORT_TASK, &resp);
6957         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
6958                 if (!err) {
6959                         err = resp; /* service response error */
6960                         dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
6961                                 __func__, tag, err);
6962                 }
6963                 goto out;
6964         }
6965
6966         err = ufshcd_clear_cmd(hba, tag);
6967         if (err)
6968                 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
6969                         __func__, tag, err);
6970
6971 out:
6972         return err;
6973 }
6974
6975 /**
6976  * ufshcd_abort - scsi host template eh_abort_handler callback
6977  * @cmd: SCSI command pointer
6978  *
6979  * Returns SUCCESS/FAILED
6980  */
6981 static int ufshcd_abort(struct scsi_cmnd *cmd)
6982 {
6983         struct Scsi_Host *host;
6984         struct ufs_hba *hba;
6985         unsigned long flags;
6986         unsigned int tag;
6987         int err = 0;
6988         struct ufshcd_lrb *lrbp;
6989         u32 reg;
6990
6991         host = cmd->device->host;
6992         hba = shost_priv(host);
6993         tag = cmd->request->tag;
6994         lrbp = &hba->lrb[tag];
6995         if (!ufshcd_valid_tag(hba, tag)) {
6996                 dev_err(hba->dev,
6997                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
6998                         __func__, tag, cmd, cmd->request);
6999                 BUG();
7000         }
7001
7002         ufshcd_hold(hba, false);
7003         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7004         /* If command is already aborted/completed, return SUCCESS */
7005         if (!(test_bit(tag, &hba->outstanding_reqs))) {
7006                 dev_err(hba->dev,
7007                         "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7008                         __func__, tag, hba->outstanding_reqs, reg);
7009                 goto out;
7010         }
7011
7012         /* Print Transfer Request of aborted task */
7013         dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7014
7015         /*
7016          * Print detailed info about aborted request.
7017          * As more than one request might get aborted at the same time,
7018          * print full information only for the first aborted request in order
7019          * to reduce repeated printouts. For other aborted requests only print
7020          * basic details.
7021          */
7022         scsi_print_command(cmd);
7023         if (!hba->req_abort_count) {
7024                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7025                 ufshcd_print_evt_hist(hba);
7026                 ufshcd_print_host_state(hba);
7027                 ufshcd_print_pwr_info(hba);
7028                 ufshcd_print_trs(hba, 1 << tag, true);
7029         } else {
7030                 ufshcd_print_trs(hba, 1 << tag, false);
7031         }
7032         hba->req_abort_count++;
7033
7034         if (!(reg & (1 << tag))) {
7035                 dev_err(hba->dev,
7036                 "%s: cmd was completed, but without a notifying intr, tag = %d",
7037                 __func__, tag);
7038                 goto cleanup;
7039         }
7040
7041         /*
7042          * Task abort to the device W-LUN is illegal. When this command
7043          * will fail, due to spec violation, scsi err handling next step
7044          * will be to send LU reset which, again, is a spec violation.
7045          * To avoid these unnecessary/illegal steps, first we clean up
7046          * the lrb taken by this cmd and re-set it in outstanding_reqs,
7047          * then queue the eh_work and bail.
7048          */
7049         if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7050                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7051                 __ufshcd_transfer_req_compl(hba, (1UL << tag));
7052                 set_bit(tag, &hba->outstanding_reqs);
7053                 spin_lock_irqsave(host->host_lock, flags);
7054                 hba->force_reset = true;
7055                 ufshcd_schedule_eh_work(hba);
7056                 spin_unlock_irqrestore(host->host_lock, flags);
7057                 goto out;
7058         }
7059
7060         /* Skip task abort in case previous aborts failed and report failure */
7061         if (lrbp->req_abort_skip)
7062                 err = -EIO;
7063         else
7064                 err = ufshcd_try_to_abort_task(hba, tag);
7065
7066         if (!err) {
7067 cleanup:
7068                 __ufshcd_transfer_req_compl(hba, (1UL << tag));
7069 out:
7070                 err = SUCCESS;
7071         } else {
7072                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7073                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7074                 err = FAILED;
7075         }
7076
7077         /*
7078          * This ufshcd_release() corresponds to the original scsi cmd that got
7079          * aborted here (as we won't get any IRQ for it).
7080          */
7081         ufshcd_release(hba);
7082         return err;
7083 }
7084
7085 /**
7086  * ufshcd_host_reset_and_restore - reset and restore host controller
7087  * @hba: per-adapter instance
7088  *
7089  * Note that host controller reset may issue DME_RESET to
7090  * local and remote (device) Uni-Pro stack and the attributes
7091  * are reset to default state.
7092  *
7093  * Returns zero on success, non-zero on failure
7094  */
7095 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7096 {
7097         int err;
7098
7099         /*
7100          * Stop the host controller and complete the requests
7101          * cleared by h/w
7102          */
7103         ufshcd_hba_stop(hba);
7104         hba->silence_err_logs = true;
7105         ufshcd_complete_requests(hba);
7106         hba->silence_err_logs = false;
7107
7108         /* scale up clocks to max frequency before full reinitialization */
7109         ufshcd_set_clk_freq(hba, true);
7110
7111         err = ufshcd_hba_enable(hba);
7112
7113         /* Establish the link again and restore the device */
7114         if (!err)
7115                 err = ufshcd_probe_hba(hba, false);
7116
7117         if (err)
7118                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7119         ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7120         return err;
7121 }
7122
7123 /**
7124  * ufshcd_reset_and_restore - reset and re-initialize host/device
7125  * @hba: per-adapter instance
7126  *
7127  * Reset and recover device, host and re-establish link. This
7128  * is helpful to recover the communication in fatal error conditions.
7129  *
7130  * Returns zero on success, non-zero on failure
7131  */
7132 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7133 {
7134         u32 saved_err;
7135         u32 saved_uic_err;
7136         int err = 0;
7137         unsigned long flags;
7138         int retries = MAX_HOST_RESET_RETRIES;
7139
7140         /*
7141          * This is a fresh start, cache and clear saved error first,
7142          * in case new error generated during reset and restore.
7143          */
7144         spin_lock_irqsave(hba->host->host_lock, flags);
7145         saved_err = hba->saved_err;
7146         saved_uic_err = hba->saved_uic_err;
7147         hba->saved_err = 0;
7148         hba->saved_uic_err = 0;
7149         spin_unlock_irqrestore(hba->host->host_lock, flags);
7150
7151         do {
7152                 /* Reset the attached device */
7153                 ufshcd_device_reset(hba);
7154
7155                 err = ufshcd_host_reset_and_restore(hba);
7156         } while (err && --retries);
7157
7158         spin_lock_irqsave(hba->host->host_lock, flags);
7159         /*
7160          * Inform scsi mid-layer that we did reset and allow to handle
7161          * Unit Attention properly.
7162          */
7163         scsi_report_bus_reset(hba->host, 0);
7164         if (err) {
7165                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7166                 hba->saved_err |= saved_err;
7167                 hba->saved_uic_err |= saved_uic_err;
7168         }
7169         spin_unlock_irqrestore(hba->host->host_lock, flags);
7170
7171         return err;
7172 }
7173
7174 /**
7175  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7176  * @cmd: SCSI command pointer
7177  *
7178  * Returns SUCCESS/FAILED
7179  */
7180 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7181 {
7182         int err = SUCCESS;
7183         unsigned long flags;
7184         struct ufs_hba *hba;
7185
7186         hba = shost_priv(cmd->device->host);
7187
7188         spin_lock_irqsave(hba->host->host_lock, flags);
7189         hba->force_reset = true;
7190         ufshcd_schedule_eh_work(hba);
7191         dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7192         spin_unlock_irqrestore(hba->host->host_lock, flags);
7193
7194         flush_work(&hba->eh_work);
7195
7196         spin_lock_irqsave(hba->host->host_lock, flags);
7197         if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7198                 err = FAILED;
7199         spin_unlock_irqrestore(hba->host->host_lock, flags);
7200
7201         return err;
7202 }
7203
7204 /**
7205  * ufshcd_get_max_icc_level - calculate the ICC level
7206  * @sup_curr_uA: max. current supported by the regulator
7207  * @start_scan: row at the desc table to start scan from
7208  * @buff: power descriptor buffer
7209  *
7210  * Returns calculated max ICC level for specific regulator
7211  */
7212 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
7213 {
7214         int i;
7215         int curr_uA;
7216         u16 data;
7217         u16 unit;
7218
7219         for (i = start_scan; i >= 0; i--) {
7220                 data = be16_to_cpup((__be16 *)&buff[2 * i]);
7221                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7222                                                 ATTR_ICC_LVL_UNIT_OFFSET;
7223                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7224                 switch (unit) {
7225                 case UFSHCD_NANO_AMP:
7226                         curr_uA = curr_uA / 1000;
7227                         break;
7228                 case UFSHCD_MILI_AMP:
7229                         curr_uA = curr_uA * 1000;
7230                         break;
7231                 case UFSHCD_AMP:
7232                         curr_uA = curr_uA * 1000 * 1000;
7233                         break;
7234                 case UFSHCD_MICRO_AMP:
7235                 default:
7236                         break;
7237                 }
7238                 if (sup_curr_uA >= curr_uA)
7239                         break;
7240         }
7241         if (i < 0) {
7242                 i = 0;
7243                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7244         }
7245
7246         return (u32)i;
7247 }
7248
7249 /**
7250  * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7251  * In case regulators are not initialized we'll return 0
7252  * @hba: per-adapter instance
7253  * @desc_buf: power descriptor buffer to extract ICC levels from.
7254  * @len: length of desc_buff
7255  *
7256  * Returns calculated ICC level
7257  */
7258 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7259                                                         u8 *desc_buf, int len)
7260 {
7261         u32 icc_level = 0;
7262
7263         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7264                                                 !hba->vreg_info.vccq2) {
7265                 dev_err(hba->dev,
7266                         "%s: Regulator capability was not set, actvIccLevel=%d",
7267                                                         __func__, icc_level);
7268                 goto out;
7269         }
7270
7271         if (hba->vreg_info.vcc->max_uA)
7272                 icc_level = ufshcd_get_max_icc_level(
7273                                 hba->vreg_info.vcc->max_uA,
7274                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7275                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7276
7277         if (hba->vreg_info.vccq->max_uA)
7278                 icc_level = ufshcd_get_max_icc_level(
7279                                 hba->vreg_info.vccq->max_uA,
7280                                 icc_level,
7281                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7282
7283         if (hba->vreg_info.vccq2->max_uA)
7284                 icc_level = ufshcd_get_max_icc_level(
7285                                 hba->vreg_info.vccq2->max_uA,
7286                                 icc_level,
7287                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7288 out:
7289         return icc_level;
7290 }
7291
7292 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7293 {
7294         int ret;
7295         int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER];
7296         u8 *desc_buf;
7297         u32 icc_level;
7298
7299         desc_buf = kmalloc(buff_len, GFP_KERNEL);
7300         if (!desc_buf)
7301                 return;
7302
7303         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7304                                      desc_buf, buff_len);
7305         if (ret) {
7306                 dev_err(hba->dev,
7307                         "%s: Failed reading power descriptor.len = %d ret = %d",
7308                         __func__, buff_len, ret);
7309                 goto out;
7310         }
7311
7312         icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf,
7313                                                          buff_len);
7314         dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7315
7316         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7317                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7318
7319         if (ret)
7320                 dev_err(hba->dev,
7321                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7322                         __func__, icc_level, ret);
7323
7324 out:
7325         kfree(desc_buf);
7326 }
7327
7328 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7329 {
7330         scsi_autopm_get_device(sdev);
7331         blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7332         if (sdev->rpm_autosuspend)
7333                 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7334                                                  RPM_AUTOSUSPEND_DELAY_MS);
7335         scsi_autopm_put_device(sdev);
7336 }
7337
7338 /**
7339  * ufshcd_scsi_add_wlus - Adds required W-LUs
7340  * @hba: per-adapter instance
7341  *
7342  * UFS device specification requires the UFS devices to support 4 well known
7343  * logical units:
7344  *      "REPORT_LUNS" (address: 01h)
7345  *      "UFS Device" (address: 50h)
7346  *      "RPMB" (address: 44h)
7347  *      "BOOT" (address: 30h)
7348  * UFS device's power management needs to be controlled by "POWER CONDITION"
7349  * field of SSU (START STOP UNIT) command. But this "power condition" field
7350  * will take effect only when its sent to "UFS device" well known logical unit
7351  * hence we require the scsi_device instance to represent this logical unit in
7352  * order for the UFS host driver to send the SSU command for power management.
7353  *
7354  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7355  * Block) LU so user space process can control this LU. User space may also
7356  * want to have access to BOOT LU.
7357  *
7358  * This function adds scsi device instances for each of all well known LUs
7359  * (except "REPORT LUNS" LU).
7360  *
7361  * Returns zero on success (all required W-LUs are added successfully),
7362  * non-zero error value on failure (if failed to add any of the required W-LU).
7363  */
7364 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
7365 {
7366         int ret = 0;
7367         struct scsi_device *sdev_boot;
7368
7369         hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
7370                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
7371         if (IS_ERR(hba->sdev_ufs_device)) {
7372                 ret = PTR_ERR(hba->sdev_ufs_device);
7373                 hba->sdev_ufs_device = NULL;
7374                 goto out;
7375         }
7376         scsi_device_put(hba->sdev_ufs_device);
7377
7378         hba->sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
7379                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
7380         if (IS_ERR(hba->sdev_rpmb)) {
7381                 ret = PTR_ERR(hba->sdev_rpmb);
7382                 goto remove_sdev_ufs_device;
7383         }
7384         ufshcd_blk_pm_runtime_init(hba->sdev_rpmb);
7385         scsi_device_put(hba->sdev_rpmb);
7386
7387         sdev_boot = __scsi_add_device(hba->host, 0, 0,
7388                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
7389         if (IS_ERR(sdev_boot)) {
7390                 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
7391         } else {
7392                 ufshcd_blk_pm_runtime_init(sdev_boot);
7393                 scsi_device_put(sdev_boot);
7394         }
7395         goto out;
7396
7397 remove_sdev_ufs_device:
7398         scsi_remove_device(hba->sdev_ufs_device);
7399 out:
7400         return ret;
7401 }
7402
7403 static void ufshcd_wb_probe(struct ufs_hba *hba, u8 *desc_buf)
7404 {
7405         struct ufs_dev_info *dev_info = &hba->dev_info;
7406         u8 lun;
7407         u32 d_lu_wb_buf_alloc;
7408         u32 ext_ufs_feature;
7409
7410         if (!ufshcd_is_wb_allowed(hba))
7411                 return;
7412         /*
7413          * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
7414          * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
7415          * enabled
7416          */
7417         if (!(dev_info->wspecversion >= 0x310 ||
7418               dev_info->wspecversion == 0x220 ||
7419              (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
7420                 goto wb_disabled;
7421
7422         if (hba->desc_size[QUERY_DESC_IDN_DEVICE] <
7423             DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4)
7424                 goto wb_disabled;
7425
7426         ext_ufs_feature = get_unaligned_be32(desc_buf +
7427                                         DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
7428
7429         if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
7430                 goto wb_disabled;
7431
7432         /*
7433          * WB may be supported but not configured while provisioning. The spec
7434          * says, in dedicated wb buffer mode, a max of 1 lun would have wb
7435          * buffer configured.
7436          */
7437         dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
7438
7439         dev_info->b_presrv_uspc_en =
7440                 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
7441
7442         if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
7443                 if (!get_unaligned_be32(desc_buf +
7444                                    DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
7445                         goto wb_disabled;
7446         } else {
7447                 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
7448                         d_lu_wb_buf_alloc = 0;
7449                         ufshcd_read_unit_desc_param(hba,
7450                                         lun,
7451                                         UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
7452                                         (u8 *)&d_lu_wb_buf_alloc,
7453                                         sizeof(d_lu_wb_buf_alloc));
7454                         if (d_lu_wb_buf_alloc) {
7455                                 dev_info->wb_dedicated_lu = lun;
7456                                 break;
7457                         }
7458                 }
7459
7460                 if (!d_lu_wb_buf_alloc)
7461                         goto wb_disabled;
7462         }
7463         return;
7464
7465 wb_disabled:
7466         hba->caps &= ~UFSHCD_CAP_WB_EN;
7467 }
7468
7469 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups)
7470 {
7471         struct ufs_dev_fix *f;
7472         struct ufs_dev_info *dev_info = &hba->dev_info;
7473
7474         if (!fixups)
7475                 return;
7476
7477         for (f = fixups; f->quirk; f++) {
7478                 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
7479                      f->wmanufacturerid == UFS_ANY_VENDOR) &&
7480                      ((dev_info->model &&
7481                        STR_PRFX_EQUAL(f->model, dev_info->model)) ||
7482                       !strcmp(f->model, UFS_ANY_MODEL)))
7483                         hba->dev_quirks |= f->quirk;
7484         }
7485 }
7486 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
7487
7488 static void ufs_fixup_device_setup(struct ufs_hba *hba)
7489 {
7490         /* fix by general quirk table */
7491         ufshcd_fixup_dev_quirks(hba, ufs_fixups);
7492
7493         /* allow vendors to fix quirks */
7494         ufshcd_vops_fixup_dev_quirks(hba);
7495 }
7496
7497 static int ufs_get_device_desc(struct ufs_hba *hba)
7498 {
7499         int err;
7500         u8 model_index;
7501         u8 *desc_buf;
7502         struct ufs_dev_info *dev_info = &hba->dev_info;
7503
7504         desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7505         if (!desc_buf) {
7506                 err = -ENOMEM;
7507                 goto out;
7508         }
7509
7510         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
7511                                      hba->desc_size[QUERY_DESC_IDN_DEVICE]);
7512         if (err) {
7513                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
7514                         __func__, err);
7515                 goto out;
7516         }
7517
7518         /*
7519          * getting vendor (manufacturerID) and Bank Index in big endian
7520          * format
7521          */
7522         dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
7523                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
7524
7525         /* getting Specification Version in big endian format */
7526         dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
7527                                       desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
7528
7529         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
7530
7531         err = ufshcd_read_string_desc(hba, model_index,
7532                                       &dev_info->model, SD_ASCII_STD);
7533         if (err < 0) {
7534                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
7535                         __func__, err);
7536                 goto out;
7537         }
7538
7539         hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
7540                 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
7541
7542         ufs_fixup_device_setup(hba);
7543
7544         ufshcd_wb_probe(hba, desc_buf);
7545
7546         /*
7547          * ufshcd_read_string_desc returns size of the string
7548          * reset the error value
7549          */
7550         err = 0;
7551
7552 out:
7553         kfree(desc_buf);
7554         return err;
7555 }
7556
7557 static void ufs_put_device_desc(struct ufs_hba *hba)
7558 {
7559         struct ufs_dev_info *dev_info = &hba->dev_info;
7560
7561         kfree(dev_info->model);
7562         dev_info->model = NULL;
7563 }
7564
7565 /**
7566  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
7567  * @hba: per-adapter instance
7568  *
7569  * PA_TActivate parameter can be tuned manually if UniPro version is less than
7570  * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
7571  * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
7572  * the hibern8 exit latency.
7573  *
7574  * Returns zero on success, non-zero error value on failure.
7575  */
7576 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
7577 {
7578         int ret = 0;
7579         u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
7580
7581         ret = ufshcd_dme_peer_get(hba,
7582                                   UIC_ARG_MIB_SEL(
7583                                         RX_MIN_ACTIVATETIME_CAPABILITY,
7584                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7585                                   &peer_rx_min_activatetime);
7586         if (ret)
7587                 goto out;
7588
7589         /* make sure proper unit conversion is applied */
7590         tuned_pa_tactivate =
7591                 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
7592                  / PA_TACTIVATE_TIME_UNIT_US);
7593         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7594                              tuned_pa_tactivate);
7595
7596 out:
7597         return ret;
7598 }
7599
7600 /**
7601  * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
7602  * @hba: per-adapter instance
7603  *
7604  * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
7605  * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
7606  * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
7607  * This optimal value can help reduce the hibern8 exit latency.
7608  *
7609  * Returns zero on success, non-zero error value on failure.
7610  */
7611 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
7612 {
7613         int ret = 0;
7614         u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
7615         u32 max_hibern8_time, tuned_pa_hibern8time;
7616
7617         ret = ufshcd_dme_get(hba,
7618                              UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
7619                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
7620                                   &local_tx_hibern8_time_cap);
7621         if (ret)
7622                 goto out;
7623
7624         ret = ufshcd_dme_peer_get(hba,
7625                                   UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
7626                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
7627                                   &peer_rx_hibern8_time_cap);
7628         if (ret)
7629                 goto out;
7630
7631         max_hibern8_time = max(local_tx_hibern8_time_cap,
7632                                peer_rx_hibern8_time_cap);
7633         /* make sure proper unit conversion is applied */
7634         tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
7635                                 / PA_HIBERN8_TIME_UNIT_US);
7636         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
7637                              tuned_pa_hibern8time);
7638 out:
7639         return ret;
7640 }
7641
7642 /**
7643  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
7644  * less than device PA_TACTIVATE time.
7645  * @hba: per-adapter instance
7646  *
7647  * Some UFS devices require host PA_TACTIVATE to be lower than device
7648  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
7649  * for such devices.
7650  *
7651  * Returns zero on success, non-zero error value on failure.
7652  */
7653 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
7654 {
7655         int ret = 0;
7656         u32 granularity, peer_granularity;
7657         u32 pa_tactivate, peer_pa_tactivate;
7658         u32 pa_tactivate_us, peer_pa_tactivate_us;
7659         u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
7660
7661         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7662                                   &granularity);
7663         if (ret)
7664                 goto out;
7665
7666         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
7667                                   &peer_granularity);
7668         if (ret)
7669                 goto out;
7670
7671         if ((granularity < PA_GRANULARITY_MIN_VAL) ||
7672             (granularity > PA_GRANULARITY_MAX_VAL)) {
7673                 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
7674                         __func__, granularity);
7675                 return -EINVAL;
7676         }
7677
7678         if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
7679             (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
7680                 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
7681                         __func__, peer_granularity);
7682                 return -EINVAL;
7683         }
7684
7685         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
7686         if (ret)
7687                 goto out;
7688
7689         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
7690                                   &peer_pa_tactivate);
7691         if (ret)
7692                 goto out;
7693
7694         pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
7695         peer_pa_tactivate_us = peer_pa_tactivate *
7696                              gran_to_us_table[peer_granularity - 1];
7697
7698         if (pa_tactivate_us > peer_pa_tactivate_us) {
7699                 u32 new_peer_pa_tactivate;
7700
7701                 new_peer_pa_tactivate = pa_tactivate_us /
7702                                       gran_to_us_table[peer_granularity - 1];
7703                 new_peer_pa_tactivate++;
7704                 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
7705                                           new_peer_pa_tactivate);
7706         }
7707
7708 out:
7709         return ret;
7710 }
7711
7712 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
7713 {
7714         if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
7715                 ufshcd_tune_pa_tactivate(hba);
7716                 ufshcd_tune_pa_hibern8time(hba);
7717         }
7718
7719         ufshcd_vops_apply_dev_quirks(hba);
7720
7721         if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
7722                 /* set 1ms timeout for PA_TACTIVATE */
7723                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
7724
7725         if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
7726                 ufshcd_quirk_tune_host_pa_tactivate(hba);
7727 }
7728
7729 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
7730 {
7731         hba->ufs_stats.hibern8_exit_cnt = 0;
7732         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
7733         hba->req_abort_count = 0;
7734 }
7735
7736 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
7737 {
7738         int err;
7739         size_t buff_len;
7740         u8 *desc_buf;
7741
7742         buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY];
7743         desc_buf = kmalloc(buff_len, GFP_KERNEL);
7744         if (!desc_buf) {
7745                 err = -ENOMEM;
7746                 goto out;
7747         }
7748
7749         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
7750                                      desc_buf, buff_len);
7751         if (err) {
7752                 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
7753                                 __func__, err);
7754                 goto out;
7755         }
7756
7757         if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
7758                 hba->dev_info.max_lu_supported = 32;
7759         else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
7760                 hba->dev_info.max_lu_supported = 8;
7761
7762 out:
7763         kfree(desc_buf);
7764         return err;
7765 }
7766
7767 static struct ufs_ref_clk ufs_ref_clk_freqs[] = {
7768         {19200000, REF_CLK_FREQ_19_2_MHZ},
7769         {26000000, REF_CLK_FREQ_26_MHZ},
7770         {38400000, REF_CLK_FREQ_38_4_MHZ},
7771         {52000000, REF_CLK_FREQ_52_MHZ},
7772         {0, REF_CLK_FREQ_INVAL},
7773 };
7774
7775 static enum ufs_ref_clk_freq
7776 ufs_get_bref_clk_from_hz(unsigned long freq)
7777 {
7778         int i;
7779
7780         for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
7781                 if (ufs_ref_clk_freqs[i].freq_hz == freq)
7782                         return ufs_ref_clk_freqs[i].val;
7783
7784         return REF_CLK_FREQ_INVAL;
7785 }
7786
7787 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
7788 {
7789         unsigned long freq;
7790
7791         freq = clk_get_rate(refclk);
7792
7793         hba->dev_ref_clk_freq =
7794                 ufs_get_bref_clk_from_hz(freq);
7795
7796         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
7797                 dev_err(hba->dev,
7798                 "invalid ref_clk setting = %ld\n", freq);
7799 }
7800
7801 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
7802 {
7803         int err;
7804         u32 ref_clk;
7805         u32 freq = hba->dev_ref_clk_freq;
7806
7807         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
7808                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
7809
7810         if (err) {
7811                 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
7812                         err);
7813                 goto out;
7814         }
7815
7816         if (ref_clk == freq)
7817                 goto out; /* nothing to update */
7818
7819         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7820                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
7821
7822         if (err) {
7823                 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
7824                         ufs_ref_clk_freqs[freq].freq_hz);
7825                 goto out;
7826         }
7827
7828         dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
7829                         ufs_ref_clk_freqs[freq].freq_hz);
7830
7831 out:
7832         return err;
7833 }
7834
7835 static int ufshcd_device_params_init(struct ufs_hba *hba)
7836 {
7837         bool flag;
7838         int ret, i;
7839
7840          /* Init device descriptor sizes */
7841         for (i = 0; i < QUERY_DESC_IDN_MAX; i++)
7842                 hba->desc_size[i] = QUERY_DESC_MAX_SIZE;
7843
7844         /* Init UFS geometry descriptor related parameters */
7845         ret = ufshcd_device_geo_params_init(hba);
7846         if (ret)
7847                 goto out;
7848
7849         /* Check and apply UFS device quirks */
7850         ret = ufs_get_device_desc(hba);
7851         if (ret) {
7852                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
7853                         __func__, ret);
7854                 goto out;
7855         }
7856
7857         ufshcd_get_ref_clk_gating_wait(hba);
7858
7859         if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
7860                         QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
7861                 hba->dev_info.f_power_on_wp_en = flag;
7862
7863         /* Probe maximum power mode co-supported by both UFS host and device */
7864         if (ufshcd_get_max_pwr_mode(hba))
7865                 dev_err(hba->dev,
7866                         "%s: Failed getting max supported power mode\n",
7867                         __func__);
7868 out:
7869         return ret;
7870 }
7871
7872 /**
7873  * ufshcd_add_lus - probe and add UFS logical units
7874  * @hba: per-adapter instance
7875  */
7876 static int ufshcd_add_lus(struct ufs_hba *hba)
7877 {
7878         int ret;
7879
7880         /* Add required well known logical units to scsi mid layer */
7881         ret = ufshcd_scsi_add_wlus(hba);
7882         if (ret)
7883                 goto out;
7884
7885         ufshcd_clear_ua_wluns(hba);
7886
7887         /* Initialize devfreq after UFS device is detected */
7888         if (ufshcd_is_clkscaling_supported(hba)) {
7889                 memcpy(&hba->clk_scaling.saved_pwr_info.info,
7890                         &hba->pwr_info,
7891                         sizeof(struct ufs_pa_layer_attr));
7892                 hba->clk_scaling.saved_pwr_info.is_valid = true;
7893                 hba->clk_scaling.is_allowed = true;
7894
7895                 ret = ufshcd_devfreq_init(hba);
7896                 if (ret)
7897                         goto out;
7898
7899                 hba->clk_scaling.is_enabled = true;
7900                 ufshcd_init_clk_scaling_sysfs(hba);
7901         }
7902
7903         ufs_bsg_probe(hba);
7904         scsi_scan_host(hba->host);
7905         pm_runtime_put_sync(hba->dev);
7906
7907 out:
7908         return ret;
7909 }
7910
7911 static int
7912 ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp);
7913
7914 static int ufshcd_clear_ua_wlun(struct ufs_hba *hba, u8 wlun)
7915 {
7916         struct scsi_device *sdp;
7917         unsigned long flags;
7918         int ret = 0;
7919
7920         spin_lock_irqsave(hba->host->host_lock, flags);
7921         if (wlun == UFS_UPIU_UFS_DEVICE_WLUN)
7922                 sdp = hba->sdev_ufs_device;
7923         else if (wlun == UFS_UPIU_RPMB_WLUN)
7924                 sdp = hba->sdev_rpmb;
7925         else
7926                 BUG();
7927         if (sdp) {
7928                 ret = scsi_device_get(sdp);
7929                 if (!ret && !scsi_device_online(sdp)) {
7930                         ret = -ENODEV;
7931                         scsi_device_put(sdp);
7932                 }
7933         } else {
7934                 ret = -ENODEV;
7935         }
7936         spin_unlock_irqrestore(hba->host->host_lock, flags);
7937         if (ret)
7938                 goto out_err;
7939
7940         ret = ufshcd_send_request_sense(hba, sdp);
7941         scsi_device_put(sdp);
7942 out_err:
7943         if (ret)
7944                 dev_err(hba->dev, "%s: UAC clear LU=%x ret = %d\n",
7945                                 __func__, wlun, ret);
7946         return ret;
7947 }
7948
7949 static int ufshcd_clear_ua_wluns(struct ufs_hba *hba)
7950 {
7951         int ret = 0;
7952
7953         if (!hba->wlun_dev_clr_ua)
7954                 goto out;
7955
7956         ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN);
7957         if (!ret)
7958                 ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN);
7959         if (!ret)
7960                 hba->wlun_dev_clr_ua = false;
7961 out:
7962         if (ret)
7963                 dev_err(hba->dev, "%s: Failed to clear UAC WLUNS ret = %d\n",
7964                                 __func__, ret);
7965         return ret;
7966 }
7967
7968 /**
7969  * ufshcd_probe_hba - probe hba to detect device and initialize
7970  * @hba: per-adapter instance
7971  * @async: asynchronous execution or not
7972  *
7973  * Execute link-startup and verify device initialization
7974  */
7975 static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
7976 {
7977         int ret;
7978         unsigned long flags;
7979         ktime_t start = ktime_get();
7980
7981         hba->ufshcd_state = UFSHCD_STATE_RESET;
7982
7983         ret = ufshcd_link_startup(hba);
7984         if (ret)
7985                 goto out;
7986
7987         /* Debug counters initialization */
7988         ufshcd_clear_dbg_ufs_stats(hba);
7989
7990         /* UniPro link is active now */
7991         ufshcd_set_link_active(hba);
7992
7993         /* Verify device initialization by sending NOP OUT UPIU */
7994         ret = ufshcd_verify_dev_init(hba);
7995         if (ret)
7996                 goto out;
7997
7998         /* Initiate UFS initialization, and waiting until completion */
7999         ret = ufshcd_complete_dev_init(hba);
8000         if (ret)
8001                 goto out;
8002
8003         /*
8004          * Initialize UFS device parameters used by driver, these
8005          * parameters are associated with UFS descriptors.
8006          */
8007         if (async) {
8008                 ret = ufshcd_device_params_init(hba);
8009                 if (ret)
8010                         goto out;
8011         }
8012
8013         ufshcd_tune_unipro_params(hba);
8014
8015         /* UFS device is also active now */
8016         ufshcd_set_ufs_dev_active(hba);
8017         ufshcd_force_reset_auto_bkops(hba);
8018         hba->wlun_dev_clr_ua = true;
8019         hba->wlun_rpmb_clr_ua = true;
8020
8021         /* Gear up to HS gear if supported */
8022         if (hba->max_pwr_info.is_valid) {
8023                 /*
8024                  * Set the right value to bRefClkFreq before attempting to
8025                  * switch to HS gears.
8026                  */
8027                 if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8028                         ufshcd_set_dev_ref_clk(hba);
8029                 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8030                 if (ret) {
8031                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8032                                         __func__, ret);
8033                         goto out;
8034                 }
8035                 ufshcd_print_pwr_info(hba);
8036         }
8037
8038         /*
8039          * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8040          * and for removable UFS card as well, hence always set the parameter.
8041          * Note: Error handler may issue the device reset hence resetting
8042          * bActiveICCLevel as well so it is always safe to set this here.
8043          */
8044         ufshcd_set_active_icc_lvl(hba);
8045
8046         ufshcd_wb_config(hba);
8047         if (hba->ee_usr_mask)
8048                 ufshcd_write_ee_control(hba);
8049         /* Enable Auto-Hibernate if configured */
8050         ufshcd_auto_hibern8_enable(hba);
8051
8052 out:
8053         spin_lock_irqsave(hba->host->host_lock, flags);
8054         if (ret)
8055                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
8056         else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
8057                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
8058         spin_unlock_irqrestore(hba->host->host_lock, flags);
8059
8060         trace_ufshcd_init(dev_name(hba->dev), ret,
8061                 ktime_to_us(ktime_sub(ktime_get(), start)),
8062                 hba->curr_dev_pwr_mode, hba->uic_link_state);
8063         return ret;
8064 }
8065
8066 /**
8067  * ufshcd_async_scan - asynchronous execution for probing hba
8068  * @data: data pointer to pass to this function
8069  * @cookie: cookie data
8070  */
8071 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8072 {
8073         struct ufs_hba *hba = (struct ufs_hba *)data;
8074         int ret;
8075
8076         down(&hba->host_sem);
8077         /* Initialize hba, detect and initialize UFS device */
8078         ret = ufshcd_probe_hba(hba, true);
8079         up(&hba->host_sem);
8080         if (ret)
8081                 goto out;
8082
8083         /* Probe and add UFS logical units  */
8084         ret = ufshcd_add_lus(hba);
8085 out:
8086         /*
8087          * If we failed to initialize the device or the device is not
8088          * present, turn off the power/clocks etc.
8089          */
8090         if (ret) {
8091                 pm_runtime_put_sync(hba->dev);
8092                 ufshcd_hba_exit(hba);
8093         }
8094 }
8095
8096 static const struct attribute_group *ufshcd_driver_groups[] = {
8097         &ufs_sysfs_unit_descriptor_group,
8098         &ufs_sysfs_lun_attributes_group,
8099         NULL,
8100 };
8101
8102 static struct ufs_hba_variant_params ufs_hba_vps = {
8103         .hba_enable_delay_us            = 1000,
8104         .wb_flush_threshold             = UFS_WB_BUF_REMAIN_PERCENT(40),
8105         .devfreq_profile.polling_ms     = 100,
8106         .devfreq_profile.target         = ufshcd_devfreq_target,
8107         .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8108         .ondemand_data.upthreshold      = 70,
8109         .ondemand_data.downdifferential = 5,
8110 };
8111
8112 static struct scsi_host_template ufshcd_driver_template = {
8113         .module                 = THIS_MODULE,
8114         .name                   = UFSHCD,
8115         .proc_name              = UFSHCD,
8116         .queuecommand           = ufshcd_queuecommand,
8117         .slave_alloc            = ufshcd_slave_alloc,
8118         .slave_configure        = ufshcd_slave_configure,
8119         .slave_destroy          = ufshcd_slave_destroy,
8120         .change_queue_depth     = ufshcd_change_queue_depth,
8121         .eh_abort_handler       = ufshcd_abort,
8122         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8123         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
8124         .this_id                = -1,
8125         .sg_tablesize           = SG_ALL,
8126         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
8127         .can_queue              = UFSHCD_CAN_QUEUE,
8128         .max_segment_size       = PRDT_DATA_BYTE_COUNT_MAX,
8129         .max_host_blocked       = 1,
8130         .track_queue_depth      = 1,
8131         .sdev_groups            = ufshcd_driver_groups,
8132         .dma_boundary           = PAGE_SIZE - 1,
8133         .rpm_autosuspend_delay  = RPM_AUTOSUSPEND_DELAY_MS,
8134 };
8135
8136 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8137                                    int ua)
8138 {
8139         int ret;
8140
8141         if (!vreg)
8142                 return 0;
8143
8144         /*
8145          * "set_load" operation shall be required on those regulators
8146          * which specifically configured current limitation. Otherwise
8147          * zero max_uA may cause unexpected behavior when regulator is
8148          * enabled or set as high power mode.
8149          */
8150         if (!vreg->max_uA)
8151                 return 0;
8152
8153         ret = regulator_set_load(vreg->reg, ua);
8154         if (ret < 0) {
8155                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
8156                                 __func__, vreg->name, ua, ret);
8157         }
8158
8159         return ret;
8160 }
8161
8162 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
8163                                          struct ufs_vreg *vreg)
8164 {
8165         return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
8166 }
8167
8168 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
8169                                          struct ufs_vreg *vreg)
8170 {
8171         if (!vreg)
8172                 return 0;
8173
8174         return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
8175 }
8176
8177 static int ufshcd_config_vreg(struct device *dev,
8178                 struct ufs_vreg *vreg, bool on)
8179 {
8180         int ret = 0;
8181         struct regulator *reg;
8182         const char *name;
8183         int min_uV, uA_load;
8184
8185         BUG_ON(!vreg);
8186
8187         reg = vreg->reg;
8188         name = vreg->name;
8189
8190         if (regulator_count_voltages(reg) > 0) {
8191                 uA_load = on ? vreg->max_uA : 0;
8192                 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
8193                 if (ret)
8194                         goto out;
8195
8196                 if (vreg->min_uV && vreg->max_uV) {
8197                         min_uV = on ? vreg->min_uV : 0;
8198                         ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
8199                         if (ret)
8200                                 dev_err(dev,
8201                                         "%s: %s set voltage failed, err=%d\n",
8202                                         __func__, name, ret);
8203                 }
8204         }
8205 out:
8206         return ret;
8207 }
8208
8209 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
8210 {
8211         int ret = 0;
8212
8213         if (!vreg || vreg->enabled)
8214                 goto out;
8215
8216         ret = ufshcd_config_vreg(dev, vreg, true);
8217         if (!ret)
8218                 ret = regulator_enable(vreg->reg);
8219
8220         if (!ret)
8221                 vreg->enabled = true;
8222         else
8223                 dev_err(dev, "%s: %s enable failed, err=%d\n",
8224                                 __func__, vreg->name, ret);
8225 out:
8226         return ret;
8227 }
8228
8229 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
8230 {
8231         int ret = 0;
8232
8233         if (!vreg || !vreg->enabled || vreg->always_on)
8234                 goto out;
8235
8236         ret = regulator_disable(vreg->reg);
8237
8238         if (!ret) {
8239                 /* ignore errors on applying disable config */
8240                 ufshcd_config_vreg(dev, vreg, false);
8241                 vreg->enabled = false;
8242         } else {
8243                 dev_err(dev, "%s: %s disable failed, err=%d\n",
8244                                 __func__, vreg->name, ret);
8245         }
8246 out:
8247         return ret;
8248 }
8249
8250 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
8251 {
8252         int ret = 0;
8253         struct device *dev = hba->dev;
8254         struct ufs_vreg_info *info = &hba->vreg_info;
8255
8256         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
8257         if (ret)
8258                 goto out;
8259
8260         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
8261         if (ret)
8262                 goto out;
8263
8264         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
8265
8266 out:
8267         if (ret) {
8268                 ufshcd_toggle_vreg(dev, info->vccq2, false);
8269                 ufshcd_toggle_vreg(dev, info->vccq, false);
8270                 ufshcd_toggle_vreg(dev, info->vcc, false);
8271         }
8272         return ret;
8273 }
8274
8275 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
8276 {
8277         struct ufs_vreg_info *info = &hba->vreg_info;
8278
8279         return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
8280 }
8281
8282 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
8283 {
8284         int ret = 0;
8285
8286         if (!vreg)
8287                 goto out;
8288
8289         vreg->reg = devm_regulator_get(dev, vreg->name);
8290         if (IS_ERR(vreg->reg)) {
8291                 ret = PTR_ERR(vreg->reg);
8292                 dev_err(dev, "%s: %s get failed, err=%d\n",
8293                                 __func__, vreg->name, ret);
8294         }
8295 out:
8296         return ret;
8297 }
8298
8299 static int ufshcd_init_vreg(struct ufs_hba *hba)
8300 {
8301         int ret = 0;
8302         struct device *dev = hba->dev;
8303         struct ufs_vreg_info *info = &hba->vreg_info;
8304
8305         ret = ufshcd_get_vreg(dev, info->vcc);
8306         if (ret)
8307                 goto out;
8308
8309         ret = ufshcd_get_vreg(dev, info->vccq);
8310         if (!ret)
8311                 ret = ufshcd_get_vreg(dev, info->vccq2);
8312 out:
8313         return ret;
8314 }
8315
8316 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
8317 {
8318         struct ufs_vreg_info *info = &hba->vreg_info;
8319
8320         if (info)
8321                 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
8322
8323         return 0;
8324 }
8325
8326 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
8327 {
8328         int ret = 0;
8329         struct ufs_clk_info *clki;
8330         struct list_head *head = &hba->clk_list_head;
8331         unsigned long flags;
8332         ktime_t start = ktime_get();
8333         bool clk_state_changed = false;
8334
8335         if (list_empty(head))
8336                 goto out;
8337
8338         ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
8339         if (ret)
8340                 return ret;
8341
8342         list_for_each_entry(clki, head, list) {
8343                 if (!IS_ERR_OR_NULL(clki->clk)) {
8344                         /*
8345                          * Don't disable clocks which are needed
8346                          * to keep the link active.
8347                          */
8348                         if (ufshcd_is_link_active(hba) &&
8349                             clki->keep_link_active)
8350                                 continue;
8351
8352                         clk_state_changed = on ^ clki->enabled;
8353                         if (on && !clki->enabled) {
8354                                 ret = clk_prepare_enable(clki->clk);
8355                                 if (ret) {
8356                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
8357                                                 __func__, clki->name, ret);
8358                                         goto out;
8359                                 }
8360                         } else if (!on && clki->enabled) {
8361                                 clk_disable_unprepare(clki->clk);
8362                         }
8363                         clki->enabled = on;
8364                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
8365                                         clki->name, on ? "en" : "dis");
8366                 }
8367         }
8368
8369         ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
8370         if (ret)
8371                 return ret;
8372
8373 out:
8374         if (ret) {
8375                 list_for_each_entry(clki, head, list) {
8376                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
8377                                 clk_disable_unprepare(clki->clk);
8378                 }
8379         } else if (!ret && on) {
8380                 spin_lock_irqsave(hba->host->host_lock, flags);
8381                 hba->clk_gating.state = CLKS_ON;
8382                 trace_ufshcd_clk_gating(dev_name(hba->dev),
8383                                         hba->clk_gating.state);
8384                 spin_unlock_irqrestore(hba->host->host_lock, flags);
8385         }
8386
8387         if (clk_state_changed)
8388                 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
8389                         (on ? "on" : "off"),
8390                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
8391         return ret;
8392 }
8393
8394 static int ufshcd_init_clocks(struct ufs_hba *hba)
8395 {
8396         int ret = 0;
8397         struct ufs_clk_info *clki;
8398         struct device *dev = hba->dev;
8399         struct list_head *head = &hba->clk_list_head;
8400
8401         if (list_empty(head))
8402                 goto out;
8403
8404         list_for_each_entry(clki, head, list) {
8405                 if (!clki->name)
8406                         continue;
8407
8408                 clki->clk = devm_clk_get(dev, clki->name);
8409                 if (IS_ERR(clki->clk)) {
8410                         ret = PTR_ERR(clki->clk);
8411                         dev_err(dev, "%s: %s clk get failed, %d\n",
8412                                         __func__, clki->name, ret);
8413                         goto out;
8414                 }
8415
8416                 /*
8417                  * Parse device ref clk freq as per device tree "ref_clk".
8418                  * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
8419                  * in ufshcd_alloc_host().
8420                  */
8421                 if (!strcmp(clki->name, "ref_clk"))
8422                         ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
8423
8424                 if (clki->max_freq) {
8425                         ret = clk_set_rate(clki->clk, clki->max_freq);
8426                         if (ret) {
8427                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
8428                                         __func__, clki->name,
8429                                         clki->max_freq, ret);
8430                                 goto out;
8431                         }
8432                         clki->curr_freq = clki->max_freq;
8433                 }
8434                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
8435                                 clki->name, clk_get_rate(clki->clk));
8436         }
8437 out:
8438         return ret;
8439 }
8440
8441 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
8442 {
8443         int err = 0;
8444
8445         if (!hba->vops)
8446                 goto out;
8447
8448         err = ufshcd_vops_init(hba);
8449         if (err)
8450                 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
8451                         __func__, ufshcd_get_var_name(hba), err);
8452 out:
8453         return err;
8454 }
8455
8456 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
8457 {
8458         if (!hba->vops)
8459                 return;
8460
8461         ufshcd_vops_exit(hba);
8462 }
8463
8464 static int ufshcd_hba_init(struct ufs_hba *hba)
8465 {
8466         int err;
8467
8468         /*
8469          * Handle host controller power separately from the UFS device power
8470          * rails as it will help controlling the UFS host controller power
8471          * collapse easily which is different than UFS device power collapse.
8472          * Also, enable the host controller power before we go ahead with rest
8473          * of the initialization here.
8474          */
8475         err = ufshcd_init_hba_vreg(hba);
8476         if (err)
8477                 goto out;
8478
8479         err = ufshcd_setup_hba_vreg(hba, true);
8480         if (err)
8481                 goto out;
8482
8483         err = ufshcd_init_clocks(hba);
8484         if (err)
8485                 goto out_disable_hba_vreg;
8486
8487         err = ufshcd_setup_clocks(hba, true);
8488         if (err)
8489                 goto out_disable_hba_vreg;
8490
8491         err = ufshcd_init_vreg(hba);
8492         if (err)
8493                 goto out_disable_clks;
8494
8495         err = ufshcd_setup_vreg(hba, true);
8496         if (err)
8497                 goto out_disable_clks;
8498
8499         err = ufshcd_variant_hba_init(hba);
8500         if (err)
8501                 goto out_disable_vreg;
8502
8503         ufs_debugfs_hba_init(hba);
8504
8505         hba->is_powered = true;
8506         goto out;
8507
8508 out_disable_vreg:
8509         ufshcd_setup_vreg(hba, false);
8510 out_disable_clks:
8511         ufshcd_setup_clocks(hba, false);
8512 out_disable_hba_vreg:
8513         ufshcd_setup_hba_vreg(hba, false);
8514 out:
8515         return err;
8516 }
8517
8518 static void ufshcd_hba_exit(struct ufs_hba *hba)
8519 {
8520         if (hba->is_powered) {
8521                 ufshcd_exit_clk_scaling(hba);
8522                 ufshcd_exit_clk_gating(hba);
8523                 if (hba->eh_wq)
8524                         destroy_workqueue(hba->eh_wq);
8525                 ufs_debugfs_hba_exit(hba);
8526                 ufshcd_variant_hba_exit(hba);
8527                 ufshcd_setup_vreg(hba, false);
8528                 ufshcd_setup_clocks(hba, false);
8529                 ufshcd_setup_hba_vreg(hba, false);
8530                 hba->is_powered = false;
8531                 ufs_put_device_desc(hba);
8532         }
8533 }
8534
8535 static int
8536 ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
8537 {
8538         unsigned char cmd[6] = {REQUEST_SENSE,
8539                                 0,
8540                                 0,
8541                                 0,
8542                                 UFS_SENSE_SIZE,
8543                                 0};
8544         char *buffer;
8545         int ret;
8546
8547         buffer = kzalloc(UFS_SENSE_SIZE, GFP_KERNEL);
8548         if (!buffer) {
8549                 ret = -ENOMEM;
8550                 goto out;
8551         }
8552
8553         ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
8554                         UFS_SENSE_SIZE, NULL, NULL,
8555                         msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
8556         if (ret)
8557                 pr_err("%s: failed with err %d\n", __func__, ret);
8558
8559         kfree(buffer);
8560 out:
8561         return ret;
8562 }
8563
8564 /**
8565  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
8566  *                           power mode
8567  * @hba: per adapter instance
8568  * @pwr_mode: device power mode to set
8569  *
8570  * Returns 0 if requested power mode is set successfully
8571  * Returns non-zero if failed to set the requested power mode
8572  */
8573 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
8574                                      enum ufs_dev_pwr_mode pwr_mode)
8575 {
8576         unsigned char cmd[6] = { START_STOP };
8577         struct scsi_sense_hdr sshdr;
8578         struct scsi_device *sdp;
8579         unsigned long flags;
8580         int ret;
8581
8582         spin_lock_irqsave(hba->host->host_lock, flags);
8583         sdp = hba->sdev_ufs_device;
8584         if (sdp) {
8585                 ret = scsi_device_get(sdp);
8586                 if (!ret && !scsi_device_online(sdp)) {
8587                         ret = -ENODEV;
8588                         scsi_device_put(sdp);
8589                 }
8590         } else {
8591                 ret = -ENODEV;
8592         }
8593         spin_unlock_irqrestore(hba->host->host_lock, flags);
8594
8595         if (ret)
8596                 return ret;
8597
8598         /*
8599          * If scsi commands fail, the scsi mid-layer schedules scsi error-
8600          * handling, which would wait for host to be resumed. Since we know
8601          * we are functional while we are here, skip host resume in error
8602          * handling context.
8603          */
8604         hba->host->eh_noresume = 1;
8605         if (hba->wlun_dev_clr_ua)
8606                 ufshcd_clear_ua_wlun(hba, UFS_UPIU_UFS_DEVICE_WLUN);
8607
8608         cmd[4] = pwr_mode << 4;
8609
8610         /*
8611          * Current function would be generally called from the power management
8612          * callbacks hence set the RQF_PM flag so that it doesn't resume the
8613          * already suspended childs.
8614          */
8615         ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
8616                         START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
8617         if (ret) {
8618                 sdev_printk(KERN_WARNING, sdp,
8619                             "START_STOP failed for power mode: %d, result %x\n",
8620                             pwr_mode, ret);
8621                 if (ret > 0 && scsi_sense_valid(&sshdr))
8622                         scsi_print_sense_hdr(sdp, NULL, &sshdr);
8623         }
8624
8625         if (!ret)
8626                 hba->curr_dev_pwr_mode = pwr_mode;
8627
8628         scsi_device_put(sdp);
8629         hba->host->eh_noresume = 0;
8630         return ret;
8631 }
8632
8633 static int ufshcd_link_state_transition(struct ufs_hba *hba,
8634                                         enum uic_link_state req_link_state,
8635                                         int check_for_bkops)
8636 {
8637         int ret = 0;
8638
8639         if (req_link_state == hba->uic_link_state)
8640                 return 0;
8641
8642         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
8643                 ret = ufshcd_uic_hibern8_enter(hba);
8644                 if (!ret) {
8645                         ufshcd_set_link_hibern8(hba);
8646                 } else {
8647                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8648                                         __func__, ret);
8649                         goto out;
8650                 }
8651         }
8652         /*
8653          * If autobkops is enabled, link can't be turned off because
8654          * turning off the link would also turn off the device, except in the
8655          * case of DeepSleep where the device is expected to remain powered.
8656          */
8657         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
8658                  (!check_for_bkops || !hba->auto_bkops_enabled)) {
8659                 /*
8660                  * Let's make sure that link is in low power mode, we are doing
8661                  * this currently by putting the link in Hibern8. Otherway to
8662                  * put the link in low power mode is to send the DME end point
8663                  * to device and then send the DME reset command to local
8664                  * unipro. But putting the link in hibern8 is much faster.
8665                  *
8666                  * Note also that putting the link in Hibern8 is a requirement
8667                  * for entering DeepSleep.
8668                  */
8669                 ret = ufshcd_uic_hibern8_enter(hba);
8670                 if (ret) {
8671                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
8672                                         __func__, ret);
8673                         goto out;
8674                 }
8675                 /*
8676                  * Change controller state to "reset state" which
8677                  * should also put the link in off/reset state
8678                  */
8679                 ufshcd_hba_stop(hba);
8680                 /*
8681                  * TODO: Check if we need any delay to make sure that
8682                  * controller is reset
8683                  */
8684                 ufshcd_set_link_off(hba);
8685         }
8686
8687 out:
8688         return ret;
8689 }
8690
8691 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
8692 {
8693         bool vcc_off = false;
8694
8695         /*
8696          * It seems some UFS devices may keep drawing more than sleep current
8697          * (atleast for 500us) from UFS rails (especially from VCCQ rail).
8698          * To avoid this situation, add 2ms delay before putting these UFS
8699          * rails in LPM mode.
8700          */
8701         if (!ufshcd_is_link_active(hba) &&
8702             hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
8703                 usleep_range(2000, 2100);
8704
8705         /*
8706          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
8707          * power.
8708          *
8709          * If UFS device and link is in OFF state, all power supplies (VCC,
8710          * VCCQ, VCCQ2) can be turned off if power on write protect is not
8711          * required. If UFS link is inactive (Hibern8 or OFF state) and device
8712          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
8713          *
8714          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
8715          * in low power state which would save some power.
8716          *
8717          * If Write Booster is enabled and the device needs to flush the WB
8718          * buffer OR if bkops status is urgent for WB, keep Vcc on.
8719          */
8720         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8721             !hba->dev_info.is_lu_power_on_wp) {
8722                 ufshcd_setup_vreg(hba, false);
8723                 vcc_off = true;
8724         } else if (!ufshcd_is_ufs_dev_active(hba)) {
8725                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8726                 vcc_off = true;
8727                 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
8728                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8729                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
8730                 }
8731         }
8732
8733         /*
8734          * Some UFS devices require delay after VCC power rail is turned-off.
8735          */
8736         if (vcc_off && hba->vreg_info.vcc &&
8737                 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
8738                 usleep_range(5000, 5100);
8739 }
8740
8741 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
8742 {
8743         int ret = 0;
8744
8745         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
8746             !hba->dev_info.is_lu_power_on_wp) {
8747                 ret = ufshcd_setup_vreg(hba, true);
8748         } else if (!ufshcd_is_ufs_dev_active(hba)) {
8749                 if (!ufshcd_is_link_active(hba)) {
8750                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
8751                         if (ret)
8752                                 goto vcc_disable;
8753                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
8754                         if (ret)
8755                                 goto vccq_lpm;
8756                 }
8757                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
8758         }
8759         goto out;
8760
8761 vccq_lpm:
8762         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
8763 vcc_disable:
8764         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
8765 out:
8766         return ret;
8767 }
8768
8769 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
8770 {
8771         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8772                 ufshcd_setup_hba_vreg(hba, false);
8773 }
8774
8775 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
8776 {
8777         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
8778                 ufshcd_setup_hba_vreg(hba, true);
8779 }
8780
8781 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8782 {
8783         int ret = 0;
8784         int check_for_bkops;
8785         enum ufs_pm_level pm_lvl;
8786         enum ufs_dev_pwr_mode req_dev_pwr_mode;
8787         enum uic_link_state req_link_state;
8788
8789         hba->pm_op_in_progress = true;
8790         if (pm_op != UFS_SHUTDOWN_PM) {
8791                 pm_lvl = pm_op == UFS_RUNTIME_PM ?
8792                          hba->rpm_lvl : hba->spm_lvl;
8793                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
8794                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
8795         } else {
8796                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
8797                 req_link_state = UIC_LINK_OFF_STATE;
8798         }
8799
8800         /*
8801          * If we can't transition into any of the low power modes
8802          * just gate the clocks.
8803          */
8804         ufshcd_hold(hba, false);
8805         hba->clk_gating.is_suspended = true;
8806
8807         if (ufshcd_is_clkscaling_supported(hba))
8808                 ufshcd_clk_scaling_suspend(hba, true);
8809
8810         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
8811                         req_link_state == UIC_LINK_ACTIVE_STATE) {
8812                 goto vops_suspend;
8813         }
8814
8815         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
8816             (req_link_state == hba->uic_link_state))
8817                 goto enable_scaling;
8818
8819         /* UFS device & link must be active before we enter in this function */
8820         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
8821                 ret = -EINVAL;
8822                 goto enable_scaling;
8823         }
8824
8825         if (pm_op == UFS_RUNTIME_PM) {
8826                 if (ufshcd_can_autobkops_during_suspend(hba)) {
8827                         /*
8828                          * The device is idle with no requests in the queue,
8829                          * allow background operations if bkops status shows
8830                          * that performance might be impacted.
8831                          */
8832                         ret = ufshcd_urgent_bkops(hba);
8833                         if (ret)
8834                                 goto enable_scaling;
8835                 } else {
8836                         /* make sure that auto bkops is disabled */
8837                         ufshcd_disable_auto_bkops(hba);
8838                 }
8839                 /*
8840                  * If device needs to do BKOP or WB buffer flush during
8841                  * Hibern8, keep device power mode as "active power mode"
8842                  * and VCC supply.
8843                  */
8844                 hba->dev_info.b_rpm_dev_flush_capable =
8845                         hba->auto_bkops_enabled ||
8846                         (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
8847                         ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
8848                         ufshcd_is_auto_hibern8_enabled(hba))) &&
8849                         ufshcd_wb_need_flush(hba));
8850         }
8851
8852         flush_work(&hba->eeh_work);
8853
8854         if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
8855                 if (pm_op != UFS_RUNTIME_PM)
8856                         /* ensure that bkops is disabled */
8857                         ufshcd_disable_auto_bkops(hba);
8858
8859                 if (!hba->dev_info.b_rpm_dev_flush_capable) {
8860                         ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
8861                         if (ret)
8862                                 goto enable_scaling;
8863                 }
8864         }
8865
8866         /*
8867          * In the case of DeepSleep, the device is expected to remain powered
8868          * with the link off, so do not check for bkops.
8869          */
8870         check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
8871         ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
8872         if (ret)
8873                 goto set_dev_active;
8874
8875 vops_suspend:
8876         /*
8877          * Call vendor specific suspend callback. As these callbacks may access
8878          * vendor specific host controller register space call them before the
8879          * host clocks are ON.
8880          */
8881         ret = ufshcd_vops_suspend(hba, pm_op);
8882         if (ret)
8883                 goto set_link_active;
8884         goto out;
8885
8886 set_link_active:
8887         /*
8888          * Device hardware reset is required to exit DeepSleep. Also, for
8889          * DeepSleep, the link is off so host reset and restore will be done
8890          * further below.
8891          */
8892         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
8893                 ufshcd_device_reset(hba);
8894                 WARN_ON(!ufshcd_is_link_off(hba));
8895         }
8896         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
8897                 ufshcd_set_link_active(hba);
8898         else if (ufshcd_is_link_off(hba))
8899                 ufshcd_host_reset_and_restore(hba);
8900 set_dev_active:
8901         /* Can also get here needing to exit DeepSleep */
8902         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
8903                 ufshcd_device_reset(hba);
8904                 ufshcd_host_reset_and_restore(hba);
8905         }
8906         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
8907                 ufshcd_disable_auto_bkops(hba);
8908 enable_scaling:
8909         if (ufshcd_is_clkscaling_supported(hba))
8910                 ufshcd_clk_scaling_suspend(hba, false);
8911
8912         hba->dev_info.b_rpm_dev_flush_capable = false;
8913 out:
8914         if (hba->dev_info.b_rpm_dev_flush_capable) {
8915                 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
8916                         msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
8917         }
8918
8919         if (ret) {
8920                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
8921                 hba->clk_gating.is_suspended = false;
8922                 ufshcd_release(hba);
8923         }
8924         hba->pm_op_in_progress = false;
8925         return ret;
8926 }
8927
8928 #ifdef CONFIG_PM
8929 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
8930 {
8931         int ret;
8932         enum uic_link_state old_link_state = hba->uic_link_state;
8933
8934         hba->pm_op_in_progress = true;
8935
8936         /*
8937          * Call vendor specific resume callback. As these callbacks may access
8938          * vendor specific host controller register space call them when the
8939          * host clocks are ON.
8940          */
8941         ret = ufshcd_vops_resume(hba, pm_op);
8942         if (ret)
8943                 goto out;
8944
8945         /* For DeepSleep, the only supported option is to have the link off */
8946         WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
8947
8948         if (ufshcd_is_link_hibern8(hba)) {
8949                 ret = ufshcd_uic_hibern8_exit(hba);
8950                 if (!ret) {
8951                         ufshcd_set_link_active(hba);
8952                 } else {
8953                         dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
8954                                         __func__, ret);
8955                         goto vendor_suspend;
8956                 }
8957         } else if (ufshcd_is_link_off(hba)) {
8958                 /*
8959                  * A full initialization of the host and the device is
8960                  * required since the link was put to off during suspend.
8961                  * Note, in the case of DeepSleep, the device will exit
8962                  * DeepSleep due to device reset.
8963                  */
8964                 ret = ufshcd_reset_and_restore(hba);
8965                 /*
8966                  * ufshcd_reset_and_restore() should have already
8967                  * set the link state as active
8968                  */
8969                 if (ret || !ufshcd_is_link_active(hba))
8970                         goto vendor_suspend;
8971         }
8972
8973         if (!ufshcd_is_ufs_dev_active(hba)) {
8974                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
8975                 if (ret)
8976                         goto set_old_link_state;
8977         }
8978
8979         if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
8980                 ufshcd_enable_auto_bkops(hba);
8981         else
8982                 /*
8983                  * If BKOPs operations are urgently needed at this moment then
8984                  * keep auto-bkops enabled or else disable it.
8985                  */
8986                 ufshcd_urgent_bkops(hba);
8987
8988         if (hba->ee_usr_mask)
8989                 ufshcd_write_ee_control(hba);
8990
8991         if (ufshcd_is_clkscaling_supported(hba))
8992                 ufshcd_clk_scaling_suspend(hba, false);
8993
8994         if (hba->dev_info.b_rpm_dev_flush_capable) {
8995                 hba->dev_info.b_rpm_dev_flush_capable = false;
8996                 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
8997         }
8998
8999         /* Enable Auto-Hibernate if configured */
9000         ufshcd_auto_hibern8_enable(hba);
9001         goto out;
9002
9003 set_old_link_state:
9004         ufshcd_link_state_transition(hba, old_link_state, 0);
9005 vendor_suspend:
9006         ufshcd_vops_suspend(hba, pm_op);
9007 out:
9008         if (ret)
9009                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9010         hba->clk_gating.is_suspended = false;
9011         ufshcd_release(hba);
9012         hba->pm_op_in_progress = false;
9013         return ret;
9014 }
9015
9016 static int ufshcd_wl_runtime_suspend(struct device *dev)
9017 {
9018         struct scsi_device *sdev = to_scsi_device(dev);
9019         struct ufs_hba *hba;
9020         int ret;
9021         ktime_t start = ktime_get();
9022
9023         hba = shost_priv(sdev->host);
9024
9025         ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9026         if (ret)
9027                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9028
9029         trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9030                 ktime_to_us(ktime_sub(ktime_get(), start)),
9031                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9032
9033         return ret;
9034 }
9035
9036 static int ufshcd_wl_runtime_resume(struct device *dev)
9037 {
9038         struct scsi_device *sdev = to_scsi_device(dev);
9039         struct ufs_hba *hba;
9040         int ret = 0;
9041         ktime_t start = ktime_get();
9042
9043         hba = shost_priv(sdev->host);
9044
9045         ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9046         if (ret)
9047                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9048
9049         trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9050                 ktime_to_us(ktime_sub(ktime_get(), start)),
9051                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9052
9053         return ret;
9054 }
9055 #endif
9056
9057 #ifdef CONFIG_PM_SLEEP
9058 static int ufshcd_wl_suspend(struct device *dev)
9059 {
9060         struct scsi_device *sdev = to_scsi_device(dev);
9061         struct ufs_hba *hba;
9062         int ret = 0;
9063         ktime_t start = ktime_get();
9064
9065         hba = shost_priv(sdev->host);
9066         down(&hba->host_sem);
9067
9068         if (pm_runtime_suspended(dev))
9069                 goto out;
9070
9071         ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9072         if (ret) {
9073                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__,  ret);
9074                 up(&hba->host_sem);
9075         }
9076
9077 out:
9078         if (!ret)
9079                 hba->is_sys_suspended = true;
9080         trace_ufshcd_wl_suspend(dev_name(dev), ret,
9081                 ktime_to_us(ktime_sub(ktime_get(), start)),
9082                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9083
9084         return ret;
9085 }
9086
9087 static int ufshcd_wl_resume(struct device *dev)
9088 {
9089         struct scsi_device *sdev = to_scsi_device(dev);
9090         struct ufs_hba *hba;
9091         int ret = 0;
9092         ktime_t start = ktime_get();
9093
9094         hba = shost_priv(sdev->host);
9095
9096         if (pm_runtime_suspended(dev))
9097                 goto out;
9098
9099         ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9100         if (ret)
9101                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9102 out:
9103         trace_ufshcd_wl_resume(dev_name(dev), ret,
9104                 ktime_to_us(ktime_sub(ktime_get(), start)),
9105                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9106         if (!ret)
9107                 hba->is_sys_suspended = false;
9108         up(&hba->host_sem);
9109         return ret;
9110 }
9111 #endif
9112
9113 static void ufshcd_wl_shutdown(struct device *dev)
9114 {
9115         struct scsi_device *sdev = to_scsi_device(dev);
9116         struct ufs_hba *hba;
9117
9118         hba = shost_priv(sdev->host);
9119
9120         down(&hba->host_sem);
9121         hba->shutting_down = true;
9122         up(&hba->host_sem);
9123
9124         /* Turn on everything while shutting down */
9125         ufshcd_rpm_get_sync(hba);
9126         scsi_device_quiesce(sdev);
9127         shost_for_each_device(sdev, hba->host) {
9128                 if (sdev == hba->sdev_ufs_device)
9129                         continue;
9130                 scsi_device_quiesce(sdev);
9131         }
9132         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9133 }
9134
9135 /**
9136  * ufshcd_suspend - helper function for suspend operations
9137  * @hba: per adapter instance
9138  *
9139  * This function will put disable irqs, turn off clocks
9140  * and set vreg and hba-vreg in lpm mode.
9141  */
9142 static int ufshcd_suspend(struct ufs_hba *hba)
9143 {
9144         int ret;
9145
9146         if (!hba->is_powered)
9147                 return 0;
9148         /*
9149          * Disable the host irq as host controller as there won't be any
9150          * host controller transaction expected till resume.
9151          */
9152         ufshcd_disable_irq(hba);
9153         ret = ufshcd_setup_clocks(hba, false);
9154         if (ret) {
9155                 ufshcd_enable_irq(hba);
9156                 return ret;
9157         }
9158         if (ufshcd_is_clkgating_allowed(hba)) {
9159                 hba->clk_gating.state = CLKS_OFF;
9160                 trace_ufshcd_clk_gating(dev_name(hba->dev),
9161                                         hba->clk_gating.state);
9162         }
9163
9164         ufshcd_vreg_set_lpm(hba);
9165         /* Put the host controller in low power mode if possible */
9166         ufshcd_hba_vreg_set_lpm(hba);
9167         return ret;
9168 }
9169
9170 /**
9171  * ufshcd_resume - helper function for resume operations
9172  * @hba: per adapter instance
9173  *
9174  * This function basically turns on the regulators, clocks and
9175  * irqs of the hba.
9176  *
9177  * Returns 0 for success and non-zero for failure
9178  */
9179 static int ufshcd_resume(struct ufs_hba *hba)
9180 {
9181         int ret;
9182
9183         if (!hba->is_powered)
9184                 return 0;
9185
9186         ufshcd_hba_vreg_set_hpm(hba);
9187         ret = ufshcd_vreg_set_hpm(hba);
9188         if (ret)
9189                 goto out;
9190
9191         /* Make sure clocks are enabled before accessing controller */
9192         ret = ufshcd_setup_clocks(hba, true);
9193         if (ret)
9194                 goto disable_vreg;
9195
9196         /* enable the host irq as host controller would be active soon */
9197         ufshcd_enable_irq(hba);
9198         goto out;
9199
9200 disable_vreg:
9201         ufshcd_vreg_set_lpm(hba);
9202 out:
9203         if (ret)
9204                 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
9205         return ret;
9206 }
9207
9208 /**
9209  * ufshcd_system_suspend - system suspend routine
9210  * @hba: per adapter instance
9211  *
9212  * Check the description of ufshcd_suspend() function for more details.
9213  *
9214  * Returns 0 for success and non-zero for failure
9215  */
9216 int ufshcd_system_suspend(struct ufs_hba *hba)
9217 {
9218         int ret = 0;
9219         ktime_t start = ktime_get();
9220
9221         if (pm_runtime_suspended(hba->dev))
9222                 goto out;
9223
9224         ret = ufshcd_suspend(hba);
9225 out:
9226         trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
9227                 ktime_to_us(ktime_sub(ktime_get(), start)),
9228                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9229         return ret;
9230 }
9231 EXPORT_SYMBOL(ufshcd_system_suspend);
9232
9233 /**
9234  * ufshcd_system_resume - system resume routine
9235  * @hba: per adapter instance
9236  *
9237  * Returns 0 for success and non-zero for failure
9238  */
9239
9240 int ufshcd_system_resume(struct ufs_hba *hba)
9241 {
9242         int ret = 0;
9243         ktime_t start = ktime_get();
9244
9245         if (pm_runtime_suspended(hba->dev))
9246                 goto out;
9247
9248         ret = ufshcd_resume(hba);
9249
9250 out:
9251         trace_ufshcd_system_resume(dev_name(hba->dev), ret,
9252                 ktime_to_us(ktime_sub(ktime_get(), start)),
9253                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9254
9255         return ret;
9256 }
9257 EXPORT_SYMBOL(ufshcd_system_resume);
9258
9259 /**
9260  * ufshcd_runtime_suspend - runtime suspend routine
9261  * @hba: per adapter instance
9262  *
9263  * Check the description of ufshcd_suspend() function for more details.
9264  *
9265  * Returns 0 for success and non-zero for failure
9266  */
9267 int ufshcd_runtime_suspend(struct ufs_hba *hba)
9268 {
9269         int ret;
9270         ktime_t start = ktime_get();
9271
9272         ret = ufshcd_suspend(hba);
9273
9274         trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
9275                 ktime_to_us(ktime_sub(ktime_get(), start)),
9276                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9277         return ret;
9278 }
9279 EXPORT_SYMBOL(ufshcd_runtime_suspend);
9280
9281 /**
9282  * ufshcd_runtime_resume - runtime resume routine
9283  * @hba: per adapter instance
9284  *
9285  * This function basically brings controller
9286  * to active state. Following operations are done in this function:
9287  *
9288  * 1. Turn on all the controller related clocks
9289  * 2. Turn ON VCC rail
9290  */
9291 int ufshcd_runtime_resume(struct ufs_hba *hba)
9292 {
9293         int ret;
9294         ktime_t start = ktime_get();
9295
9296         ret = ufshcd_resume(hba);
9297
9298         trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
9299                 ktime_to_us(ktime_sub(ktime_get(), start)),
9300                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9301         return ret;
9302 }
9303 EXPORT_SYMBOL(ufshcd_runtime_resume);
9304
9305 int ufshcd_runtime_idle(struct ufs_hba *hba)
9306 {
9307         return 0;
9308 }
9309 EXPORT_SYMBOL(ufshcd_runtime_idle);
9310
9311 /**
9312  * ufshcd_shutdown - shutdown routine
9313  * @hba: per adapter instance
9314  *
9315  * This function would turn off both UFS device and UFS hba
9316  * regulators. It would also disable clocks.
9317  *
9318  * Returns 0 always to allow force shutdown even in case of errors.
9319  */
9320 int ufshcd_shutdown(struct ufs_hba *hba)
9321 {
9322         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
9323                 goto out;
9324
9325         pm_runtime_get_sync(hba->dev);
9326
9327         ufshcd_suspend(hba);
9328 out:
9329         hba->is_powered = false;
9330         /* allow force shutdown even in case of errors */
9331         return 0;
9332 }
9333 EXPORT_SYMBOL(ufshcd_shutdown);
9334
9335 /**
9336  * ufshcd_remove - de-allocate SCSI host and host memory space
9337  *              data structure memory
9338  * @hba: per adapter instance
9339  */
9340 void ufshcd_remove(struct ufs_hba *hba)
9341 {
9342         if (hba->sdev_ufs_device)
9343                 ufshcd_rpm_get_sync(hba);
9344         ufs_bsg_remove(hba);
9345         ufs_sysfs_remove_nodes(hba->dev);
9346         blk_cleanup_queue(hba->tmf_queue);
9347         blk_mq_free_tag_set(&hba->tmf_tag_set);
9348         blk_cleanup_queue(hba->cmd_queue);
9349         scsi_remove_host(hba->host);
9350         /* disable interrupts */
9351         ufshcd_disable_intr(hba, hba->intr_mask);
9352         ufshcd_hba_stop(hba);
9353         ufshcd_hba_exit(hba);
9354 }
9355 EXPORT_SYMBOL_GPL(ufshcd_remove);
9356
9357 /**
9358  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
9359  * @hba: pointer to Host Bus Adapter (HBA)
9360  */
9361 void ufshcd_dealloc_host(struct ufs_hba *hba)
9362 {
9363         scsi_host_put(hba->host);
9364 }
9365 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
9366
9367 /**
9368  * ufshcd_set_dma_mask - Set dma mask based on the controller
9369  *                       addressing capability
9370  * @hba: per adapter instance
9371  *
9372  * Returns 0 for success, non-zero for failure
9373  */
9374 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
9375 {
9376         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
9377                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
9378                         return 0;
9379         }
9380         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
9381 }
9382
9383 /**
9384  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
9385  * @dev: pointer to device handle
9386  * @hba_handle: driver private handle
9387  * Returns 0 on success, non-zero value on failure
9388  */
9389 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
9390 {
9391         struct Scsi_Host *host;
9392         struct ufs_hba *hba;
9393         int err = 0;
9394
9395         if (!dev) {
9396                 dev_err(dev,
9397                 "Invalid memory reference for dev is NULL\n");
9398                 err = -ENODEV;
9399                 goto out_error;
9400         }
9401
9402         host = scsi_host_alloc(&ufshcd_driver_template,
9403                                 sizeof(struct ufs_hba));
9404         if (!host) {
9405                 dev_err(dev, "scsi_host_alloc failed\n");
9406                 err = -ENOMEM;
9407                 goto out_error;
9408         }
9409         hba = shost_priv(host);
9410         hba->host = host;
9411         hba->dev = dev;
9412         *hba_handle = hba;
9413         hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
9414
9415         INIT_LIST_HEAD(&hba->clk_list_head);
9416
9417 out_error:
9418         return err;
9419 }
9420 EXPORT_SYMBOL(ufshcd_alloc_host);
9421
9422 /* This function exists because blk_mq_alloc_tag_set() requires this. */
9423 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
9424                                      const struct blk_mq_queue_data *qd)
9425 {
9426         WARN_ON_ONCE(true);
9427         return BLK_STS_NOTSUPP;
9428 }
9429
9430 static const struct blk_mq_ops ufshcd_tmf_ops = {
9431         .queue_rq = ufshcd_queue_tmf,
9432 };
9433
9434 /**
9435  * ufshcd_init - Driver initialization routine
9436  * @hba: per-adapter instance
9437  * @mmio_base: base register address
9438  * @irq: Interrupt line of device
9439  * Returns 0 on success, non-zero value on failure
9440  */
9441 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
9442 {
9443         int err;
9444         struct Scsi_Host *host = hba->host;
9445         struct device *dev = hba->dev;
9446         char eh_wq_name[sizeof("ufs_eh_wq_00")];
9447
9448         if (!mmio_base) {
9449                 dev_err(hba->dev,
9450                 "Invalid memory reference for mmio_base is NULL\n");
9451                 err = -ENODEV;
9452                 goto out_error;
9453         }
9454
9455         hba->mmio_base = mmio_base;
9456         hba->irq = irq;
9457         hba->vps = &ufs_hba_vps;
9458
9459         err = ufshcd_hba_init(hba);
9460         if (err)
9461                 goto out_error;
9462
9463         /* Read capabilities registers */
9464         err = ufshcd_hba_capabilities(hba);
9465         if (err)
9466                 goto out_disable;
9467
9468         /* Get UFS version supported by the controller */
9469         hba->ufs_version = ufshcd_get_ufs_version(hba);
9470
9471         /* Get Interrupt bit mask per version */
9472         hba->intr_mask = ufshcd_get_intr_mask(hba);
9473
9474         err = ufshcd_set_dma_mask(hba);
9475         if (err) {
9476                 dev_err(hba->dev, "set dma mask failed\n");
9477                 goto out_disable;
9478         }
9479
9480         /* Allocate memory for host memory space */
9481         err = ufshcd_memory_alloc(hba);
9482         if (err) {
9483                 dev_err(hba->dev, "Memory allocation failed\n");
9484                 goto out_disable;
9485         }
9486
9487         /* Configure LRB */
9488         ufshcd_host_memory_configure(hba);
9489
9490         host->can_queue = hba->nutrs;
9491         host->cmd_per_lun = hba->nutrs;
9492         host->max_id = UFSHCD_MAX_ID;
9493         host->max_lun = UFS_MAX_LUNS;
9494         host->max_channel = UFSHCD_MAX_CHANNEL;
9495         host->unique_id = host->host_no;
9496         host->max_cmd_len = UFS_CDB_SIZE;
9497
9498         hba->max_pwr_info.is_valid = false;
9499
9500         /* Initialize work queues */
9501         snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
9502                  hba->host->host_no);
9503         hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
9504         if (!hba->eh_wq) {
9505                 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
9506                                 __func__);
9507                 err = -ENOMEM;
9508                 goto out_disable;
9509         }
9510         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
9511         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
9512
9513         sema_init(&hba->host_sem, 1);
9514
9515         /* Initialize UIC command mutex */
9516         mutex_init(&hba->uic_cmd_mutex);
9517
9518         /* Initialize mutex for device management commands */
9519         mutex_init(&hba->dev_cmd.lock);
9520
9521         /* Initialize mutex for exception event control */
9522         mutex_init(&hba->ee_ctrl_mutex);
9523
9524         init_rwsem(&hba->clk_scaling_lock);
9525
9526         ufshcd_init_clk_gating(hba);
9527
9528         ufshcd_init_clk_scaling(hba);
9529
9530         /*
9531          * In order to avoid any spurious interrupt immediately after
9532          * registering UFS controller interrupt handler, clear any pending UFS
9533          * interrupt status and disable all the UFS interrupts.
9534          */
9535         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
9536                       REG_INTERRUPT_STATUS);
9537         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
9538         /*
9539          * Make sure that UFS interrupts are disabled and any pending interrupt
9540          * status is cleared before registering UFS interrupt handler.
9541          */
9542         mb();
9543
9544         /* IRQ registration */
9545         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
9546         if (err) {
9547                 dev_err(hba->dev, "request irq failed\n");
9548                 goto out_disable;
9549         } else {
9550                 hba->is_irq_enabled = true;
9551         }
9552
9553         err = scsi_add_host(host, hba->dev);
9554         if (err) {
9555                 dev_err(hba->dev, "scsi_add_host failed\n");
9556                 goto out_disable;
9557         }
9558
9559         hba->cmd_queue = blk_mq_init_queue(&hba->host->tag_set);
9560         if (IS_ERR(hba->cmd_queue)) {
9561                 err = PTR_ERR(hba->cmd_queue);
9562                 goto out_remove_scsi_host;
9563         }
9564
9565         hba->tmf_tag_set = (struct blk_mq_tag_set) {
9566                 .nr_hw_queues   = 1,
9567                 .queue_depth    = hba->nutmrs,
9568                 .ops            = &ufshcd_tmf_ops,
9569                 .flags          = BLK_MQ_F_NO_SCHED,
9570         };
9571         err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
9572         if (err < 0)
9573                 goto free_cmd_queue;
9574         hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set);
9575         if (IS_ERR(hba->tmf_queue)) {
9576                 err = PTR_ERR(hba->tmf_queue);
9577                 goto free_tmf_tag_set;
9578         }
9579
9580         /* Reset the attached device */
9581         ufshcd_device_reset(hba);
9582
9583         ufshcd_init_crypto(hba);
9584
9585         /* Host controller enable */
9586         err = ufshcd_hba_enable(hba);
9587         if (err) {
9588                 dev_err(hba->dev, "Host controller enable failed\n");
9589                 ufshcd_print_evt_hist(hba);
9590                 ufshcd_print_host_state(hba);
9591                 goto free_tmf_queue;
9592         }
9593
9594         /*
9595          * Set the default power management level for runtime and system PM.
9596          * Default power saving mode is to keep UFS link in Hibern8 state
9597          * and UFS device in sleep state.
9598          */
9599         hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9600                                                 UFS_SLEEP_PWR_MODE,
9601                                                 UIC_LINK_HIBERN8_STATE);
9602         hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
9603                                                 UFS_SLEEP_PWR_MODE,
9604                                                 UIC_LINK_HIBERN8_STATE);
9605
9606         INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work,
9607                           ufshcd_rpm_dev_flush_recheck_work);
9608
9609         /* Set the default auto-hiberate idle timer value to 150 ms */
9610         if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
9611                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
9612                             FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
9613         }
9614
9615         /* Hold auto suspend until async scan completes */
9616         pm_runtime_get_sync(dev);
9617         atomic_set(&hba->scsi_block_reqs_cnt, 0);
9618         /*
9619          * We are assuming that device wasn't put in sleep/power-down
9620          * state exclusively during the boot stage before kernel.
9621          * This assumption helps avoid doing link startup twice during
9622          * ufshcd_probe_hba().
9623          */
9624         ufshcd_set_ufs_dev_active(hba);
9625
9626         async_schedule(ufshcd_async_scan, hba);
9627         ufs_sysfs_add_nodes(hba->dev);
9628
9629         device_enable_async_suspend(dev);
9630         return 0;
9631
9632 free_tmf_queue:
9633         blk_cleanup_queue(hba->tmf_queue);
9634 free_tmf_tag_set:
9635         blk_mq_free_tag_set(&hba->tmf_tag_set);
9636 free_cmd_queue:
9637         blk_cleanup_queue(hba->cmd_queue);
9638 out_remove_scsi_host:
9639         scsi_remove_host(hba->host);
9640 out_disable:
9641         hba->is_irq_enabled = false;
9642         ufshcd_hba_exit(hba);
9643 out_error:
9644         return err;
9645 }
9646 EXPORT_SYMBOL_GPL(ufshcd_init);
9647
9648 void ufshcd_resume_complete(struct device *dev)
9649 {
9650         struct ufs_hba *hba = dev_get_drvdata(dev);
9651
9652         if (hba->complete_put) {
9653                 ufshcd_rpm_put(hba);
9654                 hba->complete_put = false;
9655         }
9656         if (hba->rpmb_complete_put) {
9657                 ufshcd_rpmb_rpm_put(hba);
9658                 hba->rpmb_complete_put = false;
9659         }
9660 }
9661 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
9662
9663 int ufshcd_suspend_prepare(struct device *dev)
9664 {
9665         struct ufs_hba *hba = dev_get_drvdata(dev);
9666         int ret;
9667
9668         /*
9669          * SCSI assumes that runtime-pm and system-pm for scsi drivers
9670          * are same. And it doesn't wake up the device for system-suspend
9671          * if it's runtime suspended. But ufs doesn't follow that.
9672          * Refer ufshcd_resume_complete()
9673          */
9674         if (hba->sdev_ufs_device) {
9675                 ret = ufshcd_rpm_get_sync(hba);
9676                 if (ret < 0 && ret != -EACCES) {
9677                         ufshcd_rpm_put(hba);
9678                         return ret;
9679                 }
9680                 hba->complete_put = true;
9681         }
9682         if (hba->sdev_rpmb) {
9683                 ufshcd_rpmb_rpm_get_sync(hba);
9684                 hba->rpmb_complete_put = true;
9685         }
9686         return 0;
9687 }
9688 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
9689
9690 #ifdef CONFIG_PM_SLEEP
9691 static int ufshcd_wl_poweroff(struct device *dev)
9692 {
9693         struct scsi_device *sdev = to_scsi_device(dev);
9694         struct ufs_hba *hba = shost_priv(sdev->host);
9695
9696         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
9697         return 0;
9698 }
9699 #endif
9700
9701 static int ufshcd_wl_probe(struct device *dev)
9702 {
9703         struct scsi_device *sdev = to_scsi_device(dev);
9704
9705         if (!is_device_wlun(sdev))
9706                 return -ENODEV;
9707
9708         blk_pm_runtime_init(sdev->request_queue, dev);
9709         pm_runtime_set_autosuspend_delay(dev, 0);
9710         pm_runtime_allow(dev);
9711
9712         return  0;
9713 }
9714
9715 static int ufshcd_wl_remove(struct device *dev)
9716 {
9717         pm_runtime_forbid(dev);
9718         return 0;
9719 }
9720
9721 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
9722 #ifdef CONFIG_PM_SLEEP
9723         .suspend = ufshcd_wl_suspend,
9724         .resume = ufshcd_wl_resume,
9725         .freeze = ufshcd_wl_suspend,
9726         .thaw = ufshcd_wl_resume,
9727         .poweroff = ufshcd_wl_poweroff,
9728         .restore = ufshcd_wl_resume,
9729 #endif
9730         SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
9731 };
9732
9733 /*
9734  * ufs_dev_wlun_template - describes ufs device wlun
9735  * ufs-device wlun - used to send pm commands
9736  * All luns are consumers of ufs-device wlun.
9737  *
9738  * Currently, no sd driver is present for wluns.
9739  * Hence the no specific pm operations are performed.
9740  * With ufs design, SSU should be sent to ufs-device wlun.
9741  * Hence register a scsi driver for ufs wluns only.
9742  */
9743 static struct scsi_driver ufs_dev_wlun_template = {
9744         .gendrv = {
9745                 .name = "ufs_device_wlun",
9746                 .owner = THIS_MODULE,
9747                 .probe = ufshcd_wl_probe,
9748                 .remove = ufshcd_wl_remove,
9749                 .pm = &ufshcd_wl_pm_ops,
9750                 .shutdown = ufshcd_wl_shutdown,
9751         },
9752 };
9753
9754 static int ufshcd_rpmb_probe(struct device *dev)
9755 {
9756         return is_rpmb_wlun(to_scsi_device(dev)) ? 0 : -ENODEV;
9757 }
9758
9759 static inline int ufshcd_clear_rpmb_uac(struct ufs_hba *hba)
9760 {
9761         int ret = 0;
9762
9763         if (!hba->wlun_rpmb_clr_ua)
9764                 return 0;
9765         ret = ufshcd_clear_ua_wlun(hba, UFS_UPIU_RPMB_WLUN);
9766         if (!ret)
9767                 hba->wlun_rpmb_clr_ua = 0;
9768         return ret;
9769 }
9770
9771 #ifdef CONFIG_PM
9772 static int ufshcd_rpmb_resume(struct device *dev)
9773 {
9774         struct ufs_hba *hba = wlun_dev_to_hba(dev);
9775
9776         if (hba->sdev_rpmb)
9777                 ufshcd_clear_rpmb_uac(hba);
9778         return 0;
9779 }
9780 #endif
9781
9782 static const struct dev_pm_ops ufs_rpmb_pm_ops = {
9783         SET_RUNTIME_PM_OPS(NULL, ufshcd_rpmb_resume, NULL)
9784         SET_SYSTEM_SLEEP_PM_OPS(NULL, ufshcd_rpmb_resume)
9785 };
9786
9787 /* ufs_rpmb_wlun_template - Describes UFS RPMB WLUN. Used only to send UAC. */
9788 static struct scsi_driver ufs_rpmb_wlun_template = {
9789         .gendrv = {
9790                 .name = "ufs_rpmb_wlun",
9791                 .owner = THIS_MODULE,
9792                 .probe = ufshcd_rpmb_probe,
9793                 .pm = &ufs_rpmb_pm_ops,
9794         },
9795 };
9796
9797 static int __init ufshcd_core_init(void)
9798 {
9799         int ret;
9800
9801         ufs_debugfs_init();
9802
9803         ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
9804         if (ret)
9805                 goto debugfs_exit;
9806
9807         ret = scsi_register_driver(&ufs_rpmb_wlun_template.gendrv);
9808         if (ret)
9809                 goto unregister;
9810
9811         return ret;
9812 unregister:
9813         scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
9814 debugfs_exit:
9815         ufs_debugfs_exit();
9816         return ret;
9817 }
9818
9819 static void __exit ufshcd_core_exit(void)
9820 {
9821         ufs_debugfs_exit();
9822         scsi_unregister_driver(&ufs_rpmb_wlun_template.gendrv);
9823         scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
9824 }
9825
9826 module_init(ufshcd_core_init);
9827 module_exit(ufshcd_core_exit);
9828
9829 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
9830 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
9831 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
9832 MODULE_LICENSE("GPL");
9833 MODULE_VERSION(UFSHCD_DRIVER_VERSION);