1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Universal Flash Storage Host controller driver
4 * Copyright (C) 2011-2013 Samsung India Software Operations
5 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
15 #include <linux/module.h>
16 #include <linux/kernel.h>
17 #include <linux/init.h>
18 #include <linux/interrupt.h>
20 #include <linux/delay.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/rwsem.h>
24 #include <linux/workqueue.h>
25 #include <linux/errno.h>
26 #include <linux/types.h>
27 #include <linux/wait.h>
28 #include <linux/bitops.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/clk.h>
31 #include <linux/completion.h>
32 #include <linux/regulator/consumer.h>
33 #include <linux/bitfield.h>
34 #include <linux/devfreq.h>
35 #include <linux/keyslot-manager.h>
39 #include <asm/byteorder.h>
40 #include <scsi/scsi.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <scsi/scsi_host.h>
43 #include <scsi/scsi_tcq.h>
44 #include <scsi/scsi_dbg.h>
45 #include <scsi/scsi_eh.h>
48 #include "ufs_quirks.h"
51 #define UFSHCD "ufshcd"
52 #define UFSHCD_DRIVER_VERSION "0.2"
57 DEV_CMD_TYPE_NOP = 0x0,
58 DEV_CMD_TYPE_QUERY = 0x1,
62 /* uic specific errors */
70 UFS_EVT_AUTO_HIBERN8_ERR,
72 UFS_EVT_LINK_STARTUP_FAIL,
85 * struct uic_command - UIC command structure
86 * @command: UIC command
87 * @argument1: UIC command argument 1
88 * @argument2: UIC command argument 2
89 * @argument3: UIC command argument 3
90 * @cmd_active: Indicate if UIC command is outstanding
91 * @done: UIC command completion
99 struct completion done;
102 /* Used to differentiate the power management options */
109 #define ufshcd_is_runtime_pm(op) ((op) == UFS_RUNTIME_PM)
110 #define ufshcd_is_system_pm(op) ((op) == UFS_SYSTEM_PM)
111 #define ufshcd_is_shutdown_pm(op) ((op) == UFS_SHUTDOWN_PM)
113 /* Host <-> Device UniPro Link state */
114 enum uic_link_state {
115 UIC_LINK_OFF_STATE = 0, /* Link powered down or disabled */
116 UIC_LINK_ACTIVE_STATE = 1, /* Link is in Fast/Slow/Sleep state */
117 UIC_LINK_HIBERN8_STATE = 2, /* Link is in Hibernate state */
118 UIC_LINK_BROKEN_STATE = 3, /* Link is in broken state */
121 #define ufshcd_is_link_off(hba) ((hba)->uic_link_state == UIC_LINK_OFF_STATE)
122 #define ufshcd_is_link_active(hba) ((hba)->uic_link_state == \
123 UIC_LINK_ACTIVE_STATE)
124 #define ufshcd_is_link_hibern8(hba) ((hba)->uic_link_state == \
125 UIC_LINK_HIBERN8_STATE)
126 #define ufshcd_is_link_broken(hba) ((hba)->uic_link_state == \
127 UIC_LINK_BROKEN_STATE)
128 #define ufshcd_set_link_off(hba) ((hba)->uic_link_state = UIC_LINK_OFF_STATE)
129 #define ufshcd_set_link_active(hba) ((hba)->uic_link_state = \
130 UIC_LINK_ACTIVE_STATE)
131 #define ufshcd_set_link_hibern8(hba) ((hba)->uic_link_state = \
132 UIC_LINK_HIBERN8_STATE)
133 #define ufshcd_set_link_broken(hba) ((hba)->uic_link_state = \
134 UIC_LINK_BROKEN_STATE)
136 #define ufshcd_set_ufs_dev_active(h) \
137 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
138 #define ufshcd_set_ufs_dev_sleep(h) \
139 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
140 #define ufshcd_set_ufs_dev_poweroff(h) \
141 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
142 #define ufshcd_set_ufs_dev_deepsleep(h) \
143 ((h)->curr_dev_pwr_mode = UFS_DEEPSLEEP_PWR_MODE)
144 #define ufshcd_is_ufs_dev_active(h) \
145 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
146 #define ufshcd_is_ufs_dev_sleep(h) \
147 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
148 #define ufshcd_is_ufs_dev_poweroff(h) \
149 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
150 #define ufshcd_is_ufs_dev_deepsleep(h) \
151 ((h)->curr_dev_pwr_mode == UFS_DEEPSLEEP_PWR_MODE)
154 * UFS Power management levels.
155 * Each level is in increasing order of power savings, except DeepSleep
156 * which is lower than PowerDown with power on but not PowerDown with
160 UFS_PM_LVL_0, /* UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE */
161 UFS_PM_LVL_1, /* UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE */
162 UFS_PM_LVL_2, /* UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE */
163 UFS_PM_LVL_3, /* UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE */
164 UFS_PM_LVL_4, /* UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE */
165 UFS_PM_LVL_5, /* UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE */
166 UFS_PM_LVL_6, /* UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE */
170 struct ufs_pm_lvl_states {
171 enum ufs_dev_pwr_mode dev_state;
172 enum uic_link_state link_state;
176 * struct ufshcd_lrb - local reference block
177 * @utr_descriptor_ptr: UTRD address of the command
178 * @ucd_req_ptr: UCD address of the command
179 * @ucd_rsp_ptr: Response UPIU address for this command
180 * @ucd_prdt_ptr: PRDT address of the command
181 * @utrd_dma_addr: UTRD dma address for debug
182 * @ucd_prdt_dma_addr: PRDT dma address for debug
183 * @ucd_rsp_dma_addr: UPIU response dma address for debug
184 * @ucd_req_dma_addr: UPIU request dma address for debug
185 * @cmd: pointer to SCSI command
186 * @sense_buffer: pointer to sense buffer address of the SCSI command
187 * @sense_bufflen: Length of the sense buffer
188 * @scsi_status: SCSI status of the command
189 * @command_type: SCSI, UFS, Query.
190 * @task_tag: Task tag of the command
191 * @lun: LUN of the command
192 * @intr_cmd: Interrupt command (doesn't participate in interrupt aggregation)
193 * @issue_time_stamp: time stamp for debug purposes
194 * @compl_time_stamp: time stamp for statistics
195 * @crypto_key_slot: the key slot to use for inline crypto (-1 if none)
196 * @data_unit_num: the data unit number for the first block for inline crypto
197 * @req_abort_skip: skip request abort task flag
198 * @in_use: indicates that this lrb is still in use
201 struct utp_transfer_req_desc *utr_descriptor_ptr;
202 struct utp_upiu_req *ucd_req_ptr;
203 struct utp_upiu_rsp *ucd_rsp_ptr;
204 struct ufshcd_sg_entry *ucd_prdt_ptr;
206 dma_addr_t utrd_dma_addr;
207 dma_addr_t ucd_req_dma_addr;
208 dma_addr_t ucd_rsp_dma_addr;
209 dma_addr_t ucd_prdt_dma_addr;
211 struct scsi_cmnd *cmd;
213 unsigned int sense_bufflen;
218 u8 lun; /* UPIU LUN id field is only 8-bit wide */
220 ktime_t issue_time_stamp;
221 ktime_t compl_time_stamp;
222 #ifdef CONFIG_SCSI_UFS_CRYPTO
232 * struct ufs_query - holds relevant data structures for query request
233 * @request: request upiu and function
234 * @descriptor: buffer for sending/receiving descriptor
235 * @response: response upiu and response
238 struct ufs_query_req request;
240 struct ufs_query_res response;
244 * struct ufs_dev_cmd - all assosiated fields with device management commands
245 * @type: device management command type - Query, NOP OUT
246 * @lock: lock to allow one command at a time
247 * @complete: internal commands completion
250 enum dev_cmd_type type;
252 struct completion *complete;
253 struct ufs_query query;
257 * struct ufs_clk_info - UFS clock related info
258 * @list: list headed by hba->clk_list_head
261 * @max_freq: maximum frequency supported by the clock
262 * @min_freq: min frequency that can be used for clock scaling
263 * @curr_freq: indicates the current frequency that it is set to
264 * @keep_link_active: indicates that the clk should not be disabled if
266 * @enabled: variable to check against multiple enable/disable
268 struct ufs_clk_info {
269 struct list_head list;
275 bool keep_link_active;
279 enum ufs_notify_change_status {
284 struct ufs_pa_layer_attr {
294 struct ufs_pwr_mode_info {
296 struct ufs_pa_layer_attr info;
300 * struct ufs_hba_variant_ops - variant specific callbacks
301 * @name: variant name
302 * @init: called when the driver is initialized
303 * @exit: called to cleanup everything done in init
304 * @get_ufs_hci_version: called to get UFS HCI version
305 * @clk_scale_notify: notifies that clks are scaled up/down
306 * @setup_clocks: called before touching any of the controller registers
307 * @hce_enable_notify: called before and after HCE enable bit is set to allow
308 * variant specific Uni-Pro initialization.
309 * @link_startup_notify: called before and after Link startup is carried out
310 * to allow variant specific Uni-Pro initialization.
311 * @pwr_change_notify: called before and after a power mode change
312 * is carried out to allow vendor spesific capabilities
314 * @setup_xfer_req: called before any transfer request is issued
316 * @setup_task_mgmt: called before any task management request is issued
318 * @hibern8_notify: called around hibern8 enter/exit
319 * @apply_dev_quirks: called to apply device specific quirks
320 * @suspend: called during host controller PM callback
321 * @resume: called during host controller PM callback
322 * @dbg_register_dump: used to dump controller debug information
323 * @phy_initialization: used to initialize phys
324 * @device_reset: called to issue a reset pulse on the UFS device
325 * @program_key: program or evict an inline encryption key
326 * @event_notify: called to notify important events
328 struct ufs_hba_variant_ops {
330 int (*init)(struct ufs_hba *);
331 void (*exit)(struct ufs_hba *);
332 u32 (*get_ufs_hci_version)(struct ufs_hba *);
333 int (*clk_scale_notify)(struct ufs_hba *, bool,
334 enum ufs_notify_change_status);
335 int (*setup_clocks)(struct ufs_hba *, bool,
336 enum ufs_notify_change_status);
337 int (*hce_enable_notify)(struct ufs_hba *,
338 enum ufs_notify_change_status);
339 int (*link_startup_notify)(struct ufs_hba *,
340 enum ufs_notify_change_status);
341 int (*pwr_change_notify)(struct ufs_hba *,
342 enum ufs_notify_change_status status,
343 struct ufs_pa_layer_attr *,
344 struct ufs_pa_layer_attr *);
345 void (*setup_xfer_req)(struct ufs_hba *, int, bool);
346 void (*setup_task_mgmt)(struct ufs_hba *, int, u8);
347 void (*hibern8_notify)(struct ufs_hba *, enum uic_cmd_dme,
348 enum ufs_notify_change_status);
349 int (*apply_dev_quirks)(struct ufs_hba *hba);
350 void (*fixup_dev_quirks)(struct ufs_hba *hba);
351 int (*suspend)(struct ufs_hba *, enum ufs_pm_op);
352 int (*resume)(struct ufs_hba *, enum ufs_pm_op);
353 void (*dbg_register_dump)(struct ufs_hba *hba);
354 int (*phy_initialization)(struct ufs_hba *);
355 int (*device_reset)(struct ufs_hba *hba);
356 void (*config_scaling_param)(struct ufs_hba *hba,
357 struct devfreq_dev_profile *profile,
359 int (*program_key)(struct ufs_hba *hba,
360 const union ufs_crypto_cfg_entry *cfg, int slot);
361 void (*event_notify)(struct ufs_hba *hba,
362 enum ufs_event_type evt, void *data);
365 /* clock gating state */
366 enum clk_gating_state {
374 * struct ufs_clk_gating - UFS clock gating related info
375 * @gate_work: worker to turn off clocks after some delay as specified in
377 * @ungate_work: worker to turn on clocks that will be used in case of
379 * @state: the current clocks state
380 * @delay_ms: gating delay in ms
381 * @is_suspended: clk gating is suspended when set to 1 which can be used
382 * during suspend/resume
383 * @delay_attr: sysfs attribute to control delay_attr
384 * @enable_attr: sysfs attribute to enable/disable clock gating
385 * @is_enabled: Indicates the current status of clock gating
386 * @active_reqs: number of requests that are pending and should be waited for
387 * completion before gating clocks.
389 struct ufs_clk_gating {
390 struct delayed_work gate_work;
391 struct work_struct ungate_work;
392 enum clk_gating_state state;
393 unsigned long delay_ms;
395 struct device_attribute delay_attr;
396 struct device_attribute enable_attr;
399 struct workqueue_struct *clk_gating_workq;
402 struct ufs_saved_pwr_info {
403 struct ufs_pa_layer_attr info;
408 * struct ufs_clk_scaling - UFS clock scaling related data
409 * @active_reqs: number of requests that are pending. If this is zero when
410 * devfreq ->target() function is called then schedule "suspend_work" to
412 * @tot_busy_t: Total busy time in current polling window
413 * @window_start_t: Start time (in jiffies) of the current polling window
414 * @busy_start_t: Start time of current busy period
415 * @enable_attr: sysfs attribute to enable/disable clock scaling
416 * @saved_pwr_info: UFS power mode may also be changed during scaling and this
417 * one keeps track of previous power mode.
418 * @workq: workqueue to schedule devfreq suspend/resume work
419 * @suspend_work: worker to suspend devfreq
420 * @resume_work: worker to resume devfreq
421 * @min_gear: lowest HS gear to scale down to
422 * @is_allowed: tracks if scaling is currently allowed or not
423 * @is_busy_started: tracks if busy period has started or not
424 * @is_suspended: tracks if devfreq is suspended or not
426 struct ufs_clk_scaling {
428 unsigned long tot_busy_t;
429 ktime_t window_start_t;
430 ktime_t busy_start_t;
431 struct device_attribute enable_attr;
432 struct ufs_saved_pwr_info saved_pwr_info;
433 struct workqueue_struct *workq;
434 struct work_struct suspend_work;
435 struct work_struct resume_work;
438 bool is_busy_started;
442 #define UFS_EVENT_HIST_LENGTH 8
444 * struct ufs_event_hist - keeps history of errors
445 * @pos: index to indicate cyclic buffer position
446 * @reg: cyclic buffer for registers value
447 * @tstamp: cyclic buffer for time stamp
449 struct ufs_event_hist {
451 u32 val[UFS_EVENT_HIST_LENGTH];
452 ktime_t tstamp[UFS_EVENT_HIST_LENGTH];
456 * struct ufs_stats - keeps usage/err statistics
457 * @last_intr_status: record the last interrupt status.
458 * @last_intr_ts: record the last interrupt timestamp.
459 * @hibern8_exit_cnt: Counter to keep track of number of exits,
460 * reset this after link-startup.
461 * @last_hibern8_exit_tstamp: Set time after the hibern8 exit.
462 * Clear after the first successful command completion.
465 u32 last_intr_status;
466 ktime_t last_intr_ts;
468 u32 hibern8_exit_cnt;
469 ktime_t last_hibern8_exit_tstamp;
470 struct ufs_event_hist event[UFS_EVT_CNT];
474 /* Interrupt aggregation support is broken */
475 UFSHCD_QUIRK_BROKEN_INTR_AGGR = 1 << 0,
478 * delay before each dme command is required as the unipro
479 * layer has shown instabilities
481 UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS = 1 << 1,
484 * If UFS host controller is having issue in processing LCC (Line
485 * Control Command) coming from device then enable this quirk.
486 * When this quirk is enabled, host controller driver should disable
487 * the LCC transmission on UFS device (by clearing TX_LCC_ENABLE
488 * attribute of device to 0).
490 UFSHCD_QUIRK_BROKEN_LCC = 1 << 2,
493 * The attribute PA_RXHSUNTERMCAP specifies whether or not the
494 * inbound Link supports unterminated line in HS mode. Setting this
495 * attribute to 1 fixes moving to HS gear.
497 UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP = 1 << 3,
500 * This quirk needs to be enabled if the host controller only allows
501 * accessing the peer dme attributes in AUTO mode (FAST AUTO or
504 UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE = 1 << 4,
507 * This quirk needs to be enabled if the host controller doesn't
508 * advertise the correct version in UFS_VER register. If this quirk
509 * is enabled, standard UFS host driver will call the vendor specific
510 * ops (get_ufs_hci_version) to get the correct version.
512 UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION = 1 << 5,
515 * Clear handling for transfer/task request list is just opposite.
517 UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR = 1 << 6,
520 * This quirk needs to be enabled if host controller doesn't allow
521 * that the interrupt aggregation timer and counter are reset by s/w.
523 UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR = 1 << 7,
526 * This quirks needs to be enabled if host controller cannot be
527 * enabled via HCE register.
529 UFSHCI_QUIRK_BROKEN_HCE = 1 << 8,
532 * This quirk needs to be enabled if the host controller regards
533 * resolution of the values of PRDTO and PRDTL in UTRD as byte.
535 UFSHCD_QUIRK_PRDT_BYTE_GRAN = 1 << 9,
538 * This quirk needs to be enabled if the host controller reports
539 * OCS FATAL ERROR with device error through sense data
541 UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR = 1 << 10,
544 * This quirk needs to be enabled if the host controller has
545 * auto-hibernate capability but it doesn't work.
547 UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 = 1 << 11,
550 * This quirk needs to disable manual flush for write booster
552 UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL = 1 << 12,
557 /* Allow dynamic clk gating */
558 UFSHCD_CAP_CLK_GATING = 1 << 0,
560 /* Allow hiberb8 with clk gating */
561 UFSHCD_CAP_HIBERN8_WITH_CLK_GATING = 1 << 1,
563 /* Allow dynamic clk scaling */
564 UFSHCD_CAP_CLK_SCALING = 1 << 2,
566 /* Allow auto bkops to enabled during runtime suspend */
567 UFSHCD_CAP_AUTO_BKOPS_SUSPEND = 1 << 3,
570 * This capability allows host controller driver to use the UFS HCI's
571 * interrupt aggregation capability.
572 * CAUTION: Enabling this might reduce overall UFS throughput.
574 UFSHCD_CAP_INTR_AGGR = 1 << 4,
577 * This capability allows the device auto-bkops to be always enabled
578 * except during suspend (both runtime and suspend).
579 * Enabling this capability means that device will always be allowed
580 * to do background operation when it's active but it might degrade
581 * the performance of ongoing read/write operations.
583 UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND = 1 << 5,
586 * This capability allows host controller driver to automatically
587 * enable runtime power management by itself instead of waiting
588 * for userspace to control the power management.
590 UFSHCD_CAP_RPM_AUTOSUSPEND = 1 << 6,
593 * This capability allows the host controller driver to turn-on
594 * WriteBooster, if the underlying device supports it and is
595 * provisioned to be used. This would increase the write performance.
597 UFSHCD_CAP_WB_EN = 1 << 7,
600 * This capability allows the host controller driver to use the
601 * inline crypto engine, if it is present
603 UFSHCD_CAP_CRYPTO = 1 << 8,
606 * This capability allows the controller regulators to be put into
607 * lpm mode aggressively during clock gating.
608 * This would increase power savings.
610 UFSHCD_CAP_AGGR_POWER_COLLAPSE = 1 << 9,
613 * This capability allows the host controller driver to use DeepSleep,
614 * if it is supported by the UFS device. The host controller driver must
615 * support device hardware reset via the hba->device_reset() callback,
616 * in order to exit DeepSleep state.
618 UFSHCD_CAP_DEEPSLEEP = 1 << 10,
621 struct ufs_hba_variant_params {
622 struct devfreq_dev_profile devfreq_profile;
623 struct devfreq_simple_ondemand_data ondemand_data;
624 u16 hba_enable_delay_us;
625 u32 wb_flush_threshold;
629 * struct ufs_hba - per adapter private structure
630 * @mmio_base: UFSHCI base register address
631 * @ucdl_base_addr: UFS Command Descriptor base address
632 * @utrdl_base_addr: UTP Transfer Request Descriptor base address
633 * @utmrdl_base_addr: UTP Task Management Descriptor base address
634 * @ucdl_dma_addr: UFS Command Descriptor DMA address
635 * @utrdl_dma_addr: UTRDL DMA address
636 * @utmrdl_dma_addr: UTMRDL DMA address
637 * @host: Scsi_Host instance of the driver
638 * @dev: device handle
639 * @lrb: local reference block
640 * @cmd_queue: Used to allocate command tags from hba->host->tag_set.
641 * @outstanding_tasks: Bits representing outstanding task requests
642 * @outstanding_reqs: Bits representing outstanding transfer requests
643 * @capabilities: UFS Controller Capabilities
644 * @nutrs: Transfer Request Queue depth supported by controller
645 * @nutmrs: Task Management Queue depth supported by controller
646 * @ufs_version: UFS Version to which controller complies
647 * @vops: pointer to variant specific operations
648 * @priv: pointer to variant specific private data
649 * @irq: Irq number of the controller
650 * @active_uic_cmd: handle of active UIC command
651 * @uic_cmd_mutex: mutex for uic command
652 * @tmf_tag_set: TMF tag set.
653 * @tmf_queue: Used to allocate TMF tags.
654 * @pwr_done: completion for power mode change
655 * @ufshcd_state: UFSHCD states
656 * @eh_flags: Error handling flags
657 * @intr_mask: Interrupt Mask Bits
658 * @ee_ctrl_mask: Exception event control mask
659 * @is_powered: flag to check if HBA is powered
660 * @eh_wq: Workqueue that eh_work works on
661 * @eh_work: Worker to handle UFS errors that require s/w attention
662 * @eeh_work: Worker to handle exception events
663 * @errors: HBA errors
664 * @uic_error: UFS interconnect layer error status
665 * @saved_err: sticky error mask
666 * @saved_uic_err: sticky UIC error mask
667 * @force_reset: flag to force eh_work perform a full reset
668 * @force_pmc: flag to force a power mode change
669 * @silence_err_logs: flag to silence error logs
670 * @dev_cmd: ufs device management command information
671 * @last_dme_cmd_tstamp: time stamp of the last completed DME command
672 * @auto_bkops_enabled: to track whether bkops is enabled in device
673 * @vreg_info: UFS device voltage regulator information
674 * @clk_list_head: UFS host controller clocks list node head
675 * @pwr_info: holds current power mode
676 * @max_pwr_info: keeps the device max valid pwm
677 * @desc_size: descriptor sizes reported by device
678 * @urgent_bkops_lvl: keeps track of urgent bkops level for device
679 * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
680 * device is known or not.
681 * @scsi_block_reqs_cnt: reference counting for scsi block requests
682 * @crypto_capabilities: Content of crypto capabilities register (0x100)
683 * @crypto_cap_array: Array of crypto capabilities
684 * @crypto_cfg_register: Start of the crypto cfg array
685 * @ksm: the keyslot manager tied to this hba
688 void __iomem *mmio_base;
690 /* Virtual memory reference */
691 struct utp_transfer_cmd_desc *ucdl_base_addr;
692 struct utp_transfer_req_desc *utrdl_base_addr;
693 struct utp_task_req_desc *utmrdl_base_addr;
695 /* DMA memory reference */
696 dma_addr_t ucdl_dma_addr;
697 dma_addr_t utrdl_dma_addr;
698 dma_addr_t utmrdl_dma_addr;
700 struct Scsi_Host *host;
702 struct request_queue *cmd_queue;
704 * This field is to keep a reference to "scsi_device" corresponding to
707 struct scsi_device *sdev_ufs_device;
708 struct scsi_device *sdev_rpmb;
710 enum ufs_dev_pwr_mode curr_dev_pwr_mode;
711 enum uic_link_state uic_link_state;
712 /* Desired UFS power management level during runtime PM */
713 enum ufs_pm_level rpm_lvl;
714 /* Desired UFS power management level during system PM */
715 enum ufs_pm_level spm_lvl;
716 struct device_attribute rpm_lvl_attr;
717 struct device_attribute spm_lvl_attr;
718 int pm_op_in_progress;
720 /* Auto-Hibernate Idle Timer register value */
723 struct ufshcd_lrb *lrb;
725 unsigned long outstanding_tasks;
726 unsigned long outstanding_reqs;
732 const struct ufs_hba_variant_ops *vops;
733 struct ufs_hba_variant_params *vps;
737 enum ufs_ref_clk_freq dev_ref_clk_freq;
739 unsigned int quirks; /* Deviations from standard UFSHCI spec. */
741 /* Device deviations from standard UFS device spec. */
742 unsigned int dev_quirks;
744 struct blk_mq_tag_set tmf_tag_set;
745 struct request_queue *tmf_queue;
747 struct uic_command *active_uic_cmd;
748 struct mutex uic_cmd_mutex;
749 struct completion *uic_async_done;
756 struct semaphore eh_sem;
759 struct workqueue_struct *eh_wq;
760 struct work_struct eh_work;
761 struct work_struct eeh_work;
768 struct ufs_stats ufs_stats;
771 bool silence_err_logs;
773 /* Device management request data */
774 struct ufs_dev_cmd dev_cmd;
775 ktime_t last_dme_cmd_tstamp;
777 /* Keeps information of the UFS device connected to this host */
778 struct ufs_dev_info dev_info;
779 bool auto_bkops_enabled;
780 struct ufs_vreg_info vreg_info;
781 struct list_head clk_list_head;
783 bool wlun_dev_clr_ua;
785 /* Number of requests aborts */
788 /* Number of lanes available (1 or 2) for Rx/Tx */
789 u32 lanes_per_direction;
790 struct ufs_pa_layer_attr pwr_info;
791 struct ufs_pwr_mode_info max_pwr_info;
793 struct ufs_clk_gating clk_gating;
794 /* Control to enable/disable host capabilities */
797 struct devfreq *devfreq;
798 struct ufs_clk_scaling clk_scaling;
799 bool is_sys_suspended;
801 enum bkops_status urgent_bkops_lvl;
802 bool is_urgent_bkops_lvl_checked;
804 struct rw_semaphore clk_scaling_lock;
805 unsigned char desc_size[QUERY_DESC_IDN_MAX];
806 atomic_t scsi_block_reqs_cnt;
808 struct device bsg_dev;
809 struct request_queue *bsg_queue;
810 bool wb_buf_flush_enabled;
812 struct delayed_work rpm_dev_flush_recheck_work;
814 #ifdef CONFIG_SCSI_UFS_CRYPTO
815 union ufs_crypto_capabilities crypto_capabilities;
816 union ufs_crypto_cap_entry *crypto_cap_array;
817 u32 crypto_cfg_register;
818 struct blk_keyslot_manager ksm;
822 /* Returns true if clocks can be gated. Otherwise false */
823 static inline bool ufshcd_is_clkgating_allowed(struct ufs_hba *hba)
825 return hba->caps & UFSHCD_CAP_CLK_GATING;
827 static inline bool ufshcd_can_hibern8_during_gating(struct ufs_hba *hba)
829 return hba->caps & UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
831 static inline int ufshcd_is_clkscaling_supported(struct ufs_hba *hba)
833 return hba->caps & UFSHCD_CAP_CLK_SCALING;
835 static inline bool ufshcd_can_autobkops_during_suspend(struct ufs_hba *hba)
837 return hba->caps & UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
839 static inline bool ufshcd_is_rpm_autosuspend_allowed(struct ufs_hba *hba)
841 return hba->caps & UFSHCD_CAP_RPM_AUTOSUSPEND;
844 static inline bool ufshcd_is_intr_aggr_allowed(struct ufs_hba *hba)
846 /* DWC UFS Core has the Interrupt aggregation feature but is not detectable*/
847 #ifndef CONFIG_SCSI_UFS_DWC
848 if ((hba->caps & UFSHCD_CAP_INTR_AGGR) &&
849 !(hba->quirks & UFSHCD_QUIRK_BROKEN_INTR_AGGR))
858 static inline bool ufshcd_can_aggressive_pc(struct ufs_hba *hba)
860 return !!(ufshcd_is_link_hibern8(hba) &&
861 (hba->caps & UFSHCD_CAP_AGGR_POWER_COLLAPSE));
864 static inline bool ufshcd_is_auto_hibern8_supported(struct ufs_hba *hba)
866 return (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) &&
867 !(hba->quirks & UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8);
870 static inline bool ufshcd_is_auto_hibern8_enabled(struct ufs_hba *hba)
872 return FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, hba->ahit) ? true : false;
875 static inline bool ufshcd_is_wb_allowed(struct ufs_hba *hba)
877 return hba->caps & UFSHCD_CAP_WB_EN;
880 #define ufshcd_writel(hba, val, reg) \
881 writel((val), (hba)->mmio_base + (reg))
882 #define ufshcd_readl(hba, reg) \
883 readl((hba)->mmio_base + (reg))
886 * ufshcd_rmwl - read modify write into a register
887 * @hba - per adapter instance
888 * @mask - mask to apply on read value
889 * @val - actual value to write
890 * @reg - register address
892 static inline void ufshcd_rmwl(struct ufs_hba *hba, u32 mask, u32 val, u32 reg)
896 tmp = ufshcd_readl(hba, reg);
899 ufshcd_writel(hba, tmp, reg);
902 int ufshcd_alloc_host(struct device *, struct ufs_hba **);
903 void ufshcd_dealloc_host(struct ufs_hba *);
904 int ufshcd_hba_enable(struct ufs_hba *hba);
905 int ufshcd_init(struct ufs_hba * , void __iomem * , unsigned int);
906 int ufshcd_link_recovery(struct ufs_hba *hba);
907 int ufshcd_make_hba_operational(struct ufs_hba *hba);
908 void ufshcd_remove(struct ufs_hba *);
909 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
910 void ufshcd_delay_us(unsigned long us, unsigned long tolerance);
911 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
912 u32 val, unsigned long interval_us,
913 unsigned long timeout_ms);
914 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk);
915 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val);
917 static inline void check_upiu_size(void)
919 BUILD_BUG_ON(ALIGNED_UPIU_SIZE <
920 GENERAL_UPIU_REQUEST_SIZE + QUERY_DESC_MAX_SIZE);
924 * ufshcd_set_variant - set variant specific data to the hba
925 * @hba - per adapter instance
926 * @variant - pointer to variant specific data
928 static inline void ufshcd_set_variant(struct ufs_hba *hba, void *variant)
935 * ufshcd_get_variant - get variant specific data from the hba
936 * @hba - per adapter instance
938 static inline void *ufshcd_get_variant(struct ufs_hba *hba)
943 static inline bool ufshcd_keep_autobkops_enabled_except_suspend(
946 return hba->caps & UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND;
949 static inline u8 ufshcd_wb_get_query_index(struct ufs_hba *hba)
951 if (hba->dev_info.b_wb_buffer_type == WB_BUF_MODE_LU_DEDICATED)
952 return hba->dev_info.wb_dedicated_lu;
956 extern int ufshcd_runtime_suspend(struct ufs_hba *hba);
957 extern int ufshcd_runtime_resume(struct ufs_hba *hba);
958 extern int ufshcd_runtime_idle(struct ufs_hba *hba);
959 extern int ufshcd_system_suspend(struct ufs_hba *hba);
960 extern int ufshcd_system_resume(struct ufs_hba *hba);
961 extern int ufshcd_shutdown(struct ufs_hba *hba);
962 extern int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
965 extern int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
966 u8 attr_set, u32 mib_val, u8 peer);
967 extern int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
968 u32 *mib_val, u8 peer);
969 extern int ufshcd_config_pwr_mode(struct ufs_hba *hba,
970 struct ufs_pa_layer_attr *desired_pwr_mode);
972 /* UIC command interfaces for DME primitives */
975 #define ATTR_SET_NOR 0 /* NORMAL */
976 #define ATTR_SET_ST 1 /* STATIC */
978 static inline int ufshcd_dme_set(struct ufs_hba *hba, u32 attr_sel,
981 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_NOR,
985 static inline int ufshcd_dme_st_set(struct ufs_hba *hba, u32 attr_sel,
988 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_ST,
992 static inline int ufshcd_dme_peer_set(struct ufs_hba *hba, u32 attr_sel,
995 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_NOR,
999 static inline int ufshcd_dme_peer_st_set(struct ufs_hba *hba, u32 attr_sel,
1002 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_ST,
1006 static inline int ufshcd_dme_get(struct ufs_hba *hba,
1007 u32 attr_sel, u32 *mib_val)
1009 return ufshcd_dme_get_attr(hba, attr_sel, mib_val, DME_LOCAL);
1012 static inline int ufshcd_dme_peer_get(struct ufs_hba *hba,
1013 u32 attr_sel, u32 *mib_val)
1015 return ufshcd_dme_get_attr(hba, attr_sel, mib_val, DME_PEER);
1018 static inline bool ufshcd_is_hs_mode(struct ufs_pa_layer_attr *pwr_info)
1020 return (pwr_info->pwr_rx == FAST_MODE ||
1021 pwr_info->pwr_rx == FASTAUTO_MODE) &&
1022 (pwr_info->pwr_tx == FAST_MODE ||
1023 pwr_info->pwr_tx == FASTAUTO_MODE);
1026 static inline int ufshcd_disable_host_tx_lcc(struct ufs_hba *hba)
1028 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0);
1031 /* Expose Query-Request API */
1032 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
1033 enum query_opcode opcode,
1034 enum desc_idn idn, u8 index,
1036 u8 *desc_buf, int *buf_len);
1037 int ufshcd_read_desc_param(struct ufs_hba *hba,
1038 enum desc_idn desc_id,
1043 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
1044 enum attr_idn idn, u8 index, u8 selector, u32 *attr_val);
1045 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1046 enum flag_idn idn, u8 index, bool *flag_res);
1048 void ufshcd_auto_hibern8_enable(struct ufs_hba *hba);
1049 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit);
1050 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, struct ufs_dev_fix *fixups);
1051 #define SD_ASCII_STD true
1052 #define SD_RAW false
1053 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
1054 u8 **buf, bool ascii);
1056 int ufshcd_hold(struct ufs_hba *hba, bool async);
1057 void ufshcd_release(struct ufs_hba *hba);
1059 void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
1062 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba);
1064 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
1066 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
1067 struct utp_upiu_req *req_upiu,
1068 struct utp_upiu_req *rsp_upiu,
1070 u8 *desc_buff, int *buff_len,
1071 enum query_opcode desc_op);
1073 /* Wrapper functions for safely calling variant operations */
1074 static inline const char *ufshcd_get_var_name(struct ufs_hba *hba)
1077 return hba->vops->name;
1081 static inline int ufshcd_vops_init(struct ufs_hba *hba)
1083 if (hba->vops && hba->vops->init)
1084 return hba->vops->init(hba);
1089 static inline void ufshcd_vops_exit(struct ufs_hba *hba)
1091 if (hba->vops && hba->vops->exit)
1092 return hba->vops->exit(hba);
1095 static inline u32 ufshcd_vops_get_ufs_hci_version(struct ufs_hba *hba)
1097 if (hba->vops && hba->vops->get_ufs_hci_version)
1098 return hba->vops->get_ufs_hci_version(hba);
1100 return ufshcd_readl(hba, REG_UFS_VERSION);
1103 static inline int ufshcd_vops_clk_scale_notify(struct ufs_hba *hba,
1104 bool up, enum ufs_notify_change_status status)
1106 if (hba->vops && hba->vops->clk_scale_notify)
1107 return hba->vops->clk_scale_notify(hba, up, status);
1111 static inline void ufshcd_vops_event_notify(struct ufs_hba *hba,
1112 enum ufs_event_type evt,
1115 if (hba->vops && hba->vops->event_notify)
1116 hba->vops->event_notify(hba, evt, data);
1119 static inline int ufshcd_vops_setup_clocks(struct ufs_hba *hba, bool on,
1120 enum ufs_notify_change_status status)
1122 if (hba->vops && hba->vops->setup_clocks)
1123 return hba->vops->setup_clocks(hba, on, status);
1127 static inline int ufshcd_vops_hce_enable_notify(struct ufs_hba *hba,
1130 if (hba->vops && hba->vops->hce_enable_notify)
1131 return hba->vops->hce_enable_notify(hba, status);
1135 static inline int ufshcd_vops_link_startup_notify(struct ufs_hba *hba,
1138 if (hba->vops && hba->vops->link_startup_notify)
1139 return hba->vops->link_startup_notify(hba, status);
1144 static inline int ufshcd_vops_phy_initialization(struct ufs_hba *hba)
1146 if (hba->vops && hba->vops->phy_initialization)
1147 return hba->vops->phy_initialization(hba);
1152 static inline int ufshcd_vops_pwr_change_notify(struct ufs_hba *hba,
1154 struct ufs_pa_layer_attr *dev_max_params,
1155 struct ufs_pa_layer_attr *dev_req_params)
1157 if (hba->vops && hba->vops->pwr_change_notify)
1158 return hba->vops->pwr_change_notify(hba, status,
1159 dev_max_params, dev_req_params);
1164 static inline void ufshcd_vops_setup_xfer_req(struct ufs_hba *hba, int tag,
1167 if (hba->vops && hba->vops->setup_xfer_req)
1168 return hba->vops->setup_xfer_req(hba, tag, is_scsi_cmd);
1171 static inline void ufshcd_vops_setup_task_mgmt(struct ufs_hba *hba,
1172 int tag, u8 tm_function)
1174 if (hba->vops && hba->vops->setup_task_mgmt)
1175 return hba->vops->setup_task_mgmt(hba, tag, tm_function);
1178 static inline void ufshcd_vops_hibern8_notify(struct ufs_hba *hba,
1179 enum uic_cmd_dme cmd,
1180 enum ufs_notify_change_status status)
1182 if (hba->vops && hba->vops->hibern8_notify)
1183 return hba->vops->hibern8_notify(hba, cmd, status);
1186 static inline int ufshcd_vops_apply_dev_quirks(struct ufs_hba *hba)
1188 if (hba->vops && hba->vops->apply_dev_quirks)
1189 return hba->vops->apply_dev_quirks(hba);
1193 static inline void ufshcd_vops_fixup_dev_quirks(struct ufs_hba *hba)
1195 if (hba->vops && hba->vops->fixup_dev_quirks)
1196 hba->vops->fixup_dev_quirks(hba);
1199 static inline int ufshcd_vops_suspend(struct ufs_hba *hba, enum ufs_pm_op op)
1201 if (hba->vops && hba->vops->suspend)
1202 return hba->vops->suspend(hba, op);
1207 static inline int ufshcd_vops_resume(struct ufs_hba *hba, enum ufs_pm_op op)
1209 if (hba->vops && hba->vops->resume)
1210 return hba->vops->resume(hba, op);
1215 static inline void ufshcd_vops_dbg_register_dump(struct ufs_hba *hba)
1217 if (hba->vops && hba->vops->dbg_register_dump)
1218 hba->vops->dbg_register_dump(hba);
1221 static inline void ufshcd_vops_device_reset(struct ufs_hba *hba)
1223 if (hba->vops && hba->vops->device_reset) {
1224 int err = hba->vops->device_reset(hba);
1227 ufshcd_set_ufs_dev_active(hba);
1228 if (err != -EOPNOTSUPP)
1229 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
1233 static inline void ufshcd_vops_config_scaling_param(struct ufs_hba *hba,
1234 struct devfreq_dev_profile
1235 *profile, void *data)
1237 if (hba->vops && hba->vops->config_scaling_param)
1238 hba->vops->config_scaling_param(hba, profile, data);
1241 extern struct ufs_pm_lvl_states ufs_pm_lvl_states[];
1244 * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN
1245 * @scsi_lun: scsi LUN id
1247 * Returns UPIU LUN id
1249 static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun)
1251 if (scsi_is_wlun(scsi_lun))
1252 return (scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID)
1255 return scsi_lun & UFS_UPIU_MAX_UNIT_NUM_ID;
1258 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
1259 const char *prefix);
1261 #endif /* End of Header */