1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * MHI PCI driver - MHI over PCI controller driver
5 * This module is a generic driver for registering MHI-over-PCI devices,
6 * such as PCIe QCOM modems.
8 * Copyright (C) 2020 Linaro Ltd <loic.poulain@linaro.org>
11 #include <linux/aer.h>
12 #include <linux/delay.h>
13 #include <linux/device.h>
14 #include <linux/mhi.h>
15 #include <linux/module.h>
16 #include <linux/pci.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/timer.h>
19 #include <linux/workqueue.h>
21 #define MHI_PCI_DEFAULT_BAR_NUM 0
23 #define MHI_POST_RESET_DELAY_MS 500
25 #define HEALTH_CHECK_PERIOD (HZ * 2)
28 * struct mhi_pci_dev_info - MHI PCI device specific information
29 * @config: MHI controller configuration
30 * @name: name of the PCI module
31 * @fw: firmware path (if any)
32 * @edl: emergency download mode firmware path (if any)
33 * @bar_num: PCI base address register to use for MHI MMIO register space
34 * @dma_data_width: DMA transfer word size (32 or 64 bits)
36 struct mhi_pci_dev_info {
37 const struct mhi_controller_config *config;
42 unsigned int dma_data_width;
45 #define MHI_CHANNEL_CONFIG_UL(ch_num, ch_name, el_count, ev_ring) \
49 .num_elements = el_count, \
50 .event_ring = ev_ring, \
51 .dir = DMA_TO_DEVICE, \
52 .ee_mask = BIT(MHI_EE_AMSS), \
54 .doorbell = MHI_DB_BRST_DISABLE, \
55 .lpm_notify = false, \
56 .offload_channel = false, \
57 .doorbell_mode_switch = false, \
60 #define MHI_CHANNEL_CONFIG_DL(ch_num, ch_name, el_count, ev_ring) \
64 .num_elements = el_count, \
65 .event_ring = ev_ring, \
66 .dir = DMA_FROM_DEVICE, \
67 .ee_mask = BIT(MHI_EE_AMSS), \
69 .doorbell = MHI_DB_BRST_DISABLE, \
70 .lpm_notify = false, \
71 .offload_channel = false, \
72 .doorbell_mode_switch = false, \
75 #define MHI_EVENT_CONFIG_CTRL(ev_ring, el_count) \
77 .num_elements = el_count, \
78 .irq_moderation_ms = 0, \
79 .irq = (ev_ring) + 1, \
81 .mode = MHI_DB_BRST_DISABLE, \
82 .data_type = MHI_ER_CTRL, \
83 .hardware_event = false, \
84 .client_managed = false, \
85 .offload_channel = false, \
88 #define MHI_CHANNEL_CONFIG_HW_UL(ch_num, ch_name, el_count, ev_ring) \
92 .num_elements = el_count, \
93 .event_ring = ev_ring, \
94 .dir = DMA_TO_DEVICE, \
95 .ee_mask = BIT(MHI_EE_AMSS), \
97 .doorbell = MHI_DB_BRST_ENABLE, \
98 .lpm_notify = false, \
99 .offload_channel = false, \
100 .doorbell_mode_switch = true, \
103 #define MHI_CHANNEL_CONFIG_HW_DL(ch_num, ch_name, el_count, ev_ring) \
107 .num_elements = el_count, \
108 .event_ring = ev_ring, \
109 .dir = DMA_FROM_DEVICE, \
110 .ee_mask = BIT(MHI_EE_AMSS), \
112 .doorbell = MHI_DB_BRST_ENABLE, \
113 .lpm_notify = false, \
114 .offload_channel = false, \
115 .doorbell_mode_switch = true, \
118 #define MHI_CHANNEL_CONFIG_UL_SBL(ch_num, ch_name, el_count, ev_ring) \
122 .num_elements = el_count, \
123 .event_ring = ev_ring, \
124 .dir = DMA_TO_DEVICE, \
125 .ee_mask = BIT(MHI_EE_SBL), \
127 .doorbell = MHI_DB_BRST_DISABLE, \
128 .lpm_notify = false, \
129 .offload_channel = false, \
130 .doorbell_mode_switch = false, \
133 #define MHI_CHANNEL_CONFIG_DL_SBL(ch_num, ch_name, el_count, ev_ring) \
137 .num_elements = el_count, \
138 .event_ring = ev_ring, \
139 .dir = DMA_FROM_DEVICE, \
140 .ee_mask = BIT(MHI_EE_SBL), \
142 .doorbell = MHI_DB_BRST_DISABLE, \
143 .lpm_notify = false, \
144 .offload_channel = false, \
145 .doorbell_mode_switch = false, \
148 #define MHI_CHANNEL_CONFIG_UL_FP(ch_num, ch_name, el_count, ev_ring) \
152 .num_elements = el_count, \
153 .event_ring = ev_ring, \
154 .dir = DMA_TO_DEVICE, \
155 .ee_mask = BIT(MHI_EE_FP), \
157 .doorbell = MHI_DB_BRST_DISABLE, \
158 .lpm_notify = false, \
159 .offload_channel = false, \
160 .doorbell_mode_switch = false, \
163 #define MHI_CHANNEL_CONFIG_DL_FP(ch_num, ch_name, el_count, ev_ring) \
167 .num_elements = el_count, \
168 .event_ring = ev_ring, \
169 .dir = DMA_FROM_DEVICE, \
170 .ee_mask = BIT(MHI_EE_FP), \
172 .doorbell = MHI_DB_BRST_DISABLE, \
173 .lpm_notify = false, \
174 .offload_channel = false, \
175 .doorbell_mode_switch = false, \
178 #define MHI_EVENT_CONFIG_DATA(ev_ring, el_count) \
180 .num_elements = el_count, \
181 .irq_moderation_ms = 5, \
182 .irq = (ev_ring) + 1, \
184 .mode = MHI_DB_BRST_DISABLE, \
185 .data_type = MHI_ER_DATA, \
186 .hardware_event = false, \
187 .client_managed = false, \
188 .offload_channel = false, \
191 #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, el_count, ch_num) \
193 .num_elements = el_count, \
194 .irq_moderation_ms = 1, \
195 .irq = (ev_ring) + 1, \
197 .mode = MHI_DB_BRST_DISABLE, \
198 .data_type = MHI_ER_DATA, \
199 .hardware_event = true, \
200 .client_managed = false, \
201 .offload_channel = false, \
205 static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = {
206 MHI_CHANNEL_CONFIG_UL(4, "DIAG", 16, 1),
207 MHI_CHANNEL_CONFIG_DL(5, "DIAG", 16, 1),
208 MHI_CHANNEL_CONFIG_UL(12, "MBIM", 4, 0),
209 MHI_CHANNEL_CONFIG_DL(13, "MBIM", 4, 0),
210 MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0),
211 MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0),
212 MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0),
213 MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0),
214 MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
215 MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
216 MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2),
217 MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3),
220 static struct mhi_event_config modem_qcom_v1_mhi_events[] = {
221 /* first ring is control+data ring */
222 MHI_EVENT_CONFIG_CTRL(0, 64),
223 /* DIAG dedicated event ring */
224 MHI_EVENT_CONFIG_DATA(1, 128),
225 /* Hardware channels request dedicated hardware event rings */
226 MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
227 MHI_EVENT_CONFIG_HW_DATA(3, 2048, 101)
230 static const struct mhi_controller_config modem_qcom_v1_mhiv_config = {
233 .num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels),
234 .ch_cfg = modem_qcom_v1_mhi_channels,
235 .num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events),
236 .event_cfg = modem_qcom_v1_mhi_events,
239 static const struct mhi_pci_dev_info mhi_qcom_sdx65_info = {
240 .name = "qcom-sdx65m",
241 .fw = "qcom/sdx65m/xbl.elf",
242 .edl = "qcom/sdx65m/edl.mbn",
243 .config = &modem_qcom_v1_mhiv_config,
244 .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
248 static const struct mhi_pci_dev_info mhi_qcom_sdx55_info = {
249 .name = "qcom-sdx55m",
250 .fw = "qcom/sdx55m/sbl1.mbn",
251 .edl = "qcom/sdx55m/edl.mbn",
252 .config = &modem_qcom_v1_mhiv_config,
253 .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
257 static const struct mhi_pci_dev_info mhi_qcom_sdx24_info = {
258 .name = "qcom-sdx24",
259 .edl = "qcom/prog_firehose_sdx24.mbn",
260 .config = &modem_qcom_v1_mhiv_config,
261 .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
265 static const struct mhi_channel_config mhi_quectel_em1xx_channels[] = {
266 MHI_CHANNEL_CONFIG_UL(0, "NMEA", 32, 0),
267 MHI_CHANNEL_CONFIG_DL(1, "NMEA", 32, 0),
268 MHI_CHANNEL_CONFIG_UL_SBL(2, "SAHARA", 32, 0),
269 MHI_CHANNEL_CONFIG_DL_SBL(3, "SAHARA", 32, 0),
270 MHI_CHANNEL_CONFIG_UL(4, "DIAG", 32, 1),
271 MHI_CHANNEL_CONFIG_DL(5, "DIAG", 32, 1),
272 MHI_CHANNEL_CONFIG_UL(12, "MBIM", 32, 0),
273 MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
274 MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
275 MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
276 /* The EDL firmware is a flash-programmer exposing firehose protocol */
277 MHI_CHANNEL_CONFIG_UL_FP(34, "FIREHOSE", 32, 0),
278 MHI_CHANNEL_CONFIG_DL_FP(35, "FIREHOSE", 32, 0),
279 MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
280 MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
283 static struct mhi_event_config mhi_quectel_em1xx_events[] = {
284 MHI_EVENT_CONFIG_CTRL(0, 128),
285 MHI_EVENT_CONFIG_DATA(1, 128),
286 MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
287 MHI_EVENT_CONFIG_HW_DATA(3, 1024, 101)
290 static const struct mhi_controller_config modem_quectel_em1xx_config = {
293 .num_channels = ARRAY_SIZE(mhi_quectel_em1xx_channels),
294 .ch_cfg = mhi_quectel_em1xx_channels,
295 .num_events = ARRAY_SIZE(mhi_quectel_em1xx_events),
296 .event_cfg = mhi_quectel_em1xx_events,
299 static const struct mhi_pci_dev_info mhi_quectel_em1xx_info = {
300 .name = "quectel-em1xx",
301 .edl = "qcom/prog_firehose_sdx24.mbn",
302 .config = &modem_quectel_em1xx_config,
303 .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
307 static const struct mhi_channel_config mhi_foxconn_sdx55_channels[] = {
308 MHI_CHANNEL_CONFIG_UL(0, "LOOPBACK", 32, 0),
309 MHI_CHANNEL_CONFIG_DL(1, "LOOPBACK", 32, 0),
310 MHI_CHANNEL_CONFIG_UL(4, "DIAG", 32, 1),
311 MHI_CHANNEL_CONFIG_DL(5, "DIAG", 32, 1),
312 MHI_CHANNEL_CONFIG_UL(12, "MBIM", 32, 0),
313 MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
314 MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
315 MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
316 MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
317 MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
320 static struct mhi_event_config mhi_foxconn_sdx55_events[] = {
321 MHI_EVENT_CONFIG_CTRL(0, 128),
322 MHI_EVENT_CONFIG_DATA(1, 128),
323 MHI_EVENT_CONFIG_HW_DATA(2, 1024, 100),
324 MHI_EVENT_CONFIG_HW_DATA(3, 1024, 101)
327 static const struct mhi_controller_config modem_foxconn_sdx55_config = {
330 .num_channels = ARRAY_SIZE(mhi_foxconn_sdx55_channels),
331 .ch_cfg = mhi_foxconn_sdx55_channels,
332 .num_events = ARRAY_SIZE(mhi_foxconn_sdx55_events),
333 .event_cfg = mhi_foxconn_sdx55_events,
336 static const struct mhi_pci_dev_info mhi_foxconn_sdx55_info = {
337 .name = "foxconn-sdx55",
338 .fw = "qcom/sdx55m/sbl1.mbn",
339 .edl = "qcom/sdx55m/edl.mbn",
340 .config = &modem_foxconn_sdx55_config,
341 .bar_num = MHI_PCI_DEFAULT_BAR_NUM,
345 static const struct pci_device_id mhi_pci_id_table[] = {
346 { PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0306),
347 .driver_data = (kernel_ulong_t) &mhi_qcom_sdx55_info },
348 { PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0304),
349 .driver_data = (kernel_ulong_t) &mhi_qcom_sdx24_info },
350 { PCI_DEVICE(0x1eac, 0x1001), /* EM120R-GL (sdx24) */
351 .driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
352 { PCI_DEVICE(0x1eac, 0x1002), /* EM160R-GL (sdx24) */
353 .driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
354 { PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0308),
355 .driver_data = (kernel_ulong_t) &mhi_qcom_sdx65_info },
356 /* T99W175 (sdx55), Both for eSIM and Non-eSIM */
357 { PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0ab),
358 .driver_data = (kernel_ulong_t) &mhi_foxconn_sdx55_info },
359 /* DW5930e (sdx55), With eSIM, It's also T99W175 */
360 { PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0b0),
361 .driver_data = (kernel_ulong_t) &mhi_foxconn_sdx55_info },
362 /* DW5930e (sdx55), Non-eSIM, It's also T99W175 */
363 { PCI_DEVICE(PCI_VENDOR_ID_FOXCONN, 0xe0b1),
364 .driver_data = (kernel_ulong_t) &mhi_foxconn_sdx55_info },
367 MODULE_DEVICE_TABLE(pci, mhi_pci_id_table);
369 enum mhi_pci_device_status {
371 MHI_PCI_DEV_SUSPENDED,
374 struct mhi_pci_device {
375 struct mhi_controller mhi_cntrl;
376 struct pci_saved_state *pci_state;
377 struct work_struct recovery_work;
378 struct timer_list health_check_timer;
379 unsigned long status;
382 static int mhi_pci_read_reg(struct mhi_controller *mhi_cntrl,
383 void __iomem *addr, u32 *out)
389 static void mhi_pci_write_reg(struct mhi_controller *mhi_cntrl,
390 void __iomem *addr, u32 val)
395 static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl,
396 enum mhi_callback cb)
398 struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
400 /* Nothing to do for now */
402 case MHI_CB_FATAL_ERROR:
403 case MHI_CB_SYS_ERROR:
404 dev_warn(&pdev->dev, "firmware crashed (%u)\n", cb);
405 pm_runtime_forbid(&pdev->dev);
407 case MHI_CB_EE_MISSION_MODE:
408 pm_runtime_allow(&pdev->dev);
415 static void mhi_pci_wake_get_nop(struct mhi_controller *mhi_cntrl, bool force)
420 static void mhi_pci_wake_put_nop(struct mhi_controller *mhi_cntrl, bool override)
425 static void mhi_pci_wake_toggle_nop(struct mhi_controller *mhi_cntrl)
430 static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl)
432 struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
435 if (pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor))
438 if (vendor == (u16) ~0 || vendor == 0)
444 static int mhi_pci_claim(struct mhi_controller *mhi_cntrl,
445 unsigned int bar_num, u64 dma_mask)
447 struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
450 err = pci_assign_resource(pdev, bar_num);
454 err = pcim_enable_device(pdev);
456 dev_err(&pdev->dev, "failed to enable pci device: %d\n", err);
460 err = pcim_iomap_regions(pdev, 1 << bar_num, pci_name(pdev));
462 dev_err(&pdev->dev, "failed to map pci region: %d\n", err);
465 mhi_cntrl->regs = pcim_iomap_table(pdev)[bar_num];
467 err = pci_set_dma_mask(pdev, dma_mask);
469 dev_err(&pdev->dev, "Cannot set proper DMA mask\n");
473 err = pci_set_consistent_dma_mask(pdev, dma_mask);
475 dev_err(&pdev->dev, "set consistent dma mask failed\n");
479 pci_set_master(pdev);
484 static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl,
485 const struct mhi_controller_config *mhi_cntrl_config)
487 struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
492 * Alloc one MSI vector for BHI + one vector per event ring, ideally...
493 * No explicit pci_free_irq_vectors required, done by pcim_release.
495 mhi_cntrl->nr_irqs = 1 + mhi_cntrl_config->num_events;
497 nr_vectors = pci_alloc_irq_vectors(pdev, 1, mhi_cntrl->nr_irqs, PCI_IRQ_MSI);
498 if (nr_vectors < 0) {
499 dev_err(&pdev->dev, "Error allocating MSI vectors %d\n",
504 if (nr_vectors < mhi_cntrl->nr_irqs) {
505 dev_warn(&pdev->dev, "using shared MSI\n");
507 /* Patch msi vectors, use only one (shared) */
508 for (i = 0; i < mhi_cntrl_config->num_events; i++)
509 mhi_cntrl_config->event_cfg[i].irq = 0;
510 mhi_cntrl->nr_irqs = 1;
513 irq = devm_kcalloc(&pdev->dev, mhi_cntrl->nr_irqs, sizeof(int), GFP_KERNEL);
517 for (i = 0; i < mhi_cntrl->nr_irqs; i++) {
518 int vector = i >= nr_vectors ? (nr_vectors - 1) : i;
520 irq[i] = pci_irq_vector(pdev, vector);
523 mhi_cntrl->irq = irq;
528 static int mhi_pci_runtime_get(struct mhi_controller *mhi_cntrl)
530 /* The runtime_get() MHI callback means:
531 * Do whatever is requested to leave M3.
533 return pm_runtime_get(mhi_cntrl->cntrl_dev);
536 static void mhi_pci_runtime_put(struct mhi_controller *mhi_cntrl)
538 /* The runtime_put() MHI callback means:
539 * Device can be moved in M3 state.
541 pm_runtime_mark_last_busy(mhi_cntrl->cntrl_dev);
542 pm_runtime_put(mhi_cntrl->cntrl_dev);
545 static void mhi_pci_recovery_work(struct work_struct *work)
547 struct mhi_pci_device *mhi_pdev = container_of(work, struct mhi_pci_device,
549 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
550 struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev);
553 dev_warn(&pdev->dev, "device recovery started\n");
555 del_timer(&mhi_pdev->health_check_timer);
556 pm_runtime_forbid(&pdev->dev);
558 /* Clean up MHI state */
559 if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) {
560 mhi_power_down(mhi_cntrl, false);
561 mhi_unprepare_after_power_down(mhi_cntrl);
564 pci_set_power_state(pdev, PCI_D0);
565 pci_load_saved_state(pdev, mhi_pdev->pci_state);
566 pci_restore_state(pdev);
568 if (!mhi_pci_is_alive(mhi_cntrl))
571 err = mhi_prepare_for_power_up(mhi_cntrl);
575 err = mhi_sync_power_up(mhi_cntrl);
579 dev_dbg(&pdev->dev, "Recovery completed\n");
581 set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status);
582 mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD);
586 mhi_unprepare_after_power_down(mhi_cntrl);
588 if (pci_reset_function(pdev))
589 dev_err(&pdev->dev, "Recovery failed\n");
592 static void health_check(struct timer_list *t)
594 struct mhi_pci_device *mhi_pdev = from_timer(mhi_pdev, t, health_check_timer);
595 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
597 if (!test_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status) ||
598 test_bit(MHI_PCI_DEV_SUSPENDED, &mhi_pdev->status))
601 if (!mhi_pci_is_alive(mhi_cntrl)) {
602 dev_err(mhi_cntrl->cntrl_dev, "Device died\n");
603 queue_work(system_long_wq, &mhi_pdev->recovery_work);
607 /* reschedule in two seconds */
608 mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD);
611 static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
613 const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data;
614 const struct mhi_controller_config *mhi_cntrl_config;
615 struct mhi_pci_device *mhi_pdev;
616 struct mhi_controller *mhi_cntrl;
619 dev_dbg(&pdev->dev, "MHI PCI device found: %s\n", info->name);
621 /* mhi_pdev.mhi_cntrl must be zero-initialized */
622 mhi_pdev = devm_kzalloc(&pdev->dev, sizeof(*mhi_pdev), GFP_KERNEL);
626 INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work);
627 timer_setup(&mhi_pdev->health_check_timer, health_check, 0);
629 mhi_cntrl_config = info->config;
630 mhi_cntrl = &mhi_pdev->mhi_cntrl;
632 mhi_cntrl->cntrl_dev = &pdev->dev;
633 mhi_cntrl->iova_start = 0;
634 mhi_cntrl->iova_stop = (dma_addr_t)DMA_BIT_MASK(info->dma_data_width);
635 mhi_cntrl->fw_image = info->fw;
636 mhi_cntrl->edl_image = info->edl;
638 mhi_cntrl->read_reg = mhi_pci_read_reg;
639 mhi_cntrl->write_reg = mhi_pci_write_reg;
640 mhi_cntrl->status_cb = mhi_pci_status_cb;
641 mhi_cntrl->runtime_get = mhi_pci_runtime_get;
642 mhi_cntrl->runtime_put = mhi_pci_runtime_put;
643 mhi_cntrl->wake_get = mhi_pci_wake_get_nop;
644 mhi_cntrl->wake_put = mhi_pci_wake_put_nop;
645 mhi_cntrl->wake_toggle = mhi_pci_wake_toggle_nop;
647 err = mhi_pci_claim(mhi_cntrl, info->bar_num, DMA_BIT_MASK(info->dma_data_width));
651 err = mhi_pci_get_irqs(mhi_cntrl, mhi_cntrl_config);
655 pci_set_drvdata(pdev, mhi_pdev);
657 /* Have stored pci confspace at hand for restore in sudden PCI error.
658 * cache the state locally and discard the PCI core one.
660 pci_save_state(pdev);
661 mhi_pdev->pci_state = pci_store_saved_state(pdev);
662 pci_load_saved_state(pdev, NULL);
664 pci_enable_pcie_error_reporting(pdev);
666 err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config);
668 goto err_disable_reporting;
670 /* MHI bus does not power up the controller by default */
671 err = mhi_prepare_for_power_up(mhi_cntrl);
673 dev_err(&pdev->dev, "failed to prepare MHI controller\n");
677 err = mhi_sync_power_up(mhi_cntrl);
679 dev_err(&pdev->dev, "failed to power up MHI controller\n");
683 set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status);
685 /* start health check */
686 mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD);
688 /* Only allow runtime-suspend if PME capable (for wakeup) */
689 if (pci_pme_capable(pdev, PCI_D3hot)) {
690 pm_runtime_set_autosuspend_delay(&pdev->dev, 2000);
691 pm_runtime_use_autosuspend(&pdev->dev);
692 pm_runtime_mark_last_busy(&pdev->dev);
693 pm_runtime_put_noidle(&pdev->dev);
699 mhi_unprepare_after_power_down(mhi_cntrl);
701 mhi_unregister_controller(mhi_cntrl);
702 err_disable_reporting:
703 pci_disable_pcie_error_reporting(pdev);
708 static void mhi_pci_remove(struct pci_dev *pdev)
710 struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
711 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
713 del_timer_sync(&mhi_pdev->health_check_timer);
714 cancel_work_sync(&mhi_pdev->recovery_work);
716 if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) {
717 mhi_power_down(mhi_cntrl, true);
718 mhi_unprepare_after_power_down(mhi_cntrl);
721 /* balancing probe put_noidle */
722 if (pci_pme_capable(pdev, PCI_D3hot))
723 pm_runtime_get_noresume(&pdev->dev);
725 mhi_unregister_controller(mhi_cntrl);
726 pci_disable_pcie_error_reporting(pdev);
729 static void mhi_pci_shutdown(struct pci_dev *pdev)
731 mhi_pci_remove(pdev);
732 pci_set_power_state(pdev, PCI_D3hot);
735 static void mhi_pci_reset_prepare(struct pci_dev *pdev)
737 struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
738 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
740 dev_info(&pdev->dev, "reset\n");
742 del_timer(&mhi_pdev->health_check_timer);
744 /* Clean up MHI state */
745 if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) {
746 mhi_power_down(mhi_cntrl, false);
747 mhi_unprepare_after_power_down(mhi_cntrl);
750 /* cause internal device reset */
751 mhi_soc_reset(mhi_cntrl);
753 /* Be sure device reset has been executed */
754 msleep(MHI_POST_RESET_DELAY_MS);
757 static void mhi_pci_reset_done(struct pci_dev *pdev)
759 struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
760 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
763 /* Restore initial known working PCI state */
764 pci_load_saved_state(pdev, mhi_pdev->pci_state);
765 pci_restore_state(pdev);
767 /* Is device status available ? */
768 if (!mhi_pci_is_alive(mhi_cntrl)) {
769 dev_err(&pdev->dev, "reset failed\n");
773 err = mhi_prepare_for_power_up(mhi_cntrl);
775 dev_err(&pdev->dev, "failed to prepare MHI controller\n");
779 err = mhi_sync_power_up(mhi_cntrl);
781 dev_err(&pdev->dev, "failed to power up MHI controller\n");
782 mhi_unprepare_after_power_down(mhi_cntrl);
786 set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status);
787 mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD);
790 static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev,
791 pci_channel_state_t state)
793 struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
794 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
796 dev_err(&pdev->dev, "PCI error detected, state = %u\n", state);
798 if (state == pci_channel_io_perm_failure)
799 return PCI_ERS_RESULT_DISCONNECT;
801 /* Clean up MHI state */
802 if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) {
803 mhi_power_down(mhi_cntrl, false);
804 mhi_unprepare_after_power_down(mhi_cntrl);
807 return PCI_ERS_RESULT_RECOVERED;
810 pci_disable_device(pdev);
812 return PCI_ERS_RESULT_NEED_RESET;
815 static pci_ers_result_t mhi_pci_slot_reset(struct pci_dev *pdev)
817 if (pci_enable_device(pdev)) {
818 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n");
819 return PCI_ERS_RESULT_DISCONNECT;
822 return PCI_ERS_RESULT_RECOVERED;
825 static void mhi_pci_io_resume(struct pci_dev *pdev)
827 struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev);
829 dev_err(&pdev->dev, "PCI slot reset done\n");
831 queue_work(system_long_wq, &mhi_pdev->recovery_work);
834 static const struct pci_error_handlers mhi_pci_err_handler = {
835 .error_detected = mhi_pci_error_detected,
836 .slot_reset = mhi_pci_slot_reset,
837 .resume = mhi_pci_io_resume,
838 .reset_prepare = mhi_pci_reset_prepare,
839 .reset_done = mhi_pci_reset_done,
842 static int __maybe_unused mhi_pci_runtime_suspend(struct device *dev)
844 struct pci_dev *pdev = to_pci_dev(dev);
845 struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
846 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
849 if (test_and_set_bit(MHI_PCI_DEV_SUSPENDED, &mhi_pdev->status))
852 del_timer(&mhi_pdev->health_check_timer);
853 cancel_work_sync(&mhi_pdev->recovery_work);
855 if (!test_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status) ||
856 mhi_cntrl->ee != MHI_EE_AMSS)
857 goto pci_suspend; /* Nothing to do at MHI level */
859 /* Transition to M3 state */
860 err = mhi_pm_suspend(mhi_cntrl);
862 dev_err(&pdev->dev, "failed to suspend device: %d\n", err);
863 clear_bit(MHI_PCI_DEV_SUSPENDED, &mhi_pdev->status);
868 pci_disable_device(pdev);
869 pci_wake_from_d3(pdev, true);
874 static int __maybe_unused mhi_pci_runtime_resume(struct device *dev)
876 struct pci_dev *pdev = to_pci_dev(dev);
877 struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
878 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
881 if (!test_and_clear_bit(MHI_PCI_DEV_SUSPENDED, &mhi_pdev->status))
884 err = pci_enable_device(pdev);
888 pci_set_master(pdev);
889 pci_wake_from_d3(pdev, false);
891 if (!test_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status) ||
892 mhi_cntrl->ee != MHI_EE_AMSS)
893 return 0; /* Nothing to do at MHI level */
895 /* Exit M3, transition to M0 state */
896 err = mhi_pm_resume(mhi_cntrl);
898 dev_err(&pdev->dev, "failed to resume device: %d\n", err);
902 /* Resume health check */
903 mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD);
905 /* It can be a remote wakeup (no mhi runtime_get), update access time */
906 pm_runtime_mark_last_busy(dev);
911 /* Do not fail to not mess up our PCI device state, the device likely
912 * lost power (d3cold) and we simply need to reset it from the recovery
913 * procedure, trigger the recovery asynchronously to prevent system
914 * suspend exit delaying.
916 queue_work(system_long_wq, &mhi_pdev->recovery_work);
917 pm_runtime_mark_last_busy(dev);
922 static int __maybe_unused mhi_pci_suspend(struct device *dev)
924 pm_runtime_disable(dev);
925 return mhi_pci_runtime_suspend(dev);
928 static int __maybe_unused mhi_pci_resume(struct device *dev)
932 /* Depending the platform, device may have lost power (d3cold), we need
933 * to resume it now to check its state and recover when necessary.
935 ret = mhi_pci_runtime_resume(dev);
936 pm_runtime_enable(dev);
941 static int __maybe_unused mhi_pci_freeze(struct device *dev)
943 struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
944 struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl;
946 /* We want to stop all operations, hibernation does not guarantee that
947 * device will be in the same state as before freezing, especially if
948 * the intermediate restore kernel reinitializes MHI device with new
951 if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) {
952 mhi_power_down(mhi_cntrl, false);
953 mhi_unprepare_after_power_down(mhi_cntrl);
959 static int __maybe_unused mhi_pci_restore(struct device *dev)
961 struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev);
963 /* Reinitialize the device */
964 queue_work(system_long_wq, &mhi_pdev->recovery_work);
969 static const struct dev_pm_ops mhi_pci_pm_ops = {
970 SET_RUNTIME_PM_OPS(mhi_pci_runtime_suspend, mhi_pci_runtime_resume, NULL)
971 #ifdef CONFIG_PM_SLEEP
972 .suspend = mhi_pci_suspend,
973 .resume = mhi_pci_resume,
974 .freeze = mhi_pci_freeze,
975 .thaw = mhi_pci_restore,
976 .restore = mhi_pci_restore,
980 static struct pci_driver mhi_pci_driver = {
981 .name = "mhi-pci-generic",
982 .id_table = mhi_pci_id_table,
983 .probe = mhi_pci_probe,
984 .remove = mhi_pci_remove,
985 .shutdown = mhi_pci_shutdown,
986 .err_handler = &mhi_pci_err_handler,
987 .driver.pm = &mhi_pci_pm_ops
989 module_pci_driver(mhi_pci_driver);
991 MODULE_AUTHOR("Loic Poulain <loic.poulain@linaro.org>");
992 MODULE_DESCRIPTION("Modem Host Interface (MHI) PCI controller driver");
993 MODULE_LICENSE("GPL");