1 // SPDX-License-Identifier: GPL-2.0
3 * Apple ANS NVM Express device driver
4 * Copyright The Asahi Linux Contributors
6 * Based on the pci.c NVM Express device driver
7 * Copyright (c) 2011-2014, Intel Corporation.
8 * and on the rdma.c NVMe over Fabrics RDMA host code.
9 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
12 #include <linux/async.h>
13 #include <linux/blkdev.h>
14 #include <linux/blk-mq.h>
15 #include <linux/device.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmapool.h>
18 #include <linux/interrupt.h>
19 #include <linux/io-64-nonatomic-lo-hi.h>
21 #include <linux/iopoll.h>
22 #include <linux/jiffies.h>
23 #include <linux/mempool.h>
24 #include <linux/module.h>
26 #include <linux/of_platform.h>
27 #include <linux/once.h>
28 #include <linux/platform_device.h>
29 #include <linux/pm_domain.h>
30 #include <linux/soc/apple/rtkit.h>
31 #include <linux/soc/apple/sart.h>
32 #include <linux/reset.h>
33 #include <linux/time64.h>
37 #define APPLE_ANS_BOOT_TIMEOUT USEC_PER_SEC
38 #define APPLE_ANS_MAX_QUEUE_DEPTH 64
40 #define APPLE_ANS_COPROC_CPU_CONTROL 0x44
41 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4)
43 #define APPLE_ANS_ACQ_DB 0x1004
44 #define APPLE_ANS_IOCQ_DB 0x100c
46 #define APPLE_ANS_MAX_PEND_CMDS_CTRL 0x1210
48 #define APPLE_ANS_BOOT_STATUS 0x1300
49 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55
51 #define APPLE_ANS_UNKNOWN_CTRL 0x24008
52 #define APPLE_ANS_PRP_NULL_CHECK BIT(11)
54 #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908
55 #define APPLE_ANS_LINEAR_SQ_EN BIT(0)
57 #define APPLE_ANS_LINEAR_ASQ_DB 0x2490c
58 #define APPLE_ANS_LINEAR_IOSQ_DB 0x24910
60 #define APPLE_NVMMU_NUM_TCBS 0x28100
61 #define APPLE_NVMMU_ASQ_TCB_BASE 0x28108
62 #define APPLE_NVMMU_IOSQ_TCB_BASE 0x28110
63 #define APPLE_NVMMU_TCB_INVAL 0x28118
64 #define APPLE_NVMMU_TCB_STAT 0x28120
67 * This controller is a bit weird in the way command tags works: Both the
68 * admin and the IO queue share the same tag space. Additionally, tags
69 * cannot be higher than 0x40 which effectively limits the combined
70 * queue depth to 0x40. Instead of wasting half of that on the admin queue
71 * which gets much less traffic we instead reduce its size here.
72 * The controller also doesn't support async event such that no space must
73 * be reserved for NVME_NR_AEN_COMMANDS.
75 #define APPLE_NVME_AQ_DEPTH 2
76 #define APPLE_NVME_AQ_MQ_TAG_DEPTH (APPLE_NVME_AQ_DEPTH - 1)
79 * These can be higher, but we need to ensure that any command doesn't
80 * require an sg allocation that needs more than a page of data.
82 #define NVME_MAX_KB_SZ 4096
83 #define NVME_MAX_SEGS 127
86 * This controller comes with an embedded IOMMU known as NVMMU.
87 * The NVMMU is pointed to an array of TCBs indexed by the command tag.
88 * Each command must be configured inside this structure before it's allowed
89 * to execute, including commands that don't require DMA transfers.
91 * An exception to this are Apple's vendor-specific commands (opcode 0xD8 on the
92 * admin queue): Those commands must still be added to the NVMMU but the DMA
93 * buffers cannot be represented as PRPs and must instead be allowed using SART.
95 * Programming the PRPs to the same values as those in the submission queue
96 * looks rather silly at first. This hardware is however designed for a kernel
97 * that runs the NVMMU code in a higher exception level than the NVMe driver.
98 * In that setting the NVMe driver first programs the submission queue entry
99 * and then executes a hypercall to the code that is allowed to program the
100 * NVMMU. The NVMMU driver then creates a shadow copy of the PRPs while
101 * verifying that they don't point to kernel text, data, pagetables, or similar
102 * protected areas before programming the TCB to point to this shadow copy.
103 * Since Linux doesn't do any of that we may as well just point both the queue
104 * and the TCB PRP pointer to the same memory.
106 struct apple_nvmmu_tcb {
109 #define APPLE_ANS_TCB_DMA_FROM_DEVICE BIT(0)
110 #define APPLE_ANS_TCB_DMA_TO_DEVICE BIT(1)
125 * The Apple NVMe controller only supports a single admin and a single IO queue
126 * which are both limited to 64 entries and share a single interrupt.
128 * The completion queue works as usual. The submission "queue" instead is
129 * an array indexed by the command tag on this hardware. Commands must also be
130 * present in the NVMMU's tcb array. They are triggered by writing their tag to
133 struct apple_nvme_queue {
134 struct nvme_command *sqes;
135 struct nvme_completion *cqes;
136 struct apple_nvmmu_tcb *tcbs;
138 dma_addr_t sq_dma_addr;
139 dma_addr_t cq_dma_addr;
140 dma_addr_t tcb_dma_addr;
153 * The apple_nvme_iod describes the data in an I/O.
155 * The sg pointer contains the list of PRP chunk allocations in addition
156 * to the actual struct scatterlist.
158 struct apple_nvme_iod {
159 struct nvme_request req;
160 struct nvme_command cmd;
161 struct apple_nvme_queue *q;
162 int npages; /* In the PRP list. 0 means small pool in use */
163 int nents; /* Used in scatterlist */
164 dma_addr_t first_dma;
165 unsigned int dma_len; /* length of single DMA segment mapping */
166 struct scatterlist *sg;
172 void __iomem *mmio_coproc;
173 void __iomem *mmio_nvme;
175 struct device **pd_dev;
176 struct device_link **pd_link;
179 struct apple_sart *sart;
180 struct apple_rtkit *rtk;
181 struct reset_control *reset;
183 struct dma_pool *prp_page_pool;
184 struct dma_pool *prp_small_pool;
185 mempool_t *iod_mempool;
187 struct nvme_ctrl ctrl;
188 struct work_struct remove_work;
190 struct apple_nvme_queue adminq;
191 struct apple_nvme_queue ioq;
193 struct blk_mq_tag_set admin_tagset;
194 struct blk_mq_tag_set tagset;
200 static_assert(sizeof(struct nvme_command) == 64);
201 static_assert(sizeof(struct apple_nvmmu_tcb) == 128);
203 static inline struct apple_nvme *ctrl_to_apple_nvme(struct nvme_ctrl *ctrl)
205 return container_of(ctrl, struct apple_nvme, ctrl);
208 static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q)
211 return container_of(q, struct apple_nvme, adminq);
213 return container_of(q, struct apple_nvme, ioq);
216 static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q)
219 return APPLE_NVME_AQ_DEPTH;
221 return APPLE_ANS_MAX_QUEUE_DEPTH;
224 static void apple_nvme_rtkit_crashed(void *cookie)
226 struct apple_nvme *anv = cookie;
228 dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot");
229 nvme_reset_ctrl(&anv->ctrl);
232 static int apple_nvme_sart_dma_setup(void *cookie,
233 struct apple_rtkit_shmem *bfr)
235 struct apple_nvme *anv = cookie;
244 dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL);
248 ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size);
250 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova);
258 static void apple_nvme_sart_dma_destroy(void *cookie,
259 struct apple_rtkit_shmem *bfr)
261 struct apple_nvme *anv = cookie;
263 apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size);
264 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova);
267 static const struct apple_rtkit_ops apple_nvme_rtkit_ops = {
268 .crashed = apple_nvme_rtkit_crashed,
269 .shmem_setup = apple_nvme_sart_dma_setup,
270 .shmem_destroy = apple_nvme_sart_dma_destroy,
273 static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned int tag)
275 struct apple_nvme *anv = queue_to_apple_nvme(q);
277 writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL);
278 if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT))
279 dev_warn_ratelimited(anv->dev,
280 "NVMMU TCB invalidation failed\n");
283 static void apple_nvme_submit_cmd(struct apple_nvme_queue *q,
284 struct nvme_command *cmd)
286 struct apple_nvme *anv = queue_to_apple_nvme(q);
287 u32 tag = nvme_tag_from_cid(cmd->common.command_id);
288 struct apple_nvmmu_tcb *tcb = &q->tcbs[tag];
290 tcb->opcode = cmd->common.opcode;
291 tcb->prp1 = cmd->common.dptr.prp1;
292 tcb->prp2 = cmd->common.dptr.prp2;
293 tcb->length = cmd->rw.length;
294 tcb->command_id = tag;
296 if (nvme_is_write(cmd))
297 tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE;
299 tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE;
301 memcpy(&q->sqes[tag], cmd, sizeof(*cmd));
304 * This lock here doesn't make much sense at a first glace but
305 * removing it will result in occasional missed completetion
306 * interrupts even though the commands still appear on the CQ.
307 * It's unclear why this happens but our best guess is that
308 * there is a bug in the firmware triggered when a new command
309 * is issued while we're inside the irq handler between the
310 * NVMMU invalidation (and making the tag available again)
311 * and the final CQ update.
313 spin_lock_irq(&anv->lock);
314 writel(tag, q->sq_db);
315 spin_unlock_irq(&anv->lock);
320 * Will slightly overestimate the number of pages needed. This is OK
321 * as it only leads to a small amount of wasted memory for the lifetime of
324 static inline size_t apple_nvme_iod_alloc_size(void)
326 const unsigned int nprps = DIV_ROUND_UP(
327 NVME_MAX_KB_SZ + NVME_CTRL_PAGE_SIZE, NVME_CTRL_PAGE_SIZE);
328 const int npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
329 const size_t alloc_size = sizeof(__le64 *) * npages +
330 sizeof(struct scatterlist) * NVME_MAX_SEGS;
335 static void **apple_nvme_iod_list(struct request *req)
337 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
339 return (void **)(iod->sg + blk_rq_nr_phys_segments(req));
342 static void apple_nvme_free_prps(struct apple_nvme *anv, struct request *req)
344 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
345 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
346 dma_addr_t dma_addr = iod->first_dma;
349 for (i = 0; i < iod->npages; i++) {
350 __le64 *prp_list = apple_nvme_iod_list(req)[i];
351 dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]);
353 dma_pool_free(anv->prp_page_pool, prp_list, dma_addr);
354 dma_addr = next_dma_addr;
358 static void apple_nvme_unmap_data(struct apple_nvme *anv, struct request *req)
360 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
363 dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len,
368 WARN_ON_ONCE(!iod->nents);
370 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));
371 if (iod->npages == 0)
372 dma_pool_free(anv->prp_small_pool, apple_nvme_iod_list(req)[0],
375 apple_nvme_free_prps(anv, req);
376 mempool_free(iod->sg, anv->iod_mempool);
379 static void apple_nvme_print_sgl(struct scatterlist *sgl, int nents)
382 struct scatterlist *sg;
384 for_each_sg(sgl, sg, nents, i) {
385 dma_addr_t phys = sg_phys(sg);
387 pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d dma_address:%pad dma_length:%d\n",
388 i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
393 static blk_status_t apple_nvme_setup_prps(struct apple_nvme *anv,
395 struct nvme_rw_command *cmnd)
397 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
398 struct dma_pool *pool;
399 int length = blk_rq_payload_bytes(req);
400 struct scatterlist *sg = iod->sg;
401 int dma_len = sg_dma_len(sg);
402 u64 dma_addr = sg_dma_address(sg);
403 int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
405 void **list = apple_nvme_iod_list(req);
409 length -= (NVME_CTRL_PAGE_SIZE - offset);
415 dma_len -= (NVME_CTRL_PAGE_SIZE - offset);
417 dma_addr += (NVME_CTRL_PAGE_SIZE - offset);
420 dma_addr = sg_dma_address(sg);
421 dma_len = sg_dma_len(sg);
424 if (length <= NVME_CTRL_PAGE_SIZE) {
425 iod->first_dma = dma_addr;
429 nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
430 if (nprps <= (256 / 8)) {
431 pool = anv->prp_small_pool;
434 pool = anv->prp_page_pool;
438 prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
440 iod->first_dma = dma_addr;
442 return BLK_STS_RESOURCE;
445 iod->first_dma = prp_dma;
448 if (i == NVME_CTRL_PAGE_SIZE >> 3) {
449 __le64 *old_prp_list = prp_list;
451 prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
454 list[iod->npages++] = prp_list;
455 prp_list[0] = old_prp_list[i - 1];
456 old_prp_list[i - 1] = cpu_to_le64(prp_dma);
459 prp_list[i++] = cpu_to_le64(dma_addr);
460 dma_len -= NVME_CTRL_PAGE_SIZE;
461 dma_addr += NVME_CTRL_PAGE_SIZE;
462 length -= NVME_CTRL_PAGE_SIZE;
467 if (unlikely(dma_len < 0))
470 dma_addr = sg_dma_address(sg);
471 dma_len = sg_dma_len(sg);
474 cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
475 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
478 apple_nvme_free_prps(anv, req);
479 return BLK_STS_RESOURCE;
481 WARN(DO_ONCE(apple_nvme_print_sgl, iod->sg, iod->nents),
482 "Invalid SGL for payload:%d nents:%d\n", blk_rq_payload_bytes(req),
484 return BLK_STS_IOERR;
487 static blk_status_t apple_nvme_setup_prp_simple(struct apple_nvme *anv,
489 struct nvme_rw_command *cmnd,
492 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
493 unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
494 unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset;
496 iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0);
497 if (dma_mapping_error(anv->dev, iod->first_dma))
498 return BLK_STS_RESOURCE;
499 iod->dma_len = bv->bv_len;
501 cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);
502 if (bv->bv_len > first_prp_len)
503 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);
507 static blk_status_t apple_nvme_map_data(struct apple_nvme *anv,
509 struct nvme_command *cmnd)
511 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
512 blk_status_t ret = BLK_STS_RESOURCE;
515 if (blk_rq_nr_phys_segments(req) == 1) {
516 struct bio_vec bv = req_bvec(req);
518 if (bv.bv_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)
519 return apple_nvme_setup_prp_simple(anv, req, &cmnd->rw,
524 iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC);
526 return BLK_STS_RESOURCE;
527 sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
528 iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
532 nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents,
533 rq_dma_dir(req), DMA_ATTR_NO_WARN);
537 ret = apple_nvme_setup_prps(anv, req, &cmnd->rw);
538 if (ret != BLK_STS_OK)
543 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req));
545 mempool_free(iod->sg, anv->iod_mempool);
549 static __always_inline void apple_nvme_unmap_rq(struct request *req)
551 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
552 struct apple_nvme *anv = queue_to_apple_nvme(iod->q);
554 if (blk_rq_nr_phys_segments(req))
555 apple_nvme_unmap_data(anv, req);
558 static void apple_nvme_complete_rq(struct request *req)
560 apple_nvme_unmap_rq(req);
561 nvme_complete_rq(req);
564 static void apple_nvme_complete_batch(struct io_comp_batch *iob)
566 nvme_complete_batch(iob, apple_nvme_unmap_rq);
569 static inline bool apple_nvme_cqe_pending(struct apple_nvme_queue *q)
571 struct nvme_completion *hcqe = &q->cqes[q->cq_head];
573 return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == q->cq_phase;
576 static inline struct blk_mq_tags *
577 apple_nvme_queue_tagset(struct apple_nvme *anv, struct apple_nvme_queue *q)
580 return anv->admin_tagset.tags[0];
582 return anv->tagset.tags[0];
585 static inline void apple_nvme_handle_cqe(struct apple_nvme_queue *q,
586 struct io_comp_batch *iob, u16 idx)
588 struct apple_nvme *anv = queue_to_apple_nvme(q);
589 struct nvme_completion *cqe = &q->cqes[idx];
590 __u16 command_id = READ_ONCE(cqe->command_id);
593 apple_nvmmu_inval(q, command_id);
595 req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id);
596 if (unlikely(!req)) {
597 dev_warn(anv->dev, "invalid id %d completed", command_id);
601 if (!nvme_try_complete_req(req, cqe->status, cqe->result) &&
602 !blk_mq_add_to_batch(req, iob, nvme_req(req)->status,
603 apple_nvme_complete_batch))
604 apple_nvme_complete_rq(req);
607 static inline void apple_nvme_update_cq_head(struct apple_nvme_queue *q)
609 u32 tmp = q->cq_head + 1;
611 if (tmp == apple_nvme_queue_depth(q)) {
619 static bool apple_nvme_poll_cq(struct apple_nvme_queue *q,
620 struct io_comp_batch *iob)
624 while (apple_nvme_cqe_pending(q)) {
628 * load-load control dependency between phase and the rest of
629 * the cqe requires a full read memory barrier
632 apple_nvme_handle_cqe(q, iob, q->cq_head);
633 apple_nvme_update_cq_head(q);
637 writel(q->cq_head, q->cq_db);
642 static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force)
645 DEFINE_IO_COMP_BATCH(iob);
647 if (!READ_ONCE(q->enabled) && !force)
650 found = apple_nvme_poll_cq(q, &iob);
652 if (!rq_list_empty(iob.req_list))
653 apple_nvme_complete_batch(&iob);
658 static irqreturn_t apple_nvme_irq(int irq, void *data)
660 struct apple_nvme *anv = data;
661 bool handled = false;
664 spin_lock_irqsave(&anv->lock, flags);
665 if (apple_nvme_handle_cq(&anv->ioq, false))
667 if (apple_nvme_handle_cq(&anv->adminq, false))
669 spin_unlock_irqrestore(&anv->lock, flags);
676 static int apple_nvme_create_cq(struct apple_nvme *anv)
678 struct nvme_command c = {};
681 * Note: we (ab)use the fact that the prp fields survive if no data
682 * is attached to the request.
684 c.create_cq.opcode = nvme_admin_create_cq;
685 c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr);
686 c.create_cq.cqid = cpu_to_le16(1);
687 c.create_cq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1);
688 c.create_cq.cq_flags = cpu_to_le16(NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED);
689 c.create_cq.irq_vector = cpu_to_le16(0);
691 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
694 static int apple_nvme_remove_cq(struct apple_nvme *anv)
696 struct nvme_command c = {};
698 c.delete_queue.opcode = nvme_admin_delete_cq;
699 c.delete_queue.qid = cpu_to_le16(1);
701 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
704 static int apple_nvme_create_sq(struct apple_nvme *anv)
706 struct nvme_command c = {};
709 * Note: we (ab)use the fact that the prp fields survive if no data
710 * is attached to the request.
712 c.create_sq.opcode = nvme_admin_create_sq;
713 c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr);
714 c.create_sq.sqid = cpu_to_le16(1);
715 c.create_sq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1);
716 c.create_sq.sq_flags = cpu_to_le16(NVME_QUEUE_PHYS_CONTIG);
717 c.create_sq.cqid = cpu_to_le16(1);
719 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
722 static int apple_nvme_remove_sq(struct apple_nvme *anv)
724 struct nvme_command c = {};
726 c.delete_queue.opcode = nvme_admin_delete_sq;
727 c.delete_queue.qid = cpu_to_le16(1);
729 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0);
732 static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
733 const struct blk_mq_queue_data *bd)
735 struct nvme_ns *ns = hctx->queue->queuedata;
736 struct apple_nvme_queue *q = hctx->driver_data;
737 struct apple_nvme *anv = queue_to_apple_nvme(q);
738 struct request *req = bd->rq;
739 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
740 struct nvme_command *cmnd = &iod->cmd;
747 * We should not need to do this, but we're still using this to
748 * ensure we can drain requests on a dying queue.
750 if (unlikely(!READ_ONCE(q->enabled)))
751 return BLK_STS_IOERR;
753 if (!nvme_check_ready(&anv->ctrl, req, true))
754 return nvme_fail_nonready_command(&anv->ctrl, req);
756 ret = nvme_setup_cmd(ns, req);
760 if (blk_rq_nr_phys_segments(req)) {
761 ret = apple_nvme_map_data(anv, req, cmnd);
766 blk_mq_start_request(req);
767 apple_nvme_submit_cmd(q, cmnd);
771 nvme_cleanup_cmd(req);
775 static int apple_nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
776 unsigned int hctx_idx)
778 hctx->driver_data = data;
782 static int apple_nvme_init_request(struct blk_mq_tag_set *set,
783 struct request *req, unsigned int hctx_idx,
784 unsigned int numa_node)
786 struct apple_nvme_queue *q = set->driver_data;
787 struct apple_nvme *anv = queue_to_apple_nvme(q);
788 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
789 struct nvme_request *nreq = nvme_req(req);
792 nreq->ctrl = &anv->ctrl;
793 nreq->cmd = &iod->cmd;
798 static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown)
800 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS);
801 bool dead = false, freeze = false;
804 if (apple_rtkit_is_crashed(anv->rtk))
806 if (!(csts & NVME_CSTS_RDY))
808 if (csts & NVME_CSTS_CFS)
811 if (anv->ctrl.state == NVME_CTRL_LIVE ||
812 anv->ctrl.state == NVME_CTRL_RESETTING) {
814 nvme_start_freeze(&anv->ctrl);
818 * Give the controller a chance to complete all entered requests if
819 * doing a safe shutdown.
821 if (!dead && shutdown && freeze)
822 nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT);
824 nvme_stop_queues(&anv->ctrl);
827 if (READ_ONCE(anv->ioq.enabled)) {
828 apple_nvme_remove_sq(anv);
829 apple_nvme_remove_cq(anv);
833 nvme_shutdown_ctrl(&anv->ctrl);
834 nvme_disable_ctrl(&anv->ctrl);
837 WRITE_ONCE(anv->ioq.enabled, false);
838 WRITE_ONCE(anv->adminq.enabled, false);
839 mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */
840 nvme_stop_admin_queue(&anv->ctrl);
842 /* last chance to complete any requests before nvme_cancel_request */
843 spin_lock_irqsave(&anv->lock, flags);
844 apple_nvme_handle_cq(&anv->ioq, true);
845 apple_nvme_handle_cq(&anv->adminq, true);
846 spin_unlock_irqrestore(&anv->lock, flags);
848 blk_mq_tagset_busy_iter(&anv->tagset, nvme_cancel_request, &anv->ctrl);
849 blk_mq_tagset_busy_iter(&anv->admin_tagset, nvme_cancel_request,
851 blk_mq_tagset_wait_completed_request(&anv->tagset);
852 blk_mq_tagset_wait_completed_request(&anv->admin_tagset);
855 * The driver will not be starting up queues again if shutting down so
856 * must flush all entered requests to their failed completion to avoid
857 * deadlocking blk-mq hot-cpu notifier.
860 nvme_start_queues(&anv->ctrl);
861 nvme_start_admin_queue(&anv->ctrl);
865 static enum blk_eh_timer_return apple_nvme_timeout(struct request *req,
868 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
869 struct apple_nvme_queue *q = iod->q;
870 struct apple_nvme *anv = queue_to_apple_nvme(q);
872 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS);
874 if (anv->ctrl.state != NVME_CTRL_LIVE) {
877 * If we are resetting, connecting or deleting we should
878 * complete immediately because we may block controller
879 * teardown or setup sequence
880 * - ctrl disable/shutdown fabrics requests
882 * - initialization admin requests
883 * - I/O requests that entered after unquiescing and
884 * the controller stopped responding
886 * All other requests should be cancelled by the error
887 * recovery work, so it's fine that we fail it here.
890 "I/O %d(aq:%d) timeout while not in live state\n",
891 req->tag, q->is_adminq);
892 if (blk_mq_request_started(req) &&
893 !blk_mq_request_completed(req)) {
894 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
895 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
896 blk_mq_complete_request(req);
901 /* check if we just missed an interrupt if we're still alive */
902 if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) {
903 spin_lock_irqsave(&anv->lock, flags);
904 apple_nvme_handle_cq(q, false);
905 spin_unlock_irqrestore(&anv->lock, flags);
906 if (blk_mq_request_completed(req)) {
908 "I/O %d(aq:%d) timeout: completion polled\n",
909 req->tag, q->is_adminq);
915 * aborting commands isn't supported which leaves a full reset as our
918 dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n",
919 req->tag, q->is_adminq);
920 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
921 apple_nvme_disable(anv, false);
922 nvme_reset_ctrl(&anv->ctrl);
926 static int apple_nvme_poll(struct blk_mq_hw_ctx *hctx,
927 struct io_comp_batch *iob)
929 struct apple_nvme_queue *q = hctx->driver_data;
930 struct apple_nvme *anv = queue_to_apple_nvme(q);
934 spin_lock_irqsave(&anv->lock, flags);
935 found = apple_nvme_poll_cq(q, iob);
936 spin_unlock_irqrestore(&anv->lock, flags);
941 static const struct blk_mq_ops apple_nvme_mq_admin_ops = {
942 .queue_rq = apple_nvme_queue_rq,
943 .complete = apple_nvme_complete_rq,
944 .init_hctx = apple_nvme_init_hctx,
945 .init_request = apple_nvme_init_request,
946 .timeout = apple_nvme_timeout,
949 static const struct blk_mq_ops apple_nvme_mq_ops = {
950 .queue_rq = apple_nvme_queue_rq,
951 .complete = apple_nvme_complete_rq,
952 .init_hctx = apple_nvme_init_hctx,
953 .init_request = apple_nvme_init_request,
954 .timeout = apple_nvme_timeout,
955 .poll = apple_nvme_poll,
958 static void apple_nvme_init_queue(struct apple_nvme_queue *q)
960 unsigned int depth = apple_nvme_queue_depth(q);
965 APPLE_ANS_MAX_QUEUE_DEPTH * sizeof(struct apple_nvmmu_tcb));
966 memset(q->cqes, 0, depth * sizeof(struct nvme_completion));
967 WRITE_ONCE(q->enabled, true);
968 wmb(); /* ensure the first interrupt sees the initialization */
971 static void apple_nvme_reset_work(struct work_struct *work)
973 unsigned int nr_io_queues = 1;
975 u32 boot_status, aqa;
976 struct apple_nvme *anv =
977 container_of(work, struct apple_nvme, ctrl.reset_work);
979 if (anv->ctrl.state != NVME_CTRL_RESETTING) {
980 dev_warn(anv->dev, "ctrl state %d is not RESETTING\n",
986 /* there's unfortunately no known way to recover if RTKit crashed :( */
987 if (apple_rtkit_is_crashed(anv->rtk)) {
989 "RTKit has crashed without any way to recover.");
994 if (anv->ctrl.ctrl_config & NVME_CC_ENABLE)
995 apple_nvme_disable(anv, false);
997 /* RTKit must be shut down cleanly for the (soft)-reset to work */
998 if (apple_rtkit_is_running(anv->rtk)) {
999 dev_dbg(anv->dev, "Trying to shut down RTKit before reset.");
1000 ret = apple_rtkit_shutdown(anv->rtk);
1005 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1007 ret = reset_control_assert(anv->reset);
1011 ret = apple_rtkit_reinit(anv->rtk);
1015 ret = reset_control_deassert(anv->reset);
1019 writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN,
1020 anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1021 ret = apple_rtkit_boot(anv->rtk);
1023 dev_err(anv->dev, "ANS did not boot");
1027 ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS,
1029 boot_status == APPLE_ANS_BOOT_STATUS_OK,
1030 USEC_PER_MSEC, APPLE_ANS_BOOT_TIMEOUT);
1032 dev_err(anv->dev, "ANS did not initialize");
1036 dev_dbg(anv->dev, "ANS booted successfully.");
1039 * Limit the max command size to prevent iod->sg allocations going
1040 * over a single page.
1042 anv->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_KB_SZ << 1,
1043 dma_max_mapping_size(anv->dev) >> 9);
1044 anv->ctrl.max_segments = NVME_MAX_SEGS;
1047 * Enable NVMMU and linear submission queues.
1048 * While we could keep those disabled and pretend this is slightly
1049 * more common NVMe controller we'd still need some quirks (e.g.
1050 * sq entries will be 128 bytes) and Apple might drop support for
1051 * that mode in the future.
1053 writel(APPLE_ANS_LINEAR_SQ_EN,
1054 anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL);
1056 /* Allow as many pending command as possible for both queues */
1057 writel(APPLE_ANS_MAX_QUEUE_DEPTH | (APPLE_ANS_MAX_QUEUE_DEPTH << 16),
1058 anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL);
1060 /* Setup the NVMMU for the maximum admin and IO queue depth */
1061 writel(APPLE_ANS_MAX_QUEUE_DEPTH - 1,
1062 anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS);
1065 * This is probably a chicken bit: without it all commands where any PRP
1066 * is set to zero (including those that don't use that field) fail and
1067 * the co-processor complains about "completed with err BAD_CMD-" or
1068 * a "NULL_PRP_PTR_ERR" in the syslog
1070 writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) &
1071 ~APPLE_ANS_PRP_NULL_CHECK,
1072 anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL);
1074 /* Setup the admin queue */
1075 aqa = APPLE_NVME_AQ_DEPTH - 1;
1077 writel(aqa, anv->mmio_nvme + NVME_REG_AQA);
1078 writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ);
1079 writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ);
1081 /* Setup NVMMU for both queues */
1082 writeq(anv->adminq.tcb_dma_addr,
1083 anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE);
1084 writeq(anv->ioq.tcb_dma_addr,
1085 anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE);
1088 APPLE_ANS_MAX_QUEUE_DEPTH - 1; /* 0's based queue depth */
1089 anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP);
1091 dev_dbg(anv->dev, "Enabling controller now");
1092 ret = nvme_enable_ctrl(&anv->ctrl);
1096 dev_dbg(anv->dev, "Starting admin queue");
1097 apple_nvme_init_queue(&anv->adminq);
1098 nvme_start_admin_queue(&anv->ctrl);
1100 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) {
1101 dev_warn(anv->ctrl.device,
1102 "failed to mark controller CONNECTING\n");
1107 ret = nvme_init_ctrl_finish(&anv->ctrl);
1111 dev_dbg(anv->dev, "Creating IOCQ");
1112 ret = apple_nvme_create_cq(anv);
1115 dev_dbg(anv->dev, "Creating IOSQ");
1116 ret = apple_nvme_create_sq(anv);
1120 apple_nvme_init_queue(&anv->ioq);
1122 ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues);
1125 if (nr_io_queues != 1) {
1130 anv->ctrl.queue_count = nr_io_queues + 1;
1132 nvme_start_queues(&anv->ctrl);
1133 nvme_wait_freeze(&anv->ctrl);
1134 blk_mq_update_nr_hw_queues(&anv->tagset, 1);
1135 nvme_unfreeze(&anv->ctrl);
1137 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_LIVE)) {
1138 dev_warn(anv->ctrl.device,
1139 "failed to mark controller live state\n");
1144 nvme_start_ctrl(&anv->ctrl);
1146 dev_dbg(anv->dev, "ANS boot and NVMe init completed.");
1150 apple_nvme_remove_sq(anv);
1152 apple_nvme_remove_cq(anv);
1154 dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret);
1155 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING);
1156 nvme_get_ctrl(&anv->ctrl);
1157 apple_nvme_disable(anv, false);
1158 nvme_kill_queues(&anv->ctrl);
1159 if (!queue_work(nvme_wq, &anv->remove_work))
1160 nvme_put_ctrl(&anv->ctrl);
1163 static void apple_nvme_remove_dead_ctrl_work(struct work_struct *work)
1165 struct apple_nvme *anv =
1166 container_of(work, struct apple_nvme, remove_work);
1168 nvme_put_ctrl(&anv->ctrl);
1169 device_release_driver(anv->dev);
1172 static int apple_nvme_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
1174 *val = readl(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off);
1178 static int apple_nvme_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
1180 writel(val, ctrl_to_apple_nvme(ctrl)->mmio_nvme + off);
1184 static int apple_nvme_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
1186 *val = readq(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off);
1190 static int apple_nvme_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
1192 struct device *dev = ctrl_to_apple_nvme(ctrl)->dev;
1194 return snprintf(buf, size, "%s\n", dev_name(dev));
1197 static void apple_nvme_free_ctrl(struct nvme_ctrl *ctrl)
1199 struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl);
1201 if (anv->ctrl.admin_q)
1202 blk_put_queue(anv->ctrl.admin_q);
1203 put_device(anv->dev);
1206 static const struct nvme_ctrl_ops nvme_ctrl_ops = {
1207 .name = "apple-nvme",
1208 .module = THIS_MODULE,
1210 .reg_read32 = apple_nvme_reg_read32,
1211 .reg_write32 = apple_nvme_reg_write32,
1212 .reg_read64 = apple_nvme_reg_read64,
1213 .free_ctrl = apple_nvme_free_ctrl,
1214 .get_address = apple_nvme_get_address,
1217 static void apple_nvme_async_probe(void *data, async_cookie_t cookie)
1219 struct apple_nvme *anv = data;
1221 flush_work(&anv->ctrl.reset_work);
1222 flush_work(&anv->ctrl.scan_work);
1223 nvme_put_ctrl(&anv->ctrl);
1226 static int apple_nvme_alloc_tagsets(struct apple_nvme *anv)
1230 anv->admin_tagset.ops = &apple_nvme_mq_admin_ops;
1231 anv->admin_tagset.nr_hw_queues = 1;
1232 anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH;
1233 anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT;
1234 anv->admin_tagset.numa_node = NUMA_NO_NODE;
1235 anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod);
1236 anv->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
1237 anv->admin_tagset.driver_data = &anv->adminq;
1239 ret = blk_mq_alloc_tag_set(&anv->admin_tagset);
1242 ret = devm_add_action_or_reset(anv->dev,
1243 (void (*)(void *))blk_mq_free_tag_set,
1244 &anv->admin_tagset);
1248 anv->tagset.ops = &apple_nvme_mq_ops;
1249 anv->tagset.nr_hw_queues = 1;
1250 anv->tagset.nr_maps = 1;
1252 * Tags are used as an index to the NVMMU and must be unique across
1253 * both queues. The admin queue gets the first APPLE_NVME_AQ_DEPTH which
1254 * must be marked as reserved in the IO queue.
1256 anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH;
1257 anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1;
1258 anv->tagset.timeout = NVME_IO_TIMEOUT;
1259 anv->tagset.numa_node = NUMA_NO_NODE;
1260 anv->tagset.cmd_size = sizeof(struct apple_nvme_iod);
1261 anv->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
1262 anv->tagset.driver_data = &anv->ioq;
1264 ret = blk_mq_alloc_tag_set(&anv->tagset);
1267 ret = devm_add_action_or_reset(
1268 anv->dev, (void (*)(void *))blk_mq_free_tag_set, &anv->tagset);
1272 anv->ctrl.admin_tagset = &anv->admin_tagset;
1273 anv->ctrl.tagset = &anv->tagset;
1278 static int apple_nvme_queue_alloc(struct apple_nvme *anv,
1279 struct apple_nvme_queue *q)
1281 unsigned int depth = apple_nvme_queue_depth(q);
1283 q->cqes = dmam_alloc_coherent(anv->dev,
1284 depth * sizeof(struct nvme_completion),
1285 &q->cq_dma_addr, GFP_KERNEL);
1289 q->sqes = dmam_alloc_coherent(anv->dev,
1290 depth * sizeof(struct nvme_command),
1291 &q->sq_dma_addr, GFP_KERNEL);
1296 * We need the maximum queue depth here because the NVMMU only has a
1297 * single depth configuration shared between both queues.
1299 q->tcbs = dmam_alloc_coherent(anv->dev,
1300 APPLE_ANS_MAX_QUEUE_DEPTH *
1301 sizeof(struct apple_nvmmu_tcb),
1302 &q->tcb_dma_addr, GFP_KERNEL);
1307 * initialize phase to make sure the allocated and empty memory
1308 * doesn't look like a full cq already.
1314 static void apple_nvme_detach_genpd(struct apple_nvme *anv)
1318 if (anv->pd_count <= 1)
1321 for (i = anv->pd_count - 1; i >= 0; i--) {
1322 if (anv->pd_link[i])
1323 device_link_del(anv->pd_link[i]);
1324 if (!IS_ERR_OR_NULL(anv->pd_dev[i]))
1325 dev_pm_domain_detach(anv->pd_dev[i], true);
1329 static int apple_nvme_attach_genpd(struct apple_nvme *anv)
1331 struct device *dev = anv->dev;
1334 anv->pd_count = of_count_phandle_with_args(
1335 dev->of_node, "power-domains", "#power-domain-cells");
1336 if (anv->pd_count <= 1)
1339 anv->pd_dev = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_dev),
1344 anv->pd_link = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_link),
1349 for (i = 0; i < anv->pd_count; i++) {
1350 anv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i);
1351 if (IS_ERR(anv->pd_dev[i])) {
1352 apple_nvme_detach_genpd(anv);
1353 return PTR_ERR(anv->pd_dev[i]);
1356 anv->pd_link[i] = device_link_add(dev, anv->pd_dev[i],
1358 DL_FLAG_PM_RUNTIME |
1359 DL_FLAG_RPM_ACTIVE);
1360 if (!anv->pd_link[i]) {
1361 apple_nvme_detach_genpd(anv);
1369 static int apple_nvme_probe(struct platform_device *pdev)
1371 struct device *dev = &pdev->dev;
1372 struct apple_nvme *anv;
1375 anv = devm_kzalloc(dev, sizeof(*anv), GFP_KERNEL);
1379 anv->dev = get_device(dev);
1380 anv->adminq.is_adminq = true;
1381 platform_set_drvdata(pdev, anv);
1383 ret = apple_nvme_attach_genpd(anv);
1385 dev_err_probe(dev, ret, "Failed to attach power domains");
1388 if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) {
1393 anv->irq = platform_get_irq(pdev, 0);
1403 anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans");
1404 if (IS_ERR(anv->mmio_coproc)) {
1405 ret = PTR_ERR(anv->mmio_coproc);
1408 anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme");
1409 if (IS_ERR(anv->mmio_nvme)) {
1410 ret = PTR_ERR(anv->mmio_nvme);
1414 anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB;
1415 anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB;
1416 anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB;
1417 anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB;
1419 anv->sart = devm_apple_sart_get(dev);
1420 if (IS_ERR(anv->sart)) {
1421 ret = dev_err_probe(dev, PTR_ERR(anv->sart),
1422 "Failed to initialize SART");
1426 anv->reset = devm_reset_control_array_get_exclusive(anv->dev);
1427 if (IS_ERR(anv->reset)) {
1428 ret = dev_err_probe(dev, PTR_ERR(anv->reset),
1429 "Failed to get reset control");
1433 INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work);
1434 INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work);
1435 spin_lock_init(&anv->lock);
1437 ret = apple_nvme_queue_alloc(anv, &anv->adminq);
1440 ret = apple_nvme_queue_alloc(anv, &anv->ioq);
1444 anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev,
1445 NVME_CTRL_PAGE_SIZE,
1446 NVME_CTRL_PAGE_SIZE, 0);
1447 if (!anv->prp_page_pool) {
1452 anv->prp_small_pool =
1453 dmam_pool_create("prp list 256", anv->dev, 256, 256, 0);
1454 if (!anv->prp_small_pool) {
1459 WARN_ON_ONCE(apple_nvme_iod_alloc_size() > PAGE_SIZE);
1461 mempool_create_kmalloc_pool(1, apple_nvme_iod_alloc_size());
1462 if (!anv->iod_mempool) {
1466 ret = devm_add_action_or_reset(
1467 anv->dev, (void (*)(void *))mempool_destroy, anv->iod_mempool);
1471 ret = apple_nvme_alloc_tagsets(anv);
1475 ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0,
1478 dev_err_probe(dev, ret, "Failed to request IRQ");
1483 devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops);
1484 if (IS_ERR(anv->rtk)) {
1485 ret = dev_err_probe(dev, PTR_ERR(anv->rtk),
1486 "Failed to initialize RTKit");
1490 ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops,
1491 NVME_QUIRK_SKIP_CID_GEN);
1493 dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl");
1497 anv->ctrl.admin_q = blk_mq_init_queue(&anv->admin_tagset);
1498 if (IS_ERR(anv->ctrl.admin_q)) {
1503 if (!blk_get_queue(anv->ctrl.admin_q)) {
1504 nvme_start_admin_queue(&anv->ctrl);
1505 blk_cleanup_queue(anv->ctrl.admin_q);
1506 anv->ctrl.admin_q = NULL;
1511 nvme_reset_ctrl(&anv->ctrl);
1512 async_schedule(apple_nvme_async_probe, anv);
1517 put_device(anv->dev);
1521 static int apple_nvme_remove(struct platform_device *pdev)
1523 struct apple_nvme *anv = platform_get_drvdata(pdev);
1525 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING);
1526 flush_work(&anv->ctrl.reset_work);
1527 nvme_stop_ctrl(&anv->ctrl);
1528 nvme_remove_namespaces(&anv->ctrl);
1529 apple_nvme_disable(anv, true);
1530 nvme_uninit_ctrl(&anv->ctrl);
1532 if (apple_rtkit_is_running(anv->rtk))
1533 apple_rtkit_shutdown(anv->rtk);
1535 apple_nvme_detach_genpd(anv);
1540 static void apple_nvme_shutdown(struct platform_device *pdev)
1542 struct apple_nvme *anv = platform_get_drvdata(pdev);
1544 apple_nvme_disable(anv, true);
1545 if (apple_rtkit_is_running(anv->rtk))
1546 apple_rtkit_shutdown(anv->rtk);
1549 static int apple_nvme_resume(struct device *dev)
1551 struct apple_nvme *anv = dev_get_drvdata(dev);
1553 return nvme_reset_ctrl(&anv->ctrl);
1556 static int apple_nvme_suspend(struct device *dev)
1558 struct apple_nvme *anv = dev_get_drvdata(dev);
1561 apple_nvme_disable(anv, true);
1563 if (apple_rtkit_is_running(anv->rtk))
1564 ret = apple_rtkit_shutdown(anv->rtk);
1566 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL);
1571 static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend,
1574 static const struct of_device_id apple_nvme_of_match[] = {
1575 { .compatible = "apple,nvme-ans2" },
1578 MODULE_DEVICE_TABLE(of, apple_nvme_of_match);
1580 static struct platform_driver apple_nvme_driver = {
1582 .name = "nvme-apple",
1583 .of_match_table = apple_nvme_of_match,
1584 .pm = pm_sleep_ptr(&apple_nvme_pm_ops),
1586 .probe = apple_nvme_probe,
1587 .remove = apple_nvme_remove,
1588 .shutdown = apple_nvme_shutdown,
1590 module_platform_driver(apple_nvme_driver);
1592 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1593 MODULE_LICENSE("GPL");