1 // SPDX-License-Identifier: GPL-2.0-only
3 * Intel MIC Platform Software Stack (MPSS)
5 * Copyright(c) 2013 Intel Corporation.
7 * Intel MIC Host driver.
9 #include <linux/delay.h>
10 #include <linux/firmware.h>
11 #include <linux/pci.h>
12 #include <linux/kmod.h>
13 #include <linux/dma-map-ops.h>
14 #include <linux/mic_common.h>
15 #include <linux/mic_bus.h>
16 #include "../bus/scif_bus.h"
17 #include "../bus/vop_bus.h"
18 #include "../common/mic_dev.h"
19 #include "mic_device.h"
22 static inline struct mic_device *vpdev_to_mdev(struct device *dev)
24 return dev_get_drvdata(dev->parent);
28 _mic_dma_map_page(struct device *dev, struct page *page,
29 unsigned long offset, size_t size,
30 enum dma_data_direction dir, unsigned long attrs)
32 void *va = phys_to_virt(page_to_phys(page)) + offset;
33 struct mic_device *mdev = vpdev_to_mdev(dev);
35 return mic_map_single(mdev, va, size);
38 static void _mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
39 size_t size, enum dma_data_direction dir,
42 struct mic_device *mdev = vpdev_to_mdev(dev);
44 mic_unmap_single(mdev, dma_addr, size);
47 static const struct dma_map_ops _mic_dma_ops = {
48 .map_page = _mic_dma_map_page,
49 .unmap_page = _mic_dma_unmap_page,
52 static struct mic_irq *
53 __mic_request_irq(struct vop_device *vpdev,
54 irqreturn_t (*func)(int irq, void *data),
55 const char *name, void *data, int intr_src)
57 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
59 return mic_request_threaded_irq(mdev, func, NULL, name, data,
60 intr_src, MIC_INTR_DB);
63 static void __mic_free_irq(struct vop_device *vpdev,
64 struct mic_irq *cookie, void *data)
66 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
68 mic_free_irq(mdev, cookie, data);
71 static void __mic_ack_interrupt(struct vop_device *vpdev, int num)
73 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
75 mdev->ops->intr_workarounds(mdev);
78 static int __mic_next_db(struct vop_device *vpdev)
80 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
82 return mic_next_db(mdev);
85 static void *__mic_get_dp(struct vop_device *vpdev)
87 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
92 static void __iomem *__mic_get_remote_dp(struct vop_device *vpdev)
97 static void __mic_send_intr(struct vop_device *vpdev, int db)
99 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
101 mdev->ops->send_intr(mdev, db);
104 static void __iomem *__mic_ioremap(struct vop_device *vpdev,
105 dma_addr_t pa, size_t len)
107 struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
109 return mdev->aper.va + pa;
112 static void __mic_iounmap(struct vop_device *vpdev, void __iomem *va)
117 static struct vop_hw_ops vop_hw_ops = {
118 .request_irq = __mic_request_irq,
119 .free_irq = __mic_free_irq,
120 .ack_interrupt = __mic_ack_interrupt,
121 .next_db = __mic_next_db,
122 .get_dp = __mic_get_dp,
123 .get_remote_dp = __mic_get_remote_dp,
124 .send_intr = __mic_send_intr,
125 .remap = __mic_ioremap,
126 .unmap = __mic_iounmap,
129 static inline struct mic_device *scdev_to_mdev(struct scif_hw_dev *scdev)
131 return dev_get_drvdata(scdev->dev.parent);
134 static void *__mic_dma_alloc(struct device *dev, size_t size,
135 dma_addr_t *dma_handle, gfp_t gfp,
138 struct scif_hw_dev *scdev = dev_get_drvdata(dev);
139 struct mic_device *mdev = scdev_to_mdev(scdev);
141 void *va = kzalloc(size, gfp);
144 tmp = mic_map_single(mdev, va, size);
145 if (dma_mapping_error(dev, tmp)) {
155 static void __mic_dma_free(struct device *dev, size_t size, void *vaddr,
156 dma_addr_t dma_handle, unsigned long attrs)
158 struct scif_hw_dev *scdev = dev_get_drvdata(dev);
159 struct mic_device *mdev = scdev_to_mdev(scdev);
161 mic_unmap_single(mdev, dma_handle, size);
166 __mic_dma_map_page(struct device *dev, struct page *page, unsigned long offset,
167 size_t size, enum dma_data_direction dir,
170 void *va = phys_to_virt(page_to_phys(page)) + offset;
171 struct scif_hw_dev *scdev = dev_get_drvdata(dev);
172 struct mic_device *mdev = scdev_to_mdev(scdev);
174 return mic_map_single(mdev, va, size);
178 __mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
179 size_t size, enum dma_data_direction dir,
182 struct scif_hw_dev *scdev = dev_get_drvdata(dev);
183 struct mic_device *mdev = scdev_to_mdev(scdev);
185 mic_unmap_single(mdev, dma_addr, size);
188 static int __mic_dma_map_sg(struct device *dev, struct scatterlist *sg,
189 int nents, enum dma_data_direction dir,
192 struct scif_hw_dev *scdev = dev_get_drvdata(dev);
193 struct mic_device *mdev = scdev_to_mdev(scdev);
194 struct scatterlist *s;
198 ret = dma_map_sg(&mdev->pdev->dev, sg, nents, dir);
202 for_each_sg(sg, s, nents, i) {
203 da = mic_map(mdev, sg_dma_address(s) + s->offset, s->length);
206 sg_dma_address(s) = da;
210 for_each_sg(sg, s, i, j) {
211 mic_unmap(mdev, sg_dma_address(s), s->length);
212 sg_dma_address(s) = mic_to_dma_addr(mdev, sg_dma_address(s));
214 dma_unmap_sg(&mdev->pdev->dev, sg, nents, dir);
218 static void __mic_dma_unmap_sg(struct device *dev,
219 struct scatterlist *sg, int nents,
220 enum dma_data_direction dir,
223 struct scif_hw_dev *scdev = dev_get_drvdata(dev);
224 struct mic_device *mdev = scdev_to_mdev(scdev);
225 struct scatterlist *s;
229 for_each_sg(sg, s, nents, i) {
230 da = mic_to_dma_addr(mdev, sg_dma_address(s));
231 mic_unmap(mdev, sg_dma_address(s), s->length);
232 sg_dma_address(s) = da;
234 dma_unmap_sg(&mdev->pdev->dev, sg, nents, dir);
237 static const struct dma_map_ops __mic_dma_ops = {
238 .alloc = __mic_dma_alloc,
239 .free = __mic_dma_free,
240 .map_page = __mic_dma_map_page,
241 .unmap_page = __mic_dma_unmap_page,
242 .map_sg = __mic_dma_map_sg,
243 .unmap_sg = __mic_dma_unmap_sg,
246 static struct mic_irq *
247 ___mic_request_irq(struct scif_hw_dev *scdev,
248 irqreturn_t (*func)(int irq, void *data),
252 struct mic_device *mdev = scdev_to_mdev(scdev);
254 return mic_request_threaded_irq(mdev, func, NULL, name, data,
259 ___mic_free_irq(struct scif_hw_dev *scdev,
260 struct mic_irq *cookie, void *data)
262 struct mic_device *mdev = scdev_to_mdev(scdev);
264 mic_free_irq(mdev, cookie, data);
267 static void ___mic_ack_interrupt(struct scif_hw_dev *scdev, int num)
269 struct mic_device *mdev = scdev_to_mdev(scdev);
271 mdev->ops->intr_workarounds(mdev);
274 static int ___mic_next_db(struct scif_hw_dev *scdev)
276 struct mic_device *mdev = scdev_to_mdev(scdev);
278 return mic_next_db(mdev);
281 static void ___mic_send_intr(struct scif_hw_dev *scdev, int db)
283 struct mic_device *mdev = scdev_to_mdev(scdev);
285 mdev->ops->send_intr(mdev, db);
288 static void __iomem *___mic_ioremap(struct scif_hw_dev *scdev,
289 phys_addr_t pa, size_t len)
291 struct mic_device *mdev = scdev_to_mdev(scdev);
293 return mdev->aper.va + pa;
296 static void ___mic_iounmap(struct scif_hw_dev *scdev, void __iomem *va)
301 static struct scif_hw_ops scif_hw_ops = {
302 .request_irq = ___mic_request_irq,
303 .free_irq = ___mic_free_irq,
304 .ack_interrupt = ___mic_ack_interrupt,
305 .next_db = ___mic_next_db,
306 .send_intr = ___mic_send_intr,
307 .remap = ___mic_ioremap,
308 .unmap = ___mic_iounmap,
311 static inline struct mic_device *mbdev_to_mdev(struct mbus_device *mbdev)
313 return dev_get_drvdata(mbdev->dev.parent);
317 mic_dma_map_page(struct device *dev, struct page *page,
318 unsigned long offset, size_t size, enum dma_data_direction dir,
321 void *va = phys_to_virt(page_to_phys(page)) + offset;
322 struct mic_device *mdev = dev_get_drvdata(dev->parent);
324 return mic_map_single(mdev, va, size);
328 mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
329 size_t size, enum dma_data_direction dir,
332 struct mic_device *mdev = dev_get_drvdata(dev->parent);
333 mic_unmap_single(mdev, dma_addr, size);
336 static const struct dma_map_ops mic_dma_ops = {
337 .map_page = mic_dma_map_page,
338 .unmap_page = mic_dma_unmap_page,
341 static struct mic_irq *
342 _mic_request_threaded_irq(struct mbus_device *mbdev,
343 irq_handler_t handler, irq_handler_t thread_fn,
344 const char *name, void *data, int intr_src)
346 return mic_request_threaded_irq(mbdev_to_mdev(mbdev), handler,
347 thread_fn, name, data,
348 intr_src, MIC_INTR_DMA);
351 static void _mic_free_irq(struct mbus_device *mbdev,
352 struct mic_irq *cookie, void *data)
354 mic_free_irq(mbdev_to_mdev(mbdev), cookie, data);
357 static void _mic_ack_interrupt(struct mbus_device *mbdev, int num)
359 struct mic_device *mdev = mbdev_to_mdev(mbdev);
360 mdev->ops->intr_workarounds(mdev);
363 static struct mbus_hw_ops mbus_hw_ops = {
364 .request_threaded_irq = _mic_request_threaded_irq,
365 .free_irq = _mic_free_irq,
366 .ack_interrupt = _mic_ack_interrupt,
369 /* Initialize the MIC bootparams */
370 void mic_bootparam_init(struct mic_device *mdev)
372 struct mic_bootparam *bootparam = mdev->dp;
374 bootparam->magic = cpu_to_le32(MIC_MAGIC);
375 bootparam->h2c_config_db = -1;
376 bootparam->node_id = mdev->id + 1;
377 bootparam->scif_host_dma_addr = 0x0;
378 bootparam->scif_card_dma_addr = 0x0;
379 bootparam->c2h_scif_db = -1;
380 bootparam->h2c_scif_db = -1;
383 static inline struct mic_device *cosmdev_to_mdev(struct cosm_device *cdev)
385 return dev_get_drvdata(cdev->dev.parent);
388 static void _mic_reset(struct cosm_device *cdev)
390 struct mic_device *mdev = cosmdev_to_mdev(cdev);
392 mdev->ops->reset_fw_ready(mdev);
393 mdev->ops->reset(mdev);
396 static bool _mic_ready(struct cosm_device *cdev)
398 struct mic_device *mdev = cosmdev_to_mdev(cdev);
400 return mdev->ops->is_fw_ready(mdev);
404 * mic_request_dma_chans - Request DMA channels
405 * @mdev: pointer to mic_device instance
407 * returns number of DMA channels acquired
409 static int mic_request_dma_chans(struct mic_device *mdev)
412 struct dma_chan *chan;
415 dma_cap_set(DMA_MEMCPY, mask);
418 chan = dma_request_channel(mask, mdev->ops->dma_filter,
421 mdev->dma_ch[mdev->num_dma_ch++] = chan;
422 if (mdev->num_dma_ch >= MIC_MAX_DMA_CHAN)
426 dev_info(&mdev->pdev->dev, "DMA channels # %d\n", mdev->num_dma_ch);
427 return mdev->num_dma_ch;
431 * mic_free_dma_chans - release DMA channels
432 * @mdev: pointer to mic_device instance
436 static void mic_free_dma_chans(struct mic_device *mdev)
440 for (i = 0; i < mdev->num_dma_ch; i++) {
441 dma_release_channel(mdev->dma_ch[i]);
442 mdev->dma_ch[i] = NULL;
444 mdev->num_dma_ch = 0;
448 * _mic_start - Start the MIC.
449 * @cdev: pointer to cosm_device instance
450 * @id: MIC device id/index provided by COSM used in other drivers like SCIF
452 * This function prepares an MIC for boot and initiates boot.
453 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
455 * For all cosm_hw_ops the caller holds a mutex to ensure serialization.
457 static int _mic_start(struct cosm_device *cdev, int id)
459 struct mic_device *mdev = cosmdev_to_mdev(cdev);
462 mic_bootparam_init(mdev);
463 mdev->dma_mbdev = mbus_register_device(&mdev->pdev->dev,
464 MBUS_DEV_DMA_HOST, &mic_dma_ops,
465 &mbus_hw_ops, id, mdev->mmio.va);
466 if (IS_ERR(mdev->dma_mbdev)) {
467 rc = PTR_ERR(mdev->dma_mbdev);
470 if (!mic_request_dma_chans(mdev)) {
474 mdev->scdev = scif_register_device(&mdev->pdev->dev, MIC_SCIF_DEV,
475 &__mic_dma_ops, &scif_hw_ops,
476 id + 1, 0, &mdev->mmio,
477 &mdev->aper, mdev->dp, NULL,
478 mdev->dma_ch, mdev->num_dma_ch,
480 if (IS_ERR(mdev->scdev)) {
481 rc = PTR_ERR(mdev->scdev);
485 mdev->vpdev = vop_register_device(&mdev->pdev->dev,
486 VOP_DEV_TRNSP, &_mic_dma_ops,
487 &vop_hw_ops, id + 1, &mdev->aper,
489 if (IS_ERR(mdev->vpdev)) {
490 rc = PTR_ERR(mdev->vpdev);
494 rc = mdev->ops->load_mic_fw(mdev, NULL);
497 mic_smpt_restore(mdev);
498 mic_intr_restore(mdev);
499 mdev->intr_ops->enable_interrupts(mdev);
500 mdev->ops->write_spad(mdev, MIC_DPLO_SPAD, mdev->dp_dma_addr);
501 mdev->ops->write_spad(mdev, MIC_DPHI_SPAD, mdev->dp_dma_addr >> 32);
502 mdev->ops->send_firmware_intr(mdev);
505 vop_unregister_device(mdev->vpdev);
507 scif_unregister_device(mdev->scdev);
509 mic_free_dma_chans(mdev);
511 mbus_unregister_device(mdev->dma_mbdev);
517 * _mic_stop - Prepare the MIC for reset and trigger reset.
518 * @cdev: pointer to cosm_device instance
519 * @force: force a MIC to reset even if it is already offline.
523 static void _mic_stop(struct cosm_device *cdev, bool force)
525 struct mic_device *mdev = cosmdev_to_mdev(cdev);
528 * Since SCIF handles card shutdown and reset (using COSM), it will
529 * will be the first to be registered and the last to be
532 vop_unregister_device(mdev->vpdev);
533 scif_unregister_device(mdev->scdev);
534 mic_free_dma_chans(mdev);
535 mbus_unregister_device(mdev->dma_mbdev);
536 mic_bootparam_init(mdev);
539 static ssize_t _mic_family(struct cosm_device *cdev, char *buf)
541 struct mic_device *mdev = cosmdev_to_mdev(cdev);
542 static const char *family[MIC_FAMILY_LAST] = { "x100", "Unknown" };
544 return scnprintf(buf, PAGE_SIZE, "%s\n", family[mdev->family]);
547 static ssize_t _mic_stepping(struct cosm_device *cdev, char *buf)
549 struct mic_device *mdev = cosmdev_to_mdev(cdev);
550 const char *string = "??";
552 switch (mdev->stepping) {
568 return scnprintf(buf, PAGE_SIZE, "%s\n", string);
571 static struct mic_mw *_mic_aper(struct cosm_device *cdev)
573 struct mic_device *mdev = cosmdev_to_mdev(cdev);
578 struct cosm_hw_ops cosm_hw_ops = {
580 .force_reset = _mic_reset,
585 .family = _mic_family,
586 .stepping = _mic_stepping,