1 /* SPDX-License-Identifier: GPL-2.0+ */
2 #include <linux/module.h>
3 #include <linux/init.h>
5 #include <linux/kernel.h> /* printk() */
6 #include <linux/slab.h> /* kmalloc() */
7 #include <linux/fs.h> /* everything... */
8 #include <linux/errno.h> /* error codes */
9 #include <linux/types.h> /* size_t */
10 #include <linux/cdev.h>
11 #include <linux/uaccess.h> /* copy_*_user */
12 #include <linux/highmem.h>
13 #include <linux/pagemap.h>
14 #include "kpc_dma_driver.h"
17 /********** Helper Functions **********/
19 unsigned int count_pages(unsigned long iov_base, size_t iov_len)
21 unsigned long first = (iov_base & PAGE_MASK) >> PAGE_SHIFT;
22 unsigned long last = ((iov_base+iov_len-1) & PAGE_MASK) >> PAGE_SHIFT;
24 return last - first + 1;
28 unsigned int count_parts_for_sge(struct scatterlist *sg)
30 return DIV_ROUND_UP(sg_dma_len(sg), 0x80000);
33 /********** Transfer Helpers **********/
34 static int kpc_dma_transfer(struct dev_private_data *priv,
35 unsigned long iov_base, size_t iov_len)
39 struct kpc_dma_device *ldev;
40 struct aio_cb_data *acd;
41 DECLARE_COMPLETION_ONSTACK(done);
43 struct scatterlist *sg;
45 struct kpc_dma_descriptor *desc;
54 acd = kzalloc(sizeof(*acd), GFP_KERNEL);
56 dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for for the aio data\n");
59 memset(acd, 0x66, sizeof(struct aio_cb_data));
62 acd->ldev = priv->ldev;
66 acd->page_count = count_pages(iov_base, iov_len);
68 // Allocate an array of page pointers
69 acd->user_pages = kzalloc(sizeof(struct page *) * acd->page_count, GFP_KERNEL);
70 if (!acd->user_pages) {
71 dev_err(&priv->ldev->pldev->dev, "Couldn't kmalloc space for for the page pointers\n");
73 goto err_alloc_userpages;
76 // Lock the user buffer pages in memory, and hold on to the page pointers (for the sglist)
77 down_read(¤t->mm->mmap_sem); /* get memory map semaphore */
78 rv = get_user_pages(iov_base, acd->page_count, FOLL_TOUCH | FOLL_WRITE | FOLL_GET, acd->user_pages, NULL);
79 up_read(¤t->mm->mmap_sem); /* release the semaphore */
80 if (rv != acd->page_count) {
81 dev_err(&priv->ldev->pldev->dev, "Couldn't get_user_pages (%ld)\n", rv);
82 goto err_get_user_pages;
85 // Allocate and setup the sg_table (scatterlist entries)
86 rv = sg_alloc_table_from_pages(&acd->sgt, acd->user_pages, acd->page_count, iov_base & (PAGE_SIZE-1), iov_len, GFP_KERNEL);
88 dev_err(&priv->ldev->pldev->dev, "Couldn't alloc sg_table (%ld)\n", rv);
89 goto err_alloc_sg_table;
92 // Setup the DMA mapping for all the sg entries
93 acd->mapped_entry_count = dma_map_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir);
94 if (acd->mapped_entry_count <= 0) {
95 dev_err(&priv->ldev->pldev->dev, "Couldn't dma_map_sg (%d)\n", acd->mapped_entry_count);
99 // Calculate how many descriptors are actually needed for this transfer.
100 for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) {
101 desc_needed += count_parts_for_sge(sg);
106 // Figoure out how many descriptors are available and return an error if there aren't enough
107 num_descrs_avail = count_descriptors_available(ldev);
108 dev_dbg(&priv->ldev->pldev->dev, " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
109 if (desc_needed >= ldev->desc_pool_cnt) {
110 dev_warn(&priv->ldev->pldev->dev, " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d TOO MANY to ever complete!\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
112 goto err_descr_too_many;
114 if (desc_needed > num_descrs_avail) {
115 dev_warn(&priv->ldev->pldev->dev, " mapped_entry_count = %d num_descrs_needed = %d num_descrs_avail = %d Too many to complete right now.\n", acd->mapped_entry_count, desc_needed, num_descrs_avail);
117 goto err_descr_too_many;
120 // Loop through all the sg table entries and fill out a descriptor for each one.
121 desc = ldev->desc_next;
122 card_addr = acd->priv->card_addr;
123 for_each_sg(acd->sgt.sgl, sg, acd->mapped_entry_count, i) {
124 pcnt = count_parts_for_sge(sg);
125 for (p = 0 ; p < pcnt ; p++) {
126 // Fill out the descriptor
127 BUG_ON(desc == NULL);
130 desc->DescByteCount = 0x80000;
132 desc->DescByteCount = sg_dma_len(sg) - (p * 0x80000);
134 desc->DescBufferByteCount = desc->DescByteCount;
136 desc->DescControlFlags |= DMA_DESC_CTL_IRQONERR;
137 if (i == 0 && p == 0)
138 desc->DescControlFlags |= DMA_DESC_CTL_SOP;
139 if (i == acd->mapped_entry_count-1 && p == pcnt-1)
140 desc->DescControlFlags |= DMA_DESC_CTL_EOP | DMA_DESC_CTL_IRQONDONE;
142 desc->DescCardAddrLS = (card_addr & 0xFFFFFFFF);
143 desc->DescCardAddrMS = (card_addr >> 32) & 0xF;
144 card_addr += desc->DescByteCount;
146 dma_addr = sg_dma_address(sg) + (p * 0x80000);
147 desc->DescSystemAddrLS = (dma_addr & 0x00000000FFFFFFFFUL) >> 0;
148 desc->DescSystemAddrMS = (dma_addr & 0xFFFFFFFF00000000UL) >> 32;
150 user_ctl = acd->priv->user_ctl;
151 if (i == acd->mapped_entry_count-1 && p == pcnt-1) {
152 user_ctl = acd->priv->user_ctl_last;
154 desc->DescUserControlLS = (user_ctl & 0x00000000FFFFFFFFUL) >> 0;
155 desc->DescUserControlMS = (user_ctl & 0xFFFFFFFF00000000UL) >> 32;
157 if (i == acd->mapped_entry_count-1 && p == pcnt-1)
160 dev_dbg(&priv->ldev->pldev->dev, " Filled descriptor %p (acd = %p)\n", desc, desc->acd);
162 ldev->desc_next = desc->Next;
167 // Send the filled descriptors off to the hardware to process!
168 SetEngineSWPtr(ldev, ldev->desc_next);
172 rv = wait_for_completion_interruptible(&done);
174 * If the user aborted (rv == -ERESTARTSYS), we're no longer responsible
175 * for cleaning up the acd
177 if (rv == -ERESTARTSYS)
187 dma_unmap_sg(&ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, ldev->dir);
188 sg_free_table(&acd->sgt);
191 for (i = 0 ; i < acd->page_count ; i++) {
192 put_page(acd->user_pages[i]);
195 kfree(acd->user_pages);
198 dev_dbg(&priv->ldev->pldev->dev, "%s returning with error %ld\n", __func__, rv);
202 void transfer_complete_cb(struct aio_cb_data *acd, size_t xfr_count, u32 flags)
207 BUG_ON(acd->user_pages == NULL);
208 BUG_ON(acd->sgt.sgl == NULL);
209 BUG_ON(acd->ldev == NULL);
210 BUG_ON(acd->ldev->pldev == NULL);
212 for (i = 0 ; i < acd->page_count ; i++) {
213 if (!PageReserved(acd->user_pages[i])) {
214 set_page_dirty(acd->user_pages[i]);
218 dma_unmap_sg(&acd->ldev->pldev->dev, acd->sgt.sgl, acd->sgt.nents, acd->ldev->dir);
220 for (i = 0 ; i < acd->page_count ; i++) {
221 put_page(acd->user_pages[i]);
224 sg_free_table(&acd->sgt);
226 kfree(acd->user_pages);
234 * There's no completion, so we're responsible for cleaning up
241 /********** Fileops **********/
243 int kpc_dma_open(struct inode *inode, struct file *filp)
245 struct dev_private_data *priv;
246 struct kpc_dma_device *ldev = kpc_dma_lookup_device(iminor(inode));
251 if (!atomic_dec_and_test(&ldev->open_count)) {
252 atomic_inc(&ldev->open_count);
253 return -EBUSY; /* already open */
256 priv = kzalloc(sizeof(struct dev_private_data), GFP_KERNEL);
261 filp->private_data = priv;
267 int kpc_dma_close(struct inode *inode, struct file *filp)
269 struct kpc_dma_descriptor *cur;
270 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
271 struct kpc_dma_device *eng = priv->ldev;
275 stop_dma_engine(eng);
277 cur = eng->desc_completed->Next;
278 while (cur != eng->desc_next) {
279 dev_dbg(&eng->pldev->dev, "Aborting descriptor %p (acd = %p)\n", cur, cur->acd);
280 if (cur->DescControlFlags & DMA_DESC_CTL_EOP) {
282 transfer_complete_cb(cur->acd, 0, ACD_FLAG_ABORT);
286 eng->desc_completed = cur;
291 start_dma_engine(eng);
295 atomic_inc(&priv->ldev->open_count); /* release the device */
301 ssize_t kpc_dma_read(struct file *filp, char __user *user_buf, size_t count, loff_t *ppos)
303 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
305 if (priv->ldev->dir != DMA_FROM_DEVICE)
308 return kpc_dma_transfer(priv, (unsigned long)user_buf, count);
312 ssize_t kpc_dma_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos)
314 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
316 if (priv->ldev->dir != DMA_TO_DEVICE)
319 return kpc_dma_transfer(priv, (unsigned long)user_buf, count);
323 long kpc_dma_ioctl(struct file *filp, unsigned int ioctl_num, unsigned long ioctl_param)
325 struct dev_private_data *priv = (struct dev_private_data *)filp->private_data;
328 case KND_IOCTL_SET_CARD_ADDR:
329 priv->card_addr = ioctl_param; return priv->card_addr;
330 case KND_IOCTL_SET_USER_CTL:
331 priv->user_ctl = ioctl_param; return priv->user_ctl;
332 case KND_IOCTL_SET_USER_CTL_LAST:
333 priv->user_ctl_last = ioctl_param; return priv->user_ctl_last;
334 case KND_IOCTL_GET_USER_STS:
335 return priv->user_sts;
341 const struct file_operations kpc_dma_fops = {
342 .owner = THIS_MODULE,
343 .open = kpc_dma_open,
344 .release = kpc_dma_close,
345 .read = kpc_dma_read,
346 .write = kpc_dma_write,
347 .unlocked_ioctl = kpc_dma_ioctl,