2 * linux/drivers/misc/xillybus_core.c
4 * Copyright 2011 Xillybus Ltd, http://xillybus.com
6 * Driver for the Xillybus FPGA/host framework.
8 * This driver interfaces with a special IP core in an FPGA, setting up
9 * a pipe between a hardware FIFO in the programmable logic and a device
10 * file in the host. The number of such pipes and their attributes are
11 * set up on the logic. This driver detects these automatically and
12 * creates the device files accordingly.
14 * This program is free software; you can redistribute it and/or modify
15 * it under the smems of the GNU General Public License as published by
16 * the Free Software Foundation; version 2 of the License.
19 #include <linux/list.h>
20 #include <linux/device.h>
21 #include <linux/module.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/interrupt.h>
25 #include <linux/sched.h>
27 #include <linux/cdev.h>
28 #include <linux/spinlock.h>
29 #include <linux/mutex.h>
30 #include <linux/crc32.h>
31 #include <linux/poll.h>
32 #include <linux/delay.h>
33 #include <linux/slab.h>
34 #include <linux/workqueue.h>
37 MODULE_DESCRIPTION("Xillybus core functions");
38 MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
39 MODULE_VERSION("1.07");
40 MODULE_ALIAS("xillybus_core");
41 MODULE_LICENSE("GPL v2");
43 /* General timeout is 100 ms, rx timeout is 10 ms */
44 #define XILLY_RX_TIMEOUT (10*HZ/1000)
45 #define XILLY_TIMEOUT (100*HZ/1000)
47 #define fpga_msg_ctrl_reg 0x0008
48 #define fpga_dma_control_reg 0x0020
49 #define fpga_dma_bufno_reg 0x0024
50 #define fpga_dma_bufaddr_lowaddr_reg 0x0028
51 #define fpga_dma_bufaddr_highaddr_reg 0x002c
52 #define fpga_buf_ctrl_reg 0x0030
53 #define fpga_buf_offset_reg 0x0034
54 #define fpga_endian_reg 0x0040
56 #define XILLYMSG_OPCODE_RELEASEBUF 1
57 #define XILLYMSG_OPCODE_QUIESCEACK 2
58 #define XILLYMSG_OPCODE_FIFOEOF 3
59 #define XILLYMSG_OPCODE_FATAL_ERROR 4
60 #define XILLYMSG_OPCODE_NONEMPTY 5
62 static const char xillyname[] = "xillybus";
64 static struct class *xillybus_class;
67 * ep_list_lock is the last lock to be taken; No other lock requests are
68 * allowed while holding it. It merely protects list_of_endpoints, and not
69 * the endpoints listed in it.
72 static LIST_HEAD(list_of_endpoints);
73 static struct mutex ep_list_lock;
74 static struct workqueue_struct *xillybus_wq;
77 * Locking scheme: Mutexes protect invocations of character device methods.
78 * If both locks are taken, wr_mutex is taken first, rd_mutex second.
80 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
81 * buffers' end_offset fields against changes made by IRQ handler (and in
82 * theory, other file request handlers, but the mutex handles that). Nothing
84 * They are held for short direct memory manipulations. Needless to say,
85 * no mutex locking is allowed when a spinlock is held.
87 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
89 * register_mutex is endpoint-specific, and is held when non-atomic
90 * register operations are performed. wr_mutex and rd_mutex may be
91 * held when register_mutex is taken, but none of the spinlocks. Note that
92 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes
93 * which are unrelated to buf_offset_reg, since they are harmless.
95 * Blocking on the wait queues is allowed with mutexes held, but not with
98 * Only interruptible blocking is allowed on mutexes and wait queues.
100 * All in all, the locking order goes (with skips allowed, of course):
101 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
104 static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
107 int msg_channel, msg_bufno, msg_data, msg_dir;
109 opcode = (buf[0] >> 24) & 0xff;
110 msg_dir = buf[0] & 1;
111 msg_channel = (buf[0] >> 1) & 0x7ff;
112 msg_bufno = (buf[0] >> 12) & 0x3ff;
113 msg_data = buf[1] & 0xfffffff;
115 dev_warn(endpoint->dev,
116 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
117 opcode, msg_channel, msg_dir, msg_bufno, msg_data);
121 * xillybus_isr assumes the interrupt is allocated exclusively to it,
122 * which is the natural case MSI and several other hardware-oriented
123 * interrupts. Sharing is not allowed.
126 irqreturn_t xillybus_isr(int irq, void *data)
128 struct xilly_endpoint *ep = data;
130 unsigned int buf_size;
133 unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
134 struct xilly_channel *channel;
136 buf = ep->msgbuf_addr;
137 buf_size = ep->msg_buf_size/sizeof(u32);
139 ep->ephw->hw_sync_sgl_for_cpu(ep,
144 for (i = 0; i < buf_size; i += 2) {
145 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
146 malformed_message(ep, &buf[i]);
148 "Sending a NACK on counter %x (instead of %x) on entry %d\n",
149 ((buf[i+1] >> 28) & 0xf),
153 if (++ep->failed_messages > 10) {
155 "Lost sync with interrupt messages. Stopping.\n");
157 ep->ephw->hw_sync_sgl_for_device(
163 iowrite32(0x01, /* Message NACK */
164 ep->registers + fpga_msg_ctrl_reg);
167 } else if (buf[i] & (1 << 22)) /* Last message */
172 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
178 for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
179 opcode = (buf[i] >> 24) & 0xff;
181 msg_dir = buf[i] & 1;
182 msg_channel = (buf[i] >> 1) & 0x7ff;
183 msg_bufno = (buf[i] >> 12) & 0x3ff;
184 msg_data = buf[i+1] & 0xfffffff;
187 case XILLYMSG_OPCODE_RELEASEBUF:
188 if ((msg_channel > ep->num_channels) ||
189 (msg_channel == 0)) {
190 malformed_message(ep, &buf[i]);
194 channel = ep->channels[msg_channel];
196 if (msg_dir) { /* Write channel */
197 if (msg_bufno >= channel->num_wr_buffers) {
198 malformed_message(ep, &buf[i]);
201 spin_lock(&channel->wr_spinlock);
202 channel->wr_buffers[msg_bufno]->end_offset =
204 channel->wr_fpga_buf_idx = msg_bufno;
205 channel->wr_empty = 0;
206 channel->wr_sleepy = 0;
207 spin_unlock(&channel->wr_spinlock);
209 wake_up_interruptible(&channel->wr_wait);
214 if (msg_bufno >= channel->num_rd_buffers) {
215 malformed_message(ep, &buf[i]);
219 spin_lock(&channel->rd_spinlock);
220 channel->rd_fpga_buf_idx = msg_bufno;
221 channel->rd_full = 0;
222 spin_unlock(&channel->rd_spinlock);
224 wake_up_interruptible(&channel->rd_wait);
225 if (!channel->rd_synchronous)
228 &channel->rd_workitem,
233 case XILLYMSG_OPCODE_NONEMPTY:
234 if ((msg_channel > ep->num_channels) ||
235 (msg_channel == 0) || (!msg_dir) ||
236 !ep->channels[msg_channel]->wr_supports_nonempty) {
237 malformed_message(ep, &buf[i]);
241 channel = ep->channels[msg_channel];
243 if (msg_bufno >= channel->num_wr_buffers) {
244 malformed_message(ep, &buf[i]);
247 spin_lock(&channel->wr_spinlock);
248 if (msg_bufno == channel->wr_host_buf_idx)
249 channel->wr_ready = 1;
250 spin_unlock(&channel->wr_spinlock);
252 wake_up_interruptible(&channel->wr_ready_wait);
255 case XILLYMSG_OPCODE_QUIESCEACK:
256 ep->idtlen = msg_data;
257 wake_up_interruptible(&ep->ep_wait);
260 case XILLYMSG_OPCODE_FIFOEOF:
261 if ((msg_channel > ep->num_channels) ||
262 (msg_channel == 0) || (!msg_dir) ||
263 !ep->channels[msg_channel]->num_wr_buffers) {
264 malformed_message(ep, &buf[i]);
267 channel = ep->channels[msg_channel];
268 spin_lock(&channel->wr_spinlock);
269 channel->wr_eof = msg_bufno;
270 channel->wr_sleepy = 0;
272 channel->wr_hangup = channel->wr_empty &&
273 (channel->wr_host_buf_idx == msg_bufno);
275 spin_unlock(&channel->wr_spinlock);
277 wake_up_interruptible(&channel->wr_wait);
280 case XILLYMSG_OPCODE_FATAL_ERROR:
282 wake_up_interruptible(&ep->ep_wait); /* For select() */
284 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
287 malformed_message(ep, &buf[i]);
292 ep->ephw->hw_sync_sgl_for_device(ep,
297 ep->msg_counter = (ep->msg_counter + 1) & 0xf;
298 ep->failed_messages = 0;
299 iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
303 EXPORT_SYMBOL(xillybus_isr);
306 * A few trivial memory management functions.
307 * NOTE: These functions are used only on probe and remove, and therefore
308 * no locks are applied!
311 static void xillybus_autoflush(struct work_struct *work);
313 struct xilly_alloc_state {
317 enum dma_data_direction direction;
321 static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
322 struct xilly_alloc_state *s,
323 struct xilly_buffer **buffers,
324 int bufnum, int bytebufsize)
328 struct device *dev = ep->dev;
329 struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
331 if (buffers) { /* Not the message buffer */
332 this_buffer = devm_kcalloc(dev, bufnum,
333 sizeof(struct xilly_buffer),
339 for (i = 0; i < bufnum; i++) {
341 * Buffers are expected in descending size order, so there
342 * is either enough space for this buffer or none at all.
345 if ((s->left_of_salami < bytebufsize) &&
346 (s->left_of_salami > 0)) {
348 "Corrupt buffer allocation in IDT. Aborting.\n");
352 if (s->left_of_salami == 0) {
353 int allocorder, allocsize;
355 allocsize = PAGE_SIZE;
357 while (bytebufsize > allocsize) {
362 s->salami = (void *) devm_get_free_pages(
364 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
369 s->left_of_salami = allocsize;
372 rc = ep->ephw->map_single(ep, s->salami,
373 bytebufsize, s->direction,
378 iowrite32((u32) (dma_addr & 0xffffffff),
379 ep->registers + fpga_dma_bufaddr_lowaddr_reg);
380 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
381 ep->registers + fpga_dma_bufaddr_highaddr_reg);
383 if (buffers) { /* Not the message buffer */
384 this_buffer->addr = s->salami;
385 this_buffer->dma_addr = dma_addr;
386 buffers[i] = this_buffer++;
388 iowrite32(s->regdirection | s->nbuffer++,
389 ep->registers + fpga_dma_bufno_reg);
391 ep->msgbuf_addr = s->salami;
392 ep->msgbuf_dma_addr = dma_addr;
393 ep->msg_buf_size = bytebufsize;
395 iowrite32(s->regdirection,
396 ep->registers + fpga_dma_bufno_reg);
399 s->left_of_salami -= bytebufsize;
400 s->salami += bytebufsize;
405 static int xilly_setupchannels(struct xilly_endpoint *ep,
406 unsigned char *chandesc,
409 struct device *dev = ep->dev;
411 struct xilly_channel *channel;
412 int channelnum, bufnum, bufsize, format, is_writebuf;
414 int synchronous, allowpartial, exclusive_open, seekable;
415 int supports_nonempty;
416 int msg_buf_done = 0;
418 struct xilly_alloc_state rd_alloc = {
422 .direction = DMA_TO_DEVICE,
426 struct xilly_alloc_state wr_alloc = {
430 .direction = DMA_FROM_DEVICE,
431 .regdirection = 0x80000000,
434 channel = devm_kcalloc(dev, ep->num_channels,
435 sizeof(struct xilly_channel), GFP_KERNEL);
439 ep->channels = devm_kcalloc(dev, ep->num_channels + 1,
440 sizeof(struct xilly_channel *),
445 ep->channels[0] = NULL; /* Channel 0 is message buf. */
447 /* Initialize all channels with defaults */
449 for (i = 1; i <= ep->num_channels; i++) {
450 channel->wr_buffers = NULL;
451 channel->rd_buffers = NULL;
452 channel->num_wr_buffers = 0;
453 channel->num_rd_buffers = 0;
454 channel->wr_fpga_buf_idx = -1;
455 channel->wr_host_buf_idx = 0;
456 channel->wr_host_buf_pos = 0;
457 channel->wr_empty = 1;
458 channel->wr_ready = 0;
459 channel->wr_sleepy = 1;
460 channel->rd_fpga_buf_idx = 0;
461 channel->rd_host_buf_idx = 0;
462 channel->rd_host_buf_pos = 0;
463 channel->rd_full = 0;
464 channel->wr_ref_count = 0;
465 channel->rd_ref_count = 0;
467 spin_lock_init(&channel->wr_spinlock);
468 spin_lock_init(&channel->rd_spinlock);
469 mutex_init(&channel->wr_mutex);
470 mutex_init(&channel->rd_mutex);
471 init_waitqueue_head(&channel->rd_wait);
472 init_waitqueue_head(&channel->wr_wait);
473 init_waitqueue_head(&channel->wr_ready_wait);
475 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
477 channel->endpoint = ep;
478 channel->chan_num = i;
480 channel->log2_element_size = 0;
482 ep->channels[i] = channel++;
485 for (entry = 0; entry < entries; entry++, chandesc += 4) {
486 struct xilly_buffer **buffers = NULL;
488 is_writebuf = chandesc[0] & 0x01;
489 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
490 format = (chandesc[1] >> 4) & 0x03;
491 allowpartial = (chandesc[1] >> 6) & 0x01;
492 synchronous = (chandesc[1] >> 7) & 0x01;
493 bufsize = 1 << (chandesc[2] & 0x1f);
494 bufnum = 1 << (chandesc[3] & 0x0f);
495 exclusive_open = (chandesc[2] >> 7) & 0x01;
496 seekable = (chandesc[2] >> 6) & 0x01;
497 supports_nonempty = (chandesc[2] >> 5) & 0x01;
499 if ((channelnum > ep->num_channels) ||
500 ((channelnum == 0) && !is_writebuf)) {
502 "IDT requests channel out of range. Aborting.\n");
506 channel = ep->channels[channelnum]; /* NULL for msg channel */
508 if (!is_writebuf || channelnum > 0) {
509 channel->log2_element_size = ((format > 2) ?
512 bytebufsize = channel->rd_buf_size = bufsize *
513 (1 << channel->log2_element_size);
515 buffers = devm_kcalloc(dev, bufnum,
516 sizeof(struct xilly_buffer *),
521 bytebufsize = bufsize << 2;
525 channel->num_rd_buffers = bufnum;
526 channel->rd_allow_partial = allowpartial;
527 channel->rd_synchronous = synchronous;
528 channel->rd_exclusive_open = exclusive_open;
529 channel->seekable = seekable;
531 channel->rd_buffers = buffers;
532 rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,
533 bufnum, bytebufsize);
534 } else if (channelnum > 0) {
535 channel->num_wr_buffers = bufnum;
537 channel->seekable = seekable;
538 channel->wr_supports_nonempty = supports_nonempty;
540 channel->wr_allow_partial = allowpartial;
541 channel->wr_synchronous = synchronous;
542 channel->wr_exclusive_open = exclusive_open;
544 channel->wr_buffers = buffers;
545 rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,
546 bufnum, bytebufsize);
548 rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,
549 bufnum, bytebufsize);
559 "Corrupt IDT: No message buffer. Aborting.\n");
565 static int xilly_scan_idt(struct xilly_endpoint *endpoint,
566 struct xilly_idt_handle *idt_handle)
569 unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
570 unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
575 idt_handle->idt = idt;
577 scan++; /* Skip version number */
579 while ((scan <= end_of_idt) && *scan) {
580 while ((scan <= end_of_idt) && *scan++)
581 /* Do nothing, just scan thru string */;
587 if (scan > end_of_idt) {
588 dev_err(endpoint->dev,
589 "IDT device name list overflow. Aborting.\n");
592 idt_handle->chandesc = scan;
594 len = endpoint->idtlen - (3 + ((int) (scan - idt)));
597 dev_err(endpoint->dev,
598 "Corrupt IDT device name list. Aborting.\n");
602 idt_handle->entries = len >> 2;
603 endpoint->num_channels = count;
608 static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
610 struct xilly_channel *channel;
611 unsigned char *version;
614 channel = endpoint->channels[1]; /* This should be generated ad-hoc */
616 channel->wr_sleepy = 1;
619 (3 << 24), /* Opcode 3 for channel 0 = Send IDT */
620 endpoint->registers + fpga_buf_ctrl_reg);
622 t = wait_event_interruptible_timeout(channel->wr_wait,
623 (!channel->wr_sleepy),
627 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
629 if (endpoint->fatal_error)
635 endpoint->ephw->hw_sync_sgl_for_cpu(
637 channel->wr_buffers[0]->dma_addr,
638 channel->wr_buf_size,
641 if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
642 dev_err(endpoint->dev,
643 "IDT length mismatch (%d != %d). Aborting.\n",
644 channel->wr_buffers[0]->end_offset, endpoint->idtlen);
648 if (crc32_le(~0, channel->wr_buffers[0]->addr,
649 endpoint->idtlen+1) != 0) {
650 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
654 version = channel->wr_buffers[0]->addr;
656 /* Check version number. Accept anything below 0x82 for now. */
657 if (*version > 0x82) {
658 dev_err(endpoint->dev,
659 "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgarde. Aborting.\n",
667 static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
668 size_t count, loff_t *f_pos)
673 int no_time_left = 0;
674 long deadline, left_to_sleep;
675 struct xilly_channel *channel = filp->private_data;
677 int empty, reached_eof, exhausted, ready;
678 /* Initializations are there only to silence warnings */
680 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
683 if (channel->endpoint->fatal_error)
686 deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
688 rc = mutex_lock_interruptible(&channel->wr_mutex);
692 while (1) { /* Note that we may drop mutex within this loop */
693 int bytes_to_do = count - bytes_done;
695 spin_lock_irqsave(&channel->wr_spinlock, flags);
697 empty = channel->wr_empty;
698 ready = !empty || channel->wr_ready;
701 bufidx = channel->wr_host_buf_idx;
702 bufpos = channel->wr_host_buf_pos;
703 howmany = ((channel->wr_buffers[bufidx]->end_offset
704 + 1) << channel->log2_element_size)
707 /* Update wr_host_* to its post-operation state */
708 if (howmany > bytes_to_do) {
711 howmany = bytes_to_do;
712 channel->wr_host_buf_pos += howmany;
716 channel->wr_host_buf_pos = 0;
718 if (bufidx == channel->wr_fpga_buf_idx) {
719 channel->wr_empty = 1;
720 channel->wr_sleepy = 1;
721 channel->wr_ready = 0;
724 if (bufidx >= (channel->num_wr_buffers - 1))
725 channel->wr_host_buf_idx = 0;
727 channel->wr_host_buf_idx++;
732 * Marking our situation after the possible changes above,
733 * for use after releasing the spinlock.
735 * empty = empty before change
736 * exhasted = empty after possible change
739 reached_eof = channel->wr_empty &&
740 (channel->wr_host_buf_idx == channel->wr_eof);
741 channel->wr_hangup = reached_eof;
742 exhausted = channel->wr_empty;
743 waiting_bufidx = channel->wr_host_buf_idx;
745 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
747 if (!empty) { /* Go on, now without the spinlock */
749 if (bufpos == 0) /* Position zero means it's virgin */
750 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
752 channel->wr_buffers[bufidx]->dma_addr,
753 channel->wr_buf_size,
758 channel->wr_buffers[bufidx]->addr
763 bytes_done += howmany;
766 channel->endpoint->ephw->hw_sync_sgl_for_device(
768 channel->wr_buffers[bufidx]->dma_addr,
769 channel->wr_buf_size,
773 * Tell FPGA the buffer is done with. It's an
774 * atomic operation to the FPGA, so what
775 * happens with other channels doesn't matter,
776 * and the certain channel is protected with
777 * the channel-specific mutex.
780 iowrite32(1 | (channel->chan_num << 1) |
782 channel->endpoint->registers +
787 mutex_unlock(&channel->wr_mutex);
792 /* This includes a zero-count return = EOF */
793 if ((bytes_done >= count) || reached_eof)
797 continue; /* More in RAM buffer(s)? Just go on. */
799 if ((bytes_done > 0) &&
801 (channel->wr_synchronous && channel->wr_allow_partial)))
805 * Nonblocking read: The "ready" flag tells us that the FPGA
806 * has data to send. In non-blocking mode, if it isn't on,
807 * just return. But if there is, we jump directly to the point
808 * where we ask for the FPGA to send all it has, and wait
809 * until that data arrives. So in a sense, we *do* block in
810 * nonblocking mode, but only for a very short time.
813 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
824 if (!no_time_left || (bytes_done > 0)) {
826 * Note that in case of an element-misaligned read
827 * request, offsetlimit will include the last element,
828 * which will be partially read from.
830 int offsetlimit = ((count - bytes_done) - 1) >>
831 channel->log2_element_size;
832 int buf_elements = channel->wr_buf_size >>
833 channel->log2_element_size;
836 * In synchronous mode, always send an offset limit.
837 * Just don't send a value too big.
840 if (channel->wr_synchronous) {
841 /* Don't request more than one buffer */
842 if (channel->wr_allow_partial &&
843 (offsetlimit >= buf_elements))
844 offsetlimit = buf_elements - 1;
846 /* Don't request more than all buffers */
847 if (!channel->wr_allow_partial &&
849 (buf_elements * channel->num_wr_buffers)))
850 offsetlimit = buf_elements *
851 channel->num_wr_buffers - 1;
855 * In asynchronous mode, force early flush of a buffer
856 * only if that will allow returning a full count. The
857 * "offsetlimit < ( ... )" rather than "<=" excludes
858 * requesting a full buffer, which would obviously
859 * cause a buffer transmission anyhow
862 if (channel->wr_synchronous ||
863 (offsetlimit < (buf_elements - 1))) {
864 mutex_lock(&channel->endpoint->register_mutex);
866 iowrite32(offsetlimit,
867 channel->endpoint->registers +
868 fpga_buf_offset_reg);
870 iowrite32(1 | (channel->chan_num << 1) |
871 (2 << 24) | /* 2 = offset limit */
872 (waiting_bufidx << 12),
873 channel->endpoint->registers +
876 mutex_unlock(&channel->endpoint->
882 * If partial completion is disallowed, there is no point in
883 * timeout sleeping. Neither if no_time_left is set and
887 if (!channel->wr_allow_partial ||
888 (no_time_left && (bytes_done == 0))) {
890 * This do-loop will run more than once if another
891 * thread reasserted wr_sleepy before we got the mutex
892 * back, so we try again.
896 mutex_unlock(&channel->wr_mutex);
898 if (wait_event_interruptible(
900 (!channel->wr_sleepy)))
903 if (mutex_lock_interruptible(
906 } while (channel->wr_sleepy);
910 interrupted: /* Mutex is not held if got here */
911 if (channel->endpoint->fatal_error)
915 if (filp->f_flags & O_NONBLOCK)
916 return -EAGAIN; /* Don't admit snoozing */
920 left_to_sleep = deadline - ((long) jiffies);
923 * If our time is out, skip the waiting. We may miss wr_sleepy
924 * being deasserted but hey, almost missing the train is like
928 if (left_to_sleep > 0) {
930 wait_event_interruptible_timeout(
932 (!channel->wr_sleepy),
935 if (left_to_sleep > 0) /* wr_sleepy deasserted */
938 if (left_to_sleep < 0) { /* Interrupt */
939 mutex_unlock(&channel->wr_mutex);
940 if (channel->endpoint->fatal_error)
949 no_time_left = 1; /* We're out of sleeping time. Desperate! */
951 if (bytes_done == 0) {
953 * Reaching here means that we allow partial return,
954 * that we've run out of time, and that we have
956 * So tell the FPGA to send anything it has or gets.
959 iowrite32(1 | (channel->chan_num << 1) |
960 (3 << 24) | /* Opcode 3, flush it all! */
961 (waiting_bufidx << 12),
962 channel->endpoint->registers +
967 * Reaching here means that we *do* have data in the buffer,
968 * but the "partial" flag disallows returning less than
969 * required. And we don't have as much. So loop again,
970 * which is likely to end up blocking indefinitely until
971 * enough data has arrived.
975 mutex_unlock(&channel->wr_mutex);
977 if (channel->endpoint->fatal_error)
987 * The timeout argument takes values as follows:
988 * >0 : Flush with timeout
989 * ==0 : Flush, and wait idefinitely for the flush to complete
990 * <0 : Autoflush: Flush only if there's a single buffer occupied
993 static int xillybus_myflush(struct xilly_channel *channel, long timeout)
998 int end_offset_plus1;
999 int bufidx, bufidx_minus1;
1002 int new_rd_host_buf_pos;
1004 if (channel->endpoint->fatal_error)
1006 rc = mutex_lock_interruptible(&channel->rd_mutex);
1011 * Don't flush a closed channel. This can happen when the work queued
1012 * autoflush thread fires off after the file has closed. This is not
1013 * an error, just something to dismiss.
1016 if (!channel->rd_ref_count)
1019 bufidx = channel->rd_host_buf_idx;
1021 bufidx_minus1 = (bufidx == 0) ?
1022 channel->num_rd_buffers - 1 :
1025 end_offset_plus1 = channel->rd_host_buf_pos >>
1026 channel->log2_element_size;
1028 new_rd_host_buf_pos = channel->rd_host_buf_pos -
1029 (end_offset_plus1 << channel->log2_element_size);
1031 /* Submit the current buffer if it's nonempty */
1032 if (end_offset_plus1) {
1033 unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1034 (end_offset_plus1 << channel->log2_element_size);
1036 /* Copy unflushed data, so we can put it in next buffer */
1037 for (i = 0; i < new_rd_host_buf_pos; i++)
1038 channel->rd_leftovers[i] = *tail++;
1040 spin_lock_irqsave(&channel->rd_spinlock, flags);
1042 /* Autoflush only if a single buffer is occupied */
1044 if ((timeout < 0) &&
1045 (channel->rd_full ||
1046 (bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1047 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1049 * A new work item may be queued by the ISR exactly
1050 * now, since the execution of a work item allows the
1051 * queuing of a new one while it's running.
1056 /* The 4th element is never needed for data, so it's a flag */
1057 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1059 /* Set up rd_full to reflect a certain moment's state */
1061 if (bufidx == channel->rd_fpga_buf_idx)
1062 channel->rd_full = 1;
1063 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1065 if (bufidx >= (channel->num_rd_buffers - 1))
1066 channel->rd_host_buf_idx = 0;
1068 channel->rd_host_buf_idx++;
1070 channel->endpoint->ephw->hw_sync_sgl_for_device(
1072 channel->rd_buffers[bufidx]->dma_addr,
1073 channel->rd_buf_size,
1076 mutex_lock(&channel->endpoint->register_mutex);
1078 iowrite32(end_offset_plus1 - 1,
1079 channel->endpoint->registers + fpga_buf_offset_reg);
1081 iowrite32((channel->chan_num << 1) | /* Channel ID */
1082 (2 << 24) | /* Opcode 2, submit buffer */
1084 channel->endpoint->registers + fpga_buf_ctrl_reg);
1086 mutex_unlock(&channel->endpoint->register_mutex);
1087 } else if (bufidx == 0) {
1088 bufidx = channel->num_rd_buffers - 1;
1093 channel->rd_host_buf_pos = new_rd_host_buf_pos;
1096 goto done; /* Autoflush */
1099 * bufidx is now the last buffer written to (or equal to
1100 * rd_fpga_buf_idx if buffer was never written to), and
1101 * channel->rd_host_buf_idx the one after it.
1103 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1106 while (1) { /* Loop waiting for draining of buffers */
1107 spin_lock_irqsave(&channel->rd_spinlock, flags);
1109 if (bufidx != channel->rd_fpga_buf_idx)
1110 channel->rd_full = 1; /*
1112 * but needs waiting.
1115 empty = !channel->rd_full;
1117 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1123 * Indefinite sleep with mutex taken. With data waiting for
1124 * flushing user should not be surprised if open() for write
1128 wait_event_interruptible(channel->rd_wait,
1129 (!channel->rd_full));
1131 else if (wait_event_interruptible_timeout(
1133 (!channel->rd_full),
1135 dev_warn(channel->endpoint->dev,
1136 "Timed out while flushing. Output data may be lost.\n");
1142 if (channel->rd_full) {
1149 mutex_unlock(&channel->rd_mutex);
1151 if (channel->endpoint->fatal_error)
1157 static int xillybus_flush(struct file *filp, fl_owner_t id)
1159 if (!(filp->f_mode & FMODE_WRITE))
1162 return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */
1165 static void xillybus_autoflush(struct work_struct *work)
1167 struct delayed_work *workitem = container_of(
1168 work, struct delayed_work, work);
1169 struct xilly_channel *channel = container_of(
1170 workitem, struct xilly_channel, rd_workitem);
1173 rc = xillybus_myflush(channel, -1);
1175 dev_warn(channel->endpoint->dev,
1176 "Autoflush failed because work queue thread got a signal.\n");
1178 dev_err(channel->endpoint->dev,
1179 "Autoflush failed under weird circumstances.\n");
1182 static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1183 size_t count, loff_t *f_pos)
1186 unsigned long flags;
1188 struct xilly_channel *channel = filp->private_data;
1190 int full, exhausted;
1191 /* Initializations are there only to silence warnings */
1193 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1194 int end_offset_plus1 = 0;
1196 if (channel->endpoint->fatal_error)
1199 rc = mutex_lock_interruptible(&channel->rd_mutex);
1204 int bytes_to_do = count - bytes_done;
1206 spin_lock_irqsave(&channel->rd_spinlock, flags);
1208 full = channel->rd_full;
1211 bufidx = channel->rd_host_buf_idx;
1212 bufpos = channel->rd_host_buf_pos;
1213 howmany = channel->rd_buf_size - bufpos;
1216 * Update rd_host_* to its state after this operation.
1217 * count=0 means committing the buffer immediately,
1218 * which is like flushing, but not necessarily block.
1221 if ((howmany > bytes_to_do) &&
1223 ((bufpos >> channel->log2_element_size) == 0))) {
1226 howmany = bytes_to_do;
1227 channel->rd_host_buf_pos += howmany;
1233 channel->rd_buf_size >>
1234 channel->log2_element_size;
1235 channel->rd_host_buf_pos = 0;
1237 unsigned char *tail;
1240 end_offset_plus1 = bufpos >>
1241 channel->log2_element_size;
1243 channel->rd_host_buf_pos -=
1245 channel->log2_element_size;
1248 rd_buffers[bufidx]->addr +
1249 (end_offset_plus1 <<
1250 channel->log2_element_size);
1253 i < channel->rd_host_buf_pos;
1255 channel->rd_leftovers[i] =
1259 if (bufidx == channel->rd_fpga_buf_idx)
1260 channel->rd_full = 1;
1262 if (bufidx >= (channel->num_rd_buffers - 1))
1263 channel->rd_host_buf_idx = 0;
1265 channel->rd_host_buf_idx++;
1270 * Marking our situation after the possible changes above,
1271 * for use after releasing the spinlock.
1273 * full = full before change
1274 * exhasted = full after possible change
1277 exhausted = channel->rd_full;
1279 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1281 if (!full) { /* Go on, now without the spinlock */
1282 unsigned char *head =
1283 channel->rd_buffers[bufidx]->addr;
1286 if ((bufpos == 0) || /* Zero means it's virgin */
1287 (channel->rd_leftovers[3] != 0)) {
1288 channel->endpoint->ephw->hw_sync_sgl_for_cpu(
1290 channel->rd_buffers[bufidx]->dma_addr,
1291 channel->rd_buf_size,
1294 /* Virgin, but leftovers are due */
1295 for (i = 0; i < bufpos; i++)
1296 *head++ = channel->rd_leftovers[i];
1298 channel->rd_leftovers[3] = 0; /* Clear flag */
1302 channel->rd_buffers[bufidx]->addr + bufpos,
1307 bytes_done += howmany;
1310 channel->endpoint->ephw->hw_sync_sgl_for_device(
1312 channel->rd_buffers[bufidx]->dma_addr,
1313 channel->rd_buf_size,
1316 mutex_lock(&channel->endpoint->register_mutex);
1318 iowrite32(end_offset_plus1 - 1,
1319 channel->endpoint->registers +
1320 fpga_buf_offset_reg);
1322 iowrite32((channel->chan_num << 1) |
1323 (2 << 24) | /* 2 = submit buffer */
1325 channel->endpoint->registers +
1328 mutex_unlock(&channel->endpoint->
1331 channel->rd_leftovers[3] =
1332 (channel->rd_host_buf_pos != 0);
1336 mutex_unlock(&channel->rd_mutex);
1338 if (channel->endpoint->fatal_error)
1341 if (!channel->rd_synchronous)
1344 &channel->rd_workitem,
1351 if (bytes_done >= count)
1355 continue; /* If there's more space, just go on */
1357 if ((bytes_done > 0) && channel->rd_allow_partial)
1361 * Indefinite sleep with mutex taken. With data waiting for
1362 * flushing, user should not be surprised if open() for write
1366 if (filp->f_flags & O_NONBLOCK) {
1371 if (wait_event_interruptible(channel->rd_wait,
1372 (!channel->rd_full))) {
1373 mutex_unlock(&channel->rd_mutex);
1375 if (channel->endpoint->fatal_error)
1384 mutex_unlock(&channel->rd_mutex);
1386 if (!channel->rd_synchronous)
1387 queue_delayed_work(xillybus_wq,
1388 &channel->rd_workitem,
1391 if (channel->endpoint->fatal_error)
1397 if ((channel->rd_synchronous) && (bytes_done > 0)) {
1398 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */
1400 if (rc && (rc != -EINTR))
1407 static int xillybus_open(struct inode *inode, struct file *filp)
1410 unsigned long flags;
1411 int minor = iminor(inode);
1412 int major = imajor(inode);
1413 struct xilly_endpoint *ep_iter, *endpoint = NULL;
1414 struct xilly_channel *channel;
1416 mutex_lock(&ep_list_lock);
1418 list_for_each_entry(ep_iter, &list_of_endpoints, ep_list) {
1419 if ((ep_iter->major == major) &&
1420 (minor >= ep_iter->lowest_minor) &&
1421 (minor < (ep_iter->lowest_minor +
1422 ep_iter->num_channels))) {
1427 mutex_unlock(&ep_list_lock);
1430 pr_err("xillybus: open() failed to find a device for major=%d and minor=%d\n",
1435 if (endpoint->fatal_error)
1438 channel = endpoint->channels[1 + minor - endpoint->lowest_minor];
1439 filp->private_data = channel;
1442 * It gets complicated because:
1443 * 1. We don't want to take a mutex we don't have to
1444 * 2. We don't want to open one direction if the other will fail.
1447 if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1450 if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1453 if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1454 (channel->wr_synchronous || !channel->wr_allow_partial ||
1455 !channel->wr_supports_nonempty)) {
1456 dev_err(endpoint->dev,
1457 "open() failed: O_NONBLOCK not allowed for read on this device\n");
1461 if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1462 (channel->rd_synchronous || !channel->rd_allow_partial)) {
1463 dev_err(endpoint->dev,
1464 "open() failed: O_NONBLOCK not allowed for write on this device\n");
1469 * Note: open() may block on getting mutexes despite O_NONBLOCK.
1470 * This shouldn't occur normally, since multiple open of the same
1471 * file descriptor is almost always prohibited anyhow
1472 * (*_exclusive_open is normally set in real-life systems).
1475 if (filp->f_mode & FMODE_READ) {
1476 rc = mutex_lock_interruptible(&channel->wr_mutex);
1481 if (filp->f_mode & FMODE_WRITE) {
1482 rc = mutex_lock_interruptible(&channel->rd_mutex);
1487 if ((filp->f_mode & FMODE_READ) &&
1488 (channel->wr_ref_count != 0) &&
1489 (channel->wr_exclusive_open)) {
1494 if ((filp->f_mode & FMODE_WRITE) &&
1495 (channel->rd_ref_count != 0) &&
1496 (channel->rd_exclusive_open)) {
1501 if (filp->f_mode & FMODE_READ) {
1502 if (channel->wr_ref_count == 0) { /* First open of file */
1503 /* Move the host to first buffer */
1504 spin_lock_irqsave(&channel->wr_spinlock, flags);
1505 channel->wr_host_buf_idx = 0;
1506 channel->wr_host_buf_pos = 0;
1507 channel->wr_fpga_buf_idx = -1;
1508 channel->wr_empty = 1;
1509 channel->wr_ready = 0;
1510 channel->wr_sleepy = 1;
1511 channel->wr_eof = -1;
1512 channel->wr_hangup = 0;
1514 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1516 iowrite32(1 | (channel->chan_num << 1) |
1517 (4 << 24) | /* Opcode 4, open channel */
1518 ((channel->wr_synchronous & 1) << 23),
1519 channel->endpoint->registers +
1523 channel->wr_ref_count++;
1526 if (filp->f_mode & FMODE_WRITE) {
1527 if (channel->rd_ref_count == 0) { /* First open of file */
1528 /* Move the host to first buffer */
1529 spin_lock_irqsave(&channel->rd_spinlock, flags);
1530 channel->rd_host_buf_idx = 0;
1531 channel->rd_host_buf_pos = 0;
1532 channel->rd_leftovers[3] = 0; /* No leftovers. */
1533 channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1534 channel->rd_full = 0;
1536 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1538 iowrite32((channel->chan_num << 1) |
1539 (4 << 24), /* Opcode 4, open channel */
1540 channel->endpoint->registers +
1544 channel->rd_ref_count++;
1548 if (filp->f_mode & FMODE_WRITE)
1549 mutex_unlock(&channel->rd_mutex);
1551 if (filp->f_mode & FMODE_READ)
1552 mutex_unlock(&channel->wr_mutex);
1554 if (!rc && (!channel->seekable))
1555 return nonseekable_open(inode, filp);
1560 static int xillybus_release(struct inode *inode, struct file *filp)
1562 unsigned long flags;
1563 struct xilly_channel *channel = filp->private_data;
1568 if (channel->endpoint->fatal_error)
1571 if (filp->f_mode & FMODE_WRITE) {
1572 mutex_lock(&channel->rd_mutex);
1574 channel->rd_ref_count--;
1576 if (channel->rd_ref_count == 0) {
1578 * We rely on the kernel calling flush()
1579 * before we get here.
1582 iowrite32((channel->chan_num << 1) | /* Channel ID */
1583 (5 << 24), /* Opcode 5, close channel */
1584 channel->endpoint->registers +
1587 mutex_unlock(&channel->rd_mutex);
1590 if (filp->f_mode & FMODE_READ) {
1591 mutex_lock(&channel->wr_mutex);
1593 channel->wr_ref_count--;
1595 if (channel->wr_ref_count == 0) {
1596 iowrite32(1 | (channel->chan_num << 1) |
1597 (5 << 24), /* Opcode 5, close channel */
1598 channel->endpoint->registers +
1602 * This is crazily cautious: We make sure that not
1603 * only that we got an EOF (be it because we closed
1604 * the channel or because of a user's EOF), but verify
1605 * that it's one beyond the last buffer arrived, so
1606 * we have no leftover buffers pending before wrapping
1607 * up (which can only happen in asynchronous channels,
1612 spin_lock_irqsave(&channel->wr_spinlock,
1614 buf_idx = channel->wr_fpga_buf_idx;
1615 eof = channel->wr_eof;
1616 channel->wr_sleepy = 1;
1617 spin_unlock_irqrestore(&channel->wr_spinlock,
1621 * Check if eof points at the buffer after
1622 * the last one the FPGA submitted. Note that
1623 * no EOF is marked by negative eof.
1627 if (buf_idx == channel->num_wr_buffers)
1634 * Steal extra 100 ms if awaken by interrupt.
1635 * This is a simple workaround for an
1636 * interrupt pending when entering, which would
1637 * otherwise result in declaring the hardware
1641 if (wait_event_interruptible(
1643 (!channel->wr_sleepy)))
1646 if (channel->wr_sleepy) {
1647 mutex_unlock(&channel->wr_mutex);
1648 dev_warn(channel->endpoint->dev,
1649 "Hardware failed to respond to close command, therefore left in messy state.\n");
1655 mutex_unlock(&channel->wr_mutex);
1661 static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1663 struct xilly_channel *channel = filp->private_data;
1664 loff_t pos = filp->f_pos;
1668 * Take both mutexes not allowing interrupts, since it seems like
1669 * common applications don't expect an -EINTR here. Besides, multiple
1670 * access to a single file descriptor on seekable devices is a mess
1674 if (channel->endpoint->fatal_error)
1677 mutex_lock(&channel->wr_mutex);
1678 mutex_lock(&channel->rd_mutex);
1688 pos = offset; /* Going to the end => to the beginning */
1695 /* In any case, we must finish on an element boundary */
1696 if (pos & ((1 << channel->log2_element_size) - 1)) {
1701 mutex_lock(&channel->endpoint->register_mutex);
1703 iowrite32(pos >> channel->log2_element_size,
1704 channel->endpoint->registers + fpga_buf_offset_reg);
1706 iowrite32((channel->chan_num << 1) |
1707 (6 << 24), /* Opcode 6, set address */
1708 channel->endpoint->registers + fpga_buf_ctrl_reg);
1710 mutex_unlock(&channel->endpoint->register_mutex);
1713 mutex_unlock(&channel->rd_mutex);
1714 mutex_unlock(&channel->wr_mutex);
1716 if (rc) /* Return error after releasing mutexes */
1722 * Since seekable devices are allowed only when the channel is
1723 * synchronous, we assume that there is no data pending in either
1724 * direction (which holds true as long as no concurrent access on the
1725 * file descriptor takes place).
1726 * The only thing we may need to throw away is leftovers from partial
1730 channel->rd_leftovers[3] = 0;
1735 static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
1737 struct xilly_channel *channel = filp->private_data;
1738 unsigned int mask = 0;
1739 unsigned long flags;
1741 poll_wait(filp, &channel->endpoint->ep_wait, wait);
1744 * poll() won't play ball regarding read() channels which
1745 * aren't asynchronous and support the nonempty message. Allowing
1746 * that will create situations where data has been delivered at
1747 * the FPGA, and users expecting select() to wake up, which it may
1751 if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1752 poll_wait(filp, &channel->wr_wait, wait);
1753 poll_wait(filp, &channel->wr_ready_wait, wait);
1755 spin_lock_irqsave(&channel->wr_spinlock, flags);
1756 if (!channel->wr_empty || channel->wr_ready)
1757 mask |= POLLIN | POLLRDNORM;
1759 if (channel->wr_hangup)
1761 * Not POLLHUP, because its behavior is in the
1762 * mist, and POLLIN does what we want: Wake up
1763 * the read file descriptor so it sees EOF.
1765 mask |= POLLIN | POLLRDNORM;
1766 spin_unlock_irqrestore(&channel->wr_spinlock, flags);
1770 * If partial data write is disallowed on a write() channel,
1771 * it's pointless to ever signal OK to write, because is could
1772 * block despite some space being available.
1775 if (channel->rd_allow_partial) {
1776 poll_wait(filp, &channel->rd_wait, wait);
1778 spin_lock_irqsave(&channel->rd_spinlock, flags);
1779 if (!channel->rd_full)
1780 mask |= POLLOUT | POLLWRNORM;
1781 spin_unlock_irqrestore(&channel->rd_spinlock, flags);
1784 if (channel->endpoint->fatal_error)
1790 static const struct file_operations xillybus_fops = {
1791 .owner = THIS_MODULE,
1792 .read = xillybus_read,
1793 .write = xillybus_write,
1794 .open = xillybus_open,
1795 .flush = xillybus_flush,
1796 .release = xillybus_release,
1797 .llseek = xillybus_llseek,
1798 .poll = xillybus_poll,
1801 static int xillybus_init_chrdev(struct xilly_endpoint *endpoint,
1802 const unsigned char *idt)
1806 int devnum, i, minor, major;
1808 struct device *device;
1810 rc = alloc_chrdev_region(&dev, 0, /* minor start */
1811 endpoint->num_channels,
1814 dev_warn(endpoint->dev, "Failed to obtain major/minors");
1818 endpoint->major = major = MAJOR(dev);
1819 endpoint->lowest_minor = minor = MINOR(dev);
1821 cdev_init(&endpoint->cdev, &xillybus_fops);
1822 endpoint->cdev.owner = endpoint->ephw->owner;
1823 rc = cdev_add(&endpoint->cdev, MKDEV(major, minor),
1824 endpoint->num_channels);
1826 dev_warn(endpoint->dev, "Failed to add cdev. Aborting.\n");
1827 goto unregister_chrdev;
1832 for (i = minor, devnum = 0;
1833 devnum < endpoint->num_channels;
1835 snprintf(devname, sizeof(devname)-1, "xillybus_%s", idt);
1837 devname[sizeof(devname)-1] = 0; /* Should never matter */
1842 device = device_create(xillybus_class,
1848 if (IS_ERR(device)) {
1849 dev_warn(endpoint->dev,
1850 "Failed to create %s device. Aborting.\n",
1853 goto unroll_device_create;
1857 dev_info(endpoint->dev, "Created %d device files.\n",
1858 endpoint->num_channels);
1859 return 0; /* succeed */
1861 unroll_device_create:
1863 for (; devnum >= 0; devnum--, i--)
1864 device_destroy(xillybus_class, MKDEV(major, i));
1866 cdev_del(&endpoint->cdev);
1868 unregister_chrdev_region(MKDEV(major, minor), endpoint->num_channels);
1873 static void xillybus_cleanup_chrdev(struct xilly_endpoint *endpoint)
1877 for (minor = endpoint->lowest_minor;
1878 minor < (endpoint->lowest_minor + endpoint->num_channels);
1880 device_destroy(xillybus_class, MKDEV(endpoint->major, minor));
1881 cdev_del(&endpoint->cdev);
1882 unregister_chrdev_region(MKDEV(endpoint->major,
1883 endpoint->lowest_minor),
1884 endpoint->num_channels);
1886 dev_info(endpoint->dev, "Removed %d device files.\n",
1887 endpoint->num_channels);
1890 struct xilly_endpoint *xillybus_init_endpoint(struct pci_dev *pdev,
1892 struct xilly_endpoint_hardware
1895 struct xilly_endpoint *endpoint;
1897 endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);
1901 endpoint->pdev = pdev;
1902 endpoint->dev = dev;
1903 endpoint->ephw = ephw;
1904 endpoint->msg_counter = 0x0b;
1905 endpoint->failed_messages = 0;
1906 endpoint->fatal_error = 0;
1908 init_waitqueue_head(&endpoint->ep_wait);
1909 mutex_init(&endpoint->register_mutex);
1913 EXPORT_SYMBOL(xillybus_init_endpoint);
1915 static int xilly_quiesce(struct xilly_endpoint *endpoint)
1919 endpoint->idtlen = -1;
1921 iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1922 endpoint->registers + fpga_dma_control_reg);
1924 t = wait_event_interruptible_timeout(endpoint->ep_wait,
1925 (endpoint->idtlen >= 0),
1928 dev_err(endpoint->dev,
1929 "Failed to quiesce the device on exit.\n");
1935 int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1940 void *bootstrap_resources;
1941 int idtbuffersize = (1 << PAGE_SHIFT);
1942 struct device *dev = endpoint->dev;
1945 * The bogus IDT is used during bootstrap for allocating the initial
1946 * message buffer, and then the message buffer and space for the IDT
1947 * itself. The initial message buffer is of a single page's size, but
1948 * it's soon replaced with a more modest one (and memory is freed).
1951 unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1952 3, 192, PAGE_SHIFT, 0 };
1953 struct xilly_idt_handle idt_handle;
1956 * Writing the value 0x00000001 to Endianness register signals which
1957 * endianness this processor is using, so the FPGA can swap words as
1961 iowrite32(1, endpoint->registers + fpga_endian_reg);
1963 /* Bootstrap phase I: Allocate temporary message buffer */
1965 bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1966 if (!bootstrap_resources)
1969 endpoint->num_channels = 0;
1971 rc = xilly_setupchannels(endpoint, bogus_idt, 1);
1975 /* Clear the message subsystem (and counter in particular) */
1976 iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1978 endpoint->idtlen = -1;
1981 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1984 iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1985 endpoint->registers + fpga_dma_control_reg);
1987 t = wait_event_interruptible_timeout(endpoint->ep_wait,
1988 (endpoint->idtlen >= 0),
1991 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1996 iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
1997 endpoint->registers + fpga_dma_control_reg);
1999 /* Bootstrap phase II: Allocate buffer for IDT and obtain it */
2000 while (endpoint->idtlen >= idtbuffersize) {
2005 endpoint->num_channels = 1;
2007 rc = xilly_setupchannels(endpoint, bogus_idt, 2);
2011 rc = xilly_obtain_idt(endpoint);
2015 rc = xilly_scan_idt(endpoint, &idt_handle);
2019 devres_close_group(dev, bootstrap_resources);
2021 /* Bootstrap phase III: Allocate buffers according to IDT */
2023 rc = xilly_setupchannels(endpoint,
2024 idt_handle.chandesc,
2025 idt_handle.entries);
2030 * endpoint is now completely configured. We put it on the list
2031 * available to open() before registering the char device(s)
2034 mutex_lock(&ep_list_lock);
2035 list_add_tail(&endpoint->ep_list, &list_of_endpoints);
2036 mutex_unlock(&ep_list_lock);
2038 rc = xillybus_init_chrdev(endpoint, idt_handle.idt);
2040 goto failed_chrdevs;
2042 devres_release_group(dev, bootstrap_resources);
2047 mutex_lock(&ep_list_lock);
2048 list_del(&endpoint->ep_list);
2049 mutex_unlock(&ep_list_lock);
2052 xilly_quiesce(endpoint);
2053 flush_workqueue(xillybus_wq);
2057 EXPORT_SYMBOL(xillybus_endpoint_discovery);
2059 void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
2061 xillybus_cleanup_chrdev(endpoint);
2063 mutex_lock(&ep_list_lock);
2064 list_del(&endpoint->ep_list);
2065 mutex_unlock(&ep_list_lock);
2067 xilly_quiesce(endpoint);
2070 * Flushing is done upon endpoint release to prevent access to memory
2071 * just about to be released. This makes the quiesce complete.
2073 flush_workqueue(xillybus_wq);
2075 EXPORT_SYMBOL(xillybus_endpoint_remove);
2077 static int __init xillybus_init(void)
2079 mutex_init(&ep_list_lock);
2081 xillybus_class = class_create(THIS_MODULE, xillyname);
2082 if (IS_ERR(xillybus_class))
2083 return PTR_ERR(xillybus_class);
2085 xillybus_wq = alloc_workqueue(xillyname, 0, 0);
2087 class_destroy(xillybus_class);
2094 static void __exit xillybus_exit(void)
2096 /* flush_workqueue() was called for each endpoint released */
2097 destroy_workqueue(xillybus_wq);
2099 class_destroy(xillybus_class);
2102 module_init(xillybus_init);
2103 module_exit(xillybus_exit);