4 * Xen models interrupts with abstract event channels. Because each
5 * domain gets 1024 event channels, but NR_IRQ is not that large, we
6 * must dynamically map irqs<->event channels. The event channels
7 * interface with the rest of the kernel by defining a xen interrupt
8 * chip. When an event is received, it is mapped to an irq and sent
9 * through the normal interrupt processing path.
11 * There are four kinds of events which can be mapped to an event
14 * 1. Inter-domain notifications. This includes all the virtual
15 * device events, since they're driven by front-ends in another domain
17 * 2. VIRQs, typically used for timers. These are per-cpu events.
19 * 4. PIRQs - Hardware interrupts.
21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
24 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
26 #include <linux/linkage.h>
27 #include <linux/interrupt.h>
28 #include <linux/irq.h>
29 #include <linux/module.h>
30 #include <linux/string.h>
31 #include <linux/bootmem.h>
32 #include <linux/slab.h>
33 #include <linux/irqnr.h>
34 #include <linux/pci.h>
38 #include <asm/ptrace.h>
41 #include <asm/io_apic.h>
42 #include <asm/xen/page.h>
43 #include <asm/xen/pci.h>
45 #include <asm/sync_bitops.h>
46 #include <asm/xen/hypercall.h>
47 #include <asm/xen/hypervisor.h>
51 #include <xen/xen-ops.h>
52 #include <xen/events.h>
53 #include <xen/interface/xen.h>
54 #include <xen/interface/event_channel.h>
55 #include <xen/interface/hvm/hvm_op.h>
56 #include <xen/interface/hvm/params.h>
57 #include <xen/interface/physdev.h>
58 #include <xen/interface/sched.h>
59 #include <asm/hw_irq.h>
62 * This lock protects updates to the following mapping and reference-count
63 * arrays. The lock does not need to be acquired to read the mapping tables.
65 static DEFINE_MUTEX(irq_mapping_update_lock);
67 static LIST_HEAD(xen_irq_list_head);
69 /* IRQ <-> VIRQ mapping. */
70 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
72 /* IRQ <-> IPI mapping */
73 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
75 /* Interrupt types. */
85 * Packed IRQ information:
86 * type - enum xen_irq_type
87 * event channel - irq->event channel mapping
88 * cpu - cpu this event channel is bound to
89 * index - type-specific information:
90 * PIRQ - physical IRQ, GSI, flags, and owner domain
96 struct list_head list;
98 enum xen_irq_type type; /* type */
100 unsigned short evtchn; /* event channel */
101 unsigned short cpu; /* cpu bound */
114 #define PIRQ_NEEDS_EOI (1 << 0)
115 #define PIRQ_SHAREABLE (1 << 1)
117 static int *evtchn_to_irq;
119 static unsigned long *pirq_eoi_map;
121 static bool (*pirq_needs_eoi)(unsigned irq);
124 * Note sizeof(xen_ulong_t) can be more than sizeof(unsigned long). Be
125 * careful to only use bitops which allow for this (e.g
126 * test_bit/find_first_bit and friends but not __ffs) and to pass
127 * BITS_PER_EVTCHN_WORD as the bitmask length.
129 #define BITS_PER_EVTCHN_WORD (sizeof(xen_ulong_t)*8)
131 * Make a bitmask (i.e. unsigned long *) of a xen_ulong_t
132 * array. Primarily to avoid long lines (hence the terse name).
134 #define BM(x) (unsigned long *)(x)
135 /* Find the first set bit in a evtchn mask */
136 #define EVTCHN_FIRST_BIT(w) find_first_bit(BM(&(w)), BITS_PER_EVTCHN_WORD)
138 static DEFINE_PER_CPU(xen_ulong_t [NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD],
141 /* Xen will never allocate port zero for any purpose. */
142 #define VALID_EVTCHN(chn) ((chn) != 0)
144 static struct irq_chip xen_dynamic_chip;
145 static struct irq_chip xen_percpu_chip;
146 static struct irq_chip xen_pirq_chip;
147 static void enable_dynirq(struct irq_data *data);
148 static void disable_dynirq(struct irq_data *data);
150 /* Get info for IRQ */
151 static struct irq_info *info_for_irq(unsigned irq)
153 return irq_get_handler_data(irq);
156 /* Constructors for packed IRQ information. */
157 static void xen_irq_info_common_init(struct irq_info *info,
159 enum xen_irq_type type,
160 unsigned short evtchn,
164 BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
168 info->evtchn = evtchn;
171 evtchn_to_irq[evtchn] = irq;
173 irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
176 static void xen_irq_info_evtchn_init(unsigned irq,
177 unsigned short evtchn)
179 struct irq_info *info = info_for_irq(irq);
181 xen_irq_info_common_init(info, irq, IRQT_EVTCHN, evtchn, 0);
184 static void xen_irq_info_ipi_init(unsigned cpu,
186 unsigned short evtchn,
189 struct irq_info *info = info_for_irq(irq);
191 xen_irq_info_common_init(info, irq, IRQT_IPI, evtchn, 0);
195 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
198 static void xen_irq_info_virq_init(unsigned cpu,
200 unsigned short evtchn,
203 struct irq_info *info = info_for_irq(irq);
205 xen_irq_info_common_init(info, irq, IRQT_VIRQ, evtchn, 0);
209 per_cpu(virq_to_irq, cpu)[virq] = irq;
212 static void xen_irq_info_pirq_init(unsigned irq,
213 unsigned short evtchn,
219 struct irq_info *info = info_for_irq(irq);
221 xen_irq_info_common_init(info, irq, IRQT_PIRQ, evtchn, 0);
223 info->u.pirq.pirq = pirq;
224 info->u.pirq.gsi = gsi;
225 info->u.pirq.domid = domid;
226 info->u.pirq.flags = flags;
230 * Accessors for packed IRQ information.
232 static unsigned int evtchn_from_irq(unsigned irq)
234 if (unlikely(WARN(irq < 0 || irq >= nr_irqs, "Invalid irq %d!\n", irq)))
237 return info_for_irq(irq)->evtchn;
240 unsigned irq_from_evtchn(unsigned int evtchn)
242 return evtchn_to_irq[evtchn];
244 EXPORT_SYMBOL_GPL(irq_from_evtchn);
246 static enum ipi_vector ipi_from_irq(unsigned irq)
248 struct irq_info *info = info_for_irq(irq);
250 BUG_ON(info == NULL);
251 BUG_ON(info->type != IRQT_IPI);
256 static unsigned virq_from_irq(unsigned irq)
258 struct irq_info *info = info_for_irq(irq);
260 BUG_ON(info == NULL);
261 BUG_ON(info->type != IRQT_VIRQ);
266 static unsigned pirq_from_irq(unsigned irq)
268 struct irq_info *info = info_for_irq(irq);
270 BUG_ON(info == NULL);
271 BUG_ON(info->type != IRQT_PIRQ);
273 return info->u.pirq.pirq;
276 static enum xen_irq_type type_from_irq(unsigned irq)
278 return info_for_irq(irq)->type;
281 static unsigned cpu_from_irq(unsigned irq)
283 return info_for_irq(irq)->cpu;
286 static unsigned int cpu_from_evtchn(unsigned int evtchn)
288 int irq = evtchn_to_irq[evtchn];
292 ret = cpu_from_irq(irq);
298 static bool pirq_check_eoi_map(unsigned irq)
300 return test_bit(pirq_from_irq(irq), pirq_eoi_map);
304 static bool pirq_needs_eoi_flag(unsigned irq)
306 struct irq_info *info = info_for_irq(irq);
307 BUG_ON(info->type != IRQT_PIRQ);
309 return info->u.pirq.flags & PIRQ_NEEDS_EOI;
312 static inline xen_ulong_t active_evtchns(unsigned int cpu,
313 struct shared_info *sh,
316 return sh->evtchn_pending[idx] &
317 per_cpu(cpu_evtchn_mask, cpu)[idx] &
318 ~sh->evtchn_mask[idx];
321 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
323 int irq = evtchn_to_irq[chn];
327 cpumask_copy(irq_to_desc(irq)->irq_data.affinity, cpumask_of(cpu));
330 clear_bit(chn, BM(per_cpu(cpu_evtchn_mask, cpu_from_irq(irq))));
331 set_bit(chn, BM(per_cpu(cpu_evtchn_mask, cpu)));
333 info_for_irq(irq)->cpu = cpu;
336 static void init_evtchn_cpu_bindings(void)
340 struct irq_info *info;
342 /* By default all event channels notify CPU#0. */
343 list_for_each_entry(info, &xen_irq_list_head, list) {
344 struct irq_desc *desc = irq_to_desc(info->irq);
345 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
349 for_each_possible_cpu(i)
350 memset(per_cpu(cpu_evtchn_mask, i),
351 (i == 0) ? ~0 : 0, NR_EVENT_CHANNELS/8);
354 static inline void clear_evtchn(int port)
356 struct shared_info *s = HYPERVISOR_shared_info;
357 sync_clear_bit(port, BM(&s->evtchn_pending[0]));
360 static inline void set_evtchn(int port)
362 struct shared_info *s = HYPERVISOR_shared_info;
363 sync_set_bit(port, BM(&s->evtchn_pending[0]));
366 static inline int test_evtchn(int port)
368 struct shared_info *s = HYPERVISOR_shared_info;
369 return sync_test_bit(port, BM(&s->evtchn_pending[0]));
374 * notify_remote_via_irq - send event to remote end of event channel via irq
375 * @irq: irq of event channel to send event to
377 * Unlike notify_remote_via_evtchn(), this is safe to use across
378 * save/restore. Notifications on a broken connection are silently
381 void notify_remote_via_irq(int irq)
383 int evtchn = evtchn_from_irq(irq);
385 if (VALID_EVTCHN(evtchn))
386 notify_remote_via_evtchn(evtchn);
388 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
390 static void mask_evtchn(int port)
392 struct shared_info *s = HYPERVISOR_shared_info;
393 sync_set_bit(port, BM(&s->evtchn_mask[0]));
396 static void unmask_evtchn(int port)
398 struct shared_info *s = HYPERVISOR_shared_info;
399 unsigned int cpu = get_cpu();
400 int do_hypercall = 0, evtchn_pending = 0;
402 BUG_ON(!irqs_disabled());
404 if (unlikely((cpu != cpu_from_evtchn(port))))
408 * Need to clear the mask before checking pending to
409 * avoid a race with an event becoming pending.
411 * EVTCHNOP_unmask will only trigger an upcall if the
412 * mask bit was set, so if a hypercall is needed
415 sync_clear_bit(port, BM(&s->evtchn_mask[0]));
416 evtchn_pending = sync_test_bit(port, BM(&s->evtchn_pending[0]));
418 if (unlikely(evtchn_pending && xen_hvm_domain())) {
419 sync_set_bit(port, BM(&s->evtchn_mask[0]));
424 /* Slow path (hypercall) if this is a non-local port or if this is
425 * an hvm domain and an event is pending (hvm domains don't have
426 * their own implementation of irq_enable). */
428 struct evtchn_unmask unmask = { .port = port };
429 (void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
431 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
434 * The following is basically the equivalent of
435 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
436 * the interrupt edge' if the channel is masked.
438 if (evtchn_pending &&
439 !sync_test_and_set_bit(port / BITS_PER_EVTCHN_WORD,
440 BM(&vcpu_info->evtchn_pending_sel)))
441 vcpu_info->evtchn_upcall_pending = 1;
447 static void xen_irq_init(unsigned irq)
449 struct irq_info *info;
451 struct irq_desc *desc = irq_to_desc(irq);
453 /* By default all event channels notify CPU#0. */
454 cpumask_copy(desc->irq_data.affinity, cpumask_of(0));
457 info = kzalloc(sizeof(*info), GFP_KERNEL);
459 panic("Unable to allocate metadata for IRQ%d\n", irq);
461 info->type = IRQT_UNBOUND;
464 irq_set_handler_data(irq, info);
466 list_add_tail(&info->list, &xen_irq_list_head);
469 static int __must_check xen_allocate_irq_dynamic(void)
474 #ifdef CONFIG_X86_IO_APIC
476 * For an HVM guest or domain 0 which see "real" (emulated or
477 * actual respectively) GSIs we allocate dynamic IRQs
478 * e.g. those corresponding to event channels or MSIs
479 * etc. from the range above those "real" GSIs to avoid
482 if (xen_initial_domain() || xen_hvm_domain())
483 first = get_nr_irqs_gsi();
486 irq = irq_alloc_desc_from(first, -1);
494 static int __must_check xen_allocate_irq_gsi(unsigned gsi)
499 * A PV guest has no concept of a GSI (since it has no ACPI
500 * nor access to/knowledge of the physical APICs). Therefore
501 * all IRQs are dynamically allocated from the entire IRQ
504 if (xen_pv_domain() && !xen_initial_domain())
505 return xen_allocate_irq_dynamic();
507 /* Legacy IRQ descriptors are already allocated by the arch. */
508 if (gsi < NR_IRQS_LEGACY)
511 irq = irq_alloc_desc_at(gsi, -1);
518 static void xen_free_irq(unsigned irq)
520 struct irq_info *info = irq_get_handler_data(irq);
525 list_del(&info->list);
527 irq_set_handler_data(irq, NULL);
529 WARN_ON(info->refcnt > 0);
533 /* Legacy IRQ descriptors are managed by the arch. */
534 if (irq < NR_IRQS_LEGACY)
540 static void pirq_query_unmask(int irq)
542 struct physdev_irq_status_query irq_status;
543 struct irq_info *info = info_for_irq(irq);
545 BUG_ON(info->type != IRQT_PIRQ);
547 irq_status.irq = pirq_from_irq(irq);
548 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
549 irq_status.flags = 0;
551 info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
552 if (irq_status.flags & XENIRQSTAT_needs_eoi)
553 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
556 static bool probing_irq(int irq)
558 struct irq_desc *desc = irq_to_desc(irq);
560 return desc && desc->action == NULL;
563 static void eoi_pirq(struct irq_data *data)
565 int evtchn = evtchn_from_irq(data->irq);
566 struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
571 if (VALID_EVTCHN(evtchn))
572 clear_evtchn(evtchn);
574 if (pirq_needs_eoi(data->irq)) {
575 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
580 static void mask_ack_pirq(struct irq_data *data)
582 disable_dynirq(data);
586 static unsigned int __startup_pirq(unsigned int irq)
588 struct evtchn_bind_pirq bind_pirq;
589 struct irq_info *info = info_for_irq(irq);
590 int evtchn = evtchn_from_irq(irq);
593 BUG_ON(info->type != IRQT_PIRQ);
595 if (VALID_EVTCHN(evtchn))
598 bind_pirq.pirq = pirq_from_irq(irq);
599 /* NB. We are happy to share unless we are probing. */
600 bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
601 BIND_PIRQ__WILL_SHARE : 0;
602 rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
604 if (!probing_irq(irq))
605 pr_info("Failed to obtain physical IRQ %d\n", irq);
608 evtchn = bind_pirq.port;
610 pirq_query_unmask(irq);
612 evtchn_to_irq[evtchn] = irq;
613 bind_evtchn_to_cpu(evtchn, 0);
614 info->evtchn = evtchn;
617 unmask_evtchn(evtchn);
618 eoi_pirq(irq_get_irq_data(irq));
623 static unsigned int startup_pirq(struct irq_data *data)
625 return __startup_pirq(data->irq);
628 static void shutdown_pirq(struct irq_data *data)
630 struct evtchn_close close;
631 unsigned int irq = data->irq;
632 struct irq_info *info = info_for_irq(irq);
633 int evtchn = evtchn_from_irq(irq);
635 BUG_ON(info->type != IRQT_PIRQ);
637 if (!VALID_EVTCHN(evtchn))
643 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
646 bind_evtchn_to_cpu(evtchn, 0);
647 evtchn_to_irq[evtchn] = -1;
651 static void enable_pirq(struct irq_data *data)
656 static void disable_pirq(struct irq_data *data)
658 disable_dynirq(data);
661 int xen_irq_from_gsi(unsigned gsi)
663 struct irq_info *info;
665 list_for_each_entry(info, &xen_irq_list_head, list) {
666 if (info->type != IRQT_PIRQ)
669 if (info->u.pirq.gsi == gsi)
675 EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
678 * Do not make any assumptions regarding the relationship between the
679 * IRQ number returned here and the Xen pirq argument.
681 * Note: We don't assign an event channel until the irq actually started
682 * up. Return an existing irq if we've already got one for the gsi.
684 * Shareable implies level triggered, not shareable implies edge
687 int xen_bind_pirq_gsi_to_irq(unsigned gsi,
688 unsigned pirq, int shareable, char *name)
691 struct physdev_irq irq_op;
693 mutex_lock(&irq_mapping_update_lock);
695 irq = xen_irq_from_gsi(gsi);
697 pr_info("%s: returning irq %d for gsi %u\n",
702 irq = xen_allocate_irq_gsi(gsi);
709 /* Only the privileged domain can do this. For non-priv, the pcifront
710 * driver provides a PCI bus that does the call to do exactly
711 * this in the priv domain. */
712 if (xen_initial_domain() &&
713 HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
719 xen_irq_info_pirq_init(irq, 0, pirq, gsi, DOMID_SELF,
720 shareable ? PIRQ_SHAREABLE : 0);
722 pirq_query_unmask(irq);
723 /* We try to use the handler with the appropriate semantic for the
724 * type of interrupt: if the interrupt is an edge triggered
725 * interrupt we use handle_edge_irq.
727 * On the other hand if the interrupt is level triggered we use
728 * handle_fasteoi_irq like the native code does for this kind of
731 * Depending on the Xen version, pirq_needs_eoi might return true
732 * not only for level triggered interrupts but for edge triggered
733 * interrupts too. In any case Xen always honors the eoi mechanism,
734 * not injecting any more pirqs of the same kind if the first one
735 * hasn't received an eoi yet. Therefore using the fasteoi handler
736 * is the right choice either way.
739 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
740 handle_fasteoi_irq, name);
742 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
743 handle_edge_irq, name);
746 mutex_unlock(&irq_mapping_update_lock);
751 #ifdef CONFIG_PCI_MSI
752 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
755 struct physdev_get_free_pirq op_get_free_pirq;
757 op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
758 rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
760 WARN_ONCE(rc == -ENOSYS,
761 "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
763 return rc ? -1 : op_get_free_pirq.pirq;
766 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
767 int pirq, const char *name, domid_t domid)
771 mutex_lock(&irq_mapping_update_lock);
773 irq = xen_allocate_irq_dynamic();
777 irq_set_chip_and_handler_name(irq, &xen_pirq_chip, handle_edge_irq,
780 xen_irq_info_pirq_init(irq, 0, pirq, 0, domid, 0);
781 ret = irq_set_msi_desc(irq, msidesc);
785 mutex_unlock(&irq_mapping_update_lock);
788 mutex_unlock(&irq_mapping_update_lock);
794 int xen_destroy_irq(int irq)
796 struct irq_desc *desc;
797 struct physdev_unmap_pirq unmap_irq;
798 struct irq_info *info = info_for_irq(irq);
801 mutex_lock(&irq_mapping_update_lock);
803 desc = irq_to_desc(irq);
807 if (xen_initial_domain()) {
808 unmap_irq.pirq = info->u.pirq.pirq;
809 unmap_irq.domid = info->u.pirq.domid;
810 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
811 /* If another domain quits without making the pci_disable_msix
812 * call, the Xen hypervisor takes care of freeing the PIRQs
813 * (free_domain_pirqs).
815 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
816 pr_info("domain %d does not have %d anymore\n",
817 info->u.pirq.domid, info->u.pirq.pirq);
819 pr_warn("unmap irq failed %d\n", rc);
827 mutex_unlock(&irq_mapping_update_lock);
831 int xen_irq_from_pirq(unsigned pirq)
835 struct irq_info *info;
837 mutex_lock(&irq_mapping_update_lock);
839 list_for_each_entry(info, &xen_irq_list_head, list) {
840 if (info->type != IRQT_PIRQ)
843 if (info->u.pirq.pirq == pirq)
848 mutex_unlock(&irq_mapping_update_lock);
854 int xen_pirq_from_irq(unsigned irq)
856 return pirq_from_irq(irq);
858 EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
859 int bind_evtchn_to_irq(unsigned int evtchn)
863 mutex_lock(&irq_mapping_update_lock);
865 irq = evtchn_to_irq[evtchn];
868 irq = xen_allocate_irq_dynamic();
872 irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
873 handle_edge_irq, "event");
875 xen_irq_info_evtchn_init(irq, evtchn);
877 struct irq_info *info = info_for_irq(irq);
878 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
882 mutex_unlock(&irq_mapping_update_lock);
886 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
888 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
890 struct evtchn_bind_ipi bind_ipi;
893 mutex_lock(&irq_mapping_update_lock);
895 irq = per_cpu(ipi_to_irq, cpu)[ipi];
898 irq = xen_allocate_irq_dynamic();
902 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
903 handle_percpu_irq, "ipi");
906 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
909 evtchn = bind_ipi.port;
911 xen_irq_info_ipi_init(cpu, irq, evtchn, ipi);
913 bind_evtchn_to_cpu(evtchn, cpu);
915 struct irq_info *info = info_for_irq(irq);
916 WARN_ON(info == NULL || info->type != IRQT_IPI);
920 mutex_unlock(&irq_mapping_update_lock);
924 static int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
925 unsigned int remote_port)
927 struct evtchn_bind_interdomain bind_interdomain;
930 bind_interdomain.remote_dom = remote_domain;
931 bind_interdomain.remote_port = remote_port;
933 err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
936 return err ? : bind_evtchn_to_irq(bind_interdomain.local_port);
939 static int find_virq(unsigned int virq, unsigned int cpu)
941 struct evtchn_status status;
942 int port, rc = -ENOENT;
944 memset(&status, 0, sizeof(status));
945 for (port = 0; port <= NR_EVENT_CHANNELS; port++) {
946 status.dom = DOMID_SELF;
948 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
951 if (status.status != EVTCHNSTAT_virq)
953 if (status.u.virq == virq && status.vcpu == cpu) {
961 int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
963 struct evtchn_bind_virq bind_virq;
964 int evtchn, irq, ret;
966 mutex_lock(&irq_mapping_update_lock);
968 irq = per_cpu(virq_to_irq, cpu)[virq];
971 irq = xen_allocate_irq_dynamic();
975 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
976 handle_percpu_irq, "virq");
978 bind_virq.virq = virq;
979 bind_virq.vcpu = cpu;
980 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
983 evtchn = bind_virq.port;
986 ret = find_virq(virq, cpu);
991 xen_irq_info_virq_init(cpu, irq, evtchn, virq);
993 bind_evtchn_to_cpu(evtchn, cpu);
995 struct irq_info *info = info_for_irq(irq);
996 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1000 mutex_unlock(&irq_mapping_update_lock);
1005 static void unbind_from_irq(unsigned int irq)
1007 struct evtchn_close close;
1008 int evtchn = evtchn_from_irq(irq);
1009 struct irq_info *info = irq_get_handler_data(irq);
1014 mutex_lock(&irq_mapping_update_lock);
1016 if (info->refcnt > 0) {
1018 if (info->refcnt != 0)
1022 if (VALID_EVTCHN(evtchn)) {
1023 close.port = evtchn;
1024 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
1027 switch (type_from_irq(irq)) {
1029 per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
1030 [virq_from_irq(irq)] = -1;
1033 per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
1034 [ipi_from_irq(irq)] = -1;
1040 /* Closed ports are implicitly re-bound to VCPU0. */
1041 bind_evtchn_to_cpu(evtchn, 0);
1043 evtchn_to_irq[evtchn] = -1;
1046 BUG_ON(info_for_irq(irq)->type == IRQT_UNBOUND);
1051 mutex_unlock(&irq_mapping_update_lock);
1054 int bind_evtchn_to_irqhandler(unsigned int evtchn,
1055 irq_handler_t handler,
1056 unsigned long irqflags,
1057 const char *devname, void *dev_id)
1061 irq = bind_evtchn_to_irq(evtchn);
1064 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1066 unbind_from_irq(irq);
1072 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1074 int bind_interdomain_evtchn_to_irqhandler(unsigned int remote_domain,
1075 unsigned int remote_port,
1076 irq_handler_t handler,
1077 unsigned long irqflags,
1078 const char *devname,
1083 irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
1087 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1089 unbind_from_irq(irq);
1095 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
1097 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1098 irq_handler_t handler,
1099 unsigned long irqflags, const char *devname, void *dev_id)
1103 irq = bind_virq_to_irq(virq, cpu);
1106 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1108 unbind_from_irq(irq);
1114 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1116 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1118 irq_handler_t handler,
1119 unsigned long irqflags,
1120 const char *devname,
1125 irq = bind_ipi_to_irq(ipi, cpu);
1129 irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1130 retval = request_irq(irq, handler, irqflags, devname, dev_id);
1132 unbind_from_irq(irq);
1139 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1141 struct irq_info *info = irq_get_handler_data(irq);
1145 free_irq(irq, dev_id);
1146 unbind_from_irq(irq);
1148 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1150 int evtchn_make_refcounted(unsigned int evtchn)
1152 int irq = evtchn_to_irq[evtchn];
1153 struct irq_info *info;
1158 info = irq_get_handler_data(irq);
1163 WARN_ON(info->refcnt != -1);
1169 EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1171 int evtchn_get(unsigned int evtchn)
1174 struct irq_info *info;
1177 if (evtchn >= NR_EVENT_CHANNELS)
1180 mutex_lock(&irq_mapping_update_lock);
1182 irq = evtchn_to_irq[evtchn];
1186 info = irq_get_handler_data(irq);
1192 if (info->refcnt <= 0)
1198 mutex_unlock(&irq_mapping_update_lock);
1202 EXPORT_SYMBOL_GPL(evtchn_get);
1204 void evtchn_put(unsigned int evtchn)
1206 int irq = evtchn_to_irq[evtchn];
1207 if (WARN_ON(irq == -1))
1209 unbind_from_irq(irq);
1211 EXPORT_SYMBOL_GPL(evtchn_put);
1213 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1215 int irq = per_cpu(ipi_to_irq, cpu)[vector];
1217 notify_remote_via_irq(irq);
1220 irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
1222 struct shared_info *sh = HYPERVISOR_shared_info;
1223 int cpu = smp_processor_id();
1224 xen_ulong_t *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
1226 unsigned long flags;
1227 static DEFINE_SPINLOCK(debug_lock);
1228 struct vcpu_info *v;
1230 spin_lock_irqsave(&debug_lock, flags);
1232 printk("\nvcpu %d\n ", cpu);
1234 for_each_online_cpu(i) {
1236 v = per_cpu(xen_vcpu, i);
1237 pending = (get_irq_regs() && i == cpu)
1238 ? xen_irqs_disabled(get_irq_regs())
1239 : v->evtchn_upcall_mask;
1240 printk("%d: masked=%d pending=%d event_sel %0*"PRI_xen_ulong"\n ", i,
1241 pending, v->evtchn_upcall_pending,
1242 (int)(sizeof(v->evtchn_pending_sel)*2),
1243 v->evtchn_pending_sel);
1245 v = per_cpu(xen_vcpu, cpu);
1247 printk("\npending:\n ");
1248 for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
1249 printk("%0*"PRI_xen_ulong"%s",
1250 (int)sizeof(sh->evtchn_pending[0])*2,
1251 sh->evtchn_pending[i],
1252 i % 8 == 0 ? "\n " : " ");
1253 printk("\nglobal mask:\n ");
1254 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
1255 printk("%0*"PRI_xen_ulong"%s",
1256 (int)(sizeof(sh->evtchn_mask[0])*2),
1258 i % 8 == 0 ? "\n " : " ");
1260 printk("\nglobally unmasked:\n ");
1261 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
1262 printk("%0*"PRI_xen_ulong"%s",
1263 (int)(sizeof(sh->evtchn_mask[0])*2),
1264 sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
1265 i % 8 == 0 ? "\n " : " ");
1267 printk("\nlocal cpu%d mask:\n ", cpu);
1268 for (i = (NR_EVENT_CHANNELS/BITS_PER_EVTCHN_WORD)-1; i >= 0; i--)
1269 printk("%0*"PRI_xen_ulong"%s", (int)(sizeof(cpu_evtchn[0])*2),
1271 i % 8 == 0 ? "\n " : " ");
1273 printk("\nlocally unmasked:\n ");
1274 for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
1275 xen_ulong_t pending = sh->evtchn_pending[i]
1276 & ~sh->evtchn_mask[i]
1278 printk("%0*"PRI_xen_ulong"%s",
1279 (int)(sizeof(sh->evtchn_mask[0])*2),
1280 pending, i % 8 == 0 ? "\n " : " ");
1283 printk("\npending list:\n");
1284 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
1285 if (sync_test_bit(i, BM(sh->evtchn_pending))) {
1286 int word_idx = i / BITS_PER_EVTCHN_WORD;
1287 printk(" %d: event %d -> irq %d%s%s%s\n",
1288 cpu_from_evtchn(i), i,
1290 sync_test_bit(word_idx, BM(&v->evtchn_pending_sel))
1292 !sync_test_bit(i, BM(sh->evtchn_mask))
1293 ? "" : " globally-masked",
1294 sync_test_bit(i, BM(cpu_evtchn))
1295 ? "" : " locally-masked");
1299 spin_unlock_irqrestore(&debug_lock, flags);
1304 static DEFINE_PER_CPU(unsigned, xed_nesting_count);
1305 static DEFINE_PER_CPU(unsigned int, current_word_idx);
1306 static DEFINE_PER_CPU(unsigned int, current_bit_idx);
1309 * Mask out the i least significant bits of w
1311 #define MASK_LSBS(w, i) (w & ((~((xen_ulong_t)0UL)) << i))
1314 * Search the CPUs pending events bitmasks. For each one found, map
1315 * the event number to an irq, and feed it into do_IRQ() for
1318 * Xen uses a two-level bitmap to speed searching. The first level is
1319 * a bitset of words which contain pending event bits. The second
1320 * level is a bitset of pending events themselves.
1322 static void __xen_evtchn_do_upcall(void)
1324 int start_word_idx, start_bit_idx;
1325 int word_idx, bit_idx;
1327 int cpu = get_cpu();
1328 struct shared_info *s = HYPERVISOR_shared_info;
1329 struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1333 xen_ulong_t pending_words;
1334 xen_ulong_t pending_bits;
1335 struct irq_desc *desc;
1337 vcpu_info->evtchn_upcall_pending = 0;
1339 if (__this_cpu_inc_return(xed_nesting_count) - 1)
1343 * Master flag must be cleared /before/ clearing
1344 * selector flag. xchg_xen_ulong must contain an
1345 * appropriate barrier.
1347 if ((irq = per_cpu(virq_to_irq, cpu)[VIRQ_TIMER]) != -1) {
1348 int evtchn = evtchn_from_irq(irq);
1349 word_idx = evtchn / BITS_PER_LONG;
1350 pending_bits = evtchn % BITS_PER_LONG;
1351 if (active_evtchns(cpu, s, word_idx) & (1ULL << pending_bits)) {
1352 desc = irq_to_desc(irq);
1354 generic_handle_irq_desc(irq, desc);
1358 pending_words = xchg_xen_ulong(&vcpu_info->evtchn_pending_sel, 0);
1360 start_word_idx = __this_cpu_read(current_word_idx);
1361 start_bit_idx = __this_cpu_read(current_bit_idx);
1363 word_idx = start_word_idx;
1365 for (i = 0; pending_words != 0; i++) {
1368 words = MASK_LSBS(pending_words, word_idx);
1371 * If we masked out all events, wrap to beginning.
1378 word_idx = EVTCHN_FIRST_BIT(words);
1380 pending_bits = active_evtchns(cpu, s, word_idx);
1381 bit_idx = 0; /* usually scan entire word from start */
1382 if (word_idx == start_word_idx) {
1383 /* We scan the starting word in two parts */
1385 /* 1st time: start in the middle */
1386 bit_idx = start_bit_idx;
1388 /* 2nd time: mask bits done already */
1389 bit_idx &= (1UL << start_bit_idx) - 1;
1396 bits = MASK_LSBS(pending_bits, bit_idx);
1398 /* If we masked out all events, move on. */
1402 bit_idx = EVTCHN_FIRST_BIT(bits);
1405 port = (word_idx * BITS_PER_EVTCHN_WORD) + bit_idx;
1406 irq = evtchn_to_irq[port];
1409 desc = irq_to_desc(irq);
1411 generic_handle_irq_desc(irq, desc);
1414 bit_idx = (bit_idx + 1) % BITS_PER_EVTCHN_WORD;
1416 /* Next caller starts at last processed + 1 */
1417 __this_cpu_write(current_word_idx,
1418 bit_idx ? word_idx :
1419 (word_idx+1) % BITS_PER_EVTCHN_WORD);
1420 __this_cpu_write(current_bit_idx, bit_idx);
1421 } while (bit_idx != 0);
1423 /* Scan start_l1i twice; all others once. */
1424 if ((word_idx != start_word_idx) || (i != 0))
1425 pending_words &= ~(1UL << word_idx);
1427 word_idx = (word_idx + 1) % BITS_PER_EVTCHN_WORD;
1430 BUG_ON(!irqs_disabled());
1432 count = __this_cpu_read(xed_nesting_count);
1433 __this_cpu_write(xed_nesting_count, 0);
1434 } while (count != 1 || vcpu_info->evtchn_upcall_pending);
1441 void xen_evtchn_do_upcall(struct pt_regs *regs)
1443 struct pt_regs *old_regs = set_irq_regs(regs);
1450 __xen_evtchn_do_upcall();
1453 set_irq_regs(old_regs);
1456 void xen_hvm_evtchn_do_upcall(void)
1458 __xen_evtchn_do_upcall();
1460 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1462 /* Rebind a new event channel to an existing irq. */
1463 void rebind_evtchn_irq(int evtchn, int irq)
1465 struct irq_info *info = info_for_irq(irq);
1470 /* Make sure the irq is masked, since the new event channel
1471 will also be masked. */
1474 mutex_lock(&irq_mapping_update_lock);
1476 /* After resume the irq<->evtchn mappings are all cleared out */
1477 BUG_ON(evtchn_to_irq[evtchn] != -1);
1478 /* Expect irq to have been bound before,
1479 so there should be a proper type */
1480 BUG_ON(info->type == IRQT_UNBOUND);
1482 xen_irq_info_evtchn_init(irq, evtchn);
1484 mutex_unlock(&irq_mapping_update_lock);
1486 /* new event channels are always bound to cpu 0 */
1487 irq_set_affinity(irq, cpumask_of(0));
1489 /* Unmask the event channel. */
1493 /* Rebind an evtchn so that it gets delivered to a specific cpu */
1494 static int rebind_irq_to_cpu(unsigned irq, unsigned tcpu)
1496 struct shared_info *s = HYPERVISOR_shared_info;
1497 struct evtchn_bind_vcpu bind_vcpu;
1498 int evtchn = evtchn_from_irq(irq);
1501 if (!VALID_EVTCHN(evtchn))
1505 * Events delivered via platform PCI interrupts are always
1506 * routed to vcpu 0 and hence cannot be rebound.
1508 if (xen_hvm_domain() && !xen_have_vector_callback)
1511 /* Send future instances of this interrupt to other vcpu. */
1512 bind_vcpu.port = evtchn;
1513 bind_vcpu.vcpu = tcpu;
1516 * Mask the event while changing the VCPU binding to prevent
1517 * it being delivered on an unexpected VCPU.
1519 masked = sync_test_and_set_bit(evtchn, BM(s->evtchn_mask));
1522 * If this fails, it usually just indicates that we're dealing with a
1523 * virq or IPI channel, which don't actually need to be rebound. Ignore
1524 * it, but don't do the xenlinux-level rebind in that case.
1526 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1527 bind_evtchn_to_cpu(evtchn, tcpu);
1530 unmask_evtchn(evtchn);
1535 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1538 unsigned tcpu = cpumask_first(dest);
1540 return rebind_irq_to_cpu(data->irq, tcpu);
1543 int resend_irq_on_evtchn(unsigned int irq)
1545 int masked, evtchn = evtchn_from_irq(irq);
1546 struct shared_info *s = HYPERVISOR_shared_info;
1548 if (!VALID_EVTCHN(evtchn))
1551 masked = sync_test_and_set_bit(evtchn, BM(s->evtchn_mask));
1552 sync_set_bit(evtchn, BM(s->evtchn_pending));
1554 unmask_evtchn(evtchn);
1559 static void enable_dynirq(struct irq_data *data)
1561 int evtchn = evtchn_from_irq(data->irq);
1563 if (VALID_EVTCHN(evtchn))
1564 unmask_evtchn(evtchn);
1567 static void disable_dynirq(struct irq_data *data)
1569 int evtchn = evtchn_from_irq(data->irq);
1571 if (VALID_EVTCHN(evtchn))
1572 mask_evtchn(evtchn);
1575 static void ack_dynirq(struct irq_data *data)
1577 int evtchn = evtchn_from_irq(data->irq);
1581 if (VALID_EVTCHN(evtchn))
1582 clear_evtchn(evtchn);
1585 static void mask_ack_dynirq(struct irq_data *data)
1587 disable_dynirq(data);
1591 static int retrigger_dynirq(struct irq_data *data)
1593 int evtchn = evtchn_from_irq(data->irq);
1594 struct shared_info *sh = HYPERVISOR_shared_info;
1597 if (VALID_EVTCHN(evtchn)) {
1600 masked = sync_test_and_set_bit(evtchn, BM(sh->evtchn_mask));
1601 sync_set_bit(evtchn, BM(sh->evtchn_pending));
1603 unmask_evtchn(evtchn);
1610 static void restore_pirqs(void)
1612 int pirq, rc, irq, gsi;
1613 struct physdev_map_pirq map_irq;
1614 struct irq_info *info;
1616 list_for_each_entry(info, &xen_irq_list_head, list) {
1617 if (info->type != IRQT_PIRQ)
1620 pirq = info->u.pirq.pirq;
1621 gsi = info->u.pirq.gsi;
1624 /* save/restore of PT devices doesn't work, so at this point the
1625 * only devices present are GSI based emulated devices */
1629 map_irq.domid = DOMID_SELF;
1630 map_irq.type = MAP_PIRQ_TYPE_GSI;
1631 map_irq.index = gsi;
1632 map_irq.pirq = pirq;
1634 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1636 pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1637 gsi, irq, pirq, rc);
1642 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1644 __startup_pirq(irq);
1648 static void restore_cpu_virqs(unsigned int cpu)
1650 struct evtchn_bind_virq bind_virq;
1651 int virq, irq, evtchn;
1653 for (virq = 0; virq < NR_VIRQS; virq++) {
1654 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1657 BUG_ON(virq_from_irq(irq) != virq);
1659 /* Get a new binding from Xen. */
1660 bind_virq.virq = virq;
1661 bind_virq.vcpu = cpu;
1662 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1665 evtchn = bind_virq.port;
1667 /* Record the new mapping. */
1668 xen_irq_info_virq_init(cpu, irq, evtchn, virq);
1669 bind_evtchn_to_cpu(evtchn, cpu);
1673 static void restore_cpu_ipis(unsigned int cpu)
1675 struct evtchn_bind_ipi bind_ipi;
1676 int ipi, irq, evtchn;
1678 for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
1679 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1682 BUG_ON(ipi_from_irq(irq) != ipi);
1684 /* Get a new binding from Xen. */
1685 bind_ipi.vcpu = cpu;
1686 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1689 evtchn = bind_ipi.port;
1691 /* Record the new mapping. */
1692 xen_irq_info_ipi_init(cpu, irq, evtchn, ipi);
1693 bind_evtchn_to_cpu(evtchn, cpu);
1697 /* Clear an irq's pending state, in preparation for polling on it */
1698 void xen_clear_irq_pending(int irq)
1700 int evtchn = evtchn_from_irq(irq);
1702 if (VALID_EVTCHN(evtchn))
1703 clear_evtchn(evtchn);
1705 EXPORT_SYMBOL(xen_clear_irq_pending);
1706 void xen_set_irq_pending(int irq)
1708 int evtchn = evtchn_from_irq(irq);
1710 if (VALID_EVTCHN(evtchn))
1714 bool xen_test_irq_pending(int irq)
1716 int evtchn = evtchn_from_irq(irq);
1719 if (VALID_EVTCHN(evtchn))
1720 ret = test_evtchn(evtchn);
1725 /* Poll waiting for an irq to become pending with timeout. In the usual case,
1726 * the irq will be disabled so it won't deliver an interrupt. */
1727 void xen_poll_irq_timeout(int irq, u64 timeout)
1729 evtchn_port_t evtchn = evtchn_from_irq(irq);
1731 if (VALID_EVTCHN(evtchn)) {
1732 struct sched_poll poll;
1735 poll.timeout = timeout;
1736 set_xen_guest_handle(poll.ports, &evtchn);
1738 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
1742 EXPORT_SYMBOL(xen_poll_irq_timeout);
1743 /* Poll waiting for an irq to become pending. In the usual case, the
1744 * irq will be disabled so it won't deliver an interrupt. */
1745 void xen_poll_irq(int irq)
1747 xen_poll_irq_timeout(irq, 0 /* no timeout */);
1750 /* Check whether the IRQ line is shared with other guests. */
1751 int xen_test_irq_shared(int irq)
1753 struct irq_info *info = info_for_irq(irq);
1754 struct physdev_irq_status_query irq_status;
1759 irq_status.irq = info->u.pirq.pirq;
1761 if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1763 return !(irq_status.flags & XENIRQSTAT_shared);
1765 EXPORT_SYMBOL_GPL(xen_test_irq_shared);
1767 void xen_irq_resume(void)
1769 unsigned int cpu, evtchn;
1770 struct irq_info *info;
1772 init_evtchn_cpu_bindings();
1774 /* New event-channel space is not 'live' yet. */
1775 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1776 mask_evtchn(evtchn);
1778 /* No IRQ <-> event-channel mappings. */
1779 list_for_each_entry(info, &xen_irq_list_head, list)
1780 info->evtchn = 0; /* zap event-channel binding */
1782 for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1783 evtchn_to_irq[evtchn] = -1;
1785 for_each_possible_cpu(cpu) {
1786 restore_cpu_virqs(cpu);
1787 restore_cpu_ipis(cpu);
1793 static struct irq_chip xen_dynamic_chip __read_mostly = {
1796 .irq_disable = disable_dynirq,
1797 .irq_mask = disable_dynirq,
1798 .irq_unmask = enable_dynirq,
1800 .irq_ack = ack_dynirq,
1801 .irq_mask_ack = mask_ack_dynirq,
1803 .irq_set_affinity = set_affinity_irq,
1804 .irq_retrigger = retrigger_dynirq,
1807 static struct irq_chip xen_pirq_chip __read_mostly = {
1810 .irq_startup = startup_pirq,
1811 .irq_shutdown = shutdown_pirq,
1812 .irq_enable = enable_pirq,
1813 .irq_disable = disable_pirq,
1815 .irq_mask = disable_dynirq,
1816 .irq_unmask = enable_dynirq,
1818 .irq_ack = eoi_pirq,
1819 .irq_eoi = eoi_pirq,
1820 .irq_mask_ack = mask_ack_pirq,
1822 .irq_set_affinity = set_affinity_irq,
1824 .irq_retrigger = retrigger_dynirq,
1827 static struct irq_chip xen_percpu_chip __read_mostly = {
1828 .name = "xen-percpu",
1830 .irq_disable = disable_dynirq,
1831 .irq_mask = disable_dynirq,
1832 .irq_unmask = enable_dynirq,
1834 .irq_ack = ack_dynirq,
1837 int xen_set_callback_via(uint64_t via)
1839 struct xen_hvm_param a;
1840 a.domid = DOMID_SELF;
1841 a.index = HVM_PARAM_CALLBACK_IRQ;
1843 return HYPERVISOR_hvm_op(HVMOP_set_param, &a);
1845 EXPORT_SYMBOL_GPL(xen_set_callback_via);
1847 #ifdef CONFIG_XEN_PVHVM
1848 /* Vector callbacks are better than PCI interrupts to receive event
1849 * channel notifications because we can receive vector callbacks on any
1850 * vcpu and we don't need PCI support or APIC interactions. */
1851 void xen_callback_vector(void)
1854 uint64_t callback_via;
1855 if (xen_have_vector_callback) {
1856 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
1857 rc = xen_set_callback_via(callback_via);
1859 pr_err("Request for Xen HVM callback vector failed\n");
1860 xen_have_vector_callback = 0;
1863 pr_info("Xen HVM callback vector for event delivery is enabled\n");
1864 /* in the restore case the vector has already been allocated */
1865 if (!test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors))
1866 alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR,
1867 xen_hvm_callback_vector);
1871 void xen_callback_vector(void) {}
1874 void __init xen_init_IRQ(void)
1878 evtchn_to_irq = kcalloc(NR_EVENT_CHANNELS, sizeof(*evtchn_to_irq),
1880 BUG_ON(!evtchn_to_irq);
1881 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1882 evtchn_to_irq[i] = -1;
1884 init_evtchn_cpu_bindings();
1886 /* No event channels are 'live' right now. */
1887 for (i = 0; i < NR_EVENT_CHANNELS; i++)
1890 pirq_needs_eoi = pirq_needs_eoi_flag;
1893 if (xen_hvm_domain()) {
1894 xen_callback_vector();
1896 /* pci_xen_hvm_init must be called after native_init_IRQ so that
1897 * __acpi_register_gsi can point at the right function */
1901 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1903 irq_ctx_init(smp_processor_id());
1904 if (xen_initial_domain())
1905 pci_xen_initial_domain();
1907 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
1908 eoi_gmfn.gmfn = virt_to_mfn(pirq_eoi_map);
1909 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
1911 free_page((unsigned long) pirq_eoi_map);
1912 pirq_eoi_map = NULL;
1914 pirq_needs_eoi = pirq_check_eoi_map;