drivers/xen/events/events_base.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Xen event channels
   4  *
   5  * Xen models interrupts with abstract event channels.  Because each
   6  * domain gets 1024 event channels, but NR_IRQ is not that large, we
   7  * must dynamically map irqs<->event channels.  The event channels
   8  * interface with the rest of the kernel by defining a xen interrupt
   9  * chip.  When an event is received, it is mapped to an irq and sent
  10  * through the normal interrupt processing path.
  11  *
  12  * There are four kinds of events which can be mapped to an event
  13  * channel:
  14  *
  15  * 1. Inter-domain notifications.  This includes all the virtual
  16  *    device events, since they're driven by front-ends in another domain
  17  *    (typically dom0).
  18  * 2. VIRQs, typically used for timers.  These are per-cpu events.
  19  * 3. IPIs.
  20  * 4. PIRQs - Hardware interrupts.
  21  *
  22  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
  23  */
  24
  25 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
  26
  27 #include <linux/linkage.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/irq.h>
  30 #include <linux/moduleparam.h>
  31 #include <linux/string.h>
  32 #include <linux/memblock.h>
  33 #include <linux/slab.h>
  34 #include <linux/irqnr.h>
  35 #include <linux/pci.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/cpuhotplug.h>
  38 #include <linux/atomic.h>
  39 #include <linux/ktime.h>
  40
  41 #ifdef CONFIG_X86
  42 #include <asm/desc.h>
  43 #include <asm/ptrace.h>
  44 #include <asm/idtentry.h>
  45 #include <asm/irq.h>
  46 #include <asm/io_apic.h>
  47 #include <asm/i8259.h>
  48 #include <asm/xen/pci.h>
  49 #endif
  50 #include <asm/sync_bitops.h>
  51 #include <asm/xen/hypercall.h>
  52 #include <asm/xen/hypervisor.h>
  53 #include <xen/page.h>
  54
  55 #include <xen/xen.h>
  56 #include <xen/hvm.h>
  57 #include <xen/xen-ops.h>
  58 #include <xen/events.h>
  59 #include <xen/interface/xen.h>
  60 #include <xen/interface/event_channel.h>
  61 #include <xen/interface/hvm/hvm_op.h>
  62 #include <xen/interface/hvm/params.h>
  63 #include <xen/interface/physdev.h>
  64 #include <xen/interface/sched.h>
  65 #include <xen/interface/vcpu.h>
  66 #include <xen/xenbus.h>
  67 #include <asm/hw_irq.h>
  68
  69 #include "events_internal.h"
  70
  71 #undef MODULE_PARAM_PREFIX
  72 #define MODULE_PARAM_PREFIX "xen."
  73
  74 /* Interrupt types. */
  75 enum xen_irq_type {
  76         IRQT_UNBOUND = 0,
  77         IRQT_PIRQ,
  78         IRQT_VIRQ,
  79         IRQT_IPI,
  80         IRQT_EVTCHN
  81 };
  82
  83 /*
  84  * Packed IRQ information:
  85  * type - enum xen_irq_type
  86  * event channel - irq->event channel mapping
  87  * cpu - cpu this event channel is bound to
  88  * index - type-specific information:
  89  *    PIRQ - vector, with MSB being "needs EIO", or physical IRQ of the HVM
  90  *           guest, or GSI (real passthrough IRQ) of the device.
  91  *    VIRQ - virq number
  92  *    IPI - IPI vector
  93  *    EVTCHN -
  94  */
  95 struct irq_info {
  96         struct list_head list;
  97         struct list_head eoi_list;
  98         short refcnt;
  99         u8 spurious_cnt;
 100         u8 is_accounted;
 101         short type;             /* type: IRQT_* */
 102         u8 mask_reason;         /* Why is event channel masked */
 103 #define EVT_MASK_REASON_EXPLICIT        0x01
 104 #define EVT_MASK_REASON_TEMPORARY       0x02
 105 #define EVT_MASK_REASON_EOI_PENDING     0x04
 106         u8 is_active;           /* Is event just being handled? */
 107         unsigned irq;
 108         evtchn_port_t evtchn;   /* event channel */
 109         unsigned short cpu;     /* cpu bound */
 110         unsigned short eoi_cpu; /* EOI must happen on this cpu-1 */
 111         unsigned int irq_epoch; /* If eoi_cpu valid: irq_epoch of event */
 112         u64 eoi_time;           /* Time in jiffies when to EOI. */
 113         raw_spinlock_t lock;
 114
 115         union {
 116                 unsigned short virq;
 117                 enum ipi_vector ipi;
 118                 struct {
 119                         unsigned short pirq;
 120                         unsigned short gsi;
 121                         unsigned char vector;
 122                         unsigned char flags;
 123                         uint16_t domid;
 124                 } pirq;
 125                 struct xenbus_device *interdomain;
 126         } u;
 127 };
 128
 129 #define PIRQ_NEEDS_EOI  (1 << 0)
 130 #define PIRQ_SHAREABLE  (1 << 1)
 131 #define PIRQ_MSI_GROUP  (1 << 2)
 132
 133 static uint __read_mostly event_loop_timeout = 2;
 134 module_param(event_loop_timeout, uint, 0644);
 135
 136 static uint __read_mostly event_eoi_delay = 10;
 137 module_param(event_eoi_delay, uint, 0644);
 138
 139 const struct evtchn_ops *evtchn_ops;
 140
 141 /*
 142  * This lock protects updates to the following mapping and reference-count
 143  * arrays. The lock does not need to be acquired to read the mapping tables.
 144  */
 145 static DEFINE_MUTEX(irq_mapping_update_lock);
 146
 147 /*
 148  * Lock protecting event handling loop against removing event channels.
 149  * Adding of event channels is no issue as the associated IRQ becomes active
 150  * only after everything is setup (before request_[threaded_]irq() the handler
 151  * can't be entered for an event, as the event channel will be unmasked only
 152  * then).
 153  */
 154 static DEFINE_RWLOCK(evtchn_rwlock);
 155
 156 /*
 157  * Lock hierarchy:
 158  *
 159  * irq_mapping_update_lock
 160  *   evtchn_rwlock
 161  *     IRQ-desc lock
 162  *       percpu eoi_list_lock
 163  *         irq_info->lock
 164  */
 165
 166 static LIST_HEAD(xen_irq_list_head);
 167
 168 /* IRQ <-> VIRQ mapping. */
 169 static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
 170
 171 /* IRQ <-> IPI mapping */
 172 static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
 173
 174 /* Event channel distribution data */
 175 static atomic_t channels_on_cpu[NR_CPUS];
 176
 177 static int **evtchn_to_irq;
 178 #ifdef CONFIG_X86
 179 static unsigned long *pirq_eoi_map;
 180 #endif
 181 static bool (*pirq_needs_eoi)(unsigned irq);
 182
 183 #define EVTCHN_ROW(e)  (e / (PAGE_SIZE/sizeof(**evtchn_to_irq)))
 184 #define EVTCHN_COL(e)  (e % (PAGE_SIZE/sizeof(**evtchn_to_irq)))
 185 #define EVTCHN_PER_ROW (PAGE_SIZE / sizeof(**evtchn_to_irq))
 186
 187 /* Xen will never allocate port zero for any purpose. */
 188 #define VALID_EVTCHN(chn)       ((chn) != 0)
 189
 190 static struct irq_info *legacy_info_ptrs[NR_IRQS_LEGACY];
 191
 192 static struct irq_chip xen_dynamic_chip;
 193 static struct irq_chip xen_lateeoi_chip;
 194 static struct irq_chip xen_percpu_chip;
 195 static struct irq_chip xen_pirq_chip;
 196 static void enable_dynirq(struct irq_data *data);
 197 static void disable_dynirq(struct irq_data *data);
 198
 199 static DEFINE_PER_CPU(unsigned int, irq_epoch);
 200
 201 static void clear_evtchn_to_irq_row(int *evtchn_row)
 202 {
 203         unsigned col;
 204
 205         for (col = 0; col < EVTCHN_PER_ROW; col++)
 206                 WRITE_ONCE(evtchn_row[col], -1);
 207 }
 208
 209 static void clear_evtchn_to_irq_all(void)
 210 {
 211         unsigned row;
 212
 213         for (row = 0; row < EVTCHN_ROW(xen_evtchn_max_channels()); row++) {
 214                 if (evtchn_to_irq[row] == NULL)
 215                         continue;
 216                 clear_evtchn_to_irq_row(evtchn_to_irq[row]);
 217         }
 218 }
 219
 220 static int set_evtchn_to_irq(evtchn_port_t evtchn, unsigned int irq)
 221 {
 222         unsigned row;
 223         unsigned col;
 224         int *evtchn_row;
 225
 226         if (evtchn >= xen_evtchn_max_channels())
 227                 return -EINVAL;
 228
 229         row = EVTCHN_ROW(evtchn);
 230         col = EVTCHN_COL(evtchn);
 231
 232         if (evtchn_to_irq[row] == NULL) {
 233                 /* Unallocated irq entries return -1 anyway */
 234                 if (irq == -1)
 235                         return 0;
 236
 237                 evtchn_row = (int *) __get_free_pages(GFP_KERNEL, 0);
 238                 if (evtchn_row == NULL)
 239                         return -ENOMEM;
 240
 241                 clear_evtchn_to_irq_row(evtchn_row);
 242
 243                 /*
 244                  * We've prepared an empty row for the mapping. If a different
 245                  * thread was faster inserting it, we can drop ours.
 246                  */
 247                 if (cmpxchg(&evtchn_to_irq[row], NULL, evtchn_row) != NULL)
 248                         free_page((unsigned long) evtchn_row);
 249         }
 250
 251         WRITE_ONCE(evtchn_to_irq[row][col], irq);
 252         return 0;
 253 }
 254
 255 int get_evtchn_to_irq(evtchn_port_t evtchn)
 256 {
 257         if (evtchn >= xen_evtchn_max_channels())
 258                 return -1;
 259         if (evtchn_to_irq[EVTCHN_ROW(evtchn)] == NULL)
 260                 return -1;
 261         return READ_ONCE(evtchn_to_irq[EVTCHN_ROW(evtchn)][EVTCHN_COL(evtchn)]);
 262 }
 263
 264 /* Get info for IRQ */
 265 static struct irq_info *info_for_irq(unsigned irq)
 266 {
 267         if (irq < nr_legacy_irqs())
 268                 return legacy_info_ptrs[irq];
 269         else
 270                 return irq_get_chip_data(irq);
 271 }
 272
 273 static void set_info_for_irq(unsigned int irq, struct irq_info *info)
 274 {
 275         if (irq < nr_legacy_irqs())
 276                 legacy_info_ptrs[irq] = info;
 277         else
 278                 irq_set_chip_data(irq, info);
 279 }
 280
 281 /* Per CPU channel accounting */
 282 static void channels_on_cpu_dec(struct irq_info *info)
 283 {
 284         if (!info->is_accounted)
 285                 return;
 286
 287         info->is_accounted = 0;
 288
 289         if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))
 290                 return;
 291
 292         WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], -1 , 0));
 293 }
 294
 295 static void channels_on_cpu_inc(struct irq_info *info)
 296 {
 297         if (WARN_ON_ONCE(info->cpu >= nr_cpu_ids))
 298                 return;
 299
 300         if (WARN_ON_ONCE(!atomic_add_unless(&channels_on_cpu[info->cpu], 1,
 301                                             INT_MAX)))
 302                 return;
 303
 304         info->is_accounted = 1;
 305 }
 306
 307 /* Constructors for packed IRQ information. */
 308 static int xen_irq_info_common_setup(struct irq_info *info,
 309                                      unsigned irq,
 310                                      enum xen_irq_type type,
 311                                      evtchn_port_t evtchn,
 312                                      unsigned short cpu)
 313 {
 314         int ret;
 315
 316         BUG_ON(info->type != IRQT_UNBOUND && info->type != type);
 317
 318         info->type = type;
 319         info->irq = irq;
 320         info->evtchn = evtchn;
 321         info->cpu = cpu;
 322         info->mask_reason = EVT_MASK_REASON_EXPLICIT;
 323         raw_spin_lock_init(&info->lock);
 324
 325         ret = set_evtchn_to_irq(evtchn, irq);
 326         if (ret < 0)
 327                 return ret;
 328
 329         irq_clear_status_flags(irq, IRQ_NOREQUEST|IRQ_NOAUTOEN);
 330
 331         return xen_evtchn_port_setup(evtchn);
 332 }
 333
 334 static int xen_irq_info_evtchn_setup(unsigned irq,
 335                                      evtchn_port_t evtchn,
 336                                      struct xenbus_device *dev)
 337 {
 338         struct irq_info *info = info_for_irq(irq);
 339         int ret;
 340
 341         ret = xen_irq_info_common_setup(info, irq, IRQT_EVTCHN, evtchn, 0);
 342         info->u.interdomain = dev;
 343         if (dev)
 344                 atomic_inc(&dev->event_channels);
 345
 346         return ret;
 347 }
 348
 349 static int xen_irq_info_ipi_setup(unsigned cpu,
 350                                   unsigned irq,
 351                                   evtchn_port_t evtchn,
 352                                   enum ipi_vector ipi)
 353 {
 354         struct irq_info *info = info_for_irq(irq);
 355
 356         info->u.ipi = ipi;
 357
 358         per_cpu(ipi_to_irq, cpu)[ipi] = irq;
 359
 360         return xen_irq_info_common_setup(info, irq, IRQT_IPI, evtchn, 0);
 361 }
 362
 363 static int xen_irq_info_virq_setup(unsigned cpu,
 364                                    unsigned irq,
 365                                    evtchn_port_t evtchn,
 366                                    unsigned virq)
 367 {
 368         struct irq_info *info = info_for_irq(irq);
 369
 370         info->u.virq = virq;
 371
 372         per_cpu(virq_to_irq, cpu)[virq] = irq;
 373
 374         return xen_irq_info_common_setup(info, irq, IRQT_VIRQ, evtchn, 0);
 375 }
 376
 377 static int xen_irq_info_pirq_setup(unsigned irq,
 378                                    evtchn_port_t evtchn,
 379                                    unsigned pirq,
 380                                    unsigned gsi,
 381                                    uint16_t domid,
 382                                    unsigned char flags)
 383 {
 384         struct irq_info *info = info_for_irq(irq);
 385
 386         info->u.pirq.pirq = pirq;
 387         info->u.pirq.gsi = gsi;
 388         info->u.pirq.domid = domid;
 389         info->u.pirq.flags = flags;
 390
 391         return xen_irq_info_common_setup(info, irq, IRQT_PIRQ, evtchn, 0);
 392 }
 393
 394 static void xen_irq_info_cleanup(struct irq_info *info)
 395 {
 396         set_evtchn_to_irq(info->evtchn, -1);
 397         xen_evtchn_port_remove(info->evtchn, info->cpu);
 398         info->evtchn = 0;
 399         channels_on_cpu_dec(info);
 400 }
 401
 402 /*
 403  * Accessors for packed IRQ information.
 404  */
 405 evtchn_port_t evtchn_from_irq(unsigned irq)
 406 {
 407         const struct irq_info *info = NULL;
 408
 409         if (likely(irq < nr_irqs))
 410                 info = info_for_irq(irq);
 411         if (!info)
 412                 return 0;
 413
 414         return info->evtchn;
 415 }
 416
 417 unsigned int irq_from_evtchn(evtchn_port_t evtchn)
 418 {
 419         return get_evtchn_to_irq(evtchn);
 420 }
 421 EXPORT_SYMBOL_GPL(irq_from_evtchn);
 422
 423 int irq_from_virq(unsigned int cpu, unsigned int virq)
 424 {
 425         return per_cpu(virq_to_irq, cpu)[virq];
 426 }
 427
 428 static enum ipi_vector ipi_from_irq(unsigned irq)
 429 {
 430         struct irq_info *info = info_for_irq(irq);
 431
 432         BUG_ON(info == NULL);
 433         BUG_ON(info->type != IRQT_IPI);
 434
 435         return info->u.ipi;
 436 }
 437
 438 static unsigned virq_from_irq(unsigned irq)
 439 {
 440         struct irq_info *info = info_for_irq(irq);
 441
 442         BUG_ON(info == NULL);
 443         BUG_ON(info->type != IRQT_VIRQ);
 444
 445         return info->u.virq;
 446 }
 447
 448 static unsigned pirq_from_irq(unsigned irq)
 449 {
 450         struct irq_info *info = info_for_irq(irq);
 451
 452         BUG_ON(info == NULL);
 453         BUG_ON(info->type != IRQT_PIRQ);
 454
 455         return info->u.pirq.pirq;
 456 }
 457
 458 static enum xen_irq_type type_from_irq(unsigned irq)
 459 {
 460         return info_for_irq(irq)->type;
 461 }
 462
 463 static unsigned cpu_from_irq(unsigned irq)
 464 {
 465         return info_for_irq(irq)->cpu;
 466 }
 467
 468 unsigned int cpu_from_evtchn(evtchn_port_t evtchn)
 469 {
 470         int irq = get_evtchn_to_irq(evtchn);
 471         unsigned ret = 0;
 472
 473         if (irq != -1)
 474                 ret = cpu_from_irq(irq);
 475
 476         return ret;
 477 }
 478
 479 static void do_mask(struct irq_info *info, u8 reason)
 480 {
 481         unsigned long flags;
 482
 483         raw_spin_lock_irqsave(&info->lock, flags);
 484
 485         if (!info->mask_reason)
 486                 mask_evtchn(info->evtchn);
 487
 488         info->mask_reason |= reason;
 489
 490         raw_spin_unlock_irqrestore(&info->lock, flags);
 491 }
 492
 493 static void do_unmask(struct irq_info *info, u8 reason)
 494 {
 495         unsigned long flags;
 496
 497         raw_spin_lock_irqsave(&info->lock, flags);
 498
 499         info->mask_reason &= ~reason;
 500
 501         if (!info->mask_reason)
 502                 unmask_evtchn(info->evtchn);
 503
 504         raw_spin_unlock_irqrestore(&info->lock, flags);
 505 }
 506
 507 #ifdef CONFIG_X86
 508 static bool pirq_check_eoi_map(unsigned irq)
 509 {
 510         return test_bit(pirq_from_irq(irq), pirq_eoi_map);
 511 }
 512 #endif
 513
 514 static bool pirq_needs_eoi_flag(unsigned irq)
 515 {
 516         struct irq_info *info = info_for_irq(irq);
 517         BUG_ON(info->type != IRQT_PIRQ);
 518
 519         return info->u.pirq.flags & PIRQ_NEEDS_EOI;
 520 }
 521
 522 static void bind_evtchn_to_cpu(evtchn_port_t evtchn, unsigned int cpu,
 523                                bool force_affinity)
 524 {
 525         int irq = get_evtchn_to_irq(evtchn);
 526         struct irq_info *info = info_for_irq(irq);
 527
 528         BUG_ON(irq == -1);
 529
 530         if (IS_ENABLED(CONFIG_SMP) && force_affinity) {
 531                 cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu));
 532                 cpumask_copy(irq_get_effective_affinity_mask(irq),
 533                              cpumask_of(cpu));
 534         }
 535
 536         xen_evtchn_port_bind_to_cpu(evtchn, cpu, info->cpu);
 537
 538         channels_on_cpu_dec(info);
 539         info->cpu = cpu;
 540         channels_on_cpu_inc(info);
 541 }
 542
 543 /**
 544  * notify_remote_via_irq - send event to remote end of event channel via irq
 545  * @irq: irq of event channel to send event to
 546  *
 547  * Unlike notify_remote_via_evtchn(), this is safe to use across
 548  * save/restore. Notifications on a broken connection are silently
 549  * dropped.
 550  */
 551 void notify_remote_via_irq(int irq)
 552 {
 553         evtchn_port_t evtchn = evtchn_from_irq(irq);
 554
 555         if (VALID_EVTCHN(evtchn))
 556                 notify_remote_via_evtchn(evtchn);
 557 }
 558 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
 559
 560 struct lateeoi_work {
 561         struct delayed_work delayed;
 562         spinlock_t eoi_list_lock;
 563         struct list_head eoi_list;
 564 };
 565
 566 static DEFINE_PER_CPU(struct lateeoi_work, lateeoi);
 567
 568 static void lateeoi_list_del(struct irq_info *info)
 569 {
 570         struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
 571         unsigned long flags;
 572
 573         spin_lock_irqsave(&eoi->eoi_list_lock, flags);
 574         list_del_init(&info->eoi_list);
 575         spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
 576 }
 577
 578 static void lateeoi_list_add(struct irq_info *info)
 579 {
 580         struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
 581         struct irq_info *elem;
 582         u64 now = get_jiffies_64();
 583         unsigned long delay;
 584         unsigned long flags;
 585
 586         if (now < info->eoi_time)
 587                 delay = info->eoi_time - now;
 588         else
 589                 delay = 1;
 590
 591         spin_lock_irqsave(&eoi->eoi_list_lock, flags);
 592
 593         if (list_empty(&eoi->eoi_list)) {
 594                 list_add(&info->eoi_list, &eoi->eoi_list);
 595                 mod_delayed_work_on(info->eoi_cpu, system_wq,
 596                                     &eoi->delayed, delay);
 597         } else {
 598                 list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) {
 599                         if (elem->eoi_time <= info->eoi_time)
 600                                 break;
 601                 }
 602                 list_add(&info->eoi_list, &elem->eoi_list);
 603         }
 604
 605         spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
 606 }
 607
 608 static void xen_irq_lateeoi_locked(struct irq_info *info, bool spurious)
 609 {
 610         evtchn_port_t evtchn;
 611         unsigned int cpu;
 612         unsigned int delay = 0;
 613
 614         evtchn = info->evtchn;
 615         if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))
 616                 return;
 617
 618         if (spurious) {
 619                 struct xenbus_device *dev = info->u.interdomain;
 620                 unsigned int threshold = 1;
 621
 622                 if (dev && dev->spurious_threshold)
 623                         threshold = dev->spurious_threshold;
 624
 625                 if ((1 << info->spurious_cnt) < (HZ << 2)) {
 626                         if (info->spurious_cnt != 0xFF)
 627                                 info->spurious_cnt++;
 628                 }
 629                 if (info->spurious_cnt > threshold) {
 630                         delay = 1 << (info->spurious_cnt - 1 - threshold);
 631                         if (delay > HZ)
 632                                 delay = HZ;
 633                         if (!info->eoi_time)
 634                                 info->eoi_cpu = smp_processor_id();
 635                         info->eoi_time = get_jiffies_64() + delay;
 636                         if (dev)
 637                                 atomic_add(delay, &dev->jiffies_eoi_delayed);
 638                 }
 639                 if (dev)
 640                         atomic_inc(&dev->spurious_events);
 641         } else {
 642                 info->spurious_cnt = 0;
 643         }
 644
 645         cpu = info->eoi_cpu;
 646         if (info->eoi_time &&
 647             (info->irq_epoch == per_cpu(irq_epoch, cpu) || delay)) {
 648                 lateeoi_list_add(info);
 649                 return;
 650         }
 651
 652         info->eoi_time = 0;
 653
 654         /* is_active hasn't been reset yet, do it now. */
 655         smp_store_release(&info->is_active, 0);
 656         do_unmask(info, EVT_MASK_REASON_EOI_PENDING);
 657 }
 658
 659 static void xen_irq_lateeoi_worker(struct work_struct *work)
 660 {
 661         struct lateeoi_work *eoi;
 662         struct irq_info *info;
 663         u64 now = get_jiffies_64();
 664         unsigned long flags;
 665
 666         eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed);
 667
 668         read_lock_irqsave(&evtchn_rwlock, flags);
 669
 670         while (true) {
 671                 spin_lock(&eoi->eoi_list_lock);
 672
 673                 info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
 674                                                 eoi_list);
 675
 676                 if (info == NULL || now < info->eoi_time) {
 677                         spin_unlock(&eoi->eoi_list_lock);
 678                         break;
 679                 }
 680
 681                 list_del_init(&info->eoi_list);
 682
 683                 spin_unlock(&eoi->eoi_list_lock);
 684
 685                 info->eoi_time = 0;
 686
 687                 xen_irq_lateeoi_locked(info, false);
 688         }
 689
 690         if (info)
 691                 mod_delayed_work_on(info->eoi_cpu, system_wq,
 692                                     &eoi->delayed, info->eoi_time - now);
 693
 694         read_unlock_irqrestore(&evtchn_rwlock, flags);
 695 }
 696
 697 static void xen_cpu_init_eoi(unsigned int cpu)
 698 {
 699         struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu);
 700
 701         INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker);
 702         spin_lock_init(&eoi->eoi_list_lock);
 703         INIT_LIST_HEAD(&eoi->eoi_list);
 704 }
 705
 706 void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags)
 707 {
 708         struct irq_info *info;
 709         unsigned long flags;
 710
 711         read_lock_irqsave(&evtchn_rwlock, flags);
 712
 713         info = info_for_irq(irq);
 714
 715         if (info)
 716                 xen_irq_lateeoi_locked(info, eoi_flags & XEN_EOI_FLAG_SPURIOUS);
 717
 718         read_unlock_irqrestore(&evtchn_rwlock, flags);
 719 }
 720 EXPORT_SYMBOL_GPL(xen_irq_lateeoi);
 721
 722 static void xen_irq_init(unsigned irq)
 723 {
 724         struct irq_info *info;
 725
 726         info = kzalloc(sizeof(*info), GFP_KERNEL);
 727         if (info == NULL)
 728                 panic("Unable to allocate metadata for IRQ%d\n", irq);
 729
 730         info->type = IRQT_UNBOUND;
 731         info->refcnt = -1;
 732
 733         set_info_for_irq(irq, info);
 734         /*
 735          * Interrupt affinity setting can be immediate. No point
 736          * in delaying it until an interrupt is handled.
 737          */
 738         irq_set_status_flags(irq, IRQ_MOVE_PCNTXT);
 739
 740         INIT_LIST_HEAD(&info->eoi_list);
 741         list_add_tail(&info->list, &xen_irq_list_head);
 742 }
 743
 744 static int __must_check xen_allocate_irqs_dynamic(int nvec)
 745 {
 746         int i, irq = irq_alloc_descs(-1, 0, nvec, -1);
 747
 748         if (irq >= 0) {
 749                 for (i = 0; i < nvec; i++)
 750                         xen_irq_init(irq + i);
 751         }
 752
 753         return irq;
 754 }
 755
 756 static inline int __must_check xen_allocate_irq_dynamic(void)
 757 {
 758
 759         return xen_allocate_irqs_dynamic(1);
 760 }
 761
 762 static int __must_check xen_allocate_irq_gsi(unsigned gsi)
 763 {
 764         int irq;
 765
 766         /*
 767          * A PV guest has no concept of a GSI (since it has no ACPI
 768          * nor access to/knowledge of the physical APICs). Therefore
 769          * all IRQs are dynamically allocated from the entire IRQ
 770          * space.
 771          */
 772         if (xen_pv_domain() && !xen_initial_domain())
 773                 return xen_allocate_irq_dynamic();
 774
 775         /* Legacy IRQ descriptors are already allocated by the arch. */
 776         if (gsi < nr_legacy_irqs())
 777                 irq = gsi;
 778         else
 779                 irq = irq_alloc_desc_at(gsi, -1);
 780
 781         xen_irq_init(irq);
 782
 783         return irq;
 784 }
 785
 786 static void xen_free_irq(unsigned irq)
 787 {
 788         struct irq_info *info = info_for_irq(irq);
 789         unsigned long flags;
 790
 791         if (WARN_ON(!info))
 792                 return;
 793
 794         write_lock_irqsave(&evtchn_rwlock, flags);
 795
 796         if (!list_empty(&info->eoi_list))
 797                 lateeoi_list_del(info);
 798
 799         list_del(&info->list);
 800
 801         set_info_for_irq(irq, NULL);
 802
 803         WARN_ON(info->refcnt > 0);
 804
 805         write_unlock_irqrestore(&evtchn_rwlock, flags);
 806
 807         kfree(info);
 808
 809         /* Legacy IRQ descriptors are managed by the arch. */
 810         if (irq < nr_legacy_irqs())
 811                 return;
 812
 813         irq_free_desc(irq);
 814 }
 815
 816 static void xen_evtchn_close(evtchn_port_t port)
 817 {
 818         struct evtchn_close close;
 819
 820         close.port = port;
 821         if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
 822                 BUG();
 823 }
 824
 825 /* Not called for lateeoi events. */
 826 static void event_handler_exit(struct irq_info *info)
 827 {
 828         smp_store_release(&info->is_active, 0);
 829         clear_evtchn(info->evtchn);
 830 }
 831
 832 static void pirq_query_unmask(int irq)
 833 {
 834         struct physdev_irq_status_query irq_status;
 835         struct irq_info *info = info_for_irq(irq);
 836
 837         BUG_ON(info->type != IRQT_PIRQ);
 838
 839         irq_status.irq = pirq_from_irq(irq);
 840         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
 841                 irq_status.flags = 0;
 842
 843         info->u.pirq.flags &= ~PIRQ_NEEDS_EOI;
 844         if (irq_status.flags & XENIRQSTAT_needs_eoi)
 845                 info->u.pirq.flags |= PIRQ_NEEDS_EOI;
 846 }
 847
 848 static void eoi_pirq(struct irq_data *data)
 849 {
 850         struct irq_info *info = info_for_irq(data->irq);
 851         evtchn_port_t evtchn = info ? info->evtchn : 0;
 852         struct physdev_eoi eoi = { .irq = pirq_from_irq(data->irq) };
 853         int rc = 0;
 854
 855         if (!VALID_EVTCHN(evtchn))
 856                 return;
 857
 858         event_handler_exit(info);
 859
 860         if (pirq_needs_eoi(data->irq)) {
 861                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
 862                 WARN_ON(rc);
 863         }
 864 }
 865
 866 static void mask_ack_pirq(struct irq_data *data)
 867 {
 868         disable_dynirq(data);
 869         eoi_pirq(data);
 870 }
 871
 872 static unsigned int __startup_pirq(unsigned int irq)
 873 {
 874         struct evtchn_bind_pirq bind_pirq;
 875         struct irq_info *info = info_for_irq(irq);
 876         evtchn_port_t evtchn = evtchn_from_irq(irq);
 877         int rc;
 878
 879         BUG_ON(info->type != IRQT_PIRQ);
 880
 881         if (VALID_EVTCHN(evtchn))
 882                 goto out;
 883
 884         bind_pirq.pirq = pirq_from_irq(irq);
 885         /* NB. We are happy to share unless we are probing. */
 886         bind_pirq.flags = info->u.pirq.flags & PIRQ_SHAREABLE ?
 887                                         BIND_PIRQ__WILL_SHARE : 0;
 888         rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
 889         if (rc != 0) {
 890                 pr_warn("Failed to obtain physical IRQ %d\n", irq);
 891                 return 0;
 892         }
 893         evtchn = bind_pirq.port;
 894
 895         pirq_query_unmask(irq);
 896
 897         rc = set_evtchn_to_irq(evtchn, irq);
 898         if (rc)
 899                 goto err;
 900
 901         info->evtchn = evtchn;
 902         bind_evtchn_to_cpu(evtchn, 0, false);
 903
 904         rc = xen_evtchn_port_setup(evtchn);
 905         if (rc)
 906                 goto err;
 907
 908 out:
 909         do_unmask(info, EVT_MASK_REASON_EXPLICIT);
 910
 911         eoi_pirq(irq_get_irq_data(irq));
 912
 913         return 0;
 914
 915 err:
 916         pr_err("irq%d: Failed to set port to irq mapping (%d)\n", irq, rc);
 917         xen_evtchn_close(evtchn);
 918         return 0;
 919 }
 920
 921 static unsigned int startup_pirq(struct irq_data *data)
 922 {
 923         return __startup_pirq(data->irq);
 924 }
 925
 926 static void shutdown_pirq(struct irq_data *data)
 927 {
 928         unsigned int irq = data->irq;
 929         struct irq_info *info = info_for_irq(irq);
 930         evtchn_port_t evtchn = evtchn_from_irq(irq);
 931
 932         BUG_ON(info->type != IRQT_PIRQ);
 933
 934         if (!VALID_EVTCHN(evtchn))
 935                 return;
 936
 937         do_mask(info, EVT_MASK_REASON_EXPLICIT);
 938         xen_evtchn_close(evtchn);
 939         xen_irq_info_cleanup(info);
 940 }
 941
 942 static void enable_pirq(struct irq_data *data)
 943 {
 944         enable_dynirq(data);
 945 }
 946
 947 static void disable_pirq(struct irq_data *data)
 948 {
 949         disable_dynirq(data);
 950 }
 951
 952 int xen_irq_from_gsi(unsigned gsi)
 953 {
 954         struct irq_info *info;
 955
 956         list_for_each_entry(info, &xen_irq_list_head, list) {
 957                 if (info->type != IRQT_PIRQ)
 958                         continue;
 959
 960                 if (info->u.pirq.gsi == gsi)
 961                         return info->irq;
 962         }
 963
 964         return -1;
 965 }
 966 EXPORT_SYMBOL_GPL(xen_irq_from_gsi);
 967
 968 static void __unbind_from_irq(unsigned int irq)
 969 {
 970         evtchn_port_t evtchn = evtchn_from_irq(irq);
 971         struct irq_info *info = info_for_irq(irq);
 972
 973         if (info->refcnt > 0) {
 974                 info->refcnt--;
 975                 if (info->refcnt != 0)
 976                         return;
 977         }
 978
 979         if (VALID_EVTCHN(evtchn)) {
 980                 unsigned int cpu = cpu_from_irq(irq);
 981                 struct xenbus_device *dev;
 982
 983                 xen_evtchn_close(evtchn);
 984
 985                 switch (type_from_irq(irq)) {
 986                 case IRQT_VIRQ:
 987                         per_cpu(virq_to_irq, cpu)[virq_from_irq(irq)] = -1;
 988                         break;
 989                 case IRQT_IPI:
 990                         per_cpu(ipi_to_irq, cpu)[ipi_from_irq(irq)] = -1;
 991                         break;
 992                 case IRQT_EVTCHN:
 993                         dev = info->u.interdomain;
 994                         if (dev)
 995                                 atomic_dec(&dev->event_channels);
 996                         break;
 997                 default:
 998                         break;
 999                 }
1000
1001                 xen_irq_info_cleanup(info);
1002         }
1003
1004         xen_free_irq(irq);
1005 }
1006
1007 /*
1008  * Do not make any assumptions regarding the relationship between the
1009  * IRQ number returned here and the Xen pirq argument.
1010  *
1011  * Note: We don't assign an event channel until the irq actually started
1012  * up.  Return an existing irq if we've already got one for the gsi.
1013  *
1014  * Shareable implies level triggered, not shareable implies edge
1015  * triggered here.
1016  */
1017 int xen_bind_pirq_gsi_to_irq(unsigned gsi,
1018                              unsigned pirq, int shareable, char *name)
1019 {
1020         int irq;
1021         struct physdev_irq irq_op;
1022         int ret;
1023
1024         mutex_lock(&irq_mapping_update_lock);
1025
1026         irq = xen_irq_from_gsi(gsi);
1027         if (irq != -1) {
1028                 pr_info("%s: returning irq %d for gsi %u\n",
1029                         __func__, irq, gsi);
1030                 goto out;
1031         }
1032
1033         irq = xen_allocate_irq_gsi(gsi);
1034         if (irq < 0)
1035                 goto out;
1036
1037         irq_op.irq = irq;
1038         irq_op.vector = 0;
1039
1040         /* Only the privileged domain can do this. For non-priv, the pcifront
1041          * driver provides a PCI bus that does the call to do exactly
1042          * this in the priv domain. */
1043         if (xen_initial_domain() &&
1044             HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op)) {
1045                 xen_free_irq(irq);
1046                 irq = -ENOSPC;
1047                 goto out;
1048         }
1049
1050         ret = xen_irq_info_pirq_setup(irq, 0, pirq, gsi, DOMID_SELF,
1051                                shareable ? PIRQ_SHAREABLE : 0);
1052         if (ret < 0) {
1053                 __unbind_from_irq(irq);
1054                 irq = ret;
1055                 goto out;
1056         }
1057
1058         pirq_query_unmask(irq);
1059         /* We try to use the handler with the appropriate semantic for the
1060          * type of interrupt: if the interrupt is an edge triggered
1061          * interrupt we use handle_edge_irq.
1062          *
1063          * On the other hand if the interrupt is level triggered we use
1064          * handle_fasteoi_irq like the native code does for this kind of
1065          * interrupts.
1066          *
1067          * Depending on the Xen version, pirq_needs_eoi might return true
1068          * not only for level triggered interrupts but for edge triggered
1069          * interrupts too. In any case Xen always honors the eoi mechanism,
1070          * not injecting any more pirqs of the same kind if the first one
1071          * hasn't received an eoi yet. Therefore using the fasteoi handler
1072          * is the right choice either way.
1073          */
1074         if (shareable)
1075                 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
1076                                 handle_fasteoi_irq, name);
1077         else
1078                 irq_set_chip_and_handler_name(irq, &xen_pirq_chip,
1079                                 handle_edge_irq, name);
1080
1081 out:
1082         mutex_unlock(&irq_mapping_update_lock);
1083
1084         return irq;
1085 }
1086
1087 #ifdef CONFIG_PCI_MSI
1088 int xen_allocate_pirq_msi(struct pci_dev *dev, struct msi_desc *msidesc)
1089 {
1090         int rc;
1091         struct physdev_get_free_pirq op_get_free_pirq;
1092
1093         op_get_free_pirq.type = MAP_PIRQ_TYPE_MSI;
1094         rc = HYPERVISOR_physdev_op(PHYSDEVOP_get_free_pirq, &op_get_free_pirq);
1095
1096         WARN_ONCE(rc == -ENOSYS,
1097                   "hypervisor does not support the PHYSDEVOP_get_free_pirq interface\n");
1098
1099         return rc ? -1 : op_get_free_pirq.pirq;
1100 }
1101
1102 int xen_bind_pirq_msi_to_irq(struct pci_dev *dev, struct msi_desc *msidesc,
1103                              int pirq, int nvec, const char *name, domid_t domid)
1104 {
1105         int i, irq, ret;
1106
1107         mutex_lock(&irq_mapping_update_lock);
1108
1109         irq = xen_allocate_irqs_dynamic(nvec);
1110         if (irq < 0)
1111                 goto out;
1112
1113         for (i = 0; i < nvec; i++) {
1114                 irq_set_chip_and_handler_name(irq + i, &xen_pirq_chip, handle_edge_irq, name);
1115
1116                 ret = xen_irq_info_pirq_setup(irq + i, 0, pirq + i, 0, domid,
1117                                               i == 0 ? 0 : PIRQ_MSI_GROUP);
1118                 if (ret < 0)
1119                         goto error_irq;
1120         }
1121
1122         ret = irq_set_msi_desc(irq, msidesc);
1123         if (ret < 0)
1124                 goto error_irq;
1125 out:
1126         mutex_unlock(&irq_mapping_update_lock);
1127         return irq;
1128 error_irq:
1129         while (nvec--)
1130                 __unbind_from_irq(irq + nvec);
1131         mutex_unlock(&irq_mapping_update_lock);
1132         return ret;
1133 }
1134 #endif
1135
1136 int xen_destroy_irq(int irq)
1137 {
1138         struct physdev_unmap_pirq unmap_irq;
1139         struct irq_info *info = info_for_irq(irq);
1140         int rc = -ENOENT;
1141
1142         mutex_lock(&irq_mapping_update_lock);
1143
1144         /*
1145          * If trying to remove a vector in a MSI group different
1146          * than the first one skip the PIRQ unmap unless this vector
1147          * is the first one in the group.
1148          */
1149         if (xen_initial_domain() && !(info->u.pirq.flags & PIRQ_MSI_GROUP)) {
1150                 unmap_irq.pirq = info->u.pirq.pirq;
1151                 unmap_irq.domid = info->u.pirq.domid;
1152                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap_irq);
1153                 /* If another domain quits without making the pci_disable_msix
1154                  * call, the Xen hypervisor takes care of freeing the PIRQs
1155                  * (free_domain_pirqs).
1156                  */
1157                 if ((rc == -ESRCH && info->u.pirq.domid != DOMID_SELF))
1158                         pr_info("domain %d does not have %d anymore\n",
1159                                 info->u.pirq.domid, info->u.pirq.pirq);
1160                 else if (rc) {
1161                         pr_warn("unmap irq failed %d\n", rc);
1162                         goto out;
1163                 }
1164         }
1165
1166         xen_free_irq(irq);
1167
1168 out:
1169         mutex_unlock(&irq_mapping_update_lock);
1170         return rc;
1171 }
1172
1173 int xen_irq_from_pirq(unsigned pirq)
1174 {
1175         int irq;
1176
1177         struct irq_info *info;
1178
1179         mutex_lock(&irq_mapping_update_lock);
1180
1181         list_for_each_entry(info, &xen_irq_list_head, list) {
1182                 if (info->type != IRQT_PIRQ)
1183                         continue;
1184                 irq = info->irq;
1185                 if (info->u.pirq.pirq == pirq)
1186                         goto out;
1187         }
1188         irq = -1;
1189 out:
1190         mutex_unlock(&irq_mapping_update_lock);
1191
1192         return irq;
1193 }
1194
1195
1196 int xen_pirq_from_irq(unsigned irq)
1197 {
1198         return pirq_from_irq(irq);
1199 }
1200 EXPORT_SYMBOL_GPL(xen_pirq_from_irq);
1201
1202 static int bind_evtchn_to_irq_chip(evtchn_port_t evtchn, struct irq_chip *chip,
1203                                    struct xenbus_device *dev)
1204 {
1205         int irq;
1206         int ret;
1207
1208         if (evtchn >= xen_evtchn_max_channels())
1209                 return -ENOMEM;
1210
1211         mutex_lock(&irq_mapping_update_lock);
1212
1213         irq = get_evtchn_to_irq(evtchn);
1214
1215         if (irq == -1) {
1216                 irq = xen_allocate_irq_dynamic();
1217                 if (irq < 0)
1218                         goto out;
1219
1220                 irq_set_chip_and_handler_name(irq, chip,
1221                                               handle_edge_irq, "event");
1222
1223                 ret = xen_irq_info_evtchn_setup(irq, evtchn, dev);
1224                 if (ret < 0) {
1225                         __unbind_from_irq(irq);
1226                         irq = ret;
1227                         goto out;
1228                 }
1229                 /*
1230                  * New interdomain events are initially bound to vCPU0 This
1231                  * is required to setup the event channel in the first
1232                  * place and also important for UP guests because the
1233                  * affinity setting is not invoked on them so nothing would
1234                  * bind the channel.
1235                  */
1236                 bind_evtchn_to_cpu(evtchn, 0, false);
1237         } else {
1238                 struct irq_info *info = info_for_irq(irq);
1239                 WARN_ON(info == NULL || info->type != IRQT_EVTCHN);
1240         }
1241
1242 out:
1243         mutex_unlock(&irq_mapping_update_lock);
1244
1245         return irq;
1246 }
1247
1248 int bind_evtchn_to_irq(evtchn_port_t evtchn)
1249 {
1250         return bind_evtchn_to_irq_chip(evtchn, &xen_dynamic_chip, NULL);
1251 }
1252 EXPORT_SYMBOL_GPL(bind_evtchn_to_irq);
1253
1254 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
1255 {
1256         struct evtchn_bind_ipi bind_ipi;
1257         evtchn_port_t evtchn;
1258         int ret, irq;
1259
1260         mutex_lock(&irq_mapping_update_lock);
1261
1262         irq = per_cpu(ipi_to_irq, cpu)[ipi];
1263
1264         if (irq == -1) {
1265                 irq = xen_allocate_irq_dynamic();
1266                 if (irq < 0)
1267                         goto out;
1268
1269                 irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
1270                                               handle_percpu_irq, "ipi");
1271
1272                 bind_ipi.vcpu = xen_vcpu_nr(cpu);
1273                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1274                                                 &bind_ipi) != 0)
1275                         BUG();
1276                 evtchn = bind_ipi.port;
1277
1278                 ret = xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
1279                 if (ret < 0) {
1280                         __unbind_from_irq(irq);
1281                         irq = ret;
1282                         goto out;
1283                 }
1284                 /*
1285                  * Force the affinity mask to the target CPU so proc shows
1286                  * the correct target.
1287                  */
1288                 bind_evtchn_to_cpu(evtchn, cpu, true);
1289         } else {
1290                 struct irq_info *info = info_for_irq(irq);
1291                 WARN_ON(info == NULL || info->type != IRQT_IPI);
1292         }
1293
1294  out:
1295         mutex_unlock(&irq_mapping_update_lock);
1296         return irq;
1297 }
1298
1299 static int bind_interdomain_evtchn_to_irq_chip(struct xenbus_device *dev,
1300                                                evtchn_port_t remote_port,
1301                                                struct irq_chip *chip)
1302 {
1303         struct evtchn_bind_interdomain bind_interdomain;
1304         int err;
1305
1306         bind_interdomain.remote_dom  = dev->otherend_id;
1307         bind_interdomain.remote_port = remote_port;
1308
1309         err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1310                                           &bind_interdomain);
1311
1312         return err ? : bind_evtchn_to_irq_chip(bind_interdomain.local_port,
1313                                                chip, dev);
1314 }
1315
1316 int bind_interdomain_evtchn_to_irq_lateeoi(struct xenbus_device *dev,
1317                                            evtchn_port_t remote_port)
1318 {
1319         return bind_interdomain_evtchn_to_irq_chip(dev, remote_port,
1320                                                    &xen_lateeoi_chip);
1321 }
1322 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irq_lateeoi);
1323
1324 static int find_virq(unsigned int virq, unsigned int cpu, evtchn_port_t *evtchn)
1325 {
1326         struct evtchn_status status;
1327         evtchn_port_t port;
1328         int rc = -ENOENT;
1329
1330         memset(&status, 0, sizeof(status));
1331         for (port = 0; port < xen_evtchn_max_channels(); port++) {
1332                 status.dom = DOMID_SELF;
1333                 status.port = port;
1334                 rc = HYPERVISOR_event_channel_op(EVTCHNOP_status, &status);
1335                 if (rc < 0)
1336                         continue;
1337                 if (status.status != EVTCHNSTAT_virq)
1338                         continue;
1339                 if (status.u.virq == virq && status.vcpu == xen_vcpu_nr(cpu)) {
1340                         *evtchn = port;
1341                         break;
1342                 }
1343         }
1344         return rc;
1345 }
1346
1347 /**
1348  * xen_evtchn_nr_channels - number of usable event channel ports
1349  *
1350  * This may be less than the maximum supported by the current
1351  * hypervisor ABI. Use xen_evtchn_max_channels() for the maximum
1352  * supported.
1353  */
1354 unsigned xen_evtchn_nr_channels(void)
1355 {
1356         return evtchn_ops->nr_channels();
1357 }
1358 EXPORT_SYMBOL_GPL(xen_evtchn_nr_channels);
1359
1360 int bind_virq_to_irq(unsigned int virq, unsigned int cpu, bool percpu)
1361 {
1362         struct evtchn_bind_virq bind_virq;
1363         evtchn_port_t evtchn = 0;
1364         int irq, ret;
1365
1366         mutex_lock(&irq_mapping_update_lock);
1367
1368         irq = per_cpu(virq_to_irq, cpu)[virq];
1369
1370         if (irq == -1) {
1371                 irq = xen_allocate_irq_dynamic();
1372                 if (irq < 0)
1373                         goto out;
1374
1375                 if (percpu)
1376                         irq_set_chip_and_handler_name(irq, &xen_percpu_chip,
1377                                                       handle_percpu_irq, "virq");
1378                 else
1379                         irq_set_chip_and_handler_name(irq, &xen_dynamic_chip,
1380                                                       handle_edge_irq, "virq");
1381
1382                 bind_virq.virq = virq;
1383                 bind_virq.vcpu = xen_vcpu_nr(cpu);
1384                 ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1385                                                 &bind_virq);
1386                 if (ret == 0)
1387                         evtchn = bind_virq.port;
1388                 else {
1389                         if (ret == -EEXIST)
1390                                 ret = find_virq(virq, cpu, &evtchn);
1391                         BUG_ON(ret < 0);
1392                 }
1393
1394                 ret = xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1395                 if (ret < 0) {
1396                         __unbind_from_irq(irq);
1397                         irq = ret;
1398                         goto out;
1399                 }
1400
1401                 /*
1402                  * Force the affinity mask for percpu interrupts so proc
1403                  * shows the correct target.
1404                  */
1405                 bind_evtchn_to_cpu(evtchn, cpu, percpu);
1406         } else {
1407                 struct irq_info *info = info_for_irq(irq);
1408                 WARN_ON(info == NULL || info->type != IRQT_VIRQ);
1409         }
1410
1411 out:
1412         mutex_unlock(&irq_mapping_update_lock);
1413
1414         return irq;
1415 }
1416
1417 static void unbind_from_irq(unsigned int irq)
1418 {
1419         mutex_lock(&irq_mapping_update_lock);
1420         __unbind_from_irq(irq);
1421         mutex_unlock(&irq_mapping_update_lock);
1422 }
1423
1424 static int bind_evtchn_to_irqhandler_chip(evtchn_port_t evtchn,
1425                                           irq_handler_t handler,
1426                                           unsigned long irqflags,
1427                                           const char *devname, void *dev_id,
1428                                           struct irq_chip *chip)
1429 {
1430         int irq, retval;
1431
1432         irq = bind_evtchn_to_irq_chip(evtchn, chip, NULL);
1433         if (irq < 0)
1434                 return irq;
1435         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1436         if (retval != 0) {
1437                 unbind_from_irq(irq);
1438                 return retval;
1439         }
1440
1441         return irq;
1442 }
1443
1444 int bind_evtchn_to_irqhandler(evtchn_port_t evtchn,
1445                               irq_handler_t handler,
1446                               unsigned long irqflags,
1447                               const char *devname, void *dev_id)
1448 {
1449         return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1450                                               devname, dev_id,
1451                                               &xen_dynamic_chip);
1452 }
1453 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler);
1454
1455 int bind_evtchn_to_irqhandler_lateeoi(evtchn_port_t evtchn,
1456                                       irq_handler_t handler,
1457                                       unsigned long irqflags,
1458                                       const char *devname, void *dev_id)
1459 {
1460         return bind_evtchn_to_irqhandler_chip(evtchn, handler, irqflags,
1461                                               devname, dev_id,
1462                                               &xen_lateeoi_chip);
1463 }
1464 EXPORT_SYMBOL_GPL(bind_evtchn_to_irqhandler_lateeoi);
1465
1466 static int bind_interdomain_evtchn_to_irqhandler_chip(
1467                 struct xenbus_device *dev, evtchn_port_t remote_port,
1468                 irq_handler_t handler, unsigned long irqflags,
1469                 const char *devname, void *dev_id, struct irq_chip *chip)
1470 {
1471         int irq, retval;
1472
1473         irq = bind_interdomain_evtchn_to_irq_chip(dev, remote_port, chip);
1474         if (irq < 0)
1475                 return irq;
1476
1477         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1478         if (retval != 0) {
1479                 unbind_from_irq(irq);
1480                 return retval;
1481         }
1482
1483         return irq;
1484 }
1485
1486 int bind_interdomain_evtchn_to_irqhandler_lateeoi(struct xenbus_device *dev,
1487                                                   evtchn_port_t remote_port,
1488                                                   irq_handler_t handler,
1489                                                   unsigned long irqflags,
1490                                                   const char *devname,
1491                                                   void *dev_id)
1492 {
1493         return bind_interdomain_evtchn_to_irqhandler_chip(dev,
1494                                 remote_port, handler, irqflags, devname,
1495                                 dev_id, &xen_lateeoi_chip);
1496 }
1497 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler_lateeoi);
1498
1499 int bind_virq_to_irqhandler(unsigned int virq, unsigned int cpu,
1500                             irq_handler_t handler,
1501                             unsigned long irqflags, const char *devname, void *dev_id)
1502 {
1503         int irq, retval;
1504
1505         irq = bind_virq_to_irq(virq, cpu, irqflags & IRQF_PERCPU);
1506         if (irq < 0)
1507                 return irq;
1508         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1509         if (retval != 0) {
1510                 unbind_from_irq(irq);
1511                 return retval;
1512         }
1513
1514         return irq;
1515 }
1516 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
1517
1518 int bind_ipi_to_irqhandler(enum ipi_vector ipi,
1519                            unsigned int cpu,
1520                            irq_handler_t handler,
1521                            unsigned long irqflags,
1522                            const char *devname,
1523                            void *dev_id)
1524 {
1525         int irq, retval;
1526
1527         irq = bind_ipi_to_irq(ipi, cpu);
1528         if (irq < 0)
1529                 return irq;
1530
1531         irqflags |= IRQF_NO_SUSPEND | IRQF_FORCE_RESUME | IRQF_EARLY_RESUME;
1532         retval = request_irq(irq, handler, irqflags, devname, dev_id);
1533         if (retval != 0) {
1534                 unbind_from_irq(irq);
1535                 return retval;
1536         }
1537
1538         return irq;
1539 }
1540
1541 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1542 {
1543         struct irq_info *info = info_for_irq(irq);
1544
1545         if (WARN_ON(!info))
1546                 return;
1547         free_irq(irq, dev_id);
1548         unbind_from_irq(irq);
1549 }
1550 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1551
1552 /**
1553  * xen_set_irq_priority() - set an event channel priority.
1554  * @irq:irq bound to an event channel.
1555  * @priority: priority between XEN_IRQ_PRIORITY_MAX and XEN_IRQ_PRIORITY_MIN.
1556  */
1557 int xen_set_irq_priority(unsigned irq, unsigned priority)
1558 {
1559         struct evtchn_set_priority set_priority;
1560
1561         set_priority.port = evtchn_from_irq(irq);
1562         set_priority.priority = priority;
1563
1564         return HYPERVISOR_event_channel_op(EVTCHNOP_set_priority,
1565                                            &set_priority);
1566 }
1567 EXPORT_SYMBOL_GPL(xen_set_irq_priority);
1568
1569 int evtchn_make_refcounted(evtchn_port_t evtchn)
1570 {
1571         int irq = get_evtchn_to_irq(evtchn);
1572         struct irq_info *info;
1573
1574         if (irq == -1)
1575                 return -ENOENT;
1576
1577         info = info_for_irq(irq);
1578
1579         if (!info)
1580                 return -ENOENT;
1581
1582         WARN_ON(info->refcnt != -1);
1583
1584         info->refcnt = 1;
1585
1586         return 0;
1587 }
1588 EXPORT_SYMBOL_GPL(evtchn_make_refcounted);
1589
1590 int evtchn_get(evtchn_port_t evtchn)
1591 {
1592         int irq;
1593         struct irq_info *info;
1594         int err = -ENOENT;
1595
1596         if (evtchn >= xen_evtchn_max_channels())
1597                 return -EINVAL;
1598
1599         mutex_lock(&irq_mapping_update_lock);
1600
1601         irq = get_evtchn_to_irq(evtchn);
1602         if (irq == -1)
1603                 goto done;
1604
1605         info = info_for_irq(irq);
1606
1607         if (!info)
1608                 goto done;
1609
1610         err = -EINVAL;
1611         if (info->refcnt <= 0 || info->refcnt == SHRT_MAX)
1612                 goto done;
1613
1614         info->refcnt++;
1615         err = 0;
1616  done:
1617         mutex_unlock(&irq_mapping_update_lock);
1618
1619         return err;
1620 }
1621 EXPORT_SYMBOL_GPL(evtchn_get);
1622
1623 void evtchn_put(evtchn_port_t evtchn)
1624 {
1625         int irq = get_evtchn_to_irq(evtchn);
1626         if (WARN_ON(irq == -1))
1627                 return;
1628         unbind_from_irq(irq);
1629 }
1630 EXPORT_SYMBOL_GPL(evtchn_put);
1631
1632 void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
1633 {
1634         int irq;
1635
1636 #ifdef CONFIG_X86
1637         if (unlikely(vector == XEN_NMI_VECTOR)) {
1638                 int rc =  HYPERVISOR_vcpu_op(VCPUOP_send_nmi, xen_vcpu_nr(cpu),
1639                                              NULL);
1640                 if (rc < 0)
1641                         printk(KERN_WARNING "Sending nmi to CPU%d failed (rc:%d)\n", cpu, rc);
1642                 return;
1643         }
1644 #endif
1645         irq = per_cpu(ipi_to_irq, cpu)[vector];
1646         BUG_ON(irq < 0);
1647         notify_remote_via_irq(irq);
1648 }
1649
1650 struct evtchn_loop_ctrl {
1651         ktime_t timeout;
1652         unsigned count;
1653         bool defer_eoi;
1654 };
1655
1656 void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl)
1657 {
1658         int irq;
1659         struct irq_info *info;
1660         struct xenbus_device *dev;
1661
1662         irq = get_evtchn_to_irq(port);
1663         if (irq == -1)
1664                 return;
1665
1666         /*
1667          * Check for timeout every 256 events.
1668          * We are setting the timeout value only after the first 256
1669          * events in order to not hurt the common case of few loop
1670          * iterations. The 256 is basically an arbitrary value.
1671          *
1672          * In case we are hitting the timeout we need to defer all further
1673          * EOIs in order to ensure to leave the event handling loop rather
1674          * sooner than later.
1675          */
1676         if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) {
1677                 ktime_t kt = ktime_get();
1678
1679                 if (!ctrl->timeout) {
1680                         kt = ktime_add_ms(kt,
1681                                           jiffies_to_msecs(event_loop_timeout));
1682                         ctrl->timeout = kt;
1683                 } else if (kt > ctrl->timeout) {
1684                         ctrl->defer_eoi = true;
1685                 }
1686         }
1687
1688         info = info_for_irq(irq);
1689         if (xchg_acquire(&info->is_active, 1))
1690                 return;
1691
1692         dev = (info->type == IRQT_EVTCHN) ? info->u.interdomain : NULL;
1693         if (dev)
1694                 atomic_inc(&dev->events);
1695
1696         if (ctrl->defer_eoi) {
1697                 info->eoi_cpu = smp_processor_id();
1698                 info->irq_epoch = __this_cpu_read(irq_epoch);
1699                 info->eoi_time = get_jiffies_64() + event_eoi_delay;
1700         }
1701
1702         generic_handle_irq(irq);
1703 }
1704
1705 static void __xen_evtchn_do_upcall(void)
1706 {
1707         struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
1708         int cpu = smp_processor_id();
1709         struct evtchn_loop_ctrl ctrl = { 0 };
1710
1711         read_lock(&evtchn_rwlock);
1712
1713         do {
1714                 vcpu_info->evtchn_upcall_pending = 0;
1715
1716                 xen_evtchn_handle_events(cpu, &ctrl);
1717
1718                 BUG_ON(!irqs_disabled());
1719
1720                 virt_rmb(); /* Hypervisor can set upcall pending. */
1721
1722         } while (vcpu_info->evtchn_upcall_pending);
1723
1724         read_unlock(&evtchn_rwlock);
1725
1726         /*
1727          * Increment irq_epoch only now to defer EOIs only for
1728          * xen_irq_lateeoi() invocations occurring from inside the loop
1729          * above.
1730          */
1731         __this_cpu_inc(irq_epoch);
1732 }
1733
1734 void xen_evtchn_do_upcall(struct pt_regs *regs)
1735 {
1736         struct pt_regs *old_regs = set_irq_regs(regs);
1737
1738         irq_enter();
1739
1740         __xen_evtchn_do_upcall();
1741
1742         irq_exit();
1743         set_irq_regs(old_regs);
1744 }
1745
1746 void xen_hvm_evtchn_do_upcall(void)
1747 {
1748         __xen_evtchn_do_upcall();
1749 }
1750 EXPORT_SYMBOL_GPL(xen_hvm_evtchn_do_upcall);
1751
1752 /* Rebind a new event channel to an existing irq. */
1753 void rebind_evtchn_irq(evtchn_port_t evtchn, int irq)
1754 {
1755         struct irq_info *info = info_for_irq(irq);
1756
1757         if (WARN_ON(!info))
1758                 return;
1759
1760         /* Make sure the irq is masked, since the new event channel
1761            will also be masked. */
1762         disable_irq(irq);
1763
1764         mutex_lock(&irq_mapping_update_lock);
1765
1766         /* After resume the irq<->evtchn mappings are all cleared out */
1767         BUG_ON(get_evtchn_to_irq(evtchn) != -1);
1768         /* Expect irq to have been bound before,
1769            so there should be a proper type */
1770         BUG_ON(info->type == IRQT_UNBOUND);
1771
1772         (void)xen_irq_info_evtchn_setup(irq, evtchn, NULL);
1773
1774         mutex_unlock(&irq_mapping_update_lock);
1775
1776         bind_evtchn_to_cpu(evtchn, info->cpu, false);
1777
1778         /* Unmask the event channel. */
1779         enable_irq(irq);
1780 }
1781
1782 /* Rebind an evtchn so that it gets delivered to a specific cpu */
1783 static int xen_rebind_evtchn_to_cpu(struct irq_info *info, unsigned int tcpu)
1784 {
1785         struct evtchn_bind_vcpu bind_vcpu;
1786         evtchn_port_t evtchn = info ? info->evtchn : 0;
1787
1788         if (!VALID_EVTCHN(evtchn))
1789                 return -1;
1790
1791         if (!xen_support_evtchn_rebind())
1792                 return -1;
1793
1794         /* Send future instances of this interrupt to other vcpu. */
1795         bind_vcpu.port = evtchn;
1796         bind_vcpu.vcpu = xen_vcpu_nr(tcpu);
1797
1798         /*
1799          * Mask the event while changing the VCPU binding to prevent
1800          * it being delivered on an unexpected VCPU.
1801          */
1802         do_mask(info, EVT_MASK_REASON_TEMPORARY);
1803
1804         /*
1805          * If this fails, it usually just indicates that we're dealing with a
1806          * virq or IPI channel, which don't actually need to be rebound. Ignore
1807          * it, but don't do the xenlinux-level rebind in that case.
1808          */
1809         if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu) >= 0)
1810                 bind_evtchn_to_cpu(evtchn, tcpu, false);
1811
1812         do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1813
1814         return 0;
1815 }
1816
1817 /*
1818  * Find the CPU within @dest mask which has the least number of channels
1819  * assigned. This is not precise as the per cpu counts can be modified
1820  * concurrently.
1821  */
1822 static unsigned int select_target_cpu(const struct cpumask *dest)
1823 {
1824         unsigned int cpu, best_cpu = UINT_MAX, minch = UINT_MAX;
1825
1826         for_each_cpu_and(cpu, dest, cpu_online_mask) {
1827                 unsigned int curch = atomic_read(&channels_on_cpu[cpu]);
1828
1829                 if (curch < minch) {
1830                         minch = curch;
1831                         best_cpu = cpu;
1832                 }
1833         }
1834
1835         /*
1836          * Catch the unlikely case that dest contains no online CPUs. Can't
1837          * recurse.
1838          */
1839         if (best_cpu == UINT_MAX)
1840                 return select_target_cpu(cpu_online_mask);
1841
1842         return best_cpu;
1843 }
1844
1845 static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1846                             bool force)
1847 {
1848         unsigned int tcpu = select_target_cpu(dest);
1849         int ret;
1850
1851         ret = xen_rebind_evtchn_to_cpu(info_for_irq(data->irq), tcpu);
1852         if (!ret)
1853                 irq_data_update_effective_affinity(data, cpumask_of(tcpu));
1854
1855         return ret;
1856 }
1857
1858 static void enable_dynirq(struct irq_data *data)
1859 {
1860         struct irq_info *info = info_for_irq(data->irq);
1861         evtchn_port_t evtchn = info ? info->evtchn : 0;
1862
1863         if (VALID_EVTCHN(evtchn))
1864                 do_unmask(info, EVT_MASK_REASON_EXPLICIT);
1865 }
1866
1867 static void disable_dynirq(struct irq_data *data)
1868 {
1869         struct irq_info *info = info_for_irq(data->irq);
1870         evtchn_port_t evtchn = info ? info->evtchn : 0;
1871
1872         if (VALID_EVTCHN(evtchn))
1873                 do_mask(info, EVT_MASK_REASON_EXPLICIT);
1874 }
1875
1876 static void ack_dynirq(struct irq_data *data)
1877 {
1878         struct irq_info *info = info_for_irq(data->irq);
1879         evtchn_port_t evtchn = info ? info->evtchn : 0;
1880
1881         if (VALID_EVTCHN(evtchn))
1882                 event_handler_exit(info);
1883 }
1884
1885 static void mask_ack_dynirq(struct irq_data *data)
1886 {
1887         disable_dynirq(data);
1888         ack_dynirq(data);
1889 }
1890
1891 static void lateeoi_ack_dynirq(struct irq_data *data)
1892 {
1893         struct irq_info *info = info_for_irq(data->irq);
1894         evtchn_port_t evtchn = info ? info->evtchn : 0;
1895
1896         if (VALID_EVTCHN(evtchn)) {
1897                 do_mask(info, EVT_MASK_REASON_EOI_PENDING);
1898                 /*
1899                  * Don't call event_handler_exit().
1900                  * Need to keep is_active non-zero in order to ignore re-raised
1901                  * events after cpu affinity changes while a lateeoi is pending.
1902                  */
1903                 clear_evtchn(evtchn);
1904         }
1905 }
1906
1907 static void lateeoi_mask_ack_dynirq(struct irq_data *data)
1908 {
1909         struct irq_info *info = info_for_irq(data->irq);
1910         evtchn_port_t evtchn = info ? info->evtchn : 0;
1911
1912         if (VALID_EVTCHN(evtchn)) {
1913                 do_mask(info, EVT_MASK_REASON_EXPLICIT);
1914                 event_handler_exit(info);
1915         }
1916 }
1917
1918 static int retrigger_dynirq(struct irq_data *data)
1919 {
1920         struct irq_info *info = info_for_irq(data->irq);
1921         evtchn_port_t evtchn = info ? info->evtchn : 0;
1922
1923         if (!VALID_EVTCHN(evtchn))
1924                 return 0;
1925
1926         do_mask(info, EVT_MASK_REASON_TEMPORARY);
1927         set_evtchn(evtchn);
1928         do_unmask(info, EVT_MASK_REASON_TEMPORARY);
1929
1930         return 1;
1931 }
1932
1933 static void restore_pirqs(void)
1934 {
1935         int pirq, rc, irq, gsi;
1936         struct physdev_map_pirq map_irq;
1937         struct irq_info *info;
1938
1939         list_for_each_entry(info, &xen_irq_list_head, list) {
1940                 if (info->type != IRQT_PIRQ)
1941                         continue;
1942
1943                 pirq = info->u.pirq.pirq;
1944                 gsi = info->u.pirq.gsi;
1945                 irq = info->irq;
1946
1947                 /* save/restore of PT devices doesn't work, so at this point the
1948                  * only devices present are GSI based emulated devices */
1949                 if (!gsi)
1950                         continue;
1951
1952                 map_irq.domid = DOMID_SELF;
1953                 map_irq.type = MAP_PIRQ_TYPE_GSI;
1954                 map_irq.index = gsi;
1955                 map_irq.pirq = pirq;
1956
1957                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq);
1958                 if (rc) {
1959                         pr_warn("xen map irq failed gsi=%d irq=%d pirq=%d rc=%d\n",
1960                                 gsi, irq, pirq, rc);
1961                         xen_free_irq(irq);
1962                         continue;
1963                 }
1964
1965                 printk(KERN_DEBUG "xen: --> irq=%d, pirq=%d\n", irq, map_irq.pirq);
1966
1967                 __startup_pirq(irq);
1968         }
1969 }
1970
1971 static void restore_cpu_virqs(unsigned int cpu)
1972 {
1973         struct evtchn_bind_virq bind_virq;
1974         evtchn_port_t evtchn;
1975         int virq, irq;
1976
1977         for (virq = 0; virq < NR_VIRQS; virq++) {
1978                 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1979                         continue;
1980
1981                 BUG_ON(virq_from_irq(irq) != virq);
1982
1983                 /* Get a new binding from Xen. */
1984                 bind_virq.virq = virq;
1985                 bind_virq.vcpu = xen_vcpu_nr(cpu);
1986                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1987                                                 &bind_virq) != 0)
1988                         BUG();
1989                 evtchn = bind_virq.port;
1990
1991                 /* Record the new mapping. */
1992                 (void)xen_irq_info_virq_setup(cpu, irq, evtchn, virq);
1993                 /* The affinity mask is still valid */
1994                 bind_evtchn_to_cpu(evtchn, cpu, false);
1995         }
1996 }
1997
1998 static void restore_cpu_ipis(unsigned int cpu)
1999 {
2000         struct evtchn_bind_ipi bind_ipi;
2001         evtchn_port_t evtchn;
2002         int ipi, irq;
2003
2004         for (ipi = 0; ipi < XEN_NR_IPIS; ipi++) {
2005                 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
2006                         continue;
2007
2008                 BUG_ON(ipi_from_irq(irq) != ipi);
2009
2010                 /* Get a new binding from Xen. */
2011                 bind_ipi.vcpu = xen_vcpu_nr(cpu);
2012                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
2013                                                 &bind_ipi) != 0)
2014                         BUG();
2015                 evtchn = bind_ipi.port;
2016
2017                 /* Record the new mapping. */
2018                 (void)xen_irq_info_ipi_setup(cpu, irq, evtchn, ipi);
2019                 /* The affinity mask is still valid */
2020                 bind_evtchn_to_cpu(evtchn, cpu, false);
2021         }
2022 }
2023
2024 /* Clear an irq's pending state, in preparation for polling on it */
2025 void xen_clear_irq_pending(int irq)
2026 {
2027         struct irq_info *info = info_for_irq(irq);
2028         evtchn_port_t evtchn = info ? info->evtchn : 0;
2029
2030         if (VALID_EVTCHN(evtchn))
2031                 event_handler_exit(info);
2032 }
2033 EXPORT_SYMBOL(xen_clear_irq_pending);
2034 void xen_set_irq_pending(int irq)
2035 {
2036         evtchn_port_t evtchn = evtchn_from_irq(irq);
2037
2038         if (VALID_EVTCHN(evtchn))
2039                 set_evtchn(evtchn);
2040 }
2041
2042 bool xen_test_irq_pending(int irq)
2043 {
2044         evtchn_port_t evtchn = evtchn_from_irq(irq);
2045         bool ret = false;
2046
2047         if (VALID_EVTCHN(evtchn))
2048                 ret = test_evtchn(evtchn);
2049
2050         return ret;
2051 }
2052
2053 /* Poll waiting for an irq to become pending with timeout.  In the usual case,
2054  * the irq will be disabled so it won't deliver an interrupt. */
2055 void xen_poll_irq_timeout(int irq, u64 timeout)
2056 {
2057         evtchn_port_t evtchn = evtchn_from_irq(irq);
2058
2059         if (VALID_EVTCHN(evtchn)) {
2060                 struct sched_poll poll;
2061
2062                 poll.nr_ports = 1;
2063                 poll.timeout = timeout;
2064                 set_xen_guest_handle(poll.ports, &evtchn);
2065
2066                 if (HYPERVISOR_sched_op(SCHEDOP_poll, &poll) != 0)
2067                         BUG();
2068         }
2069 }
2070 EXPORT_SYMBOL(xen_poll_irq_timeout);
2071 /* Poll waiting for an irq to become pending.  In the usual case, the
2072  * irq will be disabled so it won't deliver an interrupt. */
2073 void xen_poll_irq(int irq)
2074 {
2075         xen_poll_irq_timeout(irq, 0 /* no timeout */);
2076 }
2077
2078 /* Check whether the IRQ line is shared with other guests. */
2079 int xen_test_irq_shared(int irq)
2080 {
2081         struct irq_info *info = info_for_irq(irq);
2082         struct physdev_irq_status_query irq_status;
2083
2084         if (WARN_ON(!info))
2085                 return -ENOENT;
2086
2087         irq_status.irq = info->u.pirq.pirq;
2088
2089         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
2090                 return 0;
2091         return !(irq_status.flags & XENIRQSTAT_shared);
2092 }
2093 EXPORT_SYMBOL_GPL(xen_test_irq_shared);
2094
2095 void xen_irq_resume(void)
2096 {
2097         unsigned int cpu;
2098         struct irq_info *info;
2099
2100         /* New event-channel space is not 'live' yet. */
2101         xen_evtchn_resume();
2102
2103         /* No IRQ <-> event-channel mappings. */
2104         list_for_each_entry(info, &xen_irq_list_head, list) {
2105                 /* Zap event-channel binding */
2106                 info->evtchn = 0;
2107                 /* Adjust accounting */
2108                 channels_on_cpu_dec(info);
2109         }
2110
2111         clear_evtchn_to_irq_all();
2112
2113         for_each_possible_cpu(cpu) {
2114                 restore_cpu_virqs(cpu);
2115                 restore_cpu_ipis(cpu);
2116         }
2117
2118         restore_pirqs();
2119 }
2120
2121 static struct irq_chip xen_dynamic_chip __read_mostly = {
2122         .name                   = "xen-dyn",
2123
2124         .irq_disable            = disable_dynirq,
2125         .irq_mask               = disable_dynirq,
2126         .irq_unmask             = enable_dynirq,
2127
2128         .irq_ack                = ack_dynirq,
2129         .irq_mask_ack           = mask_ack_dynirq,
2130
2131         .irq_set_affinity       = set_affinity_irq,
2132         .irq_retrigger          = retrigger_dynirq,
2133 };
2134
2135 static struct irq_chip xen_lateeoi_chip __read_mostly = {
2136         /* The chip name needs to contain "xen-dyn" for irqbalance to work. */
2137         .name                   = "xen-dyn-lateeoi",
2138
2139         .irq_disable            = disable_dynirq,
2140         .irq_mask               = disable_dynirq,
2141         .irq_unmask             = enable_dynirq,
2142
2143         .irq_ack                = lateeoi_ack_dynirq,
2144         .irq_mask_ack           = lateeoi_mask_ack_dynirq,
2145
2146         .irq_set_affinity       = set_affinity_irq,
2147         .irq_retrigger          = retrigger_dynirq,
2148 };
2149
2150 static struct irq_chip xen_pirq_chip __read_mostly = {
2151         .name                   = "xen-pirq",
2152
2153         .irq_startup            = startup_pirq,
2154         .irq_shutdown           = shutdown_pirq,
2155         .irq_enable             = enable_pirq,
2156         .irq_disable            = disable_pirq,
2157
2158         .irq_mask               = disable_dynirq,
2159         .irq_unmask             = enable_dynirq,
2160
2161         .irq_ack                = eoi_pirq,
2162         .irq_eoi                = eoi_pirq,
2163         .irq_mask_ack           = mask_ack_pirq,
2164
2165         .irq_set_affinity       = set_affinity_irq,
2166
2167         .irq_retrigger          = retrigger_dynirq,
2168 };
2169
2170 static struct irq_chip xen_percpu_chip __read_mostly = {
2171         .name                   = "xen-percpu",
2172
2173         .irq_disable            = disable_dynirq,
2174         .irq_mask               = disable_dynirq,
2175         .irq_unmask             = enable_dynirq,
2176
2177         .irq_ack                = ack_dynirq,
2178 };
2179
2180 #ifdef CONFIG_XEN_PVHVM
2181 /* Vector callbacks are better than PCI interrupts to receive event
2182  * channel notifications because we can receive vector callbacks on any
2183  * vcpu and we don't need PCI support or APIC interactions. */
2184 void xen_setup_callback_vector(void)
2185 {
2186         uint64_t callback_via;
2187
2188         if (xen_have_vector_callback) {
2189                 callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR);
2190                 if (xen_set_callback_via(callback_via)) {
2191                         pr_err("Request for Xen HVM callback vector failed\n");
2192                         xen_have_vector_callback = 0;
2193                 }
2194         }
2195 }
2196
2197 static __init void xen_alloc_callback_vector(void)
2198 {
2199         if (!xen_have_vector_callback)
2200                 return;
2201
2202         pr_info("Xen HVM callback vector for event delivery is enabled\n");
2203         alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback);
2204 }
2205 #else
2206 void xen_setup_callback_vector(void) {}
2207 static inline void xen_alloc_callback_vector(void) {}
2208 #endif
2209
2210 bool xen_fifo_events = true;
2211 module_param_named(fifo_events, xen_fifo_events, bool, 0);
2212
2213 static int xen_evtchn_cpu_prepare(unsigned int cpu)
2214 {
2215         int ret = 0;
2216
2217         xen_cpu_init_eoi(cpu);
2218
2219         if (evtchn_ops->percpu_init)
2220                 ret = evtchn_ops->percpu_init(cpu);
2221
2222         return ret;
2223 }
2224
2225 static int xen_evtchn_cpu_dead(unsigned int cpu)
2226 {
2227         int ret = 0;
2228
2229         if (evtchn_ops->percpu_deinit)
2230                 ret = evtchn_ops->percpu_deinit(cpu);
2231
2232         return ret;
2233 }
2234
2235 void __init xen_init_IRQ(void)
2236 {
2237         int ret = -EINVAL;
2238         evtchn_port_t evtchn;
2239
2240         if (xen_fifo_events)
2241                 ret = xen_evtchn_fifo_init();
2242         if (ret < 0) {
2243                 xen_evtchn_2l_init();
2244                 xen_fifo_events = false;
2245         }
2246
2247         xen_cpu_init_eoi(smp_processor_id());
2248
2249         cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
2250                                   "xen/evtchn:prepare",
2251                                   xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
2252
2253         evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
2254                                 sizeof(*evtchn_to_irq), GFP_KERNEL);
2255         BUG_ON(!evtchn_to_irq);
2256
2257         /* No event channels are 'live' right now. */
2258         for (evtchn = 0; evtchn < xen_evtchn_nr_channels(); evtchn++)
2259                 mask_evtchn(evtchn);
2260
2261         pirq_needs_eoi = pirq_needs_eoi_flag;
2262
2263 #ifdef CONFIG_X86
2264         if (xen_pv_domain()) {
2265                 if (xen_initial_domain())
2266                         pci_xen_initial_domain();
2267         }
2268         if (xen_feature(XENFEAT_hvm_callback_vector)) {
2269                 xen_setup_callback_vector();
2270                 xen_alloc_callback_vector();
2271         }
2272
2273         if (xen_hvm_domain()) {
2274                 native_init_IRQ();
2275                 /* pci_xen_hvm_init must be called after native_init_IRQ so that
2276                  * __acpi_register_gsi can point at the right function */
2277                 pci_xen_hvm_init();
2278         } else {
2279                 int rc;
2280                 struct physdev_pirq_eoi_gmfn eoi_gmfn;
2281
2282                 pirq_eoi_map = (void *)__get_free_page(GFP_KERNEL|__GFP_ZERO);
2283                 eoi_gmfn.gmfn = virt_to_gfn(pirq_eoi_map);
2284                 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
2285                 if (rc != 0) {
2286                         free_page((unsigned long) pirq_eoi_map);
2287                         pirq_eoi_map = NULL;
2288                 } else
2289                         pirq_needs_eoi = pirq_check_eoi_map;
2290         }
2291 #endif
2292 }