1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2017 Benjamin Herrenschmidt, IBM Corporation.
6 #define pr_fmt(fmt) "xive-kvm: " fmt
8 #include <linux/kernel.h>
9 #include <linux/kvm_host.h>
10 #include <linux/err.h>
11 #include <linux/gfp.h>
12 #include <linux/spinlock.h>
13 #include <linux/delay.h>
14 #include <linux/percpu.h>
15 #include <linux/cpumask.h>
16 #include <linux/uaccess.h>
17 #include <linux/irqdomain.h>
18 #include <asm/kvm_book3s.h>
19 #include <asm/kvm_ppc.h>
20 #include <asm/hvcall.h>
23 #include <asm/xive-regs.h>
24 #include <asm/debug.h>
28 #include <linux/debugfs.h>
29 #include <linux/seq_file.h>
31 #include "book3s_xive.h"
35 * Virtual mode variants of the hcalls for use on radix/radix
36 * with AIL. They require the VCPU's VP to be "pushed"
38 * We still instantiate them here because we use some of the
39 * generated utility functions as well in this file.
41 #define XIVE_RUNTIME_CHECKS
42 #define X_PFX xive_vm_
43 #define X_STATIC static
44 #define X_STAT_PFX stat_vm_
45 #define __x_tima xive_tima
46 #define __x_eoi_page(xd) ((void __iomem *)((xd)->eoi_mmio))
47 #define __x_trig_page(xd) ((void __iomem *)((xd)->trig_mmio))
48 #define __x_writeb __raw_writeb
49 #define __x_readw __raw_readw
50 #define __x_readq __raw_readq
51 #define __x_writeq __raw_writeq
53 #include "book3s_xive_template.c"
56 * We leave a gap of a couple of interrupts in the queue to
57 * account for the IPI and additional safety guard.
61 static bool kvmppc_xive_vcpu_has_save_restore(struct kvm_vcpu *vcpu)
63 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
65 /* Check enablement at VP level */
66 return xc->vp_cam & TM_QW1W2_HO;
69 bool kvmppc_xive_check_save_restore(struct kvm_vcpu *vcpu)
71 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
72 struct kvmppc_xive *xive = xc->xive;
74 if (xive->flags & KVMPPC_XIVE_FLAG_SAVE_RESTORE)
75 return kvmppc_xive_vcpu_has_save_restore(vcpu);
81 * Push a vcpu's context to the XIVE on guest entry.
82 * This assumes we are in virtual mode (MMU on)
84 void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu)
86 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
90 * Nothing to do if the platform doesn't have a XIVE
91 * or this vCPU doesn't have its own XIVE context
92 * (e.g. because it's not using an in-kernel interrupt controller).
94 if (!tima || !vcpu->arch.xive_cam_word)
98 if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
99 __raw_writeq(vcpu->arch.xive_saved_state.w01, tima + TM_QW1_OS);
100 __raw_writel(vcpu->arch.xive_cam_word, tima + TM_QW1_OS + TM_WORD2);
101 vcpu->arch.xive_pushed = 1;
105 * We clear the irq_pending flag. There is a small chance of a
106 * race vs. the escalation interrupt happening on another
107 * processor setting it again, but the only consequence is to
108 * cause a spurious wakeup on the next H_CEDE, which is not an
111 vcpu->arch.irq_pending = 0;
114 * In single escalation mode, if the escalation interrupt is
117 if (vcpu->arch.xive_esc_on) {
118 pq = __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
119 XIVE_ESB_SET_PQ_01));
123 * We have a possible subtle race here: The escalation
124 * interrupt might have fired and be on its way to the
125 * host queue while we mask it, and if we unmask it
126 * early enough (re-cede right away), there is a
127 * theorical possibility that it fires again, thus
128 * landing in the target queue more than once which is
131 * Fortunately, solving this is rather easy. If the
132 * above load setting PQ to 01 returns a previous
133 * value where P is set, then we know the escalation
134 * interrupt is somewhere on its way to the host. In
135 * that case we simply don't clear the xive_esc_on
136 * flag below. It will be eventually cleared by the
137 * handler for the escalation interrupt.
139 * Then, when doing a cede, we check that flag again
140 * before re-enabling the escalation interrupt, and if
141 * set, we abort the cede.
143 if (!(pq & XIVE_ESB_VAL_P))
144 /* Now P is 0, we can clear the flag */
145 vcpu->arch.xive_esc_on = 0;
148 EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu);
151 * Pull a vcpu's context from the XIVE on guest exit.
152 * This assumes we are in virtual mode (MMU on)
154 void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu)
156 void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt;
158 if (!vcpu->arch.xive_pushed)
162 * Should not have been pushed if there is no tima
168 /* First load to pull the context, we ignore the value */
169 __raw_readl(tima + TM_SPC_PULL_OS_CTX);
170 /* Second load to recover the context state (Words 0 and 1) */
171 if (!kvmppc_xive_vcpu_has_save_restore(vcpu))
172 vcpu->arch.xive_saved_state.w01 = __raw_readq(tima + TM_QW1_OS);
174 /* Fixup some of the state for the next load */
175 vcpu->arch.xive_saved_state.lsmfb = 0;
176 vcpu->arch.xive_saved_state.ack = 0xff;
177 vcpu->arch.xive_pushed = 0;
180 EXPORT_SYMBOL_GPL(kvmppc_xive_pull_vcpu);
182 void kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu)
184 void __iomem *esc_vaddr = (void __iomem *)vcpu->arch.xive_esc_vaddr;
189 /* we are using XIVE with single escalation */
191 if (vcpu->arch.xive_esc_on) {
193 * If we still have a pending escalation, abort the cede,
194 * and we must set PQ to 10 rather than 00 so that we don't
195 * potentially end up with two entries for the escalation
196 * interrupt in the XIVE interrupt queue. In that case
197 * we also don't want to set xive_esc_on to 1 here in
198 * case we race with xive_esc_irq().
200 vcpu->arch.ceded = 0;
202 * The escalation interrupts are special as we don't EOI them.
203 * There is no need to use the load-after-store ordering offset
204 * to set PQ to 10 as we won't use StoreEOI.
206 __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_10);
208 vcpu->arch.xive_esc_on = true;
210 __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_00);
214 EXPORT_SYMBOL_GPL(kvmppc_xive_rearm_escalation);
217 * This is a simple trigger for a generic XIVE IRQ. This must
218 * only be called for interrupts that support a trigger page
220 static bool xive_irq_trigger(struct xive_irq_data *xd)
222 /* This should be only for MSIs */
223 if (WARN_ON(xd->flags & XIVE_IRQ_FLAG_LSI))
226 /* Those interrupts should always have a trigger page */
227 if (WARN_ON(!xd->trig_mmio))
230 out_be64(xd->trig_mmio, 0);
235 static irqreturn_t xive_esc_irq(int irq, void *data)
237 struct kvm_vcpu *vcpu = data;
239 vcpu->arch.irq_pending = 1;
241 if (vcpu->arch.ceded)
242 kvmppc_fast_vcpu_kick(vcpu);
244 /* Since we have the no-EOI flag, the interrupt is effectively
245 * disabled now. Clearing xive_esc_on means we won't bother
246 * doing so on the next entry.
248 * This also allows the entry code to know that if a PQ combination
249 * of 10 is observed while xive_esc_on is true, it means the queue
250 * contains an unprocessed escalation interrupt. We don't make use of
251 * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
253 vcpu->arch.xive_esc_on = false;
255 /* This orders xive_esc_on = false vs. subsequent stale_p = true */
256 smp_wmb(); /* goes with smp_mb() in cleanup_single_escalation */
261 int kvmppc_xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio,
262 bool single_escalation)
264 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
265 struct xive_q *q = &xc->queues[prio];
269 /* Already there ? */
270 if (xc->esc_virq[prio])
273 /* Hook up the escalation interrupt */
274 xc->esc_virq[prio] = irq_create_mapping(NULL, q->esc_irq);
275 if (!xc->esc_virq[prio]) {
276 pr_err("Failed to map escalation interrupt for queue %d of VCPU %d\n",
277 prio, xc->server_num);
281 if (single_escalation)
282 name = kasprintf(GFP_KERNEL, "kvm-%d-%d",
283 vcpu->kvm->arch.lpid, xc->server_num);
285 name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
286 vcpu->kvm->arch.lpid, xc->server_num, prio);
288 pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
289 prio, xc->server_num);
294 pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
296 rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
297 IRQF_NO_THREAD, name, vcpu);
299 pr_err("Failed to request escalation interrupt for queue %d of VCPU %d\n",
300 prio, xc->server_num);
303 xc->esc_virq_names[prio] = name;
305 /* In single escalation mode, we grab the ESB MMIO of the
306 * interrupt and mask it. Also populate the VCPU v/raddr
307 * of the ESB page for use by asm entry/exit code. Finally
308 * set the XIVE_IRQ_FLAG_NO_EOI flag which will prevent the
309 * core code from performing an EOI on the escalation
310 * interrupt, thus leaving it effectively masked after
313 if (single_escalation) {
314 struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
315 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
317 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
318 vcpu->arch.xive_esc_raddr = xd->eoi_page;
319 vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
320 xd->flags |= XIVE_IRQ_FLAG_NO_EOI;
325 irq_dispose_mapping(xc->esc_virq[prio]);
326 xc->esc_virq[prio] = 0;
331 static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
333 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
334 struct kvmppc_xive *xive = xc->xive;
335 struct xive_q *q = &xc->queues[prio];
339 if (WARN_ON(q->qpage))
342 /* Allocate the queue and retrieve infos on current node for now */
343 qpage = (__be32 *)__get_free_pages(GFP_KERNEL, xive->q_page_order);
345 pr_err("Failed to allocate queue %d for VCPU %d\n",
346 prio, xc->server_num);
349 memset(qpage, 0, 1 << xive->q_order);
352 * Reconfigure the queue. This will set q->qpage only once the
353 * queue is fully configured. This is a requirement for prio 0
354 * as we will stop doing EOIs for every IPI as soon as we observe
355 * qpage being non-NULL, and instead will only EOI when we receive
356 * corresponding queue 0 entries
358 rc = xive_native_configure_queue(xc->vp_id, q, prio, qpage,
359 xive->q_order, true);
361 pr_err("Failed to configure queue %d for VCPU %d\n",
362 prio, xc->server_num);
366 /* Called with xive->lock held */
367 static int xive_check_provisioning(struct kvm *kvm, u8 prio)
369 struct kvmppc_xive *xive = kvm->arch.xive;
370 struct kvm_vcpu *vcpu;
374 lockdep_assert_held(&xive->lock);
376 /* Already provisioned ? */
377 if (xive->qmap & (1 << prio))
380 pr_devel("Provisioning prio... %d\n", prio);
382 /* Provision each VCPU and enable escalations if needed */
383 kvm_for_each_vcpu(i, vcpu, kvm) {
384 if (!vcpu->arch.xive_vcpu)
386 rc = xive_provision_queue(vcpu, prio);
387 if (rc == 0 && !kvmppc_xive_has_single_escalation(xive))
388 kvmppc_xive_attach_escalation(vcpu, prio,
389 kvmppc_xive_has_single_escalation(xive));
394 /* Order previous stores and mark it as provisioned */
396 xive->qmap |= (1 << prio);
400 static void xive_inc_q_pending(struct kvm *kvm, u32 server, u8 prio)
402 struct kvm_vcpu *vcpu;
403 struct kvmppc_xive_vcpu *xc;
406 /* Locate target server */
407 vcpu = kvmppc_xive_find_server(kvm, server);
409 pr_warn("%s: Can't find server %d\n", __func__, server);
412 xc = vcpu->arch.xive_vcpu;
416 q = &xc->queues[prio];
417 atomic_inc(&q->pending_count);
420 static int xive_try_pick_queue(struct kvm_vcpu *vcpu, u8 prio)
422 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
431 q = &xc->queues[prio];
432 if (WARN_ON(!q->qpage))
435 /* Calculate max number of interrupts in that queue. */
436 max = (q->msk + 1) - XIVE_Q_GAP;
437 return atomic_add_unless(&q->count, 1, max) ? 0 : -EBUSY;
440 int kvmppc_xive_select_target(struct kvm *kvm, u32 *server, u8 prio)
442 struct kvm_vcpu *vcpu;
446 /* Locate target server */
447 vcpu = kvmppc_xive_find_server(kvm, *server);
449 pr_devel("Can't find server %d\n", *server);
453 pr_devel("Finding irq target on 0x%x/%d...\n", *server, prio);
456 rc = xive_try_pick_queue(vcpu, prio);
460 pr_devel(" .. failed, looking up candidate...\n");
462 /* Failed, pick another VCPU */
463 kvm_for_each_vcpu(i, vcpu, kvm) {
464 if (!vcpu->arch.xive_vcpu)
466 rc = xive_try_pick_queue(vcpu, prio);
468 *server = vcpu->arch.xive_vcpu->server_num;
469 pr_devel(" found on 0x%x/%d\n", *server, prio);
473 pr_devel(" no available target !\n");
475 /* No available target ! */
479 static u8 xive_lock_and_mask(struct kvmppc_xive *xive,
480 struct kvmppc_xive_src_block *sb,
481 struct kvmppc_xive_irq_state *state)
483 struct xive_irq_data *xd;
489 * Take the lock, set masked, try again if racing
493 arch_spin_lock(&sb->lock);
494 old_prio = state->guest_priority;
495 state->guest_priority = MASKED;
499 state->guest_priority = old_prio;
500 arch_spin_unlock(&sb->lock);
503 /* No change ? Bail */
504 if (old_prio == MASKED)
507 /* Get the right irq */
508 kvmppc_xive_select_irq(state, &hw_num, &xd);
510 /* Set PQ to 10, return old P and old Q and remember them */
511 val = xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_10);
512 state->old_p = !!(val & 2);
513 state->old_q = !!(val & 1);
516 * Synchronize hardware to sensure the queues are updated when
519 xive_native_sync_source(hw_num);
524 static void xive_lock_for_unmask(struct kvmppc_xive_src_block *sb,
525 struct kvmppc_xive_irq_state *state)
528 * Take the lock try again if racing with H_EOI
531 arch_spin_lock(&sb->lock);
534 arch_spin_unlock(&sb->lock);
538 static void xive_finish_unmask(struct kvmppc_xive *xive,
539 struct kvmppc_xive_src_block *sb,
540 struct kvmppc_xive_irq_state *state,
543 struct xive_irq_data *xd;
546 /* If we aren't changing a thing, move on */
547 if (state->guest_priority != MASKED)
550 /* Get the right irq */
551 kvmppc_xive_select_irq(state, &hw_num, &xd);
553 /* Old Q set, set PQ to 11 */
555 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_11);
558 * If not old P, then perform an "effective" EOI,
559 * on the source. This will handle the cases where
563 xive_vm_source_eoi(hw_num, xd);
565 /* Synchronize ordering and mark unmasked */
568 state->guest_priority = prio;
572 * Target an interrupt to a given server/prio, this will fallback
573 * to another server if necessary and perform the HW targetting
576 * NOTE: Must be called with the state lock held
578 static int xive_target_interrupt(struct kvm *kvm,
579 struct kvmppc_xive_irq_state *state,
582 struct kvmppc_xive *xive = kvm->arch.xive;
587 * This will return a tentative server and actual
588 * priority. The count for that new target will have
589 * already been incremented.
591 rc = kvmppc_xive_select_target(kvm, &server, prio);
594 * We failed to find a target ? Not much we can do
595 * at least until we support the GIQ.
601 * Increment the old queue pending count if there
602 * was one so that the old queue count gets adjusted later
603 * when observed to be empty.
605 if (state->act_priority != MASKED)
606 xive_inc_q_pending(kvm,
608 state->act_priority);
610 * Update state and HW
612 state->act_priority = prio;
613 state->act_server = server;
615 /* Get the right irq */
616 kvmppc_xive_select_irq(state, &hw_num, NULL);
618 return xive_native_configure_irq(hw_num,
619 kvmppc_xive_vp(xive, server),
620 prio, state->number);
624 * Targetting rules: In order to avoid losing track of
625 * pending interrupts accross mask and unmask, which would
626 * allow queue overflows, we implement the following rules:
628 * - Unless it was never enabled (or we run out of capacity)
629 * an interrupt is always targetted at a valid server/queue
630 * pair even when "masked" by the guest. This pair tends to
631 * be the last one used but it can be changed under some
632 * circumstances. That allows us to separate targetting
633 * from masking, we only handle accounting during (re)targetting,
634 * this also allows us to let an interrupt drain into its target
635 * queue after masking, avoiding complex schemes to remove
636 * interrupts out of remote processor queues.
638 * - When masking, we set PQ to 10 and save the previous value
641 * - When unmasking, if saved Q was set, we set PQ to 11
642 * otherwise we leave PQ to the HW state which will be either
643 * 10 if nothing happened or 11 if the interrupt fired while
644 * masked. Effectively we are OR'ing the previous Q into the
647 * Then if saved P is clear, we do an effective EOI (Q->P->Trigger)
648 * which will unmask the interrupt and shoot a new one if Q was
651 * Otherwise (saved P is set) we leave PQ unchanged (so 10 or 11,
652 * effectively meaning an H_EOI from the guest is still expected
653 * for that interrupt).
655 * - If H_EOI occurs while masked, we clear the saved P.
657 * - When changing target, we account on the new target and
658 * increment a separate "pending" counter on the old one.
659 * This pending counter will be used to decrement the old
660 * target's count when its queue has been observed empty.
663 int kvmppc_xive_set_xive(struct kvm *kvm, u32 irq, u32 server,
666 struct kvmppc_xive *xive = kvm->arch.xive;
667 struct kvmppc_xive_src_block *sb;
668 struct kvmppc_xive_irq_state *state;
676 pr_devel("set_xive ! irq 0x%x server 0x%x prio %d\n",
677 irq, server, priority);
679 /* First, check provisioning of queues */
680 if (priority != MASKED) {
681 mutex_lock(&xive->lock);
682 rc = xive_check_provisioning(xive->kvm,
683 xive_prio_from_guest(priority));
684 mutex_unlock(&xive->lock);
687 pr_devel(" provisioning failure %d !\n", rc);
691 sb = kvmppc_xive_find_source(xive, irq, &idx);
694 state = &sb->irq_state[idx];
697 * We first handle masking/unmasking since the locking
698 * might need to be retried due to EOIs, we'll handle
699 * targetting changes later. These functions will return
700 * with the SB lock held.
702 * xive_lock_and_mask() will also set state->guest_priority
703 * but won't otherwise change other fields of the state.
705 * xive_lock_for_unmask will not actually unmask, this will
706 * be done later by xive_finish_unmask() once the targetting
707 * has been done, so we don't try to unmask an interrupt
708 * that hasn't yet been targetted.
710 if (priority == MASKED)
711 xive_lock_and_mask(xive, sb, state);
713 xive_lock_for_unmask(sb, state);
717 * Then we handle targetting.
719 * First calculate a new "actual priority"
721 new_act_prio = state->act_priority;
722 if (priority != MASKED)
723 new_act_prio = xive_prio_from_guest(priority);
725 pr_devel(" new_act_prio=%x act_server=%x act_prio=%x\n",
726 new_act_prio, state->act_server, state->act_priority);
729 * Then check if we actually need to change anything,
731 * The condition for re-targetting the interrupt is that
732 * we have a valid new priority (new_act_prio is not 0xff)
733 * and either the server or the priority changed.
735 * Note: If act_priority was ff and the new priority is
736 * also ff, we don't do anything and leave the interrupt
737 * untargetted. An attempt of doing an int_on on an
738 * untargetted interrupt will fail. If that is a problem
739 * we could initialize interrupts with valid default
742 if (new_act_prio != MASKED &&
743 (state->act_server != server ||
744 state->act_priority != new_act_prio))
745 rc = xive_target_interrupt(kvm, state, server, new_act_prio);
748 * Perform the final unmasking of the interrupt source
751 if (priority != MASKED)
752 xive_finish_unmask(xive, sb, state, priority);
755 * Finally Update saved_priority to match. Only int_on/off
756 * set this field to a different value.
758 state->saved_priority = priority;
760 arch_spin_unlock(&sb->lock);
764 int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server,
767 struct kvmppc_xive *xive = kvm->arch.xive;
768 struct kvmppc_xive_src_block *sb;
769 struct kvmppc_xive_irq_state *state;
775 sb = kvmppc_xive_find_source(xive, irq, &idx);
778 state = &sb->irq_state[idx];
779 arch_spin_lock(&sb->lock);
780 *server = state->act_server;
781 *priority = state->guest_priority;
782 arch_spin_unlock(&sb->lock);
787 int kvmppc_xive_int_on(struct kvm *kvm, u32 irq)
789 struct kvmppc_xive *xive = kvm->arch.xive;
790 struct kvmppc_xive_src_block *sb;
791 struct kvmppc_xive_irq_state *state;
797 sb = kvmppc_xive_find_source(xive, irq, &idx);
800 state = &sb->irq_state[idx];
802 pr_devel("int_on(irq=0x%x)\n", irq);
805 * Check if interrupt was not targetted
807 if (state->act_priority == MASKED) {
808 pr_devel("int_on on untargetted interrupt\n");
812 /* If saved_priority is 0xff, do nothing */
813 if (state->saved_priority == MASKED)
817 * Lock and unmask it.
819 xive_lock_for_unmask(sb, state);
820 xive_finish_unmask(xive, sb, state, state->saved_priority);
821 arch_spin_unlock(&sb->lock);
826 int kvmppc_xive_int_off(struct kvm *kvm, u32 irq)
828 struct kvmppc_xive *xive = kvm->arch.xive;
829 struct kvmppc_xive_src_block *sb;
830 struct kvmppc_xive_irq_state *state;
836 sb = kvmppc_xive_find_source(xive, irq, &idx);
839 state = &sb->irq_state[idx];
841 pr_devel("int_off(irq=0x%x)\n", irq);
846 state->saved_priority = xive_lock_and_mask(xive, sb, state);
847 arch_spin_unlock(&sb->lock);
852 static bool xive_restore_pending_irq(struct kvmppc_xive *xive, u32 irq)
854 struct kvmppc_xive_src_block *sb;
855 struct kvmppc_xive_irq_state *state;
858 sb = kvmppc_xive_find_source(xive, irq, &idx);
861 state = &sb->irq_state[idx];
866 * Trigger the IPI. This assumes we never restore a pass-through
867 * interrupt which should be safe enough
869 xive_irq_trigger(&state->ipi_data);
874 u64 kvmppc_xive_get_icp(struct kvm_vcpu *vcpu)
876 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
881 /* Return the per-cpu state for state saving/migration */
882 return (u64)xc->cppr << KVM_REG_PPC_ICP_CPPR_SHIFT |
883 (u64)xc->mfrr << KVM_REG_PPC_ICP_MFRR_SHIFT |
884 (u64)0xff << KVM_REG_PPC_ICP_PPRI_SHIFT;
887 int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval)
889 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
890 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
897 /* Grab individual state fields. We don't use pending_pri */
898 cppr = icpval >> KVM_REG_PPC_ICP_CPPR_SHIFT;
899 xisr = (icpval >> KVM_REG_PPC_ICP_XISR_SHIFT) &
900 KVM_REG_PPC_ICP_XISR_MASK;
901 mfrr = icpval >> KVM_REG_PPC_ICP_MFRR_SHIFT;
903 pr_devel("set_icp vcpu %d cppr=0x%x mfrr=0x%x xisr=0x%x\n",
904 xc->server_num, cppr, mfrr, xisr);
907 * We can't update the state of a "pushed" VCPU, but that
908 * shouldn't happen because the vcpu->mutex makes running a
909 * vcpu mutually exclusive with doing one_reg get/set on it.
911 if (WARN_ON(vcpu->arch.xive_pushed))
914 /* Update VCPU HW saved state */
915 vcpu->arch.xive_saved_state.cppr = cppr;
916 xc->hw_cppr = xc->cppr = cppr;
919 * Update MFRR state. If it's not 0xff, we mark the VCPU as
920 * having a pending MFRR change, which will re-evaluate the
921 * target. The VCPU will thus potentially get a spurious
922 * interrupt but that's not a big deal.
926 xive_irq_trigger(&xc->vp_ipi_data);
929 * Now saved XIRR is "interesting". It means there's something in
930 * the legacy "1 element" queue... for an IPI we simply ignore it,
931 * as the MFRR restore will handle that. For anything else we need
932 * to force a resend of the source.
933 * However the source may not have been setup yet. If that's the
934 * case, we keep that info and increment a counter in the xive to
935 * tell subsequent xive_set_source() to go look.
937 if (xisr > XICS_IPI && !xive_restore_pending_irq(xive, xisr)) {
938 xc->delayed_irq = xisr;
939 xive->delayed_irqs++;
940 pr_devel(" xisr restore delayed\n");
946 int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq,
947 unsigned long host_irq)
949 struct kvmppc_xive *xive = kvm->arch.xive;
950 struct kvmppc_xive_src_block *sb;
951 struct kvmppc_xive_irq_state *state;
952 struct irq_data *host_data =
953 irq_domain_get_irq_data(irq_get_default_host(), host_irq);
954 unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data);
962 pr_debug("%s: GIRQ 0x%lx host IRQ %ld XIVE HW IRQ 0x%x\n",
963 __func__, guest_irq, host_irq, hw_irq);
965 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
968 state = &sb->irq_state[idx];
971 * Mark the passed-through interrupt as going to a VCPU,
972 * this will prevent further EOIs and similar operations
973 * from the XIVE code. It will also mask the interrupt
974 * to either PQ=10 or 11 state, the latter if the interrupt
975 * is pending. This will allow us to unmask or retrigger it
976 * after routing it to the guest with a simple EOI.
978 * The "state" argument is a "token", all it needs is to be
979 * non-NULL to switch to passed-through or NULL for the
980 * other way around. We may not yet have an actual VCPU
981 * target here and we don't really care.
983 rc = irq_set_vcpu_affinity(host_irq, state);
985 pr_err("Failed to set VCPU affinity for host IRQ %ld\n", host_irq);
990 * Mask and read state of IPI. We need to know if its P bit
991 * is set as that means it's potentially already using a
992 * queue entry in the target
994 prio = xive_lock_and_mask(xive, sb, state);
995 pr_devel(" old IPI prio %02x P:%d Q:%d\n", prio,
996 state->old_p, state->old_q);
998 /* Turn the IPI hard off */
999 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1002 * Reset ESB guest mapping. Needed when ESB pages are exposed
1003 * to the guest in XIVE native mode
1005 if (xive->ops && xive->ops->reset_mapped)
1006 xive->ops->reset_mapped(kvm, guest_irq);
1008 /* Grab info about irq */
1009 state->pt_number = hw_irq;
1010 state->pt_data = irq_data_get_irq_handler_data(host_data);
1013 * Configure the IRQ to match the existing configuration of
1014 * the IPI if it was already targetted. Otherwise this will
1015 * mask the interrupt in a lossy way (act_priority is 0xff)
1016 * which is fine for a never started interrupt.
1018 xive_native_configure_irq(hw_irq,
1019 kvmppc_xive_vp(xive, state->act_server),
1020 state->act_priority, state->number);
1023 * We do an EOI to enable the interrupt (and retrigger if needed)
1024 * if the guest has the interrupt unmasked and the P bit was *not*
1025 * set in the IPI. If it was set, we know a slot may still be in
1026 * use in the target queue thus we have to wait for a guest
1029 if (prio != MASKED && !state->old_p)
1030 xive_vm_source_eoi(hw_irq, state->pt_data);
1032 /* Clear old_p/old_q as they are no longer relevant */
1033 state->old_p = state->old_q = false;
1035 /* Restore guest prio (unlocks EOI) */
1037 state->guest_priority = prio;
1038 arch_spin_unlock(&sb->lock);
1042 EXPORT_SYMBOL_GPL(kvmppc_xive_set_mapped);
1044 int kvmppc_xive_clr_mapped(struct kvm *kvm, unsigned long guest_irq,
1045 unsigned long host_irq)
1047 struct kvmppc_xive *xive = kvm->arch.xive;
1048 struct kvmppc_xive_src_block *sb;
1049 struct kvmppc_xive_irq_state *state;
1057 pr_debug("%s: GIRQ 0x%lx host IRQ %ld\n", __func__, guest_irq, host_irq);
1059 sb = kvmppc_xive_find_source(xive, guest_irq, &idx);
1062 state = &sb->irq_state[idx];
1065 * Mask and read state of IRQ. We need to know if its P bit
1066 * is set as that means it's potentially already using a
1067 * queue entry in the target
1069 prio = xive_lock_and_mask(xive, sb, state);
1070 pr_devel(" old IRQ prio %02x P:%d Q:%d\n", prio,
1071 state->old_p, state->old_q);
1074 * If old_p is set, the interrupt is pending, we switch it to
1075 * PQ=11. This will force a resend in the host so the interrupt
1076 * isn't lost to whatver host driver may pick it up
1079 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_11);
1081 /* Release the passed-through interrupt to the host */
1082 rc = irq_set_vcpu_affinity(host_irq, NULL);
1084 pr_err("Failed to clr VCPU affinity for host IRQ %ld\n", host_irq);
1088 /* Forget about the IRQ */
1089 state->pt_number = 0;
1090 state->pt_data = NULL;
1093 * Reset ESB guest mapping. Needed when ESB pages are exposed
1094 * to the guest in XIVE native mode
1096 if (xive->ops && xive->ops->reset_mapped) {
1097 xive->ops->reset_mapped(kvm, guest_irq);
1100 /* Reconfigure the IPI */
1101 xive_native_configure_irq(state->ipi_number,
1102 kvmppc_xive_vp(xive, state->act_server),
1103 state->act_priority, state->number);
1106 * If old_p is set (we have a queue entry potentially
1107 * occupied) or the interrupt is masked, we set the IPI
1108 * to PQ=10 state. Otherwise we just re-enable it (PQ=00).
1110 if (prio == MASKED || state->old_p)
1111 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_10);
1113 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_00);
1115 /* Restore guest prio (unlocks EOI) */
1117 state->guest_priority = prio;
1118 arch_spin_unlock(&sb->lock);
1122 EXPORT_SYMBOL_GPL(kvmppc_xive_clr_mapped);
1124 void kvmppc_xive_disable_vcpu_interrupts(struct kvm_vcpu *vcpu)
1126 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1127 struct kvm *kvm = vcpu->kvm;
1128 struct kvmppc_xive *xive = kvm->arch.xive;
1131 for (i = 0; i <= xive->max_sbid; i++) {
1132 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1136 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++) {
1137 struct kvmppc_xive_irq_state *state = &sb->irq_state[j];
1141 if (state->act_priority == MASKED)
1143 if (state->act_server != xc->server_num)
1147 arch_spin_lock(&sb->lock);
1148 state->act_priority = MASKED;
1149 xive_vm_esb_load(&state->ipi_data, XIVE_ESB_SET_PQ_01);
1150 xive_native_configure_irq(state->ipi_number, 0, MASKED, 0);
1151 if (state->pt_number) {
1152 xive_vm_esb_load(state->pt_data, XIVE_ESB_SET_PQ_01);
1153 xive_native_configure_irq(state->pt_number, 0, MASKED, 0);
1155 arch_spin_unlock(&sb->lock);
1159 /* Disable vcpu's escalation interrupt */
1160 if (vcpu->arch.xive_esc_on) {
1161 __raw_readq((void __iomem *)(vcpu->arch.xive_esc_vaddr +
1162 XIVE_ESB_SET_PQ_01));
1163 vcpu->arch.xive_esc_on = false;
1167 * Clear pointers to escalation interrupt ESB.
1168 * This is safe because the vcpu->mutex is held, preventing
1169 * any other CPU from concurrently executing a KVM_RUN ioctl.
1171 vcpu->arch.xive_esc_vaddr = 0;
1172 vcpu->arch.xive_esc_raddr = 0;
1176 * In single escalation mode, the escalation interrupt is marked so
1177 * that EOI doesn't re-enable it, but just sets the stale_p flag to
1178 * indicate that the P bit has already been dealt with. However, the
1179 * assembly code that enters the guest sets PQ to 00 without clearing
1180 * stale_p (because it has no easy way to address it). Hence we have
1181 * to adjust stale_p before shutting down the interrupt.
1183 void xive_cleanup_single_escalation(struct kvm_vcpu *vcpu,
1184 struct kvmppc_xive_vcpu *xc, int irq)
1186 struct irq_data *d = irq_get_irq_data(irq);
1187 struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
1190 * This slightly odd sequence gives the right result
1191 * (i.e. stale_p set if xive_esc_on is false) even if
1192 * we race with xive_esc_irq() and xive_irq_eoi().
1194 xd->stale_p = false;
1195 smp_mb(); /* paired with smb_wmb in xive_esc_irq */
1196 if (!vcpu->arch.xive_esc_on)
1200 void kvmppc_xive_cleanup_vcpu(struct kvm_vcpu *vcpu)
1202 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1203 struct kvmppc_xive *xive = vcpu->kvm->arch.xive;
1206 if (!kvmppc_xics_enabled(vcpu))
1212 pr_devel("cleanup_vcpu(cpu=%d)\n", xc->server_num);
1214 /* Ensure no interrupt is still routed to that VP */
1216 kvmppc_xive_disable_vcpu_interrupts(vcpu);
1218 /* Mask the VP IPI */
1219 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_01);
1221 /* Free escalations */
1222 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1223 if (xc->esc_virq[i]) {
1224 if (kvmppc_xive_has_single_escalation(xc->xive))
1225 xive_cleanup_single_escalation(vcpu, xc,
1227 free_irq(xc->esc_virq[i], vcpu);
1228 irq_dispose_mapping(xc->esc_virq[i]);
1229 kfree(xc->esc_virq_names[i]);
1233 /* Disable the VP */
1234 xive_native_disable_vp(xc->vp_id);
1236 /* Clear the cam word so guest entry won't try to push context */
1237 vcpu->arch.xive_cam_word = 0;
1239 /* Free the queues */
1240 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1241 struct xive_q *q = &xc->queues[i];
1243 xive_native_disable_queue(xc->vp_id, q, i);
1245 free_pages((unsigned long)q->qpage,
1246 xive->q_page_order);
1253 xive_cleanup_irq_data(&xc->vp_ipi_data);
1254 xive_native_free_irq(xc->vp_ipi);
1259 /* Cleanup the vcpu */
1260 vcpu->arch.irq_type = KVMPPC_IRQ_DEFAULT;
1261 vcpu->arch.xive_vcpu = NULL;
1264 static bool kvmppc_xive_vcpu_id_valid(struct kvmppc_xive *xive, u32 cpu)
1266 /* We have a block of xive->nr_servers VPs. We just need to check
1267 * packed vCPU ids are below that.
1269 return kvmppc_pack_vcpu_id(xive->kvm, cpu) < xive->nr_servers;
1272 int kvmppc_xive_compute_vp_id(struct kvmppc_xive *xive, u32 cpu, u32 *vp)
1276 if (!kvmppc_xive_vcpu_id_valid(xive, cpu)) {
1277 pr_devel("Out of bounds !\n");
1281 if (xive->vp_base == XIVE_INVALID_VP) {
1282 xive->vp_base = xive_native_alloc_vp_block(xive->nr_servers);
1283 pr_devel("VP_Base=%x nr_servers=%d\n", xive->vp_base, xive->nr_servers);
1285 if (xive->vp_base == XIVE_INVALID_VP)
1289 vp_id = kvmppc_xive_vp(xive, cpu);
1290 if (kvmppc_xive_vp_in_use(xive->kvm, vp_id)) {
1291 pr_devel("Duplicate !\n");
1300 int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
1301 struct kvm_vcpu *vcpu, u32 cpu)
1303 struct kvmppc_xive *xive = dev->private;
1304 struct kvmppc_xive_vcpu *xc;
1308 pr_devel("connect_vcpu(cpu=%d)\n", cpu);
1310 if (dev->ops != &kvm_xive_ops) {
1311 pr_devel("Wrong ops !\n");
1314 if (xive->kvm != vcpu->kvm)
1316 if (vcpu->arch.irq_type != KVMPPC_IRQ_DEFAULT)
1319 /* We need to synchronize with queue provisioning */
1320 mutex_lock(&xive->lock);
1322 r = kvmppc_xive_compute_vp_id(xive, cpu, &vp_id);
1326 xc = kzalloc(sizeof(*xc), GFP_KERNEL);
1332 vcpu->arch.xive_vcpu = xc;
1335 xc->server_num = cpu;
1340 r = xive_native_get_vp_info(xc->vp_id, &xc->vp_cam, &xc->vp_chip_id);
1344 if (!kvmppc_xive_check_save_restore(vcpu)) {
1345 pr_err("inconsistent save-restore setup for VCPU %d\n", cpu);
1350 /* Configure VCPU fields for use by assembly push/pull */
1351 vcpu->arch.xive_saved_state.w01 = cpu_to_be64(0xff000000);
1352 vcpu->arch.xive_cam_word = cpu_to_be32(xc->vp_cam | TM_QW1W2_VO);
1355 xc->vp_ipi = xive_native_alloc_irq();
1357 pr_err("Failed to allocate xive irq for VCPU IPI\n");
1361 pr_devel(" IPI=0x%x\n", xc->vp_ipi);
1363 r = xive_native_populate_irq_data(xc->vp_ipi, &xc->vp_ipi_data);
1368 * Enable the VP first as the single escalation mode will
1369 * affect escalation interrupts numbering
1371 r = xive_native_enable_vp(xc->vp_id, kvmppc_xive_has_single_escalation(xive));
1373 pr_err("Failed to enable VP in OPAL, err %d\n", r);
1378 * Initialize queues. Initially we set them all for no queueing
1379 * and we enable escalation for queue 0 only which we'll use for
1380 * our mfrr change notifications. If the VCPU is hot-plugged, we
1381 * do handle provisioning however based on the existing "map"
1382 * of enabled queues.
1384 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
1385 struct xive_q *q = &xc->queues[i];
1387 /* Single escalation, no queue 7 */
1388 if (i == 7 && kvmppc_xive_has_single_escalation(xive))
1391 /* Is queue already enabled ? Provision it */
1392 if (xive->qmap & (1 << i)) {
1393 r = xive_provision_queue(vcpu, i);
1394 if (r == 0 && !kvmppc_xive_has_single_escalation(xive))
1395 kvmppc_xive_attach_escalation(
1396 vcpu, i, kvmppc_xive_has_single_escalation(xive));
1400 r = xive_native_configure_queue(xc->vp_id,
1401 q, i, NULL, 0, true);
1403 pr_err("Failed to configure queue %d for VCPU %d\n",
1410 /* If not done above, attach priority 0 escalation */
1411 r = kvmppc_xive_attach_escalation(vcpu, 0, kvmppc_xive_has_single_escalation(xive));
1416 r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
1418 xive_vm_esb_load(&xc->vp_ipi_data, XIVE_ESB_SET_PQ_00);
1421 mutex_unlock(&xive->lock);
1423 kvmppc_xive_cleanup_vcpu(vcpu);
1427 vcpu->arch.irq_type = KVMPPC_IRQ_XICS;
1432 * Scanning of queues before/after migration save
1434 static void xive_pre_save_set_queued(struct kvmppc_xive *xive, u32 irq)
1436 struct kvmppc_xive_src_block *sb;
1437 struct kvmppc_xive_irq_state *state;
1440 sb = kvmppc_xive_find_source(xive, irq, &idx);
1444 state = &sb->irq_state[idx];
1446 /* Some sanity checking */
1447 if (!state->valid) {
1448 pr_err("invalid irq 0x%x in cpu queue!\n", irq);
1453 * If the interrupt is in a queue it should have P set.
1454 * We warn so that gets reported. A backtrace isn't useful
1455 * so no need to use a WARN_ON.
1457 if (!state->saved_p)
1458 pr_err("Interrupt 0x%x is marked in a queue but P not set !\n", irq);
1461 state->in_queue = true;
1464 static void xive_pre_save_mask_irq(struct kvmppc_xive *xive,
1465 struct kvmppc_xive_src_block *sb,
1468 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
1473 /* Mask and save state, this will also sync HW queues */
1474 state->saved_scan_prio = xive_lock_and_mask(xive, sb, state);
1476 /* Transfer P and Q */
1477 state->saved_p = state->old_p;
1478 state->saved_q = state->old_q;
1481 arch_spin_unlock(&sb->lock);
1484 static void xive_pre_save_unmask_irq(struct kvmppc_xive *xive,
1485 struct kvmppc_xive_src_block *sb,
1488 struct kvmppc_xive_irq_state *state = &sb->irq_state[irq];
1494 * Lock / exclude EOI (not technically necessary if the
1495 * guest isn't running concurrently. If this becomes a
1496 * performance issue we can probably remove the lock.
1498 xive_lock_for_unmask(sb, state);
1500 /* Restore mask/prio if it wasn't masked */
1501 if (state->saved_scan_prio != MASKED)
1502 xive_finish_unmask(xive, sb, state, state->saved_scan_prio);
1505 arch_spin_unlock(&sb->lock);
1508 static void xive_pre_save_queue(struct kvmppc_xive *xive, struct xive_q *q)
1511 u32 toggle = q->toggle;
1515 irq = __xive_read_eq(q->qpage, q->msk, &idx, &toggle);
1517 xive_pre_save_set_queued(xive, irq);
1521 static void xive_pre_save_scan(struct kvmppc_xive *xive)
1523 struct kvm_vcpu *vcpu = NULL;
1528 * See comment in xive_get_source() about how this
1529 * work. Collect a stable state for all interrupts
1531 for (i = 0; i <= xive->max_sbid; i++) {
1532 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1535 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1536 xive_pre_save_mask_irq(xive, sb, j);
1539 /* Then scan the queues and update the "in_queue" flag */
1540 kvm_for_each_vcpu(i, vcpu, xive->kvm) {
1541 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1544 for (j = 0; j < KVMPPC_XIVE_Q_COUNT; j++) {
1545 if (xc->queues[j].qpage)
1546 xive_pre_save_queue(xive, &xc->queues[j]);
1550 /* Finally restore interrupt states */
1551 for (i = 0; i <= xive->max_sbid; i++) {
1552 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1555 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1556 xive_pre_save_unmask_irq(xive, sb, j);
1560 static void xive_post_save_scan(struct kvmppc_xive *xive)
1564 /* Clear all the in_queue flags */
1565 for (i = 0; i <= xive->max_sbid; i++) {
1566 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
1569 for (j = 0; j < KVMPPC_XICS_IRQ_PER_ICS; j++)
1570 sb->irq_state[j].in_queue = false;
1573 /* Next get_source() will do a new scan */
1574 xive->saved_src_count = 0;
1578 * This returns the source configuration and state to user space.
1580 static int xive_get_source(struct kvmppc_xive *xive, long irq, u64 addr)
1582 struct kvmppc_xive_src_block *sb;
1583 struct kvmppc_xive_irq_state *state;
1584 u64 __user *ubufp = (u64 __user *) addr;
1588 sb = kvmppc_xive_find_source(xive, irq, &idx);
1592 state = &sb->irq_state[idx];
1597 pr_devel("get_source(%ld)...\n", irq);
1600 * So to properly save the state into something that looks like a
1601 * XICS migration stream we cannot treat interrupts individually.
1603 * We need, instead, mask them all (& save their previous PQ state)
1604 * to get a stable state in the HW, then sync them to ensure that
1605 * any interrupt that had already fired hits its queue, and finally
1606 * scan all the queues to collect which interrupts are still present
1607 * in the queues, so we can set the "pending" flag on them and
1608 * they can be resent on restore.
1610 * So we do it all when the "first" interrupt gets saved, all the
1611 * state is collected at that point, the rest of xive_get_source()
1612 * will merely collect and convert that state to the expected
1613 * userspace bit mask.
1615 if (xive->saved_src_count == 0)
1616 xive_pre_save_scan(xive);
1617 xive->saved_src_count++;
1619 /* Convert saved state into something compatible with xics */
1620 val = state->act_server;
1621 prio = state->saved_scan_prio;
1623 if (prio == MASKED) {
1624 val |= KVM_XICS_MASKED;
1625 prio = state->saved_priority;
1627 val |= prio << KVM_XICS_PRIORITY_SHIFT;
1629 val |= KVM_XICS_LEVEL_SENSITIVE;
1631 val |= KVM_XICS_PENDING;
1634 val |= KVM_XICS_PRESENTED;
1637 val |= KVM_XICS_QUEUED;
1640 * We mark it pending (which will attempt a re-delivery)
1641 * if we are in a queue *or* we were masked and had
1642 * Q set which is equivalent to the XICS "masked pending"
1645 if (state->in_queue || (prio == MASKED && state->saved_q))
1646 val |= KVM_XICS_PENDING;
1650 * If that was the last interrupt saved, reset the
1653 if (xive->saved_src_count == xive->src_count)
1654 xive_post_save_scan(xive);
1656 /* Copy the result to userspace */
1657 if (put_user(val, ubufp))
1663 struct kvmppc_xive_src_block *kvmppc_xive_create_src_block(
1664 struct kvmppc_xive *xive, int irq)
1666 struct kvmppc_xive_src_block *sb;
1669 bid = irq >> KVMPPC_XICS_ICS_SHIFT;
1671 mutex_lock(&xive->lock);
1673 /* block already exists - somebody else got here first */
1674 if (xive->src_blocks[bid])
1677 /* Create the ICS */
1678 sb = kzalloc(sizeof(*sb), GFP_KERNEL);
1684 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
1685 sb->irq_state[i].number = (bid << KVMPPC_XICS_ICS_SHIFT) | i;
1686 sb->irq_state[i].eisn = 0;
1687 sb->irq_state[i].guest_priority = MASKED;
1688 sb->irq_state[i].saved_priority = MASKED;
1689 sb->irq_state[i].act_priority = MASKED;
1692 xive->src_blocks[bid] = sb;
1694 if (bid > xive->max_sbid)
1695 xive->max_sbid = bid;
1698 mutex_unlock(&xive->lock);
1699 return xive->src_blocks[bid];
1702 static bool xive_check_delayed_irq(struct kvmppc_xive *xive, u32 irq)
1704 struct kvm *kvm = xive->kvm;
1705 struct kvm_vcpu *vcpu = NULL;
1708 kvm_for_each_vcpu(i, vcpu, kvm) {
1709 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
1714 if (xc->delayed_irq == irq) {
1715 xc->delayed_irq = 0;
1716 xive->delayed_irqs--;
1723 static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
1725 struct kvmppc_xive_src_block *sb;
1726 struct kvmppc_xive_irq_state *state;
1727 u64 __user *ubufp = (u64 __user *) addr;
1730 u8 act_prio, guest_prio;
1734 if (irq < KVMPPC_XICS_FIRST_IRQ || irq >= KVMPPC_XICS_NR_IRQS)
1737 pr_devel("set_source(irq=0x%lx)\n", irq);
1739 /* Find the source */
1740 sb = kvmppc_xive_find_source(xive, irq, &idx);
1742 pr_devel("No source, creating source block...\n");
1743 sb = kvmppc_xive_create_src_block(xive, irq);
1745 pr_devel("Failed to create block...\n");
1749 state = &sb->irq_state[idx];
1751 /* Read user passed data */
1752 if (get_user(val, ubufp)) {
1753 pr_devel("fault getting user info !\n");
1757 server = val & KVM_XICS_DESTINATION_MASK;
1758 guest_prio = val >> KVM_XICS_PRIORITY_SHIFT;
1760 pr_devel(" val=0x016%llx (server=0x%x, guest_prio=%d)\n",
1761 val, server, guest_prio);
1764 * If the source doesn't already have an IPI, allocate
1765 * one and get the corresponding data
1767 if (!state->ipi_number) {
1768 state->ipi_number = xive_native_alloc_irq();
1769 if (state->ipi_number == 0) {
1770 pr_devel("Failed to allocate IPI !\n");
1773 xive_native_populate_irq_data(state->ipi_number, &state->ipi_data);
1774 pr_devel(" src_ipi=0x%x\n", state->ipi_number);
1778 * We use lock_and_mask() to set us in the right masked
1779 * state. We will override that state from the saved state
1780 * further down, but this will handle the cases of interrupts
1781 * that need FW masking. We set the initial guest_priority to
1782 * 0 before calling it to ensure it actually performs the masking.
1784 state->guest_priority = 0;
1785 xive_lock_and_mask(xive, sb, state);
1788 * Now, we select a target if we have one. If we don't we
1789 * leave the interrupt untargetted. It means that an interrupt
1790 * can become "untargetted" accross migration if it was masked
1791 * by set_xive() but there is little we can do about it.
1794 /* First convert prio and mark interrupt as untargetted */
1795 act_prio = xive_prio_from_guest(guest_prio);
1796 state->act_priority = MASKED;
1799 * We need to drop the lock due to the mutex below. Hopefully
1800 * nothing is touching that interrupt yet since it hasn't been
1801 * advertized to a running guest yet
1803 arch_spin_unlock(&sb->lock);
1805 /* If we have a priority target the interrupt */
1806 if (act_prio != MASKED) {
1807 /* First, check provisioning of queues */
1808 mutex_lock(&xive->lock);
1809 rc = xive_check_provisioning(xive->kvm, act_prio);
1810 mutex_unlock(&xive->lock);
1812 /* Target interrupt */
1814 rc = xive_target_interrupt(xive->kvm, state,
1817 * If provisioning or targetting failed, leave it
1818 * alone and masked. It will remain disabled until
1819 * the guest re-targets it.
1824 * Find out if this was a delayed irq stashed in an ICP,
1825 * in which case, treat it as pending
1827 if (xive->delayed_irqs && xive_check_delayed_irq(xive, irq)) {
1828 val |= KVM_XICS_PENDING;
1829 pr_devel(" Found delayed ! forcing PENDING !\n");
1832 /* Cleanup the SW state */
1833 state->old_p = false;
1834 state->old_q = false;
1836 state->asserted = false;
1838 /* Restore LSI state */
1839 if (val & KVM_XICS_LEVEL_SENSITIVE) {
1841 if (val & KVM_XICS_PENDING)
1842 state->asserted = true;
1843 pr_devel(" LSI ! Asserted=%d\n", state->asserted);
1847 * Restore P and Q. If the interrupt was pending, we
1848 * force Q and !P, which will trigger a resend.
1850 * That means that a guest that had both an interrupt
1851 * pending (queued) and Q set will restore with only
1852 * one instance of that interrupt instead of 2, but that
1853 * is perfectly fine as coalescing interrupts that haven't
1854 * been presented yet is always allowed.
1856 if (val & KVM_XICS_PRESENTED && !(val & KVM_XICS_PENDING))
1857 state->old_p = true;
1858 if (val & KVM_XICS_QUEUED || val & KVM_XICS_PENDING)
1859 state->old_q = true;
1861 pr_devel(" P=%d, Q=%d\n", state->old_p, state->old_q);
1864 * If the interrupt was unmasked, update guest priority and
1865 * perform the appropriate state transition and do a
1866 * re-trigger if necessary.
1868 if (val & KVM_XICS_MASKED) {
1869 pr_devel(" masked, saving prio\n");
1870 state->guest_priority = MASKED;
1871 state->saved_priority = guest_prio;
1873 pr_devel(" unmasked, restoring to prio %d\n", guest_prio);
1874 xive_finish_unmask(xive, sb, state, guest_prio);
1875 state->saved_priority = guest_prio;
1878 /* Increment the number of valid sources and mark this one valid */
1881 state->valid = true;
1886 int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level,
1889 struct kvmppc_xive *xive = kvm->arch.xive;
1890 struct kvmppc_xive_src_block *sb;
1891 struct kvmppc_xive_irq_state *state;
1897 sb = kvmppc_xive_find_source(xive, irq, &idx);
1901 /* Perform locklessly .... (we need to do some RCUisms here...) */
1902 state = &sb->irq_state[idx];
1906 /* We don't allow a trigger on a passed-through interrupt */
1907 if (state->pt_number)
1910 if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL)
1911 state->asserted = true;
1912 else if (level == 0 || level == KVM_INTERRUPT_UNSET) {
1913 state->asserted = false;
1917 /* Trigger the IPI */
1918 xive_irq_trigger(&state->ipi_data);
1923 int kvmppc_xive_set_nr_servers(struct kvmppc_xive *xive, u64 addr)
1925 u32 __user *ubufp = (u32 __user *) addr;
1929 if (get_user(nr_servers, ubufp))
1932 pr_devel("%s nr_servers=%u\n", __func__, nr_servers);
1934 if (!nr_servers || nr_servers > KVM_MAX_VCPU_IDS)
1937 mutex_lock(&xive->lock);
1938 if (xive->vp_base != XIVE_INVALID_VP)
1939 /* The VP block is allocated once and freed when the device
1940 * is released. Better not allow to change its size since its
1941 * used by connect_vcpu to validate vCPU ids are valid (eg,
1942 * setting it back to a higher value could allow connect_vcpu
1943 * to come up with a VP id that goes beyond the VP block, which
1944 * is likely to cause a crash in OPAL).
1947 else if (nr_servers > KVM_MAX_VCPUS)
1948 /* We don't need more servers. Higher vCPU ids get packed
1949 * down below KVM_MAX_VCPUS by kvmppc_pack_vcpu_id().
1951 xive->nr_servers = KVM_MAX_VCPUS;
1953 xive->nr_servers = nr_servers;
1955 mutex_unlock(&xive->lock);
1960 static int xive_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1962 struct kvmppc_xive *xive = dev->private;
1964 /* We honor the existing XICS ioctl */
1965 switch (attr->group) {
1966 case KVM_DEV_XICS_GRP_SOURCES:
1967 return xive_set_source(xive, attr->attr, attr->addr);
1968 case KVM_DEV_XICS_GRP_CTRL:
1969 switch (attr->attr) {
1970 case KVM_DEV_XICS_NR_SERVERS:
1971 return kvmppc_xive_set_nr_servers(xive, attr->addr);
1977 static int xive_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1979 struct kvmppc_xive *xive = dev->private;
1981 /* We honor the existing XICS ioctl */
1982 switch (attr->group) {
1983 case KVM_DEV_XICS_GRP_SOURCES:
1984 return xive_get_source(xive, attr->attr, attr->addr);
1989 static int xive_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
1991 /* We honor the same limits as XICS, at least for now */
1992 switch (attr->group) {
1993 case KVM_DEV_XICS_GRP_SOURCES:
1994 if (attr->attr >= KVMPPC_XICS_FIRST_IRQ &&
1995 attr->attr < KVMPPC_XICS_NR_IRQS)
1998 case KVM_DEV_XICS_GRP_CTRL:
1999 switch (attr->attr) {
2000 case KVM_DEV_XICS_NR_SERVERS:
2007 static void kvmppc_xive_cleanup_irq(u32 hw_num, struct xive_irq_data *xd)
2009 xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
2010 xive_native_configure_irq(hw_num, 0, MASKED, 0);
2013 void kvmppc_xive_free_sources(struct kvmppc_xive_src_block *sb)
2017 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2018 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2023 kvmppc_xive_cleanup_irq(state->ipi_number, &state->ipi_data);
2024 xive_cleanup_irq_data(&state->ipi_data);
2025 xive_native_free_irq(state->ipi_number);
2027 /* Pass-through, cleanup too but keep IRQ hw data */
2028 if (state->pt_number)
2029 kvmppc_xive_cleanup_irq(state->pt_number, state->pt_data);
2031 state->valid = false;
2036 * Called when device fd is closed. kvm->lock is held.
2038 static void kvmppc_xive_release(struct kvm_device *dev)
2040 struct kvmppc_xive *xive = dev->private;
2041 struct kvm *kvm = xive->kvm;
2042 struct kvm_vcpu *vcpu;
2045 pr_devel("Releasing xive device\n");
2048 * Since this is the device release function, we know that
2049 * userspace does not have any open fd referring to the
2050 * device. Therefore there can not be any of the device
2051 * attribute set/get functions being executed concurrently,
2052 * and similarly, the connect_vcpu and set/clr_mapped
2053 * functions also cannot be being executed.
2056 debugfs_remove(xive->dentry);
2059 * We should clean up the vCPU interrupt presenters first.
2061 kvm_for_each_vcpu(i, vcpu, kvm) {
2063 * Take vcpu->mutex to ensure that no one_reg get/set ioctl
2064 * (i.e. kvmppc_xive_[gs]et_icp) can be done concurrently.
2065 * Holding the vcpu->mutex also means that the vcpu cannot
2066 * be executing the KVM_RUN ioctl, and therefore it cannot
2067 * be executing the XIVE push or pull code or accessing
2068 * the XIVE MMIO regions.
2070 mutex_lock(&vcpu->mutex);
2071 kvmppc_xive_cleanup_vcpu(vcpu);
2072 mutex_unlock(&vcpu->mutex);
2076 * Now that we have cleared vcpu->arch.xive_vcpu, vcpu->arch.irq_type
2077 * and vcpu->arch.xive_esc_[vr]addr on each vcpu, we are safe
2078 * against xive code getting called during vcpu execution or
2079 * set/get one_reg operations.
2081 kvm->arch.xive = NULL;
2083 /* Mask and free interrupts */
2084 for (i = 0; i <= xive->max_sbid; i++) {
2085 if (xive->src_blocks[i])
2086 kvmppc_xive_free_sources(xive->src_blocks[i]);
2087 kfree(xive->src_blocks[i]);
2088 xive->src_blocks[i] = NULL;
2091 if (xive->vp_base != XIVE_INVALID_VP)
2092 xive_native_free_vp_block(xive->vp_base);
2095 * A reference of the kvmppc_xive pointer is now kept under
2096 * the xive_devices struct of the machine for reuse. It is
2097 * freed when the VM is destroyed for now until we fix all the
2105 * When the guest chooses the interrupt mode (XICS legacy or XIVE
2106 * native), the VM will switch of KVM device. The previous device will
2107 * be "released" before the new one is created.
2109 * Until we are sure all execution paths are well protected, provide a
2110 * fail safe (transitional) method for device destruction, in which
2111 * the XIVE device pointer is recycled and not directly freed.
2113 struct kvmppc_xive *kvmppc_xive_get_device(struct kvm *kvm, u32 type)
2115 struct kvmppc_xive **kvm_xive_device = type == KVM_DEV_TYPE_XIVE ?
2116 &kvm->arch.xive_devices.native :
2117 &kvm->arch.xive_devices.xics_on_xive;
2118 struct kvmppc_xive *xive = *kvm_xive_device;
2121 xive = kzalloc(sizeof(*xive), GFP_KERNEL);
2122 *kvm_xive_device = xive;
2124 memset(xive, 0, sizeof(*xive));
2131 * Create a XICS device with XIVE backend. kvm->lock is held.
2133 static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
2135 struct kvmppc_xive *xive;
2136 struct kvm *kvm = dev->kvm;
2138 pr_devel("Creating xive for partition\n");
2140 /* Already there ? */
2144 xive = kvmppc_xive_get_device(kvm, type);
2148 dev->private = xive;
2151 mutex_init(&xive->lock);
2153 /* We use the default queue size set by the host */
2154 xive->q_order = xive_native_default_eq_shift();
2155 if (xive->q_order < PAGE_SHIFT)
2156 xive->q_page_order = 0;
2158 xive->q_page_order = xive->q_order - PAGE_SHIFT;
2160 /* VP allocation is delayed to the first call to connect_vcpu */
2161 xive->vp_base = XIVE_INVALID_VP;
2162 /* KVM_MAX_VCPUS limits the number of VMs to roughly 64 per sockets
2163 * on a POWER9 system.
2165 xive->nr_servers = KVM_MAX_VCPUS;
2167 if (xive_native_has_single_escalation())
2168 xive->flags |= KVMPPC_XIVE_FLAG_SINGLE_ESCALATION;
2170 if (xive_native_has_save_restore())
2171 xive->flags |= KVMPPC_XIVE_FLAG_SAVE_RESTORE;
2173 kvm->arch.xive = xive;
2177 int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req)
2179 struct kvmppc_vcore *vc = vcpu->arch.vcore;
2181 /* The VM should have configured XICS mode before doing XICS hcalls. */
2182 if (!kvmppc_xics_enabled(vcpu))
2187 return xive_vm_h_xirr(vcpu);
2189 return xive_vm_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4));
2191 return xive_vm_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4));
2193 return xive_vm_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4),
2194 kvmppc_get_gpr(vcpu, 5));
2196 return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4));
2198 xive_vm_h_xirr(vcpu);
2199 kvmppc_set_gpr(vcpu, 5, get_tb() + vc->tb_offset);
2203 return H_UNSUPPORTED;
2205 EXPORT_SYMBOL_GPL(kvmppc_xive_xics_hcall);
2207 int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu)
2209 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2212 for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
2213 struct xive_q *q = &xc->queues[i];
2216 if (!q->qpage && !xc->esc_virq[i])
2220 seq_printf(m, " q[%d]: ", i);
2222 i0 = be32_to_cpup(q->qpage + idx);
2223 idx = (idx + 1) & q->msk;
2224 i1 = be32_to_cpup(q->qpage + idx);
2225 seq_printf(m, "T=%d %08x %08x...\n", q->toggle,
2228 if (xc->esc_virq[i]) {
2229 struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
2230 struct xive_irq_data *xd =
2231 irq_data_get_irq_handler_data(d);
2232 u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2234 seq_printf(m, " ESC %d %c%c EOI @%llx",
2236 (pq & XIVE_ESB_VAL_P) ? 'P' : '-',
2237 (pq & XIVE_ESB_VAL_Q) ? 'Q' : '-',
2245 void kvmppc_xive_debug_show_sources(struct seq_file *m,
2246 struct kvmppc_xive_src_block *sb)
2250 seq_puts(m, " LISN HW/CHIP TYPE PQ EISN CPU/PRIO\n");
2251 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) {
2252 struct kvmppc_xive_irq_state *state = &sb->irq_state[i];
2253 struct xive_irq_data *xd;
2260 kvmppc_xive_select_irq(state, &hw_num, &xd);
2262 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
2264 seq_printf(m, "%08x %08x/%02x", state->number, hw_num,
2267 seq_printf(m, " %cLSI", state->asserted ? '^' : ' ');
2269 seq_puts(m, " MSI");
2271 seq_printf(m, " %s %c%c %08x % 4d/%d",
2272 state->ipi_number == hw_num ? "IPI" : " PT",
2273 pq & XIVE_ESB_VAL_P ? 'P' : '-',
2274 pq & XIVE_ESB_VAL_Q ? 'Q' : '-',
2275 state->eisn, state->act_server,
2276 state->act_priority);
2282 static int xive_debug_show(struct seq_file *m, void *private)
2284 struct kvmppc_xive *xive = m->private;
2285 struct kvm *kvm = xive->kvm;
2286 struct kvm_vcpu *vcpu;
2287 u64 t_rm_h_xirr = 0;
2288 u64 t_rm_h_ipoll = 0;
2289 u64 t_rm_h_cppr = 0;
2292 u64 t_vm_h_xirr = 0;
2293 u64 t_vm_h_ipoll = 0;
2294 u64 t_vm_h_cppr = 0;
2302 seq_puts(m, "=========\nVCPU state\n=========\n");
2304 kvm_for_each_vcpu(i, vcpu, kvm) {
2305 struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
2310 seq_printf(m, "VCPU %d: VP:%#x/%02x\n"
2311 " CPPR:%#x HWCPPR:%#x MFRR:%#x PEND:%#x h_xirr: R=%lld V=%lld\n",
2312 xc->server_num, xc->vp_id, xc->vp_chip_id,
2313 xc->cppr, xc->hw_cppr,
2314 xc->mfrr, xc->pending,
2315 xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
2317 kvmppc_xive_debug_show_queues(m, vcpu);
2319 t_rm_h_xirr += xc->stat_rm_h_xirr;
2320 t_rm_h_ipoll += xc->stat_rm_h_ipoll;
2321 t_rm_h_cppr += xc->stat_rm_h_cppr;
2322 t_rm_h_eoi += xc->stat_rm_h_eoi;
2323 t_rm_h_ipi += xc->stat_rm_h_ipi;
2324 t_vm_h_xirr += xc->stat_vm_h_xirr;
2325 t_vm_h_ipoll += xc->stat_vm_h_ipoll;
2326 t_vm_h_cppr += xc->stat_vm_h_cppr;
2327 t_vm_h_eoi += xc->stat_vm_h_eoi;
2328 t_vm_h_ipi += xc->stat_vm_h_ipi;
2331 seq_puts(m, "Hcalls totals\n");
2332 seq_printf(m, " H_XIRR R=%10lld V=%10lld\n", t_rm_h_xirr, t_vm_h_xirr);
2333 seq_printf(m, " H_IPOLL R=%10lld V=%10lld\n", t_rm_h_ipoll, t_vm_h_ipoll);
2334 seq_printf(m, " H_CPPR R=%10lld V=%10lld\n", t_rm_h_cppr, t_vm_h_cppr);
2335 seq_printf(m, " H_EOI R=%10lld V=%10lld\n", t_rm_h_eoi, t_vm_h_eoi);
2336 seq_printf(m, " H_IPI R=%10lld V=%10lld\n", t_rm_h_ipi, t_vm_h_ipi);
2338 seq_puts(m, "=========\nSources\n=========\n");
2340 for (i = 0; i <= xive->max_sbid; i++) {
2341 struct kvmppc_xive_src_block *sb = xive->src_blocks[i];
2344 arch_spin_lock(&sb->lock);
2345 kvmppc_xive_debug_show_sources(m, sb);
2346 arch_spin_unlock(&sb->lock);
2353 DEFINE_SHOW_ATTRIBUTE(xive_debug);
2355 static void xive_debugfs_init(struct kvmppc_xive *xive)
2357 xive->dentry = debugfs_create_file("xive", S_IRUGO, xive->kvm->debugfs_dentry,
2358 xive, &xive_debug_fops);
2360 pr_debug("%s: created\n", __func__);
2363 static void kvmppc_xive_init(struct kvm_device *dev)
2365 struct kvmppc_xive *xive = dev->private;
2367 /* Register some debug interfaces */
2368 xive_debugfs_init(xive);
2371 struct kvm_device_ops kvm_xive_ops = {
2373 .create = kvmppc_xive_create,
2374 .init = kvmppc_xive_init,
2375 .release = kvmppc_xive_release,
2376 .set_attr = xive_set_attr,
2377 .get_attr = xive_get_attr,
2378 .has_attr = xive_has_attr,