arch/x86/xen/xen-asm_64.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Asm versions of Xen pv-ops, suitable for direct use.
   4  *
   5  * We only bother with direct forms (ie, vcpu in pda) of the
   6  * operations here; the indirect forms are better handled in C.
   7  */
   8
   9 #include <asm/errno.h>
  10 #include <asm/percpu.h>
  11 #include <asm/processor-flags.h>
  12 #include <asm/segment.h>
  13 #include <asm/asm-offsets.h>
  14 #include <asm/thread_info.h>
  15 #include <asm/asm.h>
  16
  17 #include <xen/interface/xen.h>
  18
  19 #include <linux/init.h>
  20 #include <linux/linkage.h>
  21
  22 .macro xen_pv_trap name
  23 SYM_CODE_START(xen_\name)
  24         pop %rcx
  25         pop %r11
  26         jmp  \name
  27 SYM_CODE_END(xen_\name)
  28 _ASM_NOKPROBE(xen_\name)
  29 .endm
  30
  31 xen_pv_trap asm_exc_divide_error
  32 xen_pv_trap asm_exc_debug
  33 xen_pv_trap asm_exc_xendebug
  34 xen_pv_trap asm_exc_int3
  35 xen_pv_trap asm_exc_xennmi
  36 xen_pv_trap asm_exc_overflow
  37 xen_pv_trap asm_exc_bounds
  38 xen_pv_trap asm_exc_invalid_op
  39 xen_pv_trap asm_exc_device_not_available
  40 xen_pv_trap asm_exc_double_fault
  41 xen_pv_trap asm_exc_coproc_segment_overrun
  42 xen_pv_trap asm_exc_invalid_tss
  43 xen_pv_trap asm_exc_segment_not_present
  44 xen_pv_trap asm_exc_stack_segment
  45 xen_pv_trap asm_exc_general_protection
  46 xen_pv_trap asm_exc_page_fault
  47 xen_pv_trap asm_exc_spurious_interrupt_bug
  48 xen_pv_trap asm_exc_coprocessor_error
  49 xen_pv_trap asm_exc_alignment_check
  50 #ifdef CONFIG_X86_MCE
  51 xen_pv_trap asm_exc_machine_check
  52 #endif /* CONFIG_X86_MCE */
  53 xen_pv_trap asm_exc_simd_coprocessor_error
  54 #ifdef CONFIG_IA32_EMULATION
  55 xen_pv_trap entry_INT80_compat
  56 #endif
  57 xen_pv_trap asm_exc_xen_hypervisor_callback
  58
  59         __INIT
  60 SYM_CODE_START(xen_early_idt_handler_array)
  61         i = 0
  62         .rept NUM_EXCEPTION_VECTORS
  63         pop %rcx
  64         pop %r11
  65         jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
  66         i = i + 1
  67         .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
  68         .endr
  69 SYM_CODE_END(xen_early_idt_handler_array)
  70         __FINIT
  71
  72 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
  73 /*
  74  * Xen64 iret frame:
  75  *
  76  *      ss
  77  *      rsp
  78  *      rflags
  79  *      cs
  80  *      rip             <-- standard iret frame
  81  *
  82  *      flags
  83  *
  84  *      rcx             }
  85  *      r11             }<-- pushed by hypercall page
  86  * rsp->rax             }
  87  */
  88 SYM_CODE_START(xen_iret)
  89         pushq $0
  90         jmp hypercall_iret
  91 SYM_CODE_END(xen_iret)
  92
  93 SYM_CODE_START(xen_sysret64)
  94         /*
  95          * We're already on the usermode stack at this point, but
  96          * still with the kernel gs, so we can easily switch back.
  97          *
  98          * tss.sp2 is scratch space.
  99          */
 100         movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
 101         movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 102
 103         pushq $__USER_DS
 104         pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
 105         pushq %r11
 106         pushq $__USER_CS
 107         pushq %rcx
 108
 109         pushq $VGCF_in_syscall
 110         jmp hypercall_iret
 111 SYM_CODE_END(xen_sysret64)
 112
 113 /*
 114  * Xen handles syscall callbacks much like ordinary exceptions, which
 115  * means we have:
 116  * - kernel gs
 117  * - kernel rsp
 118  * - an iret-like stack frame on the stack (including rcx and r11):
 119  *      ss
 120  *      rsp
 121  *      rflags
 122  *      cs
 123  *      rip
 124  *      r11
 125  * rsp->rcx
 126  */
 127
 128 /* Normal 64-bit system call target */
 129 SYM_FUNC_START(xen_syscall_target)
 130         popq %rcx
 131         popq %r11
 132
 133         /*
 134          * Neither Xen nor the kernel really knows what the old SS and
 135          * CS were.  The kernel expects __USER_DS and __USER_CS, so
 136          * report those values even though Xen will guess its own values.
 137          */
 138         movq $__USER_DS, 4*8(%rsp)
 139         movq $__USER_CS, 1*8(%rsp)
 140
 141         jmp entry_SYSCALL_64_after_hwframe
 142 SYM_FUNC_END(xen_syscall_target)
 143
 144 #ifdef CONFIG_IA32_EMULATION
 145
 146 /* 32-bit compat syscall target */
 147 SYM_FUNC_START(xen_syscall32_target)
 148         popq %rcx
 149         popq %r11
 150
 151         /*
 152          * Neither Xen nor the kernel really knows what the old SS and
 153          * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
 154          * report those values even though Xen will guess its own values.
 155          */
 156         movq $__USER32_DS, 4*8(%rsp)
 157         movq $__USER32_CS, 1*8(%rsp)
 158
 159         jmp entry_SYSCALL_compat_after_hwframe
 160 SYM_FUNC_END(xen_syscall32_target)
 161
 162 /* 32-bit compat sysenter target */
 163 SYM_FUNC_START(xen_sysenter_target)
 164         mov 0*8(%rsp), %rcx
 165         mov 1*8(%rsp), %r11
 166         mov 5*8(%rsp), %rsp
 167         jmp entry_SYSENTER_compat
 168 SYM_FUNC_END(xen_sysenter_target)
 169
 170 #else /* !CONFIG_IA32_EMULATION */
 171
 172 SYM_FUNC_START_ALIAS(xen_syscall32_target)
 173 SYM_FUNC_START(xen_sysenter_target)
 174         lea 16(%rsp), %rsp      /* strip %rcx, %r11 */
 175         mov $-ENOSYS, %rax
 176         pushq $0
 177         jmp hypercall_iret
 178 SYM_FUNC_END(xen_sysenter_target)
 179 SYM_FUNC_END_ALIAS(xen_syscall32_target)
 180
 181 #endif  /* CONFIG_IA32_EMULATION */