arch/s390/kvm/kvm-s390.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * hosting IBM Z kernel virtual machines (s390x)
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  *               Christian Borntraeger <borntraeger@de.ibm.com>
   9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  11  *               Jason J. Herne <jjherne@us.ibm.com>
  12  */
  13
  14 #define KMSG_COMPONENT "kvm-s390"
  15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/random.h>
  28 #include <linux/slab.h>
  29 #include <linux/timer.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/bitmap.h>
  32 #include <linux/sched/signal.h>
  33 #include <linux/string.h>
  34
  35 #include <asm/asm-offsets.h>
  36 #include <asm/lowcore.h>
  37 #include <asm/stp.h>
  38 #include <asm/pgtable.h>
  39 #include <asm/gmap.h>
  40 #include <asm/nmi.h>
  41 #include <asm/switch_to.h>
  42 #include <asm/isc.h>
  43 #include <asm/sclp.h>
  44 #include <asm/cpacf.h>
  45 #include <asm/timex.h>
  46 #include <asm/ap.h>
  47 #include <asm/uv.h>
  48 #include "kvm-s390.h"
  49 #include "gaccess.h"
  50
  51 #define CREATE_TRACE_POINTS
  52 #include "trace.h"
  53 #include "trace-s390.h"
  54
  55 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  56 #define LOCAL_IRQS 32
  57 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  58                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  59
  60 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  61 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  62
  63 struct kvm_stats_debugfs_item debugfs_entries[] = {
  64         { "userspace_handled", VCPU_STAT(exit_userspace) },
  65         { "exit_null", VCPU_STAT(exit_null) },
  66         { "exit_validity", VCPU_STAT(exit_validity) },
  67         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  68         { "exit_external_request", VCPU_STAT(exit_external_request) },
  69         { "exit_io_request", VCPU_STAT(exit_io_request) },
  70         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  71         { "exit_instruction", VCPU_STAT(exit_instruction) },
  72         { "exit_pei", VCPU_STAT(exit_pei) },
  73         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  74         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  75         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  76         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  77         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  78         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  79         { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
  80         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  81         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  82         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  83         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  84         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  85         { "deliver_ckc", VCPU_STAT(deliver_ckc) },
  86         { "deliver_cputm", VCPU_STAT(deliver_cputm) },
  87         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  88         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  89         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  90         { "deliver_virtio", VCPU_STAT(deliver_virtio) },
  91         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  92         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  93         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  94         { "deliver_program", VCPU_STAT(deliver_program) },
  95         { "deliver_io", VCPU_STAT(deliver_io) },
  96         { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
  97         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  98         { "inject_ckc", VCPU_STAT(inject_ckc) },
  99         { "inject_cputm", VCPU_STAT(inject_cputm) },
 100         { "inject_external_call", VCPU_STAT(inject_external_call) },
 101         { "inject_float_mchk", VM_STAT(inject_float_mchk) },
 102         { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
 103         { "inject_io", VM_STAT(inject_io) },
 104         { "inject_mchk", VCPU_STAT(inject_mchk) },
 105         { "inject_pfault_done", VM_STAT(inject_pfault_done) },
 106         { "inject_program", VCPU_STAT(inject_program) },
 107         { "inject_restart", VCPU_STAT(inject_restart) },
 108         { "inject_service_signal", VM_STAT(inject_service_signal) },
 109         { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
 110         { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
 111         { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
 112         { "inject_virtio", VM_STAT(inject_virtio) },
 113         { "instruction_epsw", VCPU_STAT(instruction_epsw) },
 114         { "instruction_gs", VCPU_STAT(instruction_gs) },
 115         { "instruction_io_other", VCPU_STAT(instruction_io_other) },
 116         { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
 117         { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
 118         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
 119         { "instruction_ptff", VCPU_STAT(instruction_ptff) },
 120         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
 121         { "instruction_sck", VCPU_STAT(instruction_sck) },
 122         { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
 123         { "instruction_spx", VCPU_STAT(instruction_spx) },
 124         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
 125         { "instruction_stap", VCPU_STAT(instruction_stap) },
 126         { "instruction_iske", VCPU_STAT(instruction_iske) },
 127         { "instruction_ri", VCPU_STAT(instruction_ri) },
 128         { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
 129         { "instruction_sske", VCPU_STAT(instruction_sske) },
 130         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
 131         { "instruction_essa", VCPU_STAT(instruction_essa) },
 132         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 133         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 134         { "instruction_tb", VCPU_STAT(instruction_tb) },
 135         { "instruction_tpi", VCPU_STAT(instruction_tpi) },
 136         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 137         { "instruction_tsch", VCPU_STAT(instruction_tsch) },
 138         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 139         { "instruction_sie", VCPU_STAT(instruction_sie) },
 140         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 141         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 142         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 143         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 144         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 145         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 146         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 147         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 148         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 149         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 150         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 151         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 152         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 153         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 154         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 155         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 156         { "instruction_diag_10", VCPU_STAT(diagnose_10) },
 157         { "instruction_diag_44", VCPU_STAT(diagnose_44) },
 158         { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
 159         { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) },
 160         { "instruction_diag_258", VCPU_STAT(diagnose_258) },
 161         { "instruction_diag_308", VCPU_STAT(diagnose_308) },
 162         { "instruction_diag_500", VCPU_STAT(diagnose_500) },
 163         { "instruction_diag_other", VCPU_STAT(diagnose_other) },
 164         { NULL }
 165 };
 166
 167 struct kvm_s390_tod_clock_ext {
 168         __u8 epoch_idx;
 169         __u64 tod;
 170         __u8 reserved[7];
 171 } __packed;
 172
 173 /* allow nested virtualization in KVM (if enabled by user space) */
 174 static int nested;
 175 module_param(nested, int, S_IRUGO);
 176 MODULE_PARM_DESC(nested, "Nested virtualization support");
 177
 178 /* allow 1m huge page guest backing, if !nested */
 179 static int hpage;
 180 module_param(hpage, int, 0444);
 181 MODULE_PARM_DESC(hpage, "1m huge page backing support");
 182
 183 /* maximum percentage of steal time for polling.  >100 is treated like 100 */
 184 static u8 halt_poll_max_steal = 10;
 185 module_param(halt_poll_max_steal, byte, 0644);
 186 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
 187
 188 /* if set to true, the GISA will be initialized and used if available */
 189 static bool use_gisa  = true;
 190 module_param(use_gisa, bool, 0644);
 191 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
 192
 193 /*
 194  * For now we handle at most 16 double words as this is what the s390 base
 195  * kernel handles and stores in the prefix page. If we ever need to go beyond
 196  * this, this requires changes to code, but the external uapi can stay.
 197  */
 198 #define SIZE_INTERNAL 16
 199
 200 /*
 201  * Base feature mask that defines default mask for facilities. Consists of the
 202  * defines in FACILITIES_KVM and the non-hypervisor managed bits.
 203  */
 204 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
 205 /*
 206  * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
 207  * and defines the facilities that can be enabled via a cpu model.
 208  */
 209 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
 210
 211 static unsigned long kvm_s390_fac_size(void)
 212 {
 213         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
 214         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
 215         BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
 216                 sizeof(S390_lowcore.stfle_fac_list));
 217
 218         return SIZE_INTERNAL;
 219 }
 220
 221 /* available cpu features supported by kvm */
 222 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 223 /* available subfunctions indicated via query / "test bit" */
 224 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 225
 226 static struct gmap_notifier gmap_notifier;
 227 static struct gmap_notifier vsie_gmap_notifier;
 228 debug_info_t *kvm_s390_dbf;
 229 debug_info_t *kvm_s390_dbf_uv;
 230
 231 /* Section: not file related */
 232 int kvm_arch_hardware_enable(void)
 233 {
 234         /* every s390 is virtualization enabled ;-) */
 235         return 0;
 236 }
 237
 238 int kvm_arch_check_processor_compat(void *opaque)
 239 {
 240         return 0;
 241 }
 242
 243 /* forward declarations */
 244 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 245                               unsigned long end);
 246 static int sca_switch_to_extended(struct kvm *kvm);
 247
 248 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
 249 {
 250         u8 delta_idx = 0;
 251
 252         /*
 253          * The TOD jumps by delta, we have to compensate this by adding
 254          * -delta to the epoch.
 255          */
 256         delta = -delta;
 257
 258         /* sign-extension - we're adding to signed values below */
 259         if ((s64)delta < 0)
 260                 delta_idx = -1;
 261
 262         scb->epoch += delta;
 263         if (scb->ecd & ECD_MEF) {
 264                 scb->epdx += delta_idx;
 265                 if (scb->epoch < delta)
 266                         scb->epdx += 1;
 267         }
 268 }
 269
 270 /*
 271  * This callback is executed during stop_machine(). All CPUs are therefore
 272  * temporarily stopped. In order not to change guest behavior, we have to
 273  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 274  * so a CPU won't be stopped while calculating with the epoch.
 275  */
 276 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 277                           void *v)
 278 {
 279         struct kvm *kvm;
 280         struct kvm_vcpu *vcpu;
 281         int i;
 282         unsigned long long *delta = v;
 283
 284         list_for_each_entry(kvm, &vm_list, vm_list) {
 285                 kvm_for_each_vcpu(i, vcpu, kvm) {
 286                         kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
 287                         if (i == 0) {
 288                                 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
 289                                 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
 290                         }
 291                         if (vcpu->arch.cputm_enabled)
 292                                 vcpu->arch.cputm_start += *delta;
 293                         if (vcpu->arch.vsie_block)
 294                                 kvm_clock_sync_scb(vcpu->arch.vsie_block,
 295                                                    *delta);
 296                 }
 297         }
 298         return NOTIFY_OK;
 299 }
 300
 301 static struct notifier_block kvm_clock_notifier = {
 302         .notifier_call = kvm_clock_sync,
 303 };
 304
 305 int kvm_arch_hardware_setup(void *opaque)
 306 {
 307         gmap_notifier.notifier_call = kvm_gmap_notifier;
 308         gmap_register_pte_notifier(&gmap_notifier);
 309         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 310         gmap_register_pte_notifier(&vsie_gmap_notifier);
 311         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 312                                        &kvm_clock_notifier);
 313         return 0;
 314 }
 315
 316 void kvm_arch_hardware_unsetup(void)
 317 {
 318         gmap_unregister_pte_notifier(&gmap_notifier);
 319         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 320         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 321                                          &kvm_clock_notifier);
 322 }
 323
 324 static void allow_cpu_feat(unsigned long nr)
 325 {
 326         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 327 }
 328
 329 static inline int plo_test_bit(unsigned char nr)
 330 {
 331         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 332         int cc;
 333
 334         asm volatile(
 335                 /* Parameter registers are ignored for "test bit" */
 336                 "       plo     0,0,0,0(0)\n"
 337                 "       ipm     %0\n"
 338                 "       srl     %0,28\n"
 339                 : "=d" (cc)
 340                 : "d" (r0)
 341                 : "cc");
 342         return cc == 0;
 343 }
 344
 345 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
 346 {
 347         register unsigned long r0 asm("0") = 0; /* query function */
 348         register unsigned long r1 asm("1") = (unsigned long) query;
 349
 350         asm volatile(
 351                 /* Parameter regs are ignored */
 352                 "       .insn   rrf,%[opc] << 16,2,4,6,0\n"
 353                 :
 354                 : "d" (r0), "a" (r1), [opc] "i" (opcode)
 355                 : "cc", "memory");
 356 }
 357
 358 #define INSN_SORTL 0xb938
 359 #define INSN_DFLTCC 0xb939
 360
 361 static void kvm_s390_cpu_feat_init(void)
 362 {
 363         int i;
 364
 365         for (i = 0; i < 256; ++i) {
 366                 if (plo_test_bit(i))
 367                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 368         }
 369
 370         if (test_facility(28)) /* TOD-clock steering */
 371                 ptff(kvm_s390_available_subfunc.ptff,
 372                      sizeof(kvm_s390_available_subfunc.ptff),
 373                      PTFF_QAF);
 374
 375         if (test_facility(17)) { /* MSA */
 376                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 377                               kvm_s390_available_subfunc.kmac);
 378                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 379                               kvm_s390_available_subfunc.kmc);
 380                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 381                               kvm_s390_available_subfunc.km);
 382                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 383                               kvm_s390_available_subfunc.kimd);
 384                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 385                               kvm_s390_available_subfunc.klmd);
 386         }
 387         if (test_facility(76)) /* MSA3 */
 388                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 389                               kvm_s390_available_subfunc.pckmo);
 390         if (test_facility(77)) { /* MSA4 */
 391                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 392                               kvm_s390_available_subfunc.kmctr);
 393                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 394                               kvm_s390_available_subfunc.kmf);
 395                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 396                               kvm_s390_available_subfunc.kmo);
 397                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 398                               kvm_s390_available_subfunc.pcc);
 399         }
 400         if (test_facility(57)) /* MSA5 */
 401                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
 402                               kvm_s390_available_subfunc.ppno);
 403
 404         if (test_facility(146)) /* MSA8 */
 405                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
 406                               kvm_s390_available_subfunc.kma);
 407
 408         if (test_facility(155)) /* MSA9 */
 409                 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
 410                               kvm_s390_available_subfunc.kdsa);
 411
 412         if (test_facility(150)) /* SORTL */
 413                 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
 414
 415         if (test_facility(151)) /* DFLTCC */
 416                 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
 417
 418         if (MACHINE_HAS_ESOP)
 419                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 420         /*
 421          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 422          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 423          */
 424         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 425             !test_facility(3) || !nested)
 426                 return;
 427         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 428         if (sclp.has_64bscao)
 429                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 430         if (sclp.has_siif)
 431                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 432         if (sclp.has_gpere)
 433                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 434         if (sclp.has_gsls)
 435                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 436         if (sclp.has_ib)
 437                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 438         if (sclp.has_cei)
 439                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 440         if (sclp.has_ibs)
 441                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 442         if (sclp.has_kss)
 443                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
 444         /*
 445          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 446          * all skey handling functions read/set the skey from the PGSTE
 447          * instead of the real storage key.
 448          *
 449          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 450          * pages being detected as preserved although they are resident.
 451          *
 452          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 453          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 454          *
 455          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 456          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 457          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 458          *
 459          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 460          * cannot easily shadow the SCA because of the ipte lock.
 461          */
 462 }
 463
 464 int kvm_arch_init(void *opaque)
 465 {
 466         int rc = -ENOMEM;
 467
 468         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 469         if (!kvm_s390_dbf)
 470                 return -ENOMEM;
 471
 472         kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
 473         if (!kvm_s390_dbf_uv)
 474                 goto out;
 475
 476         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
 477             debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
 478                 goto out;
 479
 480         kvm_s390_cpu_feat_init();
 481
 482         /* Register floating interrupt controller interface. */
 483         rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 484         if (rc) {
 485                 pr_err("A FLIC registration call failed with rc=%d\n", rc);
 486                 goto out;
 487         }
 488
 489         rc = kvm_s390_gib_init(GAL_ISC);
 490         if (rc)
 491                 goto out;
 492
 493         return 0;
 494
 495 out:
 496         kvm_arch_exit();
 497         return rc;
 498 }
 499
 500 void kvm_arch_exit(void)
 501 {
 502         kvm_s390_gib_destroy();
 503         debug_unregister(kvm_s390_dbf);
 504         debug_unregister(kvm_s390_dbf_uv);
 505 }
 506
 507 /* Section: device related */
 508 long kvm_arch_dev_ioctl(struct file *filp,
 509                         unsigned int ioctl, unsigned long arg)
 510 {
 511         if (ioctl == KVM_S390_ENABLE_SIE)
 512                 return s390_enable_sie();
 513         return -EINVAL;
 514 }
 515
 516 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 517 {
 518         int r;
 519
 520         switch (ext) {
 521         case KVM_CAP_S390_PSW:
 522         case KVM_CAP_S390_GMAP:
 523         case KVM_CAP_SYNC_MMU:
 524 #ifdef CONFIG_KVM_S390_UCONTROL
 525         case KVM_CAP_S390_UCONTROL:
 526 #endif
 527         case KVM_CAP_ASYNC_PF:
 528         case KVM_CAP_SYNC_REGS:
 529         case KVM_CAP_ONE_REG:
 530         case KVM_CAP_ENABLE_CAP:
 531         case KVM_CAP_S390_CSS_SUPPORT:
 532         case KVM_CAP_IOEVENTFD:
 533         case KVM_CAP_DEVICE_CTRL:
 534         case KVM_CAP_S390_IRQCHIP:
 535         case KVM_CAP_VM_ATTRIBUTES:
 536         case KVM_CAP_MP_STATE:
 537         case KVM_CAP_IMMEDIATE_EXIT:
 538         case KVM_CAP_S390_INJECT_IRQ:
 539         case KVM_CAP_S390_USER_SIGP:
 540         case KVM_CAP_S390_USER_STSI:
 541         case KVM_CAP_S390_SKEYS:
 542         case KVM_CAP_S390_IRQ_STATE:
 543         case KVM_CAP_S390_USER_INSTR0:
 544         case KVM_CAP_S390_CMMA_MIGRATION:
 545         case KVM_CAP_S390_AIS:
 546         case KVM_CAP_S390_AIS_MIGRATION:
 547         case KVM_CAP_S390_VCPU_RESETS:
 548                 r = 1;
 549                 break;
 550         case KVM_CAP_S390_HPAGE_1M:
 551                 r = 0;
 552                 if (hpage && !kvm_is_ucontrol(kvm))
 553                         r = 1;
 554                 break;
 555         case KVM_CAP_S390_MEM_OP:
 556                 r = MEM_OP_MAX_SIZE;
 557                 break;
 558         case KVM_CAP_NR_VCPUS:
 559         case KVM_CAP_MAX_VCPUS:
 560         case KVM_CAP_MAX_VCPU_ID:
 561                 r = KVM_S390_BSCA_CPU_SLOTS;
 562                 if (!kvm_s390_use_sca_entries())
 563                         r = KVM_MAX_VCPUS;
 564                 else if (sclp.has_esca && sclp.has_64bscao)
 565                         r = KVM_S390_ESCA_CPU_SLOTS;
 566                 break;
 567         case KVM_CAP_S390_COW:
 568                 r = MACHINE_HAS_ESOP;
 569                 break;
 570         case KVM_CAP_S390_VECTOR_REGISTERS:
 571                 r = MACHINE_HAS_VX;
 572                 break;
 573         case KVM_CAP_S390_RI:
 574                 r = test_facility(64);
 575                 break;
 576         case KVM_CAP_S390_GS:
 577                 r = test_facility(133);
 578                 break;
 579         case KVM_CAP_S390_BPB:
 580                 r = test_facility(82);
 581                 break;
 582         case KVM_CAP_S390_PROTECTED:
 583                 r = is_prot_virt_host();
 584                 break;
 585         default:
 586                 r = 0;
 587         }
 588         return r;
 589 }
 590
 591 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
 592 {
 593         int i;
 594         gfn_t cur_gfn, last_gfn;
 595         unsigned long gaddr, vmaddr;
 596         struct gmap *gmap = kvm->arch.gmap;
 597         DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
 598
 599         /* Loop over all guest segments */
 600         cur_gfn = memslot->base_gfn;
 601         last_gfn = memslot->base_gfn + memslot->npages;
 602         for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
 603                 gaddr = gfn_to_gpa(cur_gfn);
 604                 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
 605                 if (kvm_is_error_hva(vmaddr))
 606                         continue;
 607
 608                 bitmap_zero(bitmap, _PAGE_ENTRIES);
 609                 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
 610                 for (i = 0; i < _PAGE_ENTRIES; i++) {
 611                         if (test_bit(i, bitmap))
 612                                 mark_page_dirty(kvm, cur_gfn + i);
 613                 }
 614
 615                 if (fatal_signal_pending(current))
 616                         return;
 617                 cond_resched();
 618         }
 619 }
 620
 621 /* Section: vm related */
 622 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 623
 624 /*
 625  * Get (and clear) the dirty memory log for a memory slot.
 626  */
 627 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 628                                struct kvm_dirty_log *log)
 629 {
 630         int r;
 631         unsigned long n;
 632         struct kvm_memory_slot *memslot;
 633         int is_dirty;
 634
 635         if (kvm_is_ucontrol(kvm))
 636                 return -EINVAL;
 637
 638         mutex_lock(&kvm->slots_lock);
 639
 640         r = -EINVAL;
 641         if (log->slot >= KVM_USER_MEM_SLOTS)
 642                 goto out;
 643
 644         r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
 645         if (r)
 646                 goto out;
 647
 648         /* Clear the dirty log */
 649         if (is_dirty) {
 650                 n = kvm_dirty_bitmap_bytes(memslot);
 651                 memset(memslot->dirty_bitmap, 0, n);
 652         }
 653         r = 0;
 654 out:
 655         mutex_unlock(&kvm->slots_lock);
 656         return r;
 657 }
 658
 659 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 660 {
 661         unsigned int i;
 662         struct kvm_vcpu *vcpu;
 663
 664         kvm_for_each_vcpu(i, vcpu, kvm) {
 665                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 666         }
 667 }
 668
 669 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 670 {
 671         int r;
 672
 673         if (cap->flags)
 674                 return -EINVAL;
 675
 676         switch (cap->cap) {
 677         case KVM_CAP_S390_IRQCHIP:
 678                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 679                 kvm->arch.use_irqchip = 1;
 680                 r = 0;
 681                 break;
 682         case KVM_CAP_S390_USER_SIGP:
 683                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 684                 kvm->arch.user_sigp = 1;
 685                 r = 0;
 686                 break;
 687         case KVM_CAP_S390_VECTOR_REGISTERS:
 688                 mutex_lock(&kvm->lock);
 689                 if (kvm->created_vcpus) {
 690                         r = -EBUSY;
 691                 } else if (MACHINE_HAS_VX) {
 692                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 693                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 694                         if (test_facility(134)) {
 695                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 696                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 697                         }
 698                         if (test_facility(135)) {
 699                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 700                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 701                         }
 702                         if (test_facility(148)) {
 703                                 set_kvm_facility(kvm->arch.model.fac_mask, 148);
 704                                 set_kvm_facility(kvm->arch.model.fac_list, 148);
 705                         }
 706                         if (test_facility(152)) {
 707                                 set_kvm_facility(kvm->arch.model.fac_mask, 152);
 708                                 set_kvm_facility(kvm->arch.model.fac_list, 152);
 709                         }
 710                         r = 0;
 711                 } else
 712                         r = -EINVAL;
 713                 mutex_unlock(&kvm->lock);
 714                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 715                          r ? "(not available)" : "(success)");
 716                 break;
 717         case KVM_CAP_S390_RI:
 718                 r = -EINVAL;
 719                 mutex_lock(&kvm->lock);
 720                 if (kvm->created_vcpus) {
 721                         r = -EBUSY;
 722                 } else if (test_facility(64)) {
 723                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 724                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 725                         r = 0;
 726                 }
 727                 mutex_unlock(&kvm->lock);
 728                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 729                          r ? "(not available)" : "(success)");
 730                 break;
 731         case KVM_CAP_S390_AIS:
 732                 mutex_lock(&kvm->lock);
 733                 if (kvm->created_vcpus) {
 734                         r = -EBUSY;
 735                 } else {
 736                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
 737                         set_kvm_facility(kvm->arch.model.fac_list, 72);
 738                         r = 0;
 739                 }
 740                 mutex_unlock(&kvm->lock);
 741                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
 742                          r ? "(not available)" : "(success)");
 743                 break;
 744         case KVM_CAP_S390_GS:
 745                 r = -EINVAL;
 746                 mutex_lock(&kvm->lock);
 747                 if (kvm->created_vcpus) {
 748                         r = -EBUSY;
 749                 } else if (test_facility(133)) {
 750                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
 751                         set_kvm_facility(kvm->arch.model.fac_list, 133);
 752                         r = 0;
 753                 }
 754                 mutex_unlock(&kvm->lock);
 755                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
 756                          r ? "(not available)" : "(success)");
 757                 break;
 758         case KVM_CAP_S390_HPAGE_1M:
 759                 mutex_lock(&kvm->lock);
 760                 if (kvm->created_vcpus)
 761                         r = -EBUSY;
 762                 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
 763                         r = -EINVAL;
 764                 else {
 765                         r = 0;
 766                         down_write(&kvm->mm->mmap_sem);
 767                         kvm->mm->context.allow_gmap_hpage_1m = 1;
 768                         up_write(&kvm->mm->mmap_sem);
 769                         /*
 770                          * We might have to create fake 4k page
 771                          * tables. To avoid that the hardware works on
 772                          * stale PGSTEs, we emulate these instructions.
 773                          */
 774                         kvm->arch.use_skf = 0;
 775                         kvm->arch.use_pfmfi = 0;
 776                 }
 777                 mutex_unlock(&kvm->lock);
 778                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
 779                          r ? "(not available)" : "(success)");
 780                 break;
 781         case KVM_CAP_S390_USER_STSI:
 782                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 783                 kvm->arch.user_stsi = 1;
 784                 r = 0;
 785                 break;
 786         case KVM_CAP_S390_USER_INSTR0:
 787                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 788                 kvm->arch.user_instr0 = 1;
 789                 icpt_operexc_on_all_vcpus(kvm);
 790                 r = 0;
 791                 break;
 792         default:
 793                 r = -EINVAL;
 794                 break;
 795         }
 796         return r;
 797 }
 798
 799 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 800 {
 801         int ret;
 802
 803         switch (attr->attr) {
 804         case KVM_S390_VM_MEM_LIMIT_SIZE:
 805                 ret = 0;
 806                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 807                          kvm->arch.mem_limit);
 808                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 809                         ret = -EFAULT;
 810                 break;
 811         default:
 812                 ret = -ENXIO;
 813                 break;
 814         }
 815         return ret;
 816 }
 817
 818 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 819 {
 820         int ret;
 821         unsigned int idx;
 822         switch (attr->attr) {
 823         case KVM_S390_VM_MEM_ENABLE_CMMA:
 824                 ret = -ENXIO;
 825                 if (!sclp.has_cmma)
 826                         break;
 827
 828                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 829                 mutex_lock(&kvm->lock);
 830                 if (kvm->created_vcpus)
 831                         ret = -EBUSY;
 832                 else if (kvm->mm->context.allow_gmap_hpage_1m)
 833                         ret = -EINVAL;
 834                 else {
 835                         kvm->arch.use_cmma = 1;
 836                         /* Not compatible with cmma. */
 837                         kvm->arch.use_pfmfi = 0;
 838                         ret = 0;
 839                 }
 840                 mutex_unlock(&kvm->lock);
 841                 break;
 842         case KVM_S390_VM_MEM_CLR_CMMA:
 843                 ret = -ENXIO;
 844                 if (!sclp.has_cmma)
 845                         break;
 846                 ret = -EINVAL;
 847                 if (!kvm->arch.use_cmma)
 848                         break;
 849
 850                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 851                 mutex_lock(&kvm->lock);
 852                 idx = srcu_read_lock(&kvm->srcu);
 853                 s390_reset_cmma(kvm->arch.gmap->mm);
 854                 srcu_read_unlock(&kvm->srcu, idx);
 855                 mutex_unlock(&kvm->lock);
 856                 ret = 0;
 857                 break;
 858         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 859                 unsigned long new_limit;
 860
 861                 if (kvm_is_ucontrol(kvm))
 862                         return -EINVAL;
 863
 864                 if (get_user(new_limit, (u64 __user *)attr->addr))
 865                         return -EFAULT;
 866
 867                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 868                     new_limit > kvm->arch.mem_limit)
 869                         return -E2BIG;
 870
 871                 if (!new_limit)
 872                         return -EINVAL;
 873
 874                 /* gmap_create takes last usable address */
 875                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 876                         new_limit -= 1;
 877
 878                 ret = -EBUSY;
 879                 mutex_lock(&kvm->lock);
 880                 if (!kvm->created_vcpus) {
 881                         /* gmap_create will round the limit up */
 882                         struct gmap *new = gmap_create(current->mm, new_limit);
 883
 884                         if (!new) {
 885                                 ret = -ENOMEM;
 886                         } else {
 887                                 gmap_remove(kvm->arch.gmap);
 888                                 new->private = kvm;
 889                                 kvm->arch.gmap = new;
 890                                 ret = 0;
 891                         }
 892                 }
 893                 mutex_unlock(&kvm->lock);
 894                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 895                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 896                          (void *) kvm->arch.gmap->asce);
 897                 break;
 898         }
 899         default:
 900                 ret = -ENXIO;
 901                 break;
 902         }
 903         return ret;
 904 }
 905
 906 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 907
 908 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
 909 {
 910         struct kvm_vcpu *vcpu;
 911         int i;
 912
 913         kvm_s390_vcpu_block_all(kvm);
 914
 915         kvm_for_each_vcpu(i, vcpu, kvm) {
 916                 kvm_s390_vcpu_crypto_setup(vcpu);
 917                 /* recreate the shadow crycb by leaving the VSIE handler */
 918                 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
 919         }
 920
 921         kvm_s390_vcpu_unblock_all(kvm);
 922 }
 923
 924 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 925 {
 926         mutex_lock(&kvm->lock);
 927         switch (attr->attr) {
 928         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 929                 if (!test_kvm_facility(kvm, 76)) {
 930                         mutex_unlock(&kvm->lock);
 931                         return -EINVAL;
 932                 }
 933                 get_random_bytes(
 934                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 935                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 936                 kvm->arch.crypto.aes_kw = 1;
 937                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 938                 break;
 939         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 940                 if (!test_kvm_facility(kvm, 76)) {
 941                         mutex_unlock(&kvm->lock);
 942                         return -EINVAL;
 943                 }
 944                 get_random_bytes(
 945                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 946                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 947                 kvm->arch.crypto.dea_kw = 1;
 948                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 949                 break;
 950         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 951                 if (!test_kvm_facility(kvm, 76)) {
 952                         mutex_unlock(&kvm->lock);
 953                         return -EINVAL;
 954                 }
 955                 kvm->arch.crypto.aes_kw = 0;
 956                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 957                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 958                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 959                 break;
 960         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 961                 if (!test_kvm_facility(kvm, 76)) {
 962                         mutex_unlock(&kvm->lock);
 963                         return -EINVAL;
 964                 }
 965                 kvm->arch.crypto.dea_kw = 0;
 966                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 967                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 968                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 969                 break;
 970         case KVM_S390_VM_CRYPTO_ENABLE_APIE:
 971                 if (!ap_instructions_available()) {
 972                         mutex_unlock(&kvm->lock);
 973                         return -EOPNOTSUPP;
 974                 }
 975                 kvm->arch.crypto.apie = 1;
 976                 break;
 977         case KVM_S390_VM_CRYPTO_DISABLE_APIE:
 978                 if (!ap_instructions_available()) {
 979                         mutex_unlock(&kvm->lock);
 980                         return -EOPNOTSUPP;
 981                 }
 982                 kvm->arch.crypto.apie = 0;
 983                 break;
 984         default:
 985                 mutex_unlock(&kvm->lock);
 986                 return -ENXIO;
 987         }
 988
 989         kvm_s390_vcpu_crypto_reset_all(kvm);
 990         mutex_unlock(&kvm->lock);
 991         return 0;
 992 }
 993
 994 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
 995 {
 996         int cx;
 997         struct kvm_vcpu *vcpu;
 998
 999         kvm_for_each_vcpu(cx, vcpu, kvm)
1000                 kvm_s390_sync_request(req, vcpu);
1001 }
1002
1003 /*
1004  * Must be called with kvm->srcu held to avoid races on memslots, and with
1005  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
1006  */
1007 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1008 {
1009         struct kvm_memory_slot *ms;
1010         struct kvm_memslots *slots;
1011         unsigned long ram_pages = 0;
1012         int slotnr;
1013
1014         /* migration mode already enabled */
1015         if (kvm->arch.migration_mode)
1016                 return 0;
1017         slots = kvm_memslots(kvm);
1018         if (!slots || !slots->used_slots)
1019                 return -EINVAL;
1020
1021         if (!kvm->arch.use_cmma) {
1022                 kvm->arch.migration_mode = 1;
1023                 return 0;
1024         }
1025         /* mark all the pages in active slots as dirty */
1026         for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1027                 ms = slots->memslots + slotnr;
1028                 if (!ms->dirty_bitmap)
1029                         return -EINVAL;
1030                 /*
1031                  * The second half of the bitmap is only used on x86,
1032                  * and would be wasted otherwise, so we put it to good
1033                  * use here to keep track of the state of the storage
1034                  * attributes.
1035                  */
1036                 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1037                 ram_pages += ms->npages;
1038         }
1039         atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1040         kvm->arch.migration_mode = 1;
1041         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1042         return 0;
1043 }
1044
1045 /*
1046  * Must be called with kvm->slots_lock to avoid races with ourselves and
1047  * kvm_s390_vm_start_migration.
1048  */
1049 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1050 {
1051         /* migration mode already disabled */
1052         if (!kvm->arch.migration_mode)
1053                 return 0;
1054         kvm->arch.migration_mode = 0;
1055         if (kvm->arch.use_cmma)
1056                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1057         return 0;
1058 }
1059
1060 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1061                                      struct kvm_device_attr *attr)
1062 {
1063         int res = -ENXIO;
1064
1065         mutex_lock(&kvm->slots_lock);
1066         switch (attr->attr) {
1067         case KVM_S390_VM_MIGRATION_START:
1068                 res = kvm_s390_vm_start_migration(kvm);
1069                 break;
1070         case KVM_S390_VM_MIGRATION_STOP:
1071                 res = kvm_s390_vm_stop_migration(kvm);
1072                 break;
1073         default:
1074                 break;
1075         }
1076         mutex_unlock(&kvm->slots_lock);
1077
1078         return res;
1079 }
1080
1081 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1082                                      struct kvm_device_attr *attr)
1083 {
1084         u64 mig = kvm->arch.migration_mode;
1085
1086         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1087                 return -ENXIO;
1088
1089         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1090                 return -EFAULT;
1091         return 0;
1092 }
1093
1094 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1095 {
1096         struct kvm_s390_vm_tod_clock gtod;
1097
1098         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1099                 return -EFAULT;
1100
1101         if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1102                 return -EINVAL;
1103         kvm_s390_set_tod_clock(kvm, &gtod);
1104
1105         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1106                 gtod.epoch_idx, gtod.tod);
1107
1108         return 0;
1109 }
1110
1111 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1112 {
1113         u8 gtod_high;
1114
1115         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1116                                            sizeof(gtod_high)))
1117                 return -EFAULT;
1118
1119         if (gtod_high != 0)
1120                 return -EINVAL;
1121         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1122
1123         return 0;
1124 }
1125
1126 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1127 {
1128         struct kvm_s390_vm_tod_clock gtod = { 0 };
1129
1130         if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1131                            sizeof(gtod.tod)))
1132                 return -EFAULT;
1133
1134         kvm_s390_set_tod_clock(kvm, &gtod);
1135         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1136         return 0;
1137 }
1138
1139 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1140 {
1141         int ret;
1142
1143         if (attr->flags)
1144                 return -EINVAL;
1145
1146         switch (attr->attr) {
1147         case KVM_S390_VM_TOD_EXT:
1148                 ret = kvm_s390_set_tod_ext(kvm, attr);
1149                 break;
1150         case KVM_S390_VM_TOD_HIGH:
1151                 ret = kvm_s390_set_tod_high(kvm, attr);
1152                 break;
1153         case KVM_S390_VM_TOD_LOW:
1154                 ret = kvm_s390_set_tod_low(kvm, attr);
1155                 break;
1156         default:
1157                 ret = -ENXIO;
1158                 break;
1159         }
1160         return ret;
1161 }
1162
1163 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1164                                    struct kvm_s390_vm_tod_clock *gtod)
1165 {
1166         struct kvm_s390_tod_clock_ext htod;
1167
1168         preempt_disable();
1169
1170         get_tod_clock_ext((char *)&htod);
1171
1172         gtod->tod = htod.tod + kvm->arch.epoch;
1173         gtod->epoch_idx = 0;
1174         if (test_kvm_facility(kvm, 139)) {
1175                 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1176                 if (gtod->tod < htod.tod)
1177                         gtod->epoch_idx += 1;
1178         }
1179
1180         preempt_enable();
1181 }
1182
1183 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1184 {
1185         struct kvm_s390_vm_tod_clock gtod;
1186
1187         memset(&gtod, 0, sizeof(gtod));
1188         kvm_s390_get_tod_clock(kvm, &gtod);
1189         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1190                 return -EFAULT;
1191
1192         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1193                 gtod.epoch_idx, gtod.tod);
1194         return 0;
1195 }
1196
1197 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1198 {
1199         u8 gtod_high = 0;
1200
1201         if (copy_to_user((void __user *)attr->addr, &gtod_high,
1202                                          sizeof(gtod_high)))
1203                 return -EFAULT;
1204         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1205
1206         return 0;
1207 }
1208
1209 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1210 {
1211         u64 gtod;
1212
1213         gtod = kvm_s390_get_tod_clock_fast(kvm);
1214         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1215                 return -EFAULT;
1216         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1217
1218         return 0;
1219 }
1220
1221 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1222 {
1223         int ret;
1224
1225         if (attr->flags)
1226                 return -EINVAL;
1227
1228         switch (attr->attr) {
1229         case KVM_S390_VM_TOD_EXT:
1230                 ret = kvm_s390_get_tod_ext(kvm, attr);
1231                 break;
1232         case KVM_S390_VM_TOD_HIGH:
1233                 ret = kvm_s390_get_tod_high(kvm, attr);
1234                 break;
1235         case KVM_S390_VM_TOD_LOW:
1236                 ret = kvm_s390_get_tod_low(kvm, attr);
1237                 break;
1238         default:
1239                 ret = -ENXIO;
1240                 break;
1241         }
1242         return ret;
1243 }
1244
1245 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1246 {
1247         struct kvm_s390_vm_cpu_processor *proc;
1248         u16 lowest_ibc, unblocked_ibc;
1249         int ret = 0;
1250
1251         mutex_lock(&kvm->lock);
1252         if (kvm->created_vcpus) {
1253                 ret = -EBUSY;
1254                 goto out;
1255         }
1256         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1257         if (!proc) {
1258                 ret = -ENOMEM;
1259                 goto out;
1260         }
1261         if (!copy_from_user(proc, (void __user *)attr->addr,
1262                             sizeof(*proc))) {
1263                 kvm->arch.model.cpuid = proc->cpuid;
1264                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1265                 unblocked_ibc = sclp.ibc & 0xfff;
1266                 if (lowest_ibc && proc->ibc) {
1267                         if (proc->ibc > unblocked_ibc)
1268                                 kvm->arch.model.ibc = unblocked_ibc;
1269                         else if (proc->ibc < lowest_ibc)
1270                                 kvm->arch.model.ibc = lowest_ibc;
1271                         else
1272                                 kvm->arch.model.ibc = proc->ibc;
1273                 }
1274                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1275                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1276                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1277                          kvm->arch.model.ibc,
1278                          kvm->arch.model.cpuid);
1279                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1280                          kvm->arch.model.fac_list[0],
1281                          kvm->arch.model.fac_list[1],
1282                          kvm->arch.model.fac_list[2]);
1283         } else
1284                 ret = -EFAULT;
1285         kfree(proc);
1286 out:
1287         mutex_unlock(&kvm->lock);
1288         return ret;
1289 }
1290
1291 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1292                                        struct kvm_device_attr *attr)
1293 {
1294         struct kvm_s390_vm_cpu_feat data;
1295
1296         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1297                 return -EFAULT;
1298         if (!bitmap_subset((unsigned long *) data.feat,
1299                            kvm_s390_available_cpu_feat,
1300                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1301                 return -EINVAL;
1302
1303         mutex_lock(&kvm->lock);
1304         if (kvm->created_vcpus) {
1305                 mutex_unlock(&kvm->lock);
1306                 return -EBUSY;
1307         }
1308         bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1309                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1310         mutex_unlock(&kvm->lock);
1311         VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1312                          data.feat[0],
1313                          data.feat[1],
1314                          data.feat[2]);
1315         return 0;
1316 }
1317
1318 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1319                                           struct kvm_device_attr *attr)
1320 {
1321         mutex_lock(&kvm->lock);
1322         if (kvm->created_vcpus) {
1323                 mutex_unlock(&kvm->lock);
1324                 return -EBUSY;
1325         }
1326
1327         if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1328                            sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1329                 mutex_unlock(&kvm->lock);
1330                 return -EFAULT;
1331         }
1332         mutex_unlock(&kvm->lock);
1333
1334         VM_EVENT(kvm, 3, "SET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1335                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1336                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1337                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1338                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1339         VM_EVENT(kvm, 3, "SET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1340                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1341                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1342         VM_EVENT(kvm, 3, "SET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1343                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1344                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1345         VM_EVENT(kvm, 3, "SET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1346                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1347                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1348         VM_EVENT(kvm, 3, "SET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1349                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1350                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1351         VM_EVENT(kvm, 3, "SET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1352                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1353                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1354         VM_EVENT(kvm, 3, "SET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1355                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1356                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1357         VM_EVENT(kvm, 3, "SET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1358                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1359                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1360         VM_EVENT(kvm, 3, "SET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1361                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1362                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1363         VM_EVENT(kvm, 3, "SET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1364                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1365                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1366         VM_EVENT(kvm, 3, "SET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1367                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1368                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1369         VM_EVENT(kvm, 3, "SET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1370                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1371                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1372         VM_EVENT(kvm, 3, "SET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1373                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1374                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1375         VM_EVENT(kvm, 3, "SET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1376                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1377                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1378         VM_EVENT(kvm, 3, "SET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1379                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1380                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1381         VM_EVENT(kvm, 3, "SET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1382                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1383                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1384                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1385                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1386         VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1387                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1388                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1389                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1390                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1391
1392         return 0;
1393 }
1394
1395 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1396 {
1397         int ret = -ENXIO;
1398
1399         switch (attr->attr) {
1400         case KVM_S390_VM_CPU_PROCESSOR:
1401                 ret = kvm_s390_set_processor(kvm, attr);
1402                 break;
1403         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1404                 ret = kvm_s390_set_processor_feat(kvm, attr);
1405                 break;
1406         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1407                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1408                 break;
1409         }
1410         return ret;
1411 }
1412
1413 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1414 {
1415         struct kvm_s390_vm_cpu_processor *proc;
1416         int ret = 0;
1417
1418         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1419         if (!proc) {
1420                 ret = -ENOMEM;
1421                 goto out;
1422         }
1423         proc->cpuid = kvm->arch.model.cpuid;
1424         proc->ibc = kvm->arch.model.ibc;
1425         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1426                S390_ARCH_FAC_LIST_SIZE_BYTE);
1427         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1428                  kvm->arch.model.ibc,
1429                  kvm->arch.model.cpuid);
1430         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1431                  kvm->arch.model.fac_list[0],
1432                  kvm->arch.model.fac_list[1],
1433                  kvm->arch.model.fac_list[2]);
1434         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1435                 ret = -EFAULT;
1436         kfree(proc);
1437 out:
1438         return ret;
1439 }
1440
1441 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1442 {
1443         struct kvm_s390_vm_cpu_machine *mach;
1444         int ret = 0;
1445
1446         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1447         if (!mach) {
1448                 ret = -ENOMEM;
1449                 goto out;
1450         }
1451         get_cpu_id((struct cpuid *) &mach->cpuid);
1452         mach->ibc = sclp.ibc;
1453         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1454                S390_ARCH_FAC_LIST_SIZE_BYTE);
1455         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1456                sizeof(S390_lowcore.stfle_fac_list));
1457         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1458                  kvm->arch.model.ibc,
1459                  kvm->arch.model.cpuid);
1460         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1461                  mach->fac_mask[0],
1462                  mach->fac_mask[1],
1463                  mach->fac_mask[2]);
1464         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1465                  mach->fac_list[0],
1466                  mach->fac_list[1],
1467                  mach->fac_list[2]);
1468         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1469                 ret = -EFAULT;
1470         kfree(mach);
1471 out:
1472         return ret;
1473 }
1474
1475 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1476                                        struct kvm_device_attr *attr)
1477 {
1478         struct kvm_s390_vm_cpu_feat data;
1479
1480         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1481                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1482         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1483                 return -EFAULT;
1484         VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1485                          data.feat[0],
1486                          data.feat[1],
1487                          data.feat[2]);
1488         return 0;
1489 }
1490
1491 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1492                                      struct kvm_device_attr *attr)
1493 {
1494         struct kvm_s390_vm_cpu_feat data;
1495
1496         bitmap_copy((unsigned long *) data.feat,
1497                     kvm_s390_available_cpu_feat,
1498                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1499         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1500                 return -EFAULT;
1501         VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
1502                          data.feat[0],
1503                          data.feat[1],
1504                          data.feat[2]);
1505         return 0;
1506 }
1507
1508 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1509                                           struct kvm_device_attr *attr)
1510 {
1511         if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1512             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1513                 return -EFAULT;
1514
1515         VM_EVENT(kvm, 3, "GET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1516                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1517                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1518                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1519                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1520         VM_EVENT(kvm, 3, "GET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1521                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1522                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1523         VM_EVENT(kvm, 3, "GET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1524                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1525                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1526         VM_EVENT(kvm, 3, "GET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1527                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1528                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1529         VM_EVENT(kvm, 3, "GET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1530                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1531                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1532         VM_EVENT(kvm, 3, "GET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1533                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1534                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1535         VM_EVENT(kvm, 3, "GET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1536                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1537                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1538         VM_EVENT(kvm, 3, "GET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1539                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1540                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1541         VM_EVENT(kvm, 3, "GET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1542                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1543                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1544         VM_EVENT(kvm, 3, "GET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1545                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1546                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1547         VM_EVENT(kvm, 3, "GET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1548                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1549                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1550         VM_EVENT(kvm, 3, "GET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1551                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1552                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1553         VM_EVENT(kvm, 3, "GET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1554                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1555                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1556         VM_EVENT(kvm, 3, "GET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1557                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1558                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1559         VM_EVENT(kvm, 3, "GET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1560                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1561                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1562         VM_EVENT(kvm, 3, "GET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1563                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1564                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1565                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1566                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1567         VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1568                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1569                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1570                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1571                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1572
1573         return 0;
1574 }
1575
1576 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1577                                         struct kvm_device_attr *attr)
1578 {
1579         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1580             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1581                 return -EFAULT;
1582
1583         VM_EVENT(kvm, 3, "GET: host  PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1584                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1585                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1586                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1587                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1588         VM_EVENT(kvm, 3, "GET: host  PTFF   subfunc 0x%16.16lx.%16.16lx",
1589                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1590                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1591         VM_EVENT(kvm, 3, "GET: host  KMAC   subfunc 0x%16.16lx.%16.16lx",
1592                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1593                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1594         VM_EVENT(kvm, 3, "GET: host  KMC    subfunc 0x%16.16lx.%16.16lx",
1595                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1596                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1597         VM_EVENT(kvm, 3, "GET: host  KM     subfunc 0x%16.16lx.%16.16lx",
1598                  ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1599                  ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1600         VM_EVENT(kvm, 3, "GET: host  KIMD   subfunc 0x%16.16lx.%16.16lx",
1601                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1602                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1603         VM_EVENT(kvm, 3, "GET: host  KLMD   subfunc 0x%16.16lx.%16.16lx",
1604                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1605                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1606         VM_EVENT(kvm, 3, "GET: host  PCKMO  subfunc 0x%16.16lx.%16.16lx",
1607                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1608                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1609         VM_EVENT(kvm, 3, "GET: host  KMCTR  subfunc 0x%16.16lx.%16.16lx",
1610                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1611                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1612         VM_EVENT(kvm, 3, "GET: host  KMF    subfunc 0x%16.16lx.%16.16lx",
1613                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1614                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1615         VM_EVENT(kvm, 3, "GET: host  KMO    subfunc 0x%16.16lx.%16.16lx",
1616                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1617                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1618         VM_EVENT(kvm, 3, "GET: host  PCC    subfunc 0x%16.16lx.%16.16lx",
1619                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1620                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1621         VM_EVENT(kvm, 3, "GET: host  PPNO   subfunc 0x%16.16lx.%16.16lx",
1622                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1623                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1624         VM_EVENT(kvm, 3, "GET: host  KMA    subfunc 0x%16.16lx.%16.16lx",
1625                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1626                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1627         VM_EVENT(kvm, 3, "GET: host  KDSA   subfunc 0x%16.16lx.%16.16lx",
1628                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1629                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1630         VM_EVENT(kvm, 3, "GET: host  SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1631                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1632                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1633                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1634                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1635         VM_EVENT(kvm, 3, "GET: host  DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1636                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1637                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1638                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1639                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1640
1641         return 0;
1642 }
1643
1644 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1645 {
1646         int ret = -ENXIO;
1647
1648         switch (attr->attr) {
1649         case KVM_S390_VM_CPU_PROCESSOR:
1650                 ret = kvm_s390_get_processor(kvm, attr);
1651                 break;
1652         case KVM_S390_VM_CPU_MACHINE:
1653                 ret = kvm_s390_get_machine(kvm, attr);
1654                 break;
1655         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1656                 ret = kvm_s390_get_processor_feat(kvm, attr);
1657                 break;
1658         case KVM_S390_VM_CPU_MACHINE_FEAT:
1659                 ret = kvm_s390_get_machine_feat(kvm, attr);
1660                 break;
1661         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1662                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1663                 break;
1664         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1665                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1666                 break;
1667         }
1668         return ret;
1669 }
1670
1671 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1672 {
1673         int ret;
1674
1675         switch (attr->group) {
1676         case KVM_S390_VM_MEM_CTRL:
1677                 ret = kvm_s390_set_mem_control(kvm, attr);
1678                 break;
1679         case KVM_S390_VM_TOD:
1680                 ret = kvm_s390_set_tod(kvm, attr);
1681                 break;
1682         case KVM_S390_VM_CPU_MODEL:
1683                 ret = kvm_s390_set_cpu_model(kvm, attr);
1684                 break;
1685         case KVM_S390_VM_CRYPTO:
1686                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1687                 break;
1688         case KVM_S390_VM_MIGRATION:
1689                 ret = kvm_s390_vm_set_migration(kvm, attr);
1690                 break;
1691         default:
1692                 ret = -ENXIO;
1693                 break;
1694         }
1695
1696         return ret;
1697 }
1698
1699 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1700 {
1701         int ret;
1702
1703         switch (attr->group) {
1704         case KVM_S390_VM_MEM_CTRL:
1705                 ret = kvm_s390_get_mem_control(kvm, attr);
1706                 break;
1707         case KVM_S390_VM_TOD:
1708                 ret = kvm_s390_get_tod(kvm, attr);
1709                 break;
1710         case KVM_S390_VM_CPU_MODEL:
1711                 ret = kvm_s390_get_cpu_model(kvm, attr);
1712                 break;
1713         case KVM_S390_VM_MIGRATION:
1714                 ret = kvm_s390_vm_get_migration(kvm, attr);
1715                 break;
1716         default:
1717                 ret = -ENXIO;
1718                 break;
1719         }
1720
1721         return ret;
1722 }
1723
1724 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1725 {
1726         int ret;
1727
1728         switch (attr->group) {
1729         case KVM_S390_VM_MEM_CTRL:
1730                 switch (attr->attr) {
1731                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1732                 case KVM_S390_VM_MEM_CLR_CMMA:
1733                         ret = sclp.has_cmma ? 0 : -ENXIO;
1734                         break;
1735                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1736                         ret = 0;
1737                         break;
1738                 default:
1739                         ret = -ENXIO;
1740                         break;
1741                 }
1742                 break;
1743         case KVM_S390_VM_TOD:
1744                 switch (attr->attr) {
1745                 case KVM_S390_VM_TOD_LOW:
1746                 case KVM_S390_VM_TOD_HIGH:
1747                         ret = 0;
1748                         break;
1749                 default:
1750                         ret = -ENXIO;
1751                         break;
1752                 }
1753                 break;
1754         case KVM_S390_VM_CPU_MODEL:
1755                 switch (attr->attr) {
1756                 case KVM_S390_VM_CPU_PROCESSOR:
1757                 case KVM_S390_VM_CPU_MACHINE:
1758                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1759                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1760                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1761                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1762                         ret = 0;
1763                         break;
1764                 default:
1765                         ret = -ENXIO;
1766                         break;
1767                 }
1768                 break;
1769         case KVM_S390_VM_CRYPTO:
1770                 switch (attr->attr) {
1771                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1772                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1773                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1774                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1775                         ret = 0;
1776                         break;
1777                 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1778                 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1779                         ret = ap_instructions_available() ? 0 : -ENXIO;
1780                         break;
1781                 default:
1782                         ret = -ENXIO;
1783                         break;
1784                 }
1785                 break;
1786         case KVM_S390_VM_MIGRATION:
1787                 ret = 0;
1788                 break;
1789         default:
1790                 ret = -ENXIO;
1791                 break;
1792         }
1793
1794         return ret;
1795 }
1796
1797 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1798 {
1799         uint8_t *keys;
1800         uint64_t hva;
1801         int srcu_idx, i, r = 0;
1802
1803         if (args->flags != 0)
1804                 return -EINVAL;
1805
1806         /* Is this guest using storage keys? */
1807         if (!mm_uses_skeys(current->mm))
1808                 return KVM_S390_GET_SKEYS_NONE;
1809
1810         /* Enforce sane limit on memory allocation */
1811         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1812                 return -EINVAL;
1813
1814         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1815         if (!keys)
1816                 return -ENOMEM;
1817
1818         down_read(&current->mm->mmap_sem);
1819         srcu_idx = srcu_read_lock(&kvm->srcu);
1820         for (i = 0; i < args->count; i++) {
1821                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1822                 if (kvm_is_error_hva(hva)) {
1823                         r = -EFAULT;
1824                         break;
1825                 }
1826
1827                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1828                 if (r)
1829                         break;
1830         }
1831         srcu_read_unlock(&kvm->srcu, srcu_idx);
1832         up_read(&current->mm->mmap_sem);
1833
1834         if (!r) {
1835                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1836                                  sizeof(uint8_t) * args->count);
1837                 if (r)
1838                         r = -EFAULT;
1839         }
1840
1841         kvfree(keys);
1842         return r;
1843 }
1844
1845 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1846 {
1847         uint8_t *keys;
1848         uint64_t hva;
1849         int srcu_idx, i, r = 0;
1850         bool unlocked;
1851
1852         if (args->flags != 0)
1853                 return -EINVAL;
1854
1855         /* Enforce sane limit on memory allocation */
1856         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1857                 return -EINVAL;
1858
1859         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1860         if (!keys)
1861                 return -ENOMEM;
1862
1863         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1864                            sizeof(uint8_t) * args->count);
1865         if (r) {
1866                 r = -EFAULT;
1867                 goto out;
1868         }
1869
1870         /* Enable storage key handling for the guest */
1871         r = s390_enable_skey();
1872         if (r)
1873                 goto out;
1874
1875         i = 0;
1876         down_read(&current->mm->mmap_sem);
1877         srcu_idx = srcu_read_lock(&kvm->srcu);
1878         while (i < args->count) {
1879                 unlocked = false;
1880                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1881                 if (kvm_is_error_hva(hva)) {
1882                         r = -EFAULT;
1883                         break;
1884                 }
1885
1886                 /* Lowest order bit is reserved */
1887                 if (keys[i] & 0x01) {
1888                         r = -EINVAL;
1889                         break;
1890                 }
1891
1892                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1893                 if (r) {
1894                         r = fixup_user_fault(current, current->mm, hva,
1895                                              FAULT_FLAG_WRITE, &unlocked);
1896                         if (r)
1897                                 break;
1898                 }
1899                 if (!r)
1900                         i++;
1901         }
1902         srcu_read_unlock(&kvm->srcu, srcu_idx);
1903         up_read(&current->mm->mmap_sem);
1904 out:
1905         kvfree(keys);
1906         return r;
1907 }
1908
1909 /*
1910  * Base address and length must be sent at the start of each block, therefore
1911  * it's cheaper to send some clean data, as long as it's less than the size of
1912  * two longs.
1913  */
1914 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1915 /* for consistency */
1916 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1917
1918 /*
1919  * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1920  * address falls in a hole. In that case the index of one of the memslots
1921  * bordering the hole is returned.
1922  */
1923 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1924 {
1925         int start = 0, end = slots->used_slots;
1926         int slot = atomic_read(&slots->lru_slot);
1927         struct kvm_memory_slot *memslots = slots->memslots;
1928
1929         if (gfn >= memslots[slot].base_gfn &&
1930             gfn < memslots[slot].base_gfn + memslots[slot].npages)
1931                 return slot;
1932
1933         while (start < end) {
1934                 slot = start + (end - start) / 2;
1935
1936                 if (gfn >= memslots[slot].base_gfn)
1937                         end = slot;
1938                 else
1939                         start = slot + 1;
1940         }
1941
1942         if (start >= slots->used_slots)
1943                 return slots->used_slots - 1;
1944
1945         if (gfn >= memslots[start].base_gfn &&
1946             gfn < memslots[start].base_gfn + memslots[start].npages) {
1947                 atomic_set(&slots->lru_slot, start);
1948         }
1949
1950         return start;
1951 }
1952
1953 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1954                               u8 *res, unsigned long bufsize)
1955 {
1956         unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1957
1958         args->count = 0;
1959         while (args->count < bufsize) {
1960                 hva = gfn_to_hva(kvm, cur_gfn);
1961                 /*
1962                  * We return an error if the first value was invalid, but we
1963                  * return successfully if at least one value was copied.
1964                  */
1965                 if (kvm_is_error_hva(hva))
1966                         return args->count ? 0 : -EFAULT;
1967                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1968                         pgstev = 0;
1969                 res[args->count++] = (pgstev >> 24) & 0x43;
1970                 cur_gfn++;
1971         }
1972
1973         return 0;
1974 }
1975
1976 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1977                                               unsigned long cur_gfn)
1978 {
1979         int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1980         struct kvm_memory_slot *ms = slots->memslots + slotidx;
1981         unsigned long ofs = cur_gfn - ms->base_gfn;
1982
1983         if (ms->base_gfn + ms->npages <= cur_gfn) {
1984                 slotidx--;
1985                 /* If we are above the highest slot, wrap around */
1986                 if (slotidx < 0)
1987                         slotidx = slots->used_slots - 1;
1988
1989                 ms = slots->memslots + slotidx;
1990                 ofs = 0;
1991         }
1992         ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1993         while ((slotidx > 0) && (ofs >= ms->npages)) {
1994                 slotidx--;
1995                 ms = slots->memslots + slotidx;
1996                 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1997         }
1998         return ms->base_gfn + ofs;
1999 }
2000
2001 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2002                              u8 *res, unsigned long bufsize)
2003 {
2004         unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2005         struct kvm_memslots *slots = kvm_memslots(kvm);
2006         struct kvm_memory_slot *ms;
2007
2008         if (unlikely(!slots->used_slots))
2009                 return 0;
2010
2011         cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2012         ms = gfn_to_memslot(kvm, cur_gfn);
2013         args->count = 0;
2014         args->start_gfn = cur_gfn;
2015         if (!ms)
2016                 return 0;
2017         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2018         mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2019
2020         while (args->count < bufsize) {
2021                 hva = gfn_to_hva(kvm, cur_gfn);
2022                 if (kvm_is_error_hva(hva))
2023                         return 0;
2024                 /* Decrement only if we actually flipped the bit to 0 */
2025                 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2026                         atomic64_dec(&kvm->arch.cmma_dirty_pages);
2027                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2028                         pgstev = 0;
2029                 /* Save the value */
2030                 res[args->count++] = (pgstev >> 24) & 0x43;
2031                 /* If the next bit is too far away, stop. */
2032                 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2033                         return 0;
2034                 /* If we reached the previous "next", find the next one */
2035                 if (cur_gfn == next_gfn)
2036                         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2037                 /* Reached the end of memory or of the buffer, stop */
2038                 if ((next_gfn >= mem_end) ||
2039                     (next_gfn - args->start_gfn >= bufsize))
2040                         return 0;
2041                 cur_gfn++;
2042                 /* Reached the end of the current memslot, take the next one. */
2043                 if (cur_gfn - ms->base_gfn >= ms->npages) {
2044                         ms = gfn_to_memslot(kvm, cur_gfn);
2045                         if (!ms)
2046                                 return 0;
2047                 }
2048         }
2049         return 0;
2050 }
2051
2052 /*
2053  * This function searches for the next page with dirty CMMA attributes, and
2054  * saves the attributes in the buffer up to either the end of the buffer or
2055  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2056  * no trailing clean bytes are saved.
2057  * In case no dirty bits were found, or if CMMA was not enabled or used, the
2058  * output buffer will indicate 0 as length.
2059  */
2060 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2061                                   struct kvm_s390_cmma_log *args)
2062 {
2063         unsigned long bufsize;
2064         int srcu_idx, peek, ret;
2065         u8 *values;
2066
2067         if (!kvm->arch.use_cmma)
2068                 return -ENXIO;
2069         /* Invalid/unsupported flags were specified */
2070         if (args->flags & ~KVM_S390_CMMA_PEEK)
2071                 return -EINVAL;
2072         /* Migration mode query, and we are not doing a migration */
2073         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2074         if (!peek && !kvm->arch.migration_mode)
2075                 return -EINVAL;
2076         /* CMMA is disabled or was not used, or the buffer has length zero */
2077         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2078         if (!bufsize || !kvm->mm->context.uses_cmm) {
2079                 memset(args, 0, sizeof(*args));
2080                 return 0;
2081         }
2082         /* We are not peeking, and there are no dirty pages */
2083         if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2084                 memset(args, 0, sizeof(*args));
2085                 return 0;
2086         }
2087
2088         values = vmalloc(bufsize);
2089         if (!values)
2090                 return -ENOMEM;
2091
2092         down_read(&kvm->mm->mmap_sem);
2093         srcu_idx = srcu_read_lock(&kvm->srcu);
2094         if (peek)
2095                 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2096         else
2097                 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2098         srcu_read_unlock(&kvm->srcu, srcu_idx);
2099         up_read(&kvm->mm->mmap_sem);
2100
2101         if (kvm->arch.migration_mode)
2102                 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2103         else
2104                 args->remaining = 0;
2105
2106         if (copy_to_user((void __user *)args->values, values, args->count))
2107                 ret = -EFAULT;
2108
2109         vfree(values);
2110         return ret;
2111 }
2112
2113 /*
2114  * This function sets the CMMA attributes for the given pages. If the input
2115  * buffer has zero length, no action is taken, otherwise the attributes are
2116  * set and the mm->context.uses_cmm flag is set.
2117  */
2118 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2119                                   const struct kvm_s390_cmma_log *args)
2120 {
2121         unsigned long hva, mask, pgstev, i;
2122         uint8_t *bits;
2123         int srcu_idx, r = 0;
2124
2125         mask = args->mask;
2126
2127         if (!kvm->arch.use_cmma)
2128                 return -ENXIO;
2129         /* invalid/unsupported flags */
2130         if (args->flags != 0)
2131                 return -EINVAL;
2132         /* Enforce sane limit on memory allocation */
2133         if (args->count > KVM_S390_CMMA_SIZE_MAX)
2134                 return -EINVAL;
2135         /* Nothing to do */
2136         if (args->count == 0)
2137                 return 0;
2138
2139         bits = vmalloc(array_size(sizeof(*bits), args->count));
2140         if (!bits)
2141                 return -ENOMEM;
2142
2143         r = copy_from_user(bits, (void __user *)args->values, args->count);
2144         if (r) {
2145                 r = -EFAULT;
2146                 goto out;
2147         }
2148
2149         down_read(&kvm->mm->mmap_sem);
2150         srcu_idx = srcu_read_lock(&kvm->srcu);
2151         for (i = 0; i < args->count; i++) {
2152                 hva = gfn_to_hva(kvm, args->start_gfn + i);
2153                 if (kvm_is_error_hva(hva)) {
2154                         r = -EFAULT;
2155                         break;
2156                 }
2157
2158                 pgstev = bits[i];
2159                 pgstev = pgstev << 24;
2160                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2161                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2162         }
2163         srcu_read_unlock(&kvm->srcu, srcu_idx);
2164         up_read(&kvm->mm->mmap_sem);
2165
2166         if (!kvm->mm->context.uses_cmm) {
2167                 down_write(&kvm->mm->mmap_sem);
2168                 kvm->mm->context.uses_cmm = 1;
2169                 up_write(&kvm->mm->mmap_sem);
2170         }
2171 out:
2172         vfree(bits);
2173         return r;
2174 }
2175
2176 static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp)
2177 {
2178         struct kvm_vcpu *vcpu;
2179         u16 rc, rrc;
2180         int ret = 0;
2181         int i;
2182
2183         /*
2184          * We ignore failures and try to destroy as many CPUs as possible.
2185          * At the same time we must not free the assigned resources when
2186          * this fails, as the ultravisor has still access to that memory.
2187          * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2188          * behind.
2189          * We want to return the first failure rc and rrc, though.
2190          */
2191         kvm_for_each_vcpu(i, vcpu, kvm) {
2192                 mutex_lock(&vcpu->mutex);
2193                 if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
2194                         *rcp = rc;
2195                         *rrcp = rrc;
2196                         ret = -EIO;
2197                 }
2198                 mutex_unlock(&vcpu->mutex);
2199         }
2200         return ret;
2201 }
2202
2203 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2204 {
2205         int i, r = 0;
2206         u16 dummy;
2207
2208         struct kvm_vcpu *vcpu;
2209
2210         kvm_for_each_vcpu(i, vcpu, kvm) {
2211                 mutex_lock(&vcpu->mutex);
2212                 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2213                 mutex_unlock(&vcpu->mutex);
2214                 if (r)
2215                         break;
2216         }
2217         if (r)
2218                 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2219         return r;
2220 }
2221
2222 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2223 {
2224         int r = 0;
2225         u16 dummy;
2226         void __user *argp = (void __user *)cmd->data;
2227
2228         switch (cmd->cmd) {
2229         case KVM_PV_ENABLE: {
2230                 r = -EINVAL;
2231                 if (kvm_s390_pv_is_protected(kvm))
2232                         break;
2233
2234                 /*
2235                  *  FMT 4 SIE needs esca. As we never switch back to bsca from
2236                  *  esca, we need no cleanup in the error cases below
2237                  */
2238                 r = sca_switch_to_extended(kvm);
2239                 if (r)
2240                         break;
2241
2242                 down_write(&current->mm->mmap_sem);
2243                 r = gmap_mark_unmergeable();
2244                 up_write(&current->mm->mmap_sem);
2245                 if (r)
2246                         break;
2247
2248                 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2249                 if (r)
2250                         break;
2251
2252                 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2253                 if (r)
2254                         kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2255
2256                 /* we need to block service interrupts from now on */
2257                 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2258                 break;
2259         }
2260         case KVM_PV_DISABLE: {
2261                 r = -EINVAL;
2262                 if (!kvm_s390_pv_is_protected(kvm))
2263                         break;
2264
2265                 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2266                 /*
2267                  * If a CPU could not be destroyed, destroy VM will also fail.
2268                  * There is no point in trying to destroy it. Instead return
2269                  * the rc and rrc from the first CPU that failed destroying.
2270                  */
2271                 if (r)
2272                         break;
2273                 r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
2274
2275                 /* no need to block service interrupts any more */
2276                 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2277                 break;
2278         }
2279         case KVM_PV_SET_SEC_PARMS: {
2280                 struct kvm_s390_pv_sec_parm parms = {};
2281                 void *hdr;
2282
2283                 r = -EINVAL;
2284                 if (!kvm_s390_pv_is_protected(kvm))
2285                         break;
2286
2287                 r = -EFAULT;
2288                 if (copy_from_user(&parms, argp, sizeof(parms)))
2289                         break;
2290
2291                 /* Currently restricted to 8KB */
2292                 r = -EINVAL;
2293                 if (parms.length > PAGE_SIZE * 2)
2294                         break;
2295
2296                 r = -ENOMEM;
2297                 hdr = vmalloc(parms.length);
2298                 if (!hdr)
2299                         break;
2300
2301                 r = -EFAULT;
2302                 if (!copy_from_user(hdr, (void __user *)parms.origin,
2303                                     parms.length))
2304                         r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2305                                                       &cmd->rc, &cmd->rrc);
2306
2307                 vfree(hdr);
2308                 break;
2309         }
2310         case KVM_PV_UNPACK: {
2311                 struct kvm_s390_pv_unp unp = {};
2312
2313                 r = -EINVAL;
2314                 if (!kvm_s390_pv_is_protected(kvm))
2315                         break;
2316
2317                 r = -EFAULT;
2318                 if (copy_from_user(&unp, argp, sizeof(unp)))
2319                         break;
2320
2321                 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2322                                        &cmd->rc, &cmd->rrc);
2323                 break;
2324         }
2325         case KVM_PV_VERIFY: {
2326                 r = -EINVAL;
2327                 if (!kvm_s390_pv_is_protected(kvm))
2328                         break;
2329
2330                 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2331                                   UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2332                 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2333                              cmd->rrc);
2334                 break;
2335         }
2336         case KVM_PV_PREP_RESET: {
2337                 r = -EINVAL;
2338                 if (!kvm_s390_pv_is_protected(kvm))
2339                         break;
2340
2341                 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2342                                   UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2343                 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2344                              cmd->rc, cmd->rrc);
2345                 break;
2346         }
2347         case KVM_PV_UNSHARE_ALL: {
2348                 r = -EINVAL;
2349                 if (!kvm_s390_pv_is_protected(kvm))
2350                         break;
2351
2352                 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2353                                   UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2354                 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2355                              cmd->rc, cmd->rrc);
2356                 break;
2357         }
2358         default:
2359                 r = -ENOTTY;
2360         }
2361         return r;
2362 }
2363
2364 long kvm_arch_vm_ioctl(struct file *filp,
2365                        unsigned int ioctl, unsigned long arg)
2366 {
2367         struct kvm *kvm = filp->private_data;
2368         void __user *argp = (void __user *)arg;
2369         struct kvm_device_attr attr;
2370         int r;
2371
2372         switch (ioctl) {
2373         case KVM_S390_INTERRUPT: {
2374                 struct kvm_s390_interrupt s390int;
2375
2376                 r = -EFAULT;
2377                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2378                         break;
2379                 r = kvm_s390_inject_vm(kvm, &s390int);
2380                 break;
2381         }
2382         case KVM_CREATE_IRQCHIP: {
2383                 struct kvm_irq_routing_entry routing;
2384
2385                 r = -EINVAL;
2386                 if (kvm->arch.use_irqchip) {
2387                         /* Set up dummy routing. */
2388                         memset(&routing, 0, sizeof(routing));
2389                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2390                 }
2391                 break;
2392         }
2393         case KVM_SET_DEVICE_ATTR: {
2394                 r = -EFAULT;
2395                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2396                         break;
2397                 r = kvm_s390_vm_set_attr(kvm, &attr);
2398                 break;
2399         }
2400         case KVM_GET_DEVICE_ATTR: {
2401                 r = -EFAULT;
2402                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2403                         break;
2404                 r = kvm_s390_vm_get_attr(kvm, &attr);
2405                 break;
2406         }
2407         case KVM_HAS_DEVICE_ATTR: {
2408                 r = -EFAULT;
2409                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2410                         break;
2411                 r = kvm_s390_vm_has_attr(kvm, &attr);
2412                 break;
2413         }
2414         case KVM_S390_GET_SKEYS: {
2415                 struct kvm_s390_skeys args;
2416
2417                 r = -EFAULT;
2418                 if (copy_from_user(&args, argp,
2419                                    sizeof(struct kvm_s390_skeys)))
2420                         break;
2421                 r = kvm_s390_get_skeys(kvm, &args);
2422                 break;
2423         }
2424         case KVM_S390_SET_SKEYS: {
2425                 struct kvm_s390_skeys args;
2426
2427                 r = -EFAULT;
2428                 if (copy_from_user(&args, argp,
2429                                    sizeof(struct kvm_s390_skeys)))
2430                         break;
2431                 r = kvm_s390_set_skeys(kvm, &args);
2432                 break;
2433         }
2434         case KVM_S390_GET_CMMA_BITS: {
2435                 struct kvm_s390_cmma_log args;
2436
2437                 r = -EFAULT;
2438                 if (copy_from_user(&args, argp, sizeof(args)))
2439                         break;
2440                 mutex_lock(&kvm->slots_lock);
2441                 r = kvm_s390_get_cmma_bits(kvm, &args);
2442                 mutex_unlock(&kvm->slots_lock);
2443                 if (!r) {
2444                         r = copy_to_user(argp, &args, sizeof(args));
2445                         if (r)
2446                                 r = -EFAULT;
2447                 }
2448                 break;
2449         }
2450         case KVM_S390_SET_CMMA_BITS: {
2451                 struct kvm_s390_cmma_log args;
2452
2453                 r = -EFAULT;
2454                 if (copy_from_user(&args, argp, sizeof(args)))
2455                         break;
2456                 mutex_lock(&kvm->slots_lock);
2457                 r = kvm_s390_set_cmma_bits(kvm, &args);
2458                 mutex_unlock(&kvm->slots_lock);
2459                 break;
2460         }
2461         case KVM_S390_PV_COMMAND: {
2462                 struct kvm_pv_cmd args;
2463
2464                 /* protvirt means user sigp */
2465                 kvm->arch.user_cpu_state_ctrl = 1;
2466                 r = 0;
2467                 if (!is_prot_virt_host()) {
2468                         r = -EINVAL;
2469                         break;
2470                 }
2471                 if (copy_from_user(&args, argp, sizeof(args))) {
2472                         r = -EFAULT;
2473                         break;
2474                 }
2475                 if (args.flags) {
2476                         r = -EINVAL;
2477                         break;
2478                 }
2479                 mutex_lock(&kvm->lock);
2480                 r = kvm_s390_handle_pv(kvm, &args);
2481                 mutex_unlock(&kvm->lock);
2482                 if (copy_to_user(argp, &args, sizeof(args))) {
2483                         r = -EFAULT;
2484                         break;
2485                 }
2486                 break;
2487         }
2488         default:
2489                 r = -ENOTTY;
2490         }
2491
2492         return r;
2493 }
2494
2495 static int kvm_s390_apxa_installed(void)
2496 {
2497         struct ap_config_info info;
2498
2499         if (ap_instructions_available()) {
2500                 if (ap_qci(&info) == 0)
2501                         return info.apxa;
2502         }
2503
2504         return 0;
2505 }
2506
2507 /*
2508  * The format of the crypto control block (CRYCB) is specified in the 3 low
2509  * order bits of the CRYCB designation (CRYCBD) field as follows:
2510  * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2511  *           AP extended addressing (APXA) facility are installed.
2512  * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2513  * Format 2: Both the APXA and MSAX3 facilities are installed
2514  */
2515 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2516 {
2517         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2518
2519         /* Clear the CRYCB format bits - i.e., set format 0 by default */
2520         kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2521
2522         /* Check whether MSAX3 is installed */
2523         if (!test_kvm_facility(kvm, 76))
2524                 return;
2525
2526         if (kvm_s390_apxa_installed())
2527                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2528         else
2529                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2530 }
2531
2532 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2533                                unsigned long *aqm, unsigned long *adm)
2534 {
2535         struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2536
2537         mutex_lock(&kvm->lock);
2538         kvm_s390_vcpu_block_all(kvm);
2539
2540         switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2541         case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2542                 memcpy(crycb->apcb1.apm, apm, 32);
2543                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2544                          apm[0], apm[1], apm[2], apm[3]);
2545                 memcpy(crycb->apcb1.aqm, aqm, 32);
2546                 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2547                          aqm[0], aqm[1], aqm[2], aqm[3]);
2548                 memcpy(crycb->apcb1.adm, adm, 32);
2549                 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2550                          adm[0], adm[1], adm[2], adm[3]);
2551                 break;
2552         case CRYCB_FORMAT1:
2553         case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2554                 memcpy(crycb->apcb0.apm, apm, 8);
2555                 memcpy(crycb->apcb0.aqm, aqm, 2);
2556                 memcpy(crycb->apcb0.adm, adm, 2);
2557                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2558                          apm[0], *((unsigned short *)aqm),
2559                          *((unsigned short *)adm));
2560                 break;
2561         default:        /* Can not happen */
2562                 break;
2563         }
2564
2565         /* recreate the shadow crycb for each vcpu */
2566         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2567         kvm_s390_vcpu_unblock_all(kvm);
2568         mutex_unlock(&kvm->lock);
2569 }
2570 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2571
2572 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2573 {
2574         mutex_lock(&kvm->lock);
2575         kvm_s390_vcpu_block_all(kvm);
2576
2577         memset(&kvm->arch.crypto.crycb->apcb0, 0,
2578                sizeof(kvm->arch.crypto.crycb->apcb0));
2579         memset(&kvm->arch.crypto.crycb->apcb1, 0,
2580                sizeof(kvm->arch.crypto.crycb->apcb1));
2581
2582         VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2583         /* recreate the shadow crycb for each vcpu */
2584         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2585         kvm_s390_vcpu_unblock_all(kvm);
2586         mutex_unlock(&kvm->lock);
2587 }
2588 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2589
2590 static u64 kvm_s390_get_initial_cpuid(void)
2591 {
2592         struct cpuid cpuid;
2593
2594         get_cpu_id(&cpuid);
2595         cpuid.version = 0xff;
2596         return *((u64 *) &cpuid);
2597 }
2598
2599 static void kvm_s390_crypto_init(struct kvm *kvm)
2600 {
2601         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2602         kvm_s390_set_crycb_format(kvm);
2603
2604         if (!test_kvm_facility(kvm, 76))
2605                 return;
2606
2607         /* Enable AES/DEA protected key functions by default */
2608         kvm->arch.crypto.aes_kw = 1;
2609         kvm->arch.crypto.dea_kw = 1;
2610         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2611                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2612         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2613                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2614 }
2615
2616 static void sca_dispose(struct kvm *kvm)
2617 {
2618         if (kvm->arch.use_esca)
2619                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2620         else
2621                 free_page((unsigned long)(kvm->arch.sca));
2622         kvm->arch.sca = NULL;
2623 }
2624
2625 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2626 {
2627         gfp_t alloc_flags = GFP_KERNEL;
2628         int i, rc;
2629         char debug_name[16];
2630         static unsigned long sca_offset;
2631
2632         rc = -EINVAL;
2633 #ifdef CONFIG_KVM_S390_UCONTROL
2634         if (type & ~KVM_VM_S390_UCONTROL)
2635                 goto out_err;
2636         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2637                 goto out_err;
2638 #else
2639         if (type)
2640                 goto out_err;
2641 #endif
2642
2643         rc = s390_enable_sie();
2644         if (rc)
2645                 goto out_err;
2646
2647         rc = -ENOMEM;
2648
2649         if (!sclp.has_64bscao)
2650                 alloc_flags |= GFP_DMA;
2651         rwlock_init(&kvm->arch.sca_lock);
2652         /* start with basic SCA */
2653         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2654         if (!kvm->arch.sca)
2655                 goto out_err;
2656         mutex_lock(&kvm_lock);
2657         sca_offset += 16;
2658         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2659                 sca_offset = 0;
2660         kvm->arch.sca = (struct bsca_block *)
2661                         ((char *) kvm->arch.sca + sca_offset);
2662         mutex_unlock(&kvm_lock);
2663
2664         sprintf(debug_name, "kvm-%u", current->pid);
2665
2666         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2667         if (!kvm->arch.dbf)
2668                 goto out_err;
2669
2670         BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2671         kvm->arch.sie_page2 =
2672              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2673         if (!kvm->arch.sie_page2)
2674                 goto out_err;
2675
2676         kvm->arch.sie_page2->kvm = kvm;
2677         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2678
2679         for (i = 0; i < kvm_s390_fac_size(); i++) {
2680                 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2681                                               (kvm_s390_fac_base[i] |
2682                                                kvm_s390_fac_ext[i]);
2683                 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2684                                               kvm_s390_fac_base[i];
2685         }
2686         kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2687
2688         /* we are always in czam mode - even on pre z14 machines */
2689         set_kvm_facility(kvm->arch.model.fac_mask, 138);
2690         set_kvm_facility(kvm->arch.model.fac_list, 138);
2691         /* we emulate STHYI in kvm */
2692         set_kvm_facility(kvm->arch.model.fac_mask, 74);
2693         set_kvm_facility(kvm->arch.model.fac_list, 74);
2694         if (MACHINE_HAS_TLB_GUEST) {
2695                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2696                 set_kvm_facility(kvm->arch.model.fac_list, 147);
2697         }
2698
2699         if (css_general_characteristics.aiv && test_facility(65))
2700                 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2701
2702         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2703         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2704
2705         kvm_s390_crypto_init(kvm);
2706
2707         mutex_init(&kvm->arch.float_int.ais_lock);
2708         spin_lock_init(&kvm->arch.float_int.lock);
2709         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2710                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2711         init_waitqueue_head(&kvm->arch.ipte_wq);
2712         mutex_init(&kvm->arch.ipte_mutex);
2713
2714         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2715         VM_EVENT(kvm, 3, "vm created with type %lu", type);
2716
2717         if (type & KVM_VM_S390_UCONTROL) {
2718                 kvm->arch.gmap = NULL;
2719                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2720         } else {
2721                 if (sclp.hamax == U64_MAX)
2722                         kvm->arch.mem_limit = TASK_SIZE_MAX;
2723                 else
2724                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2725                                                     sclp.hamax + 1);
2726                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2727                 if (!kvm->arch.gmap)
2728                         goto out_err;
2729                 kvm->arch.gmap->private = kvm;
2730                 kvm->arch.gmap->pfault_enabled = 0;
2731         }
2732
2733         kvm->arch.use_pfmfi = sclp.has_pfmfi;
2734         kvm->arch.use_skf = sclp.has_skey;
2735         spin_lock_init(&kvm->arch.start_stop_lock);
2736         kvm_s390_vsie_init(kvm);
2737         if (use_gisa)
2738                 kvm_s390_gisa_init(kvm);
2739         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2740
2741         return 0;
2742 out_err:
2743         free_page((unsigned long)kvm->arch.sie_page2);
2744         debug_unregister(kvm->arch.dbf);
2745         sca_dispose(kvm);
2746         KVM_EVENT(3, "creation of vm failed: %d", rc);
2747         return rc;
2748 }
2749
2750 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2751 {
2752         u16 rc, rrc;
2753
2754         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2755         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2756         kvm_s390_clear_local_irqs(vcpu);
2757         kvm_clear_async_pf_completion_queue(vcpu);
2758         if (!kvm_is_ucontrol(vcpu->kvm))
2759                 sca_del_vcpu(vcpu);
2760
2761         if (kvm_is_ucontrol(vcpu->kvm))
2762                 gmap_remove(vcpu->arch.gmap);
2763
2764         if (vcpu->kvm->arch.use_cmma)
2765                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2766         /* We can not hold the vcpu mutex here, we are already dying */
2767         if (kvm_s390_pv_cpu_get_handle(vcpu))
2768                 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
2769         free_page((unsigned long)(vcpu->arch.sie_block));
2770 }
2771
2772 static void kvm_free_vcpus(struct kvm *kvm)
2773 {
2774         unsigned int i;
2775         struct kvm_vcpu *vcpu;
2776
2777         kvm_for_each_vcpu(i, vcpu, kvm)
2778                 kvm_vcpu_destroy(vcpu);
2779
2780         mutex_lock(&kvm->lock);
2781         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2782                 kvm->vcpus[i] = NULL;
2783
2784         atomic_set(&kvm->online_vcpus, 0);
2785         mutex_unlock(&kvm->lock);
2786 }
2787
2788 void kvm_arch_destroy_vm(struct kvm *kvm)
2789 {
2790         u16 rc, rrc;
2791
2792         kvm_free_vcpus(kvm);
2793         sca_dispose(kvm);
2794         kvm_s390_gisa_destroy(kvm);
2795         /*
2796          * We are already at the end of life and kvm->lock is not taken.
2797          * This is ok as the file descriptor is closed by now and nobody
2798          * can mess with the pv state. To avoid lockdep_assert_held from
2799          * complaining we do not use kvm_s390_pv_is_protected.
2800          */
2801         if (kvm_s390_pv_get_handle(kvm))
2802                 kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
2803         debug_unregister(kvm->arch.dbf);
2804         free_page((unsigned long)kvm->arch.sie_page2);
2805         if (!kvm_is_ucontrol(kvm))
2806                 gmap_remove(kvm->arch.gmap);
2807         kvm_s390_destroy_adapters(kvm);
2808         kvm_s390_clear_float_irqs(kvm);
2809         kvm_s390_vsie_destroy(kvm);
2810         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2811 }
2812
2813 /* Section: vcpu related */
2814 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2815 {
2816         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2817         if (!vcpu->arch.gmap)
2818                 return -ENOMEM;
2819         vcpu->arch.gmap->private = vcpu->kvm;
2820
2821         return 0;
2822 }
2823
2824 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2825 {
2826         if (!kvm_s390_use_sca_entries())
2827                 return;
2828         read_lock(&vcpu->kvm->arch.sca_lock);
2829         if (vcpu->kvm->arch.use_esca) {
2830                 struct esca_block *sca = vcpu->kvm->arch.sca;
2831
2832                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2833                 sca->cpu[vcpu->vcpu_id].sda = 0;
2834         } else {
2835                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2836
2837                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2838                 sca->cpu[vcpu->vcpu_id].sda = 0;
2839         }
2840         read_unlock(&vcpu->kvm->arch.sca_lock);
2841 }
2842
2843 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2844 {
2845         if (!kvm_s390_use_sca_entries()) {
2846                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2847
2848                 /* we still need the basic sca for the ipte control */
2849                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2850                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2851                 return;
2852         }
2853         read_lock(&vcpu->kvm->arch.sca_lock);
2854         if (vcpu->kvm->arch.use_esca) {
2855                 struct esca_block *sca = vcpu->kvm->arch.sca;
2856
2857                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2858                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2859                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2860                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2861                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2862         } else {
2863                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2864
2865                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2866                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2867                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2868                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2869         }
2870         read_unlock(&vcpu->kvm->arch.sca_lock);
2871 }
2872
2873 /* Basic SCA to Extended SCA data copy routines */
2874 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2875 {
2876         d->sda = s->sda;
2877         d->sigp_ctrl.c = s->sigp_ctrl.c;
2878         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2879 }
2880
2881 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2882 {
2883         int i;
2884
2885         d->ipte_control = s->ipte_control;
2886         d->mcn[0] = s->mcn;
2887         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2888                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2889 }
2890
2891 static int sca_switch_to_extended(struct kvm *kvm)
2892 {
2893         struct bsca_block *old_sca = kvm->arch.sca;
2894         struct esca_block *new_sca;
2895         struct kvm_vcpu *vcpu;
2896         unsigned int vcpu_idx;
2897         u32 scaol, scaoh;
2898
2899         if (kvm->arch.use_esca)
2900                 return 0;
2901
2902         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2903         if (!new_sca)
2904                 return -ENOMEM;
2905
2906         scaoh = (u32)((u64)(new_sca) >> 32);
2907         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2908
2909         kvm_s390_vcpu_block_all(kvm);
2910         write_lock(&kvm->arch.sca_lock);
2911
2912         sca_copy_b_to_e(new_sca, old_sca);
2913
2914         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2915                 vcpu->arch.sie_block->scaoh = scaoh;
2916                 vcpu->arch.sie_block->scaol = scaol;
2917                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2918         }
2919         kvm->arch.sca = new_sca;
2920         kvm->arch.use_esca = 1;
2921
2922         write_unlock(&kvm->arch.sca_lock);
2923         kvm_s390_vcpu_unblock_all(kvm);
2924
2925         free_page((unsigned long)old_sca);
2926
2927         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2928                  old_sca, kvm->arch.sca);
2929         return 0;
2930 }
2931
2932 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2933 {
2934         int rc;
2935
2936         if (!kvm_s390_use_sca_entries()) {
2937                 if (id < KVM_MAX_VCPUS)
2938                         return true;
2939                 return false;
2940         }
2941         if (id < KVM_S390_BSCA_CPU_SLOTS)
2942                 return true;
2943         if (!sclp.has_esca || !sclp.has_64bscao)
2944                 return false;
2945
2946         mutex_lock(&kvm->lock);
2947         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2948         mutex_unlock(&kvm->lock);
2949
2950         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2951 }
2952
2953 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2954 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2955 {
2956         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2957         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2958         vcpu->arch.cputm_start = get_tod_clock_fast();
2959         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2960 }
2961
2962 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2963 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2964 {
2965         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2966         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2967         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2968         vcpu->arch.cputm_start = 0;
2969         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2970 }
2971
2972 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2973 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2974 {
2975         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2976         vcpu->arch.cputm_enabled = true;
2977         __start_cpu_timer_accounting(vcpu);
2978 }
2979
2980 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2981 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2982 {
2983         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2984         __stop_cpu_timer_accounting(vcpu);
2985         vcpu->arch.cputm_enabled = false;
2986 }
2987
2988 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2989 {
2990         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2991         __enable_cpu_timer_accounting(vcpu);
2992         preempt_enable();
2993 }
2994
2995 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2996 {
2997         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2998         __disable_cpu_timer_accounting(vcpu);
2999         preempt_enable();
3000 }
3001
3002 /* set the cpu timer - may only be called from the VCPU thread itself */
3003 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3004 {
3005         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3006         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3007         if (vcpu->arch.cputm_enabled)
3008                 vcpu->arch.cputm_start = get_tod_clock_fast();
3009         vcpu->arch.sie_block->cputm = cputm;
3010         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3011         preempt_enable();
3012 }
3013
3014 /* update and get the cpu timer - can also be called from other VCPU threads */
3015 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3016 {
3017         unsigned int seq;
3018         __u64 value;
3019
3020         if (unlikely(!vcpu->arch.cputm_enabled))
3021                 return vcpu->arch.sie_block->cputm;
3022
3023         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3024         do {
3025                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3026                 /*
3027                  * If the writer would ever execute a read in the critical
3028                  * section, e.g. in irq context, we have a deadlock.
3029                  */
3030                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3031                 value = vcpu->arch.sie_block->cputm;
3032                 /* if cputm_start is 0, accounting is being started/stopped */
3033                 if (likely(vcpu->arch.cputm_start))
3034                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3035         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3036         preempt_enable();
3037         return value;
3038 }
3039
3040 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3041 {
3042
3043         gmap_enable(vcpu->arch.enabled_gmap);
3044         kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3045         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3046                 __start_cpu_timer_accounting(vcpu);
3047         vcpu->cpu = cpu;
3048 }
3049
3050 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3051 {
3052         vcpu->cpu = -1;
3053         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3054                 __stop_cpu_timer_accounting(vcpu);
3055         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3056         vcpu->arch.enabled_gmap = gmap_get_enabled();
3057         gmap_disable(vcpu->arch.enabled_gmap);
3058
3059 }
3060
3061 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3062 {
3063         mutex_lock(&vcpu->kvm->lock);
3064         preempt_disable();
3065         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3066         vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3067         preempt_enable();
3068         mutex_unlock(&vcpu->kvm->lock);
3069         if (!kvm_is_ucontrol(vcpu->kvm)) {
3070                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3071                 sca_add_vcpu(vcpu);
3072         }
3073         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3074                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3075         /* make vcpu_load load the right gmap on the first trigger */
3076         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
3077 }
3078
3079 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3080 {
3081         if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3082             test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3083                 return true;
3084         return false;
3085 }
3086
3087 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3088 {
3089         /* At least one ECC subfunction must be present */
3090         return kvm_has_pckmo_subfunc(kvm, 32) ||
3091                kvm_has_pckmo_subfunc(kvm, 33) ||
3092                kvm_has_pckmo_subfunc(kvm, 34) ||
3093                kvm_has_pckmo_subfunc(kvm, 40) ||
3094                kvm_has_pckmo_subfunc(kvm, 41);
3095
3096 }
3097
3098 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3099 {
3100         /*
3101          * If the AP instructions are not being interpreted and the MSAX3
3102          * facility is not configured for the guest, there is nothing to set up.
3103          */
3104         if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3105                 return;
3106
3107         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3108         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3109         vcpu->arch.sie_block->eca &= ~ECA_APIE;
3110         vcpu->arch.sie_block->ecd &= ~ECD_ECC;
3111
3112         if (vcpu->kvm->arch.crypto.apie)
3113                 vcpu->arch.sie_block->eca |= ECA_APIE;
3114
3115         /* Set up protected key support */
3116         if (vcpu->kvm->arch.crypto.aes_kw) {
3117                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3118                 /* ecc is also wrapped with AES key */
3119                 if (kvm_has_pckmo_ecc(vcpu->kvm))
3120                         vcpu->arch.sie_block->ecd |= ECD_ECC;
3121         }
3122
3123         if (vcpu->kvm->arch.crypto.dea_kw)
3124                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3125 }
3126
3127 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3128 {
3129         free_page(vcpu->arch.sie_block->cbrlo);
3130         vcpu->arch.sie_block->cbrlo = 0;
3131 }
3132
3133 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3134 {
3135         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
3136         if (!vcpu->arch.sie_block->cbrlo)
3137                 return -ENOMEM;
3138         return 0;
3139 }
3140
3141 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3142 {
3143         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3144
3145         vcpu->arch.sie_block->ibc = model->ibc;
3146         if (test_kvm_facility(vcpu->kvm, 7))
3147                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
3148 }
3149
3150 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3151 {
3152         int rc = 0;
3153         u16 uvrc, uvrrc;
3154
3155         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3156                                                     CPUSTAT_SM |
3157                                                     CPUSTAT_STOPPED);
3158
3159         if (test_kvm_facility(vcpu->kvm, 78))
3160                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3161         else if (test_kvm_facility(vcpu->kvm, 8))
3162                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3163
3164         kvm_s390_vcpu_setup_model(vcpu);
3165
3166         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
3167         if (MACHINE_HAS_ESOP)
3168                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3169         if (test_kvm_facility(vcpu->kvm, 9))
3170                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3171         if (test_kvm_facility(vcpu->kvm, 73))
3172                 vcpu->arch.sie_block->ecb |= ECB_TE;
3173
3174         if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3175                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3176         if (test_kvm_facility(vcpu->kvm, 130))
3177                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3178         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3179         if (sclp.has_cei)
3180                 vcpu->arch.sie_block->eca |= ECA_CEI;
3181         if (sclp.has_ib)
3182                 vcpu->arch.sie_block->eca |= ECA_IB;
3183         if (sclp.has_siif)
3184                 vcpu->arch.sie_block->eca |= ECA_SII;
3185         if (sclp.has_sigpif)
3186                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3187         if (test_kvm_facility(vcpu->kvm, 129)) {
3188                 vcpu->arch.sie_block->eca |= ECA_VX;
3189                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3190         }
3191         if (test_kvm_facility(vcpu->kvm, 139))
3192                 vcpu->arch.sie_block->ecd |= ECD_MEF;
3193         if (test_kvm_facility(vcpu->kvm, 156))
3194                 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3195         if (vcpu->arch.sie_block->gd) {
3196                 vcpu->arch.sie_block->eca |= ECA_AIV;
3197                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3198                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3199         }
3200         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
3201                                         | SDNXC;
3202         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
3203
3204         if (sclp.has_kss)
3205                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3206         else
3207                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3208
3209         if (vcpu->kvm->arch.use_cmma) {
3210                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3211                 if (rc)
3212                         return rc;
3213         }
3214         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3215         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
3216
3217         vcpu->arch.sie_block->hpid = HPID_KVM;
3218
3219         kvm_s390_vcpu_crypto_setup(vcpu);
3220
3221         mutex_lock(&vcpu->kvm->lock);
3222         if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3223                 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3224                 if (rc)
3225                         kvm_s390_vcpu_unsetup_cmma(vcpu);
3226         }
3227         mutex_unlock(&vcpu->kvm->lock);
3228
3229         return rc;
3230 }
3231
3232 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3233 {
3234         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3235                 return -EINVAL;
3236         return 0;
3237 }
3238
3239 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3240 {
3241         struct sie_page *sie_page;
3242         int rc;
3243
3244         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3245         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
3246         if (!sie_page)
3247                 return -ENOMEM;
3248
3249         vcpu->arch.sie_block = &sie_page->sie_block;
3250         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3251
3252         /* the real guest size will always be smaller than msl */
3253         vcpu->arch.sie_block->mso = 0;
3254         vcpu->arch.sie_block->msl = sclp.hamax;
3255
3256         vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3257         spin_lock_init(&vcpu->arch.local_int.lock);
3258         vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
3259         if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3260                 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3261         seqcount_init(&vcpu->arch.cputm_seqcount);
3262
3263         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3264         kvm_clear_async_pf_completion_queue(vcpu);
3265         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3266                                     KVM_SYNC_GPRS |
3267                                     KVM_SYNC_ACRS |
3268                                     KVM_SYNC_CRS |
3269                                     KVM_SYNC_ARCH0 |
3270                                     KVM_SYNC_PFAULT;
3271         kvm_s390_set_prefix(vcpu, 0);
3272         if (test_kvm_facility(vcpu->kvm, 64))
3273                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3274         if (test_kvm_facility(vcpu->kvm, 82))
3275                 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3276         if (test_kvm_facility(vcpu->kvm, 133))
3277                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3278         if (test_kvm_facility(vcpu->kvm, 156))
3279                 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3280         /* fprs can be synchronized via vrs, even if the guest has no vx. With
3281          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3282          */
3283         if (MACHINE_HAS_VX)
3284                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3285         else
3286                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3287
3288         if (kvm_is_ucontrol(vcpu->kvm)) {
3289                 rc = __kvm_ucontrol_vcpu_init(vcpu);
3290                 if (rc)
3291                         goto out_free_sie_block;
3292         }
3293
3294         VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3295                  vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3296         trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3297
3298         rc = kvm_s390_vcpu_setup(vcpu);
3299         if (rc)
3300                 goto out_ucontrol_uninit;
3301         return 0;
3302
3303 out_ucontrol_uninit:
3304         if (kvm_is_ucontrol(vcpu->kvm))
3305                 gmap_remove(vcpu->arch.gmap);
3306 out_free_sie_block:
3307         free_page((unsigned long)(vcpu->arch.sie_block));
3308         return rc;
3309 }
3310
3311 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3312 {
3313         return kvm_s390_vcpu_has_irq(vcpu, 0);
3314 }
3315
3316 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3317 {
3318         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3319 }
3320
3321 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3322 {
3323         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3324         exit_sie(vcpu);
3325 }
3326
3327 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3328 {
3329         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3330 }
3331
3332 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3333 {
3334         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3335         exit_sie(vcpu);
3336 }
3337
3338 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3339 {
3340         return atomic_read(&vcpu->arch.sie_block->prog20) &
3341                (PROG_BLOCK_SIE | PROG_REQUEST);
3342 }
3343
3344 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3345 {
3346         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3347 }
3348
3349 /*
3350  * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3351  * If the CPU is not running (e.g. waiting as idle) the function will
3352  * return immediately. */
3353 void exit_sie(struct kvm_vcpu *vcpu)
3354 {
3355         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3356         kvm_s390_vsie_kick(vcpu);
3357         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3358                 cpu_relax();
3359 }
3360
3361 /* Kick a guest cpu out of SIE to process a request synchronously */
3362 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3363 {
3364         kvm_make_request(req, vcpu);
3365         kvm_s390_vcpu_request(vcpu);
3366 }
3367
3368 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3369                               unsigned long end)
3370 {
3371         struct kvm *kvm = gmap->private;
3372         struct kvm_vcpu *vcpu;
3373         unsigned long prefix;
3374         int i;
3375
3376         if (gmap_is_shadow(gmap))
3377                 return;
3378         if (start >= 1UL << 31)
3379                 /* We are only interested in prefix pages */
3380                 return;
3381         kvm_for_each_vcpu(i, vcpu, kvm) {
3382                 /* match against both prefix pages */
3383                 prefix = kvm_s390_get_prefix(vcpu);
3384                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3385                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3386                                    start, end);
3387                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3388                 }
3389         }
3390 }
3391
3392 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3393 {
3394         /* do not poll with more than halt_poll_max_steal percent of steal time */
3395         if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3396             halt_poll_max_steal) {
3397                 vcpu->stat.halt_no_poll_steal++;
3398                 return true;
3399         }
3400         return false;
3401 }
3402
3403 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3404 {
3405         /* kvm common code refers to this, but never calls it */
3406         BUG();
3407         return 0;
3408 }
3409
3410 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3411                                            struct kvm_one_reg *reg)
3412 {
3413         int r = -EINVAL;
3414
3415         switch (reg->id) {
3416         case KVM_REG_S390_TODPR:
3417                 r = put_user(vcpu->arch.sie_block->todpr,
3418                              (u32 __user *)reg->addr);
3419                 break;
3420         case KVM_REG_S390_EPOCHDIFF:
3421                 r = put_user(vcpu->arch.sie_block->epoch,
3422                              (u64 __user *)reg->addr);
3423                 break;
3424         case KVM_REG_S390_CPU_TIMER:
3425                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3426                              (u64 __user *)reg->addr);
3427                 break;
3428         case KVM_REG_S390_CLOCK_COMP:
3429                 r = put_user(vcpu->arch.sie_block->ckc,
3430                              (u64 __user *)reg->addr);
3431                 break;
3432         case KVM_REG_S390_PFTOKEN:
3433                 r = put_user(vcpu->arch.pfault_token,
3434                              (u64 __user *)reg->addr);
3435                 break;
3436         case KVM_REG_S390_PFCOMPARE:
3437                 r = put_user(vcpu->arch.pfault_compare,
3438                              (u64 __user *)reg->addr);
3439                 break;
3440         case KVM_REG_S390_PFSELECT:
3441                 r = put_user(vcpu->arch.pfault_select,
3442                              (u64 __user *)reg->addr);
3443                 break;
3444         case KVM_REG_S390_PP:
3445                 r = put_user(vcpu->arch.sie_block->pp,
3446                              (u64 __user *)reg->addr);
3447                 break;
3448         case KVM_REG_S390_GBEA:
3449                 r = put_user(vcpu->arch.sie_block->gbea,
3450                              (u64 __user *)reg->addr);
3451                 break;
3452         default:
3453                 break;
3454         }
3455
3456         return r;
3457 }
3458
3459 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3460                                            struct kvm_one_reg *reg)
3461 {
3462         int r = -EINVAL;
3463         __u64 val;
3464
3465         switch (reg->id) {
3466         case KVM_REG_S390_TODPR:
3467                 r = get_user(vcpu->arch.sie_block->todpr,
3468                              (u32 __user *)reg->addr);
3469                 break;
3470         case KVM_REG_S390_EPOCHDIFF:
3471                 r = get_user(vcpu->arch.sie_block->epoch,
3472                              (u64 __user *)reg->addr);
3473                 break;
3474         case KVM_REG_S390_CPU_TIMER:
3475                 r = get_user(val, (u64 __user *)reg->addr);
3476                 if (!r)
3477                         kvm_s390_set_cpu_timer(vcpu, val);
3478                 break;
3479         case KVM_REG_S390_CLOCK_COMP:
3480                 r = get_user(vcpu->arch.sie_block->ckc,
3481                              (u64 __user *)reg->addr);
3482                 break;
3483         case KVM_REG_S390_PFTOKEN:
3484                 r = get_user(vcpu->arch.pfault_token,
3485                              (u64 __user *)reg->addr);
3486                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3487                         kvm_clear_async_pf_completion_queue(vcpu);
3488                 break;
3489         case KVM_REG_S390_PFCOMPARE:
3490                 r = get_user(vcpu->arch.pfault_compare,
3491                              (u64 __user *)reg->addr);
3492                 break;
3493         case KVM_REG_S390_PFSELECT:
3494                 r = get_user(vcpu->arch.pfault_select,
3495                              (u64 __user *)reg->addr);
3496                 break;
3497         case KVM_REG_S390_PP:
3498                 r = get_user(vcpu->arch.sie_block->pp,
3499                              (u64 __user *)reg->addr);
3500                 break;
3501         case KVM_REG_S390_GBEA:
3502                 r = get_user(vcpu->arch.sie_block->gbea,
3503                              (u64 __user *)reg->addr);
3504                 break;
3505         default:
3506                 break;
3507         }
3508
3509         return r;
3510 }
3511
3512 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
3513 {
3514         vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3515         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3516         memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3517
3518         kvm_clear_async_pf_completion_queue(vcpu);
3519         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3520                 kvm_s390_vcpu_stop(vcpu);
3521         kvm_s390_clear_local_irqs(vcpu);
3522 }
3523
3524 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3525 {
3526         /* Initial reset is a superset of the normal reset */
3527         kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3528
3529         /*
3530          * This equals initial cpu reset in pop, but we don't switch to ESA.
3531          * We do not only reset the internal data, but also ...
3532          */
3533         vcpu->arch.sie_block->gpsw.mask = 0;
3534         vcpu->arch.sie_block->gpsw.addr = 0;
3535         kvm_s390_set_prefix(vcpu, 0);
3536         kvm_s390_set_cpu_timer(vcpu, 0);
3537         vcpu->arch.sie_block->ckc = 0;
3538         memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3539         vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3540         vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
3541
3542         /* ... the data in sync regs */
3543         memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
3544         vcpu->run->s.regs.ckc = 0;
3545         vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
3546         vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
3547         vcpu->run->psw_addr = 0;
3548         vcpu->run->psw_mask = 0;
3549         vcpu->run->s.regs.todpr = 0;
3550         vcpu->run->s.regs.cputm = 0;
3551         vcpu->run->s.regs.ckc = 0;
3552         vcpu->run->s.regs.pp = 0;
3553         vcpu->run->s.regs.gbea = 1;
3554         vcpu->run->s.regs.fpc = 0;
3555         /*
3556          * Do not reset these registers in the protected case, as some of
3557          * them are overlayed and they are not accessible in this case
3558          * anyway.
3559          */
3560         if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3561                 vcpu->arch.sie_block->gbea = 1;
3562                 vcpu->arch.sie_block->pp = 0;
3563                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3564                 vcpu->arch.sie_block->todpr = 0;
3565         }
3566 }
3567
3568 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3569 {
3570         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3571
3572         /* Clear reset is a superset of the initial reset */
3573         kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3574
3575         memset(&regs->gprs, 0, sizeof(regs->gprs));
3576         memset(&regs->vrs, 0, sizeof(regs->vrs));
3577         memset(&regs->acrs, 0, sizeof(regs->acrs));
3578         memset(&regs->gscb, 0, sizeof(regs->gscb));
3579
3580         regs->etoken = 0;
3581         regs->etoken_extension = 0;
3582 }
3583
3584 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3585 {
3586         vcpu_load(vcpu);
3587         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
3588         vcpu_put(vcpu);
3589         return 0;
3590 }
3591
3592 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3593 {
3594         vcpu_load(vcpu);
3595         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3596         vcpu_put(vcpu);
3597         return 0;
3598 }
3599
3600 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3601                                   struct kvm_sregs *sregs)
3602 {
3603         vcpu_load(vcpu);
3604
3605         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3606         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3607
3608         vcpu_put(vcpu);
3609         return 0;
3610 }
3611
3612 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3613                                   struct kvm_sregs *sregs)
3614 {
3615         vcpu_load(vcpu);
3616
3617         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3618         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3619
3620         vcpu_put(vcpu);
3621         return 0;
3622 }
3623
3624 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3625 {
3626         int ret = 0;
3627
3628         vcpu_load(vcpu);
3629
3630         if (test_fp_ctl(fpu->fpc)) {
3631                 ret = -EINVAL;
3632                 goto out;
3633         }
3634         vcpu->run->s.regs.fpc = fpu->fpc;
3635         if (MACHINE_HAS_VX)
3636                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3637                                  (freg_t *) fpu->fprs);
3638         else
3639                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3640
3641 out:
3642         vcpu_put(vcpu);
3643         return ret;
3644 }
3645
3646 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3647 {
3648         vcpu_load(vcpu);
3649
3650         /* make sure we have the latest values */
3651         save_fpu_regs();
3652         if (MACHINE_HAS_VX)
3653                 convert_vx_to_fp((freg_t *) fpu->fprs,
3654                                  (__vector128 *) vcpu->run->s.regs.vrs);
3655         else
3656                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3657         fpu->fpc = vcpu->run->s.regs.fpc;
3658
3659         vcpu_put(vcpu);
3660         return 0;
3661 }
3662
3663 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3664 {
3665         int rc = 0;
3666
3667         if (!is_vcpu_stopped(vcpu))
3668                 rc = -EBUSY;
3669         else {
3670                 vcpu->run->psw_mask = psw.mask;
3671                 vcpu->run->psw_addr = psw.addr;
3672         }
3673         return rc;
3674 }
3675
3676 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3677                                   struct kvm_translation *tr)
3678 {
3679         return -EINVAL; /* not implemented yet */
3680 }
3681
3682 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3683                               KVM_GUESTDBG_USE_HW_BP | \
3684                               KVM_GUESTDBG_ENABLE)
3685
3686 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3687                                         struct kvm_guest_debug *dbg)
3688 {
3689         int rc = 0;
3690
3691         vcpu_load(vcpu);
3692
3693         vcpu->guest_debug = 0;
3694         kvm_s390_clear_bp_data(vcpu);
3695
3696         if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3697                 rc = -EINVAL;
3698                 goto out;
3699         }
3700         if (!sclp.has_gpere) {
3701                 rc = -EINVAL;
3702                 goto out;
3703         }
3704
3705         if (dbg->control & KVM_GUESTDBG_ENABLE) {
3706                 vcpu->guest_debug = dbg->control;
3707                 /* enforce guest PER */
3708                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3709
3710                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3711                         rc = kvm_s390_import_bp_data(vcpu, dbg);
3712         } else {
3713                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3714                 vcpu->arch.guestdbg.last_bp = 0;
3715         }
3716
3717         if (rc) {
3718                 vcpu->guest_debug = 0;
3719                 kvm_s390_clear_bp_data(vcpu);
3720                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3721         }
3722
3723 out:
3724         vcpu_put(vcpu);
3725         return rc;
3726 }
3727
3728 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3729                                     struct kvm_mp_state *mp_state)
3730 {
3731         int ret;
3732
3733         vcpu_load(vcpu);
3734
3735         /* CHECK_STOP and LOAD are not supported yet */
3736         ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3737                                       KVM_MP_STATE_OPERATING;
3738
3739         vcpu_put(vcpu);
3740         return ret;
3741 }
3742
3743 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3744                                     struct kvm_mp_state *mp_state)
3745 {
3746         int rc = 0;
3747
3748         vcpu_load(vcpu);
3749
3750         /* user space knows about this interface - let it control the state */
3751         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3752
3753         switch (mp_state->mp_state) {
3754         case KVM_MP_STATE_STOPPED:
3755                 rc = kvm_s390_vcpu_stop(vcpu);
3756                 break;
3757         case KVM_MP_STATE_OPERATING:
3758                 rc = kvm_s390_vcpu_start(vcpu);
3759                 break;
3760         case KVM_MP_STATE_LOAD:
3761                 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3762                         rc = -ENXIO;
3763                         break;
3764                 }
3765                 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
3766                 break;
3767         case KVM_MP_STATE_CHECK_STOP:
3768                 fallthrough;    /* CHECK_STOP and LOAD are not supported yet */
3769         default:
3770                 rc = -ENXIO;
3771         }
3772
3773         vcpu_put(vcpu);
3774         return rc;
3775 }
3776
3777 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3778 {
3779         return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3780 }
3781
3782 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3783 {
3784 retry:
3785         kvm_s390_vcpu_request_handled(vcpu);
3786         if (!kvm_request_pending(vcpu))
3787                 return 0;
3788         /*
3789          * We use MMU_RELOAD just to re-arm the ipte notifier for the
3790          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3791          * This ensures that the ipte instruction for this request has
3792          * already finished. We might race against a second unmapper that
3793          * wants to set the blocking bit. Lets just retry the request loop.
3794          */
3795         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3796                 int rc;
3797                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3798                                           kvm_s390_get_prefix(vcpu),
3799                                           PAGE_SIZE * 2, PROT_WRITE);
3800                 if (rc) {
3801                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3802                         return rc;
3803                 }
3804                 goto retry;
3805         }
3806
3807         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3808                 vcpu->arch.sie_block->ihcpu = 0xffff;
3809                 goto retry;
3810         }
3811
3812         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3813                 if (!ibs_enabled(vcpu)) {
3814                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3815                         kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3816                 }
3817                 goto retry;
3818         }
3819
3820         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3821                 if (ibs_enabled(vcpu)) {
3822                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3823                         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3824                 }
3825                 goto retry;
3826         }
3827
3828         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3829                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3830                 goto retry;
3831         }
3832
3833         if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3834                 /*
3835                  * Disable CMM virtualization; we will emulate the ESSA
3836                  * instruction manually, in order to provide additional
3837                  * functionalities needed for live migration.
3838                  */
3839                 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3840                 goto retry;
3841         }
3842
3843         if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3844                 /*
3845                  * Re-enable CMM virtualization if CMMA is available and
3846                  * CMM has been used.
3847                  */
3848                 if ((vcpu->kvm->arch.use_cmma) &&
3849                     (vcpu->kvm->mm->context.uses_cmm))
3850                         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3851                 goto retry;
3852         }
3853
3854         /* nothing to do, just clear the request */
3855         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3856         /* we left the vsie handler, nothing to do, just clear the request */
3857         kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3858
3859         return 0;
3860 }
3861
3862 void kvm_s390_set_tod_clock(struct kvm *kvm,
3863                             const struct kvm_s390_vm_tod_clock *gtod)
3864 {
3865         struct kvm_vcpu *vcpu;
3866         struct kvm_s390_tod_clock_ext htod;
3867         int i;
3868
3869         mutex_lock(&kvm->lock);
3870         preempt_disable();
3871
3872         get_tod_clock_ext((char *)&htod);
3873
3874         kvm->arch.epoch = gtod->tod - htod.tod;
3875         kvm->arch.epdx = 0;
3876         if (test_kvm_facility(kvm, 139)) {
3877                 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3878                 if (kvm->arch.epoch > gtod->tod)
3879                         kvm->arch.epdx -= 1;
3880         }
3881
3882         kvm_s390_vcpu_block_all(kvm);
3883         kvm_for_each_vcpu(i, vcpu, kvm) {
3884                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3885                 vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
3886         }
3887
3888         kvm_s390_vcpu_unblock_all(kvm);
3889         preempt_enable();
3890         mutex_unlock(&kvm->lock);
3891 }
3892
3893 /**
3894  * kvm_arch_fault_in_page - fault-in guest page if necessary
3895  * @vcpu: The corresponding virtual cpu
3896  * @gpa: Guest physical address
3897  * @writable: Whether the page should be writable or not
3898  *
3899  * Make sure that a guest page has been faulted-in on the host.
3900  *
3901  * Return: Zero on success, negative error code otherwise.
3902  */
3903 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3904 {
3905         return gmap_fault(vcpu->arch.gmap, gpa,
3906                           writable ? FAULT_FLAG_WRITE : 0);
3907 }
3908
3909 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3910                                       unsigned long token)
3911 {
3912         struct kvm_s390_interrupt inti;
3913         struct kvm_s390_irq irq;
3914
3915         if (start_token) {
3916                 irq.u.ext.ext_params2 = token;
3917                 irq.type = KVM_S390_INT_PFAULT_INIT;
3918                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3919         } else {
3920                 inti.type = KVM_S390_INT_PFAULT_DONE;
3921                 inti.parm64 = token;
3922                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3923         }
3924 }
3925
3926 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3927                                      struct kvm_async_pf *work)
3928 {
3929         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3930         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3931 }
3932
3933 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3934                                  struct kvm_async_pf *work)
3935 {
3936         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3937         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3938 }
3939
3940 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3941                                struct kvm_async_pf *work)
3942 {
3943         /* s390 will always inject the page directly */
3944 }
3945
3946 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3947 {
3948         /*
3949          * s390 will always inject the page directly,
3950          * but we still want check_async_completion to cleanup
3951          */
3952         return true;
3953 }
3954
3955 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3956 {
3957         hva_t hva;
3958         struct kvm_arch_async_pf arch;
3959         int rc;
3960
3961         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3962                 return 0;
3963         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3964             vcpu->arch.pfault_compare)
3965                 return 0;
3966         if (psw_extint_disabled(vcpu))
3967                 return 0;
3968         if (kvm_s390_vcpu_has_irq(vcpu, 0))
3969                 return 0;
3970         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3971                 return 0;
3972         if (!vcpu->arch.gmap->pfault_enabled)
3973                 return 0;
3974
3975         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3976         hva += current->thread.gmap_addr & ~PAGE_MASK;
3977         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3978                 return 0;
3979
3980         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3981         return rc;
3982 }
3983
3984 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3985 {
3986         int rc, cpuflags;
3987
3988         /*
3989          * On s390 notifications for arriving pages will be delivered directly
3990          * to the guest but the house keeping for completed pfaults is
3991          * handled outside the worker.
3992          */
3993         kvm_check_async_pf_completion(vcpu);
3994
3995         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3996         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3997
3998         if (need_resched())
3999                 schedule();
4000
4001         if (test_cpu_flag(CIF_MCCK_PENDING))
4002                 s390_handle_mcck();
4003
4004         if (!kvm_is_ucontrol(vcpu->kvm)) {
4005                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4006                 if (rc)
4007                         return rc;
4008         }
4009
4010         rc = kvm_s390_handle_requests(vcpu);
4011         if (rc)
4012                 return rc;
4013
4014         if (guestdbg_enabled(vcpu)) {
4015                 kvm_s390_backup_guest_per_regs(vcpu);
4016                 kvm_s390_patch_guest_per_regs(vcpu);
4017         }
4018
4019         clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask);
4020
4021         vcpu->arch.sie_block->icptcode = 0;
4022         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4023         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4024         trace_kvm_s390_sie_enter(vcpu, cpuflags);
4025
4026         return 0;
4027 }
4028
4029 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
4030 {
4031         struct kvm_s390_pgm_info pgm_info = {
4032                 .code = PGM_ADDRESSING,
4033         };
4034         u8 opcode, ilen;
4035         int rc;
4036
4037         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4038         trace_kvm_s390_sie_fault(vcpu);
4039
4040         /*
4041          * We want to inject an addressing exception, which is defined as a
4042          * suppressing or terminating exception. However, since we came here
4043          * by a DAT access exception, the PSW still points to the faulting
4044          * instruction since DAT exceptions are nullifying. So we've got
4045          * to look up the current opcode to get the length of the instruction
4046          * to be able to forward the PSW.
4047          */
4048         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4049         ilen = insn_length(opcode);
4050         if (rc < 0) {
4051                 return rc;
4052         } else if (rc) {
4053                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4054                  * Forward by arbitrary ilc, injection will take care of
4055                  * nullification if necessary.
4056                  */
4057                 pgm_info = vcpu->arch.pgm;
4058                 ilen = 4;
4059         }
4060         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4061         kvm_s390_forward_psw(vcpu, ilen);
4062         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4063 }
4064
4065 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4066 {
4067         struct mcck_volatile_info *mcck_info;
4068         struct sie_page *sie_page;
4069
4070         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4071                    vcpu->arch.sie_block->icptcode);
4072         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4073
4074         if (guestdbg_enabled(vcpu))
4075                 kvm_s390_restore_guest_per_regs(vcpu);
4076
4077         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4078         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4079
4080         if (exit_reason == -EINTR) {
4081                 VCPU_EVENT(vcpu, 3, "%s", "machine check");
4082                 sie_page = container_of(vcpu->arch.sie_block,
4083                                         struct sie_page, sie_block);
4084                 mcck_info = &sie_page->mcck_info;
4085                 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4086                 return 0;
4087         }
4088
4089         if (vcpu->arch.sie_block->icptcode > 0) {
4090                 int rc = kvm_handle_sie_intercept(vcpu);
4091
4092                 if (rc != -EOPNOTSUPP)
4093                         return rc;
4094                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4095                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4096                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4097                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4098                 return -EREMOTE;
4099         } else if (exit_reason != -EFAULT) {
4100                 vcpu->stat.exit_null++;
4101                 return 0;
4102         } else if (kvm_is_ucontrol(vcpu->kvm)) {
4103                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4104                 vcpu->run->s390_ucontrol.trans_exc_code =
4105                                                 current->thread.gmap_addr;
4106                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
4107                 return -EREMOTE;
4108         } else if (current->thread.gmap_pfault) {
4109                 trace_kvm_s390_major_guest_pfault(vcpu);
4110                 current->thread.gmap_pfault = 0;
4111                 if (kvm_arch_setup_async_pf(vcpu))
4112                         return 0;
4113                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
4114         }
4115         return vcpu_post_run_fault_in_sie(vcpu);
4116 }
4117
4118 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
4119 static int __vcpu_run(struct kvm_vcpu *vcpu)
4120 {
4121         int rc, exit_reason;
4122         struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4123
4124         /*
4125          * We try to hold kvm->srcu during most of vcpu_run (except when run-
4126          * ning the guest), so that memslots (and other stuff) are protected
4127          */
4128         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4129
4130         do {
4131                 rc = vcpu_pre_run(vcpu);
4132                 if (rc)
4133                         break;
4134
4135                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
4136                 /*
4137                  * As PF_VCPU will be used in fault handler, between
4138                  * guest_enter and guest_exit should be no uaccess.
4139                  */
4140                 local_irq_disable();
4141                 guest_enter_irqoff();
4142                 __disable_cpu_timer_accounting(vcpu);
4143                 local_irq_enable();
4144                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4145                         memcpy(sie_page->pv_grregs,
4146                                vcpu->run->s.regs.gprs,
4147                                sizeof(sie_page->pv_grregs));
4148                 }
4149                 exit_reason = sie64a(vcpu->arch.sie_block,
4150                                      vcpu->run->s.regs.gprs);
4151                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4152                         memcpy(vcpu->run->s.regs.gprs,
4153                                sie_page->pv_grregs,
4154                                sizeof(sie_page->pv_grregs));
4155                         /*
4156                          * We're not allowed to inject interrupts on intercepts
4157                          * that leave the guest state in an "in-between" state
4158                          * where the next SIE entry will do a continuation.
4159                          * Fence interrupts in our "internal" PSW.
4160                          */
4161                         if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4162                             vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4163                                 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4164                         }
4165                 }
4166                 local_irq_disable();
4167                 __enable_cpu_timer_accounting(vcpu);
4168                 guest_exit_irqoff();
4169                 local_irq_enable();
4170                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4171
4172                 rc = vcpu_post_run(vcpu, exit_reason);
4173         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
4174
4175         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
4176         return rc;
4177 }
4178
4179 static void sync_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4180 {
4181         struct runtime_instr_cb *riccb;
4182         struct gs_cb *gscb;
4183
4184         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4185         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
4186         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4187         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
4188         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4189                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4190                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4191                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4192         }
4193         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4194                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4195                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4196                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
4197                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4198                         kvm_clear_async_pf_completion_queue(vcpu);
4199         }
4200         /*
4201          * If userspace sets the riccb (e.g. after migration) to a valid state,
4202          * we should enable RI here instead of doing the lazy enablement.
4203          */
4204         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4205             test_kvm_facility(vcpu->kvm, 64) &&
4206             riccb->v &&
4207             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4208                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
4209                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
4210         }
4211         /*
4212          * If userspace sets the gscb (e.g. after migration) to non-zero,
4213          * we should enable GS here instead of doing the lazy enablement.
4214          */
4215         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4216             test_kvm_facility(vcpu->kvm, 133) &&
4217             gscb->gssm &&
4218             !vcpu->arch.gs_enabled) {
4219                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4220                 vcpu->arch.sie_block->ecb |= ECB_GS;
4221                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4222                 vcpu->arch.gs_enabled = 1;
4223         }
4224         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4225             test_kvm_facility(vcpu->kvm, 82)) {
4226                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4227                 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4228         }
4229         if (MACHINE_HAS_GS) {
4230                 preempt_disable();
4231                 __ctl_set_bit(2, 4);
4232                 if (current->thread.gs_cb) {
4233                         vcpu->arch.host_gscb = current->thread.gs_cb;
4234                         save_gs_cb(vcpu->arch.host_gscb);
4235                 }
4236                 if (vcpu->arch.gs_enabled) {
4237                         current->thread.gs_cb = (struct gs_cb *)
4238                                                 &vcpu->run->s.regs.gscb;
4239                         restore_gs_cb(current->thread.gs_cb);
4240                 }
4241                 preempt_enable();
4242         }
4243         /* SIE will load etoken directly from SDNX and therefore kvm_run */
4244 }
4245
4246 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4247 {
4248         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4249                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4250         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4251                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4252                 /* some control register changes require a tlb flush */
4253                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4254         }
4255         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4256                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4257                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4258         }
4259         save_access_regs(vcpu->arch.host_acrs);
4260         restore_access_regs(vcpu->run->s.regs.acrs);
4261         /* save host (userspace) fprs/vrs */
4262         save_fpu_regs();
4263         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
4264         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
4265         if (MACHINE_HAS_VX)
4266                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
4267         else
4268                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
4269         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
4270         if (test_fp_ctl(current->thread.fpu.fpc))
4271                 /* User space provided an invalid FPC, let's clear it */
4272                 current->thread.fpu.fpc = 0;
4273
4274         /* Sync fmt2 only data */
4275         if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4276                 sync_regs_fmt2(vcpu, kvm_run);
4277         } else {
4278                 /*
4279                  * In several places we have to modify our internal view to
4280                  * not do things that are disallowed by the ultravisor. For
4281                  * example we must not inject interrupts after specific exits
4282                  * (e.g. 112 prefix page not secure). We do this by turning
4283                  * off the machine check, external and I/O interrupt bits
4284                  * of our PSW copy. To avoid getting validity intercepts, we
4285                  * do only accept the condition code from userspace.
4286                  */
4287                 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4288                 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4289                                                    PSW_MASK_CC;
4290         }
4291
4292         kvm_run->kvm_dirty_regs = 0;
4293 }
4294
4295 static void store_regs_fmt2(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4296 {
4297         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4298         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4299         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4300         kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4301         if (MACHINE_HAS_GS) {
4302                 __ctl_set_bit(2, 4);
4303                 if (vcpu->arch.gs_enabled)
4304                         save_gs_cb(current->thread.gs_cb);
4305                 preempt_disable();
4306                 current->thread.gs_cb = vcpu->arch.host_gscb;
4307                 restore_gs_cb(vcpu->arch.host_gscb);
4308                 preempt_enable();
4309                 if (!vcpu->arch.host_gscb)
4310                         __ctl_clear_bit(2, 4);
4311                 vcpu->arch.host_gscb = NULL;
4312         }
4313         /* SIE will save etoken directly into SDNX and therefore kvm_run */
4314 }
4315
4316 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4317 {
4318         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4319         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4320         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4321         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4322         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
4323         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
4324         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4325         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4326         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
4327         save_access_regs(vcpu->run->s.regs.acrs);
4328         restore_access_regs(vcpu->arch.host_acrs);
4329         /* Save guest register state */
4330         save_fpu_regs();
4331         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4332         /* Restore will be done lazily at return */
4333         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4334         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4335         if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
4336                 store_regs_fmt2(vcpu, kvm_run);
4337 }
4338
4339 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4340 {
4341         int rc;
4342
4343         if (kvm_run->immediate_exit)
4344                 return -EINTR;
4345
4346         if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4347             kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4348                 return -EINVAL;
4349
4350         vcpu_load(vcpu);
4351
4352         if (guestdbg_exit_pending(vcpu)) {
4353                 kvm_s390_prepare_debug_exit(vcpu);
4354                 rc = 0;
4355                 goto out;
4356         }
4357
4358         kvm_sigset_activate(vcpu);
4359
4360         /*
4361          * no need to check the return value of vcpu_start as it can only have
4362          * an error for protvirt, but protvirt means user cpu state
4363          */
4364         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4365                 kvm_s390_vcpu_start(vcpu);
4366         } else if (is_vcpu_stopped(vcpu)) {
4367                 pr_err_ratelimited("can't run stopped vcpu %d\n",
4368                                    vcpu->vcpu_id);
4369                 rc = -EINVAL;
4370                 goto out;
4371         }
4372
4373         sync_regs(vcpu, kvm_run);
4374         enable_cpu_timer_accounting(vcpu);
4375
4376         might_fault();
4377         rc = __vcpu_run(vcpu);
4378
4379         if (signal_pending(current) && !rc) {
4380                 kvm_run->exit_reason = KVM_EXIT_INTR;
4381                 rc = -EINTR;
4382         }
4383
4384         if (guestdbg_exit_pending(vcpu) && !rc)  {
4385                 kvm_s390_prepare_debug_exit(vcpu);
4386                 rc = 0;
4387         }
4388
4389         if (rc == -EREMOTE) {
4390                 /* userspace support is needed, kvm_run has been prepared */
4391                 rc = 0;
4392         }
4393
4394         disable_cpu_timer_accounting(vcpu);
4395         store_regs(vcpu, kvm_run);
4396
4397         kvm_sigset_deactivate(vcpu);
4398
4399         vcpu->stat.exit_userspace++;
4400 out:
4401         vcpu_put(vcpu);
4402         return rc;
4403 }
4404
4405 /*
4406  * store status at address
4407  * we use have two special cases:
4408  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4409  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4410  */
4411 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
4412 {
4413         unsigned char archmode = 1;
4414         freg_t fprs[NUM_FPRS];
4415         unsigned int px;
4416         u64 clkcomp, cputm;
4417         int rc;
4418
4419         px = kvm_s390_get_prefix(vcpu);
4420         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4421                 if (write_guest_abs(vcpu, 163, &archmode, 1))
4422                         return -EFAULT;
4423                 gpa = 0;
4424         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4425                 if (write_guest_real(vcpu, 163, &archmode, 1))
4426                         return -EFAULT;
4427                 gpa = px;
4428         } else
4429                 gpa -= __LC_FPREGS_SAVE_AREA;
4430
4431         /* manually convert vector registers if necessary */
4432         if (MACHINE_HAS_VX) {
4433                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
4434                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4435                                      fprs, 128);
4436         } else {
4437                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4438                                      vcpu->run->s.regs.fprs, 128);
4439         }
4440         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
4441                               vcpu->run->s.regs.gprs, 128);
4442         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
4443                               &vcpu->arch.sie_block->gpsw, 16);
4444         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
4445                               &px, 4);
4446         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
4447                               &vcpu->run->s.regs.fpc, 4);
4448         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
4449                               &vcpu->arch.sie_block->todpr, 4);
4450         cputm = kvm_s390_get_cpu_timer(vcpu);
4451         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4452                               &cputm, 8);
4453         clkcomp = vcpu->arch.sie_block->ckc >> 8;
4454         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
4455                               &clkcomp, 8);
4456         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
4457                               &vcpu->run->s.regs.acrs, 64);
4458         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
4459                               &vcpu->arch.sie_block->gcr, 128);
4460         return rc ? -EFAULT : 0;
4461 }
4462
4463 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4464 {
4465         /*
4466          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
4467          * switch in the run ioctl. Let's update our copies before we save
4468          * it into the save area
4469          */
4470         save_fpu_regs();
4471         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4472         save_access_regs(vcpu->run->s.regs.acrs);
4473
4474         return kvm_s390_store_status_unloaded(vcpu, addr);
4475 }
4476
4477 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4478 {
4479         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
4480         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
4481 }
4482
4483 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4484 {
4485         unsigned int i;
4486         struct kvm_vcpu *vcpu;
4487
4488         kvm_for_each_vcpu(i, vcpu, kvm) {
4489                 __disable_ibs_on_vcpu(vcpu);
4490         }
4491 }
4492
4493 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4494 {
4495         if (!sclp.has_ibs)
4496                 return;
4497         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
4498         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
4499 }
4500
4501 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4502 {
4503         int i, online_vcpus, r = 0, started_vcpus = 0;
4504
4505         if (!is_vcpu_stopped(vcpu))
4506                 return 0;
4507
4508         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
4509         /* Only one cpu at a time may enter/leave the STOPPED state. */
4510         spin_lock(&vcpu->kvm->arch.start_stop_lock);
4511         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4512
4513         /* Let's tell the UV that we want to change into the operating state */
4514         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4515                 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
4516                 if (r) {
4517                         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4518                         return r;
4519                 }
4520         }
4521
4522         for (i = 0; i < online_vcpus; i++) {
4523                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4524                         started_vcpus++;
4525         }
4526
4527         if (started_vcpus == 0) {
4528                 /* we're the only active VCPU -> speed it up */
4529                 __enable_ibs_on_vcpu(vcpu);
4530         } else if (started_vcpus == 1) {
4531                 /*
4532                  * As we are starting a second VCPU, we have to disable
4533                  * the IBS facility on all VCPUs to remove potentially
4534                  * oustanding ENABLE requests.
4535                  */
4536                 __disable_ibs_on_all_vcpus(vcpu->kvm);
4537         }
4538
4539         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
4540         /*
4541          * The real PSW might have changed due to a RESTART interpreted by the
4542          * ultravisor. We block all interrupts and let the next sie exit
4543          * refresh our view.
4544          */
4545         if (kvm_s390_pv_cpu_is_protected(vcpu))
4546                 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4547         /*
4548          * Another VCPU might have used IBS while we were offline.
4549          * Let's play safe and flush the VCPU at startup.
4550          */
4551         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4552         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4553         return 0;
4554 }
4555
4556 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4557 {
4558         int i, online_vcpus, r = 0, started_vcpus = 0;
4559         struct kvm_vcpu *started_vcpu = NULL;
4560
4561         if (is_vcpu_stopped(vcpu))
4562                 return 0;
4563
4564         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
4565         /* Only one cpu at a time may enter/leave the STOPPED state. */
4566         spin_lock(&vcpu->kvm->arch.start_stop_lock);
4567         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4568
4569         /* Let's tell the UV that we want to change into the stopped state */
4570         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4571                 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
4572                 if (r) {
4573                         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4574                         return r;
4575                 }
4576         }
4577
4578         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
4579         kvm_s390_clear_stop_irq(vcpu);
4580
4581         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
4582         __disable_ibs_on_vcpu(vcpu);
4583
4584         for (i = 0; i < online_vcpus; i++) {
4585                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4586                         started_vcpus++;
4587                         started_vcpu = vcpu->kvm->vcpus[i];
4588                 }
4589         }
4590
4591         if (started_vcpus == 1) {
4592                 /*
4593                  * As we only have one VCPU left, we want to enable the
4594                  * IBS facility for that VCPU to speed it up.
4595                  */
4596                 __enable_ibs_on_vcpu(started_vcpu);
4597         }
4598
4599         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4600         return 0;
4601 }
4602
4603 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4604                                      struct kvm_enable_cap *cap)
4605 {
4606         int r;
4607
4608         if (cap->flags)
4609                 return -EINVAL;
4610
4611         switch (cap->cap) {
4612         case KVM_CAP_S390_CSS_SUPPORT:
4613                 if (!vcpu->kvm->arch.css_support) {
4614                         vcpu->kvm->arch.css_support = 1;
4615                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4616                         trace_kvm_s390_enable_css(vcpu->kvm);
4617                 }
4618                 r = 0;
4619                 break;
4620         default:
4621                 r = -EINVAL;
4622                 break;
4623         }
4624         return r;
4625 }
4626
4627 static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu,
4628                                    struct kvm_s390_mem_op *mop)
4629 {
4630         void __user *uaddr = (void __user *)mop->buf;
4631         int r = 0;
4632
4633         if (mop->flags || !mop->size)
4634                 return -EINVAL;
4635         if (mop->size + mop->sida_offset < mop->size)
4636                 return -EINVAL;
4637         if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
4638                 return -E2BIG;
4639
4640         switch (mop->op) {
4641         case KVM_S390_MEMOP_SIDA_READ:
4642                 if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
4643                                  mop->sida_offset), mop->size))
4644                         r = -EFAULT;
4645
4646                 break;
4647         case KVM_S390_MEMOP_SIDA_WRITE:
4648                 if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
4649                                    mop->sida_offset), uaddr, mop->size))
4650                         r = -EFAULT;
4651                 break;
4652         }
4653         return r;
4654 }
4655 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4656                                   struct kvm_s390_mem_op *mop)
4657 {
4658         void __user *uaddr = (void __user *)mop->buf;
4659         void *tmpbuf = NULL;
4660         int r = 0;
4661         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4662                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
4663
4664         if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4665                 return -EINVAL;
4666
4667         if (mop->size > MEM_OP_MAX_SIZE)
4668                 return -E2BIG;
4669
4670         if (kvm_s390_pv_cpu_is_protected(vcpu))
4671                 return -EINVAL;
4672
4673         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4674                 tmpbuf = vmalloc(mop->size);
4675                 if (!tmpbuf)
4676                         return -ENOMEM;
4677         }
4678
4679         switch (mop->op) {
4680         case KVM_S390_MEMOP_LOGICAL_READ:
4681                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4682                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4683                                             mop->size, GACC_FETCH);
4684                         break;
4685                 }
4686                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4687                 if (r == 0) {
4688                         if (copy_to_user(uaddr, tmpbuf, mop->size))
4689                                 r = -EFAULT;
4690                 }
4691                 break;
4692         case KVM_S390_MEMOP_LOGICAL_WRITE:
4693                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4694                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4695                                             mop->size, GACC_STORE);
4696                         break;
4697                 }
4698                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4699                         r = -EFAULT;
4700                         break;
4701                 }
4702                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4703                 break;
4704         }
4705
4706         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4707                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4708
4709         vfree(tmpbuf);
4710         return r;
4711 }
4712
4713 static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu,
4714                                       struct kvm_s390_mem_op *mop)
4715 {
4716         int r, srcu_idx;
4717
4718         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4719
4720         switch (mop->op) {
4721         case KVM_S390_MEMOP_LOGICAL_READ:
4722         case KVM_S390_MEMOP_LOGICAL_WRITE:
4723                 r = kvm_s390_guest_mem_op(vcpu, mop);
4724                 break;
4725         case KVM_S390_MEMOP_SIDA_READ:
4726         case KVM_S390_MEMOP_SIDA_WRITE:
4727                 /* we are locked against sida going away by the vcpu->mutex */
4728                 r = kvm_s390_guest_sida_op(vcpu, mop);
4729                 break;
4730         default:
4731                 r = -EINVAL;
4732         }
4733
4734         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4735         return r;
4736 }
4737
4738 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4739                                unsigned int ioctl, unsigned long arg)
4740 {
4741         struct kvm_vcpu *vcpu = filp->private_data;
4742         void __user *argp = (void __user *)arg;
4743
4744         switch (ioctl) {
4745         case KVM_S390_IRQ: {
4746                 struct kvm_s390_irq s390irq;
4747
4748                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4749                         return -EFAULT;
4750                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4751         }
4752         case KVM_S390_INTERRUPT: {
4753                 struct kvm_s390_interrupt s390int;
4754                 struct kvm_s390_irq s390irq = {};
4755
4756                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4757                         return -EFAULT;
4758                 if (s390int_to_s390irq(&s390int, &s390irq))
4759                         return -EINVAL;
4760                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4761         }
4762         }
4763         return -ENOIOCTLCMD;
4764 }
4765
4766 long kvm_arch_vcpu_ioctl(struct file *filp,
4767                          unsigned int ioctl, unsigned long arg)
4768 {
4769         struct kvm_vcpu *vcpu = filp->private_data;
4770         void __user *argp = (void __user *)arg;
4771         int idx;
4772         long r;
4773         u16 rc, rrc;
4774
4775         vcpu_load(vcpu);
4776
4777         switch (ioctl) {
4778         case KVM_S390_STORE_STATUS:
4779                 idx = srcu_read_lock(&vcpu->kvm->srcu);
4780                 r = kvm_s390_store_status_unloaded(vcpu, arg);
4781                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4782                 break;
4783         case KVM_S390_SET_INITIAL_PSW: {
4784                 psw_t psw;
4785
4786                 r = -EFAULT;
4787                 if (copy_from_user(&psw, argp, sizeof(psw)))
4788                         break;
4789                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4790                 break;
4791         }
4792         case KVM_S390_CLEAR_RESET:
4793                 r = 0;
4794                 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4795                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4796                         r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4797                                           UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
4798                         VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
4799                                    rc, rrc);
4800                 }
4801                 break;
4802         case KVM_S390_INITIAL_RESET:
4803                 r = 0;
4804                 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4805                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4806                         r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4807                                           UVC_CMD_CPU_RESET_INITIAL,
4808                                           &rc, &rrc);
4809                         VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
4810                                    rc, rrc);
4811                 }
4812                 break;
4813         case KVM_S390_NORMAL_RESET:
4814                 r = 0;
4815                 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4816                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4817                         r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4818                                           UVC_CMD_CPU_RESET, &rc, &rrc);
4819                         VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
4820                                    rc, rrc);
4821                 }
4822                 break;
4823         case KVM_SET_ONE_REG:
4824         case KVM_GET_ONE_REG: {
4825                 struct kvm_one_reg reg;
4826                 r = -EINVAL;
4827                 if (kvm_s390_pv_cpu_is_protected(vcpu))
4828                         break;
4829                 r = -EFAULT;
4830                 if (copy_from_user(&reg, argp, sizeof(reg)))
4831                         break;
4832                 if (ioctl == KVM_SET_ONE_REG)
4833                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4834                 else
4835                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4836                 break;
4837         }
4838 #ifdef CONFIG_KVM_S390_UCONTROL
4839         case KVM_S390_UCAS_MAP: {
4840                 struct kvm_s390_ucas_mapping ucasmap;
4841
4842                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4843                         r = -EFAULT;
4844                         break;
4845                 }
4846
4847                 if (!kvm_is_ucontrol(vcpu->kvm)) {
4848                         r = -EINVAL;
4849                         break;
4850                 }
4851
4852                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4853                                      ucasmap.vcpu_addr, ucasmap.length);
4854                 break;
4855         }
4856         case KVM_S390_UCAS_UNMAP: {
4857                 struct kvm_s390_ucas_mapping ucasmap;
4858
4859                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4860                         r = -EFAULT;
4861                         break;
4862                 }
4863
4864                 if (!kvm_is_ucontrol(vcpu->kvm)) {
4865                         r = -EINVAL;
4866                         break;
4867                 }
4868
4869                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4870                         ucasmap.length);
4871                 break;
4872         }
4873 #endif
4874         case KVM_S390_VCPU_FAULT: {
4875                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4876                 break;
4877         }
4878         case KVM_ENABLE_CAP:
4879         {
4880                 struct kvm_enable_cap cap;
4881                 r = -EFAULT;
4882                 if (copy_from_user(&cap, argp, sizeof(cap)))
4883                         break;
4884                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4885                 break;
4886         }
4887         case KVM_S390_MEM_OP: {
4888                 struct kvm_s390_mem_op mem_op;
4889
4890                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4891                         r = kvm_s390_guest_memsida_op(vcpu, &mem_op);
4892                 else
4893                         r = -EFAULT;
4894                 break;
4895         }
4896         case KVM_S390_SET_IRQ_STATE: {
4897                 struct kvm_s390_irq_state irq_state;
4898
4899                 r = -EFAULT;
4900                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4901                         break;
4902                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4903                     irq_state.len == 0 ||
4904                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4905                         r = -EINVAL;
4906                         break;
4907                 }
4908                 /* do not use irq_state.flags, it will break old QEMUs */
4909                 r = kvm_s390_set_irq_state(vcpu,
4910                                            (void __user *) irq_state.buf,
4911                                            irq_state.len);
4912                 break;
4913         }
4914         case KVM_S390_GET_IRQ_STATE: {
4915                 struct kvm_s390_irq_state irq_state;
4916
4917                 r = -EFAULT;
4918                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4919                         break;
4920                 if (irq_state.len == 0) {
4921                         r = -EINVAL;
4922                         break;
4923                 }
4924                 /* do not use irq_state.flags, it will break old QEMUs */
4925                 r = kvm_s390_get_irq_state(vcpu,
4926                                            (__u8 __user *)  irq_state.buf,
4927                                            irq_state.len);
4928                 break;
4929         }
4930         default:
4931                 r = -ENOTTY;
4932         }
4933
4934         vcpu_put(vcpu);
4935         return r;
4936 }
4937
4938 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4939 {
4940 #ifdef CONFIG_KVM_S390_UCONTROL
4941         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4942                  && (kvm_is_ucontrol(vcpu->kvm))) {
4943                 vmf->page = virt_to_page(vcpu->arch.sie_block);
4944                 get_page(vmf->page);
4945                 return 0;
4946         }
4947 #endif
4948         return VM_FAULT_SIGBUS;
4949 }
4950
4951 /* Section: memory related */
4952 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4953                                    struct kvm_memory_slot *memslot,
4954                                    const struct kvm_userspace_memory_region *mem,
4955                                    enum kvm_mr_change change)
4956 {
4957         /* A few sanity checks. We can have memory slots which have to be
4958            located/ended at a segment boundary (1MB). The memory in userland is
4959            ok to be fragmented into various different vmas. It is okay to mmap()
4960            and munmap() stuff in this slot after doing this call at any time */
4961
4962         if (mem->userspace_addr & 0xffffful)
4963                 return -EINVAL;
4964
4965         if (mem->memory_size & 0xffffful)
4966                 return -EINVAL;
4967
4968         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4969                 return -EINVAL;
4970
4971         /* When we are protected, we should not change the memory slots */
4972         if (kvm_s390_pv_get_handle(kvm))
4973                 return -EINVAL;
4974         return 0;
4975 }
4976
4977 void kvm_arch_commit_memory_region(struct kvm *kvm,
4978                                 const struct kvm_userspace_memory_region *mem,
4979                                 struct kvm_memory_slot *old,
4980                                 const struct kvm_memory_slot *new,
4981                                 enum kvm_mr_change change)
4982 {
4983         int rc = 0;
4984
4985         switch (change) {
4986         case KVM_MR_DELETE:
4987                 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4988                                         old->npages * PAGE_SIZE);
4989                 break;
4990         case KVM_MR_MOVE:
4991                 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4992                                         old->npages * PAGE_SIZE);
4993                 if (rc)
4994                         break;
4995                 fallthrough;
4996         case KVM_MR_CREATE:
4997                 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4998                                       mem->guest_phys_addr, mem->memory_size);
4999                 break;
5000         case KVM_MR_FLAGS_ONLY:
5001                 break;
5002         default:
5003                 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
5004         }
5005         if (rc)
5006                 pr_warn("failed to commit memory region\n");
5007         return;
5008 }
5009
5010 static inline unsigned long nonhyp_mask(int i)
5011 {
5012         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5013
5014         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5015 }
5016
5017 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
5018 {
5019         vcpu->valid_wakeup = false;
5020 }
5021
5022 static int __init kvm_s390_init(void)
5023 {
5024         int i;
5025
5026         if (!sclp.has_sief2) {
5027                 pr_info("SIE is not available\n");
5028                 return -ENODEV;
5029         }
5030
5031         if (nested && hpage) {
5032                 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
5033                 return -EINVAL;
5034         }
5035
5036         for (i = 0; i < 16; i++)
5037                 kvm_s390_fac_base[i] |=
5038                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
5039
5040         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
5041 }
5042
5043 static void __exit kvm_s390_exit(void)
5044 {
5045         kvm_exit();
5046 }
5047
5048 module_init(kvm_s390_init);
5049 module_exit(kvm_s390_exit);
5050
5051 /*
5052  * Enable autoloading of the kvm module.
5053  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5054  * since x86 takes a different approach.
5055  */
5056 #include <linux/miscdevice.h>
5057 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5058 MODULE_ALIAS("devname:kvm");