arch/s390/kvm/kvm-s390.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * hosting IBM Z kernel virtual machines (s390x)
   4  *
   5  * Copyright IBM Corp. 2008, 2020
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  *               Christian Borntraeger <borntraeger@de.ibm.com>
   9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  11  *               Jason J. Herne <jjherne@us.ibm.com>
  12  */
  13
  14 #define KMSG_COMPONENT "kvm-s390"
  15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/random.h>
  28 #include <linux/slab.h>
  29 #include <linux/timer.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/bitmap.h>
  32 #include <linux/sched/signal.h>
  33 #include <linux/string.h>
  34 #include <linux/pgtable.h>
  35
  36 #include <asm/asm-offsets.h>
  37 #include <asm/lowcore.h>
  38 #include <asm/stp.h>
  39 #include <asm/gmap.h>
  40 #include <asm/nmi.h>
  41 #include <asm/switch_to.h>
  42 #include <asm/isc.h>
  43 #include <asm/sclp.h>
  44 #include <asm/cpacf.h>
  45 #include <asm/timex.h>
  46 #include <asm/ap.h>
  47 #include <asm/uv.h>
  48 #include "kvm-s390.h"
  49 #include "gaccess.h"
  50
  51 #define CREATE_TRACE_POINTS
  52 #include "trace.h"
  53 #include "trace-s390.h"
  54
  55 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  56 #define LOCAL_IRQS 32
  57 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  58                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  59
  60 struct kvm_stats_debugfs_item debugfs_entries[] = {
  61         VCPU_STAT("userspace_handled", exit_userspace),
  62         VCPU_STAT("exit_null", exit_null),
  63         VCPU_STAT("pfault_sync", pfault_sync),
  64         VCPU_STAT("exit_validity", exit_validity),
  65         VCPU_STAT("exit_stop_request", exit_stop_request),
  66         VCPU_STAT("exit_external_request", exit_external_request),
  67         VCPU_STAT("exit_io_request", exit_io_request),
  68         VCPU_STAT("exit_external_interrupt", exit_external_interrupt),
  69         VCPU_STAT("exit_instruction", exit_instruction),
  70         VCPU_STAT("exit_pei", exit_pei),
  71         VCPU_STAT("exit_program_interruption", exit_program_interruption),
  72         VCPU_STAT("exit_instr_and_program_int", exit_instr_and_program),
  73         VCPU_STAT("exit_operation_exception", exit_operation_exception),
  74         VCPU_STAT("halt_successful_poll", halt_successful_poll),
  75         VCPU_STAT("halt_attempted_poll", halt_attempted_poll),
  76         VCPU_STAT("halt_poll_invalid", halt_poll_invalid),
  77         VCPU_STAT("halt_no_poll_steal", halt_no_poll_steal),
  78         VCPU_STAT("halt_wakeup", halt_wakeup),
  79         VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns),
  80         VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns),
  81         VCPU_STAT("instruction_lctlg", instruction_lctlg),
  82         VCPU_STAT("instruction_lctl", instruction_lctl),
  83         VCPU_STAT("instruction_stctl", instruction_stctl),
  84         VCPU_STAT("instruction_stctg", instruction_stctg),
  85         VCPU_STAT("deliver_ckc", deliver_ckc),
  86         VCPU_STAT("deliver_cputm", deliver_cputm),
  87         VCPU_STAT("deliver_emergency_signal", deliver_emergency_signal),
  88         VCPU_STAT("deliver_external_call", deliver_external_call),
  89         VCPU_STAT("deliver_service_signal", deliver_service_signal),
  90         VCPU_STAT("deliver_virtio", deliver_virtio),
  91         VCPU_STAT("deliver_stop_signal", deliver_stop_signal),
  92         VCPU_STAT("deliver_prefix_signal", deliver_prefix_signal),
  93         VCPU_STAT("deliver_restart_signal", deliver_restart_signal),
  94         VCPU_STAT("deliver_program", deliver_program),
  95         VCPU_STAT("deliver_io", deliver_io),
  96         VCPU_STAT("deliver_machine_check", deliver_machine_check),
  97         VCPU_STAT("exit_wait_state", exit_wait_state),
  98         VCPU_STAT("inject_ckc", inject_ckc),
  99         VCPU_STAT("inject_cputm", inject_cputm),
 100         VCPU_STAT("inject_external_call", inject_external_call),
 101         VM_STAT("inject_float_mchk", inject_float_mchk),
 102         VCPU_STAT("inject_emergency_signal", inject_emergency_signal),
 103         VM_STAT("inject_io", inject_io),
 104         VCPU_STAT("inject_mchk", inject_mchk),
 105         VM_STAT("inject_pfault_done", inject_pfault_done),
 106         VCPU_STAT("inject_program", inject_program),
 107         VCPU_STAT("inject_restart", inject_restart),
 108         VM_STAT("inject_service_signal", inject_service_signal),
 109         VCPU_STAT("inject_set_prefix", inject_set_prefix),
 110         VCPU_STAT("inject_stop_signal", inject_stop_signal),
 111         VCPU_STAT("inject_pfault_init", inject_pfault_init),
 112         VM_STAT("inject_virtio", inject_virtio),
 113         VCPU_STAT("instruction_epsw", instruction_epsw),
 114         VCPU_STAT("instruction_gs", instruction_gs),
 115         VCPU_STAT("instruction_io_other", instruction_io_other),
 116         VCPU_STAT("instruction_lpsw", instruction_lpsw),
 117         VCPU_STAT("instruction_lpswe", instruction_lpswe),
 118         VCPU_STAT("instruction_pfmf", instruction_pfmf),
 119         VCPU_STAT("instruction_ptff", instruction_ptff),
 120         VCPU_STAT("instruction_stidp", instruction_stidp),
 121         VCPU_STAT("instruction_sck", instruction_sck),
 122         VCPU_STAT("instruction_sckpf", instruction_sckpf),
 123         VCPU_STAT("instruction_spx", instruction_spx),
 124         VCPU_STAT("instruction_stpx", instruction_stpx),
 125         VCPU_STAT("instruction_stap", instruction_stap),
 126         VCPU_STAT("instruction_iske", instruction_iske),
 127         VCPU_STAT("instruction_ri", instruction_ri),
 128         VCPU_STAT("instruction_rrbe", instruction_rrbe),
 129         VCPU_STAT("instruction_sske", instruction_sske),
 130         VCPU_STAT("instruction_ipte_interlock", instruction_ipte_interlock),
 131         VCPU_STAT("instruction_essa", instruction_essa),
 132         VCPU_STAT("instruction_stsi", instruction_stsi),
 133         VCPU_STAT("instruction_stfl", instruction_stfl),
 134         VCPU_STAT("instruction_tb", instruction_tb),
 135         VCPU_STAT("instruction_tpi", instruction_tpi),
 136         VCPU_STAT("instruction_tprot", instruction_tprot),
 137         VCPU_STAT("instruction_tsch", instruction_tsch),
 138         VCPU_STAT("instruction_sthyi", instruction_sthyi),
 139         VCPU_STAT("instruction_sie", instruction_sie),
 140         VCPU_STAT("instruction_sigp_sense", instruction_sigp_sense),
 141         VCPU_STAT("instruction_sigp_sense_running", instruction_sigp_sense_running),
 142         VCPU_STAT("instruction_sigp_external_call", instruction_sigp_external_call),
 143         VCPU_STAT("instruction_sigp_emergency", instruction_sigp_emergency),
 144         VCPU_STAT("instruction_sigp_cond_emergency", instruction_sigp_cond_emergency),
 145         VCPU_STAT("instruction_sigp_start", instruction_sigp_start),
 146         VCPU_STAT("instruction_sigp_stop", instruction_sigp_stop),
 147         VCPU_STAT("instruction_sigp_stop_store_status", instruction_sigp_stop_store_status),
 148         VCPU_STAT("instruction_sigp_store_status", instruction_sigp_store_status),
 149         VCPU_STAT("instruction_sigp_store_adtl_status", instruction_sigp_store_adtl_status),
 150         VCPU_STAT("instruction_sigp_set_arch", instruction_sigp_arch),
 151         VCPU_STAT("instruction_sigp_set_prefix", instruction_sigp_prefix),
 152         VCPU_STAT("instruction_sigp_restart", instruction_sigp_restart),
 153         VCPU_STAT("instruction_sigp_cpu_reset", instruction_sigp_cpu_reset),
 154         VCPU_STAT("instruction_sigp_init_cpu_reset", instruction_sigp_init_cpu_reset),
 155         VCPU_STAT("instruction_sigp_unknown", instruction_sigp_unknown),
 156         VCPU_STAT("instruction_diag_10", diagnose_10),
 157         VCPU_STAT("instruction_diag_44", diagnose_44),
 158         VCPU_STAT("instruction_diag_9c", diagnose_9c),
 159         VCPU_STAT("diag_9c_ignored", diagnose_9c_ignored),
 160         VCPU_STAT("instruction_diag_258", diagnose_258),
 161         VCPU_STAT("instruction_diag_308", diagnose_308),
 162         VCPU_STAT("instruction_diag_500", diagnose_500),
 163         VCPU_STAT("instruction_diag_other", diagnose_other),
 164         { NULL }
 165 };
 166
 167 struct kvm_s390_tod_clock_ext {
 168         __u8 epoch_idx;
 169         __u64 tod;
 170         __u8 reserved[7];
 171 } __packed;
 172
 173 /* allow nested virtualization in KVM (if enabled by user space) */
 174 static int nested;
 175 module_param(nested, int, S_IRUGO);
 176 MODULE_PARM_DESC(nested, "Nested virtualization support");
 177
 178 /* allow 1m huge page guest backing, if !nested */
 179 static int hpage;
 180 module_param(hpage, int, 0444);
 181 MODULE_PARM_DESC(hpage, "1m huge page backing support");
 182
 183 /* maximum percentage of steal time for polling.  >100 is treated like 100 */
 184 static u8 halt_poll_max_steal = 10;
 185 module_param(halt_poll_max_steal, byte, 0644);
 186 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
 187
 188 /* if set to true, the GISA will be initialized and used if available */
 189 static bool use_gisa  = true;
 190 module_param(use_gisa, bool, 0644);
 191 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
 192
 193 /*
 194  * For now we handle at most 16 double words as this is what the s390 base
 195  * kernel handles and stores in the prefix page. If we ever need to go beyond
 196  * this, this requires changes to code, but the external uapi can stay.
 197  */
 198 #define SIZE_INTERNAL 16
 199
 200 /*
 201  * Base feature mask that defines default mask for facilities. Consists of the
 202  * defines in FACILITIES_KVM and the non-hypervisor managed bits.
 203  */
 204 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
 205 /*
 206  * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
 207  * and defines the facilities that can be enabled via a cpu model.
 208  */
 209 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
 210
 211 static unsigned long kvm_s390_fac_size(void)
 212 {
 213         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
 214         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
 215         BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
 216                 sizeof(S390_lowcore.stfle_fac_list));
 217
 218         return SIZE_INTERNAL;
 219 }
 220
 221 /* available cpu features supported by kvm */
 222 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 223 /* available subfunctions indicated via query / "test bit" */
 224 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 225
 226 static struct gmap_notifier gmap_notifier;
 227 static struct gmap_notifier vsie_gmap_notifier;
 228 debug_info_t *kvm_s390_dbf;
 229 debug_info_t *kvm_s390_dbf_uv;
 230
 231 /* Section: not file related */
 232 int kvm_arch_hardware_enable(void)
 233 {
 234         /* every s390 is virtualization enabled ;-) */
 235         return 0;
 236 }
 237
 238 int kvm_arch_check_processor_compat(void *opaque)
 239 {
 240         return 0;
 241 }
 242
 243 /* forward declarations */
 244 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 245                               unsigned long end);
 246 static int sca_switch_to_extended(struct kvm *kvm);
 247
 248 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
 249 {
 250         u8 delta_idx = 0;
 251
 252         /*
 253          * The TOD jumps by delta, we have to compensate this by adding
 254          * -delta to the epoch.
 255          */
 256         delta = -delta;
 257
 258         /* sign-extension - we're adding to signed values below */
 259         if ((s64)delta < 0)
 260                 delta_idx = -1;
 261
 262         scb->epoch += delta;
 263         if (scb->ecd & ECD_MEF) {
 264                 scb->epdx += delta_idx;
 265                 if (scb->epoch < delta)
 266                         scb->epdx += 1;
 267         }
 268 }
 269
 270 /*
 271  * This callback is executed during stop_machine(). All CPUs are therefore
 272  * temporarily stopped. In order not to change guest behavior, we have to
 273  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 274  * so a CPU won't be stopped while calculating with the epoch.
 275  */
 276 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 277                           void *v)
 278 {
 279         struct kvm *kvm;
 280         struct kvm_vcpu *vcpu;
 281         int i;
 282         unsigned long long *delta = v;
 283
 284         list_for_each_entry(kvm, &vm_list, vm_list) {
 285                 kvm_for_each_vcpu(i, vcpu, kvm) {
 286                         kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
 287                         if (i == 0) {
 288                                 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
 289                                 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
 290                         }
 291                         if (vcpu->arch.cputm_enabled)
 292                                 vcpu->arch.cputm_start += *delta;
 293                         if (vcpu->arch.vsie_block)
 294                                 kvm_clock_sync_scb(vcpu->arch.vsie_block,
 295                                                    *delta);
 296                 }
 297         }
 298         return NOTIFY_OK;
 299 }
 300
 301 static struct notifier_block kvm_clock_notifier = {
 302         .notifier_call = kvm_clock_sync,
 303 };
 304
 305 int kvm_arch_hardware_setup(void *opaque)
 306 {
 307         gmap_notifier.notifier_call = kvm_gmap_notifier;
 308         gmap_register_pte_notifier(&gmap_notifier);
 309         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 310         gmap_register_pte_notifier(&vsie_gmap_notifier);
 311         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 312                                        &kvm_clock_notifier);
 313         return 0;
 314 }
 315
 316 void kvm_arch_hardware_unsetup(void)
 317 {
 318         gmap_unregister_pte_notifier(&gmap_notifier);
 319         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 320         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 321                                          &kvm_clock_notifier);
 322 }
 323
 324 static void allow_cpu_feat(unsigned long nr)
 325 {
 326         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 327 }
 328
 329 static inline int plo_test_bit(unsigned char nr)
 330 {
 331         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 332         int cc;
 333
 334         asm volatile(
 335                 /* Parameter registers are ignored for "test bit" */
 336                 "       plo     0,0,0,0(0)\n"
 337                 "       ipm     %0\n"
 338                 "       srl     %0,28\n"
 339                 : "=d" (cc)
 340                 : "d" (r0)
 341                 : "cc");
 342         return cc == 0;
 343 }
 344
 345 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
 346 {
 347         register unsigned long r0 asm("0") = 0; /* query function */
 348         register unsigned long r1 asm("1") = (unsigned long) query;
 349
 350         asm volatile(
 351                 /* Parameter regs are ignored */
 352                 "       .insn   rrf,%[opc] << 16,2,4,6,0\n"
 353                 :
 354                 : "d" (r0), "a" (r1), [opc] "i" (opcode)
 355                 : "cc", "memory");
 356 }
 357
 358 #define INSN_SORTL 0xb938
 359 #define INSN_DFLTCC 0xb939
 360
 361 static void kvm_s390_cpu_feat_init(void)
 362 {
 363         int i;
 364
 365         for (i = 0; i < 256; ++i) {
 366                 if (plo_test_bit(i))
 367                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 368         }
 369
 370         if (test_facility(28)) /* TOD-clock steering */
 371                 ptff(kvm_s390_available_subfunc.ptff,
 372                      sizeof(kvm_s390_available_subfunc.ptff),
 373                      PTFF_QAF);
 374
 375         if (test_facility(17)) { /* MSA */
 376                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 377                               kvm_s390_available_subfunc.kmac);
 378                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 379                               kvm_s390_available_subfunc.kmc);
 380                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 381                               kvm_s390_available_subfunc.km);
 382                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 383                               kvm_s390_available_subfunc.kimd);
 384                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 385                               kvm_s390_available_subfunc.klmd);
 386         }
 387         if (test_facility(76)) /* MSA3 */
 388                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 389                               kvm_s390_available_subfunc.pckmo);
 390         if (test_facility(77)) { /* MSA4 */
 391                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 392                               kvm_s390_available_subfunc.kmctr);
 393                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 394                               kvm_s390_available_subfunc.kmf);
 395                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 396                               kvm_s390_available_subfunc.kmo);
 397                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 398                               kvm_s390_available_subfunc.pcc);
 399         }
 400         if (test_facility(57)) /* MSA5 */
 401                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
 402                               kvm_s390_available_subfunc.ppno);
 403
 404         if (test_facility(146)) /* MSA8 */
 405                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
 406                               kvm_s390_available_subfunc.kma);
 407
 408         if (test_facility(155)) /* MSA9 */
 409                 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
 410                               kvm_s390_available_subfunc.kdsa);
 411
 412         if (test_facility(150)) /* SORTL */
 413                 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
 414
 415         if (test_facility(151)) /* DFLTCC */
 416                 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
 417
 418         if (MACHINE_HAS_ESOP)
 419                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 420         /*
 421          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 422          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 423          */
 424         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 425             !test_facility(3) || !nested)
 426                 return;
 427         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 428         if (sclp.has_64bscao)
 429                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 430         if (sclp.has_siif)
 431                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 432         if (sclp.has_gpere)
 433                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 434         if (sclp.has_gsls)
 435                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 436         if (sclp.has_ib)
 437                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 438         if (sclp.has_cei)
 439                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 440         if (sclp.has_ibs)
 441                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 442         if (sclp.has_kss)
 443                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
 444         /*
 445          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 446          * all skey handling functions read/set the skey from the PGSTE
 447          * instead of the real storage key.
 448          *
 449          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 450          * pages being detected as preserved although they are resident.
 451          *
 452          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 453          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 454          *
 455          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 456          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 457          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 458          *
 459          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 460          * cannot easily shadow the SCA because of the ipte lock.
 461          */
 462 }
 463
 464 int kvm_arch_init(void *opaque)
 465 {
 466         int rc = -ENOMEM;
 467
 468         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 469         if (!kvm_s390_dbf)
 470                 return -ENOMEM;
 471
 472         kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
 473         if (!kvm_s390_dbf_uv)
 474                 goto out;
 475
 476         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
 477             debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
 478                 goto out;
 479
 480         kvm_s390_cpu_feat_init();
 481
 482         /* Register floating interrupt controller interface. */
 483         rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 484         if (rc) {
 485                 pr_err("A FLIC registration call failed with rc=%d\n", rc);
 486                 goto out;
 487         }
 488
 489         rc = kvm_s390_gib_init(GAL_ISC);
 490         if (rc)
 491                 goto out;
 492
 493         return 0;
 494
 495 out:
 496         kvm_arch_exit();
 497         return rc;
 498 }
 499
 500 void kvm_arch_exit(void)
 501 {
 502         kvm_s390_gib_destroy();
 503         debug_unregister(kvm_s390_dbf);
 504         debug_unregister(kvm_s390_dbf_uv);
 505 }
 506
 507 /* Section: device related */
 508 long kvm_arch_dev_ioctl(struct file *filp,
 509                         unsigned int ioctl, unsigned long arg)
 510 {
 511         if (ioctl == KVM_S390_ENABLE_SIE)
 512                 return s390_enable_sie();
 513         return -EINVAL;
 514 }
 515
 516 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 517 {
 518         int r;
 519
 520         switch (ext) {
 521         case KVM_CAP_S390_PSW:
 522         case KVM_CAP_S390_GMAP:
 523         case KVM_CAP_SYNC_MMU:
 524 #ifdef CONFIG_KVM_S390_UCONTROL
 525         case KVM_CAP_S390_UCONTROL:
 526 #endif
 527         case KVM_CAP_ASYNC_PF:
 528         case KVM_CAP_SYNC_REGS:
 529         case KVM_CAP_ONE_REG:
 530         case KVM_CAP_ENABLE_CAP:
 531         case KVM_CAP_S390_CSS_SUPPORT:
 532         case KVM_CAP_IOEVENTFD:
 533         case KVM_CAP_DEVICE_CTRL:
 534         case KVM_CAP_S390_IRQCHIP:
 535         case KVM_CAP_VM_ATTRIBUTES:
 536         case KVM_CAP_MP_STATE:
 537         case KVM_CAP_IMMEDIATE_EXIT:
 538         case KVM_CAP_S390_INJECT_IRQ:
 539         case KVM_CAP_S390_USER_SIGP:
 540         case KVM_CAP_S390_USER_STSI:
 541         case KVM_CAP_S390_SKEYS:
 542         case KVM_CAP_S390_IRQ_STATE:
 543         case KVM_CAP_S390_USER_INSTR0:
 544         case KVM_CAP_S390_CMMA_MIGRATION:
 545         case KVM_CAP_S390_AIS:
 546         case KVM_CAP_S390_AIS_MIGRATION:
 547         case KVM_CAP_S390_VCPU_RESETS:
 548         case KVM_CAP_SET_GUEST_DEBUG:
 549         case KVM_CAP_S390_DIAG318:
 550                 r = 1;
 551                 break;
 552         case KVM_CAP_S390_HPAGE_1M:
 553                 r = 0;
 554                 if (hpage && !kvm_is_ucontrol(kvm))
 555                         r = 1;
 556                 break;
 557         case KVM_CAP_S390_MEM_OP:
 558                 r = MEM_OP_MAX_SIZE;
 559                 break;
 560         case KVM_CAP_NR_VCPUS:
 561         case KVM_CAP_MAX_VCPUS:
 562         case KVM_CAP_MAX_VCPU_ID:
 563                 r = KVM_S390_BSCA_CPU_SLOTS;
 564                 if (!kvm_s390_use_sca_entries())
 565                         r = KVM_MAX_VCPUS;
 566                 else if (sclp.has_esca && sclp.has_64bscao)
 567                         r = KVM_S390_ESCA_CPU_SLOTS;
 568                 break;
 569         case KVM_CAP_S390_COW:
 570                 r = MACHINE_HAS_ESOP;
 571                 break;
 572         case KVM_CAP_S390_VECTOR_REGISTERS:
 573                 r = MACHINE_HAS_VX;
 574                 break;
 575         case KVM_CAP_S390_RI:
 576                 r = test_facility(64);
 577                 break;
 578         case KVM_CAP_S390_GS:
 579                 r = test_facility(133);
 580                 break;
 581         case KVM_CAP_S390_BPB:
 582                 r = test_facility(82);
 583                 break;
 584         case KVM_CAP_S390_PROTECTED:
 585                 r = is_prot_virt_host();
 586                 break;
 587         default:
 588                 r = 0;
 589         }
 590         return r;
 591 }
 592
 593 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
 594 {
 595         int i;
 596         gfn_t cur_gfn, last_gfn;
 597         unsigned long gaddr, vmaddr;
 598         struct gmap *gmap = kvm->arch.gmap;
 599         DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
 600
 601         /* Loop over all guest segments */
 602         cur_gfn = memslot->base_gfn;
 603         last_gfn = memslot->base_gfn + memslot->npages;
 604         for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
 605                 gaddr = gfn_to_gpa(cur_gfn);
 606                 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
 607                 if (kvm_is_error_hva(vmaddr))
 608                         continue;
 609
 610                 bitmap_zero(bitmap, _PAGE_ENTRIES);
 611                 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
 612                 for (i = 0; i < _PAGE_ENTRIES; i++) {
 613                         if (test_bit(i, bitmap))
 614                                 mark_page_dirty(kvm, cur_gfn + i);
 615                 }
 616
 617                 if (fatal_signal_pending(current))
 618                         return;
 619                 cond_resched();
 620         }
 621 }
 622
 623 /* Section: vm related */
 624 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 625
 626 /*
 627  * Get (and clear) the dirty memory log for a memory slot.
 628  */
 629 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 630                                struct kvm_dirty_log *log)
 631 {
 632         int r;
 633         unsigned long n;
 634         struct kvm_memory_slot *memslot;
 635         int is_dirty;
 636
 637         if (kvm_is_ucontrol(kvm))
 638                 return -EINVAL;
 639
 640         mutex_lock(&kvm->slots_lock);
 641
 642         r = -EINVAL;
 643         if (log->slot >= KVM_USER_MEM_SLOTS)
 644                 goto out;
 645
 646         r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
 647         if (r)
 648                 goto out;
 649
 650         /* Clear the dirty log */
 651         if (is_dirty) {
 652                 n = kvm_dirty_bitmap_bytes(memslot);
 653                 memset(memslot->dirty_bitmap, 0, n);
 654         }
 655         r = 0;
 656 out:
 657         mutex_unlock(&kvm->slots_lock);
 658         return r;
 659 }
 660
 661 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 662 {
 663         unsigned int i;
 664         struct kvm_vcpu *vcpu;
 665
 666         kvm_for_each_vcpu(i, vcpu, kvm) {
 667                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 668         }
 669 }
 670
 671 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 672 {
 673         int r;
 674
 675         if (cap->flags)
 676                 return -EINVAL;
 677
 678         switch (cap->cap) {
 679         case KVM_CAP_S390_IRQCHIP:
 680                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 681                 kvm->arch.use_irqchip = 1;
 682                 r = 0;
 683                 break;
 684         case KVM_CAP_S390_USER_SIGP:
 685                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 686                 kvm->arch.user_sigp = 1;
 687                 r = 0;
 688                 break;
 689         case KVM_CAP_S390_VECTOR_REGISTERS:
 690                 mutex_lock(&kvm->lock);
 691                 if (kvm->created_vcpus) {
 692                         r = -EBUSY;
 693                 } else if (MACHINE_HAS_VX) {
 694                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 695                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 696                         if (test_facility(134)) {
 697                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 698                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 699                         }
 700                         if (test_facility(135)) {
 701                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 702                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 703                         }
 704                         if (test_facility(148)) {
 705                                 set_kvm_facility(kvm->arch.model.fac_mask, 148);
 706                                 set_kvm_facility(kvm->arch.model.fac_list, 148);
 707                         }
 708                         if (test_facility(152)) {
 709                                 set_kvm_facility(kvm->arch.model.fac_mask, 152);
 710                                 set_kvm_facility(kvm->arch.model.fac_list, 152);
 711                         }
 712                         r = 0;
 713                 } else
 714                         r = -EINVAL;
 715                 mutex_unlock(&kvm->lock);
 716                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 717                          r ? "(not available)" : "(success)");
 718                 break;
 719         case KVM_CAP_S390_RI:
 720                 r = -EINVAL;
 721                 mutex_lock(&kvm->lock);
 722                 if (kvm->created_vcpus) {
 723                         r = -EBUSY;
 724                 } else if (test_facility(64)) {
 725                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 726                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 727                         r = 0;
 728                 }
 729                 mutex_unlock(&kvm->lock);
 730                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 731                          r ? "(not available)" : "(success)");
 732                 break;
 733         case KVM_CAP_S390_AIS:
 734                 mutex_lock(&kvm->lock);
 735                 if (kvm->created_vcpus) {
 736                         r = -EBUSY;
 737                 } else {
 738                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
 739                         set_kvm_facility(kvm->arch.model.fac_list, 72);
 740                         r = 0;
 741                 }
 742                 mutex_unlock(&kvm->lock);
 743                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
 744                          r ? "(not available)" : "(success)");
 745                 break;
 746         case KVM_CAP_S390_GS:
 747                 r = -EINVAL;
 748                 mutex_lock(&kvm->lock);
 749                 if (kvm->created_vcpus) {
 750                         r = -EBUSY;
 751                 } else if (test_facility(133)) {
 752                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
 753                         set_kvm_facility(kvm->arch.model.fac_list, 133);
 754                         r = 0;
 755                 }
 756                 mutex_unlock(&kvm->lock);
 757                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
 758                          r ? "(not available)" : "(success)");
 759                 break;
 760         case KVM_CAP_S390_HPAGE_1M:
 761                 mutex_lock(&kvm->lock);
 762                 if (kvm->created_vcpus)
 763                         r = -EBUSY;
 764                 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
 765                         r = -EINVAL;
 766                 else {
 767                         r = 0;
 768                         mmap_write_lock(kvm->mm);
 769                         kvm->mm->context.allow_gmap_hpage_1m = 1;
 770                         mmap_write_unlock(kvm->mm);
 771                         /*
 772                          * We might have to create fake 4k page
 773                          * tables. To avoid that the hardware works on
 774                          * stale PGSTEs, we emulate these instructions.
 775                          */
 776                         kvm->arch.use_skf = 0;
 777                         kvm->arch.use_pfmfi = 0;
 778                 }
 779                 mutex_unlock(&kvm->lock);
 780                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
 781                          r ? "(not available)" : "(success)");
 782                 break;
 783         case KVM_CAP_S390_USER_STSI:
 784                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 785                 kvm->arch.user_stsi = 1;
 786                 r = 0;
 787                 break;
 788         case KVM_CAP_S390_USER_INSTR0:
 789                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 790                 kvm->arch.user_instr0 = 1;
 791                 icpt_operexc_on_all_vcpus(kvm);
 792                 r = 0;
 793                 break;
 794         default:
 795                 r = -EINVAL;
 796                 break;
 797         }
 798         return r;
 799 }
 800
 801 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 802 {
 803         int ret;
 804
 805         switch (attr->attr) {
 806         case KVM_S390_VM_MEM_LIMIT_SIZE:
 807                 ret = 0;
 808                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 809                          kvm->arch.mem_limit);
 810                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 811                         ret = -EFAULT;
 812                 break;
 813         default:
 814                 ret = -ENXIO;
 815                 break;
 816         }
 817         return ret;
 818 }
 819
 820 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 821 {
 822         int ret;
 823         unsigned int idx;
 824         switch (attr->attr) {
 825         case KVM_S390_VM_MEM_ENABLE_CMMA:
 826                 ret = -ENXIO;
 827                 if (!sclp.has_cmma)
 828                         break;
 829
 830                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 831                 mutex_lock(&kvm->lock);
 832                 if (kvm->created_vcpus)
 833                         ret = -EBUSY;
 834                 else if (kvm->mm->context.allow_gmap_hpage_1m)
 835                         ret = -EINVAL;
 836                 else {
 837                         kvm->arch.use_cmma = 1;
 838                         /* Not compatible with cmma. */
 839                         kvm->arch.use_pfmfi = 0;
 840                         ret = 0;
 841                 }
 842                 mutex_unlock(&kvm->lock);
 843                 break;
 844         case KVM_S390_VM_MEM_CLR_CMMA:
 845                 ret = -ENXIO;
 846                 if (!sclp.has_cmma)
 847                         break;
 848                 ret = -EINVAL;
 849                 if (!kvm->arch.use_cmma)
 850                         break;
 851
 852                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 853                 mutex_lock(&kvm->lock);
 854                 idx = srcu_read_lock(&kvm->srcu);
 855                 s390_reset_cmma(kvm->arch.gmap->mm);
 856                 srcu_read_unlock(&kvm->srcu, idx);
 857                 mutex_unlock(&kvm->lock);
 858                 ret = 0;
 859                 break;
 860         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 861                 unsigned long new_limit;
 862
 863                 if (kvm_is_ucontrol(kvm))
 864                         return -EINVAL;
 865
 866                 if (get_user(new_limit, (u64 __user *)attr->addr))
 867                         return -EFAULT;
 868
 869                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 870                     new_limit > kvm->arch.mem_limit)
 871                         return -E2BIG;
 872
 873                 if (!new_limit)
 874                         return -EINVAL;
 875
 876                 /* gmap_create takes last usable address */
 877                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 878                         new_limit -= 1;
 879
 880                 ret = -EBUSY;
 881                 mutex_lock(&kvm->lock);
 882                 if (!kvm->created_vcpus) {
 883                         /* gmap_create will round the limit up */
 884                         struct gmap *new = gmap_create(current->mm, new_limit);
 885
 886                         if (!new) {
 887                                 ret = -ENOMEM;
 888                         } else {
 889                                 gmap_remove(kvm->arch.gmap);
 890                                 new->private = kvm;
 891                                 kvm->arch.gmap = new;
 892                                 ret = 0;
 893                         }
 894                 }
 895                 mutex_unlock(&kvm->lock);
 896                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 897                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 898                          (void *) kvm->arch.gmap->asce);
 899                 break;
 900         }
 901         default:
 902                 ret = -ENXIO;
 903                 break;
 904         }
 905         return ret;
 906 }
 907
 908 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 909
 910 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
 911 {
 912         struct kvm_vcpu *vcpu;
 913         int i;
 914
 915         kvm_s390_vcpu_block_all(kvm);
 916
 917         kvm_for_each_vcpu(i, vcpu, kvm) {
 918                 kvm_s390_vcpu_crypto_setup(vcpu);
 919                 /* recreate the shadow crycb by leaving the VSIE handler */
 920                 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
 921         }
 922
 923         kvm_s390_vcpu_unblock_all(kvm);
 924 }
 925
 926 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 927 {
 928         mutex_lock(&kvm->lock);
 929         switch (attr->attr) {
 930         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 931                 if (!test_kvm_facility(kvm, 76)) {
 932                         mutex_unlock(&kvm->lock);
 933                         return -EINVAL;
 934                 }
 935                 get_random_bytes(
 936                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 937                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 938                 kvm->arch.crypto.aes_kw = 1;
 939                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 940                 break;
 941         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 942                 if (!test_kvm_facility(kvm, 76)) {
 943                         mutex_unlock(&kvm->lock);
 944                         return -EINVAL;
 945                 }
 946                 get_random_bytes(
 947                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 948                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 949                 kvm->arch.crypto.dea_kw = 1;
 950                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 951                 break;
 952         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 953                 if (!test_kvm_facility(kvm, 76)) {
 954                         mutex_unlock(&kvm->lock);
 955                         return -EINVAL;
 956                 }
 957                 kvm->arch.crypto.aes_kw = 0;
 958                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 959                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 960                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 961                 break;
 962         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 963                 if (!test_kvm_facility(kvm, 76)) {
 964                         mutex_unlock(&kvm->lock);
 965                         return -EINVAL;
 966                 }
 967                 kvm->arch.crypto.dea_kw = 0;
 968                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 969                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 970                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 971                 break;
 972         case KVM_S390_VM_CRYPTO_ENABLE_APIE:
 973                 if (!ap_instructions_available()) {
 974                         mutex_unlock(&kvm->lock);
 975                         return -EOPNOTSUPP;
 976                 }
 977                 kvm->arch.crypto.apie = 1;
 978                 break;
 979         case KVM_S390_VM_CRYPTO_DISABLE_APIE:
 980                 if (!ap_instructions_available()) {
 981                         mutex_unlock(&kvm->lock);
 982                         return -EOPNOTSUPP;
 983                 }
 984                 kvm->arch.crypto.apie = 0;
 985                 break;
 986         default:
 987                 mutex_unlock(&kvm->lock);
 988                 return -ENXIO;
 989         }
 990
 991         kvm_s390_vcpu_crypto_reset_all(kvm);
 992         mutex_unlock(&kvm->lock);
 993         return 0;
 994 }
 995
 996 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
 997 {
 998         int cx;
 999         struct kvm_vcpu *vcpu;
1000
1001         kvm_for_each_vcpu(cx, vcpu, kvm)
1002                 kvm_s390_sync_request(req, vcpu);
1003 }
1004
1005 /*
1006  * Must be called with kvm->srcu held to avoid races on memslots, and with
1007  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
1008  */
1009 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1010 {
1011         struct kvm_memory_slot *ms;
1012         struct kvm_memslots *slots;
1013         unsigned long ram_pages = 0;
1014         int slotnr;
1015
1016         /* migration mode already enabled */
1017         if (kvm->arch.migration_mode)
1018                 return 0;
1019         slots = kvm_memslots(kvm);
1020         if (!slots || !slots->used_slots)
1021                 return -EINVAL;
1022
1023         if (!kvm->arch.use_cmma) {
1024                 kvm->arch.migration_mode = 1;
1025                 return 0;
1026         }
1027         /* mark all the pages in active slots as dirty */
1028         for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1029                 ms = slots->memslots + slotnr;
1030                 if (!ms->dirty_bitmap)
1031                         return -EINVAL;
1032                 /*
1033                  * The second half of the bitmap is only used on x86,
1034                  * and would be wasted otherwise, so we put it to good
1035                  * use here to keep track of the state of the storage
1036                  * attributes.
1037                  */
1038                 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1039                 ram_pages += ms->npages;
1040         }
1041         atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1042         kvm->arch.migration_mode = 1;
1043         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1044         return 0;
1045 }
1046
1047 /*
1048  * Must be called with kvm->slots_lock to avoid races with ourselves and
1049  * kvm_s390_vm_start_migration.
1050  */
1051 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1052 {
1053         /* migration mode already disabled */
1054         if (!kvm->arch.migration_mode)
1055                 return 0;
1056         kvm->arch.migration_mode = 0;
1057         if (kvm->arch.use_cmma)
1058                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1059         return 0;
1060 }
1061
1062 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1063                                      struct kvm_device_attr *attr)
1064 {
1065         int res = -ENXIO;
1066
1067         mutex_lock(&kvm->slots_lock);
1068         switch (attr->attr) {
1069         case KVM_S390_VM_MIGRATION_START:
1070                 res = kvm_s390_vm_start_migration(kvm);
1071                 break;
1072         case KVM_S390_VM_MIGRATION_STOP:
1073                 res = kvm_s390_vm_stop_migration(kvm);
1074                 break;
1075         default:
1076                 break;
1077         }
1078         mutex_unlock(&kvm->slots_lock);
1079
1080         return res;
1081 }
1082
1083 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1084                                      struct kvm_device_attr *attr)
1085 {
1086         u64 mig = kvm->arch.migration_mode;
1087
1088         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1089                 return -ENXIO;
1090
1091         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1092                 return -EFAULT;
1093         return 0;
1094 }
1095
1096 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1097 {
1098         struct kvm_s390_vm_tod_clock gtod;
1099
1100         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1101                 return -EFAULT;
1102
1103         if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1104                 return -EINVAL;
1105         kvm_s390_set_tod_clock(kvm, &gtod);
1106
1107         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1108                 gtod.epoch_idx, gtod.tod);
1109
1110         return 0;
1111 }
1112
1113 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1114 {
1115         u8 gtod_high;
1116
1117         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1118                                            sizeof(gtod_high)))
1119                 return -EFAULT;
1120
1121         if (gtod_high != 0)
1122                 return -EINVAL;
1123         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1124
1125         return 0;
1126 }
1127
1128 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1129 {
1130         struct kvm_s390_vm_tod_clock gtod = { 0 };
1131
1132         if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1133                            sizeof(gtod.tod)))
1134                 return -EFAULT;
1135
1136         kvm_s390_set_tod_clock(kvm, &gtod);
1137         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1138         return 0;
1139 }
1140
1141 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1142 {
1143         int ret;
1144
1145         if (attr->flags)
1146                 return -EINVAL;
1147
1148         switch (attr->attr) {
1149         case KVM_S390_VM_TOD_EXT:
1150                 ret = kvm_s390_set_tod_ext(kvm, attr);
1151                 break;
1152         case KVM_S390_VM_TOD_HIGH:
1153                 ret = kvm_s390_set_tod_high(kvm, attr);
1154                 break;
1155         case KVM_S390_VM_TOD_LOW:
1156                 ret = kvm_s390_set_tod_low(kvm, attr);
1157                 break;
1158         default:
1159                 ret = -ENXIO;
1160                 break;
1161         }
1162         return ret;
1163 }
1164
1165 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1166                                    struct kvm_s390_vm_tod_clock *gtod)
1167 {
1168         struct kvm_s390_tod_clock_ext htod;
1169
1170         preempt_disable();
1171
1172         get_tod_clock_ext((char *)&htod);
1173
1174         gtod->tod = htod.tod + kvm->arch.epoch;
1175         gtod->epoch_idx = 0;
1176         if (test_kvm_facility(kvm, 139)) {
1177                 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1178                 if (gtod->tod < htod.tod)
1179                         gtod->epoch_idx += 1;
1180         }
1181
1182         preempt_enable();
1183 }
1184
1185 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1186 {
1187         struct kvm_s390_vm_tod_clock gtod;
1188
1189         memset(&gtod, 0, sizeof(gtod));
1190         kvm_s390_get_tod_clock(kvm, &gtod);
1191         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1192                 return -EFAULT;
1193
1194         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1195                 gtod.epoch_idx, gtod.tod);
1196         return 0;
1197 }
1198
1199 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1200 {
1201         u8 gtod_high = 0;
1202
1203         if (copy_to_user((void __user *)attr->addr, &gtod_high,
1204                                          sizeof(gtod_high)))
1205                 return -EFAULT;
1206         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1207
1208         return 0;
1209 }
1210
1211 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1212 {
1213         u64 gtod;
1214
1215         gtod = kvm_s390_get_tod_clock_fast(kvm);
1216         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1217                 return -EFAULT;
1218         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1219
1220         return 0;
1221 }
1222
1223 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1224 {
1225         int ret;
1226
1227         if (attr->flags)
1228                 return -EINVAL;
1229
1230         switch (attr->attr) {
1231         case KVM_S390_VM_TOD_EXT:
1232                 ret = kvm_s390_get_tod_ext(kvm, attr);
1233                 break;
1234         case KVM_S390_VM_TOD_HIGH:
1235                 ret = kvm_s390_get_tod_high(kvm, attr);
1236                 break;
1237         case KVM_S390_VM_TOD_LOW:
1238                 ret = kvm_s390_get_tod_low(kvm, attr);
1239                 break;
1240         default:
1241                 ret = -ENXIO;
1242                 break;
1243         }
1244         return ret;
1245 }
1246
1247 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1248 {
1249         struct kvm_s390_vm_cpu_processor *proc;
1250         u16 lowest_ibc, unblocked_ibc;
1251         int ret = 0;
1252
1253         mutex_lock(&kvm->lock);
1254         if (kvm->created_vcpus) {
1255                 ret = -EBUSY;
1256                 goto out;
1257         }
1258         proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1259         if (!proc) {
1260                 ret = -ENOMEM;
1261                 goto out;
1262         }
1263         if (!copy_from_user(proc, (void __user *)attr->addr,
1264                             sizeof(*proc))) {
1265                 kvm->arch.model.cpuid = proc->cpuid;
1266                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1267                 unblocked_ibc = sclp.ibc & 0xfff;
1268                 if (lowest_ibc && proc->ibc) {
1269                         if (proc->ibc > unblocked_ibc)
1270                                 kvm->arch.model.ibc = unblocked_ibc;
1271                         else if (proc->ibc < lowest_ibc)
1272                                 kvm->arch.model.ibc = lowest_ibc;
1273                         else
1274                                 kvm->arch.model.ibc = proc->ibc;
1275                 }
1276                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1277                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1278                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1279                          kvm->arch.model.ibc,
1280                          kvm->arch.model.cpuid);
1281                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1282                          kvm->arch.model.fac_list[0],
1283                          kvm->arch.model.fac_list[1],
1284                          kvm->arch.model.fac_list[2]);
1285         } else
1286                 ret = -EFAULT;
1287         kfree(proc);
1288 out:
1289         mutex_unlock(&kvm->lock);
1290         return ret;
1291 }
1292
1293 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1294                                        struct kvm_device_attr *attr)
1295 {
1296         struct kvm_s390_vm_cpu_feat data;
1297
1298         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1299                 return -EFAULT;
1300         if (!bitmap_subset((unsigned long *) data.feat,
1301                            kvm_s390_available_cpu_feat,
1302                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1303                 return -EINVAL;
1304
1305         mutex_lock(&kvm->lock);
1306         if (kvm->created_vcpus) {
1307                 mutex_unlock(&kvm->lock);
1308                 return -EBUSY;
1309         }
1310         bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1311                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1312         mutex_unlock(&kvm->lock);
1313         VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1314                          data.feat[0],
1315                          data.feat[1],
1316                          data.feat[2]);
1317         return 0;
1318 }
1319
1320 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1321                                           struct kvm_device_attr *attr)
1322 {
1323         mutex_lock(&kvm->lock);
1324         if (kvm->created_vcpus) {
1325                 mutex_unlock(&kvm->lock);
1326                 return -EBUSY;
1327         }
1328
1329         if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1330                            sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1331                 mutex_unlock(&kvm->lock);
1332                 return -EFAULT;
1333         }
1334         mutex_unlock(&kvm->lock);
1335
1336         VM_EVENT(kvm, 3, "SET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1337                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1338                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1339                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1340                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1341         VM_EVENT(kvm, 3, "SET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1342                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1343                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1344         VM_EVENT(kvm, 3, "SET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1345                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1346                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1347         VM_EVENT(kvm, 3, "SET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1348                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1349                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1350         VM_EVENT(kvm, 3, "SET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1351                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1352                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1353         VM_EVENT(kvm, 3, "SET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1354                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1355                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1356         VM_EVENT(kvm, 3, "SET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1357                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1358                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1359         VM_EVENT(kvm, 3, "SET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1360                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1361                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1362         VM_EVENT(kvm, 3, "SET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1363                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1364                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1365         VM_EVENT(kvm, 3, "SET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1366                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1367                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1368         VM_EVENT(kvm, 3, "SET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1369                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1370                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1371         VM_EVENT(kvm, 3, "SET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1372                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1373                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1374         VM_EVENT(kvm, 3, "SET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1375                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1376                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1377         VM_EVENT(kvm, 3, "SET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1378                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1379                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1380         VM_EVENT(kvm, 3, "SET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1381                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1382                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1383         VM_EVENT(kvm, 3, "SET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1384                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1385                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1386                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1387                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1388         VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1389                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1390                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1391                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1392                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1393
1394         return 0;
1395 }
1396
1397 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1398 {
1399         int ret = -ENXIO;
1400
1401         switch (attr->attr) {
1402         case KVM_S390_VM_CPU_PROCESSOR:
1403                 ret = kvm_s390_set_processor(kvm, attr);
1404                 break;
1405         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1406                 ret = kvm_s390_set_processor_feat(kvm, attr);
1407                 break;
1408         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1409                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1410                 break;
1411         }
1412         return ret;
1413 }
1414
1415 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1416 {
1417         struct kvm_s390_vm_cpu_processor *proc;
1418         int ret = 0;
1419
1420         proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1421         if (!proc) {
1422                 ret = -ENOMEM;
1423                 goto out;
1424         }
1425         proc->cpuid = kvm->arch.model.cpuid;
1426         proc->ibc = kvm->arch.model.ibc;
1427         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1428                S390_ARCH_FAC_LIST_SIZE_BYTE);
1429         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1430                  kvm->arch.model.ibc,
1431                  kvm->arch.model.cpuid);
1432         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1433                  kvm->arch.model.fac_list[0],
1434                  kvm->arch.model.fac_list[1],
1435                  kvm->arch.model.fac_list[2]);
1436         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1437                 ret = -EFAULT;
1438         kfree(proc);
1439 out:
1440         return ret;
1441 }
1442
1443 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1444 {
1445         struct kvm_s390_vm_cpu_machine *mach;
1446         int ret = 0;
1447
1448         mach = kzalloc(sizeof(*mach), GFP_KERNEL_ACCOUNT);
1449         if (!mach) {
1450                 ret = -ENOMEM;
1451                 goto out;
1452         }
1453         get_cpu_id((struct cpuid *) &mach->cpuid);
1454         mach->ibc = sclp.ibc;
1455         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1456                S390_ARCH_FAC_LIST_SIZE_BYTE);
1457         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1458                sizeof(S390_lowcore.stfle_fac_list));
1459         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1460                  kvm->arch.model.ibc,
1461                  kvm->arch.model.cpuid);
1462         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1463                  mach->fac_mask[0],
1464                  mach->fac_mask[1],
1465                  mach->fac_mask[2]);
1466         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1467                  mach->fac_list[0],
1468                  mach->fac_list[1],
1469                  mach->fac_list[2]);
1470         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1471                 ret = -EFAULT;
1472         kfree(mach);
1473 out:
1474         return ret;
1475 }
1476
1477 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1478                                        struct kvm_device_attr *attr)
1479 {
1480         struct kvm_s390_vm_cpu_feat data;
1481
1482         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1483                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1484         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1485                 return -EFAULT;
1486         VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1487                          data.feat[0],
1488                          data.feat[1],
1489                          data.feat[2]);
1490         return 0;
1491 }
1492
1493 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1494                                      struct kvm_device_attr *attr)
1495 {
1496         struct kvm_s390_vm_cpu_feat data;
1497
1498         bitmap_copy((unsigned long *) data.feat,
1499                     kvm_s390_available_cpu_feat,
1500                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1501         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1502                 return -EFAULT;
1503         VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
1504                          data.feat[0],
1505                          data.feat[1],
1506                          data.feat[2]);
1507         return 0;
1508 }
1509
1510 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1511                                           struct kvm_device_attr *attr)
1512 {
1513         if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1514             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1515                 return -EFAULT;
1516
1517         VM_EVENT(kvm, 3, "GET: guest PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1518                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1519                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1520                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1521                  ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1522         VM_EVENT(kvm, 3, "GET: guest PTFF   subfunc 0x%16.16lx.%16.16lx",
1523                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1524                  ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1525         VM_EVENT(kvm, 3, "GET: guest KMAC   subfunc 0x%16.16lx.%16.16lx",
1526                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1527                  ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1528         VM_EVENT(kvm, 3, "GET: guest KMC    subfunc 0x%16.16lx.%16.16lx",
1529                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1530                  ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1531         VM_EVENT(kvm, 3, "GET: guest KM     subfunc 0x%16.16lx.%16.16lx",
1532                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1533                  ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1534         VM_EVENT(kvm, 3, "GET: guest KIMD   subfunc 0x%16.16lx.%16.16lx",
1535                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1536                  ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1537         VM_EVENT(kvm, 3, "GET: guest KLMD   subfunc 0x%16.16lx.%16.16lx",
1538                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1539                  ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1540         VM_EVENT(kvm, 3, "GET: guest PCKMO  subfunc 0x%16.16lx.%16.16lx",
1541                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1542                  ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1543         VM_EVENT(kvm, 3, "GET: guest KMCTR  subfunc 0x%16.16lx.%16.16lx",
1544                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1545                  ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1546         VM_EVENT(kvm, 3, "GET: guest KMF    subfunc 0x%16.16lx.%16.16lx",
1547                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1548                  ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1549         VM_EVENT(kvm, 3, "GET: guest KMO    subfunc 0x%16.16lx.%16.16lx",
1550                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1551                  ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1552         VM_EVENT(kvm, 3, "GET: guest PCC    subfunc 0x%16.16lx.%16.16lx",
1553                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1554                  ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1555         VM_EVENT(kvm, 3, "GET: guest PPNO   subfunc 0x%16.16lx.%16.16lx",
1556                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1557                  ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1558         VM_EVENT(kvm, 3, "GET: guest KMA    subfunc 0x%16.16lx.%16.16lx",
1559                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1560                  ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1561         VM_EVENT(kvm, 3, "GET: guest KDSA   subfunc 0x%16.16lx.%16.16lx",
1562                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1563                  ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1564         VM_EVENT(kvm, 3, "GET: guest SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1565                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1566                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1567                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1568                  ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1569         VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1570                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1571                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1572                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1573                  ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1574
1575         return 0;
1576 }
1577
1578 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1579                                         struct kvm_device_attr *attr)
1580 {
1581         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1582             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1583                 return -EFAULT;
1584
1585         VM_EVENT(kvm, 3, "GET: host  PLO    subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1586                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1587                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1588                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1589                  ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1590         VM_EVENT(kvm, 3, "GET: host  PTFF   subfunc 0x%16.16lx.%16.16lx",
1591                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1592                  ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1593         VM_EVENT(kvm, 3, "GET: host  KMAC   subfunc 0x%16.16lx.%16.16lx",
1594                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1595                  ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1596         VM_EVENT(kvm, 3, "GET: host  KMC    subfunc 0x%16.16lx.%16.16lx",
1597                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1598                  ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1599         VM_EVENT(kvm, 3, "GET: host  KM     subfunc 0x%16.16lx.%16.16lx",
1600                  ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1601                  ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1602         VM_EVENT(kvm, 3, "GET: host  KIMD   subfunc 0x%16.16lx.%16.16lx",
1603                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1604                  ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1605         VM_EVENT(kvm, 3, "GET: host  KLMD   subfunc 0x%16.16lx.%16.16lx",
1606                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1607                  ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1608         VM_EVENT(kvm, 3, "GET: host  PCKMO  subfunc 0x%16.16lx.%16.16lx",
1609                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1610                  ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1611         VM_EVENT(kvm, 3, "GET: host  KMCTR  subfunc 0x%16.16lx.%16.16lx",
1612                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1613                  ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1614         VM_EVENT(kvm, 3, "GET: host  KMF    subfunc 0x%16.16lx.%16.16lx",
1615                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1616                  ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1617         VM_EVENT(kvm, 3, "GET: host  KMO    subfunc 0x%16.16lx.%16.16lx",
1618                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1619                  ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1620         VM_EVENT(kvm, 3, "GET: host  PCC    subfunc 0x%16.16lx.%16.16lx",
1621                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1622                  ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1623         VM_EVENT(kvm, 3, "GET: host  PPNO   subfunc 0x%16.16lx.%16.16lx",
1624                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1625                  ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1626         VM_EVENT(kvm, 3, "GET: host  KMA    subfunc 0x%16.16lx.%16.16lx",
1627                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1628                  ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1629         VM_EVENT(kvm, 3, "GET: host  KDSA   subfunc 0x%16.16lx.%16.16lx",
1630                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1631                  ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1632         VM_EVENT(kvm, 3, "GET: host  SORTL  subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1633                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1634                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1635                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1636                  ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1637         VM_EVENT(kvm, 3, "GET: host  DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1638                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1639                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1640                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1641                  ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1642
1643         return 0;
1644 }
1645
1646 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1647 {
1648         int ret = -ENXIO;
1649
1650         switch (attr->attr) {
1651         case KVM_S390_VM_CPU_PROCESSOR:
1652                 ret = kvm_s390_get_processor(kvm, attr);
1653                 break;
1654         case KVM_S390_VM_CPU_MACHINE:
1655                 ret = kvm_s390_get_machine(kvm, attr);
1656                 break;
1657         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1658                 ret = kvm_s390_get_processor_feat(kvm, attr);
1659                 break;
1660         case KVM_S390_VM_CPU_MACHINE_FEAT:
1661                 ret = kvm_s390_get_machine_feat(kvm, attr);
1662                 break;
1663         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1664                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1665                 break;
1666         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1667                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1668                 break;
1669         }
1670         return ret;
1671 }
1672
1673 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1674 {
1675         int ret;
1676
1677         switch (attr->group) {
1678         case KVM_S390_VM_MEM_CTRL:
1679                 ret = kvm_s390_set_mem_control(kvm, attr);
1680                 break;
1681         case KVM_S390_VM_TOD:
1682                 ret = kvm_s390_set_tod(kvm, attr);
1683                 break;
1684         case KVM_S390_VM_CPU_MODEL:
1685                 ret = kvm_s390_set_cpu_model(kvm, attr);
1686                 break;
1687         case KVM_S390_VM_CRYPTO:
1688                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1689                 break;
1690         case KVM_S390_VM_MIGRATION:
1691                 ret = kvm_s390_vm_set_migration(kvm, attr);
1692                 break;
1693         default:
1694                 ret = -ENXIO;
1695                 break;
1696         }
1697
1698         return ret;
1699 }
1700
1701 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1702 {
1703         int ret;
1704
1705         switch (attr->group) {
1706         case KVM_S390_VM_MEM_CTRL:
1707                 ret = kvm_s390_get_mem_control(kvm, attr);
1708                 break;
1709         case KVM_S390_VM_TOD:
1710                 ret = kvm_s390_get_tod(kvm, attr);
1711                 break;
1712         case KVM_S390_VM_CPU_MODEL:
1713                 ret = kvm_s390_get_cpu_model(kvm, attr);
1714                 break;
1715         case KVM_S390_VM_MIGRATION:
1716                 ret = kvm_s390_vm_get_migration(kvm, attr);
1717                 break;
1718         default:
1719                 ret = -ENXIO;
1720                 break;
1721         }
1722
1723         return ret;
1724 }
1725
1726 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1727 {
1728         int ret;
1729
1730         switch (attr->group) {
1731         case KVM_S390_VM_MEM_CTRL:
1732                 switch (attr->attr) {
1733                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1734                 case KVM_S390_VM_MEM_CLR_CMMA:
1735                         ret = sclp.has_cmma ? 0 : -ENXIO;
1736                         break;
1737                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1738                         ret = 0;
1739                         break;
1740                 default:
1741                         ret = -ENXIO;
1742                         break;
1743                 }
1744                 break;
1745         case KVM_S390_VM_TOD:
1746                 switch (attr->attr) {
1747                 case KVM_S390_VM_TOD_LOW:
1748                 case KVM_S390_VM_TOD_HIGH:
1749                         ret = 0;
1750                         break;
1751                 default:
1752                         ret = -ENXIO;
1753                         break;
1754                 }
1755                 break;
1756         case KVM_S390_VM_CPU_MODEL:
1757                 switch (attr->attr) {
1758                 case KVM_S390_VM_CPU_PROCESSOR:
1759                 case KVM_S390_VM_CPU_MACHINE:
1760                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1761                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1762                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1763                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1764                         ret = 0;
1765                         break;
1766                 default:
1767                         ret = -ENXIO;
1768                         break;
1769                 }
1770                 break;
1771         case KVM_S390_VM_CRYPTO:
1772                 switch (attr->attr) {
1773                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1774                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1775                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1776                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1777                         ret = 0;
1778                         break;
1779                 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1780                 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1781                         ret = ap_instructions_available() ? 0 : -ENXIO;
1782                         break;
1783                 default:
1784                         ret = -ENXIO;
1785                         break;
1786                 }
1787                 break;
1788         case KVM_S390_VM_MIGRATION:
1789                 ret = 0;
1790                 break;
1791         default:
1792                 ret = -ENXIO;
1793                 break;
1794         }
1795
1796         return ret;
1797 }
1798
1799 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1800 {
1801         uint8_t *keys;
1802         uint64_t hva;
1803         int srcu_idx, i, r = 0;
1804
1805         if (args->flags != 0)
1806                 return -EINVAL;
1807
1808         /* Is this guest using storage keys? */
1809         if (!mm_uses_skeys(current->mm))
1810                 return KVM_S390_GET_SKEYS_NONE;
1811
1812         /* Enforce sane limit on memory allocation */
1813         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1814                 return -EINVAL;
1815
1816         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
1817         if (!keys)
1818                 return -ENOMEM;
1819
1820         mmap_read_lock(current->mm);
1821         srcu_idx = srcu_read_lock(&kvm->srcu);
1822         for (i = 0; i < args->count; i++) {
1823                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1824                 if (kvm_is_error_hva(hva)) {
1825                         r = -EFAULT;
1826                         break;
1827                 }
1828
1829                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1830                 if (r)
1831                         break;
1832         }
1833         srcu_read_unlock(&kvm->srcu, srcu_idx);
1834         mmap_read_unlock(current->mm);
1835
1836         if (!r) {
1837                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1838                                  sizeof(uint8_t) * args->count);
1839                 if (r)
1840                         r = -EFAULT;
1841         }
1842
1843         kvfree(keys);
1844         return r;
1845 }
1846
1847 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1848 {
1849         uint8_t *keys;
1850         uint64_t hva;
1851         int srcu_idx, i, r = 0;
1852         bool unlocked;
1853
1854         if (args->flags != 0)
1855                 return -EINVAL;
1856
1857         /* Enforce sane limit on memory allocation */
1858         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1859                 return -EINVAL;
1860
1861         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
1862         if (!keys)
1863                 return -ENOMEM;
1864
1865         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1866                            sizeof(uint8_t) * args->count);
1867         if (r) {
1868                 r = -EFAULT;
1869                 goto out;
1870         }
1871
1872         /* Enable storage key handling for the guest */
1873         r = s390_enable_skey();
1874         if (r)
1875                 goto out;
1876
1877         i = 0;
1878         mmap_read_lock(current->mm);
1879         srcu_idx = srcu_read_lock(&kvm->srcu);
1880         while (i < args->count) {
1881                 unlocked = false;
1882                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1883                 if (kvm_is_error_hva(hva)) {
1884                         r = -EFAULT;
1885                         break;
1886                 }
1887
1888                 /* Lowest order bit is reserved */
1889                 if (keys[i] & 0x01) {
1890                         r = -EINVAL;
1891                         break;
1892                 }
1893
1894                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1895                 if (r) {
1896                         r = fixup_user_fault(current->mm, hva,
1897                                              FAULT_FLAG_WRITE, &unlocked);
1898                         if (r)
1899                                 break;
1900                 }
1901                 if (!r)
1902                         i++;
1903         }
1904         srcu_read_unlock(&kvm->srcu, srcu_idx);
1905         mmap_read_unlock(current->mm);
1906 out:
1907         kvfree(keys);
1908         return r;
1909 }
1910
1911 /*
1912  * Base address and length must be sent at the start of each block, therefore
1913  * it's cheaper to send some clean data, as long as it's less than the size of
1914  * two longs.
1915  */
1916 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1917 /* for consistency */
1918 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1919
1920 /*
1921  * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1922  * address falls in a hole. In that case the index of one of the memslots
1923  * bordering the hole is returned.
1924  */
1925 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1926 {
1927         int start = 0, end = slots->used_slots;
1928         int slot = atomic_read(&slots->lru_slot);
1929         struct kvm_memory_slot *memslots = slots->memslots;
1930
1931         if (gfn >= memslots[slot].base_gfn &&
1932             gfn < memslots[slot].base_gfn + memslots[slot].npages)
1933                 return slot;
1934
1935         while (start < end) {
1936                 slot = start + (end - start) / 2;
1937
1938                 if (gfn >= memslots[slot].base_gfn)
1939                         end = slot;
1940                 else
1941                         start = slot + 1;
1942         }
1943
1944         if (start >= slots->used_slots)
1945                 return slots->used_slots - 1;
1946
1947         if (gfn >= memslots[start].base_gfn &&
1948             gfn < memslots[start].base_gfn + memslots[start].npages) {
1949                 atomic_set(&slots->lru_slot, start);
1950         }
1951
1952         return start;
1953 }
1954
1955 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1956                               u8 *res, unsigned long bufsize)
1957 {
1958         unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1959
1960         args->count = 0;
1961         while (args->count < bufsize) {
1962                 hva = gfn_to_hva(kvm, cur_gfn);
1963                 /*
1964                  * We return an error if the first value was invalid, but we
1965                  * return successfully if at least one value was copied.
1966                  */
1967                 if (kvm_is_error_hva(hva))
1968                         return args->count ? 0 : -EFAULT;
1969                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1970                         pgstev = 0;
1971                 res[args->count++] = (pgstev >> 24) & 0x43;
1972                 cur_gfn++;
1973         }
1974
1975         return 0;
1976 }
1977
1978 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1979                                               unsigned long cur_gfn)
1980 {
1981         int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1982         struct kvm_memory_slot *ms = slots->memslots + slotidx;
1983         unsigned long ofs = cur_gfn - ms->base_gfn;
1984
1985         if (ms->base_gfn + ms->npages <= cur_gfn) {
1986                 slotidx--;
1987                 /* If we are above the highest slot, wrap around */
1988                 if (slotidx < 0)
1989                         slotidx = slots->used_slots - 1;
1990
1991                 ms = slots->memslots + slotidx;
1992                 ofs = 0;
1993         }
1994         ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1995         while ((slotidx > 0) && (ofs >= ms->npages)) {
1996                 slotidx--;
1997                 ms = slots->memslots + slotidx;
1998                 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1999         }
2000         return ms->base_gfn + ofs;
2001 }
2002
2003 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2004                              u8 *res, unsigned long bufsize)
2005 {
2006         unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2007         struct kvm_memslots *slots = kvm_memslots(kvm);
2008         struct kvm_memory_slot *ms;
2009
2010         if (unlikely(!slots->used_slots))
2011                 return 0;
2012
2013         cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2014         ms = gfn_to_memslot(kvm, cur_gfn);
2015         args->count = 0;
2016         args->start_gfn = cur_gfn;
2017         if (!ms)
2018                 return 0;
2019         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2020         mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2021
2022         while (args->count < bufsize) {
2023                 hva = gfn_to_hva(kvm, cur_gfn);
2024                 if (kvm_is_error_hva(hva))
2025                         return 0;
2026                 /* Decrement only if we actually flipped the bit to 0 */
2027                 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2028                         atomic64_dec(&kvm->arch.cmma_dirty_pages);
2029                 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2030                         pgstev = 0;
2031                 /* Save the value */
2032                 res[args->count++] = (pgstev >> 24) & 0x43;
2033                 /* If the next bit is too far away, stop. */
2034                 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2035                         return 0;
2036                 /* If we reached the previous "next", find the next one */
2037                 if (cur_gfn == next_gfn)
2038                         next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2039                 /* Reached the end of memory or of the buffer, stop */
2040                 if ((next_gfn >= mem_end) ||
2041                     (next_gfn - args->start_gfn >= bufsize))
2042                         return 0;
2043                 cur_gfn++;
2044                 /* Reached the end of the current memslot, take the next one. */
2045                 if (cur_gfn - ms->base_gfn >= ms->npages) {
2046                         ms = gfn_to_memslot(kvm, cur_gfn);
2047                         if (!ms)
2048                                 return 0;
2049                 }
2050         }
2051         return 0;
2052 }
2053
2054 /*
2055  * This function searches for the next page with dirty CMMA attributes, and
2056  * saves the attributes in the buffer up to either the end of the buffer or
2057  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2058  * no trailing clean bytes are saved.
2059  * In case no dirty bits were found, or if CMMA was not enabled or used, the
2060  * output buffer will indicate 0 as length.
2061  */
2062 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2063                                   struct kvm_s390_cmma_log *args)
2064 {
2065         unsigned long bufsize;
2066         int srcu_idx, peek, ret;
2067         u8 *values;
2068
2069         if (!kvm->arch.use_cmma)
2070                 return -ENXIO;
2071         /* Invalid/unsupported flags were specified */
2072         if (args->flags & ~KVM_S390_CMMA_PEEK)
2073                 return -EINVAL;
2074         /* Migration mode query, and we are not doing a migration */
2075         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2076         if (!peek && !kvm->arch.migration_mode)
2077                 return -EINVAL;
2078         /* CMMA is disabled or was not used, or the buffer has length zero */
2079         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2080         if (!bufsize || !kvm->mm->context.uses_cmm) {
2081                 memset(args, 0, sizeof(*args));
2082                 return 0;
2083         }
2084         /* We are not peeking, and there are no dirty pages */
2085         if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2086                 memset(args, 0, sizeof(*args));
2087                 return 0;
2088         }
2089
2090         values = vmalloc(bufsize);
2091         if (!values)
2092                 return -ENOMEM;
2093
2094         mmap_read_lock(kvm->mm);
2095         srcu_idx = srcu_read_lock(&kvm->srcu);
2096         if (peek)
2097                 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2098         else
2099                 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2100         srcu_read_unlock(&kvm->srcu, srcu_idx);
2101         mmap_read_unlock(kvm->mm);
2102
2103         if (kvm->arch.migration_mode)
2104                 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2105         else
2106                 args->remaining = 0;
2107
2108         if (copy_to_user((void __user *)args->values, values, args->count))
2109                 ret = -EFAULT;
2110
2111         vfree(values);
2112         return ret;
2113 }
2114
2115 /*
2116  * This function sets the CMMA attributes for the given pages. If the input
2117  * buffer has zero length, no action is taken, otherwise the attributes are
2118  * set and the mm->context.uses_cmm flag is set.
2119  */
2120 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2121                                   const struct kvm_s390_cmma_log *args)
2122 {
2123         unsigned long hva, mask, pgstev, i;
2124         uint8_t *bits;
2125         int srcu_idx, r = 0;
2126
2127         mask = args->mask;
2128
2129         if (!kvm->arch.use_cmma)
2130                 return -ENXIO;
2131         /* invalid/unsupported flags */
2132         if (args->flags != 0)
2133                 return -EINVAL;
2134         /* Enforce sane limit on memory allocation */
2135         if (args->count > KVM_S390_CMMA_SIZE_MAX)
2136                 return -EINVAL;
2137         /* Nothing to do */
2138         if (args->count == 0)
2139                 return 0;
2140
2141         bits = vmalloc(array_size(sizeof(*bits), args->count));
2142         if (!bits)
2143                 return -ENOMEM;
2144
2145         r = copy_from_user(bits, (void __user *)args->values, args->count);
2146         if (r) {
2147                 r = -EFAULT;
2148                 goto out;
2149         }
2150
2151         mmap_read_lock(kvm->mm);
2152         srcu_idx = srcu_read_lock(&kvm->srcu);
2153         for (i = 0; i < args->count; i++) {
2154                 hva = gfn_to_hva(kvm, args->start_gfn + i);
2155                 if (kvm_is_error_hva(hva)) {
2156                         r = -EFAULT;
2157                         break;
2158                 }
2159
2160                 pgstev = bits[i];
2161                 pgstev = pgstev << 24;
2162                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2163                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2164         }
2165         srcu_read_unlock(&kvm->srcu, srcu_idx);
2166         mmap_read_unlock(kvm->mm);
2167
2168         if (!kvm->mm->context.uses_cmm) {
2169                 mmap_write_lock(kvm->mm);
2170                 kvm->mm->context.uses_cmm = 1;
2171                 mmap_write_unlock(kvm->mm);
2172         }
2173 out:
2174         vfree(bits);
2175         return r;
2176 }
2177
2178 static int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rcp, u16 *rrcp)
2179 {
2180         struct kvm_vcpu *vcpu;
2181         u16 rc, rrc;
2182         int ret = 0;
2183         int i;
2184
2185         /*
2186          * We ignore failures and try to destroy as many CPUs as possible.
2187          * At the same time we must not free the assigned resources when
2188          * this fails, as the ultravisor has still access to that memory.
2189          * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2190          * behind.
2191          * We want to return the first failure rc and rrc, though.
2192          */
2193         kvm_for_each_vcpu(i, vcpu, kvm) {
2194                 mutex_lock(&vcpu->mutex);
2195                 if (kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc) && !ret) {
2196                         *rcp = rc;
2197                         *rrcp = rrc;
2198                         ret = -EIO;
2199                 }
2200                 mutex_unlock(&vcpu->mutex);
2201         }
2202         return ret;
2203 }
2204
2205 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2206 {
2207         int i, r = 0;
2208         u16 dummy;
2209
2210         struct kvm_vcpu *vcpu;
2211
2212         kvm_for_each_vcpu(i, vcpu, kvm) {
2213                 mutex_lock(&vcpu->mutex);
2214                 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2215                 mutex_unlock(&vcpu->mutex);
2216                 if (r)
2217                         break;
2218         }
2219         if (r)
2220                 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2221         return r;
2222 }
2223
2224 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2225 {
2226         int r = 0;
2227         u16 dummy;
2228         void __user *argp = (void __user *)cmd->data;
2229
2230         switch (cmd->cmd) {
2231         case KVM_PV_ENABLE: {
2232                 r = -EINVAL;
2233                 if (kvm_s390_pv_is_protected(kvm))
2234                         break;
2235
2236                 /*
2237                  *  FMT 4 SIE needs esca. As we never switch back to bsca from
2238                  *  esca, we need no cleanup in the error cases below
2239                  */
2240                 r = sca_switch_to_extended(kvm);
2241                 if (r)
2242                         break;
2243
2244                 mmap_write_lock(current->mm);
2245                 r = gmap_mark_unmergeable();
2246                 mmap_write_unlock(current->mm);
2247                 if (r)
2248                         break;
2249
2250                 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2251                 if (r)
2252                         break;
2253
2254                 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2255                 if (r)
2256                         kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2257
2258                 /* we need to block service interrupts from now on */
2259                 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2260                 break;
2261         }
2262         case KVM_PV_DISABLE: {
2263                 r = -EINVAL;
2264                 if (!kvm_s390_pv_is_protected(kvm))
2265                         break;
2266
2267                 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2268                 /*
2269                  * If a CPU could not be destroyed, destroy VM will also fail.
2270                  * There is no point in trying to destroy it. Instead return
2271                  * the rc and rrc from the first CPU that failed destroying.
2272                  */
2273                 if (r)
2274                         break;
2275                 r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
2276
2277                 /* no need to block service interrupts any more */
2278                 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2279                 break;
2280         }
2281         case KVM_PV_SET_SEC_PARMS: {
2282                 struct kvm_s390_pv_sec_parm parms = {};
2283                 void *hdr;
2284
2285                 r = -EINVAL;
2286                 if (!kvm_s390_pv_is_protected(kvm))
2287                         break;
2288
2289                 r = -EFAULT;
2290                 if (copy_from_user(&parms, argp, sizeof(parms)))
2291                         break;
2292
2293                 /* Currently restricted to 8KB */
2294                 r = -EINVAL;
2295                 if (parms.length > PAGE_SIZE * 2)
2296                         break;
2297
2298                 r = -ENOMEM;
2299                 hdr = vmalloc(parms.length);
2300                 if (!hdr)
2301                         break;
2302
2303                 r = -EFAULT;
2304                 if (!copy_from_user(hdr, (void __user *)parms.origin,
2305                                     parms.length))
2306                         r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2307                                                       &cmd->rc, &cmd->rrc);
2308
2309                 vfree(hdr);
2310                 break;
2311         }
2312         case KVM_PV_UNPACK: {
2313                 struct kvm_s390_pv_unp unp = {};
2314
2315                 r = -EINVAL;
2316                 if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2317                         break;
2318
2319                 r = -EFAULT;
2320                 if (copy_from_user(&unp, argp, sizeof(unp)))
2321                         break;
2322
2323                 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2324                                        &cmd->rc, &cmd->rrc);
2325                 break;
2326         }
2327         case KVM_PV_VERIFY: {
2328                 r = -EINVAL;
2329                 if (!kvm_s390_pv_is_protected(kvm))
2330                         break;
2331
2332                 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2333                                   UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2334                 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2335                              cmd->rrc);
2336                 break;
2337         }
2338         case KVM_PV_PREP_RESET: {
2339                 r = -EINVAL;
2340                 if (!kvm_s390_pv_is_protected(kvm))
2341                         break;
2342
2343                 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2344                                   UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2345                 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2346                              cmd->rc, cmd->rrc);
2347                 break;
2348         }
2349         case KVM_PV_UNSHARE_ALL: {
2350                 r = -EINVAL;
2351                 if (!kvm_s390_pv_is_protected(kvm))
2352                         break;
2353
2354                 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2355                                   UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2356                 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2357                              cmd->rc, cmd->rrc);
2358                 break;
2359         }
2360         default:
2361                 r = -ENOTTY;
2362         }
2363         return r;
2364 }
2365
2366 long kvm_arch_vm_ioctl(struct file *filp,
2367                        unsigned int ioctl, unsigned long arg)
2368 {
2369         struct kvm *kvm = filp->private_data;
2370         void __user *argp = (void __user *)arg;
2371         struct kvm_device_attr attr;
2372         int r;
2373
2374         switch (ioctl) {
2375         case KVM_S390_INTERRUPT: {
2376                 struct kvm_s390_interrupt s390int;
2377
2378                 r = -EFAULT;
2379                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2380                         break;
2381                 r = kvm_s390_inject_vm(kvm, &s390int);
2382                 break;
2383         }
2384         case KVM_CREATE_IRQCHIP: {
2385                 struct kvm_irq_routing_entry routing;
2386
2387                 r = -EINVAL;
2388                 if (kvm->arch.use_irqchip) {
2389                         /* Set up dummy routing. */
2390                         memset(&routing, 0, sizeof(routing));
2391                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2392                 }
2393                 break;
2394         }
2395         case KVM_SET_DEVICE_ATTR: {
2396                 r = -EFAULT;
2397                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2398                         break;
2399                 r = kvm_s390_vm_set_attr(kvm, &attr);
2400                 break;
2401         }
2402         case KVM_GET_DEVICE_ATTR: {
2403                 r = -EFAULT;
2404                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2405                         break;
2406                 r = kvm_s390_vm_get_attr(kvm, &attr);
2407                 break;
2408         }
2409         case KVM_HAS_DEVICE_ATTR: {
2410                 r = -EFAULT;
2411                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2412                         break;
2413                 r = kvm_s390_vm_has_attr(kvm, &attr);
2414                 break;
2415         }
2416         case KVM_S390_GET_SKEYS: {
2417                 struct kvm_s390_skeys args;
2418
2419                 r = -EFAULT;
2420                 if (copy_from_user(&args, argp,
2421                                    sizeof(struct kvm_s390_skeys)))
2422                         break;
2423                 r = kvm_s390_get_skeys(kvm, &args);
2424                 break;
2425         }
2426         case KVM_S390_SET_SKEYS: {
2427                 struct kvm_s390_skeys args;
2428
2429                 r = -EFAULT;
2430                 if (copy_from_user(&args, argp,
2431                                    sizeof(struct kvm_s390_skeys)))
2432                         break;
2433                 r = kvm_s390_set_skeys(kvm, &args);
2434                 break;
2435         }
2436         case KVM_S390_GET_CMMA_BITS: {
2437                 struct kvm_s390_cmma_log args;
2438
2439                 r = -EFAULT;
2440                 if (copy_from_user(&args, argp, sizeof(args)))
2441                         break;
2442                 mutex_lock(&kvm->slots_lock);
2443                 r = kvm_s390_get_cmma_bits(kvm, &args);
2444                 mutex_unlock(&kvm->slots_lock);
2445                 if (!r) {
2446                         r = copy_to_user(argp, &args, sizeof(args));
2447                         if (r)
2448                                 r = -EFAULT;
2449                 }
2450                 break;
2451         }
2452         case KVM_S390_SET_CMMA_BITS: {
2453                 struct kvm_s390_cmma_log args;
2454
2455                 r = -EFAULT;
2456                 if (copy_from_user(&args, argp, sizeof(args)))
2457                         break;
2458                 mutex_lock(&kvm->slots_lock);
2459                 r = kvm_s390_set_cmma_bits(kvm, &args);
2460                 mutex_unlock(&kvm->slots_lock);
2461                 break;
2462         }
2463         case KVM_S390_PV_COMMAND: {
2464                 struct kvm_pv_cmd args;
2465
2466                 /* protvirt means user sigp */
2467                 kvm->arch.user_cpu_state_ctrl = 1;
2468                 r = 0;
2469                 if (!is_prot_virt_host()) {
2470                         r = -EINVAL;
2471                         break;
2472                 }
2473                 if (copy_from_user(&args, argp, sizeof(args))) {
2474                         r = -EFAULT;
2475                         break;
2476                 }
2477                 if (args.flags) {
2478                         r = -EINVAL;
2479                         break;
2480                 }
2481                 mutex_lock(&kvm->lock);
2482                 r = kvm_s390_handle_pv(kvm, &args);
2483                 mutex_unlock(&kvm->lock);
2484                 if (copy_to_user(argp, &args, sizeof(args))) {
2485                         r = -EFAULT;
2486                         break;
2487                 }
2488                 break;
2489         }
2490         default:
2491                 r = -ENOTTY;
2492         }
2493
2494         return r;
2495 }
2496
2497 static int kvm_s390_apxa_installed(void)
2498 {
2499         struct ap_config_info info;
2500
2501         if (ap_instructions_available()) {
2502                 if (ap_qci(&info) == 0)
2503                         return info.apxa;
2504         }
2505
2506         return 0;
2507 }
2508
2509 /*
2510  * The format of the crypto control block (CRYCB) is specified in the 3 low
2511  * order bits of the CRYCB designation (CRYCBD) field as follows:
2512  * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2513  *           AP extended addressing (APXA) facility are installed.
2514  * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2515  * Format 2: Both the APXA and MSAX3 facilities are installed
2516  */
2517 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2518 {
2519         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2520
2521         /* Clear the CRYCB format bits - i.e., set format 0 by default */
2522         kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2523
2524         /* Check whether MSAX3 is installed */
2525         if (!test_kvm_facility(kvm, 76))
2526                 return;
2527
2528         if (kvm_s390_apxa_installed())
2529                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2530         else
2531                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2532 }
2533
2534 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2535                                unsigned long *aqm, unsigned long *adm)
2536 {
2537         struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2538
2539         mutex_lock(&kvm->lock);
2540         kvm_s390_vcpu_block_all(kvm);
2541
2542         switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2543         case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2544                 memcpy(crycb->apcb1.apm, apm, 32);
2545                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2546                          apm[0], apm[1], apm[2], apm[3]);
2547                 memcpy(crycb->apcb1.aqm, aqm, 32);
2548                 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2549                          aqm[0], aqm[1], aqm[2], aqm[3]);
2550                 memcpy(crycb->apcb1.adm, adm, 32);
2551                 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2552                          adm[0], adm[1], adm[2], adm[3]);
2553                 break;
2554         case CRYCB_FORMAT1:
2555         case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2556                 memcpy(crycb->apcb0.apm, apm, 8);
2557                 memcpy(crycb->apcb0.aqm, aqm, 2);
2558                 memcpy(crycb->apcb0.adm, adm, 2);
2559                 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2560                          apm[0], *((unsigned short *)aqm),
2561                          *((unsigned short *)adm));
2562                 break;
2563         default:        /* Can not happen */
2564                 break;
2565         }
2566
2567         /* recreate the shadow crycb for each vcpu */
2568         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2569         kvm_s390_vcpu_unblock_all(kvm);
2570         mutex_unlock(&kvm->lock);
2571 }
2572 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2573
2574 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2575 {
2576         mutex_lock(&kvm->lock);
2577         kvm_s390_vcpu_block_all(kvm);
2578
2579         memset(&kvm->arch.crypto.crycb->apcb0, 0,
2580                sizeof(kvm->arch.crypto.crycb->apcb0));
2581         memset(&kvm->arch.crypto.crycb->apcb1, 0,
2582                sizeof(kvm->arch.crypto.crycb->apcb1));
2583
2584         VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2585         /* recreate the shadow crycb for each vcpu */
2586         kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2587         kvm_s390_vcpu_unblock_all(kvm);
2588         mutex_unlock(&kvm->lock);
2589 }
2590 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2591
2592 static u64 kvm_s390_get_initial_cpuid(void)
2593 {
2594         struct cpuid cpuid;
2595
2596         get_cpu_id(&cpuid);
2597         cpuid.version = 0xff;
2598         return *((u64 *) &cpuid);
2599 }
2600
2601 static void kvm_s390_crypto_init(struct kvm *kvm)
2602 {
2603         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2604         kvm_s390_set_crycb_format(kvm);
2605
2606         if (!test_kvm_facility(kvm, 76))
2607                 return;
2608
2609         /* Enable AES/DEA protected key functions by default */
2610         kvm->arch.crypto.aes_kw = 1;
2611         kvm->arch.crypto.dea_kw = 1;
2612         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2613                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2614         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2615                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2616 }
2617
2618 static void sca_dispose(struct kvm *kvm)
2619 {
2620         if (kvm->arch.use_esca)
2621                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2622         else
2623                 free_page((unsigned long)(kvm->arch.sca));
2624         kvm->arch.sca = NULL;
2625 }
2626
2627 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2628 {
2629         gfp_t alloc_flags = GFP_KERNEL_ACCOUNT;
2630         int i, rc;
2631         char debug_name[16];
2632         static unsigned long sca_offset;
2633
2634         rc = -EINVAL;
2635 #ifdef CONFIG_KVM_S390_UCONTROL
2636         if (type & ~KVM_VM_S390_UCONTROL)
2637                 goto out_err;
2638         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2639                 goto out_err;
2640 #else
2641         if (type)
2642                 goto out_err;
2643 #endif
2644
2645         rc = s390_enable_sie();
2646         if (rc)
2647                 goto out_err;
2648
2649         rc = -ENOMEM;
2650
2651         if (!sclp.has_64bscao)
2652                 alloc_flags |= GFP_DMA;
2653         rwlock_init(&kvm->arch.sca_lock);
2654         /* start with basic SCA */
2655         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2656         if (!kvm->arch.sca)
2657                 goto out_err;
2658         mutex_lock(&kvm_lock);
2659         sca_offset += 16;
2660         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2661                 sca_offset = 0;
2662         kvm->arch.sca = (struct bsca_block *)
2663                         ((char *) kvm->arch.sca + sca_offset);
2664         mutex_unlock(&kvm_lock);
2665
2666         sprintf(debug_name, "kvm-%u", current->pid);
2667
2668         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2669         if (!kvm->arch.dbf)
2670                 goto out_err;
2671
2672         BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2673         kvm->arch.sie_page2 =
2674              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
2675         if (!kvm->arch.sie_page2)
2676                 goto out_err;
2677
2678         kvm->arch.sie_page2->kvm = kvm;
2679         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2680
2681         for (i = 0; i < kvm_s390_fac_size(); i++) {
2682                 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2683                                               (kvm_s390_fac_base[i] |
2684                                                kvm_s390_fac_ext[i]);
2685                 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2686                                               kvm_s390_fac_base[i];
2687         }
2688         kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2689
2690         /* we are always in czam mode - even on pre z14 machines */
2691         set_kvm_facility(kvm->arch.model.fac_mask, 138);
2692         set_kvm_facility(kvm->arch.model.fac_list, 138);
2693         /* we emulate STHYI in kvm */
2694         set_kvm_facility(kvm->arch.model.fac_mask, 74);
2695         set_kvm_facility(kvm->arch.model.fac_list, 74);
2696         if (MACHINE_HAS_TLB_GUEST) {
2697                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2698                 set_kvm_facility(kvm->arch.model.fac_list, 147);
2699         }
2700
2701         if (css_general_characteristics.aiv && test_facility(65))
2702                 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2703
2704         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2705         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2706
2707         kvm_s390_crypto_init(kvm);
2708
2709         mutex_init(&kvm->arch.float_int.ais_lock);
2710         spin_lock_init(&kvm->arch.float_int.lock);
2711         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2712                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2713         init_waitqueue_head(&kvm->arch.ipte_wq);
2714         mutex_init(&kvm->arch.ipte_mutex);
2715
2716         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2717         VM_EVENT(kvm, 3, "vm created with type %lu", type);
2718
2719         if (type & KVM_VM_S390_UCONTROL) {
2720                 kvm->arch.gmap = NULL;
2721                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2722         } else {
2723                 if (sclp.hamax == U64_MAX)
2724                         kvm->arch.mem_limit = TASK_SIZE_MAX;
2725                 else
2726                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2727                                                     sclp.hamax + 1);
2728                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2729                 if (!kvm->arch.gmap)
2730                         goto out_err;
2731                 kvm->arch.gmap->private = kvm;
2732                 kvm->arch.gmap->pfault_enabled = 0;
2733         }
2734
2735         kvm->arch.use_pfmfi = sclp.has_pfmfi;
2736         kvm->arch.use_skf = sclp.has_skey;
2737         spin_lock_init(&kvm->arch.start_stop_lock);
2738         kvm_s390_vsie_init(kvm);
2739         if (use_gisa)
2740                 kvm_s390_gisa_init(kvm);
2741         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2742
2743         return 0;
2744 out_err:
2745         free_page((unsigned long)kvm->arch.sie_page2);
2746         debug_unregister(kvm->arch.dbf);
2747         sca_dispose(kvm);
2748         KVM_EVENT(3, "creation of vm failed: %d", rc);
2749         return rc;
2750 }
2751
2752 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2753 {
2754         u16 rc, rrc;
2755
2756         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2757         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2758         kvm_s390_clear_local_irqs(vcpu);
2759         kvm_clear_async_pf_completion_queue(vcpu);
2760         if (!kvm_is_ucontrol(vcpu->kvm))
2761                 sca_del_vcpu(vcpu);
2762
2763         if (kvm_is_ucontrol(vcpu->kvm))
2764                 gmap_remove(vcpu->arch.gmap);
2765
2766         if (vcpu->kvm->arch.use_cmma)
2767                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2768         /* We can not hold the vcpu mutex here, we are already dying */
2769         if (kvm_s390_pv_cpu_get_handle(vcpu))
2770                 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
2771         free_page((unsigned long)(vcpu->arch.sie_block));
2772 }
2773
2774 static void kvm_free_vcpus(struct kvm *kvm)
2775 {
2776         unsigned int i;
2777         struct kvm_vcpu *vcpu;
2778
2779         kvm_for_each_vcpu(i, vcpu, kvm)
2780                 kvm_vcpu_destroy(vcpu);
2781
2782         mutex_lock(&kvm->lock);
2783         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2784                 kvm->vcpus[i] = NULL;
2785
2786         atomic_set(&kvm->online_vcpus, 0);
2787         mutex_unlock(&kvm->lock);
2788 }
2789
2790 void kvm_arch_destroy_vm(struct kvm *kvm)
2791 {
2792         u16 rc, rrc;
2793
2794         kvm_free_vcpus(kvm);
2795         sca_dispose(kvm);
2796         kvm_s390_gisa_destroy(kvm);
2797         /*
2798          * We are already at the end of life and kvm->lock is not taken.
2799          * This is ok as the file descriptor is closed by now and nobody
2800          * can mess with the pv state. To avoid lockdep_assert_held from
2801          * complaining we do not use kvm_s390_pv_is_protected.
2802          */
2803         if (kvm_s390_pv_get_handle(kvm))
2804                 kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
2805         debug_unregister(kvm->arch.dbf);
2806         free_page((unsigned long)kvm->arch.sie_page2);
2807         if (!kvm_is_ucontrol(kvm))
2808                 gmap_remove(kvm->arch.gmap);
2809         kvm_s390_destroy_adapters(kvm);
2810         kvm_s390_clear_float_irqs(kvm);
2811         kvm_s390_vsie_destroy(kvm);
2812         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2813 }
2814
2815 /* Section: vcpu related */
2816 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2817 {
2818         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2819         if (!vcpu->arch.gmap)
2820                 return -ENOMEM;
2821         vcpu->arch.gmap->private = vcpu->kvm;
2822
2823         return 0;
2824 }
2825
2826 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2827 {
2828         if (!kvm_s390_use_sca_entries())
2829                 return;
2830         read_lock(&vcpu->kvm->arch.sca_lock);
2831         if (vcpu->kvm->arch.use_esca) {
2832                 struct esca_block *sca = vcpu->kvm->arch.sca;
2833
2834                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2835                 sca->cpu[vcpu->vcpu_id].sda = 0;
2836         } else {
2837                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2838
2839                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2840                 sca->cpu[vcpu->vcpu_id].sda = 0;
2841         }
2842         read_unlock(&vcpu->kvm->arch.sca_lock);
2843 }
2844
2845 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2846 {
2847         if (!kvm_s390_use_sca_entries()) {
2848                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2849
2850                 /* we still need the basic sca for the ipte control */
2851                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2852                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2853                 return;
2854         }
2855         read_lock(&vcpu->kvm->arch.sca_lock);
2856         if (vcpu->kvm->arch.use_esca) {
2857                 struct esca_block *sca = vcpu->kvm->arch.sca;
2858
2859                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2860                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2861                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2862                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2863                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2864         } else {
2865                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2866
2867                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2868                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2869                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2870                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2871         }
2872         read_unlock(&vcpu->kvm->arch.sca_lock);
2873 }
2874
2875 /* Basic SCA to Extended SCA data copy routines */
2876 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2877 {
2878         d->sda = s->sda;
2879         d->sigp_ctrl.c = s->sigp_ctrl.c;
2880         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2881 }
2882
2883 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2884 {
2885         int i;
2886
2887         d->ipte_control = s->ipte_control;
2888         d->mcn[0] = s->mcn;
2889         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2890                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2891 }
2892
2893 static int sca_switch_to_extended(struct kvm *kvm)
2894 {
2895         struct bsca_block *old_sca = kvm->arch.sca;
2896         struct esca_block *new_sca;
2897         struct kvm_vcpu *vcpu;
2898         unsigned int vcpu_idx;
2899         u32 scaol, scaoh;
2900
2901         if (kvm->arch.use_esca)
2902                 return 0;
2903
2904         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL_ACCOUNT | __GFP_ZERO);
2905         if (!new_sca)
2906                 return -ENOMEM;
2907
2908         scaoh = (u32)((u64)(new_sca) >> 32);
2909         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2910
2911         kvm_s390_vcpu_block_all(kvm);
2912         write_lock(&kvm->arch.sca_lock);
2913
2914         sca_copy_b_to_e(new_sca, old_sca);
2915
2916         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2917                 vcpu->arch.sie_block->scaoh = scaoh;
2918                 vcpu->arch.sie_block->scaol = scaol;
2919                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2920         }
2921         kvm->arch.sca = new_sca;
2922         kvm->arch.use_esca = 1;
2923
2924         write_unlock(&kvm->arch.sca_lock);
2925         kvm_s390_vcpu_unblock_all(kvm);
2926
2927         free_page((unsigned long)old_sca);
2928
2929         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2930                  old_sca, kvm->arch.sca);
2931         return 0;
2932 }
2933
2934 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2935 {
2936         int rc;
2937
2938         if (!kvm_s390_use_sca_entries()) {
2939                 if (id < KVM_MAX_VCPUS)
2940                         return true;
2941                 return false;
2942         }
2943         if (id < KVM_S390_BSCA_CPU_SLOTS)
2944                 return true;
2945         if (!sclp.has_esca || !sclp.has_64bscao)
2946                 return false;
2947
2948         mutex_lock(&kvm->lock);
2949         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2950         mutex_unlock(&kvm->lock);
2951
2952         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2953 }
2954
2955 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2956 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2957 {
2958         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2959         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2960         vcpu->arch.cputm_start = get_tod_clock_fast();
2961         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2962 }
2963
2964 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2965 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2966 {
2967         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2968         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2969         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2970         vcpu->arch.cputm_start = 0;
2971         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2972 }
2973
2974 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2975 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2976 {
2977         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2978         vcpu->arch.cputm_enabled = true;
2979         __start_cpu_timer_accounting(vcpu);
2980 }
2981
2982 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2983 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2984 {
2985         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2986         __stop_cpu_timer_accounting(vcpu);
2987         vcpu->arch.cputm_enabled = false;
2988 }
2989
2990 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2991 {
2992         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2993         __enable_cpu_timer_accounting(vcpu);
2994         preempt_enable();
2995 }
2996
2997 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2998 {
2999         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3000         __disable_cpu_timer_accounting(vcpu);
3001         preempt_enable();
3002 }
3003
3004 /* set the cpu timer - may only be called from the VCPU thread itself */
3005 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3006 {
3007         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3008         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3009         if (vcpu->arch.cputm_enabled)
3010                 vcpu->arch.cputm_start = get_tod_clock_fast();
3011         vcpu->arch.sie_block->cputm = cputm;
3012         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3013         preempt_enable();
3014 }
3015
3016 /* update and get the cpu timer - can also be called from other VCPU threads */
3017 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3018 {
3019         unsigned int seq;
3020         __u64 value;
3021
3022         if (unlikely(!vcpu->arch.cputm_enabled))
3023                 return vcpu->arch.sie_block->cputm;
3024
3025         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3026         do {
3027                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3028                 /*
3029                  * If the writer would ever execute a read in the critical
3030                  * section, e.g. in irq context, we have a deadlock.
3031                  */
3032                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3033                 value = vcpu->arch.sie_block->cputm;
3034                 /* if cputm_start is 0, accounting is being started/stopped */
3035                 if (likely(vcpu->arch.cputm_start))
3036                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3037         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3038         preempt_enable();
3039         return value;
3040 }
3041
3042 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3043 {
3044
3045         gmap_enable(vcpu->arch.enabled_gmap);
3046         kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3047         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3048                 __start_cpu_timer_accounting(vcpu);
3049         vcpu->cpu = cpu;
3050 }
3051
3052 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3053 {
3054         vcpu->cpu = -1;
3055         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3056                 __stop_cpu_timer_accounting(vcpu);
3057         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3058         vcpu->arch.enabled_gmap = gmap_get_enabled();
3059         gmap_disable(vcpu->arch.enabled_gmap);
3060
3061 }
3062
3063 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3064 {
3065         mutex_lock(&vcpu->kvm->lock);
3066         preempt_disable();
3067         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3068         vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3069         preempt_enable();
3070         mutex_unlock(&vcpu->kvm->lock);
3071         if (!kvm_is_ucontrol(vcpu->kvm)) {
3072                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3073                 sca_add_vcpu(vcpu);
3074         }
3075         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3076                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3077         /* make vcpu_load load the right gmap on the first trigger */
3078         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
3079 }
3080
3081 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3082 {
3083         if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3084             test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3085                 return true;
3086         return false;
3087 }
3088
3089 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3090 {
3091         /* At least one ECC subfunction must be present */
3092         return kvm_has_pckmo_subfunc(kvm, 32) ||
3093                kvm_has_pckmo_subfunc(kvm, 33) ||
3094                kvm_has_pckmo_subfunc(kvm, 34) ||
3095                kvm_has_pckmo_subfunc(kvm, 40) ||
3096                kvm_has_pckmo_subfunc(kvm, 41);
3097
3098 }
3099
3100 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3101 {
3102         /*
3103          * If the AP instructions are not being interpreted and the MSAX3
3104          * facility is not configured for the guest, there is nothing to set up.
3105          */
3106         if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3107                 return;
3108
3109         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3110         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3111         vcpu->arch.sie_block->eca &= ~ECA_APIE;
3112         vcpu->arch.sie_block->ecd &= ~ECD_ECC;
3113
3114         if (vcpu->kvm->arch.crypto.apie)
3115                 vcpu->arch.sie_block->eca |= ECA_APIE;
3116
3117         /* Set up protected key support */
3118         if (vcpu->kvm->arch.crypto.aes_kw) {
3119                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3120                 /* ecc is also wrapped with AES key */
3121                 if (kvm_has_pckmo_ecc(vcpu->kvm))
3122                         vcpu->arch.sie_block->ecd |= ECD_ECC;
3123         }
3124
3125         if (vcpu->kvm->arch.crypto.dea_kw)
3126                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3127 }
3128
3129 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3130 {
3131         free_page(vcpu->arch.sie_block->cbrlo);
3132         vcpu->arch.sie_block->cbrlo = 0;
3133 }
3134
3135 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3136 {
3137         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL_ACCOUNT);
3138         if (!vcpu->arch.sie_block->cbrlo)
3139                 return -ENOMEM;
3140         return 0;
3141 }
3142
3143 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3144 {
3145         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3146
3147         vcpu->arch.sie_block->ibc = model->ibc;
3148         if (test_kvm_facility(vcpu->kvm, 7))
3149                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
3150 }
3151
3152 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3153 {
3154         int rc = 0;
3155         u16 uvrc, uvrrc;
3156
3157         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3158                                                     CPUSTAT_SM |
3159                                                     CPUSTAT_STOPPED);
3160
3161         if (test_kvm_facility(vcpu->kvm, 78))
3162                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3163         else if (test_kvm_facility(vcpu->kvm, 8))
3164                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3165
3166         kvm_s390_vcpu_setup_model(vcpu);
3167
3168         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
3169         if (MACHINE_HAS_ESOP)
3170                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3171         if (test_kvm_facility(vcpu->kvm, 9))
3172                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3173         if (test_kvm_facility(vcpu->kvm, 73))
3174                 vcpu->arch.sie_block->ecb |= ECB_TE;
3175
3176         if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3177                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3178         if (test_kvm_facility(vcpu->kvm, 130))
3179                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3180         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3181         if (sclp.has_cei)
3182                 vcpu->arch.sie_block->eca |= ECA_CEI;
3183         if (sclp.has_ib)
3184                 vcpu->arch.sie_block->eca |= ECA_IB;
3185         if (sclp.has_siif)
3186                 vcpu->arch.sie_block->eca |= ECA_SII;
3187         if (sclp.has_sigpif)
3188                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3189         if (test_kvm_facility(vcpu->kvm, 129)) {
3190                 vcpu->arch.sie_block->eca |= ECA_VX;
3191                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3192         }
3193         if (test_kvm_facility(vcpu->kvm, 139))
3194                 vcpu->arch.sie_block->ecd |= ECD_MEF;
3195         if (test_kvm_facility(vcpu->kvm, 156))
3196                 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3197         if (vcpu->arch.sie_block->gd) {
3198                 vcpu->arch.sie_block->eca |= ECA_AIV;
3199                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3200                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3201         }
3202         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
3203                                         | SDNXC;
3204         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
3205
3206         if (sclp.has_kss)
3207                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3208         else
3209                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3210
3211         if (vcpu->kvm->arch.use_cmma) {
3212                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3213                 if (rc)
3214                         return rc;
3215         }
3216         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3217         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
3218
3219         vcpu->arch.sie_block->hpid = HPID_KVM;
3220
3221         kvm_s390_vcpu_crypto_setup(vcpu);
3222
3223         mutex_lock(&vcpu->kvm->lock);
3224         if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3225                 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3226                 if (rc)
3227                         kvm_s390_vcpu_unsetup_cmma(vcpu);
3228         }
3229         mutex_unlock(&vcpu->kvm->lock);
3230
3231         return rc;
3232 }
3233
3234 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3235 {
3236         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3237                 return -EINVAL;
3238         return 0;
3239 }
3240
3241 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3242 {
3243         struct sie_page *sie_page;
3244         int rc;
3245
3246         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3247         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3248         if (!sie_page)
3249                 return -ENOMEM;
3250
3251         vcpu->arch.sie_block = &sie_page->sie_block;
3252         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3253
3254         /* the real guest size will always be smaller than msl */
3255         vcpu->arch.sie_block->mso = 0;
3256         vcpu->arch.sie_block->msl = sclp.hamax;
3257
3258         vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3259         spin_lock_init(&vcpu->arch.local_int.lock);
3260         vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
3261         if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3262                 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3263         seqcount_init(&vcpu->arch.cputm_seqcount);
3264
3265         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3266         kvm_clear_async_pf_completion_queue(vcpu);
3267         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3268                                     KVM_SYNC_GPRS |
3269                                     KVM_SYNC_ACRS |
3270                                     KVM_SYNC_CRS |
3271                                     KVM_SYNC_ARCH0 |
3272                                     KVM_SYNC_PFAULT |
3273                                     KVM_SYNC_DIAG318;
3274         kvm_s390_set_prefix(vcpu, 0);
3275         if (test_kvm_facility(vcpu->kvm, 64))
3276                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3277         if (test_kvm_facility(vcpu->kvm, 82))
3278                 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3279         if (test_kvm_facility(vcpu->kvm, 133))
3280                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3281         if (test_kvm_facility(vcpu->kvm, 156))
3282                 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3283         /* fprs can be synchronized via vrs, even if the guest has no vx. With
3284          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3285          */
3286         if (MACHINE_HAS_VX)
3287                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3288         else
3289                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3290
3291         if (kvm_is_ucontrol(vcpu->kvm)) {
3292                 rc = __kvm_ucontrol_vcpu_init(vcpu);
3293                 if (rc)
3294                         goto out_free_sie_block;
3295         }
3296
3297         VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3298                  vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3299         trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3300
3301         rc = kvm_s390_vcpu_setup(vcpu);
3302         if (rc)
3303                 goto out_ucontrol_uninit;
3304         return 0;
3305
3306 out_ucontrol_uninit:
3307         if (kvm_is_ucontrol(vcpu->kvm))
3308                 gmap_remove(vcpu->arch.gmap);
3309 out_free_sie_block:
3310         free_page((unsigned long)(vcpu->arch.sie_block));
3311         return rc;
3312 }
3313
3314 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3315 {
3316         return kvm_s390_vcpu_has_irq(vcpu, 0);
3317 }
3318
3319 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3320 {
3321         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3322 }
3323
3324 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3325 {
3326         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3327         exit_sie(vcpu);
3328 }
3329
3330 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3331 {
3332         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3333 }
3334
3335 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3336 {
3337         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3338         exit_sie(vcpu);
3339 }
3340
3341 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3342 {
3343         return atomic_read(&vcpu->arch.sie_block->prog20) &
3344                (PROG_BLOCK_SIE | PROG_REQUEST);
3345 }
3346
3347 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3348 {
3349         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3350 }
3351
3352 /*
3353  * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3354  * If the CPU is not running (e.g. waiting as idle) the function will
3355  * return immediately. */
3356 void exit_sie(struct kvm_vcpu *vcpu)
3357 {
3358         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3359         kvm_s390_vsie_kick(vcpu);
3360         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3361                 cpu_relax();
3362 }
3363
3364 /* Kick a guest cpu out of SIE to process a request synchronously */
3365 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3366 {
3367         kvm_make_request(req, vcpu);
3368         kvm_s390_vcpu_request(vcpu);
3369 }
3370
3371 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3372                               unsigned long end)
3373 {
3374         struct kvm *kvm = gmap->private;
3375         struct kvm_vcpu *vcpu;
3376         unsigned long prefix;
3377         int i;
3378
3379         if (gmap_is_shadow(gmap))
3380                 return;
3381         if (start >= 1UL << 31)
3382                 /* We are only interested in prefix pages */
3383                 return;
3384         kvm_for_each_vcpu(i, vcpu, kvm) {
3385                 /* match against both prefix pages */
3386                 prefix = kvm_s390_get_prefix(vcpu);
3387                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3388                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3389                                    start, end);
3390                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3391                 }
3392         }
3393 }
3394
3395 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3396 {
3397         /* do not poll with more than halt_poll_max_steal percent of steal time */
3398         if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3399             halt_poll_max_steal) {
3400                 vcpu->stat.halt_no_poll_steal++;
3401                 return true;
3402         }
3403         return false;
3404 }
3405
3406 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3407 {
3408         /* kvm common code refers to this, but never calls it */
3409         BUG();
3410         return 0;
3411 }
3412
3413 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3414                                            struct kvm_one_reg *reg)
3415 {
3416         int r = -EINVAL;
3417
3418         switch (reg->id) {
3419         case KVM_REG_S390_TODPR:
3420                 r = put_user(vcpu->arch.sie_block->todpr,
3421                              (u32 __user *)reg->addr);
3422                 break;
3423         case KVM_REG_S390_EPOCHDIFF:
3424                 r = put_user(vcpu->arch.sie_block->epoch,
3425                              (u64 __user *)reg->addr);
3426                 break;
3427         case KVM_REG_S390_CPU_TIMER:
3428                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3429                              (u64 __user *)reg->addr);
3430                 break;
3431         case KVM_REG_S390_CLOCK_COMP:
3432                 r = put_user(vcpu->arch.sie_block->ckc,
3433                              (u64 __user *)reg->addr);
3434                 break;
3435         case KVM_REG_S390_PFTOKEN:
3436                 r = put_user(vcpu->arch.pfault_token,
3437                              (u64 __user *)reg->addr);
3438                 break;
3439         case KVM_REG_S390_PFCOMPARE:
3440                 r = put_user(vcpu->arch.pfault_compare,
3441                              (u64 __user *)reg->addr);
3442                 break;
3443         case KVM_REG_S390_PFSELECT:
3444                 r = put_user(vcpu->arch.pfault_select,
3445                              (u64 __user *)reg->addr);
3446                 break;
3447         case KVM_REG_S390_PP:
3448                 r = put_user(vcpu->arch.sie_block->pp,
3449                              (u64 __user *)reg->addr);
3450                 break;
3451         case KVM_REG_S390_GBEA:
3452                 r = put_user(vcpu->arch.sie_block->gbea,
3453                              (u64 __user *)reg->addr);
3454                 break;
3455         default:
3456                 break;
3457         }
3458
3459         return r;
3460 }
3461
3462 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3463                                            struct kvm_one_reg *reg)
3464 {
3465         int r = -EINVAL;
3466         __u64 val;
3467
3468         switch (reg->id) {
3469         case KVM_REG_S390_TODPR:
3470                 r = get_user(vcpu->arch.sie_block->todpr,
3471                              (u32 __user *)reg->addr);
3472                 break;
3473         case KVM_REG_S390_EPOCHDIFF:
3474                 r = get_user(vcpu->arch.sie_block->epoch,
3475                              (u64 __user *)reg->addr);
3476                 break;
3477         case KVM_REG_S390_CPU_TIMER:
3478                 r = get_user(val, (u64 __user *)reg->addr);
3479                 if (!r)
3480                         kvm_s390_set_cpu_timer(vcpu, val);
3481                 break;
3482         case KVM_REG_S390_CLOCK_COMP:
3483                 r = get_user(vcpu->arch.sie_block->ckc,
3484                              (u64 __user *)reg->addr);
3485                 break;
3486         case KVM_REG_S390_PFTOKEN:
3487                 r = get_user(vcpu->arch.pfault_token,
3488                              (u64 __user *)reg->addr);
3489                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3490                         kvm_clear_async_pf_completion_queue(vcpu);
3491                 break;
3492         case KVM_REG_S390_PFCOMPARE:
3493                 r = get_user(vcpu->arch.pfault_compare,
3494                              (u64 __user *)reg->addr);
3495                 break;
3496         case KVM_REG_S390_PFSELECT:
3497                 r = get_user(vcpu->arch.pfault_select,
3498                              (u64 __user *)reg->addr);
3499                 break;
3500         case KVM_REG_S390_PP:
3501                 r = get_user(vcpu->arch.sie_block->pp,
3502                              (u64 __user *)reg->addr);
3503                 break;
3504         case KVM_REG_S390_GBEA:
3505                 r = get_user(vcpu->arch.sie_block->gbea,
3506                              (u64 __user *)reg->addr);
3507                 break;
3508         default:
3509                 break;
3510         }
3511
3512         return r;
3513 }
3514
3515 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
3516 {
3517         vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3518         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3519         memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3520
3521         kvm_clear_async_pf_completion_queue(vcpu);
3522         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3523                 kvm_s390_vcpu_stop(vcpu);
3524         kvm_s390_clear_local_irqs(vcpu);
3525 }
3526
3527 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3528 {
3529         /* Initial reset is a superset of the normal reset */
3530         kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3531
3532         /*
3533          * This equals initial cpu reset in pop, but we don't switch to ESA.
3534          * We do not only reset the internal data, but also ...
3535          */
3536         vcpu->arch.sie_block->gpsw.mask = 0;
3537         vcpu->arch.sie_block->gpsw.addr = 0;
3538         kvm_s390_set_prefix(vcpu, 0);
3539         kvm_s390_set_cpu_timer(vcpu, 0);
3540         vcpu->arch.sie_block->ckc = 0;
3541         memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3542         vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3543         vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
3544
3545         /* ... the data in sync regs */
3546         memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
3547         vcpu->run->s.regs.ckc = 0;
3548         vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
3549         vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
3550         vcpu->run->psw_addr = 0;
3551         vcpu->run->psw_mask = 0;
3552         vcpu->run->s.regs.todpr = 0;
3553         vcpu->run->s.regs.cputm = 0;
3554         vcpu->run->s.regs.ckc = 0;
3555         vcpu->run->s.regs.pp = 0;
3556         vcpu->run->s.regs.gbea = 1;
3557         vcpu->run->s.regs.fpc = 0;
3558         /*
3559          * Do not reset these registers in the protected case, as some of
3560          * them are overlayed and they are not accessible in this case
3561          * anyway.
3562          */
3563         if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3564                 vcpu->arch.sie_block->gbea = 1;
3565                 vcpu->arch.sie_block->pp = 0;
3566                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3567                 vcpu->arch.sie_block->todpr = 0;
3568         }
3569 }
3570
3571 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3572 {
3573         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3574
3575         /* Clear reset is a superset of the initial reset */
3576         kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3577
3578         memset(&regs->gprs, 0, sizeof(regs->gprs));
3579         memset(&regs->vrs, 0, sizeof(regs->vrs));
3580         memset(&regs->acrs, 0, sizeof(regs->acrs));
3581         memset(&regs->gscb, 0, sizeof(regs->gscb));
3582
3583         regs->etoken = 0;
3584         regs->etoken_extension = 0;
3585 }
3586
3587 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3588 {
3589         vcpu_load(vcpu);
3590         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
3591         vcpu_put(vcpu);
3592         return 0;
3593 }
3594
3595 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3596 {
3597         vcpu_load(vcpu);
3598         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3599         vcpu_put(vcpu);
3600         return 0;
3601 }
3602
3603 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3604                                   struct kvm_sregs *sregs)
3605 {
3606         vcpu_load(vcpu);
3607
3608         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3609         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3610
3611         vcpu_put(vcpu);
3612         return 0;
3613 }
3614
3615 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3616                                   struct kvm_sregs *sregs)
3617 {
3618         vcpu_load(vcpu);
3619
3620         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3621         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3622
3623         vcpu_put(vcpu);
3624         return 0;
3625 }
3626
3627 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3628 {
3629         int ret = 0;
3630
3631         vcpu_load(vcpu);
3632
3633         if (test_fp_ctl(fpu->fpc)) {
3634                 ret = -EINVAL;
3635                 goto out;
3636         }
3637         vcpu->run->s.regs.fpc = fpu->fpc;
3638         if (MACHINE_HAS_VX)
3639                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3640                                  (freg_t *) fpu->fprs);
3641         else
3642                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3643
3644 out:
3645         vcpu_put(vcpu);
3646         return ret;
3647 }
3648
3649 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3650 {
3651         vcpu_load(vcpu);
3652
3653         /* make sure we have the latest values */
3654         save_fpu_regs();
3655         if (MACHINE_HAS_VX)
3656                 convert_vx_to_fp((freg_t *) fpu->fprs,
3657                                  (__vector128 *) vcpu->run->s.regs.vrs);
3658         else
3659                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3660         fpu->fpc = vcpu->run->s.regs.fpc;
3661
3662         vcpu_put(vcpu);
3663         return 0;
3664 }
3665
3666 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3667 {
3668         int rc = 0;
3669
3670         if (!is_vcpu_stopped(vcpu))
3671                 rc = -EBUSY;
3672         else {
3673                 vcpu->run->psw_mask = psw.mask;
3674                 vcpu->run->psw_addr = psw.addr;
3675         }
3676         return rc;
3677 }
3678
3679 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3680                                   struct kvm_translation *tr)
3681 {
3682         return -EINVAL; /* not implemented yet */
3683 }
3684
3685 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3686                               KVM_GUESTDBG_USE_HW_BP | \
3687                               KVM_GUESTDBG_ENABLE)
3688
3689 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3690                                         struct kvm_guest_debug *dbg)
3691 {
3692         int rc = 0;
3693
3694         vcpu_load(vcpu);
3695
3696         vcpu->guest_debug = 0;
3697         kvm_s390_clear_bp_data(vcpu);
3698
3699         if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3700                 rc = -EINVAL;
3701                 goto out;
3702         }
3703         if (!sclp.has_gpere) {
3704                 rc = -EINVAL;
3705                 goto out;
3706         }
3707
3708         if (dbg->control & KVM_GUESTDBG_ENABLE) {
3709                 vcpu->guest_debug = dbg->control;
3710                 /* enforce guest PER */
3711                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3712
3713                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3714                         rc = kvm_s390_import_bp_data(vcpu, dbg);
3715         } else {
3716                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3717                 vcpu->arch.guestdbg.last_bp = 0;
3718         }
3719
3720         if (rc) {
3721                 vcpu->guest_debug = 0;
3722                 kvm_s390_clear_bp_data(vcpu);
3723                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3724         }
3725
3726 out:
3727         vcpu_put(vcpu);
3728         return rc;
3729 }
3730
3731 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3732                                     struct kvm_mp_state *mp_state)
3733 {
3734         int ret;
3735
3736         vcpu_load(vcpu);
3737
3738         /* CHECK_STOP and LOAD are not supported yet */
3739         ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3740                                       KVM_MP_STATE_OPERATING;
3741
3742         vcpu_put(vcpu);
3743         return ret;
3744 }
3745
3746 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3747                                     struct kvm_mp_state *mp_state)
3748 {
3749         int rc = 0;
3750
3751         vcpu_load(vcpu);
3752
3753         /* user space knows about this interface - let it control the state */
3754         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3755
3756         switch (mp_state->mp_state) {
3757         case KVM_MP_STATE_STOPPED:
3758                 rc = kvm_s390_vcpu_stop(vcpu);
3759                 break;
3760         case KVM_MP_STATE_OPERATING:
3761                 rc = kvm_s390_vcpu_start(vcpu);
3762                 break;
3763         case KVM_MP_STATE_LOAD:
3764                 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
3765                         rc = -ENXIO;
3766                         break;
3767                 }
3768                 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
3769                 break;
3770         case KVM_MP_STATE_CHECK_STOP:
3771                 fallthrough;    /* CHECK_STOP and LOAD are not supported yet */
3772         default:
3773                 rc = -ENXIO;
3774         }
3775
3776         vcpu_put(vcpu);
3777         return rc;
3778 }
3779
3780 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3781 {
3782         return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3783 }
3784
3785 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3786 {
3787 retry:
3788         kvm_s390_vcpu_request_handled(vcpu);
3789         if (!kvm_request_pending(vcpu))
3790                 return 0;
3791         /*
3792          * We use MMU_RELOAD just to re-arm the ipte notifier for the
3793          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3794          * This ensures that the ipte instruction for this request has
3795          * already finished. We might race against a second unmapper that
3796          * wants to set the blocking bit. Lets just retry the request loop.
3797          */
3798         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3799                 int rc;
3800                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3801                                           kvm_s390_get_prefix(vcpu),
3802                                           PAGE_SIZE * 2, PROT_WRITE);
3803                 if (rc) {
3804                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3805                         return rc;
3806                 }
3807                 goto retry;
3808         }
3809
3810         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3811                 vcpu->arch.sie_block->ihcpu = 0xffff;
3812                 goto retry;
3813         }
3814
3815         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3816                 if (!ibs_enabled(vcpu)) {
3817                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3818                         kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3819                 }
3820                 goto retry;
3821         }
3822
3823         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3824                 if (ibs_enabled(vcpu)) {
3825                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3826                         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3827                 }
3828                 goto retry;
3829         }
3830
3831         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3832                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3833                 goto retry;
3834         }
3835
3836         if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3837                 /*
3838                  * Disable CMM virtualization; we will emulate the ESSA
3839                  * instruction manually, in order to provide additional
3840                  * functionalities needed for live migration.
3841                  */
3842                 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3843                 goto retry;
3844         }
3845
3846         if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3847                 /*
3848                  * Re-enable CMM virtualization if CMMA is available and
3849                  * CMM has been used.
3850                  */
3851                 if ((vcpu->kvm->arch.use_cmma) &&
3852                     (vcpu->kvm->mm->context.uses_cmm))
3853                         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3854                 goto retry;
3855         }
3856
3857         /* nothing to do, just clear the request */
3858         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3859         /* we left the vsie handler, nothing to do, just clear the request */
3860         kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3861
3862         return 0;
3863 }
3864
3865 void kvm_s390_set_tod_clock(struct kvm *kvm,
3866                             const struct kvm_s390_vm_tod_clock *gtod)
3867 {
3868         struct kvm_vcpu *vcpu;
3869         struct kvm_s390_tod_clock_ext htod;
3870         int i;
3871
3872         mutex_lock(&kvm->lock);
3873         preempt_disable();
3874
3875         get_tod_clock_ext((char *)&htod);
3876
3877         kvm->arch.epoch = gtod->tod - htod.tod;
3878         kvm->arch.epdx = 0;
3879         if (test_kvm_facility(kvm, 139)) {
3880                 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3881                 if (kvm->arch.epoch > gtod->tod)
3882                         kvm->arch.epdx -= 1;
3883         }
3884
3885         kvm_s390_vcpu_block_all(kvm);
3886         kvm_for_each_vcpu(i, vcpu, kvm) {
3887                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3888                 vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
3889         }
3890
3891         kvm_s390_vcpu_unblock_all(kvm);
3892         preempt_enable();
3893         mutex_unlock(&kvm->lock);
3894 }
3895
3896 /**
3897  * kvm_arch_fault_in_page - fault-in guest page if necessary
3898  * @vcpu: The corresponding virtual cpu
3899  * @gpa: Guest physical address
3900  * @writable: Whether the page should be writable or not
3901  *
3902  * Make sure that a guest page has been faulted-in on the host.
3903  *
3904  * Return: Zero on success, negative error code otherwise.
3905  */
3906 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3907 {
3908         return gmap_fault(vcpu->arch.gmap, gpa,
3909                           writable ? FAULT_FLAG_WRITE : 0);
3910 }
3911
3912 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3913                                       unsigned long token)
3914 {
3915         struct kvm_s390_interrupt inti;
3916         struct kvm_s390_irq irq;
3917
3918         if (start_token) {
3919                 irq.u.ext.ext_params2 = token;
3920                 irq.type = KVM_S390_INT_PFAULT_INIT;
3921                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3922         } else {
3923                 inti.type = KVM_S390_INT_PFAULT_DONE;
3924                 inti.parm64 = token;
3925                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3926         }
3927 }
3928
3929 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3930                                      struct kvm_async_pf *work)
3931 {
3932         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3933         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3934
3935         return true;
3936 }
3937
3938 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3939                                  struct kvm_async_pf *work)
3940 {
3941         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3942         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3943 }
3944
3945 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3946                                struct kvm_async_pf *work)
3947 {
3948         /* s390 will always inject the page directly */
3949 }
3950
3951 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
3952 {
3953         /*
3954          * s390 will always inject the page directly,
3955          * but we still want check_async_completion to cleanup
3956          */
3957         return true;
3958 }
3959
3960 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3961 {
3962         hva_t hva;
3963         struct kvm_arch_async_pf arch;
3964
3965         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3966                 return false;
3967         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3968             vcpu->arch.pfault_compare)
3969                 return false;
3970         if (psw_extint_disabled(vcpu))
3971                 return false;
3972         if (kvm_s390_vcpu_has_irq(vcpu, 0))
3973                 return false;
3974         if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3975                 return false;
3976         if (!vcpu->arch.gmap->pfault_enabled)
3977                 return false;
3978
3979         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3980         hva += current->thread.gmap_addr & ~PAGE_MASK;
3981         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3982                 return false;
3983
3984         return kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3985 }
3986
3987 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3988 {
3989         int rc, cpuflags;
3990
3991         /*
3992          * On s390 notifications for arriving pages will be delivered directly
3993          * to the guest but the house keeping for completed pfaults is
3994          * handled outside the worker.
3995          */
3996         kvm_check_async_pf_completion(vcpu);
3997
3998         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3999         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4000
4001         if (need_resched())
4002                 schedule();
4003
4004         if (!kvm_is_ucontrol(vcpu->kvm)) {
4005                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4006                 if (rc)
4007                         return rc;
4008         }
4009
4010         rc = kvm_s390_handle_requests(vcpu);
4011         if (rc)
4012                 return rc;
4013
4014         if (guestdbg_enabled(vcpu)) {
4015                 kvm_s390_backup_guest_per_regs(vcpu);
4016                 kvm_s390_patch_guest_per_regs(vcpu);
4017         }
4018
4019         clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask);
4020
4021         vcpu->arch.sie_block->icptcode = 0;
4022         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4023         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4024         trace_kvm_s390_sie_enter(vcpu, cpuflags);
4025
4026         return 0;
4027 }
4028
4029 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
4030 {
4031         struct kvm_s390_pgm_info pgm_info = {
4032                 .code = PGM_ADDRESSING,
4033         };
4034         u8 opcode, ilen;
4035         int rc;
4036
4037         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4038         trace_kvm_s390_sie_fault(vcpu);
4039
4040         /*
4041          * We want to inject an addressing exception, which is defined as a
4042          * suppressing or terminating exception. However, since we came here
4043          * by a DAT access exception, the PSW still points to the faulting
4044          * instruction since DAT exceptions are nullifying. So we've got
4045          * to look up the current opcode to get the length of the instruction
4046          * to be able to forward the PSW.
4047          */
4048         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4049         ilen = insn_length(opcode);
4050         if (rc < 0) {
4051                 return rc;
4052         } else if (rc) {
4053                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4054                  * Forward by arbitrary ilc, injection will take care of
4055                  * nullification if necessary.
4056                  */
4057                 pgm_info = vcpu->arch.pgm;
4058                 ilen = 4;
4059         }
4060         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4061         kvm_s390_forward_psw(vcpu, ilen);
4062         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4063 }
4064
4065 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4066 {
4067         struct mcck_volatile_info *mcck_info;
4068         struct sie_page *sie_page;
4069
4070         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4071                    vcpu->arch.sie_block->icptcode);
4072         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4073
4074         if (guestdbg_enabled(vcpu))
4075                 kvm_s390_restore_guest_per_regs(vcpu);
4076
4077         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4078         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4079
4080         if (exit_reason == -EINTR) {
4081                 VCPU_EVENT(vcpu, 3, "%s", "machine check");
4082                 sie_page = container_of(vcpu->arch.sie_block,
4083                                         struct sie_page, sie_block);
4084                 mcck_info = &sie_page->mcck_info;
4085                 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4086                 return 0;
4087         }
4088
4089         if (vcpu->arch.sie_block->icptcode > 0) {
4090                 int rc = kvm_handle_sie_intercept(vcpu);
4091
4092                 if (rc != -EOPNOTSUPP)
4093                         return rc;
4094                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4095                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4096                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4097                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4098                 return -EREMOTE;
4099         } else if (exit_reason != -EFAULT) {
4100                 vcpu->stat.exit_null++;
4101                 return 0;
4102         } else if (kvm_is_ucontrol(vcpu->kvm)) {
4103                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4104                 vcpu->run->s390_ucontrol.trans_exc_code =
4105                                                 current->thread.gmap_addr;
4106                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
4107                 return -EREMOTE;
4108         } else if (current->thread.gmap_pfault) {
4109                 trace_kvm_s390_major_guest_pfault(vcpu);
4110                 current->thread.gmap_pfault = 0;
4111                 if (kvm_arch_setup_async_pf(vcpu))
4112                         return 0;
4113                 vcpu->stat.pfault_sync++;
4114                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
4115         }
4116         return vcpu_post_run_fault_in_sie(vcpu);
4117 }
4118
4119 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
4120 static int __vcpu_run(struct kvm_vcpu *vcpu)
4121 {
4122         int rc, exit_reason;
4123         struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4124
4125         /*
4126          * We try to hold kvm->srcu during most of vcpu_run (except when run-
4127          * ning the guest), so that memslots (and other stuff) are protected
4128          */
4129         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4130
4131         do {
4132                 rc = vcpu_pre_run(vcpu);
4133                 if (rc)
4134                         break;
4135
4136                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
4137                 /*
4138                  * As PF_VCPU will be used in fault handler, between
4139                  * guest_enter and guest_exit should be no uaccess.
4140                  */
4141                 local_irq_disable();
4142                 guest_enter_irqoff();
4143                 __disable_cpu_timer_accounting(vcpu);
4144                 local_irq_enable();
4145                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4146                         memcpy(sie_page->pv_grregs,
4147                                vcpu->run->s.regs.gprs,
4148                                sizeof(sie_page->pv_grregs));
4149                 }
4150                 exit_reason = sie64a(vcpu->arch.sie_block,
4151                                      vcpu->run->s.regs.gprs);
4152                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4153                         memcpy(vcpu->run->s.regs.gprs,
4154                                sie_page->pv_grregs,
4155                                sizeof(sie_page->pv_grregs));
4156                         /*
4157                          * We're not allowed to inject interrupts on intercepts
4158                          * that leave the guest state in an "in-between" state
4159                          * where the next SIE entry will do a continuation.
4160                          * Fence interrupts in our "internal" PSW.
4161                          */
4162                         if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4163                             vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4164                                 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4165                         }
4166                 }
4167                 local_irq_disable();
4168                 __enable_cpu_timer_accounting(vcpu);
4169                 guest_exit_irqoff();
4170                 local_irq_enable();
4171                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4172
4173                 rc = vcpu_post_run(vcpu, exit_reason);
4174         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
4175
4176         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
4177         return rc;
4178 }
4179
4180 static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
4181 {
4182         struct kvm_run *kvm_run = vcpu->run;
4183         struct runtime_instr_cb *riccb;
4184         struct gs_cb *gscb;
4185
4186         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4187         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
4188         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4189         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
4190         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4191                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4192                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4193                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4194         }
4195         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4196                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4197                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4198                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
4199                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4200                         kvm_clear_async_pf_completion_queue(vcpu);
4201         }
4202         if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
4203                 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
4204                 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
4205         }
4206         /*
4207          * If userspace sets the riccb (e.g. after migration) to a valid state,
4208          * we should enable RI here instead of doing the lazy enablement.
4209          */
4210         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4211             test_kvm_facility(vcpu->kvm, 64) &&
4212             riccb->v &&
4213             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4214                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
4215                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
4216         }
4217         /*
4218          * If userspace sets the gscb (e.g. after migration) to non-zero,
4219          * we should enable GS here instead of doing the lazy enablement.
4220          */
4221         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4222             test_kvm_facility(vcpu->kvm, 133) &&
4223             gscb->gssm &&
4224             !vcpu->arch.gs_enabled) {
4225                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4226                 vcpu->arch.sie_block->ecb |= ECB_GS;
4227                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4228                 vcpu->arch.gs_enabled = 1;
4229         }
4230         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4231             test_kvm_facility(vcpu->kvm, 82)) {
4232                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4233                 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4234         }
4235         if (MACHINE_HAS_GS) {
4236                 preempt_disable();
4237                 __ctl_set_bit(2, 4);
4238                 if (current->thread.gs_cb) {
4239                         vcpu->arch.host_gscb = current->thread.gs_cb;
4240                         save_gs_cb(vcpu->arch.host_gscb);
4241                 }
4242                 if (vcpu->arch.gs_enabled) {
4243                         current->thread.gs_cb = (struct gs_cb *)
4244                                                 &vcpu->run->s.regs.gscb;
4245                         restore_gs_cb(current->thread.gs_cb);
4246                 }
4247                 preempt_enable();
4248         }
4249         /* SIE will load etoken directly from SDNX and therefore kvm_run */
4250 }
4251
4252 static void sync_regs(struct kvm_vcpu *vcpu)
4253 {
4254         struct kvm_run *kvm_run = vcpu->run;
4255
4256         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4257                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4258         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4259                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4260                 /* some control register changes require a tlb flush */
4261                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4262         }
4263         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4264                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4265                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4266         }
4267         save_access_regs(vcpu->arch.host_acrs);
4268         restore_access_regs(vcpu->run->s.regs.acrs);
4269         /* save host (userspace) fprs/vrs */
4270         save_fpu_regs();
4271         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
4272         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
4273         if (MACHINE_HAS_VX)
4274                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
4275         else
4276                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
4277         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
4278         if (test_fp_ctl(current->thread.fpu.fpc))
4279                 /* User space provided an invalid FPC, let's clear it */
4280                 current->thread.fpu.fpc = 0;
4281
4282         /* Sync fmt2 only data */
4283         if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4284                 sync_regs_fmt2(vcpu);
4285         } else {
4286                 /*
4287                  * In several places we have to modify our internal view to
4288                  * not do things that are disallowed by the ultravisor. For
4289                  * example we must not inject interrupts after specific exits
4290                  * (e.g. 112 prefix page not secure). We do this by turning
4291                  * off the machine check, external and I/O interrupt bits
4292                  * of our PSW copy. To avoid getting validity intercepts, we
4293                  * do only accept the condition code from userspace.
4294                  */
4295                 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4296                 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4297                                                    PSW_MASK_CC;
4298         }
4299
4300         kvm_run->kvm_dirty_regs = 0;
4301 }
4302
4303 static void store_regs_fmt2(struct kvm_vcpu *vcpu)
4304 {
4305         struct kvm_run *kvm_run = vcpu->run;
4306
4307         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4308         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4309         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4310         kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4311         kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
4312         if (MACHINE_HAS_GS) {
4313                 __ctl_set_bit(2, 4);
4314                 if (vcpu->arch.gs_enabled)
4315                         save_gs_cb(current->thread.gs_cb);
4316                 preempt_disable();
4317                 current->thread.gs_cb = vcpu->arch.host_gscb;
4318                 restore_gs_cb(vcpu->arch.host_gscb);
4319                 preempt_enable();
4320                 if (!vcpu->arch.host_gscb)
4321                         __ctl_clear_bit(2, 4);
4322                 vcpu->arch.host_gscb = NULL;
4323         }
4324         /* SIE will save etoken directly into SDNX and therefore kvm_run */
4325 }
4326
4327 static void store_regs(struct kvm_vcpu *vcpu)
4328 {
4329         struct kvm_run *kvm_run = vcpu->run;
4330
4331         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4332         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4333         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4334         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4335         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
4336         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
4337         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4338         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4339         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
4340         save_access_regs(vcpu->run->s.regs.acrs);
4341         restore_access_regs(vcpu->arch.host_acrs);
4342         /* Save guest register state */
4343         save_fpu_regs();
4344         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4345         /* Restore will be done lazily at return */
4346         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4347         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4348         if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
4349                 store_regs_fmt2(vcpu);
4350 }
4351
4352 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
4353 {
4354         struct kvm_run *kvm_run = vcpu->run;
4355         int rc;
4356
4357         if (kvm_run->immediate_exit)
4358                 return -EINTR;
4359
4360         if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4361             kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4362                 return -EINVAL;
4363
4364         vcpu_load(vcpu);
4365
4366         if (guestdbg_exit_pending(vcpu)) {
4367                 kvm_s390_prepare_debug_exit(vcpu);
4368                 rc = 0;
4369                 goto out;
4370         }
4371
4372         kvm_sigset_activate(vcpu);
4373
4374         /*
4375          * no need to check the return value of vcpu_start as it can only have
4376          * an error for protvirt, but protvirt means user cpu state
4377          */
4378         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4379                 kvm_s390_vcpu_start(vcpu);
4380         } else if (is_vcpu_stopped(vcpu)) {
4381                 pr_err_ratelimited("can't run stopped vcpu %d\n",
4382                                    vcpu->vcpu_id);
4383                 rc = -EINVAL;
4384                 goto out;
4385         }
4386
4387         sync_regs(vcpu);
4388         enable_cpu_timer_accounting(vcpu);
4389
4390         might_fault();
4391         rc = __vcpu_run(vcpu);
4392
4393         if (signal_pending(current) && !rc) {
4394                 kvm_run->exit_reason = KVM_EXIT_INTR;
4395                 rc = -EINTR;
4396         }
4397
4398         if (guestdbg_exit_pending(vcpu) && !rc)  {
4399                 kvm_s390_prepare_debug_exit(vcpu);
4400                 rc = 0;
4401         }
4402
4403         if (rc == -EREMOTE) {
4404                 /* userspace support is needed, kvm_run has been prepared */
4405                 rc = 0;
4406         }
4407
4408         disable_cpu_timer_accounting(vcpu);
4409         store_regs(vcpu);
4410
4411         kvm_sigset_deactivate(vcpu);
4412
4413         vcpu->stat.exit_userspace++;
4414 out:
4415         vcpu_put(vcpu);
4416         return rc;
4417 }
4418
4419 /*
4420  * store status at address
4421  * we use have two special cases:
4422  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4423  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4424  */
4425 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
4426 {
4427         unsigned char archmode = 1;
4428         freg_t fprs[NUM_FPRS];
4429         unsigned int px;
4430         u64 clkcomp, cputm;
4431         int rc;
4432
4433         px = kvm_s390_get_prefix(vcpu);
4434         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4435                 if (write_guest_abs(vcpu, 163, &archmode, 1))
4436                         return -EFAULT;
4437                 gpa = 0;
4438         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4439                 if (write_guest_real(vcpu, 163, &archmode, 1))
4440                         return -EFAULT;
4441                 gpa = px;
4442         } else
4443                 gpa -= __LC_FPREGS_SAVE_AREA;
4444
4445         /* manually convert vector registers if necessary */
4446         if (MACHINE_HAS_VX) {
4447                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
4448                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4449                                      fprs, 128);
4450         } else {
4451                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4452                                      vcpu->run->s.regs.fprs, 128);
4453         }
4454         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
4455                               vcpu->run->s.regs.gprs, 128);
4456         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
4457                               &vcpu->arch.sie_block->gpsw, 16);
4458         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
4459                               &px, 4);
4460         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
4461                               &vcpu->run->s.regs.fpc, 4);
4462         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
4463                               &vcpu->arch.sie_block->todpr, 4);
4464         cputm = kvm_s390_get_cpu_timer(vcpu);
4465         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4466                               &cputm, 8);
4467         clkcomp = vcpu->arch.sie_block->ckc >> 8;
4468         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
4469                               &clkcomp, 8);
4470         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
4471                               &vcpu->run->s.regs.acrs, 64);
4472         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
4473                               &vcpu->arch.sie_block->gcr, 128);
4474         return rc ? -EFAULT : 0;
4475 }
4476
4477 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4478 {
4479         /*
4480          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
4481          * switch in the run ioctl. Let's update our copies before we save
4482          * it into the save area
4483          */
4484         save_fpu_regs();
4485         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4486         save_access_regs(vcpu->run->s.regs.acrs);
4487
4488         return kvm_s390_store_status_unloaded(vcpu, addr);
4489 }
4490
4491 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4492 {
4493         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
4494         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
4495 }
4496
4497 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4498 {
4499         unsigned int i;
4500         struct kvm_vcpu *vcpu;
4501
4502         kvm_for_each_vcpu(i, vcpu, kvm) {
4503                 __disable_ibs_on_vcpu(vcpu);
4504         }
4505 }
4506
4507 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4508 {
4509         if (!sclp.has_ibs)
4510                 return;
4511         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
4512         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
4513 }
4514
4515 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4516 {
4517         int i, online_vcpus, r = 0, started_vcpus = 0;
4518
4519         if (!is_vcpu_stopped(vcpu))
4520                 return 0;
4521
4522         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
4523         /* Only one cpu at a time may enter/leave the STOPPED state. */
4524         spin_lock(&vcpu->kvm->arch.start_stop_lock);
4525         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4526
4527         /* Let's tell the UV that we want to change into the operating state */
4528         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4529                 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
4530                 if (r) {
4531                         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4532                         return r;
4533                 }
4534         }
4535
4536         for (i = 0; i < online_vcpus; i++) {
4537                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4538                         started_vcpus++;
4539         }
4540
4541         if (started_vcpus == 0) {
4542                 /* we're the only active VCPU -> speed it up */
4543                 __enable_ibs_on_vcpu(vcpu);
4544         } else if (started_vcpus == 1) {
4545                 /*
4546                  * As we are starting a second VCPU, we have to disable
4547                  * the IBS facility on all VCPUs to remove potentially
4548                  * oustanding ENABLE requests.
4549                  */
4550                 __disable_ibs_on_all_vcpus(vcpu->kvm);
4551         }
4552
4553         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
4554         /*
4555          * The real PSW might have changed due to a RESTART interpreted by the
4556          * ultravisor. We block all interrupts and let the next sie exit
4557          * refresh our view.
4558          */
4559         if (kvm_s390_pv_cpu_is_protected(vcpu))
4560                 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4561         /*
4562          * Another VCPU might have used IBS while we were offline.
4563          * Let's play safe and flush the VCPU at startup.
4564          */
4565         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4566         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4567         return 0;
4568 }
4569
4570 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4571 {
4572         int i, online_vcpus, r = 0, started_vcpus = 0;
4573         struct kvm_vcpu *started_vcpu = NULL;
4574
4575         if (is_vcpu_stopped(vcpu))
4576                 return 0;
4577
4578         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
4579         /* Only one cpu at a time may enter/leave the STOPPED state. */
4580         spin_lock(&vcpu->kvm->arch.start_stop_lock);
4581         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4582
4583         /* Let's tell the UV that we want to change into the stopped state */
4584         if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4585                 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
4586                 if (r) {
4587                         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4588                         return r;
4589                 }
4590         }
4591
4592         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
4593         kvm_s390_clear_stop_irq(vcpu);
4594
4595         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
4596         __disable_ibs_on_vcpu(vcpu);
4597
4598         for (i = 0; i < online_vcpus; i++) {
4599                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4600                         started_vcpus++;
4601                         started_vcpu = vcpu->kvm->vcpus[i];
4602                 }
4603         }
4604
4605         if (started_vcpus == 1) {
4606                 /*
4607                  * As we only have one VCPU left, we want to enable the
4608                  * IBS facility for that VCPU to speed it up.
4609                  */
4610                 __enable_ibs_on_vcpu(started_vcpu);
4611         }
4612
4613         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4614         return 0;
4615 }
4616
4617 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4618                                      struct kvm_enable_cap *cap)
4619 {
4620         int r;
4621
4622         if (cap->flags)
4623                 return -EINVAL;
4624
4625         switch (cap->cap) {
4626         case KVM_CAP_S390_CSS_SUPPORT:
4627                 if (!vcpu->kvm->arch.css_support) {
4628                         vcpu->kvm->arch.css_support = 1;
4629                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4630                         trace_kvm_s390_enable_css(vcpu->kvm);
4631                 }
4632                 r = 0;
4633                 break;
4634         default:
4635                 r = -EINVAL;
4636                 break;
4637         }
4638         return r;
4639 }
4640
4641 static long kvm_s390_guest_sida_op(struct kvm_vcpu *vcpu,
4642                                    struct kvm_s390_mem_op *mop)
4643 {
4644         void __user *uaddr = (void __user *)mop->buf;
4645         int r = 0;
4646
4647         if (mop->flags || !mop->size)
4648                 return -EINVAL;
4649         if (mop->size + mop->sida_offset < mop->size)
4650                 return -EINVAL;
4651         if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
4652                 return -E2BIG;
4653
4654         switch (mop->op) {
4655         case KVM_S390_MEMOP_SIDA_READ:
4656                 if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
4657                                  mop->sida_offset), mop->size))
4658                         r = -EFAULT;
4659
4660                 break;
4661         case KVM_S390_MEMOP_SIDA_WRITE:
4662                 if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
4663                                    mop->sida_offset), uaddr, mop->size))
4664                         r = -EFAULT;
4665                 break;
4666         }
4667         return r;
4668 }
4669 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4670                                   struct kvm_s390_mem_op *mop)
4671 {
4672         void __user *uaddr = (void __user *)mop->buf;
4673         void *tmpbuf = NULL;
4674         int r = 0;
4675         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4676                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
4677
4678         if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4679                 return -EINVAL;
4680
4681         if (mop->size > MEM_OP_MAX_SIZE)
4682                 return -E2BIG;
4683
4684         if (kvm_s390_pv_cpu_is_protected(vcpu))
4685                 return -EINVAL;
4686
4687         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4688                 tmpbuf = vmalloc(mop->size);
4689                 if (!tmpbuf)
4690                         return -ENOMEM;
4691         }
4692
4693         switch (mop->op) {
4694         case KVM_S390_MEMOP_LOGICAL_READ:
4695                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4696                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4697                                             mop->size, GACC_FETCH);
4698                         break;
4699                 }
4700                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4701                 if (r == 0) {
4702                         if (copy_to_user(uaddr, tmpbuf, mop->size))
4703                                 r = -EFAULT;
4704                 }
4705                 break;
4706         case KVM_S390_MEMOP_LOGICAL_WRITE:
4707                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4708                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4709                                             mop->size, GACC_STORE);
4710                         break;
4711                 }
4712                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4713                         r = -EFAULT;
4714                         break;
4715                 }
4716                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4717                 break;
4718         }
4719
4720         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4721                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4722
4723         vfree(tmpbuf);
4724         return r;
4725 }
4726
4727 static long kvm_s390_guest_memsida_op(struct kvm_vcpu *vcpu,
4728                                       struct kvm_s390_mem_op *mop)
4729 {
4730         int r, srcu_idx;
4731
4732         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4733
4734         switch (mop->op) {
4735         case KVM_S390_MEMOP_LOGICAL_READ:
4736         case KVM_S390_MEMOP_LOGICAL_WRITE:
4737                 r = kvm_s390_guest_mem_op(vcpu, mop);
4738                 break;
4739         case KVM_S390_MEMOP_SIDA_READ:
4740         case KVM_S390_MEMOP_SIDA_WRITE:
4741                 /* we are locked against sida going away by the vcpu->mutex */
4742                 r = kvm_s390_guest_sida_op(vcpu, mop);
4743                 break;
4744         default:
4745                 r = -EINVAL;
4746         }
4747
4748         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4749         return r;
4750 }
4751
4752 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4753                                unsigned int ioctl, unsigned long arg)
4754 {
4755         struct kvm_vcpu *vcpu = filp->private_data;
4756         void __user *argp = (void __user *)arg;
4757
4758         switch (ioctl) {
4759         case KVM_S390_IRQ: {
4760                 struct kvm_s390_irq s390irq;
4761
4762                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4763                         return -EFAULT;
4764                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4765         }
4766         case KVM_S390_INTERRUPT: {
4767                 struct kvm_s390_interrupt s390int;
4768                 struct kvm_s390_irq s390irq = {};
4769
4770                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4771                         return -EFAULT;
4772                 if (s390int_to_s390irq(&s390int, &s390irq))
4773                         return -EINVAL;
4774                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4775         }
4776         }
4777         return -ENOIOCTLCMD;
4778 }
4779
4780 long kvm_arch_vcpu_ioctl(struct file *filp,
4781                          unsigned int ioctl, unsigned long arg)
4782 {
4783         struct kvm_vcpu *vcpu = filp->private_data;
4784         void __user *argp = (void __user *)arg;
4785         int idx;
4786         long r;
4787         u16 rc, rrc;
4788
4789         vcpu_load(vcpu);
4790
4791         switch (ioctl) {
4792         case KVM_S390_STORE_STATUS:
4793                 idx = srcu_read_lock(&vcpu->kvm->srcu);
4794                 r = kvm_s390_store_status_unloaded(vcpu, arg);
4795                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4796                 break;
4797         case KVM_S390_SET_INITIAL_PSW: {
4798                 psw_t psw;
4799
4800                 r = -EFAULT;
4801                 if (copy_from_user(&psw, argp, sizeof(psw)))
4802                         break;
4803                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4804                 break;
4805         }
4806         case KVM_S390_CLEAR_RESET:
4807                 r = 0;
4808                 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4809                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4810                         r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4811                                           UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
4812                         VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
4813                                    rc, rrc);
4814                 }
4815                 break;
4816         case KVM_S390_INITIAL_RESET:
4817                 r = 0;
4818                 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4819                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4820                         r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4821                                           UVC_CMD_CPU_RESET_INITIAL,
4822                                           &rc, &rrc);
4823                         VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
4824                                    rc, rrc);
4825                 }
4826                 break;
4827         case KVM_S390_NORMAL_RESET:
4828                 r = 0;
4829                 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4830                 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4831                         r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
4832                                           UVC_CMD_CPU_RESET, &rc, &rrc);
4833                         VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
4834                                    rc, rrc);
4835                 }
4836                 break;
4837         case KVM_SET_ONE_REG:
4838         case KVM_GET_ONE_REG: {
4839                 struct kvm_one_reg reg;
4840                 r = -EINVAL;
4841                 if (kvm_s390_pv_cpu_is_protected(vcpu))
4842                         break;
4843                 r = -EFAULT;
4844                 if (copy_from_user(&reg, argp, sizeof(reg)))
4845                         break;
4846                 if (ioctl == KVM_SET_ONE_REG)
4847                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4848                 else
4849                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4850                 break;
4851         }
4852 #ifdef CONFIG_KVM_S390_UCONTROL
4853         case KVM_S390_UCAS_MAP: {
4854                 struct kvm_s390_ucas_mapping ucasmap;
4855
4856                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4857                         r = -EFAULT;
4858                         break;
4859                 }
4860
4861                 if (!kvm_is_ucontrol(vcpu->kvm)) {
4862                         r = -EINVAL;
4863                         break;
4864                 }
4865
4866                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4867                                      ucasmap.vcpu_addr, ucasmap.length);
4868                 break;
4869         }
4870         case KVM_S390_UCAS_UNMAP: {
4871                 struct kvm_s390_ucas_mapping ucasmap;
4872
4873                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4874                         r = -EFAULT;
4875                         break;
4876                 }
4877
4878                 if (!kvm_is_ucontrol(vcpu->kvm)) {
4879                         r = -EINVAL;
4880                         break;
4881                 }
4882
4883                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4884                         ucasmap.length);
4885                 break;
4886         }
4887 #endif
4888         case KVM_S390_VCPU_FAULT: {
4889                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4890                 break;
4891         }
4892         case KVM_ENABLE_CAP:
4893         {
4894                 struct kvm_enable_cap cap;
4895                 r = -EFAULT;
4896                 if (copy_from_user(&cap, argp, sizeof(cap)))
4897                         break;
4898                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4899                 break;
4900         }
4901         case KVM_S390_MEM_OP: {
4902                 struct kvm_s390_mem_op mem_op;
4903
4904                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4905                         r = kvm_s390_guest_memsida_op(vcpu, &mem_op);
4906                 else
4907                         r = -EFAULT;
4908                 break;
4909         }
4910         case KVM_S390_SET_IRQ_STATE: {
4911                 struct kvm_s390_irq_state irq_state;
4912
4913                 r = -EFAULT;
4914                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4915                         break;
4916                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4917                     irq_state.len == 0 ||
4918                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4919                         r = -EINVAL;
4920                         break;
4921                 }
4922                 /* do not use irq_state.flags, it will break old QEMUs */
4923                 r = kvm_s390_set_irq_state(vcpu,
4924                                            (void __user *) irq_state.buf,
4925                                            irq_state.len);
4926                 break;
4927         }
4928         case KVM_S390_GET_IRQ_STATE: {
4929                 struct kvm_s390_irq_state irq_state;
4930
4931                 r = -EFAULT;
4932                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4933                         break;
4934                 if (irq_state.len == 0) {
4935                         r = -EINVAL;
4936                         break;
4937                 }
4938                 /* do not use irq_state.flags, it will break old QEMUs */
4939                 r = kvm_s390_get_irq_state(vcpu,
4940                                            (__u8 __user *)  irq_state.buf,
4941                                            irq_state.len);
4942                 break;
4943         }
4944         default:
4945                 r = -ENOTTY;
4946         }
4947
4948         vcpu_put(vcpu);
4949         return r;
4950 }
4951
4952 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4953 {
4954 #ifdef CONFIG_KVM_S390_UCONTROL
4955         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4956                  && (kvm_is_ucontrol(vcpu->kvm))) {
4957                 vmf->page = virt_to_page(vcpu->arch.sie_block);
4958                 get_page(vmf->page);
4959                 return 0;
4960         }
4961 #endif
4962         return VM_FAULT_SIGBUS;
4963 }
4964
4965 /* Section: memory related */
4966 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4967                                    struct kvm_memory_slot *memslot,
4968                                    const struct kvm_userspace_memory_region *mem,
4969                                    enum kvm_mr_change change)
4970 {
4971         /* A few sanity checks. We can have memory slots which have to be
4972            located/ended at a segment boundary (1MB). The memory in userland is
4973            ok to be fragmented into various different vmas. It is okay to mmap()
4974            and munmap() stuff in this slot after doing this call at any time */
4975
4976         if (mem->userspace_addr & 0xffffful)
4977                 return -EINVAL;
4978
4979         if (mem->memory_size & 0xffffful)
4980                 return -EINVAL;
4981
4982         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4983                 return -EINVAL;
4984
4985         /* When we are protected, we should not change the memory slots */
4986         if (kvm_s390_pv_get_handle(kvm))
4987                 return -EINVAL;
4988         return 0;
4989 }
4990
4991 void kvm_arch_commit_memory_region(struct kvm *kvm,
4992                                 const struct kvm_userspace_memory_region *mem,
4993                                 struct kvm_memory_slot *old,
4994                                 const struct kvm_memory_slot *new,
4995                                 enum kvm_mr_change change)
4996 {
4997         int rc = 0;
4998
4999         switch (change) {
5000         case KVM_MR_DELETE:
5001                 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5002                                         old->npages * PAGE_SIZE);
5003                 break;
5004         case KVM_MR_MOVE:
5005                 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5006                                         old->npages * PAGE_SIZE);
5007                 if (rc)
5008                         break;
5009                 fallthrough;
5010         case KVM_MR_CREATE:
5011                 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
5012                                       mem->guest_phys_addr, mem->memory_size);
5013                 break;
5014         case KVM_MR_FLAGS_ONLY:
5015                 break;
5016         default:
5017                 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
5018         }
5019         if (rc)
5020                 pr_warn("failed to commit memory region\n");
5021         return;
5022 }
5023
5024 static inline unsigned long nonhyp_mask(int i)
5025 {
5026         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5027
5028         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5029 }
5030
5031 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
5032 {
5033         vcpu->valid_wakeup = false;
5034 }
5035
5036 static int __init kvm_s390_init(void)
5037 {
5038         int i;
5039
5040         if (!sclp.has_sief2) {
5041                 pr_info("SIE is not available\n");
5042                 return -ENODEV;
5043         }
5044
5045         if (nested && hpage) {
5046                 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
5047                 return -EINVAL;
5048         }
5049
5050         for (i = 0; i < 16; i++)
5051                 kvm_s390_fac_base[i] |=
5052                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
5053
5054         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
5055 }
5056
5057 static void __exit kvm_s390_exit(void)
5058 {
5059         kvm_exit();
5060 }
5061
5062 module_init(kvm_s390_init);
5063 module_exit(kvm_s390_exit);
5064
5065 /*
5066  * Enable autoloading of the kvm module.
5067  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5068  * since x86 takes a different approach.
5069  */
5070 #include <linux/miscdevice.h>
5071 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5072 MODULE_ALIAS("devname:kvm");