drivers/xen/time.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Xen stolen ticks accounting.
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/kernel_stat.h>
   7 #include <linux/math64.h>
   8 #include <linux/gfp.h>
   9 #include <linux/slab.h>
  10
  11 #include <asm/paravirt.h>
  12 #include <asm/xen/hypervisor.h>
  13 #include <asm/xen/hypercall.h>
  14
  15 #include <xen/events.h>
  16 #include <xen/features.h>
  17 #include <xen/interface/xen.h>
  18 #include <xen/interface/vcpu.h>
  19 #include <xen/xen-ops.h>
  20
  21 /* runstate info updated by Xen */
  22 static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
  23
  24 static DEFINE_PER_CPU(u64[4], old_runstate_time);
  25
  26 /* return an consistent snapshot of 64-bit time/counter value */
  27 static u64 get64(const u64 *p)
  28 {
  29         u64 ret;
  30
  31         if (BITS_PER_LONG < 64) {
  32                 u32 *p32 = (u32 *)p;
  33                 u32 h, l, h2;
  34
  35                 /*
  36                  * Read high then low, and then make sure high is
  37                  * still the same; this will only loop if low wraps
  38                  * and carries into high.
  39                  * XXX some clean way to make this endian-proof?
  40                  */
  41                 do {
  42                         h = READ_ONCE(p32[1]);
  43                         l = READ_ONCE(p32[0]);
  44                         h2 = READ_ONCE(p32[1]);
  45                 } while(h2 != h);
  46
  47                 ret = (((u64)h) << 32) | l;
  48         } else
  49                 ret = READ_ONCE(*p);
  50
  51         return ret;
  52 }
  53
  54 static void xen_get_runstate_snapshot_cpu_delta(
  55                               struct vcpu_runstate_info *res, unsigned int cpu)
  56 {
  57         u64 state_time;
  58         struct vcpu_runstate_info *state;
  59
  60         BUG_ON(preemptible());
  61
  62         state = per_cpu_ptr(&xen_runstate, cpu);
  63
  64         do {
  65                 state_time = get64(&state->state_entry_time);
  66                 rmb();  /* Hypervisor might update data. */
  67                 *res = __READ_ONCE(*state);
  68                 rmb();  /* Hypervisor might update data. */
  69         } while (get64(&state->state_entry_time) != state_time ||
  70                  (state_time & XEN_RUNSTATE_UPDATE));
  71 }
  72
  73 static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res,
  74                                           unsigned int cpu)
  75 {
  76         int i;
  77
  78         xen_get_runstate_snapshot_cpu_delta(res, cpu);
  79
  80         for (i = 0; i < 4; i++)
  81                 res->time[i] += per_cpu(old_runstate_time, cpu)[i];
  82 }
  83
  84 void xen_manage_runstate_time(int action)
  85 {
  86         static struct vcpu_runstate_info *runstate_delta;
  87         struct vcpu_runstate_info state;
  88         int cpu, i;
  89
  90         switch (action) {
  91         case -1: /* backup runstate time before suspend */
  92                 if (unlikely(runstate_delta))
  93                         pr_warn_once("%s: memory leak as runstate_delta is not NULL\n",
  94                                         __func__);
  95
  96                 runstate_delta = kmalloc_array(num_possible_cpus(),
  97                                         sizeof(*runstate_delta),
  98                                         GFP_ATOMIC);
  99                 if (unlikely(!runstate_delta)) {
 100                         pr_warn("%s: failed to allocate runstate_delta\n",
 101                                         __func__);
 102                         return;
 103                 }
 104
 105                 for_each_possible_cpu(cpu) {
 106                         xen_get_runstate_snapshot_cpu_delta(&state, cpu);
 107                         memcpy(runstate_delta[cpu].time, state.time,
 108                                         sizeof(runstate_delta[cpu].time));
 109                 }
 110
 111                 break;
 112
 113         case 0: /* backup runstate time after resume */
 114                 if (unlikely(!runstate_delta)) {
 115                         pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n",
 116                                         __func__);
 117                         return;
 118                 }
 119
 120                 for_each_possible_cpu(cpu) {
 121                         for (i = 0; i < 4; i++)
 122                                 per_cpu(old_runstate_time, cpu)[i] +=
 123                                         runstate_delta[cpu].time[i];
 124                 }
 125
 126                 break;
 127
 128         default: /* do not accumulate runstate time for checkpointing */
 129                 break;
 130         }
 131
 132         if (action != -1 && runstate_delta) {
 133                 kfree(runstate_delta);
 134                 runstate_delta = NULL;
 135         }
 136 }
 137
 138 /*
 139  * Runstate accounting
 140  */
 141 void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
 142 {
 143         xen_get_runstate_snapshot_cpu(res, smp_processor_id());
 144 }
 145
 146 /* return true when a vcpu could run but has no real cpu to run on */
 147 bool xen_vcpu_stolen(int vcpu)
 148 {
 149         return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
 150 }
 151
 152 u64 xen_steal_clock(int cpu)
 153 {
 154         struct vcpu_runstate_info state;
 155
 156         xen_get_runstate_snapshot_cpu(&state, cpu);
 157         return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
 158 }
 159
 160 void xen_setup_runstate_info(int cpu)
 161 {
 162         struct vcpu_register_runstate_memory_area area;
 163
 164         area.addr.v = &per_cpu(xen_runstate, cpu);
 165
 166         if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
 167                                xen_vcpu_nr(cpu), &area))
 168                 BUG();
 169 }
 170
 171 void __init xen_time_setup_guest(void)
 172 {
 173         bool xen_runstate_remote;
 174
 175         xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
 176                                         VMASST_TYPE_runstate_update_flag);
 177
 178         pv_ops.time.steal_clock = xen_steal_clock;
 179
 180         static_key_slow_inc(&paravirt_steal_enabled);
 181         if (xen_runstate_remote)
 182                 static_key_slow_inc(&paravirt_steal_rq_enabled);
 183 }