lib/vdso/gettimeofday.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic userspace implementations of gettimeofday() and similar.
   4  */
   5 #include <linux/compiler.h>
   6 #include <linux/math64.h>
   7 #include <linux/time.h>
   8 #include <linux/kernel.h>
   9 #include <linux/hrtimer_defs.h>
  10 #include <vdso/datapage.h>
  11 #include <vdso/helpers.h>
  12
  13 /*
  14  * The generic vDSO implementation requires that gettimeofday.h
  15  * provides:
  16  * - __arch_get_vdso_data(): to get the vdso datapage.
  17  * - __arch_get_hw_counter(): to get the hw counter based on the
  18  *   clock_mode.
  19  * - gettimeofday_fallback(): fallback for gettimeofday.
  20  * - clock_gettime_fallback(): fallback for clock_gettime.
  21  * - clock_getres_fallback(): fallback for clock_getres.
  22  */
  23 #ifdef ENABLE_COMPAT_VDSO
  24 #include <asm/vdso/compat_gettimeofday.h>
  25 #else
  26 #include <asm/vdso/gettimeofday.h>
  27 #endif /* ENABLE_COMPAT_VDSO */
  28
  29 #ifndef vdso_calc_delta
  30 /*
  31  * Default implementation which works for all sane clocksources. That
  32  * obviously excludes x86/TSC.
  33  */
  34 static __always_inline
  35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
  36 {
  37         return ((cycles - last) & mask) * mult;
  38 }
  39 #endif
  40
  41 static int do_hres(const struct vdso_data *vd, clockid_t clk,
  42                    struct __kernel_timespec *ts)
  43 {
  44         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  45         u64 cycles, last, sec, ns;
  46         u32 seq;
  47
  48         do {
  49                 seq = vdso_read_begin(vd);
  50                 cycles = __arch_get_hw_counter(vd->clock_mode);
  51                 ns = vdso_ts->nsec;
  52                 last = vd->cycle_last;
  53                 if (unlikely((s64)cycles < 0))
  54                         return clock_gettime_fallback(clk, ts);
  55
  56                 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
  57                 ns >>= vd->shift;
  58                 sec = vdso_ts->sec;
  59         } while (unlikely(vdso_read_retry(vd, seq)));
  60
  61         /*
  62          * Do this outside the loop: a race inside the loop could result
  63          * in __iter_div_u64_rem() being extremely slow.
  64          */
  65         ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
  66         ts->tv_nsec = ns;
  67
  68         return 0;
  69 }
  70
  71 static void do_coarse(const struct vdso_data *vd, clockid_t clk,
  72                       struct __kernel_timespec *ts)
  73 {
  74         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  75         u32 seq;
  76
  77         do {
  78                 seq = vdso_read_begin(vd);
  79                 ts->tv_sec = vdso_ts->sec;
  80                 ts->tv_nsec = vdso_ts->nsec;
  81         } while (unlikely(vdso_read_retry(vd, seq)));
  82 }
  83
  84 static __maybe_unused int
  85 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
  86 {
  87         const struct vdso_data *vd = __arch_get_vdso_data();
  88         u32 msk;
  89
  90         /* Check for negative values or invalid clocks */
  91         if (unlikely((u32) clock >= MAX_CLOCKS))
  92                 goto fallback;
  93
  94         /*
  95          * Convert the clockid to a bitmask and use it to check which
  96          * clocks are handled in the VDSO directly.
  97          */
  98         msk = 1U << clock;
  99         if (likely(msk & VDSO_HRES)) {
 100                 return do_hres(&vd[CS_HRES_COARSE], clock, ts);
 101         } else if (msk & VDSO_COARSE) {
 102                 do_coarse(&vd[CS_HRES_COARSE], clock, ts);
 103                 return 0;
 104         } else if (msk & VDSO_RAW) {
 105                 return do_hres(&vd[CS_RAW], clock, ts);
 106         }
 107
 108 fallback:
 109         return clock_gettime_fallback(clock, ts);
 110 }
 111
 112 static __maybe_unused int
 113 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
 114 {
 115         struct __kernel_timespec ts;
 116         int ret;
 117
 118         ret = __cvdso_clock_gettime(clock, &ts);
 119
 120         if (ret == 0) {
 121                 res->tv_sec = ts.tv_sec;
 122                 res->tv_nsec = ts.tv_nsec;
 123         }
 124
 125         return ret;
 126 }
 127
 128 static __maybe_unused int
 129 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 130 {
 131         const struct vdso_data *vd = __arch_get_vdso_data();
 132
 133         if (likely(tv != NULL)) {
 134                 struct __kernel_timespec ts;
 135
 136                 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
 137                         return gettimeofday_fallback(tv, tz);
 138
 139                 tv->tv_sec = ts.tv_sec;
 140                 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
 141         }
 142
 143         if (unlikely(tz != NULL)) {
 144                 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
 145                 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
 146         }
 147
 148         return 0;
 149 }
 150
 151 #ifdef VDSO_HAS_TIME
 152 static __maybe_unused time_t __cvdso_time(time_t *time)
 153 {
 154         const struct vdso_data *vd = __arch_get_vdso_data();
 155         time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
 156
 157         if (time)
 158                 *time = t;
 159
 160         return t;
 161 }
 162 #endif /* VDSO_HAS_TIME */
 163
 164 #ifdef VDSO_HAS_CLOCK_GETRES
 165 static __maybe_unused
 166 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
 167 {
 168         const struct vdso_data *vd = __arch_get_vdso_data();
 169         u64 ns;
 170         u32 msk;
 171         u64 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
 172
 173         /* Check for negative values or invalid clocks */
 174         if (unlikely((u32) clock >= MAX_CLOCKS))
 175                 goto fallback;
 176
 177         /*
 178          * Convert the clockid to a bitmask and use it to check which
 179          * clocks are handled in the VDSO directly.
 180          */
 181         msk = 1U << clock;
 182         if (msk & VDSO_HRES) {
 183                 /*
 184                  * Preserves the behaviour of posix_get_hrtimer_res().
 185                  */
 186                 ns = hrtimer_res;
 187         } else if (msk & VDSO_COARSE) {
 188                 /*
 189                  * Preserves the behaviour of posix_get_coarse_res().
 190                  */
 191                 ns = LOW_RES_NSEC;
 192         } else if (msk & VDSO_RAW) {
 193                 /*
 194                  * Preserves the behaviour of posix_get_hrtimer_res().
 195                  */
 196                 ns = hrtimer_res;
 197         } else {
 198                 goto fallback;
 199         }
 200
 201         res->tv_sec = 0;
 202         res->tv_nsec = ns;
 203
 204         return 0;
 205
 206 fallback:
 207         return clock_getres_fallback(clock, res);
 208 }
 209
 210 static __maybe_unused int
 211 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
 212 {
 213         struct __kernel_timespec ts;
 214         int ret;
 215
 216         ret = __cvdso_clock_getres(clock, &ts);
 217
 218         if (ret == 0) {
 219                 res->tv_sec = ts.tv_sec;
 220                 res->tv_nsec = ts.tv_nsec;
 221         }
 222
 223         return ret;
 224 }
 225 #endif /* VDSO_HAS_CLOCK_GETRES */