lib/vdso/gettimeofday.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic userspace implementations of gettimeofday() and similar.
   4  */
   5 #include <linux/compiler.h>
   6 #include <linux/math64.h>
   7 #include <linux/time.h>
   8 #include <linux/kernel.h>
   9 #include <linux/hrtimer_defs.h>
  10 #include <vdso/datapage.h>
  11 #include <vdso/helpers.h>
  12
  13 /*
  14  * The generic vDSO implementation requires that gettimeofday.h
  15  * provides:
  16  * - __arch_get_vdso_data(): to get the vdso datapage.
  17  * - __arch_get_hw_counter(): to get the hw counter based on the
  18  *   clock_mode.
  19  * - gettimeofday_fallback(): fallback for gettimeofday.
  20  * - clock_gettime_fallback(): fallback for clock_gettime.
  21  * - clock_getres_fallback(): fallback for clock_getres.
  22  */
  23 #ifdef ENABLE_COMPAT_VDSO
  24 #include <asm/vdso/compat_gettimeofday.h>
  25 #else
  26 #include <asm/vdso/gettimeofday.h>
  27 #endif /* ENABLE_COMPAT_VDSO */
  28
  29 #ifndef vdso_calc_delta
  30 /*
  31  * Default implementation which works for all sane clocksources. That
  32  * obviously excludes x86/TSC.
  33  */
  34 static __always_inline
  35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
  36 {
  37         return ((cycles - last) & mask) * mult;
  38 }
  39 #endif
  40
  41 static int do_hres(const struct vdso_data *vd, clockid_t clk,
  42                    struct __kernel_timespec *ts)
  43 {
  44         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  45         u64 cycles, last, sec, ns;
  46         u32 seq;
  47
  48         do {
  49                 seq = vdso_read_begin(vd);
  50                 cycles = __arch_get_hw_counter(vd->clock_mode);
  51                 ns = vdso_ts->nsec;
  52                 last = vd->cycle_last;
  53                 if (unlikely((s64)cycles < 0))
  54                         return -1;
  55
  56                 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
  57                 ns >>= vd->shift;
  58                 sec = vdso_ts->sec;
  59         } while (unlikely(vdso_read_retry(vd, seq)));
  60
  61         /*
  62          * Do this outside the loop: a race inside the loop could result
  63          * in __iter_div_u64_rem() being extremely slow.
  64          */
  65         ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
  66         ts->tv_nsec = ns;
  67
  68         return 0;
  69 }
  70
  71 static void do_coarse(const struct vdso_data *vd, clockid_t clk,
  72                       struct __kernel_timespec *ts)
  73 {
  74         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  75         u32 seq;
  76
  77         do {
  78                 seq = vdso_read_begin(vd);
  79                 ts->tv_sec = vdso_ts->sec;
  80                 ts->tv_nsec = vdso_ts->nsec;
  81         } while (unlikely(vdso_read_retry(vd, seq)));
  82 }
  83
  84 static __maybe_unused int
  85 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
  86 {
  87         const struct vdso_data *vd = __arch_get_vdso_data();
  88         u32 msk;
  89
  90         /* Check for negative values or invalid clocks */
  91         if (unlikely((u32) clock >= MAX_CLOCKS))
  92                 return -1;
  93
  94         /*
  95          * Convert the clockid to a bitmask and use it to check which
  96          * clocks are handled in the VDSO directly.
  97          */
  98         msk = 1U << clock;
  99         if (likely(msk & VDSO_HRES)) {
 100                 return do_hres(&vd[CS_HRES_COARSE], clock, ts);
 101         } else if (msk & VDSO_COARSE) {
 102                 do_coarse(&vd[CS_HRES_COARSE], clock, ts);
 103                 return 0;
 104         } else if (msk & VDSO_RAW) {
 105                 return do_hres(&vd[CS_RAW], clock, ts);
 106         }
 107         return -1;
 108 }
 109
 110 static __maybe_unused int
 111 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
 112 {
 113         int ret = __cvdso_clock_gettime_common(clock, ts);
 114
 115         if (unlikely(ret))
 116                 return clock_gettime_fallback(clock, ts);
 117         return 0;
 118 }
 119
 120 static __maybe_unused int
 121 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
 122 {
 123         struct __kernel_timespec ts;
 124         int ret;
 125
 126         ret = __cvdso_clock_gettime_common(clock, &ts);
 127
 128 #ifdef VDSO_HAS_32BIT_FALLBACK
 129         if (unlikely(ret))
 130                 return clock_gettime32_fallback(clock, res);
 131 #else
 132         if (unlikely(ret))
 133                 ret = clock_gettime_fallback(clock, &ts);
 134 #endif
 135
 136         if (likely(!ret)) {
 137                 res->tv_sec = ts.tv_sec;
 138                 res->tv_nsec = ts.tv_nsec;
 139         }
 140         return ret;
 141 }
 142
 143 static __maybe_unused int
 144 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 145 {
 146         const struct vdso_data *vd = __arch_get_vdso_data();
 147
 148         if (likely(tv != NULL)) {
 149                 struct __kernel_timespec ts;
 150
 151                 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
 152                         return gettimeofday_fallback(tv, tz);
 153
 154                 tv->tv_sec = ts.tv_sec;
 155                 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
 156         }
 157
 158         if (unlikely(tz != NULL)) {
 159                 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
 160                 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
 161         }
 162
 163         return 0;
 164 }
 165
 166 #ifdef VDSO_HAS_TIME
 167 static __maybe_unused time_t __cvdso_time(time_t *time)
 168 {
 169         const struct vdso_data *vd = __arch_get_vdso_data();
 170         time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
 171
 172         if (time)
 173                 *time = t;
 174
 175         return t;
 176 }
 177 #endif /* VDSO_HAS_TIME */
 178
 179 #ifdef VDSO_HAS_CLOCK_GETRES
 180 static __maybe_unused
 181 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
 182 {
 183         const struct vdso_data *vd = __arch_get_vdso_data();
 184         u64 hrtimer_res;
 185         u32 msk;
 186         u64 ns;
 187
 188         /* Check for negative values or invalid clocks */
 189         if (unlikely((u32) clock >= MAX_CLOCKS))
 190                 return -1;
 191
 192         hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
 193         /*
 194          * Convert the clockid to a bitmask and use it to check which
 195          * clocks are handled in the VDSO directly.
 196          */
 197         msk = 1U << clock;
 198         if (msk & VDSO_HRES) {
 199                 /*
 200                  * Preserves the behaviour of posix_get_hrtimer_res().
 201                  */
 202                 ns = hrtimer_res;
 203         } else if (msk & VDSO_COARSE) {
 204                 /*
 205                  * Preserves the behaviour of posix_get_coarse_res().
 206                  */
 207                 ns = LOW_RES_NSEC;
 208         } else if (msk & VDSO_RAW) {
 209                 /*
 210                  * Preserves the behaviour of posix_get_hrtimer_res().
 211                  */
 212                 ns = hrtimer_res;
 213         } else {
 214                 return -1;
 215         }
 216
 217         res->tv_sec = 0;
 218         res->tv_nsec = ns;
 219
 220         return 0;
 221 }
 222
 223 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
 224 {
 225         int ret = __cvdso_clock_getres_common(clock, res);
 226
 227         if (unlikely(ret))
 228                 return clock_getres_fallback(clock, res);
 229         return 0;
 230 }
 231
 232 static __maybe_unused int
 233 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
 234 {
 235         struct __kernel_timespec ts;
 236         int ret;
 237
 238         ret = __cvdso_clock_getres_common(clock, &ts);
 239
 240 #ifdef VDSO_HAS_32BIT_FALLBACK
 241         if (unlikely(ret))
 242                 return clock_getres32_fallback(clock, res);
 243 #else
 244         if (unlikely(ret))
 245                 ret = clock_getres_fallback(clock, &ts);
 246 #endif
 247
 248         if (likely(!ret)) {
 249                 res->tv_sec = ts.tv_sec;
 250                 res->tv_nsec = ts.tv_nsec;
 251         }
 252         return ret;
 253 }
 254 #endif /* VDSO_HAS_CLOCK_GETRES */