lib/vdso/gettimeofday.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic userspace implementations of gettimeofday() and similar.
   4  */
   5 #include <linux/compiler.h>
   6 #include <linux/math64.h>
   7 #include <linux/time.h>
   8 #include <linux/kernel.h>
   9 #include <linux/hrtimer_defs.h>
  10 #include <vdso/datapage.h>
  11 #include <vdso/helpers.h>
  12
  13 /*
  14  * The generic vDSO implementation requires that gettimeofday.h
  15  * provides:
  16  * - __arch_get_vdso_data(): to get the vdso datapage.
  17  * - __arch_get_hw_counter(): to get the hw counter based on the
  18  *   clock_mode.
  19  * - gettimeofday_fallback(): fallback for gettimeofday.
  20  * - clock_gettime_fallback(): fallback for clock_gettime.
  21  * - clock_getres_fallback(): fallback for clock_getres.
  22  */
  23 #ifdef ENABLE_COMPAT_VDSO
  24 #include <asm/vdso/compat_gettimeofday.h>
  25 #else
  26 #include <asm/vdso/gettimeofday.h>
  27 #endif /* ENABLE_COMPAT_VDSO */
  28
  29 #ifndef vdso_calc_delta
  30 /*
  31  * Default implementation which works for all sane clocksources. That
  32  * obviously excludes x86/TSC.
  33  */
  34 static __always_inline
  35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
  36 {
  37         return ((cycles - last) & mask) * mult;
  38 }
  39 #endif
  40
  41 static int do_hres(const struct vdso_data *vd, clockid_t clk,
  42                    struct __kernel_timespec *ts)
  43 {
  44         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  45         u64 cycles, last, sec, ns;
  46         u32 seq;
  47
  48         do {
  49                 seq = vdso_read_begin(vd);
  50                 cycles = __arch_get_hw_counter(vd->clock_mode);
  51                 ns = vdso_ts->nsec;
  52                 last = vd->cycle_last;
  53                 if (unlikely((s64)cycles < 0))
  54                         return -1;
  55
  56                 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
  57                 ns >>= vd->shift;
  58                 sec = vdso_ts->sec;
  59         } while (unlikely(vdso_read_retry(vd, seq)));
  60
  61         /*
  62          * Do this outside the loop: a race inside the loop could result
  63          * in __iter_div_u64_rem() being extremely slow.
  64          */
  65         ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
  66         ts->tv_nsec = ns;
  67
  68         return 0;
  69 }
  70
  71 static void do_coarse(const struct vdso_data *vd, clockid_t clk,
  72                       struct __kernel_timespec *ts)
  73 {
  74         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  75         u32 seq;
  76
  77         do {
  78                 seq = vdso_read_begin(vd);
  79                 ts->tv_sec = vdso_ts->sec;
  80                 ts->tv_nsec = vdso_ts->nsec;
  81         } while (unlikely(vdso_read_retry(vd, seq)));
  82 }
  83
  84 static __maybe_unused int
  85 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
  86 {
  87         const struct vdso_data *vd = __arch_get_vdso_data();
  88         u32 msk;
  89
  90         /* Check for negative values or invalid clocks */
  91         if (unlikely((u32) clock >= MAX_CLOCKS))
  92                 return -1;
  93
  94         /*
  95          * Convert the clockid to a bitmask and use it to check which
  96          * clocks are handled in the VDSO directly.
  97          */
  98         msk = 1U << clock;
  99         if (likely(msk & VDSO_HRES)) {
 100                 return do_hres(&vd[CS_HRES_COARSE], clock, ts);
 101         } else if (msk & VDSO_COARSE) {
 102                 do_coarse(&vd[CS_HRES_COARSE], clock, ts);
 103                 return 0;
 104         } else if (msk & VDSO_RAW) {
 105                 return do_hres(&vd[CS_RAW], clock, ts);
 106         }
 107         return -1;
 108 }
 109
 110 static __maybe_unused int
 111 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
 112 {
 113         int ret = __cvdso_clock_gettime_common(clock, ts);
 114
 115         if (unlikely(ret))
 116                 return clock_gettime_fallback(clock, ts);
 117         return 0;
 118 }
 119
 120 #ifdef BUILD_VDSO32
 121 static __maybe_unused int
 122 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
 123 {
 124         struct __kernel_timespec ts;
 125         int ret;
 126
 127         ret = __cvdso_clock_gettime_common(clock, &ts);
 128
 129 #ifdef VDSO_HAS_32BIT_FALLBACK
 130         if (unlikely(ret))
 131                 return clock_gettime32_fallback(clock, res);
 132 #else
 133         if (unlikely(ret))
 134                 ret = clock_gettime_fallback(clock, &ts);
 135 #endif
 136
 137         if (likely(!ret)) {
 138                 res->tv_sec = ts.tv_sec;
 139                 res->tv_nsec = ts.tv_nsec;
 140         }
 141         return ret;
 142 }
 143 #endif /* BUILD_VDSO32 */
 144
 145 static __maybe_unused int
 146 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 147 {
 148         const struct vdso_data *vd = __arch_get_vdso_data();
 149
 150         if (likely(tv != NULL)) {
 151                 struct __kernel_timespec ts;
 152
 153                 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
 154                         return gettimeofday_fallback(tv, tz);
 155
 156                 tv->tv_sec = ts.tv_sec;
 157                 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
 158         }
 159
 160         if (unlikely(tz != NULL)) {
 161                 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
 162                 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
 163         }
 164
 165         return 0;
 166 }
 167
 168 #ifdef VDSO_HAS_TIME
 169 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
 170 {
 171         const struct vdso_data *vd = __arch_get_vdso_data();
 172         __kernel_old_time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
 173
 174         if (time)
 175                 *time = t;
 176
 177         return t;
 178 }
 179 #endif /* VDSO_HAS_TIME */
 180
 181 #ifdef VDSO_HAS_CLOCK_GETRES
 182 static __maybe_unused
 183 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
 184 {
 185         const struct vdso_data *vd = __arch_get_vdso_data();
 186         u64 hrtimer_res;
 187         u32 msk;
 188         u64 ns;
 189
 190         /* Check for negative values or invalid clocks */
 191         if (unlikely((u32) clock >= MAX_CLOCKS))
 192                 return -1;
 193
 194         hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
 195         /*
 196          * Convert the clockid to a bitmask and use it to check which
 197          * clocks are handled in the VDSO directly.
 198          */
 199         msk = 1U << clock;
 200         if (msk & VDSO_HRES) {
 201                 /*
 202                  * Preserves the behaviour of posix_get_hrtimer_res().
 203                  */
 204                 ns = hrtimer_res;
 205         } else if (msk & VDSO_COARSE) {
 206                 /*
 207                  * Preserves the behaviour of posix_get_coarse_res().
 208                  */
 209                 ns = LOW_RES_NSEC;
 210         } else if (msk & VDSO_RAW) {
 211                 /*
 212                  * Preserves the behaviour of posix_get_hrtimer_res().
 213                  */
 214                 ns = hrtimer_res;
 215         } else {
 216                 return -1;
 217         }
 218
 219         if (likely(res)) {
 220                 res->tv_sec = 0;
 221                 res->tv_nsec = ns;
 222         }
 223         return 0;
 224 }
 225
 226 static __maybe_unused
 227 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
 228 {
 229         int ret = __cvdso_clock_getres_common(clock, res);
 230
 231         if (unlikely(ret))
 232                 return clock_getres_fallback(clock, res);
 233         return 0;
 234 }
 235
 236 #ifdef BUILD_VDSO32
 237 static __maybe_unused int
 238 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
 239 {
 240         struct __kernel_timespec ts;
 241         int ret;
 242
 243         ret = __cvdso_clock_getres_common(clock, &ts);
 244
 245 #ifdef VDSO_HAS_32BIT_FALLBACK
 246         if (unlikely(ret))
 247                 return clock_getres32_fallback(clock, res);
 248 #else
 249         if (unlikely(ret))
 250                 ret = clock_getres_fallback(clock, &ts);
 251 #endif
 252
 253         if (likely(!ret && res)) {
 254                 res->tv_sec = ts.tv_sec;
 255                 res->tv_nsec = ts.tv_nsec;
 256         }
 257         return ret;
 258 }
 259 #endif /* BUILD_VDSO32 */
 260 #endif /* VDSO_HAS_CLOCK_GETRES */