lib/vdso/gettimeofday.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic userspace implementations of gettimeofday() and similar.
   4  */
   5 #include <linux/compiler.h>
   6 #include <linux/math64.h>
   7 #include <linux/time.h>
   8 #include <linux/kernel.h>
   9 #include <linux/hrtimer_defs.h>
  10 #include <vdso/datapage.h>
  11 #include <vdso/helpers.h>
  12
  13 /*
  14  * The generic vDSO implementation requires that gettimeofday.h
  15  * provides:
  16  * - __arch_get_vdso_data(): to get the vdso datapage.
  17  * - __arch_get_hw_counter(): to get the hw counter based on the
  18  *   clock_mode.
  19  * - gettimeofday_fallback(): fallback for gettimeofday.
  20  * - clock_gettime_fallback(): fallback for clock_gettime.
  21  * - clock_getres_fallback(): fallback for clock_getres.
  22  */
  23 #ifdef ENABLE_COMPAT_VDSO
  24 #include <asm/vdso/compat_gettimeofday.h>
  25 #else
  26 #include <asm/vdso/gettimeofday.h>
  27 #endif /* ENABLE_COMPAT_VDSO */
  28
  29 #ifndef vdso_calc_delta
  30 /*
  31  * Default implementation which works for all sane clocksources. That
  32  * obviously excludes x86/TSC.
  33  */
  34 static __always_inline
  35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
  36 {
  37         return ((cycles - last) & mask) * mult;
  38 }
  39 #endif
  40
  41 #ifdef CONFIG_TIME_NS
  42 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
  43                           struct __kernel_timespec *ts)
  44 {
  45         const struct vdso_data *vd = __arch_get_timens_vdso_data();
  46         const struct timens_offset *offs = &vdns->offset[clk];
  47         const struct vdso_timestamp *vdso_ts;
  48         u64 cycles, last, ns;
  49         u32 seq;
  50         s64 sec;
  51
  52         if (clk != CLOCK_MONOTONIC_RAW)
  53                 vd = &vd[CS_HRES_COARSE];
  54         else
  55                 vd = &vd[CS_RAW];
  56         vdso_ts = &vd->basetime[clk];
  57
  58         do {
  59                 seq = vdso_read_begin(vd);
  60                 cycles = __arch_get_hw_counter(vd->clock_mode);
  61                 ns = vdso_ts->nsec;
  62                 last = vd->cycle_last;
  63                 if (unlikely((s64)cycles < 0))
  64                         return -1;
  65
  66                 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
  67                 ns >>= vd->shift;
  68                 sec = vdso_ts->sec;
  69         } while (unlikely(vdso_read_retry(vd, seq)));
  70
  71         /* Add the namespace offset */
  72         sec += offs->sec;
  73         ns += offs->nsec;
  74
  75         /*
  76          * Do this outside the loop: a race inside the loop could result
  77          * in __iter_div_u64_rem() being extremely slow.
  78          */
  79         ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
  80         ts->tv_nsec = ns;
  81
  82         return 0;
  83 }
  84 #else
  85 static __always_inline const struct vdso_data *__arch_get_timens_vdso_data(void)
  86 {
  87         return NULL;
  88 }
  89
  90 static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
  91                           struct __kernel_timespec *ts)
  92 {
  93         return -EINVAL;
  94 }
  95 #endif
  96
  97 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
  98                                    struct __kernel_timespec *ts)
  99 {
 100         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
 101         u64 cycles, last, sec, ns;
 102         u32 seq;
 103
 104         do {
 105                 /*
 106                  * Open coded to handle VCLOCK_TIMENS. Time namespace
 107                  * enabled tasks have a special VVAR page installed which
 108                  * has vd->seq set to 1 and vd->clock_mode set to
 109                  * VCLOCK_TIMENS. For non time namespace affected tasks
 110                  * this does not affect performance because if vd->seq is
 111                  * odd, i.e. a concurrent update is in progress the extra
 112                  * check for vd->clock_mode is just a few extra
 113                  * instructions while spin waiting for vd->seq to become
 114                  * even again.
 115                  */
 116                 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
 117                         if (IS_ENABLED(CONFIG_TIME_NS) &&
 118                             vd->clock_mode == VCLOCK_TIMENS)
 119                                 return do_hres_timens(vd, clk, ts);
 120                         cpu_relax();
 121                 }
 122                 smp_rmb();
 123
 124                 cycles = __arch_get_hw_counter(vd->clock_mode);
 125                 ns = vdso_ts->nsec;
 126                 last = vd->cycle_last;
 127                 if (unlikely((s64)cycles < 0))
 128                         return -1;
 129
 130                 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
 131                 ns >>= vd->shift;
 132                 sec = vdso_ts->sec;
 133         } while (unlikely(vdso_read_retry(vd, seq)));
 134
 135         /*
 136          * Do this outside the loop: a race inside the loop could result
 137          * in __iter_div_u64_rem() being extremely slow.
 138          */
 139         ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 140         ts->tv_nsec = ns;
 141
 142         return 0;
 143 }
 144
 145 #ifdef CONFIG_TIME_NS
 146 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
 147                             struct __kernel_timespec *ts)
 148 {
 149         const struct vdso_data *vd = __arch_get_timens_vdso_data();
 150         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
 151         const struct timens_offset *offs = &vdns->offset[clk];
 152         u64 nsec;
 153         s64 sec;
 154         s32 seq;
 155
 156         do {
 157                 seq = vdso_read_begin(vd);
 158                 sec = vdso_ts->sec;
 159                 nsec = vdso_ts->nsec;
 160         } while (unlikely(vdso_read_retry(vd, seq)));
 161
 162         /* Add the namespace offset */
 163         sec += offs->sec;
 164         nsec += offs->nsec;
 165
 166         /*
 167          * Do this outside the loop: a race inside the loop could result
 168          * in __iter_div_u64_rem() being extremely slow.
 169          */
 170         ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
 171         ts->tv_nsec = nsec;
 172         return 0;
 173 }
 174 #else
 175 static int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
 176                             struct __kernel_timespec *ts)
 177 {
 178         return -1;
 179 }
 180 #endif
 181
 182 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
 183                                      struct __kernel_timespec *ts)
 184 {
 185         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
 186         u32 seq;
 187
 188         do {
 189                 /*
 190                  * Open coded to handle VCLOCK_TIMENS. See comment in
 191                  * do_hres().
 192                  */
 193                 while ((seq = READ_ONCE(vd->seq)) & 1) {
 194                         if (IS_ENABLED(CONFIG_TIME_NS) &&
 195                             vd->clock_mode == VCLOCK_TIMENS)
 196                                 return do_coarse_timens(vd, clk, ts);
 197                         cpu_relax();
 198                 }
 199                 smp_rmb();
 200
 201                 ts->tv_sec = vdso_ts->sec;
 202                 ts->tv_nsec = vdso_ts->nsec;
 203         } while (unlikely(vdso_read_retry(vd, seq)));
 204
 205         return 0;
 206 }
 207
 208 static __maybe_unused int
 209 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
 210 {
 211         const struct vdso_data *vd = __arch_get_vdso_data();
 212         u32 msk;
 213
 214         /* Check for negative values or invalid clocks */
 215         if (unlikely((u32) clock >= MAX_CLOCKS))
 216                 return -1;
 217
 218         /*
 219          * Convert the clockid to a bitmask and use it to check which
 220          * clocks are handled in the VDSO directly.
 221          */
 222         msk = 1U << clock;
 223         if (likely(msk & VDSO_HRES))
 224                 vd = &vd[CS_HRES_COARSE];
 225         else if (msk & VDSO_COARSE)
 226                 return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
 227         else if (msk & VDSO_RAW)
 228                 vd = &vd[CS_RAW];
 229         else
 230                 return -1;
 231
 232         return do_hres(vd, clock, ts);
 233 }
 234
 235 static __maybe_unused int
 236 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
 237 {
 238         int ret = __cvdso_clock_gettime_common(clock, ts);
 239
 240         if (unlikely(ret))
 241                 return clock_gettime_fallback(clock, ts);
 242         return 0;
 243 }
 244
 245 #ifdef BUILD_VDSO32
 246 static __maybe_unused int
 247 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
 248 {
 249         struct __kernel_timespec ts;
 250         int ret;
 251
 252         ret = __cvdso_clock_gettime_common(clock, &ts);
 253
 254         if (unlikely(ret))
 255                 return clock_gettime32_fallback(clock, res);
 256
 257         /* For ret == 0 */
 258         res->tv_sec = ts.tv_sec;
 259         res->tv_nsec = ts.tv_nsec;
 260
 261         return ret;
 262 }
 263 #endif /* BUILD_VDSO32 */
 264
 265 static __maybe_unused int
 266 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 267 {
 268         const struct vdso_data *vd = __arch_get_vdso_data();
 269
 270         if (likely(tv != NULL)) {
 271                 struct __kernel_timespec ts;
 272
 273                 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
 274                         return gettimeofday_fallback(tv, tz);
 275
 276                 tv->tv_sec = ts.tv_sec;
 277                 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
 278         }
 279
 280         if (unlikely(tz != NULL)) {
 281                 if (IS_ENABLED(CONFIG_TIME_NS) &&
 282                     vd->clock_mode == VCLOCK_TIMENS)
 283                         vd = __arch_get_timens_vdso_data();
 284
 285                 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
 286                 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
 287         }
 288
 289         return 0;
 290 }
 291
 292 #ifdef VDSO_HAS_TIME
 293 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
 294 {
 295         const struct vdso_data *vd = __arch_get_vdso_data();
 296         __kernel_old_time_t t;
 297
 298         if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS)
 299                 vd = __arch_get_timens_vdso_data();
 300
 301         t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
 302
 303         if (time)
 304                 *time = t;
 305
 306         return t;
 307 }
 308 #endif /* VDSO_HAS_TIME */
 309
 310 #ifdef VDSO_HAS_CLOCK_GETRES
 311 static __maybe_unused
 312 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
 313 {
 314         const struct vdso_data *vd = __arch_get_vdso_data();
 315         u32 msk;
 316         u64 ns;
 317
 318         /* Check for negative values or invalid clocks */
 319         if (unlikely((u32) clock >= MAX_CLOCKS))
 320                 return -1;
 321
 322         if (IS_ENABLED(CONFIG_TIME_NS) && vd->clock_mode == VCLOCK_TIMENS)
 323                 vd = __arch_get_timens_vdso_data();
 324
 325         /*
 326          * Convert the clockid to a bitmask and use it to check which
 327          * clocks are handled in the VDSO directly.
 328          */
 329         msk = 1U << clock;
 330         if (msk & (VDSO_HRES | VDSO_RAW)) {
 331                 /*
 332                  * Preserves the behaviour of posix_get_hrtimer_res().
 333                  */
 334                 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
 335         } else if (msk & VDSO_COARSE) {
 336                 /*
 337                  * Preserves the behaviour of posix_get_coarse_res().
 338                  */
 339                 ns = LOW_RES_NSEC;
 340         } else {
 341                 return -1;
 342         }
 343
 344         if (likely(res)) {
 345                 res->tv_sec = 0;
 346                 res->tv_nsec = ns;
 347         }
 348         return 0;
 349 }
 350
 351 static __maybe_unused
 352 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
 353 {
 354         int ret = __cvdso_clock_getres_common(clock, res);
 355
 356         if (unlikely(ret))
 357                 return clock_getres_fallback(clock, res);
 358         return 0;
 359 }
 360
 361 #ifdef BUILD_VDSO32
 362 static __maybe_unused int
 363 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
 364 {
 365         struct __kernel_timespec ts;
 366         int ret;
 367
 368         ret = __cvdso_clock_getres_common(clock, &ts);
 369
 370         if (unlikely(ret))
 371                 return clock_getres32_fallback(clock, res);
 372
 373         if (likely(res)) {
 374                 res->tv_sec = ts.tv_sec;
 375                 res->tv_nsec = ts.tv_nsec;
 376         }
 377         return ret;
 378 }
 379 #endif /* BUILD_VDSO32 */
 380 #endif /* VDSO_HAS_CLOCK_GETRES */