2 * Copyright 2006 Andi Kleen, SUSE Labs.
3 * Subject to the GNU Public License, v.2
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
7 * The code should have no internal unresolved relocations.
8 * Check with readelf after changing.
9 * Also alternative() doesn't work.
12 * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
15 /* Disable profiling for userspace code: */
16 #ifndef DISABLE_BRANCH_PROFILING
17 #define DISABLE_BRANCH_PROFILING
20 #include <linux/kernel.h>
21 #include <linux/time.h>
22 #include <linux/string.h>
24 #include <asm/unistd.h>
25 #include <asm/timex.h>
26 #include <asm/clocksource.h>
31 #define TICK_PRIV_BIT (1UL << 63)
33 #define TICK_PRIV_BIT (1ULL << 63)
36 #define SYSCALL_STRING \
38 "sub %%g0, %%o0, %%o0;" \
40 #define SYSCALL_CLOBBERS \
41 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \
42 "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \
43 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \
44 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \
45 "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \
46 "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \
50 * Compute the vvar page's address in the process address space, and return it
51 * as a pointer to the vvar_data.
53 static notrace noinline struct vvar_data *
59 * vdso data page is the first vDSO page so grab the return address
60 * and move up a page to get to the data page.
62 ret = (unsigned long)__builtin_return_address(0);
66 return (struct vvar_data *) ret;
70 vdso_fallback_gettime(long clock, struct timespec *ts)
72 register long num __asm__("g1") = __NR_clock_gettime;
73 register long o0 __asm__("o0") = clock;
74 register long o1 __asm__("o1") = (long) ts;
76 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
77 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
81 static notrace __always_inline long
82 vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz)
84 register long num __asm__("g1") = __NR_gettimeofday;
85 register long o0 __asm__("o0") = (long) tv;
86 register long o1 __asm__("o1") = (long) tz;
88 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
89 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
94 static notrace noinline u64
98 __asm__ __volatile__("rd %%asr24, %0 \n"
99 ".section .vread_tick_patch, \"ax\" \n"
103 return ret & ~TICK_PRIV_BIT;
106 static notrace noinline u64
111 __asm__ __volatile__("rd %%asr24, %%g1\n\t"
112 "srlx %%g1, 32, %1\n\t"
114 ".section .vread_tick_patch, \"ax\" \n"
117 : "=&r" (lo), "=&r" (hi)
120 return lo | ((u64)hi << 32);
124 static notrace inline u64
125 vgetsns(struct vvar_data *vvar)
130 cycles = vread_tick();
131 v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
132 return v * vvar->clock.mult;
135 static notrace noinline int
136 do_realtime(struct vvar_data *vvar, struct timespec *ts)
143 seq = vvar_read_begin(vvar);
144 ts->tv_sec = vvar->wall_time_sec;
145 ns = vvar->wall_time_snsec;
147 ns >>= vvar->clock.shift;
148 } while (unlikely(vvar_read_retry(vvar, seq)));
150 timespec_add_ns(ts, ns);
155 static notrace noinline int
156 do_monotonic(struct vvar_data *vvar, struct timespec *ts)
163 seq = vvar_read_begin(vvar);
164 ts->tv_sec = vvar->monotonic_time_sec;
165 ns = vvar->monotonic_time_snsec;
167 ns >>= vvar->clock.shift;
168 } while (unlikely(vvar_read_retry(vvar, seq)));
170 timespec_add_ns(ts, ns);
175 static notrace noinline int
176 do_realtime_coarse(struct vvar_data *vvar, struct timespec *ts)
181 seq = vvar_read_begin(vvar);
182 ts->tv_sec = vvar->wall_time_coarse_sec;
183 ts->tv_nsec = vvar->wall_time_coarse_nsec;
184 } while (unlikely(vvar_read_retry(vvar, seq)));
188 static notrace noinline int
189 do_monotonic_coarse(struct vvar_data *vvar, struct timespec *ts)
194 seq = vvar_read_begin(vvar);
195 ts->tv_sec = vvar->monotonic_time_coarse_sec;
196 ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
197 } while (unlikely(vvar_read_retry(vvar, seq)));
203 __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
205 struct vvar_data *vvd = get_vvar_data();
209 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
211 return do_realtime(vvd, ts);
212 case CLOCK_MONOTONIC:
213 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
215 return do_monotonic(vvd, ts);
216 case CLOCK_REALTIME_COARSE:
217 return do_realtime_coarse(vvd, ts);
218 case CLOCK_MONOTONIC_COARSE:
219 return do_monotonic_coarse(vvd, ts);
222 * Unknown clock ID ? Fall back to the syscall.
224 return vdso_fallback_gettime(clock, ts);
227 clock_gettime(clockid_t, struct timespec *)
228 __attribute__((weak, alias("__vdso_clock_gettime")));
231 __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
233 struct vvar_data *vvd = get_vvar_data();
235 if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
236 if (likely(tv != NULL)) {
240 } *tstv = (union tstv_t *) tv;
241 do_realtime(vvd, &tstv->ts);
243 * Assign before dividing to ensure that the division is
244 * done in the type of tv_usec, not tv_nsec.
246 * There cannot be > 1 billion usec in a second:
247 * do_realtime() has already distributed such overflow
248 * into tv_sec. So we can assign it to an int safely.
250 tstv->tv.tv_usec = tstv->ts.tv_nsec;
251 tstv->tv.tv_usec /= 1000;
253 if (unlikely(tz != NULL)) {
254 /* Avoid memcpy. Some old compilers fail to inline it */
255 tz->tz_minuteswest = vvd->tz_minuteswest;
256 tz->tz_dsttime = vvd->tz_dsttime;
260 return vdso_fallback_gettimeofday(tv, tz);
263 gettimeofday(struct timeval *, struct timezone *)
264 __attribute__((weak, alias("__vdso_gettimeofday")));