2 * This code largely moved from arch/i386/kernel/timer/timer_tsc.c
3 * which was originally moved from arch/i386/kernel/time.c.
4 * See comments there for proper credits.
7 #include <linux/workqueue.h>
8 #include <linux/cpufreq.h>
9 #include <linux/jiffies.h>
10 #include <linux/init.h>
13 #include <asm/delay.h>
16 #include "mach_timer.h"
19 * On some systems the TSC frequency does not
20 * change with the cpu frequency. So we need
21 * an extra value to store the TSC freq
25 int tsc_disable __cpuinitdata = 0;
28 static int __init tsc_setup(char *str)
30 printk(KERN_WARNING "notsc: Kernel compiled with CONFIG_X86_TSC, "
31 "cannot disable TSC.\n");
36 * disable flag for tsc. Takes effect by clearing the TSC cpu flag
39 static int __init tsc_setup(char *str)
47 __setup("notsc", tsc_setup);
50 * code to mark and check if the TSC is unstable
51 * due to cpufreq or due to unsynced TSCs
53 static int tsc_unstable;
55 static inline int check_tsc_unstable(void)
60 void mark_tsc_unstable(void)
64 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
66 /* Accellerators for sched_clock()
67 * convert from cycles(64bits) => nanoseconds (64bits)
69 * ns = cycles / (freq / ns_per_sec)
70 * ns = cycles * (ns_per_sec / freq)
71 * ns = cycles * (10^9 / (cpu_khz * 10^3))
72 * ns = cycles * (10^6 / cpu_khz)
74 * Then we use scaling math (suggested by george@mvista.com) to get:
75 * ns = cycles * (10^6 * SC / cpu_khz) / SC
76 * ns = cycles * cyc2ns_scale / SC
78 * And since SC is a constant power of two, we can convert the div
81 * We can use khz divisor instead of mhz to keep a better percision, since
82 * cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
83 * (mathieu.desnoyers@polymtl.ca)
85 * -johnstul@us.ibm.com "math is hard, lets go shopping!"
87 static unsigned long cyc2ns_scale __read_mostly;
89 #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */
91 static inline void set_cyc2ns_scale(unsigned long cpu_khz)
93 cyc2ns_scale = (1000000 << CYC2NS_SCALE_FACTOR)/cpu_khz;
96 static inline unsigned long long cycles_2_ns(unsigned long long cyc)
98 return (cyc * cyc2ns_scale) >> CYC2NS_SCALE_FACTOR;
102 * Scheduler clock - returns current time in nanosec units.
104 unsigned long long sched_clock(void)
106 unsigned long long this_offset;
109 * in the NUMA case we dont use the TSC as they are not
110 * synchronized across all CPUs.
113 if (!cpu_khz || check_tsc_unstable())
115 /* no locking but a rare wrong value is not a big deal */
116 return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
118 /* read the Time Stamp Counter: */
119 rdtscll(this_offset);
121 /* return the value in ns */
122 return cycles_2_ns(this_offset);
125 static unsigned long calculate_cpu_khz(void)
127 unsigned long long start, end;
133 local_irq_save(flags);
135 /* run 3 times to ensure the cache is warm */
136 for (i = 0; i < 3; i++) {
137 mach_prepare_counter();
139 mach_countup(&count);
143 * Error: ECTCNEVERSET
144 * The CTC wasn't reliable: we got a hit on the very first read,
145 * or the CPU was so fast/slow that the quotient wouldn't fit in
151 delta64 = end - start;
153 /* cpu freq too fast: */
154 if (delta64 > (1ULL<<32))
157 /* cpu freq too slow: */
158 if (delta64 <= CALIBRATE_TIME_MSEC)
161 delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */
162 do_div(delta64,CALIBRATE_TIME_MSEC);
164 local_irq_restore(flags);
165 return (unsigned long)delta64;
167 local_irq_restore(flags);
171 int recalibrate_cpu_khz(void)
174 unsigned long cpu_khz_old = cpu_khz;
177 cpu_khz = calculate_cpu_khz();
179 cpu_data[0].loops_per_jiffy =
180 cpufreq_scale(cpu_data[0].loops_per_jiffy,
181 cpu_khz_old, cpu_khz);
190 EXPORT_SYMBOL(recalibrate_cpu_khz);
194 if (!cpu_has_tsc || tsc_disable)
197 cpu_khz = calculate_cpu_khz();
203 printk("Detected %lu.%03lu MHz processor.\n",
204 (unsigned long)cpu_khz / 1000,
205 (unsigned long)cpu_khz % 1000);
207 set_cyc2ns_scale(cpu_khz);
211 #ifdef CONFIG_CPU_FREQ
213 static unsigned int cpufreq_delayed_issched = 0;
214 static unsigned int cpufreq_init = 0;
215 static struct work_struct cpufreq_delayed_get_work;
217 static void handle_cpufreq_delayed_get(void *v)
221 for_each_online_cpu(cpu)
224 cpufreq_delayed_issched = 0;
228 * if we notice cpufreq oddness, schedule a call to cpufreq_get() as it tries
229 * to verify the CPU frequency the timing core thinks the CPU is running
230 * at is still correct.
232 static inline void cpufreq_delayed_get(void)
234 if (cpufreq_init && !cpufreq_delayed_issched) {
235 cpufreq_delayed_issched = 1;
236 printk(KERN_DEBUG "Checking if CPU frequency changed.\n");
237 schedule_work(&cpufreq_delayed_get_work);
242 * if the CPU frequency is scaled, TSC-based delays will need a different
243 * loops_per_jiffy value to function properly.
245 static unsigned int ref_freq = 0;
246 static unsigned long loops_per_jiffy_ref = 0;
247 static unsigned long cpu_khz_ref = 0;
250 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
252 struct cpufreq_freqs *freq = data;
254 if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)
255 write_seqlock_irq(&xtime_lock);
259 ref_freq = freq->new;
262 ref_freq = freq->old;
263 loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy;
264 cpu_khz_ref = cpu_khz;
267 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
268 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
269 (val == CPUFREQ_RESUMECHANGE)) {
270 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
271 cpu_data[freq->cpu].loops_per_jiffy =
272 cpufreq_scale(loops_per_jiffy_ref,
273 ref_freq, freq->new);
277 if (num_online_cpus() == 1)
278 cpu_khz = cpufreq_scale(cpu_khz_ref,
279 ref_freq, freq->new);
280 if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
282 set_cyc2ns_scale(cpu_khz);
284 * TSC based sched_clock turns
292 if (val != CPUFREQ_RESUMECHANGE && val != CPUFREQ_SUSPENDCHANGE)
293 write_sequnlock_irq(&xtime_lock);
298 static struct notifier_block time_cpufreq_notifier_block = {
299 .notifier_call = time_cpufreq_notifier
302 static int __init cpufreq_tsc(void)
306 INIT_WORK(&cpufreq_delayed_get_work, handle_cpufreq_delayed_get, NULL);
307 ret = cpufreq_register_notifier(&time_cpufreq_notifier_block,
308 CPUFREQ_TRANSITION_NOTIFIER);
315 core_initcall(cpufreq_tsc);