Merge tag 'locks-v4.19-1' of git://git.kernel.org/pub/scm/linux/kernel/git/jlayton...

[linux-2.6-microblaze.git] / arch / x86 / events / intel / lbr.c

diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index cf372b9..f3e006b 100644 (file)
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -216,6 +216,8 @@ static void intel_pmu_lbr_reset_64(void)
 
 void intel_pmu_lbr_reset(void)
 {
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
        if (!x86_pmu.lbr_nr)
                return;
 
@@ -223,6 +225,9 @@ void intel_pmu_lbr_reset(void)
                intel_pmu_lbr_reset_32();
        else
                intel_pmu_lbr_reset_64();
+
+       cpuc->last_task_ctx = NULL;
+       cpuc->last_log_id = 0;
 }
 
 /*
@@ -334,6 +339,7 @@ static inline u64 rdlbr_to(unsigned int idx)
 
 static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
 {
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int i;
        unsigned lbr_idx, mask;
        u64 tos;
@@ -344,9 +350,21 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
                return;
        }
 
-       mask = x86_pmu.lbr_nr - 1;
        tos = task_ctx->tos;
-       for (i = 0; i < tos; i++) {
+       /*
+        * Does not restore the LBR registers, if
+        * - No one else touched them, and
+        * - Did not enter C6
+        */
+       if ((task_ctx == cpuc->last_task_ctx) &&
+           (task_ctx->log_id == cpuc->last_log_id) &&
+           rdlbr_from(tos)) {
+               task_ctx->lbr_stack_state = LBR_NONE;
+               return;
+       }
+
+       mask = x86_pmu.lbr_nr - 1;
+       for (i = 0; i < task_ctx->valid_lbrs; i++) {
                lbr_idx = (tos - i) & mask;
                wrlbr_from(lbr_idx, task_ctx->lbr_from[i]);
                wrlbr_to  (lbr_idx, task_ctx->lbr_to[i]);
@@ -354,14 +372,24 @@ static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
                if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
                        wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
        }
+
+       for (; i < x86_pmu.lbr_nr; i++) {
+               lbr_idx = (tos - i) & mask;
+               wrlbr_from(lbr_idx, 0);
+               wrlbr_to(lbr_idx, 0);
+               if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
+                       wrmsrl(MSR_LBR_INFO_0 + lbr_idx, 0);
+       }
+
        wrmsrl(x86_pmu.lbr_tos, tos);
        task_ctx->lbr_stack_state = LBR_NONE;
 }
 
 static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
 {
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        unsigned lbr_idx, mask;
-       u64 tos;
+       u64 tos, from;
        int i;
 
        if (task_ctx->lbr_callstack_users == 0) {
@@ -371,15 +399,22 @@ static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
 
        mask = x86_pmu.lbr_nr - 1;
        tos = intel_pmu_lbr_tos();
-       for (i = 0; i < tos; i++) {
+       for (i = 0; i < x86_pmu.lbr_nr; i++) {
                lbr_idx = (tos - i) & mask;
-               task_ctx->lbr_from[i] = rdlbr_from(lbr_idx);
+               from = rdlbr_from(lbr_idx);
+               if (!from)
+                       break;
+               task_ctx->lbr_from[i] = from;
                task_ctx->lbr_to[i]   = rdlbr_to(lbr_idx);
                if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)
                        rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]);
        }
+       task_ctx->valid_lbrs = i;
        task_ctx->tos = tos;
        task_ctx->lbr_stack_state = LBR_VALID;
+
+       cpuc->last_task_ctx = task_ctx;
+       cpuc->last_log_id = ++task_ctx->log_id;
 }
 
 void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in)
@@ -531,7 +566,7 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
  */
 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 {
-       bool need_info = false;
+       bool need_info = false, call_stack = false;
        unsigned long mask = x86_pmu.lbr_nr - 1;
        int lbr_format = x86_pmu.intel_cap.lbr_format;
        u64 tos = intel_pmu_lbr_tos();
@@ -542,7 +577,7 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
        if (cpuc->lbr_sel) {
                need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO);
                if (cpuc->lbr_sel->config & LBR_CALL_STACK)
-                       num = tos;
+                       call_stack = true;
        }
 
        for (i = 0; i < num; i++) {
@@ -555,6 +590,13 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
                from = rdlbr_from(lbr_idx);
                to   = rdlbr_to(lbr_idx);
 
+               /*
+                * Read LBR call stack entries
+                * until invalid entry (0s) is detected.
+                */
+               if (call_stack && !from)
+                       break;
+
                if (lbr_format == LBR_FORMAT_INFO && need_info) {
                        u64 info;