arch/powerpc/platforms/pseries/dtl.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Virtual Processor Dispatch Trace Log
   4  *
   5  * (C) Copyright IBM Corporation 2009
   6  *
   7  * Author: Jeremy Kerr <jk@ozlabs.org>
   8  */
   9
  10 #include <linux/slab.h>
  11 #include <linux/spinlock.h>
  12 #include <asm/smp.h>
  13 #include <linux/uaccess.h>
  14 #include <linux/debugfs.h>
  15 #include <asm/firmware.h>
  16 #include <asm/dtl.h>
  17 #include <asm/lppaca.h>
  18 #include <asm/plpar_wrappers.h>
  19 #include <asm/machdep.h>
  20
  21 #ifdef CONFIG_DTL
  22 struct dtl {
  23         struct dtl_entry        *buf;
  24         int                     cpu;
  25         int                     buf_entries;
  26         u64                     last_idx;
  27         spinlock_t              lock;
  28 };
  29 static DEFINE_PER_CPU(struct dtl, cpu_dtl);
  30
  31 static u8 dtl_event_mask = DTL_LOG_ALL;
  32
  33
  34 /*
  35  * Size of per-cpu log buffers. Firmware requires that the buffer does
  36  * not cross a 4k boundary.
  37  */
  38 static int dtl_buf_entries = N_DISPATCH_LOG;
  39
  40 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
  41
  42 /*
  43  * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls
  44  * reading from the dispatch trace log.  If other code wants to consume
  45  * DTL entries, it can set this pointer to a function that will get
  46  * called once for each DTL entry that gets processed.
  47  */
  48 static void (*dtl_consumer)(struct dtl_entry *entry, u64 index);
  49
  50 struct dtl_ring {
  51         u64     write_index;
  52         struct dtl_entry *write_ptr;
  53         struct dtl_entry *buf;
  54         struct dtl_entry *buf_end;
  55 };
  56
  57 static DEFINE_PER_CPU(struct dtl_ring, dtl_rings);
  58
  59 static atomic_t dtl_count;
  60
  61 /*
  62  * The cpu accounting code controls the DTL ring buffer, and we get
  63  * given entries as they are processed.
  64  */
  65 static void consume_dtle(struct dtl_entry *dtle, u64 index)
  66 {
  67         struct dtl_ring *dtlr = this_cpu_ptr(&dtl_rings);
  68         struct dtl_entry *wp = dtlr->write_ptr;
  69         struct lppaca *vpa = local_paca->lppaca_ptr;
  70
  71         if (!wp)
  72                 return;
  73
  74         *wp = *dtle;
  75         barrier();
  76
  77         /* check for hypervisor ring buffer overflow, ignore this entry if so */
  78         if (index + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx))
  79                 return;
  80
  81         ++wp;
  82         if (wp == dtlr->buf_end)
  83                 wp = dtlr->buf;
  84         dtlr->write_ptr = wp;
  85
  86         /* incrementing write_index makes the new entry visible */
  87         smp_wmb();
  88         ++dtlr->write_index;
  89 }
  90
  91 static int dtl_start(struct dtl *dtl)
  92 {
  93         struct dtl_ring *dtlr = &per_cpu(dtl_rings, dtl->cpu);
  94
  95         dtlr->buf = dtl->buf;
  96         dtlr->buf_end = dtl->buf + dtl->buf_entries;
  97         dtlr->write_index = 0;
  98
  99         /* setting write_ptr enables logging into our buffer */
 100         smp_wmb();
 101         dtlr->write_ptr = dtl->buf;
 102
 103         /* enable event logging */
 104         lppaca_of(dtl->cpu).dtl_enable_mask |= dtl_event_mask;
 105
 106         dtl_consumer = consume_dtle;
 107         atomic_inc(&dtl_count);
 108         return 0;
 109 }
 110
 111 static void dtl_stop(struct dtl *dtl)
 112 {
 113         struct dtl_ring *dtlr = &per_cpu(dtl_rings, dtl->cpu);
 114
 115         dtlr->write_ptr = NULL;
 116         smp_wmb();
 117
 118         dtlr->buf = NULL;
 119
 120         /* restore dtl_enable_mask */
 121         lppaca_of(dtl->cpu).dtl_enable_mask = DTL_LOG_PREEMPT;
 122
 123         if (atomic_dec_and_test(&dtl_count))
 124                 dtl_consumer = NULL;
 125 }
 126
 127 static u64 dtl_current_index(struct dtl *dtl)
 128 {
 129         return per_cpu(dtl_rings, dtl->cpu).write_index;
 130 }
 131
 132 #else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
 133
 134 static int dtl_start(struct dtl *dtl)
 135 {
 136         unsigned long addr;
 137         int ret, hwcpu;
 138
 139         /* Register our dtl buffer with the hypervisor. The HV expects the
 140          * buffer size to be passed in the second word of the buffer */
 141         ((u32 *)dtl->buf)[1] = cpu_to_be32(DISPATCH_LOG_BYTES);
 142
 143         hwcpu = get_hard_smp_processor_id(dtl->cpu);
 144         addr = __pa(dtl->buf);
 145         ret = register_dtl(hwcpu, addr);
 146         if (ret) {
 147                 printk(KERN_WARNING "%s: DTL registration for cpu %d (hw %d) "
 148                        "failed with %d\n", __func__, dtl->cpu, hwcpu, ret);
 149                 return -EIO;
 150         }
 151
 152         /* set our initial buffer indices */
 153         lppaca_of(dtl->cpu).dtl_idx = 0;
 154
 155         /* ensure that our updates to the lppaca fields have occurred before
 156          * we actually enable the logging */
 157         smp_wmb();
 158
 159         /* enable event logging */
 160         lppaca_of(dtl->cpu).dtl_enable_mask = dtl_event_mask;
 161
 162         return 0;
 163 }
 164
 165 static void dtl_stop(struct dtl *dtl)
 166 {
 167         int hwcpu = get_hard_smp_processor_id(dtl->cpu);
 168
 169         lppaca_of(dtl->cpu).dtl_enable_mask = 0x0;
 170
 171         unregister_dtl(hwcpu);
 172 }
 173
 174 static u64 dtl_current_index(struct dtl *dtl)
 175 {
 176         return be64_to_cpu(lppaca_of(dtl->cpu).dtl_idx);
 177 }
 178 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
 179
 180 static int dtl_enable(struct dtl *dtl)
 181 {
 182         long int n_entries;
 183         long int rc;
 184         struct dtl_entry *buf = NULL;
 185
 186         if (!dtl_cache)
 187                 return -ENOMEM;
 188
 189         /* only allow one reader */
 190         if (dtl->buf)
 191                 return -EBUSY;
 192
 193         /* ensure there are no other conflicting dtl users */
 194         if (!read_trylock(&dtl_access_lock))
 195                 return -EBUSY;
 196
 197         n_entries = dtl_buf_entries;
 198         buf = kmem_cache_alloc_node(dtl_cache, GFP_KERNEL, cpu_to_node(dtl->cpu));
 199         if (!buf) {
 200                 printk(KERN_WARNING "%s: buffer alloc failed for cpu %d\n",
 201                                 __func__, dtl->cpu);
 202                 read_unlock(&dtl_access_lock);
 203                 return -ENOMEM;
 204         }
 205
 206         spin_lock(&dtl->lock);
 207         rc = -EBUSY;
 208         if (!dtl->buf) {
 209                 /* store the original allocation size for use during read */
 210                 dtl->buf_entries = n_entries;
 211                 dtl->buf = buf;
 212                 dtl->last_idx = 0;
 213                 rc = dtl_start(dtl);
 214                 if (rc)
 215                         dtl->buf = NULL;
 216         }
 217         spin_unlock(&dtl->lock);
 218
 219         if (rc) {
 220                 read_unlock(&dtl_access_lock);
 221                 kmem_cache_free(dtl_cache, buf);
 222         }
 223
 224         return rc;
 225 }
 226
 227 static void dtl_disable(struct dtl *dtl)
 228 {
 229         spin_lock(&dtl->lock);
 230         dtl_stop(dtl);
 231         kmem_cache_free(dtl_cache, dtl->buf);
 232         dtl->buf = NULL;
 233         dtl->buf_entries = 0;
 234         spin_unlock(&dtl->lock);
 235         read_unlock(&dtl_access_lock);
 236 }
 237
 238 /* file interface */
 239
 240 static int dtl_file_open(struct inode *inode, struct file *filp)
 241 {
 242         struct dtl *dtl = inode->i_private;
 243         int rc;
 244
 245         rc = dtl_enable(dtl);
 246         if (rc)
 247                 return rc;
 248
 249         filp->private_data = dtl;
 250         return 0;
 251 }
 252
 253 static int dtl_file_release(struct inode *inode, struct file *filp)
 254 {
 255         struct dtl *dtl = inode->i_private;
 256         dtl_disable(dtl);
 257         return 0;
 258 }
 259
 260 static ssize_t dtl_file_read(struct file *filp, char __user *buf, size_t len,
 261                 loff_t *pos)
 262 {
 263         long int rc, n_read, n_req, read_size;
 264         struct dtl *dtl;
 265         u64 cur_idx, last_idx, i;
 266
 267         if ((len % sizeof(struct dtl_entry)) != 0)
 268                 return -EINVAL;
 269
 270         dtl = filp->private_data;
 271
 272         /* requested number of entries to read */
 273         n_req = len / sizeof(struct dtl_entry);
 274
 275         /* actual number of entries read */
 276         n_read = 0;
 277
 278         spin_lock(&dtl->lock);
 279
 280         cur_idx = dtl_current_index(dtl);
 281         last_idx = dtl->last_idx;
 282
 283         if (last_idx + dtl->buf_entries <= cur_idx)
 284                 last_idx = cur_idx - dtl->buf_entries + 1;
 285
 286         if (last_idx + n_req > cur_idx)
 287                 n_req = cur_idx - last_idx;
 288
 289         if (n_req > 0)
 290                 dtl->last_idx = last_idx + n_req;
 291
 292         spin_unlock(&dtl->lock);
 293
 294         if (n_req <= 0)
 295                 return 0;
 296
 297         i = last_idx % dtl->buf_entries;
 298
 299         /* read the tail of the buffer if we've wrapped */
 300         if (i + n_req > dtl->buf_entries) {
 301                 read_size = dtl->buf_entries - i;
 302
 303                 rc = copy_to_user(buf, &dtl->buf[i],
 304                                 read_size * sizeof(struct dtl_entry));
 305                 if (rc)
 306                         return -EFAULT;
 307
 308                 i = 0;
 309                 n_req -= read_size;
 310                 n_read += read_size;
 311                 buf += read_size * sizeof(struct dtl_entry);
 312         }
 313
 314         /* .. and now the head */
 315         rc = copy_to_user(buf, &dtl->buf[i], n_req * sizeof(struct dtl_entry));
 316         if (rc)
 317                 return -EFAULT;
 318
 319         n_read += n_req;
 320
 321         return n_read * sizeof(struct dtl_entry);
 322 }
 323
 324 static const struct file_operations dtl_fops = {
 325         .open           = dtl_file_open,
 326         .release        = dtl_file_release,
 327         .read           = dtl_file_read,
 328         .llseek         = no_llseek,
 329 };
 330
 331 static struct dentry *dtl_dir;
 332
 333 static void dtl_setup_file(struct dtl *dtl)
 334 {
 335         char name[10];
 336
 337         sprintf(name, "cpu-%d", dtl->cpu);
 338
 339         debugfs_create_file(name, 0400, dtl_dir, dtl, &dtl_fops);
 340 }
 341
 342 static int dtl_init(void)
 343 {
 344         int i;
 345
 346         if (!firmware_has_feature(FW_FEATURE_SPLPAR))
 347                 return -ENODEV;
 348
 349         /* set up common debugfs structure */
 350
 351         dtl_dir = debugfs_create_dir("dtl", arch_debugfs_dir);
 352
 353         debugfs_create_x8("dtl_event_mask", 0600, dtl_dir, &dtl_event_mask);
 354         debugfs_create_u32("dtl_buf_entries", 0400, dtl_dir, &dtl_buf_entries);
 355
 356         /* set up the per-cpu log structures */
 357         for_each_possible_cpu(i) {
 358                 struct dtl *dtl = &per_cpu(cpu_dtl, i);
 359                 spin_lock_init(&dtl->lock);
 360                 dtl->cpu = i;
 361
 362                 dtl_setup_file(dtl);
 363         }
 364
 365         return 0;
 366 }
 367 machine_arch_initcall(pseries, dtl_init);
 368 #endif /* CONFIG_DTL */
 369
 370 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 371 /*
 372  * Scan the dispatch trace log and count up the stolen time.
 373  * Should be called with interrupts disabled.
 374  */
 375 static notrace u64 scan_dispatch_log(u64 stop_tb)
 376 {
 377         u64 i = local_paca->dtl_ridx;
 378         struct dtl_entry *dtl = local_paca->dtl_curr;
 379         struct dtl_entry *dtl_end = local_paca->dispatch_log_end;
 380         struct lppaca *vpa = local_paca->lppaca_ptr;
 381         u64 tb_delta;
 382         u64 stolen = 0;
 383         u64 dtb;
 384
 385         if (!dtl)
 386                 return 0;
 387
 388         if (i == be64_to_cpu(vpa->dtl_idx))
 389                 return 0;
 390         while (i < be64_to_cpu(vpa->dtl_idx)) {
 391                 dtb = be64_to_cpu(dtl->timebase);
 392                 tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) +
 393                         be32_to_cpu(dtl->ready_to_enqueue_time);
 394                 barrier();
 395                 if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) {
 396                         /* buffer has overflowed */
 397                         i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG;
 398                         dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG);
 399                         continue;
 400                 }
 401                 if (dtb > stop_tb)
 402                         break;
 403 #ifdef CONFIG_DTL
 404                 if (dtl_consumer)
 405                         dtl_consumer(dtl, i);
 406 #endif
 407                 stolen += tb_delta;
 408                 ++i;
 409                 ++dtl;
 410                 if (dtl == dtl_end)
 411                         dtl = local_paca->dispatch_log;
 412         }
 413         local_paca->dtl_ridx = i;
 414         local_paca->dtl_curr = dtl;
 415         return stolen;
 416 }
 417
 418 /*
 419  * Accumulate stolen time by scanning the dispatch trace log.
 420  * Called on entry from user mode.
 421  */
 422 void notrace pseries_accumulate_stolen_time(void)
 423 {
 424         u64 sst, ust;
 425         struct cpu_accounting_data *acct = &local_paca->accounting;
 426
 427         sst = scan_dispatch_log(acct->starttime_user);
 428         ust = scan_dispatch_log(acct->starttime);
 429         acct->stime -= sst;
 430         acct->utime -= ust;
 431         acct->steal_time += ust + sst;
 432 }
 433
 434 u64 pseries_calculate_stolen_time(u64 stop_tb)
 435 {
 436         if (!firmware_has_feature(FW_FEATURE_SPLPAR))
 437                 return 0;
 438
 439         if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx))
 440                 return scan_dispatch_log(stop_tb);
 441
 442         return 0;
 443 }
 444
 445 #endif