mm/gup_benchmark.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7
   8 #define GUP_FAST_BENCHMARK      _IOWR('g', 1, struct gup_benchmark)
   9 #define GUP_BENCHMARK           _IOWR('g', 2, struct gup_benchmark)
  10 #define PIN_FAST_BENCHMARK      _IOWR('g', 3, struct gup_benchmark)
  11 #define PIN_BENCHMARK           _IOWR('g', 4, struct gup_benchmark)
  12 #define PIN_LONGTERM_BENCHMARK  _IOWR('g', 5, struct gup_benchmark)
  13
  14 struct gup_benchmark {
  15         __u64 get_delta_usec;
  16         __u64 put_delta_usec;
  17         __u64 addr;
  18         __u64 size;
  19         __u32 nr_pages_per_call;
  20         __u32 flags;
  21         __u64 expansion[10];    /* For future use */
  22 };
  23
  24 static void put_back_pages(unsigned int cmd, struct page **pages,
  25                            unsigned long nr_pages)
  26 {
  27         unsigned long i;
  28
  29         switch (cmd) {
  30         case GUP_FAST_BENCHMARK:
  31         case GUP_BENCHMARK:
  32                 for (i = 0; i < nr_pages; i++)
  33                         put_page(pages[i]);
  34                 break;
  35
  36         case PIN_FAST_BENCHMARK:
  37         case PIN_BENCHMARK:
  38         case PIN_LONGTERM_BENCHMARK:
  39                 unpin_user_pages(pages, nr_pages);
  40                 break;
  41         }
  42 }
  43
  44 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  45                               unsigned long nr_pages)
  46 {
  47         unsigned long i;
  48         struct page *page;
  49
  50         switch (cmd) {
  51         case PIN_FAST_BENCHMARK:
  52         case PIN_BENCHMARK:
  53         case PIN_LONGTERM_BENCHMARK:
  54                 for (i = 0; i < nr_pages; i++) {
  55                         page = pages[i];
  56                         if (WARN(!page_maybe_dma_pinned(page),
  57                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  58
  59                                 dump_page(page, "gup_benchmark failure");
  60                                 break;
  61                         }
  62                 }
  63                 break;
  64         }
  65 }
  66
  67 static int __gup_benchmark_ioctl(unsigned int cmd,
  68                 struct gup_benchmark *gup)
  69 {
  70         ktime_t start_time, end_time;
  71         unsigned long i, nr_pages, addr, next;
  72         int nr;
  73         struct page **pages;
  74         int ret = 0;
  75
  76         if (gup->size > ULONG_MAX)
  77                 return -EINVAL;
  78
  79         nr_pages = gup->size / PAGE_SIZE;
  80         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
  81         if (!pages)
  82                 return -ENOMEM;
  83
  84         i = 0;
  85         nr = gup->nr_pages_per_call;
  86         start_time = ktime_get();
  87         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
  88                 if (nr != gup->nr_pages_per_call)
  89                         break;
  90
  91                 next = addr + nr * PAGE_SIZE;
  92                 if (next > gup->addr + gup->size) {
  93                         next = gup->addr + gup->size;
  94                         nr = (next - addr) / PAGE_SIZE;
  95                 }
  96
  97                 /* Filter out most gup flags: only allow a tiny subset here: */
  98                 gup->flags &= FOLL_WRITE;
  99
 100                 switch (cmd) {
 101                 case GUP_FAST_BENCHMARK:
 102                         nr = get_user_pages_fast(addr, nr, gup->flags,
 103                                                  pages + i);
 104                         break;
 105                 case GUP_BENCHMARK:
 106                         nr = get_user_pages(addr, nr, gup->flags, pages + i,
 107                                             NULL);
 108                         break;
 109                 case PIN_FAST_BENCHMARK:
 110                         nr = pin_user_pages_fast(addr, nr, gup->flags,
 111                                                  pages + i);
 112                         break;
 113                 case PIN_BENCHMARK:
 114                         nr = pin_user_pages(addr, nr, gup->flags, pages + i,
 115                                             NULL);
 116                         break;
 117                 case PIN_LONGTERM_BENCHMARK:
 118                         nr = pin_user_pages(addr, nr,
 119                                             gup->flags | FOLL_LONGTERM,
 120                                             pages + i, NULL);
 121                         break;
 122                 default:
 123                         kvfree(pages);
 124                         ret = -EINVAL;
 125                         goto out;
 126                 }
 127
 128                 if (nr <= 0)
 129                         break;
 130                 i += nr;
 131         }
 132         end_time = ktime_get();
 133
 134         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 135         nr_pages = i;
 136
 137         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 138         gup->size = addr - gup->addr;
 139
 140         /*
 141          * Take an un-benchmark-timed moment to verify DMA pinned
 142          * state: print a warning if any non-dma-pinned pages are found:
 143          */
 144         verify_dma_pinned(cmd, pages, nr_pages);
 145
 146         start_time = ktime_get();
 147
 148         put_back_pages(cmd, pages, nr_pages);
 149
 150         end_time = ktime_get();
 151         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 152
 153         kvfree(pages);
 154 out:
 155         return ret;
 156 }
 157
 158 static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd,
 159                 unsigned long arg)
 160 {
 161         struct gup_benchmark gup;
 162         int ret;
 163
 164         switch (cmd) {
 165         case GUP_FAST_BENCHMARK:
 166         case GUP_BENCHMARK:
 167         case PIN_FAST_BENCHMARK:
 168         case PIN_BENCHMARK:
 169         case PIN_LONGTERM_BENCHMARK:
 170                 break;
 171         default:
 172                 return -EINVAL;
 173         }
 174
 175         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 176                 return -EFAULT;
 177
 178         ret = __gup_benchmark_ioctl(cmd, &gup);
 179         if (ret)
 180                 return ret;
 181
 182         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 183                 return -EFAULT;
 184
 185         return 0;
 186 }
 187
 188 static const struct file_operations gup_benchmark_fops = {
 189         .open = nonseekable_open,
 190         .unlocked_ioctl = gup_benchmark_ioctl,
 191 };
 192
 193 static int gup_benchmark_init(void)
 194 {
 195         debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL,
 196                                    &gup_benchmark_fops);
 197
 198         return 0;
 199 }
 200
 201 late_initcall(gup_benchmark_init);