1 // SPDX-License-Identifier: GPL-2.0
3 * Tests x86 Memory Protection Keys (see Documentation/x86/protection-keys.txt)
5 * There are examples in here of:
6 * * how to set protection keys on memory
7 * * how to set/clear bits in PKRU (the rights register)
8 * * how to handle SEGV_PKRU signals and extract pkey-relevant
9 * information from the siginfo
12 * make sure KSM and KSM COW breaking works
13 * prefault pages in at malloc, or not
14 * protect MPX bounds tables with protection keys?
15 * make sure VMA splitting/merging is working correctly
16 * OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys
17 * look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel
18 * do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks
21 * gcc -o protection_keys -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
22 * gcc -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
26 #include <linux/futex.h>
28 #include <sys/syscall.h>
38 #include <sys/types.h>
43 #include <sys/ptrace.h>
46 #include "pkey-helpers.h"
51 unsigned int shadow_pkru;
53 #define HPAGE_SIZE (1UL<<21)
54 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x)))
55 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
56 #define ALIGN_DOWN(x, align_to) ((x) & ~((align_to)-1))
57 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
58 #define ALIGN_PTR_DOWN(p, ptr_align_to) ((typeof(p))ALIGN_DOWN((unsigned long)(p), ptr_align_to))
59 #define __stringify_1(x...) #x
60 #define __stringify(x...) __stringify_1(x)
62 #define PTR_ERR_ENOTSUP ((void *)-ENOTSUP)
65 char dprint_in_signal_buffer[DPRINT_IN_SIGNAL_BUF_SIZE];
67 extern void abort_hooks(void);
68 #define pkey_assert(condition) do { \
70 dprintf0("assert() at %s::%d test_nr: %d iteration: %d\n", \
72 test_nr, iteration_nr); \
73 dprintf0("errno at assert: %d", errno); \
79 void cat_into_file(char *str, char *file)
81 int fd = open(file, O_RDWR);
84 dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file);
86 * these need to be raw because they are called under
90 fprintf(stderr, "error opening '%s'\n", str);
95 ret = write(fd, str, strlen(str));
96 if (ret != strlen(str)) {
97 perror("write to file failed");
98 fprintf(stderr, "filename: '%s' str: '%s'\n", file, str);
104 #if CONTROL_TRACING > 0
105 static int warned_tracing;
106 int tracing_root_ok(void)
108 if (geteuid() != 0) {
110 fprintf(stderr, "WARNING: not run as root, "
111 "can not do tracing control\n");
119 void tracing_on(void)
121 #if CONTROL_TRACING > 0
122 #define TRACEDIR "/sys/kernel/debug/tracing"
125 if (!tracing_root_ok())
128 sprintf(pidstr, "%d", getpid());
129 cat_into_file("0", TRACEDIR "/tracing_on");
130 cat_into_file("\n", TRACEDIR "/trace");
132 cat_into_file("function_graph", TRACEDIR "/current_tracer");
133 cat_into_file("1", TRACEDIR "/options/funcgraph-proc");
135 cat_into_file("nop", TRACEDIR "/current_tracer");
137 cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid");
138 cat_into_file("1", TRACEDIR "/tracing_on");
139 dprintf1("enabled tracing\n");
143 void tracing_off(void)
145 #if CONTROL_TRACING > 0
146 if (!tracing_root_ok())
148 cat_into_file("0", "/sys/kernel/debug/tracing/tracing_on");
152 void abort_hooks(void)
154 fprintf(stderr, "running %s()...\n", __func__);
156 #ifdef SLEEP_ON_ABORT
157 sleep(SLEEP_ON_ABORT);
161 static inline void __page_o_noops(void)
163 /* 8-bytes of instruction * 512 bytes = 1 page */
164 asm(".rept 512 ; nopl 0x7eeeeeee(%eax) ; .endr");
168 * This attempts to have roughly a page of instructions followed by a few
169 * instructions that do a write, and another page of instructions. That
170 * way, we are pretty sure that the write is in the second page of
171 * instructions and has at least a page of padding behind it.
173 * *That* lets us be sure to madvise() away the write instruction, which
174 * will then fault, which makes sure that the fault code handles
175 * execute-only memory properly.
177 __attribute__((__aligned__(PAGE_SIZE)))
178 void lots_o_noops_around_write(int *write_to_me)
180 dprintf3("running %s()\n", __func__);
182 /* Assume this happens in the second page of instructions: */
183 *write_to_me = __LINE__;
184 /* pad out by another page: */
186 dprintf3("%s() done\n", __func__);
189 /* Define some kernel-like types */
197 #ifndef SYS_mprotect_key
198 # define SYS_mprotect_key 380
201 #ifndef SYS_pkey_alloc
202 # define SYS_pkey_alloc 381
203 # define SYS_pkey_free 382
206 #define REG_IP_IDX REG_EIP
207 #define si_pkey_offset 0x14
211 #ifndef SYS_mprotect_key
212 # define SYS_mprotect_key 329
215 #ifndef SYS_pkey_alloc
216 # define SYS_pkey_alloc 330
217 # define SYS_pkey_free 331
220 #define REG_IP_IDX REG_RIP
221 #define si_pkey_offset 0x20
225 void dump_mem(void *dumpme, int len_bytes)
227 char *c = (void *)dumpme;
230 for (i = 0; i < len_bytes; i += sizeof(u64)) {
231 u64 *ptr = (u64 *)(c + i);
232 dprintf1("dump[%03d][@%p]: %016jx\n", i, ptr, *ptr);
236 /* Failed address bound checks: */
238 # define SEGV_BNDERR 3
242 # define SEGV_PKUERR 4
245 static char *si_code_str(int si_code)
247 if (si_code == SEGV_MAPERR)
248 return "SEGV_MAPERR";
249 if (si_code == SEGV_ACCERR)
250 return "SEGV_ACCERR";
251 if (si_code == SEGV_BNDERR)
252 return "SEGV_BNDERR";
253 if (si_code == SEGV_PKUERR)
254 return "SEGV_PKUERR";
259 int last_si_pkey = -1;
260 void signal_handler(int signum, siginfo_t *si, void *vucontext)
262 ucontext_t *uctxt = vucontext;
272 dprint_in_signal = 1;
273 dprintf1(">>>>===============SIGSEGV============================\n");
274 dprintf1("%s()::%d, pkru: 0x%x shadow: %x\n", __func__, __LINE__,
275 __rdpkru(), shadow_pkru);
277 trapno = uctxt->uc_mcontext.gregs[REG_TRAPNO];
278 ip = uctxt->uc_mcontext.gregs[REG_IP_IDX];
279 fpregset = uctxt->uc_mcontext.fpregs;
280 fpregs = (void *)fpregset;
282 dprintf2("%s() trapno: %d ip: 0x%lx info->si_code: %s/%d\n", __func__,
283 trapno, ip, si_code_str(si->si_code), si->si_code);
286 * 32-bit has some extra padding so that userspace can tell whether
287 * the XSTATE header is present in addition to the "legacy" FPU
288 * state. We just assume that it is here.
292 pkru_offset = pkru_xstate_offset();
293 pkru_ptr = (void *)(&fpregs[pkru_offset]);
295 dprintf1("siginfo: %p\n", si);
296 dprintf1(" fpregs: %p\n", fpregs);
298 * If we got a PKRU fault, we *HAVE* to have at least one bit set in
301 dprintf1("pkru_xstate_offset: %d\n", pkru_xstate_offset());
303 dump_mem(pkru_ptr - 128, 256);
304 pkey_assert(*pkru_ptr);
306 if ((si->si_code == SEGV_MAPERR) ||
307 (si->si_code == SEGV_ACCERR) ||
308 (si->si_code == SEGV_BNDERR)) {
309 printf("non-PK si_code, exiting...\n");
313 si_pkey_ptr = (u32 *)(((u8 *)si) + si_pkey_offset);
314 dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr);
315 dump_mem((u8 *)si_pkey_ptr - 8, 24);
316 siginfo_pkey = *si_pkey_ptr;
317 pkey_assert(siginfo_pkey < NR_PKEYS);
318 last_si_pkey = siginfo_pkey;
320 dprintf1("signal pkru from xsave: %08x\n", *pkru_ptr);
321 /* need __rdpkru() version so we do not do shadow_pkru checking */
322 dprintf1("signal pkru from pkru: %08x\n", __rdpkru());
323 dprintf1("pkey from siginfo: %jx\n", siginfo_pkey);
324 *(u64 *)pkru_ptr = 0x00000000;
325 dprintf1("WARNING: set PRKU=0 to allow faulting instruction to continue\n");
327 dprintf1("<<<<==================================================\n");
328 dprint_in_signal = 0;
331 int wait_all_children(void)
334 return waitpid(-1, &status, 0);
339 dprint_in_signal = 1;
340 dprintf2("[%d] SIGCHLD: %d\n", getpid(), x);
341 dprint_in_signal = 0;
344 void setup_sigsegv_handler(void)
347 struct sigaction newact;
348 struct sigaction oldact;
350 /* #PF is mapped to sigsegv */
351 int signum = SIGSEGV;
353 newact.sa_handler = 0;
354 newact.sa_sigaction = signal_handler;
356 /*sigset_t - signals to block while in the handler */
357 /* get the old signal mask. */
358 rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask);
359 pkey_assert(rs == 0);
361 /* call sa_sigaction, not sa_handler*/
362 newact.sa_flags = SA_SIGINFO;
364 newact.sa_restorer = 0; /* void(*)(), obsolete */
365 r = sigaction(signum, &newact, &oldact);
366 r = sigaction(SIGALRM, &newact, &oldact);
370 void setup_handlers(void)
372 signal(SIGCHLD, &sig_chld);
373 setup_sigsegv_handler();
376 pid_t fork_lazy_child(void)
381 pkey_assert(forkret >= 0);
382 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
387 dprintf1("child sleeping...\n");
394 #ifndef PKEY_DISABLE_ACCESS
395 # define PKEY_DISABLE_ACCESS 0x1
398 #ifndef PKEY_DISABLE_WRITE
399 # define PKEY_DISABLE_WRITE 0x2
402 static u32 hw_pkey_get(int pkey, unsigned long flags)
404 u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE);
405 u32 pkru = __rdpkru();
409 dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n",
410 __func__, pkey, flags, 0, 0);
411 dprintf2("%s() raw pkru: %x\n", __func__, pkru);
413 shifted_pkru = (pkru >> (pkey * PKRU_BITS_PER_PKEY));
414 dprintf2("%s() shifted_pkru: %x\n", __func__, shifted_pkru);
415 masked_pkru = shifted_pkru & mask;
416 dprintf2("%s() masked pkru: %x\n", __func__, masked_pkru);
418 * shift down the relevant bits to the lowest two, then
419 * mask off all the other high bits.
424 static int hw_pkey_set(int pkey, unsigned long rights, unsigned long flags)
426 u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE);
427 u32 old_pkru = __rdpkru();
430 /* make sure that 'rights' only contains the bits we expect: */
431 assert(!(rights & ~mask));
435 /* mask out bits from pkey in old value: */
436 new_pkru &= ~(mask << (pkey * PKRU_BITS_PER_PKEY));
437 /* OR in new bits for pkey: */
438 new_pkru |= (rights << (pkey * PKRU_BITS_PER_PKEY));
442 dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x pkru now: %x old_pkru: %x\n",
443 __func__, pkey, rights, flags, 0, __rdpkru(), old_pkru);
447 void pkey_disable_set(int pkey, int flags)
449 unsigned long syscall_flags = 0;
452 u32 orig_pkru = rdpkru();
454 dprintf1("START->%s(%d, 0x%x)\n", __func__,
456 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
458 pkey_rights = hw_pkey_get(pkey, syscall_flags);
460 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
461 pkey, pkey, pkey_rights);
462 pkey_assert(pkey_rights >= 0);
464 pkey_rights |= flags;
466 ret = hw_pkey_set(pkey, pkey_rights, syscall_flags);
468 /*pkru and flags have the same format */
469 shadow_pkru |= flags << (pkey * 2);
470 dprintf1("%s(%d) shadow: 0x%x\n", __func__, pkey, shadow_pkru);
472 pkey_assert(ret >= 0);
474 pkey_rights = hw_pkey_get(pkey, syscall_flags);
475 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
476 pkey, pkey, pkey_rights);
478 dprintf1("%s(%d) pkru: 0x%x\n", __func__, pkey, rdpkru());
480 pkey_assert(rdpkru() > orig_pkru);
481 dprintf1("END<---%s(%d, 0x%x)\n", __func__,
485 void pkey_disable_clear(int pkey, int flags)
487 unsigned long syscall_flags = 0;
489 int pkey_rights = hw_pkey_get(pkey, syscall_flags);
490 u32 orig_pkru = rdpkru();
492 pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
494 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
495 pkey, pkey, pkey_rights);
496 pkey_assert(pkey_rights >= 0);
498 pkey_rights |= flags;
500 ret = hw_pkey_set(pkey, pkey_rights, 0);
501 /* pkru and flags have the same format */
502 shadow_pkru &= ~(flags << (pkey * 2));
503 pkey_assert(ret >= 0);
505 pkey_rights = hw_pkey_get(pkey, syscall_flags);
506 dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
507 pkey, pkey, pkey_rights);
509 dprintf1("%s(%d) pkru: 0x%x\n", __func__, pkey, rdpkru());
511 assert(rdpkru() > orig_pkru);
514 void pkey_write_allow(int pkey)
516 pkey_disable_clear(pkey, PKEY_DISABLE_WRITE);
518 void pkey_write_deny(int pkey)
520 pkey_disable_set(pkey, PKEY_DISABLE_WRITE);
522 void pkey_access_allow(int pkey)
524 pkey_disable_clear(pkey, PKEY_DISABLE_ACCESS);
526 void pkey_access_deny(int pkey)
528 pkey_disable_set(pkey, PKEY_DISABLE_ACCESS);
531 int sys_mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
536 dprintf2("%s(0x%p, %zx, prot=%lx, pkey=%lx)\n", __func__,
537 ptr, size, orig_prot, pkey);
540 sret = syscall(SYS_mprotect_key, ptr, size, orig_prot, pkey);
542 dprintf2("SYS_mprotect_key sret: %d\n", sret);
543 dprintf2("SYS_mprotect_key prot: 0x%lx\n", orig_prot);
544 dprintf2("SYS_mprotect_key failed, errno: %d\n", errno);
545 if (DEBUG_LEVEL >= 2)
546 perror("SYS_mprotect_pkey");
551 int sys_pkey_alloc(unsigned long flags, unsigned long init_val)
553 int ret = syscall(SYS_pkey_alloc, flags, init_val);
554 dprintf1("%s(flags=%lx, init_val=%lx) syscall ret: %d errno: %d\n",
555 __func__, flags, init_val, ret, errno);
562 unsigned long init_val = 0x0;
564 dprintf1("alloc_pkey()::%d, pkru: 0x%x shadow: %x\n",
565 __LINE__, __rdpkru(), shadow_pkru);
566 ret = sys_pkey_alloc(0, init_val);
568 * pkey_alloc() sets PKRU, so we need to reflect it in
571 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
572 __LINE__, ret, __rdpkru(), shadow_pkru);
574 /* clear both the bits: */
575 shadow_pkru &= ~(0x3 << (ret * 2));
576 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
577 __LINE__, ret, __rdpkru(), shadow_pkru);
579 * move the new state in from init_val
580 * (remember, we cheated and init_val == pkru format)
582 shadow_pkru |= (init_val << (ret * 2));
584 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
585 __LINE__, ret, __rdpkru(), shadow_pkru);
586 dprintf1("alloc_pkey()::%d errno: %d\n", __LINE__, errno);
587 /* for shadow checking: */
589 dprintf4("alloc_pkey()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n",
590 __LINE__, ret, __rdpkru(), shadow_pkru);
594 int sys_pkey_free(unsigned long pkey)
596 int ret = syscall(SYS_pkey_free, pkey);
597 dprintf1("%s(pkey=%ld) syscall ret: %d\n", __func__, pkey, ret);
602 * I had a bug where pkey bits could be set by mprotect() but
603 * not cleared. This ensures we get lots of random bit sets
604 * and clears on the vma and pte pkey bits.
606 int alloc_random_pkey(void)
608 int max_nr_pkey_allocs;
611 int alloced_pkeys[NR_PKEYS];
614 memset(alloced_pkeys, 0, sizeof(alloced_pkeys));
616 /* allocate every possible key and make a note of which ones we got */
617 max_nr_pkey_allocs = NR_PKEYS;
618 max_nr_pkey_allocs = 1;
619 for (i = 0; i < max_nr_pkey_allocs; i++) {
620 int new_pkey = alloc_pkey();
623 alloced_pkeys[nr_alloced++] = new_pkey;
626 pkey_assert(nr_alloced > 0);
627 /* select a random one out of the allocated ones */
628 random_index = rand() % nr_alloced;
629 ret = alloced_pkeys[random_index];
630 /* now zero it out so we don't free it next */
631 alloced_pkeys[random_index] = 0;
633 /* go through the allocated ones that we did not want and free them */
634 for (i = 0; i < nr_alloced; i++) {
636 if (!alloced_pkeys[i])
638 free_ret = sys_pkey_free(alloced_pkeys[i]);
639 pkey_assert(!free_ret);
641 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
642 __LINE__, ret, __rdpkru(), shadow_pkru);
646 int mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
649 int nr_iterations = random() % 100;
653 int rpkey = alloc_random_pkey();
654 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
655 dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
656 ptr, size, orig_prot, pkey, ret);
657 if (nr_iterations-- < 0)
660 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
661 __LINE__, ret, __rdpkru(), shadow_pkru);
662 sys_pkey_free(rpkey);
663 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
664 __LINE__, ret, __rdpkru(), shadow_pkru);
666 pkey_assert(pkey < NR_PKEYS);
668 ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
669 dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
670 ptr, size, orig_prot, pkey, ret);
672 dprintf1("%s()::%d, ret: %d pkru: 0x%x shadow: 0x%x\n", __func__,
673 __LINE__, ret, __rdpkru(), shadow_pkru);
677 struct pkey_malloc_record {
682 struct pkey_malloc_record *pkey_malloc_records;
683 struct pkey_malloc_record *pkey_last_malloc_record;
684 long nr_pkey_malloc_records;
685 void record_pkey_malloc(void *ptr, long size, int prot)
688 struct pkey_malloc_record *rec = NULL;
690 for (i = 0; i < nr_pkey_malloc_records; i++) {
691 rec = &pkey_malloc_records[i];
692 /* find a free record */
697 /* every record is full */
698 size_t old_nr_records = nr_pkey_malloc_records;
699 size_t new_nr_records = (nr_pkey_malloc_records * 2 + 1);
700 size_t new_size = new_nr_records * sizeof(struct pkey_malloc_record);
701 dprintf2("new_nr_records: %zd\n", new_nr_records);
702 dprintf2("new_size: %zd\n", new_size);
703 pkey_malloc_records = realloc(pkey_malloc_records, new_size);
704 pkey_assert(pkey_malloc_records != NULL);
705 rec = &pkey_malloc_records[nr_pkey_malloc_records];
707 * realloc() does not initialize memory, so zero it from
708 * the first new record all the way to the end.
710 for (i = 0; i < new_nr_records - old_nr_records; i++)
711 memset(rec + i, 0, sizeof(*rec));
713 dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n",
714 (int)(rec - pkey_malloc_records), rec, ptr, size);
718 pkey_last_malloc_record = rec;
719 nr_pkey_malloc_records++;
722 void free_pkey_malloc(void *ptr)
726 dprintf3("%s(%p)\n", __func__, ptr);
727 for (i = 0; i < nr_pkey_malloc_records; i++) {
728 struct pkey_malloc_record *rec = &pkey_malloc_records[i];
729 dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n",
730 ptr, i, rec, rec->ptr, rec->size);
731 if ((ptr < rec->ptr) ||
732 (ptr >= rec->ptr + rec->size))
735 dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n",
736 ptr, i, rec, rec->ptr, rec->size);
737 nr_pkey_malloc_records--;
738 ret = munmap(rec->ptr, rec->size);
739 dprintf3("munmap ret: %d\n", ret);
741 dprintf3("clearing rec->ptr, rec: %p\n", rec);
743 dprintf3("done clearing rec->ptr, rec: %p\n", rec);
750 void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey)
756 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
758 pkey_assert(pkey < NR_PKEYS);
759 ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
760 pkey_assert(ptr != (void *)-1);
761 ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey);
763 record_pkey_malloc(ptr, size, prot);
766 dprintf1("%s() for pkey %d @ %p\n", __func__, pkey, ptr);
770 void *malloc_pkey_anon_huge(long size, int prot, u16 pkey)
775 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
778 * Guarantee we can fit at least one huge page in the resulting
779 * allocation by allocating space for 2:
781 size = ALIGN_UP(size, HPAGE_SIZE * 2);
782 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
783 pkey_assert(ptr != (void *)-1);
784 record_pkey_malloc(ptr, size, prot);
785 mprotect_pkey(ptr, size, prot, pkey);
787 dprintf1("unaligned ptr: %p\n", ptr);
788 ptr = ALIGN_PTR_UP(ptr, HPAGE_SIZE);
789 dprintf1(" aligned ptr: %p\n", ptr);
790 ret = madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE);
791 dprintf1("MADV_HUGEPAGE ret: %d\n", ret);
792 ret = madvise(ptr, HPAGE_SIZE, MADV_WILLNEED);
793 dprintf1("MADV_WILLNEED ret: %d\n", ret);
794 memset(ptr, 0, HPAGE_SIZE);
796 dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey, ptr);
800 int hugetlb_setup_ok;
801 #define GET_NR_HUGE_PAGES 10
802 void setup_hugetlbfs(void)
808 if (geteuid() != 0) {
809 fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n");
813 cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages");
816 * Now go make sure that we got the pages and that they
817 * are 2M pages. Someone might have made 1G the default.
819 fd = open("/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages", O_RDONLY);
821 perror("opening sysfs 2M hugetlb config");
825 /* -1 to guarantee leaving the trailing \0 */
826 err = read(fd, buf, sizeof(buf)-1);
829 perror("reading sysfs 2M hugetlb config");
833 if (atoi(buf) != GET_NR_HUGE_PAGES) {
834 fprintf(stderr, "could not confirm 2M pages, got: '%s' expected %d\n",
835 buf, GET_NR_HUGE_PAGES);
839 hugetlb_setup_ok = 1;
842 void *malloc_pkey_hugetlb(long size, int prot, u16 pkey)
845 int flags = MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB;
847 if (!hugetlb_setup_ok)
848 return PTR_ERR_ENOTSUP;
850 dprintf1("doing %s(%ld, %x, %x)\n", __func__, size, prot, pkey);
851 size = ALIGN_UP(size, HPAGE_SIZE * 2);
852 pkey_assert(pkey < NR_PKEYS);
853 ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0);
854 pkey_assert(ptr != (void *)-1);
855 mprotect_pkey(ptr, size, prot, pkey);
857 record_pkey_malloc(ptr, size, prot);
859 dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey, ptr);
863 void *malloc_pkey_mmap_dax(long size, int prot, u16 pkey)
868 dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
870 pkey_assert(pkey < NR_PKEYS);
871 fd = open("/dax/foo", O_RDWR);
872 pkey_assert(fd >= 0);
874 ptr = mmap(0, size, prot, MAP_SHARED, fd, 0);
875 pkey_assert(ptr != (void *)-1);
877 mprotect_pkey(ptr, size, prot, pkey);
879 record_pkey_malloc(ptr, size, prot);
881 dprintf1("mmap()'d for pkey %d @ %p\n", pkey, ptr);
886 void *(*pkey_malloc[])(long size, int prot, u16 pkey) = {
888 malloc_pkey_with_mprotect,
889 malloc_pkey_anon_huge,
891 /* can not do direct with the pkey_mprotect() API:
892 malloc_pkey_mmap_direct,
893 malloc_pkey_mmap_dax,
897 void *malloc_pkey(long size, int prot, u16 pkey)
900 static int malloc_type;
901 int nr_malloc_types = ARRAY_SIZE(pkey_malloc);
903 pkey_assert(pkey < NR_PKEYS);
906 pkey_assert(malloc_type < nr_malloc_types);
908 ret = pkey_malloc[malloc_type](size, prot, pkey);
909 pkey_assert(ret != (void *)-1);
912 if (malloc_type >= nr_malloc_types)
913 malloc_type = (random()%nr_malloc_types);
915 /* try again if the malloc_type we tried is unsupported */
916 if (ret == PTR_ERR_ENOTSUP)
922 dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__,
923 size, prot, pkey, ret);
927 int last_pkru_faults;
928 #define UNKNOWN_PKEY -2
929 void expected_pk_fault(int pkey)
931 dprintf2("%s(): last_pkru_faults: %d pkru_faults: %d\n",
932 __func__, last_pkru_faults, pkru_faults);
933 dprintf2("%s(%d): last_si_pkey: %d\n", __func__, pkey, last_si_pkey);
934 pkey_assert(last_pkru_faults + 1 == pkru_faults);
937 * For exec-only memory, we do not know the pkey in
938 * advance, so skip this check.
940 if (pkey != UNKNOWN_PKEY)
941 pkey_assert(last_si_pkey == pkey);
944 * The signal handler shold have cleared out PKRU to let the
945 * test program continue. We now have to restore it.
950 __wrpkru(shadow_pkru);
951 dprintf1("%s() set PKRU=%x to restore state after signal nuked it\n",
952 __func__, shadow_pkru);
953 last_pkru_faults = pkru_faults;
957 #define do_not_expect_pk_fault(msg) do { \
958 if (last_pkru_faults != pkru_faults) \
959 dprintf0("unexpected PK fault: %s\n", msg); \
960 pkey_assert(last_pkru_faults == pkru_faults); \
963 int test_fds[10] = { -1 };
965 void __save_test_fd(int fd)
967 pkey_assert(fd >= 0);
968 pkey_assert(nr_test_fds < ARRAY_SIZE(test_fds));
969 test_fds[nr_test_fds] = fd;
973 int get_test_read_fd(void)
975 int test_fd = open("/etc/passwd", O_RDONLY);
976 __save_test_fd(test_fd);
980 void close_test_fds(void)
984 for (i = 0; i < nr_test_fds; i++) {
993 #define barrier() __asm__ __volatile__("": : :"memory")
994 __attribute__((noinline)) int read_ptr(int *ptr)
997 * Keep GCC from optimizing this away somehow
1003 void test_read_of_write_disabled_region(int *ptr, u16 pkey)
1007 dprintf1("disabling write access to PKEY[1], doing read\n");
1008 pkey_write_deny(pkey);
1009 ptr_contents = read_ptr(ptr);
1010 dprintf1("*ptr: %d\n", ptr_contents);
1013 void test_read_of_access_disabled_region(int *ptr, u16 pkey)
1017 dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey, ptr);
1019 pkey_access_deny(pkey);
1020 ptr_contents = read_ptr(ptr);
1021 dprintf1("*ptr: %d\n", ptr_contents);
1022 expected_pk_fault(pkey);
1024 void test_write_of_write_disabled_region(int *ptr, u16 pkey)
1026 dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey);
1027 pkey_write_deny(pkey);
1029 expected_pk_fault(pkey);
1031 void test_write_of_access_disabled_region(int *ptr, u16 pkey)
1033 dprintf1("disabling access to PKEY[%02d], doing write\n", pkey);
1034 pkey_access_deny(pkey);
1036 expected_pk_fault(pkey);
1038 void test_kernel_write_of_access_disabled_region(int *ptr, u16 pkey)
1041 int test_fd = get_test_read_fd();
1043 dprintf1("disabling access to PKEY[%02d], "
1044 "having kernel read() to buffer\n", pkey);
1045 pkey_access_deny(pkey);
1046 ret = read(test_fd, ptr, 1);
1047 dprintf1("read ret: %d\n", ret);
1050 void test_kernel_write_of_write_disabled_region(int *ptr, u16 pkey)
1053 int test_fd = get_test_read_fd();
1055 pkey_write_deny(pkey);
1056 ret = read(test_fd, ptr, 100);
1057 dprintf1("read ret: %d\n", ret);
1058 if (ret < 0 && (DEBUG_LEVEL > 0))
1059 perror("verbose read result (OK for this to be bad)");
1063 void test_kernel_gup_of_access_disabled_region(int *ptr, u16 pkey)
1065 int pipe_ret, vmsplice_ret;
1069 pipe_ret = pipe(pipe_fds);
1071 pkey_assert(pipe_ret == 0);
1072 dprintf1("disabling access to PKEY[%02d], "
1073 "having kernel vmsplice from buffer\n", pkey);
1074 pkey_access_deny(pkey);
1076 iov.iov_len = PAGE_SIZE;
1077 vmsplice_ret = vmsplice(pipe_fds[1], &iov, 1, SPLICE_F_GIFT);
1078 dprintf1("vmsplice() ret: %d\n", vmsplice_ret);
1079 pkey_assert(vmsplice_ret == -1);
1085 void test_kernel_gup_write_to_write_disabled_region(int *ptr, u16 pkey)
1087 int ignored = 0xdada;
1089 int some_int = __LINE__;
1091 dprintf1("disabling write to PKEY[%02d], "
1092 "doing futex gunk in buffer\n", pkey);
1094 pkey_write_deny(pkey);
1095 futex_ret = syscall(SYS_futex, ptr, FUTEX_WAIT, some_int-1, NULL,
1097 if (DEBUG_LEVEL > 0)
1099 dprintf1("futex() ret: %d\n", futex_ret);
1102 /* Assumes that all pkeys other than 'pkey' are unallocated */
1103 void test_pkey_syscalls_on_non_allocated_pkey(int *ptr, u16 pkey)
1108 /* Note: 0 is the default pkey, so don't mess with it */
1109 for (i = 1; i < NR_PKEYS; i++) {
1113 dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i);
1114 err = sys_pkey_free(i);
1117 err = sys_pkey_free(i);
1120 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, i);
1125 /* Assumes that all pkeys other than 'pkey' are unallocated */
1126 void test_pkey_syscalls_bad_args(int *ptr, u16 pkey)
1129 int bad_pkey = NR_PKEYS+99;
1131 /* pass a known-invalid pkey in: */
1132 err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, bad_pkey);
1136 void become_child(void)
1141 pkey_assert(forkret >= 0);
1142 dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
1151 /* Assumes that all pkeys other than 'pkey' are unallocated */
1152 void test_pkey_alloc_exhaust(int *ptr, u16 pkey)
1155 int allocated_pkeys[NR_PKEYS] = {0};
1156 int nr_allocated_pkeys = 0;
1159 for (i = 0; i < NR_PKEYS*3; i++) {
1161 dprintf1("%s() alloc loop: %d\n", __func__, i);
1162 new_pkey = alloc_pkey();
1163 dprintf4("%s()::%d, err: %d pkru: 0x%x shadow: 0x%x\n", __func__,
1164 __LINE__, err, __rdpkru(), shadow_pkru);
1165 rdpkru(); /* for shadow checking */
1166 dprintf2("%s() errno: %d ENOSPC: %d\n", __func__, errno, ENOSPC);
1167 if ((new_pkey == -1) && (errno == ENOSPC)) {
1168 dprintf2("%s() failed to allocate pkey after %d tries\n",
1169 __func__, nr_allocated_pkeys);
1172 * Ensure the number of successes never
1173 * exceeds the number of keys supported
1176 pkey_assert(nr_allocated_pkeys < NR_PKEYS);
1177 allocated_pkeys[nr_allocated_pkeys++] = new_pkey;
1181 * Make sure that allocation state is properly
1182 * preserved across fork().
1184 if (i == NR_PKEYS*2)
1188 dprintf3("%s()::%d\n", __func__, __LINE__);
1191 * There are 16 pkeys supported in hardware. Three are
1192 * allocated by the time we get here:
1193 * 1. The default key (0)
1194 * 2. One possibly consumed by an execute-only mapping.
1195 * 3. One allocated by the test code and passed in via
1196 * 'pkey' to this function.
1197 * Ensure that we can allocate at least another 13 (16-3).
1199 pkey_assert(i >= NR_PKEYS-3);
1201 for (i = 0; i < nr_allocated_pkeys; i++) {
1202 err = sys_pkey_free(allocated_pkeys[i]);
1204 rdpkru(); /* for shadow checking */
1209 * pkey 0 is special. It is allocated by default, so you do not
1210 * have to call pkey_alloc() to use it first. Make sure that it
1213 void test_mprotect_with_pkey_0(int *ptr, u16 pkey)
1218 assert(pkey_last_malloc_record);
1219 size = pkey_last_malloc_record->size;
1221 * This is a bit of a hack. But mprotect() requires
1222 * huge-page-aligned sizes when operating on hugetlbfs.
1223 * So, make sure that we use something that's a multiple
1224 * of a huge page when we can.
1226 if (size >= HPAGE_SIZE)
1228 prot = pkey_last_malloc_record->prot;
1231 mprotect_pkey(ptr, size, prot, 0);
1233 /* Make sure that we can set it back to the original pkey. */
1234 mprotect_pkey(ptr, size, prot, pkey);
1237 void test_ptrace_of_child(int *ptr, u16 pkey)
1239 __attribute__((__unused__)) int peek_result;
1245 * This is the "control" for our little expermient. Make sure
1246 * we can always access it when ptracing.
1248 int *plain_ptr_unaligned = malloc(HPAGE_SIZE);
1249 int *plain_ptr = ALIGN_PTR_UP(plain_ptr_unaligned, PAGE_SIZE);
1252 * Fork a child which is an exact copy of this process, of course.
1253 * That means we can do all of our tests via ptrace() and then plain
1254 * memory access and ensure they work differently.
1256 child_pid = fork_lazy_child();
1257 dprintf1("[%d] child pid: %d\n", getpid(), child_pid);
1259 ret = ptrace(PTRACE_ATTACH, child_pid, ignored, ignored);
1262 dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret, __LINE__);
1263 pkey_assert(ret != -1);
1264 ret = waitpid(child_pid, &status, WUNTRACED);
1265 if ((ret != child_pid) || !(WIFSTOPPED(status))) {
1266 fprintf(stderr, "weird waitpid result %ld stat %x\n",
1270 dprintf2("waitpid ret: %ld\n", ret);
1271 dprintf2("waitpid status: %d\n", status);
1273 pkey_access_deny(pkey);
1274 pkey_write_deny(pkey);
1276 /* Write access, untested for now:
1277 ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data);
1278 pkey_assert(ret != -1);
1279 dprintf1("poke at %p: %ld\n", peek_at, ret);
1283 * Try to access the pkey-protected "ptr" via ptrace:
1285 ret = ptrace(PTRACE_PEEKDATA, child_pid, ptr, ignored);
1286 /* expect it to work, without an error: */
1287 pkey_assert(ret != -1);
1288 /* Now access from the current task, and expect an exception: */
1289 peek_result = read_ptr(ptr);
1290 expected_pk_fault(pkey);
1293 * Try to access the NON-pkey-protected "plain_ptr" via ptrace:
1295 ret = ptrace(PTRACE_PEEKDATA, child_pid, plain_ptr, ignored);
1296 /* expect it to work, without an error: */
1297 pkey_assert(ret != -1);
1298 /* Now access from the current task, and expect NO exception: */
1299 peek_result = read_ptr(plain_ptr);
1300 do_not_expect_pk_fault("read plain pointer after ptrace");
1302 ret = ptrace(PTRACE_DETACH, child_pid, ignored, 0);
1303 pkey_assert(ret != -1);
1305 ret = kill(child_pid, SIGKILL);
1306 pkey_assert(ret != -1);
1310 free(plain_ptr_unaligned);
1313 void *get_pointer_to_instructions(void)
1317 p1 = ALIGN_PTR_UP(&lots_o_noops_around_write, PAGE_SIZE);
1318 dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write);
1319 /* lots_o_noops_around_write should be page-aligned already */
1320 assert(p1 == &lots_o_noops_around_write);
1322 /* Point 'p1' at the *second* page of the function: */
1326 * Try to ensure we fault this in on next touch to ensure
1327 * we get an instruction fault as opposed to a data one
1329 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1334 void test_executing_on_unreadable_memory(int *ptr, u16 pkey)
1341 p1 = get_pointer_to_instructions();
1342 lots_o_noops_around_write(&scratch);
1343 ptr_contents = read_ptr(p1);
1344 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1346 ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC, (u64)pkey);
1348 pkey_access_deny(pkey);
1350 dprintf2("pkru: %x\n", rdpkru());
1353 * Make sure this is an *instruction* fault
1355 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1356 lots_o_noops_around_write(&scratch);
1357 do_not_expect_pk_fault("executing on PROT_EXEC memory");
1358 ptr_contents = read_ptr(p1);
1359 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1360 expected_pk_fault(pkey);
1363 void test_implicit_mprotect_exec_only_memory(int *ptr, u16 pkey)
1370 dprintf1("%s() start\n", __func__);
1372 p1 = get_pointer_to_instructions();
1373 lots_o_noops_around_write(&scratch);
1374 ptr_contents = read_ptr(p1);
1375 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1377 /* Use a *normal* mprotect(), not mprotect_pkey(): */
1378 ret = mprotect(p1, PAGE_SIZE, PROT_EXEC);
1381 dprintf2("pkru: %x\n", rdpkru());
1383 /* Make sure this is an *instruction* fault */
1384 madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1385 lots_o_noops_around_write(&scratch);
1386 do_not_expect_pk_fault("executing on PROT_EXEC memory");
1387 ptr_contents = read_ptr(p1);
1388 dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1389 expected_pk_fault(UNKNOWN_PKEY);
1392 * Put the memory back to non-PROT_EXEC. Should clear the
1393 * exec-only pkey off the VMA and allow it to be readable
1394 * again. Go to PROT_NONE first to check for a kernel bug
1395 * that did not clear the pkey when doing PROT_NONE.
1397 ret = mprotect(p1, PAGE_SIZE, PROT_NONE);
1400 ret = mprotect(p1, PAGE_SIZE, PROT_READ|PROT_EXEC);
1402 ptr_contents = read_ptr(p1);
1403 do_not_expect_pk_fault("plain read on recently PROT_EXEC area");
1406 void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey)
1408 int size = PAGE_SIZE;
1411 if (cpu_has_pku()) {
1412 dprintf1("SKIP: %s: no CPU support\n", __func__);
1416 sret = syscall(SYS_mprotect_key, ptr, size, PROT_READ, pkey);
1417 pkey_assert(sret < 0);
1420 void (*pkey_tests[])(int *ptr, u16 pkey) = {
1421 test_read_of_write_disabled_region,
1422 test_read_of_access_disabled_region,
1423 test_write_of_write_disabled_region,
1424 test_write_of_access_disabled_region,
1425 test_kernel_write_of_access_disabled_region,
1426 test_kernel_write_of_write_disabled_region,
1427 test_kernel_gup_of_access_disabled_region,
1428 test_kernel_gup_write_to_write_disabled_region,
1429 test_executing_on_unreadable_memory,
1430 test_implicit_mprotect_exec_only_memory,
1431 test_mprotect_with_pkey_0,
1432 test_ptrace_of_child,
1433 test_pkey_syscalls_on_non_allocated_pkey,
1434 test_pkey_syscalls_bad_args,
1435 test_pkey_alloc_exhaust,
1438 void run_tests_once(void)
1441 int prot = PROT_READ|PROT_WRITE;
1443 for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) {
1445 int orig_pkru_faults = pkru_faults;
1447 dprintf1("======================\n");
1448 dprintf1("test %d preparing...\n", test_nr);
1451 pkey = alloc_random_pkey();
1452 dprintf1("test %d starting with pkey: %d\n", test_nr, pkey);
1453 ptr = malloc_pkey(PAGE_SIZE, prot, pkey);
1454 dprintf1("test %d starting...\n", test_nr);
1455 pkey_tests[test_nr](ptr, pkey);
1456 dprintf1("freeing test memory: %p\n", ptr);
1457 free_pkey_malloc(ptr);
1458 sys_pkey_free(pkey);
1460 dprintf1("pkru_faults: %d\n", pkru_faults);
1461 dprintf1("orig_pkru_faults: %d\n", orig_pkru_faults);
1466 printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr);
1467 dprintf1("======================\n\n");
1472 void pkey_setup_shadow(void)
1474 shadow_pkru = __rdpkru();
1479 int nr_iterations = 22;
1483 printf("has pku: %d\n", cpu_has_pku());
1485 if (!cpu_has_pku()) {
1486 int size = PAGE_SIZE;
1489 printf("running PKEY tests for unsupported CPU/OS\n");
1491 ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1492 assert(ptr != (void *)-1);
1493 test_mprotect_pkey_on_unsupported_cpu(ptr, 1);
1497 pkey_setup_shadow();
1498 printf("startup pkru: %x\n", rdpkru());
1501 while (nr_iterations-- > 0)
1504 printf("done (all tests OK)\n");