Merge tag 'mfd-next-5.11' of git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd

[linux-2.6-microblaze.git] / kernel / scs.c

diff --git a/kernel/scs.c b/kernel/scs.c
index 4ff4a7b..e2a71fc 100644 (file)
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -5,26 +5,49 @@
  * Copyright (C) 2019 Google LLC
  */
 
+#include <linux/cpuhotplug.h>
 #include <linux/kasan.h>
 #include <linux/mm.h>
 #include <linux/scs.h>
-#include <linux/slab.h>
+#include <linux/vmalloc.h>
 #include <linux/vmstat.h>
 
-static struct kmem_cache *scs_cache;
-
 static void __scs_account(void *s, int account)
 {
-       struct page *scs_page = virt_to_page(s);
+       struct page *scs_page = vmalloc_to_page(s);
 
        mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
                            account * (SCS_SIZE / SZ_1K));
 }
 
-static void *scs_alloc(int node)
+/* Matches NR_CACHED_STACKS for VMAP_STACK */
+#define NR_CACHED_SCS 2
+static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
+
+static void *__scs_alloc(int node)
 {
-       void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
+       int i;
+       void *s;
+
+       for (i = 0; i < NR_CACHED_SCS; i++) {
+               s = this_cpu_xchg(scs_cache[i], NULL);
+               if (s) {
+                       kasan_unpoison_vmalloc(s, SCS_SIZE);
+                       memset(s, 0, SCS_SIZE);
+                       return s;
+               }
+       }
+
+       return __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END,
+                                   GFP_SCS, PAGE_KERNEL, 0, node,
+                                   __builtin_return_address(0));
+}
 
+void *scs_alloc(int node)
+{
+       void *s;
+
+       s = __scs_alloc(node);
        if (!s)
                return NULL;
 
@@ -34,21 +57,47 @@ static void *scs_alloc(int node)
         * Poison the allocation to catch unintentional accesses to
         * the shadow stack when KASAN is enabled.
         */
-       kasan_poison_object_data(scs_cache, s);
+       kasan_poison_vmalloc(s, SCS_SIZE);
        __scs_account(s, 1);
        return s;
 }
 
-static void scs_free(void *s)
+void scs_free(void *s)
 {
+       int i;
+
        __scs_account(s, -1);
-       kasan_unpoison_object_data(scs_cache, s);
-       kmem_cache_free(scs_cache, s);
+
+       /*
+        * We cannot sleep as this can be called in interrupt context,
+        * so use this_cpu_cmpxchg to update the cache, and vfree_atomic
+        * to free the stack.
+        */
+
+       for (i = 0; i < NR_CACHED_SCS; i++)
+               if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
+                       return;
+
+       vfree_atomic(s);
+}
+
+static int scs_cleanup(unsigned int cpu)
+{
+       int i;
+       void **cache = per_cpu_ptr(scs_cache, cpu);
+
+       for (i = 0; i < NR_CACHED_SCS; i++) {
+               vfree(cache[i]);
+               cache[i] = NULL;
+       }
+
+       return 0;
 }
 
 void __init scs_init(void)
 {
-       scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
+       cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
+                         scs_cleanup);
 }
 
 int scs_prepare(struct task_struct *tsk, int node)