mm: kfence: apply kmemleak_ignore_phys on early allocated pool

[linux-2.6-microblaze.git] / mm / kfence / core.c

diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index 9b2b5f5..6aff49f 100644 (file)
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -21,6 +21,8 @@
 #include <linux/log2.h>
 #include <linux/memblock.h>
 #include <linux/moduleparam.h>
+#include <linux/notifier.h>
+#include <linux/panic_notifier.h>
 #include <linux/random.h>
 #include <linux/rcupdate.h>
 #include <linux/sched/clock.h>
@@ -67,8 +69,11 @@ static int param_set_sample_interval(const char *val, const struct kernel_param
        if (ret < 0)
                return ret;
 
-       if (!num) /* Using 0 to indicate KFENCE is disabled. */
+       /* Using 0 to indicate KFENCE is disabled. */
+       if (!num && READ_ONCE(kfence_enabled)) {
+               pr_info("disabled\n");
                WRITE_ONCE(kfence_enabled, false);
+       }
 
        *((unsigned long *)kp->arg) = num;
 
@@ -99,6 +104,10 @@ module_param_named(skip_covered_thresh, kfence_skip_covered_thresh, ulong, 0644)
 static bool kfence_deferrable __read_mostly = IS_ENABLED(CONFIG_KFENCE_DEFERRABLE);
 module_param_named(deferrable, kfence_deferrable, bool, 0444);
 
+/* If true, check all canary bytes on panic. */
+static bool kfence_check_on_panic __read_mostly;
+module_param_named(check_on_panic, kfence_check_on_panic, bool, 0444);
+
 /* The pool of pages used for guard pages and objects. */
 char *__kfence_pool __read_mostly;
 EXPORT_SYMBOL(__kfence_pool); /* Export for test modules. */
@@ -351,6 +360,9 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
        unsigned long flags;
        struct slab *slab;
        void *addr;
+       const bool random_right_allocate = prandom_u32_max(2);
+       const bool random_fault = CONFIG_KFENCE_STRESS_TEST_FAULTS &&
+                                 !prandom_u32_max(CONFIG_KFENCE_STRESS_TEST_FAULTS);
 
        /* Try to obtain a free object. */
        raw_spin_lock_irqsave(&kfence_freelist_lock, flags);
@@ -395,7 +407,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
         * is that the out-of-bounds accesses detected are deterministic for
         * such allocations.
         */
-       if (prandom_u32_max(2)) {
+       if (random_right_allocate) {
                /* Allocate on the "right" side, re-calculate address. */
                meta->addr += PAGE_SIZE - size;
                meta->addr = ALIGN_DOWN(meta->addr, cache->align);
@@ -435,7 +447,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g
        if (cache->ctor)
                cache->ctor(addr);
 
-       if (CONFIG_KFENCE_STRESS_TEST_FAULTS && !prandom_u32_max(CONFIG_KFENCE_STRESS_TEST_FAULTS))
+       if (random_fault)
                kfence_protect(meta->addr); /* Random "faults" by protecting the object. */
 
        atomic_long_inc(&counters[KFENCE_COUNTER_ALLOCATED]);
@@ -591,14 +603,6 @@ static unsigned long kfence_init_pool(void)
                addr += 2 * PAGE_SIZE;
        }
 
-       /*
-        * The pool is live and will never be deallocated from this point on.
-        * Remove the pool object from the kmemleak object tree, as it would
-        * otherwise overlap with allocations returned by kfence_alloc(), which
-        * are registered with kmemleak through the slab post-alloc hook.
-        */
-       kmemleak_free(__kfence_pool);
-
        return 0;
 }
 
@@ -611,8 +615,16 @@ static bool __init kfence_init_pool_early(void)
 
        addr = kfence_init_pool();
 
-       if (!addr)
+       if (!addr) {
+               /*
+                * The pool is live and will never be deallocated from this point on.
+                * Ignore the pool object from the kmemleak phys object tree, as it would
+                * otherwise overlap with allocations returned by kfence_alloc(), which
+                * are registered with kmemleak through the slab post-alloc hook.
+                */
+               kmemleak_ignore_phys(__pa(__kfence_pool));
                return true;
+       }
 
        /*
         * Only release unprotected pages, and do not try to go back and change
@@ -621,6 +633,16 @@ static bool __init kfence_init_pool_early(void)
         * fails for the first page, and therefore expect addr==__kfence_pool in
         * most failure cases.
         */
+       for (char *p = (char *)addr; p < __kfence_pool + KFENCE_POOL_SIZE; p += PAGE_SIZE) {
+               struct slab *slab = virt_to_slab(p);
+
+               if (!slab)
+                       continue;
+#ifdef CONFIG_MEMCG
+               slab->memcg_data = 0;
+#endif
+               __folio_clear_slab(slab_folio(slab));
+       }
        memblock_free_late(__pa(addr), KFENCE_POOL_SIZE - (addr - (unsigned long)__kfence_pool));
        __kfence_pool = NULL;
        return false;
@@ -727,6 +749,31 @@ static int __init kfence_debugfs_init(void)
 
 late_initcall(kfence_debugfs_init);
 
+/* === Panic Notifier ====================================================== */
+
+static void kfence_check_all_canary(void)
+{
+       int i;
+
+       for (i = 0; i < CONFIG_KFENCE_NUM_OBJECTS; i++) {
+               struct kfence_metadata *meta = &kfence_metadata[i];
+
+               if (meta->state == KFENCE_OBJECT_ALLOCATED)
+                       for_each_canary(meta, check_canary_byte);
+       }
+}
+
+static int kfence_check_canary_callback(struct notifier_block *nb,
+                                       unsigned long reason, void *arg)
+{
+       kfence_check_all_canary();
+       return NOTIFY_OK;
+}
+
+static struct notifier_block kfence_check_canary_notifier = {
+       .notifier_call = kfence_check_canary_callback,
+};
+
 /* === Allocation Gate Timer ================================================ */
 
 static struct delayed_work kfence_timer;
@@ -804,6 +851,9 @@ static void kfence_init_enable(void)
        else
                INIT_DELAYED_WORK(&kfence_timer, toggle_allocation_gate);
 
+       if (kfence_check_on_panic)
+               atomic_notifier_chain_register(&panic_notifier_list, &kfence_check_canary_notifier);
+
        WRITE_ONCE(kfence_enabled, true);
        queue_delayed_work(system_unbound_wq, &kfence_timer, 0);
 
@@ -864,6 +914,7 @@ static int kfence_enable_late(void)
 
        WRITE_ONCE(kfence_enabled, true);
        queue_delayed_work(system_unbound_wq, &kfence_timer, 0);
+       pr_info("re-enabled\n");
        return 0;
 }