kfence: use TASK_IDLE when awaiting allocation

[linux-2.6-microblaze.git] / mm / kfence / core.c

diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index d53c91f..4d21ac4 100644 (file)
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -10,6 +10,7 @@
 #include <linux/atomic.h>
 #include <linux/bug.h>
 #include <linux/debugfs.h>
+#include <linux/irq_work.h>
 #include <linux/kcsan-checks.h>
 #include <linux/kfence.h>
 #include <linux/kmemleak.h>
@@ -19,6 +20,7 @@
 #include <linux/moduleparam.h>
 #include <linux/random.h>
 #include <linux/rcupdate.h>
+#include <linux/sched/sysctl.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -372,6 +374,7 @@ static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool z
 
        /* Restore page protection if there was an OOB access. */
        if (meta->unprotected_page) {
+               memzero_explicit((void *)ALIGN_DOWN(meta->unprotected_page, PAGE_SIZE), PAGE_SIZE);
                kfence_protect(meta->unprotected_page);
                meta->unprotected_page = 0;
        }
@@ -586,6 +589,17 @@ late_initcall(kfence_debugfs_init);
 
 /* === Allocation Gate Timer ================================================ */
 
+#ifdef CONFIG_KFENCE_STATIC_KEYS
+/* Wait queue to wake up allocation-gate timer task. */
+static DECLARE_WAIT_QUEUE_HEAD(allocation_wait);
+
+static void wake_up_kfence_timer(struct irq_work *work)
+{
+       wake_up(&allocation_wait);
+}
+static DEFINE_IRQ_WORK(wake_up_kfence_timer_work, wake_up_kfence_timer);
+#endif
+
 /*
  * Set up delayed work, which will enable and disable the static key. We need to
  * use a work queue (rather than a simple timer), since enabling and disabling a
@@ -603,29 +617,27 @@ static void toggle_allocation_gate(struct work_struct *work)
        if (!READ_ONCE(kfence_enabled))
                return;
 
-       /* Enable static key, and await allocation to happen. */
        atomic_set(&kfence_allocation_gate, 0);
 #ifdef CONFIG_KFENCE_STATIC_KEYS
+       /* Enable static key, and await allocation to happen. */
        static_branch_enable(&kfence_allocation_key);
-       /*
-        * Await an allocation. Timeout after 1 second, in case the kernel stops
-        * doing allocations, to avoid stalling this worker task for too long.
-        */
-       {
-               unsigned long end_wait = jiffies + HZ;
-
-               do {
-                       set_current_state(TASK_UNINTERRUPTIBLE);
-                       if (atomic_read(&kfence_allocation_gate) != 0)
-                               break;
-                       schedule_timeout(1);
-               } while (time_before(jiffies, end_wait));
-               __set_current_state(TASK_RUNNING);
+
+       if (sysctl_hung_task_timeout_secs) {
+               /*
+                * During low activity with no allocations we might wait a
+                * while; let's avoid the hung task warning.
+                */
+               wait_event_idle_timeout(allocation_wait, atomic_read(&kfence_allocation_gate),
+                                       sysctl_hung_task_timeout_secs * HZ / 2);
+       } else {
+               wait_event_idle(allocation_wait, atomic_read(&kfence_allocation_gate));
        }
+
        /* Disable static key and reset timer. */
        static_branch_disable(&kfence_allocation_key);
 #endif
-       schedule_delayed_work(&kfence_timer, msecs_to_jiffies(kfence_sample_interval));
+       queue_delayed_work(system_power_efficient_wq, &kfence_timer,
+                          msecs_to_jiffies(kfence_sample_interval));
 }
 static DECLARE_DELAYED_WORK(kfence_timer, toggle_allocation_gate);
 
@@ -654,7 +666,7 @@ void __init kfence_init(void)
        }
 
        WRITE_ONCE(kfence_enabled, true);
-       schedule_delayed_work(&kfence_timer, 0);
+       queue_delayed_work(system_power_efficient_wq, &kfence_timer, 0);
        pr_info("initialized - using %lu bytes for %d objects at 0x%p-0x%p\n", KFENCE_POOL_SIZE,
                CONFIG_KFENCE_NUM_OBJECTS, (void *)__kfence_pool,
                (void *)(__kfence_pool + KFENCE_POOL_SIZE));
@@ -728,6 +740,19 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
         */
        if (atomic_read(&kfence_allocation_gate) || atomic_inc_return(&kfence_allocation_gate) > 1)
                return NULL;
+#ifdef CONFIG_KFENCE_STATIC_KEYS
+       /*
+        * waitqueue_active() is fully ordered after the update of
+        * kfence_allocation_gate per atomic_inc_return().
+        */
+       if (waitqueue_active(&allocation_wait)) {
+               /*
+                * Calling wake_up() here may deadlock when allocations happen
+                * from within timer code. Use an irq_work to defer it.
+                */
+               irq_work_queue(&wake_up_kfence_timer_work);
+       }
+#endif
 
        if (!READ_ONCE(kfence_enabled))
                return NULL;