firmware: arm_sdei: add support for CPU private events

Private SDE events are per-cpu, and need to be registered and enabled
on each CPU.

Hide this detail from the caller by adapting our {,un}register and
{en,dis}able calls to send an IPI to each CPU if the event is private.

CPU private events are unregistered when the CPU is powered-off, and
re-registered when the CPU is brought back online. This saves bringing
secondary cores back online to call private_reset() on shutdown, kexec
and resume from hibernate.

Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c
index c503bc5..0c73536 100644 (file)
--- a/drivers/firmware/arm_sdei.c
+++ b/drivers/firmware/arm_sdei.c
@@ -5,9 +5,11 @@
 #include <linux/acpi.h>
 #include <linux/arm_sdei.h>
 #include <linux/arm-smccc.h>
+#include <linux/atomic.h>
 #include <linux/bitops.h>
 #include <linux/compiler.h>
 #include <linux/cpuhotplug.h>
+#include <linux/cpu.h>
 #include <linux/cpu_pm.h>
 #include <linux/errno.h>
 #include <linux/hardirq.h>
@@ -52,7 +54,13 @@ struct sdei_event {
        u8                      priority;
 
        /* This pointer is handed to firmware as the event argument. */
-       struct sdei_registered_event *registered;
+       union {
+               /* Shared events */
+               struct sdei_registered_event *registered;
+
+               /* CPU private events */
+               struct sdei_registered_event __percpu *private_registered;
+       };
 };
 
 /* Take the mutex for any API call or modification. Take the mutex first. */
@@ -62,6 +70,34 @@ static DEFINE_MUTEX(sdei_events_lock);
 static DEFINE_SPINLOCK(sdei_list_lock);
 static LIST_HEAD(sdei_list);
 
+/* Private events are registered/enabled via IPI passing one of these */
+struct sdei_crosscall_args {
+       struct sdei_event *event;
+       atomic_t errors;
+       int first_error;
+};
+
+#define CROSSCALL_INIT(arg, event)     (arg.event = event, \
+                                        arg.first_error = 0, \
+                                        atomic_set(&arg.errors, 0))
+
+static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
+{
+       struct sdei_crosscall_args arg;
+
+       CROSSCALL_INIT(arg, event);
+       on_each_cpu(fn, &arg, true);
+
+       return arg.first_error;
+}
+
+static inline void
+sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
+{
+       if (err && (atomic_inc_return(&arg->errors) == 1))
+               arg->first_error = err;
+}
+
 static int sdei_to_linux_errno(unsigned long sdei_err)
 {
        switch (sdei_err) {
@@ -207,6 +243,26 @@ static struct sdei_event *sdei_event_create(u32 event_num,
                reg->callback = cb;
                reg->callback_arg = cb_arg;
                event->registered = reg;
+       } else {
+               int cpu;
+               struct sdei_registered_event __percpu *regs;
+
+               regs = alloc_percpu(struct sdei_registered_event);
+               if (!regs) {
+                       kfree(event);
+                       return ERR_PTR(-ENOMEM);
+               }
+
+               for_each_possible_cpu(cpu) {
+                       reg = per_cpu_ptr(regs, cpu);
+
+                       reg->event_num = event->event_num;
+                       reg->priority = event->priority;
+                       reg->callback = cb;
+                       reg->callback_arg = cb_arg;
+               }
+
+               event->private_registered = regs;
        }
 
        if (sdei_event_find(event_num)) {
@@ -232,6 +288,8 @@ static void sdei_event_destroy(struct sdei_event *event)
 
        if (event->type == SDEI_EVENT_TYPE_SHARED)
                kfree(event->registered);
+       else
+               free_percpu(event->private_registered);
 
        kfree(event);
 }
@@ -262,11 +320,6 @@ static void _ipi_mask_cpu(void *ignored)
        sdei_mask_local_cpu();
 }
 
-static int sdei_cpuhp_down(unsigned int ignored)
-{
-       return sdei_mask_local_cpu();
-}
-
 int sdei_unmask_local_cpu(void)
 {
        int err;
@@ -288,11 +341,6 @@ static void _ipi_unmask_cpu(void *ignored)
        sdei_unmask_local_cpu();
 }
 
-static int sdei_cpuhp_up(unsigned int ignored)
-{
-       return sdei_unmask_local_cpu();
-}
-
 static void _ipi_private_reset(void *ignored)
 {
        int err;
@@ -337,6 +385,19 @@ static int sdei_api_event_enable(u32 event_num)
                              0, NULL);
 }
 
+/* Called directly by the hotplug callbacks */
+static void _local_event_enable(void *data)
+{
+       int err;
+       struct sdei_crosscall_args *arg = data;
+
+       WARN_ON_ONCE(preemptible());
+
+       err = sdei_api_event_enable(arg->event->event_num);
+
+       sdei_cross_call_return(arg, err);
+}
+
 int sdei_event_enable(u32 event_num)
 {
        int err = -EINVAL;
@@ -355,6 +416,8 @@ int sdei_event_enable(u32 event_num)
 
        if (event->type == SDEI_EVENT_TYPE_SHARED)
                err = sdei_api_event_enable(event->event_num);
+       else
+               err = sdei_do_cross_call(_local_event_enable, event);
        mutex_unlock(&sdei_events_lock);
 
        return err;
@@ -367,6 +430,16 @@ static int sdei_api_event_disable(u32 event_num)
                              0, 0, NULL);
 }
 
+static void _ipi_event_disable(void *data)
+{
+       int err;
+       struct sdei_crosscall_args *arg = data;
+
+       err = sdei_api_event_disable(arg->event->event_num);
+
+       sdei_cross_call_return(arg, err);
+}
+
 int sdei_event_disable(u32 event_num)
 {
        int err = -EINVAL;
@@ -385,6 +458,8 @@ int sdei_event_disable(u32 event_num)
 
        if (event->type == SDEI_EVENT_TYPE_SHARED)
                err = sdei_api_event_disable(event->event_num);
+       else
+               err = sdei_do_cross_call(_ipi_event_disable, event);
        mutex_unlock(&sdei_events_lock);
 
        return err;
@@ -397,6 +472,19 @@ static int sdei_api_event_unregister(u32 event_num)
                              0, 0, 0, NULL);
 }
 
+/* Called directly by the hotplug callbacks */
+static void _local_event_unregister(void *data)
+{
+       int err;
+       struct sdei_crosscall_args *arg = data;
+
+       WARN_ON_ONCE(preemptible());
+
+       err = sdei_api_event_unregister(arg->event->event_num);
+
+       sdei_cross_call_return(arg, err);
+}
+
 static int _sdei_event_unregister(struct sdei_event *event)
 {
        lockdep_assert_held(&sdei_events_lock);
@@ -409,7 +497,7 @@ static int _sdei_event_unregister(struct sdei_event *event)
        if (event->type == SDEI_EVENT_TYPE_SHARED)
                return sdei_api_event_unregister(event->event_num);
 
-       return -EINVAL;
+       return sdei_do_cross_call(_local_event_unregister, event);
 }
 
 int sdei_event_unregister(u32 event_num)
@@ -473,17 +561,50 @@ static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
                              flags, affinity, NULL);
 }
 
+/* Called directly by the hotplug callbacks */
+static void _local_event_register(void *data)
+{
+       int err;
+       struct sdei_registered_event *reg;
+       struct sdei_crosscall_args *arg = data;
+
+       WARN_ON(preemptible());
+
+       reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
+       err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
+                                     reg, 0, 0);
+
+       sdei_cross_call_return(arg, err);
+}
+
 static int _sdei_event_register(struct sdei_event *event)
 {
+       int err;
+
        lockdep_assert_held(&sdei_events_lock);
 
+       spin_lock(&sdei_list_lock);
+       event->reregister = true;
+       spin_unlock(&sdei_list_lock);
+
        if (event->type == SDEI_EVENT_TYPE_SHARED)
                return sdei_api_event_register(event->event_num,
                                               sdei_entry_point,
                                               event->registered,
                                               SDEI_EVENT_REGISTER_RM_ANY, 0);
 
-       return -EINVAL;
+
+       err = sdei_do_cross_call(_local_event_register, event);
+       if (err) {
+               spin_lock(&sdei_list_lock);
+               event->reregister = false;
+               event->reenable = false;
+               spin_unlock(&sdei_list_lock);
+
+               sdei_do_cross_call(_local_event_unregister, event);
+       }
+
+       return err;
 }
 
 int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
@@ -538,6 +659,8 @@ static int sdei_reregister_event(struct sdei_event *event)
        if (event->reenable) {
                if (event->type == SDEI_EVENT_TYPE_SHARED)
                        err = sdei_api_event_enable(event->event_num);
+               else
+                       err = sdei_do_cross_call(_local_event_enable, event);
        }
 
        if (err)
@@ -569,6 +692,62 @@ static int sdei_reregister_shared(void)
        return err;
 }
 
+static int sdei_cpuhp_down(unsigned int cpu)
+{
+       struct sdei_event *event;
+       struct sdei_crosscall_args arg;
+
+       /* un-register private events */
+       spin_lock(&sdei_list_lock);
+       list_for_each_entry(event, &sdei_list, list) {
+               if (event->type == SDEI_EVENT_TYPE_SHARED)
+                       continue;
+
+               CROSSCALL_INIT(arg, event);
+               /* call the cross-call function locally... */
+               _local_event_unregister(&arg);
+               if (arg.first_error)
+                       pr_err("Failed to unregister event %u: %d\n",
+                              event->event_num, arg.first_error);
+       }
+       spin_unlock(&sdei_list_lock);
+
+       return sdei_mask_local_cpu();
+}
+
+static int sdei_cpuhp_up(unsigned int cpu)
+{
+       struct sdei_event *event;
+       struct sdei_crosscall_args arg;
+
+       /* re-register/enable private events */
+       spin_lock(&sdei_list_lock);
+       list_for_each_entry(event, &sdei_list, list) {
+               if (event->type == SDEI_EVENT_TYPE_SHARED)
+                       continue;
+
+               if (event->reregister) {
+                       CROSSCALL_INIT(arg, event);
+                       /* call the cross-call function locally... */
+                       _local_event_register(&arg);
+                       if (arg.first_error)
+                               pr_err("Failed to re-register event %u: %d\n",
+                                      event->event_num, arg.first_error);
+               }
+
+               if (event->reenable) {
+                       CROSSCALL_INIT(arg, event);
+                       _local_event_enable(&arg);
+                       if (arg.first_error)
+                               pr_err("Failed to re-enable event %u: %d\n",
+                                      event->event_num, arg.first_error);
+               }
+       }
+       spin_unlock(&sdei_list_lock);
+
+       return sdei_unmask_local_cpu();
+}
+
 /* When entering idle, mask/unmask events for this cpu */
 static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
                            void *data)
@@ -622,6 +801,7 @@ static int sdei_device_freeze(struct device *dev)
 {
        int err;
 
+       /* unregister private events */
        cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 
        err = sdei_unregister_shared();