drivers/gpu/host1x/intr.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Interrupt Management
   4  *
   5  * Copyright (c) 2010-2013, NVIDIA Corporation.
   6  */
   7
   8 #include <linux/clk.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/slab.h>
  11 #include <linux/irq.h>
  12
  13 #include <trace/events/host1x.h>
  14 #include "channel.h"
  15 #include "dev.h"
  16 #include "intr.h"
  17
  18 /* Wait list management */
  19
  20 enum waitlist_state {
  21         WLS_PENDING,
  22         WLS_REMOVED,
  23         WLS_CANCELLED,
  24         WLS_HANDLED
  25 };
  26
  27 static void waiter_release(struct kref *kref)
  28 {
  29         kfree(container_of(kref, struct host1x_waitlist, refcount));
  30 }
  31
  32 /*
  33  * add a waiter to a waiter queue, sorted by threshold
  34  * returns true if it was added at the head of the queue
  35  */
  36 static bool add_waiter_to_queue(struct host1x_waitlist *waiter,
  37                                 struct list_head *queue)
  38 {
  39         struct host1x_waitlist *pos;
  40         u32 thresh = waiter->thresh;
  41
  42         list_for_each_entry_reverse(pos, queue, list)
  43                 if ((s32)(pos->thresh - thresh) <= 0) {
  44                         list_add(&waiter->list, &pos->list);
  45                         return false;
  46                 }
  47
  48         list_add(&waiter->list, queue);
  49         return true;
  50 }
  51
  52 /*
  53  * run through a waiter queue for a single sync point ID
  54  * and gather all completed waiters into lists by actions
  55  */
  56 static void remove_completed_waiters(struct list_head *head, u32 sync,
  57                         struct list_head completed[HOST1X_INTR_ACTION_COUNT])
  58 {
  59         struct list_head *dest;
  60         struct host1x_waitlist *waiter, *next, *prev;
  61
  62         list_for_each_entry_safe(waiter, next, head, list) {
  63                 if ((s32)(waiter->thresh - sync) > 0)
  64                         break;
  65
  66                 dest = completed + waiter->action;
  67
  68                 /* consolidate submit cleanups */
  69                 if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE &&
  70                     !list_empty(dest)) {
  71                         prev = list_entry(dest->prev,
  72                                           struct host1x_waitlist, list);
  73                         if (prev->data == waiter->data) {
  74                                 prev->count++;
  75                                 dest = NULL;
  76                         }
  77                 }
  78
  79                 /* PENDING->REMOVED or CANCELLED->HANDLED */
  80                 if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
  81                         list_del(&waiter->list);
  82                         kref_put(&waiter->refcount, waiter_release);
  83                 } else
  84                         list_move_tail(&waiter->list, dest);
  85         }
  86 }
  87
  88 static void reset_threshold_interrupt(struct host1x *host,
  89                                       struct list_head *head,
  90                                       unsigned int id)
  91 {
  92         u32 thresh =
  93                 list_first_entry(head, struct host1x_waitlist, list)->thresh;
  94
  95         host1x_hw_intr_set_syncpt_threshold(host, id, thresh);
  96         host1x_hw_intr_enable_syncpt_intr(host, id);
  97 }
  98
  99 static void action_submit_complete(struct host1x_waitlist *waiter)
 100 {
 101         struct host1x_channel *channel = waiter->data;
 102
 103         host1x_cdma_update(&channel->cdma);
 104
 105         /*  Add nr_completed to trace */
 106         trace_host1x_channel_submit_complete(dev_name(channel->dev),
 107                                              waiter->count, waiter->thresh);
 108 }
 109
 110 static void action_wakeup(struct host1x_waitlist *waiter)
 111 {
 112         wait_queue_head_t *wq = waiter->data;
 113
 114         wake_up(wq);
 115 }
 116
 117 static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
 118 {
 119         wait_queue_head_t *wq = waiter->data;
 120
 121         wake_up_interruptible(wq);
 122 }
 123
 124 typedef void (*action_handler)(struct host1x_waitlist *waiter);
 125
 126 static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
 127         action_submit_complete,
 128         action_wakeup,
 129         action_wakeup_interruptible,
 130 };
 131
 132 static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT])
 133 {
 134         struct list_head *head = completed;
 135         unsigned int i;
 136
 137         for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) {
 138                 action_handler handler = action_handlers[i];
 139                 struct host1x_waitlist *waiter, *next;
 140
 141                 list_for_each_entry_safe(waiter, next, head, list) {
 142                         list_del(&waiter->list);
 143                         handler(waiter);
 144                         WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) !=
 145                                 WLS_REMOVED);
 146                         kref_put(&waiter->refcount, waiter_release);
 147                 }
 148         }
 149 }
 150
 151 /*
 152  * Remove & handle all waiters that have completed for the given syncpt
 153  */
 154 static int process_wait_list(struct host1x *host,
 155                              struct host1x_syncpt *syncpt,
 156                              u32 threshold)
 157 {
 158         struct list_head completed[HOST1X_INTR_ACTION_COUNT];
 159         unsigned int i;
 160         int empty;
 161
 162         for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i)
 163                 INIT_LIST_HEAD(completed + i);
 164
 165         spin_lock(&syncpt->intr.lock);
 166
 167         remove_completed_waiters(&syncpt->intr.wait_head, threshold,
 168                                  completed);
 169
 170         empty = list_empty(&syncpt->intr.wait_head);
 171         if (empty)
 172                 host1x_hw_intr_disable_syncpt_intr(host, syncpt->id);
 173         else
 174                 reset_threshold_interrupt(host, &syncpt->intr.wait_head,
 175                                           syncpt->id);
 176
 177         spin_unlock(&syncpt->intr.lock);
 178
 179         run_handlers(completed);
 180
 181         return empty;
 182 }
 183
 184 /*
 185  * Sync point threshold interrupt service thread function
 186  * Handles sync point threshold triggers, in thread context
 187  */
 188
 189 static void syncpt_thresh_work(struct work_struct *work)
 190 {
 191         struct host1x_syncpt_intr *syncpt_intr =
 192                 container_of(work, struct host1x_syncpt_intr, work);
 193         struct host1x_syncpt *syncpt =
 194                 container_of(syncpt_intr, struct host1x_syncpt, intr);
 195         unsigned int id = syncpt->id;
 196         struct host1x *host = syncpt->host;
 197
 198         (void)process_wait_list(host, syncpt,
 199                                 host1x_syncpt_load(host->syncpt + id));
 200 }
 201
 202 int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt,
 203                            u32 thresh, enum host1x_intr_action action,
 204                            void *data, struct host1x_waitlist *waiter,
 205                            void **ref)
 206 {
 207         int queue_was_empty;
 208
 209         if (waiter == NULL) {
 210                 pr_warn("%s: NULL waiter\n", __func__);
 211                 return -EINVAL;
 212         }
 213
 214         /* initialize a new waiter */
 215         INIT_LIST_HEAD(&waiter->list);
 216         kref_init(&waiter->refcount);
 217         if (ref)
 218                 kref_get(&waiter->refcount);
 219         waiter->thresh = thresh;
 220         waiter->action = action;
 221         atomic_set(&waiter->state, WLS_PENDING);
 222         waiter->data = data;
 223         waiter->count = 1;
 224
 225         spin_lock(&syncpt->intr.lock);
 226
 227         queue_was_empty = list_empty(&syncpt->intr.wait_head);
 228
 229         if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) {
 230                 /* added at head of list - new threshold value */
 231                 host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh);
 232
 233                 /* added as first waiter - enable interrupt */
 234                 if (queue_was_empty)
 235                         host1x_hw_intr_enable_syncpt_intr(host, syncpt->id);
 236         }
 237
 238         if (ref)
 239                 *ref = waiter;
 240
 241         spin_unlock(&syncpt->intr.lock);
 242
 243         return 0;
 244 }
 245
 246 void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref,
 247                          bool flush)
 248 {
 249         struct host1x_waitlist *waiter = ref;
 250         struct host1x_syncpt *syncpt;
 251
 252         atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED);
 253
 254         syncpt = host->syncpt + id;
 255
 256         spin_lock(&syncpt->intr.lock);
 257         if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) ==
 258             WLS_CANCELLED) {
 259                 list_del(&waiter->list);
 260                 kref_put(&waiter->refcount, waiter_release);
 261         }
 262         spin_unlock(&syncpt->intr.lock);
 263
 264         if (flush) {
 265                 /* Wait until any concurrently executing handler has finished. */
 266                 while (atomic_read(&waiter->state) != WLS_HANDLED)
 267                         schedule();
 268         }
 269
 270         kref_put(&waiter->refcount, waiter_release);
 271 }
 272
 273 int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
 274 {
 275         unsigned int id;
 276         u32 nb_pts = host1x_syncpt_nb_pts(host);
 277
 278         mutex_init(&host->intr_mutex);
 279         host->intr_syncpt_irq = irq_sync;
 280
 281         for (id = 0; id < nb_pts; ++id) {
 282                 struct host1x_syncpt *syncpt = host->syncpt + id;
 283
 284                 spin_lock_init(&syncpt->intr.lock);
 285                 INIT_LIST_HEAD(&syncpt->intr.wait_head);
 286                 snprintf(syncpt->intr.thresh_irq_name,
 287                          sizeof(syncpt->intr.thresh_irq_name),
 288                          "host1x_sp_%02u", id);
 289         }
 290
 291         host1x_intr_start(host);
 292
 293         return 0;
 294 }
 295
 296 void host1x_intr_deinit(struct host1x *host)
 297 {
 298         host1x_intr_stop(host);
 299 }
 300
 301 void host1x_intr_start(struct host1x *host)
 302 {
 303         u32 hz = clk_get_rate(host->clk);
 304         int err;
 305
 306         mutex_lock(&host->intr_mutex);
 307         err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000),
 308                                             syncpt_thresh_work);
 309         if (err) {
 310                 mutex_unlock(&host->intr_mutex);
 311                 return;
 312         }
 313         mutex_unlock(&host->intr_mutex);
 314 }
 315
 316 void host1x_intr_stop(struct host1x *host)
 317 {
 318         unsigned int id;
 319         struct host1x_syncpt *syncpt = host->syncpt;
 320         u32 nb_pts = host1x_syncpt_nb_pts(host);
 321
 322         mutex_lock(&host->intr_mutex);
 323
 324         host1x_hw_intr_disable_all_syncpt_intrs(host);
 325
 326         for (id = 0; id < nb_pts; ++id) {
 327                 struct host1x_waitlist *waiter, *next;
 328
 329                 list_for_each_entry_safe(waiter, next,
 330                         &syncpt[id].intr.wait_head, list) {
 331                         if (atomic_cmpxchg(&waiter->state,
 332                             WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) {
 333                                 list_del(&waiter->list);
 334                                 kref_put(&waiter->refcount, waiter_release);
 335                         }
 336                 }
 337
 338                 if (!list_empty(&syncpt[id].intr.wait_head)) {
 339                         /* output diagnostics */
 340                         mutex_unlock(&host->intr_mutex);
 341                         pr_warn("%s cannot stop syncpt intr id=%u\n",
 342                                 __func__, id);
 343                         return;
 344                 }
 345         }
 346
 347         host1x_hw_intr_free_syncpt_irq(host);
 348
 349         mutex_unlock(&host->intr_mutex);
 350 }