drivers/mmc/host/mmc_hsq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *
   4  * MMC software queue support based on command queue interfaces
   5  *
   6  * Copyright (C) 2019 Linaro, Inc.
   7  * Author: Baolin Wang <baolin.wang@linaro.org>
   8  */
   9
  10 #include <linux/mmc/card.h>
  11 #include <linux/mmc/host.h>
  12 #include <linux/module.h>
  13
  14 #include "mmc_hsq.h"
  15
  16 #define HSQ_NUM_SLOTS   64
  17 #define HSQ_INVALID_TAG HSQ_NUM_SLOTS
  18
  19 static void mmc_hsq_retry_handler(struct work_struct *work)
  20 {
  21         struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);
  22         struct mmc_host *mmc = hsq->mmc;
  23
  24         mmc->ops->request(mmc, hsq->mrq);
  25 }
  26
  27 static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)
  28 {
  29         struct mmc_host *mmc = hsq->mmc;
  30         struct hsq_slot *slot;
  31         unsigned long flags;
  32         int ret = 0;
  33
  34         spin_lock_irqsave(&hsq->lock, flags);
  35
  36         /* Make sure we are not already running a request now */
  37         if (hsq->mrq) {
  38                 spin_unlock_irqrestore(&hsq->lock, flags);
  39                 return;
  40         }
  41
  42         /* Make sure there are remain requests need to pump */
  43         if (!hsq->qcnt || !hsq->enabled) {
  44                 spin_unlock_irqrestore(&hsq->lock, flags);
  45                 return;
  46         }
  47
  48         slot = &hsq->slot[hsq->next_tag];
  49         hsq->mrq = slot->mrq;
  50         hsq->qcnt--;
  51
  52         spin_unlock_irqrestore(&hsq->lock, flags);
  53
  54         if (mmc->ops->request_atomic)
  55                 ret = mmc->ops->request_atomic(mmc, hsq->mrq);
  56         else
  57                 mmc->ops->request(mmc, hsq->mrq);
  58
  59         /*
  60          * If returning BUSY from request_atomic(), which means the card
  61          * may be busy now, and we should change to non-atomic context to
  62          * try again for this unusual case, to avoid time-consuming operations
  63          * in the atomic context.
  64          *
  65          * Note: we just give a warning for other error cases, since the host
  66          * driver will handle them.
  67          */
  68         if (ret == -EBUSY)
  69                 schedule_work(&hsq->retry_work);
  70         else
  71                 WARN_ON_ONCE(ret);
  72 }
  73
  74 static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)
  75 {
  76         struct hsq_slot *slot;
  77         int tag;
  78
  79         /*
  80          * If there are no remain requests in software queue, then set a invalid
  81          * tag.
  82          */
  83         if (!remains) {
  84                 hsq->next_tag = HSQ_INVALID_TAG;
  85                 return;
  86         }
  87
  88         /*
  89          * Increasing the next tag and check if the corresponding request is
  90          * available, if yes, then we found a candidate request.
  91          */
  92         if (++hsq->next_tag != HSQ_INVALID_TAG) {
  93                 slot = &hsq->slot[hsq->next_tag];
  94                 if (slot->mrq)
  95                         return;
  96         }
  97
  98         /* Othersie we should iterate all slots to find a available tag. */
  99         for (tag = 0; tag < HSQ_NUM_SLOTS; tag++) {
 100                 slot = &hsq->slot[tag];
 101                 if (slot->mrq)
 102                         break;
 103         }
 104
 105         if (tag == HSQ_NUM_SLOTS)
 106                 tag = HSQ_INVALID_TAG;
 107
 108         hsq->next_tag = tag;
 109 }
 110
 111 static void mmc_hsq_post_request(struct mmc_hsq *hsq)
 112 {
 113         unsigned long flags;
 114         int remains;
 115
 116         spin_lock_irqsave(&hsq->lock, flags);
 117
 118         remains = hsq->qcnt;
 119         hsq->mrq = NULL;
 120
 121         /* Update the next available tag to be queued. */
 122         mmc_hsq_update_next_tag(hsq, remains);
 123
 124         if (hsq->waiting_for_idle && !remains) {
 125                 hsq->waiting_for_idle = false;
 126                 wake_up(&hsq->wait_queue);
 127         }
 128
 129         /* Do not pump new request in recovery mode. */
 130         if (hsq->recovery_halt) {
 131                 spin_unlock_irqrestore(&hsq->lock, flags);
 132                 return;
 133         }
 134
 135         spin_unlock_irqrestore(&hsq->lock, flags);
 136
 137          /*
 138           * Try to pump new request to host controller as fast as possible,
 139           * after completing previous request.
 140           */
 141         if (remains > 0)
 142                 mmc_hsq_pump_requests(hsq);
 143 }
 144
 145 /**
 146  * mmc_hsq_finalize_request - finalize one request if the request is done
 147  * @mmc: the host controller
 148  * @mrq: the request need to be finalized
 149  *
 150  * Return true if we finalized the corresponding request in software queue,
 151  * otherwise return false.
 152  */
 153 bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq)
 154 {
 155         struct mmc_hsq *hsq = mmc->cqe_private;
 156         unsigned long flags;
 157
 158         spin_lock_irqsave(&hsq->lock, flags);
 159
 160         if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {
 161                 spin_unlock_irqrestore(&hsq->lock, flags);
 162                 return false;
 163         }
 164
 165         /*
 166          * Clear current completed slot request to make a room for new request.
 167          */
 168         hsq->slot[hsq->next_tag].mrq = NULL;
 169
 170         spin_unlock_irqrestore(&hsq->lock, flags);
 171
 172         mmc_cqe_request_done(mmc, hsq->mrq);
 173
 174         mmc_hsq_post_request(hsq);
 175
 176         return true;
 177 }
 178 EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request);
 179
 180 static void mmc_hsq_recovery_start(struct mmc_host *mmc)
 181 {
 182         struct mmc_hsq *hsq = mmc->cqe_private;
 183         unsigned long flags;
 184
 185         spin_lock_irqsave(&hsq->lock, flags);
 186
 187         hsq->recovery_halt = true;
 188
 189         spin_unlock_irqrestore(&hsq->lock, flags);
 190 }
 191
 192 static void mmc_hsq_recovery_finish(struct mmc_host *mmc)
 193 {
 194         struct mmc_hsq *hsq = mmc->cqe_private;
 195         int remains;
 196
 197         spin_lock_irq(&hsq->lock);
 198
 199         hsq->recovery_halt = false;
 200         remains = hsq->qcnt;
 201
 202         spin_unlock_irq(&hsq->lock);
 203
 204         /*
 205          * Try to pump new request if there are request pending in software
 206          * queue after finishing recovery.
 207          */
 208         if (remains > 0)
 209                 mmc_hsq_pump_requests(hsq);
 210 }
 211
 212 static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq)
 213 {
 214         struct mmc_hsq *hsq = mmc->cqe_private;
 215         int tag = mrq->tag;
 216
 217         spin_lock_irq(&hsq->lock);
 218
 219         if (!hsq->enabled) {
 220                 spin_unlock_irq(&hsq->lock);
 221                 return -ESHUTDOWN;
 222         }
 223
 224         /* Do not queue any new requests in recovery mode. */
 225         if (hsq->recovery_halt) {
 226                 spin_unlock_irq(&hsq->lock);
 227                 return -EBUSY;
 228         }
 229
 230         hsq->slot[tag].mrq = mrq;
 231
 232         /*
 233          * Set the next tag as current request tag if no available
 234          * next tag.
 235          */
 236         if (hsq->next_tag == HSQ_INVALID_TAG)
 237                 hsq->next_tag = tag;
 238
 239         hsq->qcnt++;
 240
 241         spin_unlock_irq(&hsq->lock);
 242
 243         mmc_hsq_pump_requests(hsq);
 244
 245         return 0;
 246 }
 247
 248 static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq)
 249 {
 250         if (mmc->ops->post_req)
 251                 mmc->ops->post_req(mmc, mrq, 0);
 252 }
 253
 254 static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret)
 255 {
 256         bool is_idle;
 257
 258         spin_lock_irq(&hsq->lock);
 259
 260         is_idle = (!hsq->mrq && !hsq->qcnt) ||
 261                 hsq->recovery_halt;
 262
 263         *ret = hsq->recovery_halt ? -EBUSY : 0;
 264         hsq->waiting_for_idle = !is_idle;
 265
 266         spin_unlock_irq(&hsq->lock);
 267
 268         return is_idle;
 269 }
 270
 271 static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)
 272 {
 273         struct mmc_hsq *hsq = mmc->cqe_private;
 274         int ret;
 275
 276         wait_event(hsq->wait_queue,
 277                    mmc_hsq_queue_is_idle(hsq, &ret));
 278
 279         return ret;
 280 }
 281
 282 static void mmc_hsq_disable(struct mmc_host *mmc)
 283 {
 284         struct mmc_hsq *hsq = mmc->cqe_private;
 285         u32 timeout = 500;
 286         int ret;
 287
 288         spin_lock_irq(&hsq->lock);
 289
 290         if (!hsq->enabled) {
 291                 spin_unlock_irq(&hsq->lock);
 292                 return;
 293         }
 294
 295         spin_unlock_irq(&hsq->lock);
 296
 297         ret = wait_event_timeout(hsq->wait_queue,
 298                                  mmc_hsq_queue_is_idle(hsq, &ret),
 299                                  msecs_to_jiffies(timeout));
 300         if (ret == 0) {
 301                 pr_warn("could not stop mmc software queue\n");
 302                 return;
 303         }
 304
 305         spin_lock_irq(&hsq->lock);
 306
 307         hsq->enabled = false;
 308
 309         spin_unlock_irq(&hsq->lock);
 310 }
 311
 312 static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card)
 313 {
 314         struct mmc_hsq *hsq = mmc->cqe_private;
 315
 316         spin_lock_irq(&hsq->lock);
 317
 318         if (hsq->enabled) {
 319                 spin_unlock_irq(&hsq->lock);
 320                 return -EBUSY;
 321         }
 322
 323         hsq->enabled = true;
 324
 325         spin_unlock_irq(&hsq->lock);
 326
 327         return 0;
 328 }
 329
 330 static const struct mmc_cqe_ops mmc_hsq_ops = {
 331         .cqe_enable = mmc_hsq_enable,
 332         .cqe_disable = mmc_hsq_disable,
 333         .cqe_request = mmc_hsq_request,
 334         .cqe_post_req = mmc_hsq_post_req,
 335         .cqe_wait_for_idle = mmc_hsq_wait_for_idle,
 336         .cqe_recovery_start = mmc_hsq_recovery_start,
 337         .cqe_recovery_finish = mmc_hsq_recovery_finish,
 338 };
 339
 340 int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)
 341 {
 342         hsq->num_slots = HSQ_NUM_SLOTS;
 343         hsq->next_tag = HSQ_INVALID_TAG;
 344
 345         hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,
 346                                  sizeof(struct hsq_slot), GFP_KERNEL);
 347         if (!hsq->slot)
 348                 return -ENOMEM;
 349
 350         hsq->mmc = mmc;
 351         hsq->mmc->cqe_private = hsq;
 352         mmc->cqe_ops = &mmc_hsq_ops;
 353
 354         INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);
 355         spin_lock_init(&hsq->lock);
 356         init_waitqueue_head(&hsq->wait_queue);
 357
 358         return 0;
 359 }
 360 EXPORT_SYMBOL_GPL(mmc_hsq_init);
 361
 362 void mmc_hsq_suspend(struct mmc_host *mmc)
 363 {
 364         mmc_hsq_disable(mmc);
 365 }
 366 EXPORT_SYMBOL_GPL(mmc_hsq_suspend);
 367
 368 int mmc_hsq_resume(struct mmc_host *mmc)
 369 {
 370         return mmc_hsq_enable(mmc, NULL);
 371 }
 372 EXPORT_SYMBOL_GPL(mmc_hsq_resume);
 373
 374 MODULE_DESCRIPTION("MMC Host Software Queue support");
 375 MODULE_LICENSE("GPL v2");