crypto/crypto_engine.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Handle async block request by crypto hardware engine.
   4  *
   5  * Copyright (C) 2016 Linaro, Inc.
   6  *
   7  * Author: Baolin Wang <baolin.wang@linaro.org>
   8  */
   9
  10 #include <linux/err.h>
  11 #include <linux/delay.h>
  12 #include <linux/device.h>
  13 #include <crypto/engine.h>
  14 #include <uapi/linux/sched/types.h>
  15 #include "internal.h"
  16
  17 #define CRYPTO_ENGINE_MAX_QLEN 10
  18
  19 /**
  20  * crypto_finalize_request - finalize one request if the request is done
  21  * @engine: the hardware engine
  22  * @req: the request need to be finalized
  23  * @err: error number
  24  */
  25 static void crypto_finalize_request(struct crypto_engine *engine,
  26                                     struct crypto_async_request *req, int err)
  27 {
  28         unsigned long flags;
  29         bool finalize_req = false;
  30         int ret;
  31         struct crypto_engine_ctx *enginectx;
  32
  33         /*
  34          * If hardware cannot enqueue more requests
  35          * and retry mechanism is not supported
  36          * make sure we are completing the current request
  37          */
  38         if (!engine->retry_support) {
  39                 spin_lock_irqsave(&engine->queue_lock, flags);
  40                 if (engine->cur_req == req) {
  41                         finalize_req = true;
  42                         engine->cur_req = NULL;
  43                 }
  44                 spin_unlock_irqrestore(&engine->queue_lock, flags);
  45         }
  46
  47         if (finalize_req || engine->retry_support) {
  48                 enginectx = crypto_tfm_ctx(req->tfm);
  49                 if (enginectx->op.prepare_request &&
  50                     enginectx->op.unprepare_request) {
  51                         ret = enginectx->op.unprepare_request(engine, req);
  52                         if (ret)
  53                                 dev_err(engine->dev, "failed to unprepare request\n");
  54                 }
  55         }
  56         req->complete(req, err);
  57
  58         kthread_queue_work(engine->kworker, &engine->pump_requests);
  59 }
  60
  61 /**
  62  * crypto_pump_requests - dequeue one request from engine queue to process
  63  * @engine: the hardware engine
  64  * @in_kthread: true if we are in the context of the request pump thread
  65  *
  66  * This function checks if there is any request in the engine queue that
  67  * needs processing and if so call out to the driver to initialize hardware
  68  * and handle each request.
  69  */
  70 static void crypto_pump_requests(struct crypto_engine *engine,
  71                                  bool in_kthread)
  72 {
  73         struct crypto_async_request *async_req, *backlog;
  74         unsigned long flags;
  75         bool was_busy = false;
  76         int ret;
  77         struct crypto_engine_ctx *enginectx;
  78
  79         spin_lock_irqsave(&engine->queue_lock, flags);
  80
  81         /* Make sure we are not already running a request */
  82         if (!engine->retry_support && engine->cur_req)
  83                 goto out;
  84
  85         /* If another context is idling then defer */
  86         if (engine->idling) {
  87                 kthread_queue_work(engine->kworker, &engine->pump_requests);
  88                 goto out;
  89         }
  90
  91         /* Check if the engine queue is idle */
  92         if (!crypto_queue_len(&engine->queue) || !engine->running) {
  93                 if (!engine->busy)
  94                         goto out;
  95
  96                 /* Only do teardown in the thread */
  97                 if (!in_kthread) {
  98                         kthread_queue_work(engine->kworker,
  99                                            &engine->pump_requests);
 100                         goto out;
 101                 }
 102
 103                 engine->busy = false;
 104                 engine->idling = true;
 105                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 106
 107                 if (engine->unprepare_crypt_hardware &&
 108                     engine->unprepare_crypt_hardware(engine))
 109                         dev_err(engine->dev, "failed to unprepare crypt hardware\n");
 110
 111                 spin_lock_irqsave(&engine->queue_lock, flags);
 112                 engine->idling = false;
 113                 goto out;
 114         }
 115
 116 start_request:
 117         /* Get the fist request from the engine queue to handle */
 118         backlog = crypto_get_backlog(&engine->queue);
 119         async_req = crypto_dequeue_request(&engine->queue);
 120         if (!async_req)
 121                 goto out;
 122
 123         /*
 124          * If hardware doesn't support the retry mechanism,
 125          * keep track of the request we are processing now.
 126          * We'll need it on completion (crypto_finalize_request).
 127          */
 128         if (!engine->retry_support)
 129                 engine->cur_req = async_req;
 130
 131         if (backlog)
 132                 backlog->complete(backlog, -EINPROGRESS);
 133
 134         if (engine->busy)
 135                 was_busy = true;
 136         else
 137                 engine->busy = true;
 138
 139         spin_unlock_irqrestore(&engine->queue_lock, flags);
 140
 141         /* Until here we get the request need to be encrypted successfully */
 142         if (!was_busy && engine->prepare_crypt_hardware) {
 143                 ret = engine->prepare_crypt_hardware(engine);
 144                 if (ret) {
 145                         dev_err(engine->dev, "failed to prepare crypt hardware\n");
 146                         goto req_err_2;
 147                 }
 148         }
 149
 150         enginectx = crypto_tfm_ctx(async_req->tfm);
 151
 152         if (enginectx->op.prepare_request) {
 153                 ret = enginectx->op.prepare_request(engine, async_req);
 154                 if (ret) {
 155                         dev_err(engine->dev, "failed to prepare request: %d\n",
 156                                 ret);
 157                         goto req_err_2;
 158                 }
 159         }
 160         if (!enginectx->op.do_one_request) {
 161                 dev_err(engine->dev, "failed to do request\n");
 162                 ret = -EINVAL;
 163                 goto req_err_1;
 164         }
 165
 166         ret = enginectx->op.do_one_request(engine, async_req);
 167
 168         /* Request unsuccessfully executed by hardware */
 169         if (ret < 0) {
 170                 /*
 171                  * If hardware queue is full (-ENOSPC), requeue request
 172                  * regardless of backlog flag.
 173                  * Otherwise, unprepare and complete the request.
 174                  */
 175                 if (!engine->retry_support ||
 176                     (ret != -ENOSPC)) {
 177                         dev_err(engine->dev,
 178                                 "Failed to do one request from queue: %d\n",
 179                                 ret);
 180                         goto req_err_1;
 181                 }
 182                 /*
 183                  * If retry mechanism is supported,
 184                  * unprepare current request and
 185                  * enqueue it back into crypto-engine queue.
 186                  */
 187                 if (enginectx->op.unprepare_request) {
 188                         ret = enginectx->op.unprepare_request(engine,
 189                                                               async_req);
 190                         if (ret)
 191                                 dev_err(engine->dev,
 192                                         "failed to unprepare request\n");
 193                 }
 194                 spin_lock_irqsave(&engine->queue_lock, flags);
 195                 /*
 196                  * If hardware was unable to execute request, enqueue it
 197                  * back in front of crypto-engine queue, to keep the order
 198                  * of requests.
 199                  */
 200                 crypto_enqueue_request_head(&engine->queue, async_req);
 201
 202                 kthread_queue_work(engine->kworker, &engine->pump_requests);
 203                 goto out;
 204         }
 205
 206         goto retry;
 207
 208 req_err_1:
 209         if (enginectx->op.unprepare_request) {
 210                 ret = enginectx->op.unprepare_request(engine, async_req);
 211                 if (ret)
 212                         dev_err(engine->dev, "failed to unprepare request\n");
 213         }
 214
 215 req_err_2:
 216         async_req->complete(async_req, ret);
 217
 218 retry:
 219         /* If retry mechanism is supported, send new requests to engine */
 220         if (engine->retry_support) {
 221                 spin_lock_irqsave(&engine->queue_lock, flags);
 222                 goto start_request;
 223         }
 224         return;
 225
 226 out:
 227         spin_unlock_irqrestore(&engine->queue_lock, flags);
 228
 229         /*
 230          * Batch requests is possible only if
 231          * hardware can enqueue multiple requests
 232          */
 233         if (engine->do_batch_requests) {
 234                 ret = engine->do_batch_requests(engine);
 235                 if (ret)
 236                         dev_err(engine->dev, "failed to do batch requests: %d\n",
 237                                 ret);
 238         }
 239
 240         return;
 241 }
 242
 243 static void crypto_pump_work(struct kthread_work *work)
 244 {
 245         struct crypto_engine *engine =
 246                 container_of(work, struct crypto_engine, pump_requests);
 247
 248         crypto_pump_requests(engine, true);
 249 }
 250
 251 /**
 252  * crypto_transfer_request - transfer the new request into the engine queue
 253  * @engine: the hardware engine
 254  * @req: the request need to be listed into the engine queue
 255  */
 256 static int crypto_transfer_request(struct crypto_engine *engine,
 257                                    struct crypto_async_request *req,
 258                                    bool need_pump)
 259 {
 260         unsigned long flags;
 261         int ret;
 262
 263         spin_lock_irqsave(&engine->queue_lock, flags);
 264
 265         if (!engine->running) {
 266                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 267                 return -ESHUTDOWN;
 268         }
 269
 270         ret = crypto_enqueue_request(&engine->queue, req);
 271
 272         if (!engine->busy && need_pump)
 273                 kthread_queue_work(engine->kworker, &engine->pump_requests);
 274
 275         spin_unlock_irqrestore(&engine->queue_lock, flags);
 276         return ret;
 277 }
 278
 279 /**
 280  * crypto_transfer_request_to_engine - transfer one request to list
 281  * into the engine queue
 282  * @engine: the hardware engine
 283  * @req: the request need to be listed into the engine queue
 284  */
 285 static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
 286                                              struct crypto_async_request *req)
 287 {
 288         return crypto_transfer_request(engine, req, true);
 289 }
 290
 291 /**
 292  * crypto_transfer_aead_request_to_engine - transfer one aead_request
 293  * to list into the engine queue
 294  * @engine: the hardware engine
 295  * @req: the request need to be listed into the engine queue
 296  */
 297 int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
 298                                            struct aead_request *req)
 299 {
 300         return crypto_transfer_request_to_engine(engine, &req->base);
 301 }
 302 EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
 303
 304 /**
 305  * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
 306  * to list into the engine queue
 307  * @engine: the hardware engine
 308  * @req: the request need to be listed into the engine queue
 309  */
 310 int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
 311                                                struct akcipher_request *req)
 312 {
 313         return crypto_transfer_request_to_engine(engine, &req->base);
 314 }
 315 EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
 316
 317 /**
 318  * crypto_transfer_hash_request_to_engine - transfer one ahash_request
 319  * to list into the engine queue
 320  * @engine: the hardware engine
 321  * @req: the request need to be listed into the engine queue
 322  */
 323 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
 324                                            struct ahash_request *req)
 325 {
 326         return crypto_transfer_request_to_engine(engine, &req->base);
 327 }
 328 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
 329
 330 /**
 331  * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
 332  * to list into the engine queue
 333  * @engine: the hardware engine
 334  * @req: the request need to be listed into the engine queue
 335  */
 336 int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
 337                                                struct skcipher_request *req)
 338 {
 339         return crypto_transfer_request_to_engine(engine, &req->base);
 340 }
 341 EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
 342
 343 /**
 344  * crypto_finalize_aead_request - finalize one aead_request if
 345  * the request is done
 346  * @engine: the hardware engine
 347  * @req: the request need to be finalized
 348  * @err: error number
 349  */
 350 void crypto_finalize_aead_request(struct crypto_engine *engine,
 351                                   struct aead_request *req, int err)
 352 {
 353         return crypto_finalize_request(engine, &req->base, err);
 354 }
 355 EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
 356
 357 /**
 358  * crypto_finalize_akcipher_request - finalize one akcipher_request if
 359  * the request is done
 360  * @engine: the hardware engine
 361  * @req: the request need to be finalized
 362  * @err: error number
 363  */
 364 void crypto_finalize_akcipher_request(struct crypto_engine *engine,
 365                                       struct akcipher_request *req, int err)
 366 {
 367         return crypto_finalize_request(engine, &req->base, err);
 368 }
 369 EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
 370
 371 /**
 372  * crypto_finalize_hash_request - finalize one ahash_request if
 373  * the request is done
 374  * @engine: the hardware engine
 375  * @req: the request need to be finalized
 376  * @err: error number
 377  */
 378 void crypto_finalize_hash_request(struct crypto_engine *engine,
 379                                   struct ahash_request *req, int err)
 380 {
 381         return crypto_finalize_request(engine, &req->base, err);
 382 }
 383 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
 384
 385 /**
 386  * crypto_finalize_skcipher_request - finalize one skcipher_request if
 387  * the request is done
 388  * @engine: the hardware engine
 389  * @req: the request need to be finalized
 390  * @err: error number
 391  */
 392 void crypto_finalize_skcipher_request(struct crypto_engine *engine,
 393                                       struct skcipher_request *req, int err)
 394 {
 395         return crypto_finalize_request(engine, &req->base, err);
 396 }
 397 EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
 398
 399 /**
 400  * crypto_engine_start - start the hardware engine
 401  * @engine: the hardware engine need to be started
 402  *
 403  * Return 0 on success, else on fail.
 404  */
 405 int crypto_engine_start(struct crypto_engine *engine)
 406 {
 407         unsigned long flags;
 408
 409         spin_lock_irqsave(&engine->queue_lock, flags);
 410
 411         if (engine->running || engine->busy) {
 412                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 413                 return -EBUSY;
 414         }
 415
 416         engine->running = true;
 417         spin_unlock_irqrestore(&engine->queue_lock, flags);
 418
 419         kthread_queue_work(engine->kworker, &engine->pump_requests);
 420
 421         return 0;
 422 }
 423 EXPORT_SYMBOL_GPL(crypto_engine_start);
 424
 425 /**
 426  * crypto_engine_stop - stop the hardware engine
 427  * @engine: the hardware engine need to be stopped
 428  *
 429  * Return 0 on success, else on fail.
 430  */
 431 int crypto_engine_stop(struct crypto_engine *engine)
 432 {
 433         unsigned long flags;
 434         unsigned int limit = 500;
 435         int ret = 0;
 436
 437         spin_lock_irqsave(&engine->queue_lock, flags);
 438
 439         /*
 440          * If the engine queue is not empty or the engine is on busy state,
 441          * we need to wait for a while to pump the requests of engine queue.
 442          */
 443         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
 444                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 445                 msleep(20);
 446                 spin_lock_irqsave(&engine->queue_lock, flags);
 447         }
 448
 449         if (crypto_queue_len(&engine->queue) || engine->busy)
 450                 ret = -EBUSY;
 451         else
 452                 engine->running = false;
 453
 454         spin_unlock_irqrestore(&engine->queue_lock, flags);
 455
 456         if (ret)
 457                 dev_warn(engine->dev, "could not stop engine\n");
 458
 459         return ret;
 460 }
 461 EXPORT_SYMBOL_GPL(crypto_engine_stop);
 462
 463 /**
 464  * crypto_engine_alloc_init_and_set - allocate crypto hardware engine structure
 465  * and initialize it by setting the maximum number of entries in the software
 466  * crypto-engine queue.
 467  * @dev: the device attached with one hardware engine
 468  * @retry_support: whether hardware has support for retry mechanism
 469  * @cbk_do_batch: pointer to a callback function to be invoked when executing
 470  *                a batch of requests.
 471  *                This has the form:
 472  *                callback(struct crypto_engine *engine)
 473  *                where:
 474  *                @engine: the crypto engine structure.
 475  * @rt: whether this queue is set to run as a realtime task
 476  * @qlen: maximum size of the crypto-engine queue
 477  *
 478  * This must be called from context that can sleep.
 479  * Return: the crypto engine structure on success, else NULL.
 480  */
 481 struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
 482                                                        bool retry_support,
 483                                                        int (*cbk_do_batch)(struct crypto_engine *engine),
 484                                                        bool rt, int qlen)
 485 {
 486         struct crypto_engine *engine;
 487
 488         if (!dev)
 489                 return NULL;
 490
 491         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
 492         if (!engine)
 493                 return NULL;
 494
 495         engine->dev = dev;
 496         engine->rt = rt;
 497         engine->running = false;
 498         engine->busy = false;
 499         engine->idling = false;
 500         engine->retry_support = retry_support;
 501         engine->priv_data = dev;
 502         /*
 503          * Batch requests is possible only if
 504          * hardware has support for retry mechanism.
 505          */
 506         engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
 507
 508         snprintf(engine->name, sizeof(engine->name),
 509                  "%s-engine", dev_name(dev));
 510
 511         crypto_init_queue(&engine->queue, qlen);
 512         spin_lock_init(&engine->queue_lock);
 513
 514         engine->kworker = kthread_create_worker(0, "%s", engine->name);
 515         if (IS_ERR(engine->kworker)) {
 516                 dev_err(dev, "failed to create crypto request pump task\n");
 517                 return NULL;
 518         }
 519         kthread_init_work(&engine->pump_requests, crypto_pump_work);
 520
 521         if (engine->rt) {
 522                 dev_info(dev, "will run requests pump with realtime priority\n");
 523                 sched_set_fifo(engine->kworker->task);
 524         }
 525
 526         return engine;
 527 }
 528 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init_and_set);
 529
 530 /**
 531  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 532  * initialize it.
 533  * @dev: the device attached with one hardware engine
 534  * @rt: whether this queue is set to run as a realtime task
 535  *
 536  * This must be called from context that can sleep.
 537  * Return: the crypto engine structure on success, else NULL.
 538  */
 539 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
 540 {
 541         return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
 542                                                 CRYPTO_ENGINE_MAX_QLEN);
 543 }
 544 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
 545
 546 /**
 547  * crypto_engine_exit - free the resources of hardware engine when exit
 548  * @engine: the hardware engine need to be freed
 549  *
 550  * Return 0 for success.
 551  */
 552 int crypto_engine_exit(struct crypto_engine *engine)
 553 {
 554         int ret;
 555
 556         ret = crypto_engine_stop(engine);
 557         if (ret)
 558                 return ret;
 559
 560         kthread_destroy_worker(engine->kworker);
 561
 562         return 0;
 563 }
 564 EXPORT_SYMBOL_GPL(crypto_engine_exit);
 565
 566 MODULE_LICENSE("GPL");
 567 MODULE_DESCRIPTION("Crypto hardware engine framework");