Merge branch 'stable/for-linus-4.10' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6-microblaze.git] / crypto / crypto_engine.c
1 /*
2  * Handle async block request by crypto hardware engine.
3  *
4  * Copyright (C) 2016 Linaro, Inc.
5  *
6  * Author: Baolin Wang <baolin.wang@linaro.org>
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU General Public License as published by the Free
10  * Software Foundation; either version 2 of the License, or (at your option)
11  * any later version.
12  *
13  */
14
15 #include <linux/err.h>
16 #include <linux/delay.h>
17 #include <crypto/engine.h>
18 #include <crypto/internal/hash.h>
19 #include "internal.h"
20
21 #define CRYPTO_ENGINE_MAX_QLEN 10
22
23 /**
24  * crypto_pump_requests - dequeue one request from engine queue to process
25  * @engine: the hardware engine
26  * @in_kthread: true if we are in the context of the request pump thread
27  *
28  * This function checks if there is any request in the engine queue that
29  * needs processing and if so call out to the driver to initialize hardware
30  * and handle each request.
31  */
32 static void crypto_pump_requests(struct crypto_engine *engine,
33                                  bool in_kthread)
34 {
35         struct crypto_async_request *async_req, *backlog;
36         struct ahash_request *hreq;
37         struct ablkcipher_request *breq;
38         unsigned long flags;
39         bool was_busy = false;
40         int ret, rtype;
41
42         spin_lock_irqsave(&engine->queue_lock, flags);
43
44         /* Make sure we are not already running a request */
45         if (engine->cur_req)
46                 goto out;
47
48         /* If another context is idling then defer */
49         if (engine->idling) {
50                 kthread_queue_work(engine->kworker, &engine->pump_requests);
51                 goto out;
52         }
53
54         /* Check if the engine queue is idle */
55         if (!crypto_queue_len(&engine->queue) || !engine->running) {
56                 if (!engine->busy)
57                         goto out;
58
59                 /* Only do teardown in the thread */
60                 if (!in_kthread) {
61                         kthread_queue_work(engine->kworker,
62                                            &engine->pump_requests);
63                         goto out;
64                 }
65
66                 engine->busy = false;
67                 engine->idling = true;
68                 spin_unlock_irqrestore(&engine->queue_lock, flags);
69
70                 if (engine->unprepare_crypt_hardware &&
71                     engine->unprepare_crypt_hardware(engine))
72                         pr_err("failed to unprepare crypt hardware\n");
73
74                 spin_lock_irqsave(&engine->queue_lock, flags);
75                 engine->idling = false;
76                 goto out;
77         }
78
79         /* Get the fist request from the engine queue to handle */
80         backlog = crypto_get_backlog(&engine->queue);
81         async_req = crypto_dequeue_request(&engine->queue);
82         if (!async_req)
83                 goto out;
84
85         engine->cur_req = async_req;
86         if (backlog)
87                 backlog->complete(backlog, -EINPROGRESS);
88
89         if (engine->busy)
90                 was_busy = true;
91         else
92                 engine->busy = true;
93
94         spin_unlock_irqrestore(&engine->queue_lock, flags);
95
96         rtype = crypto_tfm_alg_type(engine->cur_req->tfm);
97         /* Until here we get the request need to be encrypted successfully */
98         if (!was_busy && engine->prepare_crypt_hardware) {
99                 ret = engine->prepare_crypt_hardware(engine);
100                 if (ret) {
101                         pr_err("failed to prepare crypt hardware\n");
102                         goto req_err;
103                 }
104         }
105
106         switch (rtype) {
107         case CRYPTO_ALG_TYPE_AHASH:
108                 hreq = ahash_request_cast(engine->cur_req);
109                 if (engine->prepare_hash_request) {
110                         ret = engine->prepare_hash_request(engine, hreq);
111                         if (ret) {
112                                 pr_err("failed to prepare request: %d\n", ret);
113                                 goto req_err;
114                         }
115                         engine->cur_req_prepared = true;
116                 }
117                 ret = engine->hash_one_request(engine, hreq);
118                 if (ret) {
119                         pr_err("failed to hash one request from queue\n");
120                         goto req_err;
121                 }
122                 return;
123         case CRYPTO_ALG_TYPE_ABLKCIPHER:
124                 breq = ablkcipher_request_cast(engine->cur_req);
125                 if (engine->prepare_cipher_request) {
126                         ret = engine->prepare_cipher_request(engine, breq);
127                         if (ret) {
128                                 pr_err("failed to prepare request: %d\n", ret);
129                                 goto req_err;
130                         }
131                         engine->cur_req_prepared = true;
132                 }
133                 ret = engine->cipher_one_request(engine, breq);
134                 if (ret) {
135                         pr_err("failed to cipher one request from queue\n");
136                         goto req_err;
137                 }
138                 return;
139         default:
140                 pr_err("failed to prepare request of unknown type\n");
141                 return;
142         }
143
144 req_err:
145         switch (rtype) {
146         case CRYPTO_ALG_TYPE_AHASH:
147                 hreq = ahash_request_cast(engine->cur_req);
148                 crypto_finalize_hash_request(engine, hreq, ret);
149                 break;
150         case CRYPTO_ALG_TYPE_ABLKCIPHER:
151                 breq = ablkcipher_request_cast(engine->cur_req);
152                 crypto_finalize_cipher_request(engine, breq, ret);
153                 break;
154         }
155         return;
156
157 out:
158         spin_unlock_irqrestore(&engine->queue_lock, flags);
159 }
160
161 static void crypto_pump_work(struct kthread_work *work)
162 {
163         struct crypto_engine *engine =
164                 container_of(work, struct crypto_engine, pump_requests);
165
166         crypto_pump_requests(engine, true);
167 }
168
169 /**
170  * crypto_transfer_cipher_request - transfer the new request into the
171  * enginequeue
172  * @engine: the hardware engine
173  * @req: the request need to be listed into the engine queue
174  */
175 int crypto_transfer_cipher_request(struct crypto_engine *engine,
176                                    struct ablkcipher_request *req,
177                                    bool need_pump)
178 {
179         unsigned long flags;
180         int ret;
181
182         spin_lock_irqsave(&engine->queue_lock, flags);
183
184         if (!engine->running) {
185                 spin_unlock_irqrestore(&engine->queue_lock, flags);
186                 return -ESHUTDOWN;
187         }
188
189         ret = ablkcipher_enqueue_request(&engine->queue, req);
190
191         if (!engine->busy && need_pump)
192                 kthread_queue_work(engine->kworker, &engine->pump_requests);
193
194         spin_unlock_irqrestore(&engine->queue_lock, flags);
195         return ret;
196 }
197 EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request);
198
199 /**
200  * crypto_transfer_cipher_request_to_engine - transfer one request to list
201  * into the engine queue
202  * @engine: the hardware engine
203  * @req: the request need to be listed into the engine queue
204  */
205 int crypto_transfer_cipher_request_to_engine(struct crypto_engine *engine,
206                                              struct ablkcipher_request *req)
207 {
208         return crypto_transfer_cipher_request(engine, req, true);
209 }
210 EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request_to_engine);
211
212 /**
213  * crypto_transfer_hash_request - transfer the new request into the
214  * enginequeue
215  * @engine: the hardware engine
216  * @req: the request need to be listed into the engine queue
217  */
218 int crypto_transfer_hash_request(struct crypto_engine *engine,
219                                  struct ahash_request *req, bool need_pump)
220 {
221         unsigned long flags;
222         int ret;
223
224         spin_lock_irqsave(&engine->queue_lock, flags);
225
226         if (!engine->running) {
227                 spin_unlock_irqrestore(&engine->queue_lock, flags);
228                 return -ESHUTDOWN;
229         }
230
231         ret = ahash_enqueue_request(&engine->queue, req);
232
233         if (!engine->busy && need_pump)
234                 kthread_queue_work(engine->kworker, &engine->pump_requests);
235
236         spin_unlock_irqrestore(&engine->queue_lock, flags);
237         return ret;
238 }
239 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request);
240
241 /**
242  * crypto_transfer_hash_request_to_engine - transfer one request to list
243  * into the engine queue
244  * @engine: the hardware engine
245  * @req: the request need to be listed into the engine queue
246  */
247 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
248                                            struct ahash_request *req)
249 {
250         return crypto_transfer_hash_request(engine, req, true);
251 }
252 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
253
254 /**
255  * crypto_finalize_cipher_request - finalize one request if the request is done
256  * @engine: the hardware engine
257  * @req: the request need to be finalized
258  * @err: error number
259  */
260 void crypto_finalize_cipher_request(struct crypto_engine *engine,
261                                     struct ablkcipher_request *req, int err)
262 {
263         unsigned long flags;
264         bool finalize_cur_req = false;
265         int ret;
266
267         spin_lock_irqsave(&engine->queue_lock, flags);
268         if (engine->cur_req == &req->base)
269                 finalize_cur_req = true;
270         spin_unlock_irqrestore(&engine->queue_lock, flags);
271
272         if (finalize_cur_req) {
273                 if (engine->cur_req_prepared &&
274                     engine->unprepare_cipher_request) {
275                         ret = engine->unprepare_cipher_request(engine, req);
276                         if (ret)
277                                 pr_err("failed to unprepare request\n");
278                 }
279                 spin_lock_irqsave(&engine->queue_lock, flags);
280                 engine->cur_req = NULL;
281                 engine->cur_req_prepared = false;
282                 spin_unlock_irqrestore(&engine->queue_lock, flags);
283         }
284
285         req->base.complete(&req->base, err);
286
287         kthread_queue_work(engine->kworker, &engine->pump_requests);
288 }
289 EXPORT_SYMBOL_GPL(crypto_finalize_cipher_request);
290
291 /**
292  * crypto_finalize_hash_request - finalize one request if the request is done
293  * @engine: the hardware engine
294  * @req: the request need to be finalized
295  * @err: error number
296  */
297 void crypto_finalize_hash_request(struct crypto_engine *engine,
298                                   struct ahash_request *req, int err)
299 {
300         unsigned long flags;
301         bool finalize_cur_req = false;
302         int ret;
303
304         spin_lock_irqsave(&engine->queue_lock, flags);
305         if (engine->cur_req == &req->base)
306                 finalize_cur_req = true;
307         spin_unlock_irqrestore(&engine->queue_lock, flags);
308
309         if (finalize_cur_req) {
310                 if (engine->cur_req_prepared &&
311                     engine->unprepare_hash_request) {
312                         ret = engine->unprepare_hash_request(engine, req);
313                         if (ret)
314                                 pr_err("failed to unprepare request\n");
315                 }
316                 spin_lock_irqsave(&engine->queue_lock, flags);
317                 engine->cur_req = NULL;
318                 engine->cur_req_prepared = false;
319                 spin_unlock_irqrestore(&engine->queue_lock, flags);
320         }
321
322         req->base.complete(&req->base, err);
323
324         kthread_queue_work(engine->kworker, &engine->pump_requests);
325 }
326 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
327
328 /**
329  * crypto_engine_start - start the hardware engine
330  * @engine: the hardware engine need to be started
331  *
332  * Return 0 on success, else on fail.
333  */
334 int crypto_engine_start(struct crypto_engine *engine)
335 {
336         unsigned long flags;
337
338         spin_lock_irqsave(&engine->queue_lock, flags);
339
340         if (engine->running || engine->busy) {
341                 spin_unlock_irqrestore(&engine->queue_lock, flags);
342                 return -EBUSY;
343         }
344
345         engine->running = true;
346         spin_unlock_irqrestore(&engine->queue_lock, flags);
347
348         kthread_queue_work(engine->kworker, &engine->pump_requests);
349
350         return 0;
351 }
352 EXPORT_SYMBOL_GPL(crypto_engine_start);
353
354 /**
355  * crypto_engine_stop - stop the hardware engine
356  * @engine: the hardware engine need to be stopped
357  *
358  * Return 0 on success, else on fail.
359  */
360 int crypto_engine_stop(struct crypto_engine *engine)
361 {
362         unsigned long flags;
363         unsigned int limit = 500;
364         int ret = 0;
365
366         spin_lock_irqsave(&engine->queue_lock, flags);
367
368         /*
369          * If the engine queue is not empty or the engine is on busy state,
370          * we need to wait for a while to pump the requests of engine queue.
371          */
372         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
373                 spin_unlock_irqrestore(&engine->queue_lock, flags);
374                 msleep(20);
375                 spin_lock_irqsave(&engine->queue_lock, flags);
376         }
377
378         if (crypto_queue_len(&engine->queue) || engine->busy)
379                 ret = -EBUSY;
380         else
381                 engine->running = false;
382
383         spin_unlock_irqrestore(&engine->queue_lock, flags);
384
385         if (ret)
386                 pr_warn("could not stop engine\n");
387
388         return ret;
389 }
390 EXPORT_SYMBOL_GPL(crypto_engine_stop);
391
392 /**
393  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
394  * initialize it.
395  * @dev: the device attached with one hardware engine
396  * @rt: whether this queue is set to run as a realtime task
397  *
398  * This must be called from context that can sleep.
399  * Return: the crypto engine structure on success, else NULL.
400  */
401 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
402 {
403         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
404         struct crypto_engine *engine;
405
406         if (!dev)
407                 return NULL;
408
409         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
410         if (!engine)
411                 return NULL;
412
413         engine->rt = rt;
414         engine->running = false;
415         engine->busy = false;
416         engine->idling = false;
417         engine->cur_req_prepared = false;
418         engine->priv_data = dev;
419         snprintf(engine->name, sizeof(engine->name),
420                  "%s-engine", dev_name(dev));
421
422         crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
423         spin_lock_init(&engine->queue_lock);
424
425         engine->kworker = kthread_create_worker(0, "%s", engine->name);
426         if (IS_ERR(engine->kworker)) {
427                 dev_err(dev, "failed to create crypto request pump task\n");
428                 return NULL;
429         }
430         kthread_init_work(&engine->pump_requests, crypto_pump_work);
431
432         if (engine->rt) {
433                 dev_info(dev, "will run requests pump with realtime priority\n");
434                 sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
435         }
436
437         return engine;
438 }
439 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
440
441 /**
442  * crypto_engine_exit - free the resources of hardware engine when exit
443  * @engine: the hardware engine need to be freed
444  *
445  * Return 0 for success.
446  */
447 int crypto_engine_exit(struct crypto_engine *engine)
448 {
449         int ret;
450
451         ret = crypto_engine_stop(engine);
452         if (ret)
453                 return ret;
454
455         kthread_destroy_worker(engine->kworker);
456
457         return 0;
458 }
459 EXPORT_SYMBOL_GPL(crypto_engine_exit);
460
461 MODULE_LICENSE("GPL");
462 MODULE_DESCRIPTION("Crypto hardware engine framework");