drivers/crypto/omap-aes-gcm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Cryptographic API.
   4  *
   5  * Support for OMAP AES GCM HW acceleration.
   6  *
   7  * Copyright (c) 2016 Texas Instruments Incorporated
   8  */
   9
  10 #include <linux/errno.h>
  11 #include <linux/scatterlist.h>
  12 #include <linux/dma-mapping.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/omap-dma.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/pm_runtime.h>
  17 #include <crypto/aes.h>
  18 #include <crypto/gcm.h>
  19 #include <crypto/scatterwalk.h>
  20 #include <crypto/skcipher.h>
  21 #include <crypto/internal/aead.h>
  22
  23 #include "omap-crypto.h"
  24 #include "omap-aes.h"
  25
  26 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
  27                                      struct aead_request *req);
  28
  29 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
  30 {
  31         struct aead_request *req = dd->aead_req;
  32
  33         dd->in_sg = NULL;
  34         dd->out_sg = NULL;
  35
  36         crypto_finalize_aead_request(dd->engine, req, ret);
  37
  38         pm_runtime_mark_last_busy(dd->dev);
  39         pm_runtime_put_autosuspend(dd->dev);
  40 }
  41
  42 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
  43 {
  44         u8 *tag;
  45         int alen, clen, i, ret = 0, nsg;
  46         struct omap_aes_reqctx *rctx;
  47
  48         alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
  49         clen = ALIGN(dd->total, AES_BLOCK_SIZE);
  50         rctx = aead_request_ctx(dd->aead_req);
  51
  52         nsg = !!(dd->assoc_len && dd->total);
  53
  54         dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
  55                                DMA_FROM_DEVICE);
  56         dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  57         dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
  58         omap_aes_crypt_dma_stop(dd);
  59
  60         omap_crypto_cleanup(dd->out_sg, dd->orig_out,
  61                             dd->aead_req->assoclen, dd->total,
  62                             FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
  63
  64         if (dd->flags & FLAGS_ENCRYPT)
  65                 scatterwalk_map_and_copy(rctx->auth_tag,
  66                                          dd->aead_req->dst,
  67                                          dd->total + dd->aead_req->assoclen,
  68                                          dd->authsize, 1);
  69
  70         omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
  71                             FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
  72
  73         omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
  74                             FLAGS_IN_DATA_ST_SHIFT, dd->flags);
  75
  76         if (!(dd->flags & FLAGS_ENCRYPT)) {
  77                 tag = (u8 *)rctx->auth_tag;
  78                 for (i = 0; i < dd->authsize; i++) {
  79                         if (tag[i]) {
  80                                 ret = -EBADMSG;
  81                         }
  82                 }
  83         }
  84
  85         omap_aes_gcm_finish_req(dd, ret);
  86 }
  87
  88 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
  89                                      struct aead_request *req)
  90 {
  91         int alen, clen, cryptlen, assoclen, ret;
  92         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  93         unsigned int authlen = crypto_aead_authsize(aead);
  94         struct scatterlist *tmp, sg_arr[2];
  95         int nsg;
  96         u16 flags;
  97
  98         assoclen = req->assoclen;
  99         cryptlen = req->cryptlen;
 100
 101         if (dd->flags & FLAGS_RFC4106_GCM)
 102                 assoclen -= 8;
 103
 104         if (!(dd->flags & FLAGS_ENCRYPT))
 105                 cryptlen -= authlen;
 106
 107         alen = ALIGN(assoclen, AES_BLOCK_SIZE);
 108         clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
 109
 110         nsg = !!(assoclen && cryptlen);
 111
 112         omap_aes_clear_copy_flags(dd);
 113
 114         sg_init_table(dd->in_sgl, nsg + 1);
 115         if (assoclen) {
 116                 tmp = req->src;
 117                 ret = omap_crypto_align_sg(&tmp, assoclen,
 118                                            AES_BLOCK_SIZE, dd->in_sgl,
 119                                            OMAP_CRYPTO_COPY_DATA |
 120                                            OMAP_CRYPTO_ZERO_BUF |
 121                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 122                                            FLAGS_ASSOC_DATA_ST_SHIFT,
 123                                            &dd->flags);
 124                 if (ret)
 125                         return ret;
 126         }
 127
 128         if (cryptlen) {
 129                 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
 130
 131                 if (nsg)
 132                         sg_unmark_end(dd->in_sgl);
 133
 134                 ret = omap_crypto_align_sg(&tmp, cryptlen,
 135                                            AES_BLOCK_SIZE, &dd->in_sgl[nsg],
 136                                            OMAP_CRYPTO_COPY_DATA |
 137                                            OMAP_CRYPTO_ZERO_BUF |
 138                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 139                                            FLAGS_IN_DATA_ST_SHIFT,
 140                                            &dd->flags);
 141                 if (ret)
 142                         return ret;
 143         }
 144
 145         dd->in_sg = dd->in_sgl;
 146         dd->total = cryptlen;
 147         dd->assoc_len = assoclen;
 148         dd->authsize = authlen;
 149
 150         dd->out_sg = req->dst;
 151         dd->orig_out = req->dst;
 152
 153         dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);
 154
 155         flags = 0;
 156         if (req->src == req->dst || dd->out_sg == sg_arr)
 157                 flags |= OMAP_CRYPTO_FORCE_COPY;
 158
 159         if (cryptlen) {
 160                 ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
 161                                            AES_BLOCK_SIZE, &dd->out_sgl,
 162                                            flags,
 163                                            FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
 164                 if (ret)
 165                         return ret;
 166         }
 167
 168         dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
 169         dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
 170
 171         return 0;
 172 }
 173
 174 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
 175 {
 176         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 177
 178         aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);
 179         return 0;
 180 }
 181
 182 void omap_aes_gcm_dma_out_callback(void *data)
 183 {
 184         struct omap_aes_dev *dd = data;
 185         struct omap_aes_reqctx *rctx;
 186         int i, val;
 187         u32 *auth_tag, tag[4];
 188
 189         if (!(dd->flags & FLAGS_ENCRYPT))
 190                 scatterwalk_map_and_copy(tag, dd->aead_req->src,
 191                                          dd->total + dd->aead_req->assoclen,
 192                                          dd->authsize, 0);
 193
 194         rctx = aead_request_ctx(dd->aead_req);
 195         auth_tag = (u32 *)rctx->auth_tag;
 196         for (i = 0; i < 4; i++) {
 197                 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
 198                 auth_tag[i] = val ^ auth_tag[i];
 199                 if (!(dd->flags & FLAGS_ENCRYPT))
 200                         auth_tag[i] = auth_tag[i] ^ tag[i];
 201         }
 202
 203         omap_aes_gcm_done_task(dd);
 204 }
 205
 206 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
 207                                      struct aead_request *req)
 208 {
 209         if (req)
 210                 return crypto_transfer_aead_request_to_engine(dd->engine, req);
 211
 212         return 0;
 213 }
 214
 215 static int omap_aes_gcm_prepare_req(struct crypto_engine *engine, void *areq)
 216 {
 217         struct aead_request *req = container_of(areq, struct aead_request,
 218                                                 base);
 219         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 220         struct omap_aes_dev *dd = rctx->dd;
 221         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 222         int err;
 223
 224         dd->aead_req = req;
 225
 226         rctx->mode &= FLAGS_MODE_MASK;
 227         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 228
 229         err = omap_aes_gcm_copy_buffers(dd, req);
 230         if (err)
 231                 return err;
 232
 233         dd->ctx = &ctx->octx;
 234
 235         return omap_aes_write_ctrl(dd);
 236 }
 237
 238 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
 239 {
 240         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 241         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 242         unsigned int authlen = crypto_aead_authsize(aead);
 243         struct omap_aes_dev *dd;
 244         __be32 counter = cpu_to_be32(1);
 245         int err, assoclen;
 246
 247         memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
 248         memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
 249
 250         err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
 251         if (err)
 252                 return err;
 253
 254         if (mode & FLAGS_RFC4106_GCM)
 255                 assoclen = req->assoclen - 8;
 256         else
 257                 assoclen = req->assoclen;
 258         if (assoclen + req->cryptlen == 0) {
 259                 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
 260                                          1);
 261                 return 0;
 262         }
 263
 264         dd = omap_aes_find_dev(rctx);
 265         if (!dd)
 266                 return -ENODEV;
 267         rctx->mode = mode;
 268
 269         return omap_aes_gcm_handle_queue(dd, req);
 270 }
 271
 272 int omap_aes_gcm_encrypt(struct aead_request *req)
 273 {
 274         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 275
 276         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 277         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
 278 }
 279
 280 int omap_aes_gcm_decrypt(struct aead_request *req)
 281 {
 282         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 283
 284         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 285         return omap_aes_gcm_crypt(req, FLAGS_GCM);
 286 }
 287
 288 int omap_aes_4106gcm_encrypt(struct aead_request *req)
 289 {
 290         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 291         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 292
 293         memcpy(rctx->iv, ctx->octx.nonce, 4);
 294         memcpy(rctx->iv + 4, req->iv, 8);
 295         return crypto_ipsec_check_assoclen(req->assoclen) ?:
 296                omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
 297                                   FLAGS_RFC4106_GCM);
 298 }
 299
 300 int omap_aes_4106gcm_decrypt(struct aead_request *req)
 301 {
 302         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 303         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 304
 305         memcpy(rctx->iv, ctx->octx.nonce, 4);
 306         memcpy(rctx->iv + 4, req->iv, 8);
 307         return crypto_ipsec_check_assoclen(req->assoclen) ?:
 308                omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
 309 }
 310
 311 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 312                         unsigned int keylen)
 313 {
 314         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 315         int ret;
 316
 317         ret = aes_expandkey(&ctx->actx, key, keylen);
 318         if (ret)
 319                 return ret;
 320
 321         memcpy(ctx->octx.key, key, keylen);
 322         ctx->octx.keylen = keylen;
 323
 324         return 0;
 325 }
 326
 327 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 328                             unsigned int keylen)
 329 {
 330         struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 331         int ret;
 332
 333         if (keylen < 4)
 334                 return -EINVAL;
 335         keylen -= 4;
 336
 337         ret = aes_expandkey(&ctx->actx, key, keylen);
 338         if (ret)
 339                 return ret;
 340
 341         memcpy(ctx->octx.key, key, keylen);
 342         memcpy(ctx->octx.nonce, key + keylen, 4);
 343         ctx->octx.keylen = keylen;
 344
 345         return 0;
 346 }
 347
 348 int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
 349 {
 350         return crypto_gcm_check_authsize(authsize);
 351 }
 352
 353 int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
 354                                  unsigned int authsize)
 355 {
 356         return crypto_rfc4106_check_authsize(authsize);
 357 }
 358
 359 static int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)
 360 {
 361         struct aead_request *req = container_of(areq, struct aead_request,
 362                                                 base);
 363         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 364         struct omap_aes_dev *dd = rctx->dd;
 365         int ret = 0;
 366
 367         if (!dd)
 368                 return -ENODEV;
 369
 370         if (dd->in_sg_len)
 371                 ret = omap_aes_crypt_dma_start(dd);
 372         else
 373                 omap_aes_gcm_dma_out_callback(dd);
 374
 375         return ret;
 376 }
 377
 378 int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
 379 {
 380         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 381
 382         ctx->enginectx.op.prepare_request = omap_aes_gcm_prepare_req;
 383         ctx->enginectx.op.unprepare_request = NULL;
 384         ctx->enginectx.op.do_one_request = omap_aes_gcm_crypt_req;
 385
 386         crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));
 387
 388         return 0;
 389 }