drivers/crypto/ccp/ccp-dev.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kthread.h>
  17 #include <linux/sched.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/spinlock_types.h>
  21 #include <linux/types.h>
  22 #include <linux/mutex.h>
  23 #include <linux/delay.h>
  24 #include <linux/hw_random.h>
  25 #include <linux/cpu.h>
  26 #ifdef CONFIG_X86
  27 #include <asm/cpu_device_id.h>
  28 #endif
  29 #include <linux/ccp.h>
  30
  31 #include "ccp-dev.h"
  32
  33 MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
  34 MODULE_LICENSE("GPL");
  35 MODULE_VERSION("1.0.0");
  36 MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver");
  37
  38 struct ccp_tasklet_data {
  39         struct completion completion;
  40         struct ccp_cmd *cmd;
  41 };
  42
  43 /* Human-readable error strings */
  44 static char *ccp_error_codes[] = {
  45         "",
  46         "ERR 01: ILLEGAL_ENGINE",
  47         "ERR 02: ILLEGAL_KEY_ID",
  48         "ERR 03: ILLEGAL_FUNCTION_TYPE",
  49         "ERR 04: ILLEGAL_FUNCTION_MODE",
  50         "ERR 05: ILLEGAL_FUNCTION_ENCRYPT",
  51         "ERR 06: ILLEGAL_FUNCTION_SIZE",
  52         "ERR 07: Zlib_MISSING_INIT_EOM",
  53         "ERR 08: ILLEGAL_FUNCTION_RSVD",
  54         "ERR 09: ILLEGAL_BUFFER_LENGTH",
  55         "ERR 10: VLSB_FAULT",
  56         "ERR 11: ILLEGAL_MEM_ADDR",
  57         "ERR 12: ILLEGAL_MEM_SEL",
  58         "ERR 13: ILLEGAL_CONTEXT_ID",
  59         "ERR 14: ILLEGAL_KEY_ADDR",
  60         "ERR 15: 0xF Reserved",
  61         "ERR 16: Zlib_ILLEGAL_MULTI_QUEUE",
  62         "ERR 17: Zlib_ILLEGAL_JOBID_CHANGE",
  63         "ERR 18: CMD_TIMEOUT",
  64         "ERR 19: IDMA0_AXI_SLVERR",
  65         "ERR 20: IDMA0_AXI_DECERR",
  66         "ERR 21: 0x15 Reserved",
  67         "ERR 22: IDMA1_AXI_SLAVE_FAULT",
  68         "ERR 23: IDMA1_AIXI_DECERR",
  69         "ERR 24: 0x18 Reserved",
  70         "ERR 25: ZLIBVHB_AXI_SLVERR",
  71         "ERR 26: ZLIBVHB_AXI_DECERR",
  72         "ERR 27: 0x1B Reserved",
  73         "ERR 27: ZLIB_UNEXPECTED_EOM",
  74         "ERR 27: ZLIB_EXTRA_DATA",
  75         "ERR 30: ZLIB_BTYPE",
  76         "ERR 31: ZLIB_UNDEFINED_SYMBOL",
  77         "ERR 32: ZLIB_UNDEFINED_DISTANCE_S",
  78         "ERR 33: ZLIB_CODE_LENGTH_SYMBOL",
  79         "ERR 34: ZLIB _VHB_ILLEGAL_FETCH",
  80         "ERR 35: ZLIB_UNCOMPRESSED_LEN",
  81         "ERR 36: ZLIB_LIMIT_REACHED",
  82         "ERR 37: ZLIB_CHECKSUM_MISMATCH0",
  83         "ERR 38: ODMA0_AXI_SLVERR",
  84         "ERR 39: ODMA0_AXI_DECERR",
  85         "ERR 40: 0x28 Reserved",
  86         "ERR 41: ODMA1_AXI_SLVERR",
  87         "ERR 42: ODMA1_AXI_DECERR",
  88         "ERR 43: LSB_PARITY_ERR",
  89 };
  90
  91 void ccp_log_error(struct ccp_device *d, int e)
  92 {
  93         dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e);
  94 }
  95
  96 /* List of CCPs, CCP count, read-write access lock, and access functions
  97  *
  98  * Lock structure: get ccp_unit_lock for reading whenever we need to
  99  * examine the CCP list. While holding it for reading we can acquire
 100  * the RR lock to update the round-robin next-CCP pointer. The unit lock
 101  * must be acquired before the RR lock.
 102  *
 103  * If the unit-lock is acquired for writing, we have total control over
 104  * the list, so there's no value in getting the RR lock.
 105  */
 106 static DEFINE_RWLOCK(ccp_unit_lock);
 107 static LIST_HEAD(ccp_units);
 108
 109 /* Round-robin counter */
 110 static DEFINE_SPINLOCK(ccp_rr_lock);
 111 static struct ccp_device *ccp_rr;
 112
 113 /* Ever-increasing value to produce unique unit numbers */
 114 static atomic_t ccp_unit_ordinal;
 115 static unsigned int ccp_increment_unit_ordinal(void)
 116 {
 117         return atomic_inc_return(&ccp_unit_ordinal);
 118 }
 119
 120 /**
 121  * ccp_add_device - add a CCP device to the list
 122  *
 123  * @ccp: ccp_device struct pointer
 124  *
 125  * Put this CCP on the unit list, which makes it available
 126  * for use.
 127  *
 128  * Returns zero if a CCP device is present, -ENODEV otherwise.
 129  */
 130 void ccp_add_device(struct ccp_device *ccp)
 131 {
 132         unsigned long flags;
 133
 134         write_lock_irqsave(&ccp_unit_lock, flags);
 135         list_add_tail(&ccp->entry, &ccp_units);
 136         if (!ccp_rr)
 137                 /* We already have the list lock (we're first) so this
 138                  * pointer can't change on us. Set its initial value.
 139                  */
 140                 ccp_rr = ccp;
 141         write_unlock_irqrestore(&ccp_unit_lock, flags);
 142 }
 143
 144 /**
 145  * ccp_del_device - remove a CCP device from the list
 146  *
 147  * @ccp: ccp_device struct pointer
 148  *
 149  * Remove this unit from the list of devices. If the next device
 150  * up for use is this one, adjust the pointer. If this is the last
 151  * device, NULL the pointer.
 152  */
 153 void ccp_del_device(struct ccp_device *ccp)
 154 {
 155         unsigned long flags;
 156
 157         write_lock_irqsave(&ccp_unit_lock, flags);
 158         if (ccp_rr == ccp) {
 159                 /* ccp_unit_lock is read/write; any read access
 160                  * will be suspended while we make changes to the
 161                  * list and RR pointer.
 162                  */
 163                 if (list_is_last(&ccp_rr->entry, &ccp_units))
 164                         ccp_rr = list_first_entry(&ccp_units, struct ccp_device,
 165                                                   entry);
 166                 else
 167                         ccp_rr = list_next_entry(ccp_rr, entry);
 168         }
 169         list_del(&ccp->entry);
 170         if (list_empty(&ccp_units))
 171                 ccp_rr = NULL;
 172         write_unlock_irqrestore(&ccp_unit_lock, flags);
 173 }
 174
 175
 176
 177 int ccp_register_rng(struct ccp_device *ccp)
 178 {
 179         int ret = 0;
 180
 181         dev_dbg(ccp->dev, "Registering RNG...\n");
 182         /* Register an RNG */
 183         ccp->hwrng.name = ccp->rngname;
 184         ccp->hwrng.read = ccp_trng_read;
 185         ret = hwrng_register(&ccp->hwrng);
 186         if (ret)
 187                 dev_err(ccp->dev, "error registering hwrng (%d)\n", ret);
 188
 189         return ret;
 190 }
 191
 192 void ccp_unregister_rng(struct ccp_device *ccp)
 193 {
 194         if (ccp->hwrng.name)
 195                 hwrng_unregister(&ccp->hwrng);
 196 }
 197
 198 static struct ccp_device *ccp_get_device(void)
 199 {
 200         unsigned long flags;
 201         struct ccp_device *dp = NULL;
 202
 203         /* We round-robin through the unit list.
 204          * The (ccp_rr) pointer refers to the next unit to use.
 205          */
 206         read_lock_irqsave(&ccp_unit_lock, flags);
 207         if (!list_empty(&ccp_units)) {
 208                 spin_lock(&ccp_rr_lock);
 209                 dp = ccp_rr;
 210                 if (list_is_last(&ccp_rr->entry, &ccp_units))
 211                         ccp_rr = list_first_entry(&ccp_units, struct ccp_device,
 212                                                   entry);
 213                 else
 214                         ccp_rr = list_next_entry(ccp_rr, entry);
 215                 spin_unlock(&ccp_rr_lock);
 216         }
 217         read_unlock_irqrestore(&ccp_unit_lock, flags);
 218
 219         return dp;
 220 }
 221
 222 /**
 223  * ccp_present - check if a CCP device is present
 224  *
 225  * Returns zero if a CCP device is present, -ENODEV otherwise.
 226  */
 227 int ccp_present(void)
 228 {
 229         unsigned long flags;
 230         int ret;
 231
 232         read_lock_irqsave(&ccp_unit_lock, flags);
 233         ret = list_empty(&ccp_units);
 234         read_unlock_irqrestore(&ccp_unit_lock, flags);
 235
 236         return ret ? -ENODEV : 0;
 237 }
 238 EXPORT_SYMBOL_GPL(ccp_present);
 239
 240 /**
 241  * ccp_version - get the version of the CCP device
 242  *
 243  * Returns the version from the first unit on the list;
 244  * otherwise a zero if no CCP device is present
 245  */
 246 unsigned int ccp_version(void)
 247 {
 248         struct ccp_device *dp;
 249         unsigned long flags;
 250         int ret = 0;
 251
 252         read_lock_irqsave(&ccp_unit_lock, flags);
 253         if (!list_empty(&ccp_units)) {
 254                 dp = list_first_entry(&ccp_units, struct ccp_device, entry);
 255                 ret = dp->vdata->version;
 256         }
 257         read_unlock_irqrestore(&ccp_unit_lock, flags);
 258
 259         return ret;
 260 }
 261 EXPORT_SYMBOL_GPL(ccp_version);
 262
 263 /**
 264  * ccp_enqueue_cmd - queue an operation for processing by the CCP
 265  *
 266  * @cmd: ccp_cmd struct to be processed
 267  *
 268  * Queue a cmd to be processed by the CCP. If queueing the cmd
 269  * would exceed the defined length of the cmd queue the cmd will
 270  * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will
 271  * result in a return code of -EBUSY.
 272  *
 273  * The callback routine specified in the ccp_cmd struct will be
 274  * called to notify the caller of completion (if the cmd was not
 275  * backlogged) or advancement out of the backlog. If the cmd has
 276  * advanced out of the backlog the "err" value of the callback
 277  * will be -EINPROGRESS. Any other "err" value during callback is
 278  * the result of the operation.
 279  *
 280  * The cmd has been successfully queued if:
 281  *   the return code is -EINPROGRESS or
 282  *   the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set
 283  */
 284 int ccp_enqueue_cmd(struct ccp_cmd *cmd)
 285 {
 286         struct ccp_device *ccp;
 287         unsigned long flags;
 288         unsigned int i;
 289         int ret;
 290
 291         /* Some commands might need to be sent to a specific device */
 292         ccp = cmd->ccp ? cmd->ccp : ccp_get_device();
 293
 294         if (!ccp)
 295                 return -ENODEV;
 296
 297         /* Caller must supply a callback routine */
 298         if (!cmd->callback)
 299                 return -EINVAL;
 300
 301         cmd->ccp = ccp;
 302
 303         spin_lock_irqsave(&ccp->cmd_lock, flags);
 304
 305         i = ccp->cmd_q_count;
 306
 307         if (ccp->cmd_count >= MAX_CMD_QLEN) {
 308                 ret = -EBUSY;
 309                 if (cmd->flags & CCP_CMD_MAY_BACKLOG)
 310                         list_add_tail(&cmd->entry, &ccp->backlog);
 311         } else {
 312                 ret = -EINPROGRESS;
 313                 ccp->cmd_count++;
 314                 list_add_tail(&cmd->entry, &ccp->cmd);
 315
 316                 /* Find an idle queue */
 317                 if (!ccp->suspending) {
 318                         for (i = 0; i < ccp->cmd_q_count; i++) {
 319                                 if (ccp->cmd_q[i].active)
 320                                         continue;
 321
 322                                 break;
 323                         }
 324                 }
 325         }
 326
 327         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 328
 329         /* If we found an idle queue, wake it up */
 330         if (i < ccp->cmd_q_count)
 331                 wake_up_process(ccp->cmd_q[i].kthread);
 332
 333         return ret;
 334 }
 335 EXPORT_SYMBOL_GPL(ccp_enqueue_cmd);
 336
 337 static void ccp_do_cmd_backlog(struct work_struct *work)
 338 {
 339         struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work);
 340         struct ccp_device *ccp = cmd->ccp;
 341         unsigned long flags;
 342         unsigned int i;
 343
 344         cmd->callback(cmd->data, -EINPROGRESS);
 345
 346         spin_lock_irqsave(&ccp->cmd_lock, flags);
 347
 348         ccp->cmd_count++;
 349         list_add_tail(&cmd->entry, &ccp->cmd);
 350
 351         /* Find an idle queue */
 352         for (i = 0; i < ccp->cmd_q_count; i++) {
 353                 if (ccp->cmd_q[i].active)
 354                         continue;
 355
 356                 break;
 357         }
 358
 359         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 360
 361         /* If we found an idle queue, wake it up */
 362         if (i < ccp->cmd_q_count)
 363                 wake_up_process(ccp->cmd_q[i].kthread);
 364 }
 365
 366 static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q)
 367 {
 368         struct ccp_device *ccp = cmd_q->ccp;
 369         struct ccp_cmd *cmd = NULL;
 370         struct ccp_cmd *backlog = NULL;
 371         unsigned long flags;
 372
 373         spin_lock_irqsave(&ccp->cmd_lock, flags);
 374
 375         cmd_q->active = 0;
 376
 377         if (ccp->suspending) {
 378                 cmd_q->suspended = 1;
 379
 380                 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 381                 wake_up_interruptible(&ccp->suspend_queue);
 382
 383                 return NULL;
 384         }
 385
 386         if (ccp->cmd_count) {
 387                 cmd_q->active = 1;
 388
 389                 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
 390                 list_del(&cmd->entry);
 391
 392                 ccp->cmd_count--;
 393         }
 394
 395         if (!list_empty(&ccp->backlog)) {
 396                 backlog = list_first_entry(&ccp->backlog, struct ccp_cmd,
 397                                            entry);
 398                 list_del(&backlog->entry);
 399         }
 400
 401         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 402
 403         if (backlog) {
 404                 INIT_WORK(&backlog->work, ccp_do_cmd_backlog);
 405                 schedule_work(&backlog->work);
 406         }
 407
 408         return cmd;
 409 }
 410
 411 static void ccp_do_cmd_complete(unsigned long data)
 412 {
 413         struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data;
 414         struct ccp_cmd *cmd = tdata->cmd;
 415
 416         cmd->callback(cmd->data, cmd->ret);
 417         complete(&tdata->completion);
 418 }
 419
 420 /**
 421  * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue
 422  *
 423  * @data: thread-specific data
 424  */
 425 int ccp_cmd_queue_thread(void *data)
 426 {
 427         struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data;
 428         struct ccp_cmd *cmd;
 429         struct ccp_tasklet_data tdata;
 430         struct tasklet_struct tasklet;
 431
 432         tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata);
 433
 434         set_current_state(TASK_INTERRUPTIBLE);
 435         while (!kthread_should_stop()) {
 436                 schedule();
 437
 438                 set_current_state(TASK_INTERRUPTIBLE);
 439
 440                 cmd = ccp_dequeue_cmd(cmd_q);
 441                 if (!cmd)
 442                         continue;
 443
 444                 __set_current_state(TASK_RUNNING);
 445
 446                 /* Execute the command */
 447                 cmd->ret = ccp_run_cmd(cmd_q, cmd);
 448
 449                 /* Schedule the completion callback */
 450                 tdata.cmd = cmd;
 451                 init_completion(&tdata.completion);
 452                 tasklet_schedule(&tasklet);
 453                 wait_for_completion(&tdata.completion);
 454         }
 455
 456         __set_current_state(TASK_RUNNING);
 457
 458         return 0;
 459 }
 460
 461 /**
 462  * ccp_alloc_struct - allocate and initialize the ccp_device struct
 463  *
 464  * @dev: device struct of the CCP
 465  */
 466 struct ccp_device *ccp_alloc_struct(struct device *dev)
 467 {
 468         struct ccp_device *ccp;
 469
 470         ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);
 471         if (!ccp)
 472                 return NULL;
 473         ccp->dev = dev;
 474
 475         INIT_LIST_HEAD(&ccp->cmd);
 476         INIT_LIST_HEAD(&ccp->backlog);
 477
 478         spin_lock_init(&ccp->cmd_lock);
 479         mutex_init(&ccp->req_mutex);
 480         mutex_init(&ccp->sb_mutex);
 481         ccp->sb_count = KSB_COUNT;
 482         ccp->sb_start = 0;
 483
 484         /* Initialize the wait queues */
 485         init_waitqueue_head(&ccp->sb_queue);
 486         init_waitqueue_head(&ccp->suspend_queue);
 487
 488         ccp->ord = ccp_increment_unit_ordinal();
 489         snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord);
 490         snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord);
 491
 492         return ccp;
 493 }
 494
 495 int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
 496 {
 497         struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
 498         u32 trng_value;
 499         int len = min_t(int, sizeof(trng_value), max);
 500
 501         /* Locking is provided by the caller so we can update device
 502          * hwrng-related fields safely
 503          */
 504         trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
 505         if (!trng_value) {
 506                 /* Zero is returned if not data is available or if a
 507                  * bad-entropy error is present. Assume an error if
 508                  * we exceed TRNG_RETRIES reads of zero.
 509                  */
 510                 if (ccp->hwrng_retries++ > TRNG_RETRIES)
 511                         return -EIO;
 512
 513                 return 0;
 514         }
 515
 516         /* Reset the counter and save the rng value */
 517         ccp->hwrng_retries = 0;
 518         memcpy(data, &trng_value, len);
 519
 520         return len;
 521 }
 522
 523 #ifdef CONFIG_PM
 524 bool ccp_queues_suspended(struct ccp_device *ccp)
 525 {
 526         unsigned int suspended = 0;
 527         unsigned long flags;
 528         unsigned int i;
 529
 530         spin_lock_irqsave(&ccp->cmd_lock, flags);
 531
 532         for (i = 0; i < ccp->cmd_q_count; i++)
 533                 if (ccp->cmd_q[i].suspended)
 534                         suspended++;
 535
 536         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 537
 538         return ccp->cmd_q_count == suspended;
 539 }
 540 #endif
 541
 542 static int __init ccp_mod_init(void)
 543 {
 544 #ifdef CONFIG_X86
 545         int ret;
 546
 547         ret = ccp_pci_init();
 548         if (ret)
 549                 return ret;
 550
 551         /* Don't leave the driver loaded if init failed */
 552         if (ccp_present() != 0) {
 553                 ccp_pci_exit();
 554                 return -ENODEV;
 555         }
 556
 557         return 0;
 558 #endif
 559
 560 #ifdef CONFIG_ARM64
 561         int ret;
 562
 563         ret = ccp_platform_init();
 564         if (ret)
 565                 return ret;
 566
 567         /* Don't leave the driver loaded if init failed */
 568         if (ccp_present() != 0) {
 569                 ccp_platform_exit();
 570                 return -ENODEV;
 571         }
 572
 573         return 0;
 574 #endif
 575
 576         return -ENODEV;
 577 }
 578
 579 static void __exit ccp_mod_exit(void)
 580 {
 581 #ifdef CONFIG_X86
 582         ccp_pci_exit();
 583 #endif
 584
 585 #ifdef CONFIG_ARM64
 586         ccp_platform_exit();
 587 #endif
 588 }
 589
 590 module_init(ccp_mod_init);
 591 module_exit(ccp_mod_exit);