drivers/crypto/ccree/cc_driver.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6
   7 #include <linux/crypto.h>
   8 #include <linux/moduleparam.h>
   9 #include <linux/types.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/platform_device.h>
  12 #include <linux/slab.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/of.h>
  15 #include <linux/clk.h>
  16 #include <linux/of_address.h>
  17
  18 #include "cc_driver.h"
  19 #include "cc_request_mgr.h"
  20 #include "cc_buffer_mgr.h"
  21 #include "cc_debugfs.h"
  22 #include "cc_cipher.h"
  23 #include "cc_aead.h"
  24 #include "cc_hash.h"
  25 #include "cc_ivgen.h"
  26 #include "cc_sram_mgr.h"
  27 #include "cc_pm.h"
  28 #include "cc_fips.h"
  29
  30 bool cc_dump_desc;
  31 module_param_named(dump_desc, cc_dump_desc, bool, 0600);
  32 MODULE_PARM_DESC(cc_dump_desc, "Dump descriptors to kernel log as debugging aid");
  33
  34 bool cc_dump_bytes;
  35 module_param_named(dump_bytes, cc_dump_bytes, bool, 0600);
  36 MODULE_PARM_DESC(cc_dump_bytes, "Dump buffers to kernel log as debugging aid");
  37
  38 struct cc_hw_data {
  39         char *name;
  40         enum cc_hw_rev rev;
  41         u32 sig;
  42         int std_bodies;
  43 };
  44
  45 /* Hardware revisions defs. */
  46
  47 /* The 703 is a OSCCA only variant of the 713 */
  48 static const struct cc_hw_data cc703_hw = {
  49         .name = "703", .rev = CC_HW_REV_713, .std_bodies = CC_STD_OSCCA
  50 };
  51
  52 static const struct cc_hw_data cc713_hw = {
  53         .name = "713", .rev = CC_HW_REV_713, .std_bodies = CC_STD_ALL
  54 };
  55
  56 static const struct cc_hw_data cc712_hw = {
  57         .name = "712", .rev = CC_HW_REV_712, .sig =  0xDCC71200U,
  58         .std_bodies = CC_STD_ALL
  59 };
  60
  61 static const struct cc_hw_data cc710_hw = {
  62         .name = "710", .rev = CC_HW_REV_710, .sig =  0xDCC63200U,
  63         .std_bodies = CC_STD_ALL
  64 };
  65
  66 static const struct cc_hw_data cc630p_hw = {
  67         .name = "630P", .rev = CC_HW_REV_630, .sig = 0xDCC63000U,
  68         .std_bodies = CC_STD_ALL
  69 };
  70
  71 static const struct of_device_id arm_ccree_dev_of_match[] = {
  72         { .compatible = "arm,cryptocell-703-ree", .data = &cc703_hw },
  73         { .compatible = "arm,cryptocell-713-ree", .data = &cc713_hw },
  74         { .compatible = "arm,cryptocell-712-ree", .data = &cc712_hw },
  75         { .compatible = "arm,cryptocell-710-ree", .data = &cc710_hw },
  76         { .compatible = "arm,cryptocell-630p-ree", .data = &cc630p_hw },
  77         {}
  78 };
  79 MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match);
  80
  81 void __dump_byte_array(const char *name, const u8 *buf, size_t len)
  82 {
  83         char prefix[64];
  84
  85         if (!buf)
  86                 return;
  87
  88         snprintf(prefix, sizeof(prefix), "%s[%zu]: ", name, len);
  89
  90         print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_ADDRESS, 16, 1, buf,
  91                        len, false);
  92 }
  93
  94 static irqreturn_t cc_isr(int irq, void *dev_id)
  95 {
  96         struct cc_drvdata *drvdata = (struct cc_drvdata *)dev_id;
  97         struct device *dev = drvdata_to_dev(drvdata);
  98         u32 irr;
  99         u32 imr;
 100
 101         /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
 102
 103         /* read the interrupt status */
 104         irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
 105         dev_dbg(dev, "Got IRR=0x%08X\n", irr);
 106         if (irr == 0) { /* Probably shared interrupt line */
 107                 dev_err(dev, "Got interrupt with empty IRR\n");
 108                 return IRQ_NONE;
 109         }
 110         imr = cc_ioread(drvdata, CC_REG(HOST_IMR));
 111
 112         /* clear interrupt - must be before processing events */
 113         cc_iowrite(drvdata, CC_REG(HOST_ICR), irr);
 114
 115         drvdata->irq = irr;
 116         /* Completion interrupt - most probable */
 117         if (irr & CC_COMP_IRQ_MASK) {
 118                 /* Mask AXI completion interrupt - will be unmasked in
 119                  * Deferred service handler
 120                  */
 121                 cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_COMP_IRQ_MASK);
 122                 irr &= ~CC_COMP_IRQ_MASK;
 123                 complete_request(drvdata);
 124         }
 125 #ifdef CONFIG_CRYPTO_FIPS
 126         /* TEE FIPS interrupt */
 127         if (irr & CC_GPR0_IRQ_MASK) {
 128                 /* Mask interrupt - will be unmasked in Deferred service
 129                  * handler
 130                  */
 131                 cc_iowrite(drvdata, CC_REG(HOST_IMR), imr | CC_GPR0_IRQ_MASK);
 132                 irr &= ~CC_GPR0_IRQ_MASK;
 133                 fips_handler(drvdata);
 134         }
 135 #endif
 136         /* AXI error interrupt */
 137         if (irr & CC_AXI_ERR_IRQ_MASK) {
 138                 u32 axi_err;
 139
 140                 /* Read the AXI error ID */
 141                 axi_err = cc_ioread(drvdata, CC_REG(AXIM_MON_ERR));
 142                 dev_dbg(dev, "AXI completion error: axim_mon_err=0x%08X\n",
 143                         axi_err);
 144
 145                 irr &= ~CC_AXI_ERR_IRQ_MASK;
 146         }
 147
 148         if (irr) {
 149                 dev_dbg_ratelimited(dev, "IRR includes unknown cause bits (0x%08X)\n",
 150                                     irr);
 151                 /* Just warning */
 152         }
 153
 154         return IRQ_HANDLED;
 155 }
 156
 157 int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
 158 {
 159         unsigned int val, cache_params;
 160         struct device *dev = drvdata_to_dev(drvdata);
 161
 162         /* Unmask all AXI interrupt sources AXI_CFG1 register */
 163         val = cc_ioread(drvdata, CC_REG(AXIM_CFG));
 164         cc_iowrite(drvdata, CC_REG(AXIM_CFG), val & ~CC_AXI_IRQ_MASK);
 165         dev_dbg(dev, "AXIM_CFG=0x%08X\n",
 166                 cc_ioread(drvdata, CC_REG(AXIM_CFG)));
 167
 168         /* Clear all pending interrupts */
 169         val = cc_ioread(drvdata, CC_REG(HOST_IRR));
 170         dev_dbg(dev, "IRR=0x%08X\n", val);
 171         cc_iowrite(drvdata, CC_REG(HOST_ICR), val);
 172
 173         /* Unmask relevant interrupt cause */
 174         val = CC_COMP_IRQ_MASK | CC_AXI_ERR_IRQ_MASK;
 175
 176         if (drvdata->hw_rev >= CC_HW_REV_712)
 177                 val |= CC_GPR0_IRQ_MASK;
 178
 179         cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val);
 180
 181         cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0);
 182
 183         val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 184
 185         if (is_probe)
 186                 dev_dbg(dev, "Cache params previous: 0x%08X\n", val);
 187
 188         cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params);
 189         val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS));
 190
 191         if (is_probe)
 192                 dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n",
 193                         val, cache_params);
 194
 195         return 0;
 196 }
 197
 198 static int init_cc_resources(struct platform_device *plat_dev)
 199 {
 200         struct resource *req_mem_cc_regs = NULL;
 201         struct cc_drvdata *new_drvdata;
 202         struct device *dev = &plat_dev->dev;
 203         struct device_node *np = dev->of_node;
 204         u32 signature_val;
 205         u64 dma_mask;
 206         const struct cc_hw_data *hw_rev;
 207         const struct of_device_id *dev_id;
 208         struct clk *clk;
 209         int rc = 0;
 210
 211         new_drvdata = devm_kzalloc(dev, sizeof(*new_drvdata), GFP_KERNEL);
 212         if (!new_drvdata)
 213                 return -ENOMEM;
 214
 215         dev_id = of_match_node(arm_ccree_dev_of_match, np);
 216         if (!dev_id)
 217                 return -ENODEV;
 218
 219         hw_rev = (struct cc_hw_data *)dev_id->data;
 220         new_drvdata->hw_rev_name = hw_rev->name;
 221         new_drvdata->hw_rev = hw_rev->rev;
 222         new_drvdata->std_bodies = hw_rev->std_bodies;
 223
 224         if (hw_rev->rev >= CC_HW_REV_712) {
 225                 new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP);
 226                 new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_712);
 227                 new_drvdata->ver_offset = CC_REG(HOST_VERSION_712);
 228         } else {
 229                 new_drvdata->axim_mon_offset = CC_REG(AXIM_MON_COMP8);
 230                 new_drvdata->sig_offset = CC_REG(HOST_SIGNATURE_630);
 231                 new_drvdata->ver_offset = CC_REG(HOST_VERSION_630);
 232         }
 233
 234         platform_set_drvdata(plat_dev, new_drvdata);
 235         new_drvdata->plat_dev = plat_dev;
 236
 237         clk = devm_clk_get(dev, NULL);
 238         if (IS_ERR(clk))
 239                 switch (PTR_ERR(clk)) {
 240                 /* Clock is optional so this might be fine */
 241                 case -ENOENT:
 242                         break;
 243
 244                 /* Clock not available, let's try again soon */
 245                 case -EPROBE_DEFER:
 246                         return -EPROBE_DEFER;
 247
 248                 default:
 249                         dev_err(dev, "Error getting clock: %ld\n",
 250                                 PTR_ERR(clk));
 251                         return PTR_ERR(clk);
 252                 }
 253         new_drvdata->clk = clk;
 254
 255         new_drvdata->coherent = of_dma_is_coherent(np);
 256
 257         /* Get device resources */
 258         /* First CC registers space */
 259         req_mem_cc_regs = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
 260         /* Map registers space */
 261         new_drvdata->cc_base = devm_ioremap_resource(dev, req_mem_cc_regs);
 262         if (IS_ERR(new_drvdata->cc_base)) {
 263                 dev_err(dev, "Failed to ioremap registers");
 264                 return PTR_ERR(new_drvdata->cc_base);
 265         }
 266
 267         dev_dbg(dev, "Got MEM resource (%s): %pR\n", req_mem_cc_regs->name,
 268                 req_mem_cc_regs);
 269         dev_dbg(dev, "CC registers mapped from %pa to 0x%p\n",
 270                 &req_mem_cc_regs->start, new_drvdata->cc_base);
 271
 272         /* Then IRQ */
 273         new_drvdata->irq = platform_get_irq(plat_dev, 0);
 274         if (new_drvdata->irq < 0) {
 275                 dev_err(dev, "Failed getting IRQ resource\n");
 276                 return new_drvdata->irq;
 277         }
 278
 279         rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
 280                               IRQF_SHARED, "ccree", new_drvdata);
 281         if (rc) {
 282                 dev_err(dev, "Could not register to interrupt %d\n",
 283                         new_drvdata->irq);
 284                 return rc;
 285         }
 286         dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
 287
 288         init_completion(&new_drvdata->hw_queue_avail);
 289
 290         if (!plat_dev->dev.dma_mask)
 291                 plat_dev->dev.dma_mask = &plat_dev->dev.coherent_dma_mask;
 292
 293         dma_mask = DMA_BIT_MASK(DMA_BIT_MASK_LEN);
 294         while (dma_mask > 0x7fffffffUL) {
 295                 if (dma_supported(&plat_dev->dev, dma_mask)) {
 296                         rc = dma_set_coherent_mask(&plat_dev->dev, dma_mask);
 297                         if (!rc)
 298                                 break;
 299                 }
 300                 dma_mask >>= 1;
 301         }
 302
 303         if (rc) {
 304                 dev_err(dev, "Failed in dma_set_mask, mask=%llx\n", dma_mask);
 305                 return rc;
 306         }
 307
 308         rc = cc_clk_on(new_drvdata);
 309         if (rc) {
 310                 dev_err(dev, "Failed to enable clock");
 311                 return rc;
 312         }
 313
 314         if (hw_rev->rev <= CC_HW_REV_712) {
 315                 /* Verify correct mapping */
 316                 signature_val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
 317                 if (signature_val != hw_rev->sig) {
 318                         dev_err(dev, "Invalid CC signature: SIGNATURE=0x%08X != expected=0x%08X\n",
 319                                 signature_val, hw_rev->sig);
 320                         rc = -EINVAL;
 321                         goto post_clk_err;
 322                 }
 323                 dev_dbg(dev, "CC SIGNATURE=0x%08X\n", signature_val);
 324         }
 325
 326         /* Display HW versions */
 327         dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X, Driver version %s\n",
 328                  hw_rev->name, cc_ioread(new_drvdata, new_drvdata->ver_offset),
 329                  DRV_MODULE_VERSION);
 330
 331         rc = init_cc_regs(new_drvdata, true);
 332         if (rc) {
 333                 dev_err(dev, "init_cc_regs failed\n");
 334                 goto post_clk_err;
 335         }
 336
 337         rc = cc_debugfs_init(new_drvdata);
 338         if (rc) {
 339                 dev_err(dev, "Failed registering debugfs interface\n");
 340                 goto post_regs_err;
 341         }
 342
 343         rc = cc_fips_init(new_drvdata);
 344         if (rc) {
 345                 dev_err(dev, "CC_FIPS_INIT failed 0x%x\n", rc);
 346                 goto post_debugfs_err;
 347         }
 348         rc = cc_sram_mgr_init(new_drvdata);
 349         if (rc) {
 350                 dev_err(dev, "cc_sram_mgr_init failed\n");
 351                 goto post_fips_init_err;
 352         }
 353
 354         new_drvdata->mlli_sram_addr =
 355                 cc_sram_alloc(new_drvdata, MAX_MLLI_BUFF_SIZE);
 356         if (new_drvdata->mlli_sram_addr == NULL_SRAM_ADDR) {
 357                 dev_err(dev, "Failed to alloc MLLI Sram buffer\n");
 358                 rc = -ENOMEM;
 359                 goto post_sram_mgr_err;
 360         }
 361
 362         rc = cc_req_mgr_init(new_drvdata);
 363         if (rc) {
 364                 dev_err(dev, "cc_req_mgr_init failed\n");
 365                 goto post_sram_mgr_err;
 366         }
 367
 368         rc = cc_buffer_mgr_init(new_drvdata);
 369         if (rc) {
 370                 dev_err(dev, "buffer_mgr_init failed\n");
 371                 goto post_req_mgr_err;
 372         }
 373
 374         rc = cc_pm_init(new_drvdata);
 375         if (rc) {
 376                 dev_err(dev, "ssi_power_mgr_init failed\n");
 377                 goto post_buf_mgr_err;
 378         }
 379
 380         rc = cc_ivgen_init(new_drvdata);
 381         if (rc) {
 382                 dev_err(dev, "cc_ivgen_init failed\n");
 383                 goto post_buf_mgr_err;
 384         }
 385
 386         /* Allocate crypto algs */
 387         rc = cc_cipher_alloc(new_drvdata);
 388         if (rc) {
 389                 dev_err(dev, "cc_cipher_alloc failed\n");
 390                 goto post_ivgen_err;
 391         }
 392
 393         /* hash must be allocated before aead since hash exports APIs */
 394         rc = cc_hash_alloc(new_drvdata);
 395         if (rc) {
 396                 dev_err(dev, "cc_hash_alloc failed\n");
 397                 goto post_cipher_err;
 398         }
 399
 400         rc = cc_aead_alloc(new_drvdata);
 401         if (rc) {
 402                 dev_err(dev, "cc_aead_alloc failed\n");
 403                 goto post_hash_err;
 404         }
 405
 406         /* All set, we can allow autosuspend */
 407         cc_pm_go(new_drvdata);
 408
 409         /* If we got here and FIPS mode is enabled
 410          * it means all FIPS test passed, so let TEE
 411          * know we're good.
 412          */
 413         cc_set_ree_fips_status(new_drvdata, true);
 414
 415         return 0;
 416
 417 post_hash_err:
 418         cc_hash_free(new_drvdata);
 419 post_cipher_err:
 420         cc_cipher_free(new_drvdata);
 421 post_ivgen_err:
 422         cc_ivgen_fini(new_drvdata);
 423 post_buf_mgr_err:
 424          cc_buffer_mgr_fini(new_drvdata);
 425 post_req_mgr_err:
 426         cc_req_mgr_fini(new_drvdata);
 427 post_sram_mgr_err:
 428         cc_sram_mgr_fini(new_drvdata);
 429 post_fips_init_err:
 430         cc_fips_fini(new_drvdata);
 431 post_debugfs_err:
 432         cc_debugfs_fini(new_drvdata);
 433 post_regs_err:
 434         fini_cc_regs(new_drvdata);
 435 post_clk_err:
 436         cc_clk_off(new_drvdata);
 437         return rc;
 438 }
 439
 440 void fini_cc_regs(struct cc_drvdata *drvdata)
 441 {
 442         /* Mask all interrupts */
 443         cc_iowrite(drvdata, CC_REG(HOST_IMR), 0xFFFFFFFF);
 444 }
 445
 446 static void cleanup_cc_resources(struct platform_device *plat_dev)
 447 {
 448         struct cc_drvdata *drvdata =
 449                 (struct cc_drvdata *)platform_get_drvdata(plat_dev);
 450
 451         cc_aead_free(drvdata);
 452         cc_hash_free(drvdata);
 453         cc_cipher_free(drvdata);
 454         cc_ivgen_fini(drvdata);
 455         cc_pm_fini(drvdata);
 456         cc_buffer_mgr_fini(drvdata);
 457         cc_req_mgr_fini(drvdata);
 458         cc_sram_mgr_fini(drvdata);
 459         cc_fips_fini(drvdata);
 460         cc_debugfs_fini(drvdata);
 461         fini_cc_regs(drvdata);
 462         cc_clk_off(drvdata);
 463 }
 464
 465 int cc_clk_on(struct cc_drvdata *drvdata)
 466 {
 467         struct clk *clk = drvdata->clk;
 468         int rc;
 469
 470         if (IS_ERR(clk))
 471                 /* Not all devices have a clock associated with CCREE  */
 472                 return 0;
 473
 474         rc = clk_prepare_enable(clk);
 475         if (rc)
 476                 return rc;
 477
 478         return 0;
 479 }
 480
 481 unsigned int cc_get_default_hash_len(struct cc_drvdata *drvdata)
 482 {
 483         if (drvdata->hw_rev >= CC_HW_REV_712)
 484                 return HASH_LEN_SIZE_712;
 485         else
 486                 return HASH_LEN_SIZE_630;
 487 }
 488
 489 void cc_clk_off(struct cc_drvdata *drvdata)
 490 {
 491         struct clk *clk = drvdata->clk;
 492
 493         if (IS_ERR(clk))
 494                 /* Not all devices have a clock associated with CCREE */
 495                 return;
 496
 497         clk_disable_unprepare(clk);
 498 }
 499
 500 static int ccree_probe(struct platform_device *plat_dev)
 501 {
 502         int rc;
 503         struct device *dev = &plat_dev->dev;
 504
 505         /* Map registers space */
 506         rc = init_cc_resources(plat_dev);
 507         if (rc)
 508                 return rc;
 509
 510         dev_info(dev, "ARM ccree device initialized\n");
 511
 512         return 0;
 513 }
 514
 515 static int ccree_remove(struct platform_device *plat_dev)
 516 {
 517         struct device *dev = &plat_dev->dev;
 518
 519         dev_dbg(dev, "Releasing ccree resources...\n");
 520
 521         cleanup_cc_resources(plat_dev);
 522
 523         dev_info(dev, "ARM ccree device terminated\n");
 524
 525         return 0;
 526 }
 527
 528 static struct platform_driver ccree_driver = {
 529         .driver = {
 530                    .name = "ccree",
 531                    .of_match_table = arm_ccree_dev_of_match,
 532 #ifdef CONFIG_PM
 533                    .pm = &ccree_pm,
 534 #endif
 535         },
 536         .probe = ccree_probe,
 537         .remove = ccree_remove,
 538 };
 539
 540 static int __init ccree_init(void)
 541 {
 542         int ret;
 543
 544         cc_hash_global_init();
 545
 546         ret = cc_debugfs_global_init();
 547         if (ret)
 548                 return ret;
 549
 550         return platform_driver_register(&ccree_driver);
 551 }
 552 module_init(ccree_init);
 553
 554 static void __exit ccree_exit(void)
 555 {
 556         platform_driver_unregister(&ccree_driver);
 557         cc_debugfs_global_fini();
 558 }
 559 module_exit(ccree_exit);
 560
 561 /* Module description */
 562 MODULE_DESCRIPTION("ARM TrustZone CryptoCell REE Driver");
 563 MODULE_VERSION(DRV_MODULE_VERSION);
 564 MODULE_AUTHOR("ARM");
 565 MODULE_LICENSE("GPL v2");