drivers/s390/crypto/zcrypt_api.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  *  Copyright IBM Corp. 2001, 2018
   4  *  Author(s): Robert Burroughs
   5  *             Eric Rossman (edrossma@us.ibm.com)
   6  *             Cornelia Huck <cornelia.huck@de.ibm.com>
   7  *
   8  *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
   9  *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
  10  *                                Ralph Wuerthner <rwuerthn@de.ibm.com>
  11  *  MSGTYPE restruct:             Holger Dengler <hd@linux.vnet.ibm.com>
  12  *  Multiple device nodes: Harald Freudenberger <freude@linux.ibm.com>
  13  */
  14
  15 #define KMSG_COMPONENT "zcrypt"
  16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  17
  18 #include <linux/module.h>
  19 #include <linux/init.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/miscdevice.h>
  22 #include <linux/fs.h>
  23 #include <linux/compat.h>
  24 #include <linux/slab.h>
  25 #include <linux/atomic.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/hw_random.h>
  28 #include <linux/debugfs.h>
  29 #include <linux/cdev.h>
  30 #include <linux/ctype.h>
  31 #include <linux/capability.h>
  32 #include <asm/debug.h>
  33
  34 #define CREATE_TRACE_POINTS
  35 #include <asm/trace/zcrypt.h>
  36
  37 #include "zcrypt_api.h"
  38 #include "zcrypt_debug.h"
  39
  40 #include "zcrypt_msgtype6.h"
  41 #include "zcrypt_msgtype50.h"
  42 #include "zcrypt_ccamisc.h"
  43 #include "zcrypt_ep11misc.h"
  44
  45 /*
  46  * Module description.
  47  */
  48 MODULE_AUTHOR("IBM Corporation");
  49 MODULE_DESCRIPTION("Cryptographic Coprocessor interface, " \
  50                    "Copyright IBM Corp. 2001, 2012");
  51 MODULE_LICENSE("GPL");
  52
  53 /*
  54  * zcrypt tracepoint functions
  55  */
  56 EXPORT_TRACEPOINT_SYMBOL(s390_zcrypt_req);
  57 EXPORT_TRACEPOINT_SYMBOL(s390_zcrypt_rep);
  58
  59 DEFINE_SPINLOCK(zcrypt_list_lock);
  60 LIST_HEAD(zcrypt_card_list);
  61
  62 static atomic_t zcrypt_open_count = ATOMIC_INIT(0);
  63
  64 static LIST_HEAD(zcrypt_ops_list);
  65
  66 /* Zcrypt related debug feature stuff. */
  67 debug_info_t *zcrypt_dbf_info;
  68
  69 /*
  70  * Process a rescan of the transport layer.
  71  * Runs a synchronous AP bus rescan.
  72  * Returns true if something has changed (for example the
  73  * bus scan has found and build up new devices) and it is
  74  * worth to do a retry. Otherwise false is returned meaning
  75  * no changes on the AP bus level.
  76  */
  77 static inline bool zcrypt_process_rescan(void)
  78 {
  79         return ap_bus_force_rescan();
  80 }
  81
  82 void zcrypt_msgtype_register(struct zcrypt_ops *zops)
  83 {
  84         list_add_tail(&zops->list, &zcrypt_ops_list);
  85 }
  86
  87 void zcrypt_msgtype_unregister(struct zcrypt_ops *zops)
  88 {
  89         list_del_init(&zops->list);
  90 }
  91
  92 struct zcrypt_ops *zcrypt_msgtype(unsigned char *name, int variant)
  93 {
  94         struct zcrypt_ops *zops;
  95
  96         list_for_each_entry(zops, &zcrypt_ops_list, list)
  97                 if (zops->variant == variant &&
  98                     (!strncmp(zops->name, name, sizeof(zops->name))))
  99                         return zops;
 100         return NULL;
 101 }
 102 EXPORT_SYMBOL(zcrypt_msgtype);
 103
 104 /*
 105  * Multi device nodes extension functions.
 106  */
 107
 108 struct zcdn_device;
 109
 110 static void zcdn_device_release(struct device *dev);
 111 static const struct class zcrypt_class = {
 112         .name = ZCRYPT_NAME,
 113         .dev_release = zcdn_device_release,
 114 };
 115 static dev_t zcrypt_devt;
 116 static struct cdev zcrypt_cdev;
 117
 118 struct zcdn_device {
 119         struct device device;
 120         struct ap_perms perms;
 121 };
 122
 123 #define to_zcdn_dev(x) container_of((x), struct zcdn_device, device)
 124
 125 #define ZCDN_MAX_NAME 32
 126
 127 static int zcdn_create(const char *name);
 128 static int zcdn_destroy(const char *name);
 129
 130 /*
 131  * Find zcdn device by name.
 132  * Returns reference to the zcdn device which needs to be released
 133  * with put_device() after use.
 134  */
 135 static inline struct zcdn_device *find_zcdndev_by_name(const char *name)
 136 {
 137         struct device *dev = class_find_device_by_name(&zcrypt_class, name);
 138
 139         return dev ? to_zcdn_dev(dev) : NULL;
 140 }
 141
 142 /*
 143  * Find zcdn device by devt value.
 144  * Returns reference to the zcdn device which needs to be released
 145  * with put_device() after use.
 146  */
 147 static inline struct zcdn_device *find_zcdndev_by_devt(dev_t devt)
 148 {
 149         struct device *dev = class_find_device_by_devt(&zcrypt_class, devt);
 150
 151         return dev ? to_zcdn_dev(dev) : NULL;
 152 }
 153
 154 static ssize_t ioctlmask_show(struct device *dev,
 155                               struct device_attribute *attr,
 156                               char *buf)
 157 {
 158         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 159         int i, n;
 160
 161         if (mutex_lock_interruptible(&ap_perms_mutex))
 162                 return -ERESTARTSYS;
 163
 164         n = sysfs_emit(buf, "0x");
 165         for (i = 0; i < sizeof(zcdndev->perms.ioctlm) / sizeof(long); i++)
 166                 n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.ioctlm[i]);
 167         n += sysfs_emit_at(buf, n, "\n");
 168
 169         mutex_unlock(&ap_perms_mutex);
 170
 171         return n;
 172 }
 173
 174 static ssize_t ioctlmask_store(struct device *dev,
 175                                struct device_attribute *attr,
 176                                const char *buf, size_t count)
 177 {
 178         int rc;
 179         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 180
 181         rc = ap_parse_mask_str(buf, zcdndev->perms.ioctlm,
 182                                AP_IOCTLS, &ap_perms_mutex);
 183         if (rc)
 184                 return rc;
 185
 186         return count;
 187 }
 188
 189 static DEVICE_ATTR_RW(ioctlmask);
 190
 191 static ssize_t apmask_show(struct device *dev,
 192                            struct device_attribute *attr,
 193                            char *buf)
 194 {
 195         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 196         int i, n;
 197
 198         if (mutex_lock_interruptible(&ap_perms_mutex))
 199                 return -ERESTARTSYS;
 200
 201         n = sysfs_emit(buf, "0x");
 202         for (i = 0; i < sizeof(zcdndev->perms.apm) / sizeof(long); i++)
 203                 n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.apm[i]);
 204         n += sysfs_emit_at(buf, n, "\n");
 205
 206         mutex_unlock(&ap_perms_mutex);
 207
 208         return n;
 209 }
 210
 211 static ssize_t apmask_store(struct device *dev,
 212                             struct device_attribute *attr,
 213                             const char *buf, size_t count)
 214 {
 215         int rc;
 216         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 217
 218         rc = ap_parse_mask_str(buf, zcdndev->perms.apm,
 219                                AP_DEVICES, &ap_perms_mutex);
 220         if (rc)
 221                 return rc;
 222
 223         return count;
 224 }
 225
 226 static DEVICE_ATTR_RW(apmask);
 227
 228 static ssize_t aqmask_show(struct device *dev,
 229                            struct device_attribute *attr,
 230                            char *buf)
 231 {
 232         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 233         int i, n;
 234
 235         if (mutex_lock_interruptible(&ap_perms_mutex))
 236                 return -ERESTARTSYS;
 237
 238         n = sysfs_emit(buf, "0x");
 239         for (i = 0; i < sizeof(zcdndev->perms.aqm) / sizeof(long); i++)
 240                 n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.aqm[i]);
 241         n += sysfs_emit_at(buf, n, "\n");
 242
 243         mutex_unlock(&ap_perms_mutex);
 244
 245         return n;
 246 }
 247
 248 static ssize_t aqmask_store(struct device *dev,
 249                             struct device_attribute *attr,
 250                             const char *buf, size_t count)
 251 {
 252         int rc;
 253         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 254
 255         rc = ap_parse_mask_str(buf, zcdndev->perms.aqm,
 256                                AP_DOMAINS, &ap_perms_mutex);
 257         if (rc)
 258                 return rc;
 259
 260         return count;
 261 }
 262
 263 static DEVICE_ATTR_RW(aqmask);
 264
 265 static ssize_t admask_show(struct device *dev,
 266                            struct device_attribute *attr,
 267                            char *buf)
 268 {
 269         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 270         int i, n;
 271
 272         if (mutex_lock_interruptible(&ap_perms_mutex))
 273                 return -ERESTARTSYS;
 274
 275         n = sysfs_emit(buf, "0x");
 276         for (i = 0; i < sizeof(zcdndev->perms.adm) / sizeof(long); i++)
 277                 n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.adm[i]);
 278         n += sysfs_emit_at(buf, n, "\n");
 279
 280         mutex_unlock(&ap_perms_mutex);
 281
 282         return n;
 283 }
 284
 285 static ssize_t admask_store(struct device *dev,
 286                             struct device_attribute *attr,
 287                             const char *buf, size_t count)
 288 {
 289         int rc;
 290         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 291
 292         rc = ap_parse_mask_str(buf, zcdndev->perms.adm,
 293                                AP_DOMAINS, &ap_perms_mutex);
 294         if (rc)
 295                 return rc;
 296
 297         return count;
 298 }
 299
 300 static DEVICE_ATTR_RW(admask);
 301
 302 static struct attribute *zcdn_dev_attrs[] = {
 303         &dev_attr_ioctlmask.attr,
 304         &dev_attr_apmask.attr,
 305         &dev_attr_aqmask.attr,
 306         &dev_attr_admask.attr,
 307         NULL
 308 };
 309
 310 static struct attribute_group zcdn_dev_attr_group = {
 311         .attrs = zcdn_dev_attrs
 312 };
 313
 314 static const struct attribute_group *zcdn_dev_attr_groups[] = {
 315         &zcdn_dev_attr_group,
 316         NULL
 317 };
 318
 319 static ssize_t zcdn_create_store(const struct class *class,
 320                                  const struct class_attribute *attr,
 321                                  const char *buf, size_t count)
 322 {
 323         int rc;
 324         char name[ZCDN_MAX_NAME];
 325
 326         strscpy(name, skip_spaces(buf), sizeof(name));
 327
 328         rc = zcdn_create(strim(name));
 329
 330         return rc ? rc : count;
 331 }
 332
 333 static const struct class_attribute class_attr_zcdn_create =
 334         __ATTR(create, 0600, NULL, zcdn_create_store);
 335
 336 static ssize_t zcdn_destroy_store(const struct class *class,
 337                                   const struct class_attribute *attr,
 338                                   const char *buf, size_t count)
 339 {
 340         int rc;
 341         char name[ZCDN_MAX_NAME];
 342
 343         strscpy(name, skip_spaces(buf), sizeof(name));
 344
 345         rc = zcdn_destroy(strim(name));
 346
 347         return rc ? rc : count;
 348 }
 349
 350 static const struct class_attribute class_attr_zcdn_destroy =
 351         __ATTR(destroy, 0600, NULL, zcdn_destroy_store);
 352
 353 static void zcdn_device_release(struct device *dev)
 354 {
 355         struct zcdn_device *zcdndev = to_zcdn_dev(dev);
 356
 357         ZCRYPT_DBF_INFO("%s releasing zcdn device %d:%d\n",
 358                         __func__, MAJOR(dev->devt), MINOR(dev->devt));
 359
 360         kfree(zcdndev);
 361 }
 362
 363 static int zcdn_create(const char *name)
 364 {
 365         dev_t devt;
 366         int i, rc = 0;
 367         struct zcdn_device *zcdndev;
 368
 369         if (mutex_lock_interruptible(&ap_perms_mutex))
 370                 return -ERESTARTSYS;
 371
 372         /* check if device node with this name already exists */
 373         if (name[0]) {
 374                 zcdndev = find_zcdndev_by_name(name);
 375                 if (zcdndev) {
 376                         put_device(&zcdndev->device);
 377                         rc = -EEXIST;
 378                         goto unlockout;
 379                 }
 380         }
 381
 382         /* find an unused minor number */
 383         for (i = 0; i < ZCRYPT_MAX_MINOR_NODES; i++) {
 384                 devt = MKDEV(MAJOR(zcrypt_devt), MINOR(zcrypt_devt) + i);
 385                 zcdndev = find_zcdndev_by_devt(devt);
 386                 if (zcdndev)
 387                         put_device(&zcdndev->device);
 388                 else
 389                         break;
 390         }
 391         if (i == ZCRYPT_MAX_MINOR_NODES) {
 392                 rc = -ENOSPC;
 393                 goto unlockout;
 394         }
 395
 396         /* alloc and prepare a new zcdn device */
 397         zcdndev = kzalloc(sizeof(*zcdndev), GFP_KERNEL);
 398         if (!zcdndev) {
 399                 rc = -ENOMEM;
 400                 goto unlockout;
 401         }
 402         zcdndev->device.release = zcdn_device_release;
 403         zcdndev->device.class = &zcrypt_class;
 404         zcdndev->device.devt = devt;
 405         zcdndev->device.groups = zcdn_dev_attr_groups;
 406         if (name[0])
 407                 rc = dev_set_name(&zcdndev->device, "%s", name);
 408         else
 409                 rc = dev_set_name(&zcdndev->device, ZCRYPT_NAME "_%d", (int)MINOR(devt));
 410         if (rc) {
 411                 kfree(zcdndev);
 412                 goto unlockout;
 413         }
 414         rc = device_register(&zcdndev->device);
 415         if (rc) {
 416                 put_device(&zcdndev->device);
 417                 goto unlockout;
 418         }
 419
 420         ZCRYPT_DBF_INFO("%s created zcdn device %d:%d\n",
 421                         __func__, MAJOR(devt), MINOR(devt));
 422
 423 unlockout:
 424         mutex_unlock(&ap_perms_mutex);
 425         return rc;
 426 }
 427
 428 static int zcdn_destroy(const char *name)
 429 {
 430         int rc = 0;
 431         struct zcdn_device *zcdndev;
 432
 433         if (mutex_lock_interruptible(&ap_perms_mutex))
 434                 return -ERESTARTSYS;
 435
 436         /* try to find this zcdn device */
 437         zcdndev = find_zcdndev_by_name(name);
 438         if (!zcdndev) {
 439                 rc = -ENOENT;
 440                 goto unlockout;
 441         }
 442
 443         /*
 444          * The zcdn device is not hard destroyed. It is subject to
 445          * reference counting and thus just needs to be unregistered.
 446          */
 447         put_device(&zcdndev->device);
 448         device_unregister(&zcdndev->device);
 449
 450 unlockout:
 451         mutex_unlock(&ap_perms_mutex);
 452         return rc;
 453 }
 454
 455 static void zcdn_destroy_all(void)
 456 {
 457         int i;
 458         dev_t devt;
 459         struct zcdn_device *zcdndev;
 460
 461         mutex_lock(&ap_perms_mutex);
 462         for (i = 0; i < ZCRYPT_MAX_MINOR_NODES; i++) {
 463                 devt = MKDEV(MAJOR(zcrypt_devt), MINOR(zcrypt_devt) + i);
 464                 zcdndev = find_zcdndev_by_devt(devt);
 465                 if (zcdndev) {
 466                         put_device(&zcdndev->device);
 467                         device_unregister(&zcdndev->device);
 468                 }
 469         }
 470         mutex_unlock(&ap_perms_mutex);
 471 }
 472
 473 /*
 474  * zcrypt_read (): Not supported beyond zcrypt 1.3.1.
 475  *
 476  * This function is not supported beyond zcrypt 1.3.1.
 477  */
 478 static ssize_t zcrypt_read(struct file *filp, char __user *buf,
 479                            size_t count, loff_t *f_pos)
 480 {
 481         return -EPERM;
 482 }
 483
 484 /*
 485  * zcrypt_write(): Not allowed.
 486  *
 487  * Write is not allowed
 488  */
 489 static ssize_t zcrypt_write(struct file *filp, const char __user *buf,
 490                             size_t count, loff_t *f_pos)
 491 {
 492         return -EPERM;
 493 }
 494
 495 /*
 496  * zcrypt_open(): Count number of users.
 497  *
 498  * Device open function to count number of users.
 499  */
 500 static int zcrypt_open(struct inode *inode, struct file *filp)
 501 {
 502         struct ap_perms *perms = &ap_perms;
 503
 504         if (filp->f_inode->i_cdev == &zcrypt_cdev) {
 505                 struct zcdn_device *zcdndev;
 506
 507                 if (mutex_lock_interruptible(&ap_perms_mutex))
 508                         return -ERESTARTSYS;
 509                 zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev);
 510                 /* find returns a reference, no get_device() needed */
 511                 mutex_unlock(&ap_perms_mutex);
 512                 if (zcdndev)
 513                         perms = &zcdndev->perms;
 514         }
 515         filp->private_data = (void *)perms;
 516
 517         atomic_inc(&zcrypt_open_count);
 518         return stream_open(inode, filp);
 519 }
 520
 521 /*
 522  * zcrypt_release(): Count number of users.
 523  *
 524  * Device close function to count number of users.
 525  */
 526 static int zcrypt_release(struct inode *inode, struct file *filp)
 527 {
 528         if (filp->f_inode->i_cdev == &zcrypt_cdev) {
 529                 struct zcdn_device *zcdndev;
 530
 531                 mutex_lock(&ap_perms_mutex);
 532                 zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev);
 533                 mutex_unlock(&ap_perms_mutex);
 534                 if (zcdndev) {
 535                         /* 2 puts here: one for find, one for open */
 536                         put_device(&zcdndev->device);
 537                         put_device(&zcdndev->device);
 538                 }
 539         }
 540
 541         atomic_dec(&zcrypt_open_count);
 542         return 0;
 543 }
 544
 545 static inline int zcrypt_check_ioctl(struct ap_perms *perms,
 546                                      unsigned int cmd)
 547 {
 548         int rc = -EPERM;
 549         int ioctlnr = (cmd & _IOC_NRMASK) >> _IOC_NRSHIFT;
 550
 551         if (ioctlnr > 0 && ioctlnr < AP_IOCTLS) {
 552                 if (test_bit_inv(ioctlnr, perms->ioctlm))
 553                         rc = 0;
 554         }
 555
 556         if (rc)
 557                 ZCRYPT_DBF_WARN("%s ioctl check failed: ioctlnr=0x%04x rc=%d\n",
 558                                 __func__, ioctlnr, rc);
 559
 560         return rc;
 561 }
 562
 563 static inline bool zcrypt_check_card(struct ap_perms *perms, int card)
 564 {
 565         return test_bit_inv(card, perms->apm) ? true : false;
 566 }
 567
 568 static inline bool zcrypt_check_queue(struct ap_perms *perms, int queue)
 569 {
 570         return test_bit_inv(queue, perms->aqm) ? true : false;
 571 }
 572
 573 static inline struct zcrypt_queue *zcrypt_pick_queue(struct zcrypt_card *zc,
 574                                                      struct zcrypt_queue *zq,
 575                                                      struct module **pmod,
 576                                                      unsigned int weight)
 577 {
 578         if (!zq || !try_module_get(zq->queue->ap_dev.device.driver->owner))
 579                 return NULL;
 580         zcrypt_card_get(zc);
 581         zcrypt_queue_get(zq);
 582         get_device(&zq->queue->ap_dev.device);
 583         atomic_add(weight, &zc->load);
 584         atomic_add(weight, &zq->load);
 585         zq->request_count++;
 586         *pmod = zq->queue->ap_dev.device.driver->owner;
 587         return zq;
 588 }
 589
 590 static inline void zcrypt_drop_queue(struct zcrypt_card *zc,
 591                                      struct zcrypt_queue *zq,
 592                                      struct module *mod,
 593                                      unsigned int weight)
 594 {
 595         zq->request_count--;
 596         atomic_sub(weight, &zc->load);
 597         atomic_sub(weight, &zq->load);
 598         put_device(&zq->queue->ap_dev.device);
 599         zcrypt_queue_put(zq);
 600         zcrypt_card_put(zc);
 601         module_put(mod);
 602 }
 603
 604 static inline bool zcrypt_card_compare(struct zcrypt_card *zc,
 605                                        struct zcrypt_card *pref_zc,
 606                                        unsigned int weight,
 607                                        unsigned int pref_weight)
 608 {
 609         if (!pref_zc)
 610                 return true;
 611         weight += atomic_read(&zc->load);
 612         pref_weight += atomic_read(&pref_zc->load);
 613         if (weight == pref_weight)
 614                 return atomic64_read(&zc->card->total_request_count) <
 615                         atomic64_read(&pref_zc->card->total_request_count);
 616         return weight < pref_weight;
 617 }
 618
 619 static inline bool zcrypt_queue_compare(struct zcrypt_queue *zq,
 620                                         struct zcrypt_queue *pref_zq,
 621                                         unsigned int weight,
 622                                         unsigned int pref_weight)
 623 {
 624         if (!pref_zq)
 625                 return true;
 626         weight += atomic_read(&zq->load);
 627         pref_weight += atomic_read(&pref_zq->load);
 628         if (weight == pref_weight)
 629                 return zq->queue->total_request_count <
 630                         pref_zq->queue->total_request_count;
 631         return weight < pref_weight;
 632 }
 633
 634 /*
 635  * zcrypt ioctls.
 636  */
 637 static long zcrypt_rsa_modexpo(struct ap_perms *perms,
 638                                struct zcrypt_track *tr,
 639                                struct ica_rsa_modexpo *mex)
 640 {
 641         struct zcrypt_card *zc, *pref_zc;
 642         struct zcrypt_queue *zq, *pref_zq;
 643         struct ap_message ap_msg;
 644         unsigned int wgt = 0, pref_wgt = 0;
 645         unsigned int func_code;
 646         int cpen, qpen, qid = 0, rc = -ENODEV;
 647         struct module *mod;
 648
 649         trace_s390_zcrypt_req(mex, TP_ICARSAMODEXPO);
 650
 651         ap_init_message(&ap_msg);
 652
 653         if (mex->outputdatalength < mex->inputdatalength) {
 654                 func_code = 0;
 655                 rc = -EINVAL;
 656                 goto out;
 657         }
 658
 659         /*
 660          * As long as outputdatalength is big enough, we can set the
 661          * outputdatalength equal to the inputdatalength, since that is the
 662          * number of bytes we will copy in any case
 663          */
 664         mex->outputdatalength = mex->inputdatalength;
 665
 666         rc = get_rsa_modex_fc(mex, &func_code);
 667         if (rc)
 668                 goto out;
 669
 670         pref_zc = NULL;
 671         pref_zq = NULL;
 672         spin_lock(&zcrypt_list_lock);
 673         for_each_zcrypt_card(zc) {
 674                 /* Check for usable accelerator or CCA card */
 675                 if (!zc->online || !zc->card->config || zc->card->chkstop ||
 676                     !(zc->card->hwinfo.accel || zc->card->hwinfo.cca))
 677                         continue;
 678                 /* Check for size limits */
 679                 if (zc->min_mod_size > mex->inputdatalength ||
 680                     zc->max_mod_size < mex->inputdatalength)
 681                         continue;
 682                 /* check if device node has admission for this card */
 683                 if (!zcrypt_check_card(perms, zc->card->id))
 684                         continue;
 685                 /* get weight index of the card device  */
 686                 wgt = zc->speed_rating[func_code];
 687                 /* penalty if this msg was previously sent via this card */
 688                 cpen = (tr && tr->again_counter && tr->last_qid &&
 689                         AP_QID_CARD(tr->last_qid) == zc->card->id) ?
 690                         TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
 691                 if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
 692                         continue;
 693                 for_each_zcrypt_queue(zq, zc) {
 694                         /* check if device is usable and eligible */
 695                         if (!zq->online || !zq->ops->rsa_modexpo ||
 696                             !ap_queue_usable(zq->queue))
 697                                 continue;
 698                         /* check if device node has admission for this queue */
 699                         if (!zcrypt_check_queue(perms,
 700                                                 AP_QID_QUEUE(zq->queue->qid)))
 701                                 continue;
 702                         /* penalty if the msg was previously sent at this qid */
 703                         qpen = (tr && tr->again_counter && tr->last_qid &&
 704                                 tr->last_qid == zq->queue->qid) ?
 705                                 TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
 706                         if (!zcrypt_queue_compare(zq, pref_zq,
 707                                                   wgt + cpen + qpen, pref_wgt))
 708                                 continue;
 709                         pref_zc = zc;
 710                         pref_zq = zq;
 711                         pref_wgt = wgt + cpen + qpen;
 712                 }
 713         }
 714         pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
 715         spin_unlock(&zcrypt_list_lock);
 716
 717         if (!pref_zq) {
 718                 pr_debug("%s no matching queue found => ENODEV\n", __func__);
 719                 rc = -ENODEV;
 720                 goto out;
 721         }
 722
 723         qid = pref_zq->queue->qid;
 724         rc = pref_zq->ops->rsa_modexpo(pref_zq, mex, &ap_msg);
 725
 726         spin_lock(&zcrypt_list_lock);
 727         zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
 728         spin_unlock(&zcrypt_list_lock);
 729
 730 out:
 731         ap_release_message(&ap_msg);
 732         if (tr) {
 733                 tr->last_rc = rc;
 734                 tr->last_qid = qid;
 735         }
 736         trace_s390_zcrypt_rep(mex, func_code, rc,
 737                               AP_QID_CARD(qid), AP_QID_QUEUE(qid));
 738         return rc;
 739 }
 740
 741 static long zcrypt_rsa_crt(struct ap_perms *perms,
 742                            struct zcrypt_track *tr,
 743                            struct ica_rsa_modexpo_crt *crt)
 744 {
 745         struct zcrypt_card *zc, *pref_zc;
 746         struct zcrypt_queue *zq, *pref_zq;
 747         struct ap_message ap_msg;
 748         unsigned int wgt = 0, pref_wgt = 0;
 749         unsigned int func_code;
 750         int cpen, qpen, qid = 0, rc = -ENODEV;
 751         struct module *mod;
 752
 753         trace_s390_zcrypt_req(crt, TP_ICARSACRT);
 754
 755         ap_init_message(&ap_msg);
 756
 757         if (crt->outputdatalength < crt->inputdatalength) {
 758                 func_code = 0;
 759                 rc = -EINVAL;
 760                 goto out;
 761         }
 762
 763         /*
 764          * As long as outputdatalength is big enough, we can set the
 765          * outputdatalength equal to the inputdatalength, since that is the
 766          * number of bytes we will copy in any case
 767          */
 768         crt->outputdatalength = crt->inputdatalength;
 769
 770         rc = get_rsa_crt_fc(crt, &func_code);
 771         if (rc)
 772                 goto out;
 773
 774         pref_zc = NULL;
 775         pref_zq = NULL;
 776         spin_lock(&zcrypt_list_lock);
 777         for_each_zcrypt_card(zc) {
 778                 /* Check for usable accelerator or CCA card */
 779                 if (!zc->online || !zc->card->config || zc->card->chkstop ||
 780                     !(zc->card->hwinfo.accel || zc->card->hwinfo.cca))
 781                         continue;
 782                 /* Check for size limits */
 783                 if (zc->min_mod_size > crt->inputdatalength ||
 784                     zc->max_mod_size < crt->inputdatalength)
 785                         continue;
 786                 /* check if device node has admission for this card */
 787                 if (!zcrypt_check_card(perms, zc->card->id))
 788                         continue;
 789                 /* get weight index of the card device  */
 790                 wgt = zc->speed_rating[func_code];
 791                 /* penalty if this msg was previously sent via this card */
 792                 cpen = (tr && tr->again_counter && tr->last_qid &&
 793                         AP_QID_CARD(tr->last_qid) == zc->card->id) ?
 794                         TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
 795                 if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
 796                         continue;
 797                 for_each_zcrypt_queue(zq, zc) {
 798                         /* check if device is usable and eligible */
 799                         if (!zq->online || !zq->ops->rsa_modexpo_crt ||
 800                             !ap_queue_usable(zq->queue))
 801                                 continue;
 802                         /* check if device node has admission for this queue */
 803                         if (!zcrypt_check_queue(perms,
 804                                                 AP_QID_QUEUE(zq->queue->qid)))
 805                                 continue;
 806                         /* penalty if the msg was previously sent at this qid */
 807                         qpen = (tr && tr->again_counter && tr->last_qid &&
 808                                 tr->last_qid == zq->queue->qid) ?
 809                                 TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
 810                         if (!zcrypt_queue_compare(zq, pref_zq,
 811                                                   wgt + cpen + qpen, pref_wgt))
 812                                 continue;
 813                         pref_zc = zc;
 814                         pref_zq = zq;
 815                         pref_wgt = wgt + cpen + qpen;
 816                 }
 817         }
 818         pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
 819         spin_unlock(&zcrypt_list_lock);
 820
 821         if (!pref_zq) {
 822                 pr_debug("%s no matching queue found => ENODEV\n", __func__);
 823                 rc = -ENODEV;
 824                 goto out;
 825         }
 826
 827         qid = pref_zq->queue->qid;
 828         rc = pref_zq->ops->rsa_modexpo_crt(pref_zq, crt, &ap_msg);
 829
 830         spin_lock(&zcrypt_list_lock);
 831         zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
 832         spin_unlock(&zcrypt_list_lock);
 833
 834 out:
 835         ap_release_message(&ap_msg);
 836         if (tr) {
 837                 tr->last_rc = rc;
 838                 tr->last_qid = qid;
 839         }
 840         trace_s390_zcrypt_rep(crt, func_code, rc,
 841                               AP_QID_CARD(qid), AP_QID_QUEUE(qid));
 842         return rc;
 843 }
 844
 845 static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms,
 846                               struct zcrypt_track *tr,
 847                               struct ica_xcRB *xcrb)
 848 {
 849         struct zcrypt_card *zc, *pref_zc;
 850         struct zcrypt_queue *zq, *pref_zq;
 851         struct ap_message ap_msg;
 852         unsigned int wgt = 0, pref_wgt = 0;
 853         unsigned int func_code;
 854         unsigned short *domain, tdom;
 855         int cpen, qpen, qid = 0, rc = -ENODEV;
 856         struct module *mod;
 857
 858         trace_s390_zcrypt_req(xcrb, TB_ZSECSENDCPRB);
 859
 860         xcrb->status = 0;
 861         ap_init_message(&ap_msg);
 862
 863         rc = prep_cca_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain);
 864         if (rc)
 865                 goto out;
 866         print_hex_dump_debug("ccareq: ", DUMP_PREFIX_ADDRESS, 16, 1,
 867                              ap_msg.msg, ap_msg.len, false);
 868
 869         tdom = *domain;
 870         if (perms != &ap_perms && tdom < AP_DOMAINS) {
 871                 if (ap_msg.flags & AP_MSG_FLAG_ADMIN) {
 872                         if (!test_bit_inv(tdom, perms->adm)) {
 873                                 rc = -ENODEV;
 874                                 goto out;
 875                         }
 876                 } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) {
 877                         rc = -EOPNOTSUPP;
 878                         goto out;
 879                 }
 880         }
 881         /*
 882          * If a valid target domain is set and this domain is NOT a usage
 883          * domain but a control only domain, autoselect target domain.
 884          */
 885         if (tdom < AP_DOMAINS &&
 886             !ap_test_config_usage_domain(tdom) &&
 887             ap_test_config_ctrl_domain(tdom))
 888                 tdom = AUTOSEL_DOM;
 889
 890         pref_zc = NULL;
 891         pref_zq = NULL;
 892         spin_lock(&zcrypt_list_lock);
 893         for_each_zcrypt_card(zc) {
 894                 /* Check for usable CCA card */
 895                 if (!zc->online || !zc->card->config || zc->card->chkstop ||
 896                     !zc->card->hwinfo.cca)
 897                         continue;
 898                 /* Check for user selected CCA card */
 899                 if (xcrb->user_defined != AUTOSELECT &&
 900                     xcrb->user_defined != zc->card->id)
 901                         continue;
 902                 /* check if request size exceeds card max msg size */
 903                 if (ap_msg.len > zc->card->maxmsgsize)
 904                         continue;
 905                 /* check if device node has admission for this card */
 906                 if (!zcrypt_check_card(perms, zc->card->id))
 907                         continue;
 908                 /* get weight index of the card device  */
 909                 wgt = speed_idx_cca(func_code) * zc->speed_rating[SECKEY];
 910                 /* penalty if this msg was previously sent via this card */
 911                 cpen = (tr && tr->again_counter && tr->last_qid &&
 912                         AP_QID_CARD(tr->last_qid) == zc->card->id) ?
 913                         TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
 914                 if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
 915                         continue;
 916                 for_each_zcrypt_queue(zq, zc) {
 917                         /* check for device usable and eligible */
 918                         if (!zq->online || !zq->ops->send_cprb ||
 919                             !ap_queue_usable(zq->queue) ||
 920                             (tdom != AUTOSEL_DOM &&
 921                              tdom != AP_QID_QUEUE(zq->queue->qid)))
 922                                 continue;
 923                         /* check if device node has admission for this queue */
 924                         if (!zcrypt_check_queue(perms,
 925                                                 AP_QID_QUEUE(zq->queue->qid)))
 926                                 continue;
 927                         /* penalty if the msg was previously sent at this qid */
 928                         qpen = (tr && tr->again_counter && tr->last_qid &&
 929                                 tr->last_qid == zq->queue->qid) ?
 930                                 TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
 931                         if (!zcrypt_queue_compare(zq, pref_zq,
 932                                                   wgt + cpen + qpen, pref_wgt))
 933                                 continue;
 934                         pref_zc = zc;
 935                         pref_zq = zq;
 936                         pref_wgt = wgt + cpen + qpen;
 937                 }
 938         }
 939         pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
 940         spin_unlock(&zcrypt_list_lock);
 941
 942         if (!pref_zq) {
 943                 pr_debug("%s no match for address %02x.%04x => ENODEV\n",
 944                          __func__, xcrb->user_defined, *domain);
 945                 rc = -ENODEV;
 946                 goto out;
 947         }
 948
 949         /* in case of auto select, provide the correct domain */
 950         qid = pref_zq->queue->qid;
 951         if (*domain == AUTOSEL_DOM)
 952                 *domain = AP_QID_QUEUE(qid);
 953
 954         rc = pref_zq->ops->send_cprb(userspace, pref_zq, xcrb, &ap_msg);
 955         if (!rc) {
 956                 print_hex_dump_debug("ccarpl: ", DUMP_PREFIX_ADDRESS, 16, 1,
 957                                      ap_msg.msg, ap_msg.len, false);
 958         }
 959
 960         spin_lock(&zcrypt_list_lock);
 961         zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
 962         spin_unlock(&zcrypt_list_lock);
 963
 964 out:
 965         ap_release_message(&ap_msg);
 966         if (tr) {
 967                 tr->last_rc = rc;
 968                 tr->last_qid = qid;
 969         }
 970         trace_s390_zcrypt_rep(xcrb, func_code, rc,
 971                               AP_QID_CARD(qid), AP_QID_QUEUE(qid));
 972         return rc;
 973 }
 974
 975 long zcrypt_send_cprb(struct ica_xcRB *xcrb)
 976 {
 977         struct zcrypt_track tr;
 978         int rc;
 979
 980         memset(&tr, 0, sizeof(tr));
 981
 982         do {
 983                 rc = _zcrypt_send_cprb(false, &ap_perms, &tr, xcrb);
 984         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
 985
 986         /* on ENODEV failure: retry once again after a requested rescan */
 987         if (rc == -ENODEV && zcrypt_process_rescan())
 988                 do {
 989                         rc = _zcrypt_send_cprb(false, &ap_perms, &tr, xcrb);
 990                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
 991         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
 992                 rc = -EIO;
 993         if (rc)
 994                 pr_debug("%s rc=%d\n", __func__, rc);
 995
 996         return rc;
 997 }
 998 EXPORT_SYMBOL(zcrypt_send_cprb);
 999
1000 static bool is_desired_ep11_card(unsigned int dev_id,
1001                                  unsigned short target_num,
1002                                  struct ep11_target_dev *targets)
1003 {
1004         while (target_num-- > 0) {
1005                 if (targets->ap_id == dev_id || targets->ap_id == AUTOSEL_AP)
1006                         return true;
1007                 targets++;
1008         }
1009         return false;
1010 }
1011
1012 static bool is_desired_ep11_queue(unsigned int dev_qid,
1013                                   unsigned short target_num,
1014                                   struct ep11_target_dev *targets)
1015 {
1016         int card = AP_QID_CARD(dev_qid), dom = AP_QID_QUEUE(dev_qid);
1017
1018         while (target_num-- > 0) {
1019                 if ((targets->ap_id == card || targets->ap_id == AUTOSEL_AP) &&
1020                     (targets->dom_id == dom || targets->dom_id == AUTOSEL_DOM))
1021                         return true;
1022                 targets++;
1023         }
1024         return false;
1025 }
1026
1027 static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms,
1028                                    struct zcrypt_track *tr,
1029                                    struct ep11_urb *xcrb)
1030 {
1031         struct zcrypt_card *zc, *pref_zc;
1032         struct zcrypt_queue *zq, *pref_zq;
1033         struct ep11_target_dev *targets;
1034         unsigned short target_num;
1035         unsigned int wgt = 0, pref_wgt = 0;
1036         unsigned int func_code, domain;
1037         struct ap_message ap_msg;
1038         int cpen, qpen, qid = 0, rc = -ENODEV;
1039         struct module *mod;
1040
1041         trace_s390_zcrypt_req(xcrb, TP_ZSENDEP11CPRB);
1042
1043         ap_init_message(&ap_msg);
1044
1045         target_num = (unsigned short)xcrb->targets_num;
1046
1047         /* empty list indicates autoselect (all available targets) */
1048         targets = NULL;
1049         if (target_num != 0) {
1050                 struct ep11_target_dev __user *uptr;
1051
1052                 targets = kcalloc(target_num, sizeof(*targets), GFP_KERNEL);
1053                 if (!targets) {
1054                         func_code = 0;
1055                         rc = -ENOMEM;
1056                         goto out;
1057                 }
1058
1059                 uptr = (struct ep11_target_dev __force __user *)xcrb->targets;
1060                 if (z_copy_from_user(userspace, targets, uptr,
1061                                      target_num * sizeof(*targets))) {
1062                         func_code = 0;
1063                         rc = -EFAULT;
1064                         goto out_free;
1065                 }
1066         }
1067
1068         rc = prep_ep11_ap_msg(userspace, xcrb, &ap_msg, &func_code, &domain);
1069         if (rc)
1070                 goto out_free;
1071         print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1,
1072                              ap_msg.msg, ap_msg.len, false);
1073
1074         if (perms != &ap_perms && domain < AUTOSEL_DOM) {
1075                 if (ap_msg.flags & AP_MSG_FLAG_ADMIN) {
1076                         if (!test_bit_inv(domain, perms->adm)) {
1077                                 rc = -ENODEV;
1078                                 goto out_free;
1079                         }
1080                 } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) {
1081                         rc = -EOPNOTSUPP;
1082                         goto out_free;
1083                 }
1084         }
1085
1086         pref_zc = NULL;
1087         pref_zq = NULL;
1088         spin_lock(&zcrypt_list_lock);
1089         for_each_zcrypt_card(zc) {
1090                 /* Check for usable EP11 card */
1091                 if (!zc->online || !zc->card->config || zc->card->chkstop ||
1092                     !zc->card->hwinfo.ep11)
1093                         continue;
1094                 /* Check for user selected EP11 card */
1095                 if (targets &&
1096                     !is_desired_ep11_card(zc->card->id, target_num, targets))
1097                         continue;
1098                 /* check if request size exceeds card max msg size */
1099                 if (ap_msg.len > zc->card->maxmsgsize)
1100                         continue;
1101                 /* check if device node has admission for this card */
1102                 if (!zcrypt_check_card(perms, zc->card->id))
1103                         continue;
1104                 /* get weight index of the card device  */
1105                 wgt = speed_idx_ep11(func_code) * zc->speed_rating[SECKEY];
1106                 /* penalty if this msg was previously sent via this card */
1107                 cpen = (tr && tr->again_counter && tr->last_qid &&
1108                         AP_QID_CARD(tr->last_qid) == zc->card->id) ?
1109                         TRACK_AGAIN_CARD_WEIGHT_PENALTY : 0;
1110                 if (!zcrypt_card_compare(zc, pref_zc, wgt + cpen, pref_wgt))
1111                         continue;
1112                 for_each_zcrypt_queue(zq, zc) {
1113                         /* check if device is usable and eligible */
1114                         if (!zq->online || !zq->ops->send_ep11_cprb ||
1115                             !ap_queue_usable(zq->queue) ||
1116                             (targets &&
1117                              !is_desired_ep11_queue(zq->queue->qid,
1118                                                     target_num, targets)))
1119                                 continue;
1120                         /* check if device node has admission for this queue */
1121                         if (!zcrypt_check_queue(perms,
1122                                                 AP_QID_QUEUE(zq->queue->qid)))
1123                                 continue;
1124                         /* penalty if the msg was previously sent at this qid */
1125                         qpen = (tr && tr->again_counter && tr->last_qid &&
1126                                 tr->last_qid == zq->queue->qid) ?
1127                                 TRACK_AGAIN_QUEUE_WEIGHT_PENALTY : 0;
1128                         if (!zcrypt_queue_compare(zq, pref_zq,
1129                                                   wgt + cpen + qpen, pref_wgt))
1130                                 continue;
1131                         pref_zc = zc;
1132                         pref_zq = zq;
1133                         pref_wgt = wgt + cpen + qpen;
1134                 }
1135         }
1136         pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
1137         spin_unlock(&zcrypt_list_lock);
1138
1139         if (!pref_zq) {
1140                 if (targets && target_num == 1) {
1141                         pr_debug("%s no match for address %02x.%04x => ENODEV\n",
1142                                  __func__, (int)targets->ap_id,
1143                                  (int)targets->dom_id);
1144                 } else if (targets) {
1145                         pr_debug("%s no match for %d target addrs => ENODEV\n",
1146                                  __func__, (int)target_num);
1147                 } else {
1148                         pr_debug("%s no match for address ff.ffff => ENODEV\n",
1149                                  __func__);
1150                 }
1151                 rc = -ENODEV;
1152                 goto out_free;
1153         }
1154
1155         qid = pref_zq->queue->qid;
1156         rc = pref_zq->ops->send_ep11_cprb(userspace, pref_zq, xcrb, &ap_msg);
1157         if (!rc) {
1158                 print_hex_dump_debug("ep11rpl: ", DUMP_PREFIX_ADDRESS, 16, 1,
1159                                      ap_msg.msg, ap_msg.len, false);
1160         }
1161
1162         spin_lock(&zcrypt_list_lock);
1163         zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
1164         spin_unlock(&zcrypt_list_lock);
1165
1166 out_free:
1167         kfree(targets);
1168 out:
1169         ap_release_message(&ap_msg);
1170         if (tr) {
1171                 tr->last_rc = rc;
1172                 tr->last_qid = qid;
1173         }
1174         trace_s390_zcrypt_rep(xcrb, func_code, rc,
1175                               AP_QID_CARD(qid), AP_QID_QUEUE(qid));
1176         return rc;
1177 }
1178
1179 long zcrypt_send_ep11_cprb(struct ep11_urb *xcrb)
1180 {
1181         struct zcrypt_track tr;
1182         int rc;
1183
1184         memset(&tr, 0, sizeof(tr));
1185
1186         do {
1187                 rc = _zcrypt_send_ep11_cprb(false, &ap_perms, &tr, xcrb);
1188         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1189
1190         /* on ENODEV failure: retry once again after a requested rescan */
1191         if (rc == -ENODEV && zcrypt_process_rescan())
1192                 do {
1193                         rc = _zcrypt_send_ep11_cprb(false, &ap_perms, &tr, xcrb);
1194                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1195         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1196                 rc = -EIO;
1197         if (rc)
1198                 pr_debug("%s rc=%d\n", __func__, rc);
1199
1200         return rc;
1201 }
1202 EXPORT_SYMBOL(zcrypt_send_ep11_cprb);
1203
1204 static long zcrypt_rng(char *buffer)
1205 {
1206         struct zcrypt_card *zc, *pref_zc;
1207         struct zcrypt_queue *zq, *pref_zq;
1208         unsigned int wgt = 0, pref_wgt = 0;
1209         unsigned int func_code;
1210         struct ap_message ap_msg;
1211         unsigned int domain;
1212         int qid = 0, rc = -ENODEV;
1213         struct module *mod;
1214
1215         trace_s390_zcrypt_req(buffer, TP_HWRNGCPRB);
1216
1217         ap_init_message(&ap_msg);
1218         rc = prep_rng_ap_msg(&ap_msg, &func_code, &domain);
1219         if (rc)
1220                 goto out;
1221
1222         pref_zc = NULL;
1223         pref_zq = NULL;
1224         spin_lock(&zcrypt_list_lock);
1225         for_each_zcrypt_card(zc) {
1226                 /* Check for usable CCA card */
1227                 if (!zc->online || !zc->card->config || zc->card->chkstop ||
1228                     !zc->card->hwinfo.cca)
1229                         continue;
1230                 /* get weight index of the card device  */
1231                 wgt = zc->speed_rating[func_code];
1232                 if (!zcrypt_card_compare(zc, pref_zc, wgt, pref_wgt))
1233                         continue;
1234                 for_each_zcrypt_queue(zq, zc) {
1235                         /* check if device is usable and eligible */
1236                         if (!zq->online || !zq->ops->rng ||
1237                             !ap_queue_usable(zq->queue))
1238                                 continue;
1239                         if (!zcrypt_queue_compare(zq, pref_zq, wgt, pref_wgt))
1240                                 continue;
1241                         pref_zc = zc;
1242                         pref_zq = zq;
1243                         pref_wgt = wgt;
1244                 }
1245         }
1246         pref_zq = zcrypt_pick_queue(pref_zc, pref_zq, &mod, wgt);
1247         spin_unlock(&zcrypt_list_lock);
1248
1249         if (!pref_zq) {
1250                 pr_debug("%s no matching queue found => ENODEV\n", __func__);
1251                 rc = -ENODEV;
1252                 goto out;
1253         }
1254
1255         qid = pref_zq->queue->qid;
1256         rc = pref_zq->ops->rng(pref_zq, buffer, &ap_msg);
1257
1258         spin_lock(&zcrypt_list_lock);
1259         zcrypt_drop_queue(pref_zc, pref_zq, mod, wgt);
1260         spin_unlock(&zcrypt_list_lock);
1261
1262 out:
1263         ap_release_message(&ap_msg);
1264         trace_s390_zcrypt_rep(buffer, func_code, rc,
1265                               AP_QID_CARD(qid), AP_QID_QUEUE(qid));
1266         return rc;
1267 }
1268
1269 static void zcrypt_device_status_mask(struct zcrypt_device_status *devstatus)
1270 {
1271         struct zcrypt_card *zc;
1272         struct zcrypt_queue *zq;
1273         struct zcrypt_device_status *stat;
1274         int card, queue;
1275
1276         memset(devstatus, 0, MAX_ZDEV_ENTRIES
1277                * sizeof(struct zcrypt_device_status));
1278
1279         spin_lock(&zcrypt_list_lock);
1280         for_each_zcrypt_card(zc) {
1281                 for_each_zcrypt_queue(zq, zc) {
1282                         card = AP_QID_CARD(zq->queue->qid);
1283                         if (card >= MAX_ZDEV_CARDIDS)
1284                                 continue;
1285                         queue = AP_QID_QUEUE(zq->queue->qid);
1286                         stat = &devstatus[card * AP_DOMAINS + queue];
1287                         stat->hwtype = zc->card->ap_dev.device_type;
1288                         stat->functions = zc->card->hwinfo.fac >> 26;
1289                         stat->qid = zq->queue->qid;
1290                         stat->online = zq->online ? 0x01 : 0x00;
1291                 }
1292         }
1293         spin_unlock(&zcrypt_list_lock);
1294 }
1295
1296 void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus)
1297 {
1298         struct zcrypt_card *zc;
1299         struct zcrypt_queue *zq;
1300         struct zcrypt_device_status_ext *stat;
1301         int card, queue;
1302
1303         memset(devstatus, 0, MAX_ZDEV_ENTRIES_EXT
1304                * sizeof(struct zcrypt_device_status_ext));
1305
1306         spin_lock(&zcrypt_list_lock);
1307         for_each_zcrypt_card(zc) {
1308                 for_each_zcrypt_queue(zq, zc) {
1309                         card = AP_QID_CARD(zq->queue->qid);
1310                         queue = AP_QID_QUEUE(zq->queue->qid);
1311                         stat = &devstatus[card * AP_DOMAINS + queue];
1312                         stat->hwtype = zc->card->ap_dev.device_type;
1313                         stat->functions = zc->card->hwinfo.fac >> 26;
1314                         stat->qid = zq->queue->qid;
1315                         stat->online = zq->online ? 0x01 : 0x00;
1316                 }
1317         }
1318         spin_unlock(&zcrypt_list_lock);
1319 }
1320 EXPORT_SYMBOL(zcrypt_device_status_mask_ext);
1321
1322 int zcrypt_device_status_ext(int card, int queue,
1323                              struct zcrypt_device_status_ext *devstat)
1324 {
1325         struct zcrypt_card *zc;
1326         struct zcrypt_queue *zq;
1327
1328         memset(devstat, 0, sizeof(*devstat));
1329
1330         spin_lock(&zcrypt_list_lock);
1331         for_each_zcrypt_card(zc) {
1332                 for_each_zcrypt_queue(zq, zc) {
1333                         if (card == AP_QID_CARD(zq->queue->qid) &&
1334                             queue == AP_QID_QUEUE(zq->queue->qid)) {
1335                                 devstat->hwtype = zc->card->ap_dev.device_type;
1336                                 devstat->functions = zc->card->hwinfo.fac >> 26;
1337                                 devstat->qid = zq->queue->qid;
1338                                 devstat->online = zq->online ? 0x01 : 0x00;
1339                                 spin_unlock(&zcrypt_list_lock);
1340                                 return 0;
1341                         }
1342                 }
1343         }
1344         spin_unlock(&zcrypt_list_lock);
1345
1346         return -ENODEV;
1347 }
1348 EXPORT_SYMBOL(zcrypt_device_status_ext);
1349
1350 static void zcrypt_status_mask(char status[], size_t max_adapters)
1351 {
1352         struct zcrypt_card *zc;
1353         struct zcrypt_queue *zq;
1354         int card;
1355
1356         memset(status, 0, max_adapters);
1357         spin_lock(&zcrypt_list_lock);
1358         for_each_zcrypt_card(zc) {
1359                 for_each_zcrypt_queue(zq, zc) {
1360                         card = AP_QID_CARD(zq->queue->qid);
1361                         if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index ||
1362                             card >= max_adapters)
1363                                 continue;
1364                         status[card] = zc->online ? zc->user_space_type : 0x0d;
1365                 }
1366         }
1367         spin_unlock(&zcrypt_list_lock);
1368 }
1369
1370 static void zcrypt_qdepth_mask(char qdepth[], size_t max_adapters)
1371 {
1372         struct zcrypt_card *zc;
1373         struct zcrypt_queue *zq;
1374         int card;
1375
1376         memset(qdepth, 0, max_adapters);
1377         spin_lock(&zcrypt_list_lock);
1378         local_bh_disable();
1379         for_each_zcrypt_card(zc) {
1380                 for_each_zcrypt_queue(zq, zc) {
1381                         card = AP_QID_CARD(zq->queue->qid);
1382                         if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index ||
1383                             card >= max_adapters)
1384                                 continue;
1385                         spin_lock(&zq->queue->lock);
1386                         qdepth[card] =
1387                                 zq->queue->pendingq_count +
1388                                 zq->queue->requestq_count;
1389                         spin_unlock(&zq->queue->lock);
1390                 }
1391         }
1392         local_bh_enable();
1393         spin_unlock(&zcrypt_list_lock);
1394 }
1395
1396 static void zcrypt_perdev_reqcnt(u32 reqcnt[], size_t max_adapters)
1397 {
1398         struct zcrypt_card *zc;
1399         struct zcrypt_queue *zq;
1400         int card;
1401         u64 cnt;
1402
1403         memset(reqcnt, 0, sizeof(int) * max_adapters);
1404         spin_lock(&zcrypt_list_lock);
1405         local_bh_disable();
1406         for_each_zcrypt_card(zc) {
1407                 for_each_zcrypt_queue(zq, zc) {
1408                         card = AP_QID_CARD(zq->queue->qid);
1409                         if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index ||
1410                             card >= max_adapters)
1411                                 continue;
1412                         spin_lock(&zq->queue->lock);
1413                         cnt = zq->queue->total_request_count;
1414                         spin_unlock(&zq->queue->lock);
1415                         reqcnt[card] = (cnt < UINT_MAX) ? (u32)cnt : UINT_MAX;
1416                 }
1417         }
1418         local_bh_enable();
1419         spin_unlock(&zcrypt_list_lock);
1420 }
1421
1422 static int zcrypt_pendingq_count(void)
1423 {
1424         struct zcrypt_card *zc;
1425         struct zcrypt_queue *zq;
1426         int pendingq_count;
1427
1428         pendingq_count = 0;
1429         spin_lock(&zcrypt_list_lock);
1430         local_bh_disable();
1431         for_each_zcrypt_card(zc) {
1432                 for_each_zcrypt_queue(zq, zc) {
1433                         if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
1434                                 continue;
1435                         spin_lock(&zq->queue->lock);
1436                         pendingq_count += zq->queue->pendingq_count;
1437                         spin_unlock(&zq->queue->lock);
1438                 }
1439         }
1440         local_bh_enable();
1441         spin_unlock(&zcrypt_list_lock);
1442         return pendingq_count;
1443 }
1444
1445 static int zcrypt_requestq_count(void)
1446 {
1447         struct zcrypt_card *zc;
1448         struct zcrypt_queue *zq;
1449         int requestq_count;
1450
1451         requestq_count = 0;
1452         spin_lock(&zcrypt_list_lock);
1453         local_bh_disable();
1454         for_each_zcrypt_card(zc) {
1455                 for_each_zcrypt_queue(zq, zc) {
1456                         if (AP_QID_QUEUE(zq->queue->qid) != ap_domain_index)
1457                                 continue;
1458                         spin_lock(&zq->queue->lock);
1459                         requestq_count += zq->queue->requestq_count;
1460                         spin_unlock(&zq->queue->lock);
1461                 }
1462         }
1463         local_bh_enable();
1464         spin_unlock(&zcrypt_list_lock);
1465         return requestq_count;
1466 }
1467
1468 static int icarsamodexpo_ioctl(struct ap_perms *perms, unsigned long arg)
1469 {
1470         int rc;
1471         struct zcrypt_track tr;
1472         struct ica_rsa_modexpo mex;
1473         struct ica_rsa_modexpo __user *umex = (void __user *)arg;
1474
1475         memset(&tr, 0, sizeof(tr));
1476         if (copy_from_user(&mex, umex, sizeof(mex)))
1477                 return -EFAULT;
1478
1479         do {
1480                 rc = zcrypt_rsa_modexpo(perms, &tr, &mex);
1481         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1482
1483         /* on ENODEV failure: retry once again after a requested rescan */
1484         if (rc == -ENODEV && zcrypt_process_rescan())
1485                 do {
1486                         rc = zcrypt_rsa_modexpo(perms, &tr, &mex);
1487                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1488         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1489                 rc = -EIO;
1490         if (rc) {
1491                 pr_debug("ioctl ICARSAMODEXPO rc=%d\n", rc);
1492                 return rc;
1493         }
1494         return put_user(mex.outputdatalength, &umex->outputdatalength);
1495 }
1496
1497 static int icarsacrt_ioctl(struct ap_perms *perms, unsigned long arg)
1498 {
1499         int rc;
1500         struct zcrypt_track tr;
1501         struct ica_rsa_modexpo_crt crt;
1502         struct ica_rsa_modexpo_crt __user *ucrt = (void __user *)arg;
1503
1504         memset(&tr, 0, sizeof(tr));
1505         if (copy_from_user(&crt, ucrt, sizeof(crt)))
1506                 return -EFAULT;
1507
1508         do {
1509                 rc = zcrypt_rsa_crt(perms, &tr, &crt);
1510         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1511
1512         /* on ENODEV failure: retry once again after a requested rescan */
1513         if (rc == -ENODEV && zcrypt_process_rescan())
1514                 do {
1515                         rc = zcrypt_rsa_crt(perms, &tr, &crt);
1516                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1517         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1518                 rc = -EIO;
1519         if (rc) {
1520                 pr_debug("ioctl ICARSACRT rc=%d\n", rc);
1521                 return rc;
1522         }
1523         return put_user(crt.outputdatalength, &ucrt->outputdatalength);
1524 }
1525
1526 static int zsecsendcprb_ioctl(struct ap_perms *perms, unsigned long arg)
1527 {
1528         int rc;
1529         struct ica_xcRB xcrb;
1530         struct zcrypt_track tr;
1531         struct ica_xcRB __user *uxcrb = (void __user *)arg;
1532
1533         memset(&tr, 0, sizeof(tr));
1534         if (copy_from_user(&xcrb, uxcrb, sizeof(xcrb)))
1535                 return -EFAULT;
1536
1537         do {
1538                 rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb);
1539         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1540
1541         /* on ENODEV failure: retry once again after a requested rescan */
1542         if (rc == -ENODEV && zcrypt_process_rescan())
1543                 do {
1544                         rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb);
1545                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1546         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1547                 rc = -EIO;
1548         if (rc)
1549                 pr_debug("ioctl ZSENDCPRB rc=%d status=0x%x\n",
1550                          rc, xcrb.status);
1551         if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb)))
1552                 return -EFAULT;
1553         return rc;
1554 }
1555
1556 static int zsendep11cprb_ioctl(struct ap_perms *perms, unsigned long arg)
1557 {
1558         int rc;
1559         struct ep11_urb xcrb;
1560         struct zcrypt_track tr;
1561         struct ep11_urb __user *uxcrb = (void __user *)arg;
1562
1563         memset(&tr, 0, sizeof(tr));
1564         if (copy_from_user(&xcrb, uxcrb, sizeof(xcrb)))
1565                 return -EFAULT;
1566
1567         do {
1568                 rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb);
1569         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1570
1571         /* on ENODEV failure: retry once again after a requested rescan */
1572         if (rc == -ENODEV && zcrypt_process_rescan())
1573                 do {
1574                         rc = _zcrypt_send_ep11_cprb(true, perms, &tr, &xcrb);
1575                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1576         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1577                 rc = -EIO;
1578         if (rc)
1579                 pr_debug("ioctl ZSENDEP11CPRB rc=%d\n", rc);
1580         if (copy_to_user(uxcrb, &xcrb, sizeof(xcrb)))
1581                 return -EFAULT;
1582         return rc;
1583 }
1584
1585 static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd,
1586                                   unsigned long arg)
1587 {
1588         int rc;
1589         struct ap_perms *perms =
1590                 (struct ap_perms *)filp->private_data;
1591
1592         rc = zcrypt_check_ioctl(perms, cmd);
1593         if (rc)
1594                 return rc;
1595
1596         switch (cmd) {
1597         case ICARSAMODEXPO:
1598                 return icarsamodexpo_ioctl(perms, arg);
1599         case ICARSACRT:
1600                 return icarsacrt_ioctl(perms, arg);
1601         case ZSECSENDCPRB:
1602                 return zsecsendcprb_ioctl(perms, arg);
1603         case ZSENDEP11CPRB:
1604                 return zsendep11cprb_ioctl(perms, arg);
1605         case ZCRYPT_DEVICE_STATUS: {
1606                 struct zcrypt_device_status_ext *device_status;
1607                 size_t total_size = MAX_ZDEV_ENTRIES_EXT
1608                         * sizeof(struct zcrypt_device_status_ext);
1609
1610                 device_status = kvmalloc_array(MAX_ZDEV_ENTRIES_EXT,
1611                                                sizeof(struct zcrypt_device_status_ext),
1612                                                GFP_KERNEL);
1613                 if (!device_status)
1614                         return -ENOMEM;
1615                 zcrypt_device_status_mask_ext(device_status);
1616                 if (copy_to_user((char __user *)arg, device_status,
1617                                  total_size))
1618                         rc = -EFAULT;
1619                 kvfree(device_status);
1620                 return rc;
1621         }
1622         case ZCRYPT_STATUS_MASK: {
1623                 char status[AP_DEVICES];
1624
1625                 zcrypt_status_mask(status, AP_DEVICES);
1626                 if (copy_to_user((char __user *)arg, status, sizeof(status)))
1627                         return -EFAULT;
1628                 return 0;
1629         }
1630         case ZCRYPT_QDEPTH_MASK: {
1631                 char qdepth[AP_DEVICES];
1632
1633                 zcrypt_qdepth_mask(qdepth, AP_DEVICES);
1634                 if (copy_to_user((char __user *)arg, qdepth, sizeof(qdepth)))
1635                         return -EFAULT;
1636                 return 0;
1637         }
1638         case ZCRYPT_PERDEV_REQCNT: {
1639                 u32 *reqcnt;
1640
1641                 reqcnt = kcalloc(AP_DEVICES, sizeof(u32), GFP_KERNEL);
1642                 if (!reqcnt)
1643                         return -ENOMEM;
1644                 zcrypt_perdev_reqcnt(reqcnt, AP_DEVICES);
1645                 if (copy_to_user((int __user *)arg, reqcnt,
1646                                  sizeof(u32) * AP_DEVICES))
1647                         rc = -EFAULT;
1648                 kfree(reqcnt);
1649                 return rc;
1650         }
1651         case Z90STAT_REQUESTQ_COUNT:
1652                 return put_user(zcrypt_requestq_count(), (int __user *)arg);
1653         case Z90STAT_PENDINGQ_COUNT:
1654                 return put_user(zcrypt_pendingq_count(), (int __user *)arg);
1655         case Z90STAT_TOTALOPEN_COUNT:
1656                 return put_user(atomic_read(&zcrypt_open_count),
1657                                 (int __user *)arg);
1658         case Z90STAT_DOMAIN_INDEX:
1659                 return put_user(ap_domain_index, (int __user *)arg);
1660         /*
1661          * Deprecated ioctls
1662          */
1663         case ZDEVICESTATUS: {
1664                 /* the old ioctl supports only 64 adapters */
1665                 struct zcrypt_device_status *device_status;
1666                 size_t total_size = MAX_ZDEV_ENTRIES
1667                         * sizeof(struct zcrypt_device_status);
1668
1669                 device_status = kzalloc(total_size, GFP_KERNEL);
1670                 if (!device_status)
1671                         return -ENOMEM;
1672                 zcrypt_device_status_mask(device_status);
1673                 if (copy_to_user((char __user *)arg, device_status,
1674                                  total_size))
1675                         rc = -EFAULT;
1676                 kfree(device_status);
1677                 return rc;
1678         }
1679         case Z90STAT_STATUS_MASK: {
1680                 /* the old ioctl supports only 64 adapters */
1681                 char status[MAX_ZDEV_CARDIDS];
1682
1683                 zcrypt_status_mask(status, MAX_ZDEV_CARDIDS);
1684                 if (copy_to_user((char __user *)arg, status, sizeof(status)))
1685                         return -EFAULT;
1686                 return 0;
1687         }
1688         case Z90STAT_QDEPTH_MASK: {
1689                 /* the old ioctl supports only 64 adapters */
1690                 char qdepth[MAX_ZDEV_CARDIDS];
1691
1692                 zcrypt_qdepth_mask(qdepth, MAX_ZDEV_CARDIDS);
1693                 if (copy_to_user((char __user *)arg, qdepth, sizeof(qdepth)))
1694                         return -EFAULT;
1695                 return 0;
1696         }
1697         case Z90STAT_PERDEV_REQCNT: {
1698                 /* the old ioctl supports only 64 adapters */
1699                 u32 reqcnt[MAX_ZDEV_CARDIDS];
1700
1701                 zcrypt_perdev_reqcnt(reqcnt, MAX_ZDEV_CARDIDS);
1702                 if (copy_to_user((int __user *)arg, reqcnt, sizeof(reqcnt)))
1703                         return -EFAULT;
1704                 return 0;
1705         }
1706         /* unknown ioctl number */
1707         default:
1708                 pr_debug("unknown ioctl 0x%08x\n", cmd);
1709                 return -ENOIOCTLCMD;
1710         }
1711 }
1712
1713 #ifdef CONFIG_COMPAT
1714 /*
1715  * ioctl32 conversion routines
1716  */
1717 struct compat_ica_rsa_modexpo {
1718         compat_uptr_t   inputdata;
1719         unsigned int    inputdatalength;
1720         compat_uptr_t   outputdata;
1721         unsigned int    outputdatalength;
1722         compat_uptr_t   b_key;
1723         compat_uptr_t   n_modulus;
1724 };
1725
1726 static long trans_modexpo32(struct ap_perms *perms, struct file *filp,
1727                             unsigned int cmd, unsigned long arg)
1728 {
1729         struct compat_ica_rsa_modexpo __user *umex32 = compat_ptr(arg);
1730         struct compat_ica_rsa_modexpo mex32;
1731         struct ica_rsa_modexpo mex64;
1732         struct zcrypt_track tr;
1733         long rc;
1734
1735         memset(&tr, 0, sizeof(tr));
1736         if (copy_from_user(&mex32, umex32, sizeof(mex32)))
1737                 return -EFAULT;
1738         mex64.inputdata = compat_ptr(mex32.inputdata);
1739         mex64.inputdatalength = mex32.inputdatalength;
1740         mex64.outputdata = compat_ptr(mex32.outputdata);
1741         mex64.outputdatalength = mex32.outputdatalength;
1742         mex64.b_key = compat_ptr(mex32.b_key);
1743         mex64.n_modulus = compat_ptr(mex32.n_modulus);
1744         do {
1745                 rc = zcrypt_rsa_modexpo(perms, &tr, &mex64);
1746         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1747
1748         /* on ENODEV failure: retry once again after a requested rescan */
1749         if (rc == -ENODEV && zcrypt_process_rescan())
1750                 do {
1751                         rc = zcrypt_rsa_modexpo(perms, &tr, &mex64);
1752                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1753         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1754                 rc = -EIO;
1755         if (rc)
1756                 return rc;
1757         return put_user(mex64.outputdatalength,
1758                         &umex32->outputdatalength);
1759 }
1760
1761 struct compat_ica_rsa_modexpo_crt {
1762         compat_uptr_t   inputdata;
1763         unsigned int    inputdatalength;
1764         compat_uptr_t   outputdata;
1765         unsigned int    outputdatalength;
1766         compat_uptr_t   bp_key;
1767         compat_uptr_t   bq_key;
1768         compat_uptr_t   np_prime;
1769         compat_uptr_t   nq_prime;
1770         compat_uptr_t   u_mult_inv;
1771 };
1772
1773 static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp,
1774                                 unsigned int cmd, unsigned long arg)
1775 {
1776         struct compat_ica_rsa_modexpo_crt __user *ucrt32 = compat_ptr(arg);
1777         struct compat_ica_rsa_modexpo_crt crt32;
1778         struct ica_rsa_modexpo_crt crt64;
1779         struct zcrypt_track tr;
1780         long rc;
1781
1782         memset(&tr, 0, sizeof(tr));
1783         if (copy_from_user(&crt32, ucrt32, sizeof(crt32)))
1784                 return -EFAULT;
1785         crt64.inputdata = compat_ptr(crt32.inputdata);
1786         crt64.inputdatalength = crt32.inputdatalength;
1787         crt64.outputdata = compat_ptr(crt32.outputdata);
1788         crt64.outputdatalength = crt32.outputdatalength;
1789         crt64.bp_key = compat_ptr(crt32.bp_key);
1790         crt64.bq_key = compat_ptr(crt32.bq_key);
1791         crt64.np_prime = compat_ptr(crt32.np_prime);
1792         crt64.nq_prime = compat_ptr(crt32.nq_prime);
1793         crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv);
1794         do {
1795                 rc = zcrypt_rsa_crt(perms, &tr, &crt64);
1796         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1797
1798         /* on ENODEV failure: retry once again after a requested rescan */
1799         if (rc == -ENODEV && zcrypt_process_rescan())
1800                 do {
1801                         rc = zcrypt_rsa_crt(perms, &tr, &crt64);
1802                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1803         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1804                 rc = -EIO;
1805         if (rc)
1806                 return rc;
1807         return put_user(crt64.outputdatalength,
1808                         &ucrt32->outputdatalength);
1809 }
1810
1811 struct compat_ica_xcrb {
1812         unsigned short  agent_ID;
1813         unsigned int    user_defined;
1814         unsigned short  request_ID;
1815         unsigned int    request_control_blk_length;
1816         unsigned char   padding1[16 - sizeof(compat_uptr_t)];
1817         compat_uptr_t   request_control_blk_addr;
1818         unsigned int    request_data_length;
1819         char            padding2[16 - sizeof(compat_uptr_t)];
1820         compat_uptr_t   request_data_address;
1821         unsigned int    reply_control_blk_length;
1822         char            padding3[16 - sizeof(compat_uptr_t)];
1823         compat_uptr_t   reply_control_blk_addr;
1824         unsigned int    reply_data_length;
1825         char            padding4[16 - sizeof(compat_uptr_t)];
1826         compat_uptr_t   reply_data_addr;
1827         unsigned short  priority_window;
1828         unsigned int    status;
1829 } __packed;
1830
1831 static long trans_xcrb32(struct ap_perms *perms, struct file *filp,
1832                          unsigned int cmd, unsigned long arg)
1833 {
1834         struct compat_ica_xcrb __user *uxcrb32 = compat_ptr(arg);
1835         struct compat_ica_xcrb xcrb32;
1836         struct zcrypt_track tr;
1837         struct ica_xcRB xcrb64;
1838         long rc;
1839
1840         memset(&tr, 0, sizeof(tr));
1841         if (copy_from_user(&xcrb32, uxcrb32, sizeof(xcrb32)))
1842                 return -EFAULT;
1843         xcrb64.agent_ID = xcrb32.agent_ID;
1844         xcrb64.user_defined = xcrb32.user_defined;
1845         xcrb64.request_ID = xcrb32.request_ID;
1846         xcrb64.request_control_blk_length =
1847                 xcrb32.request_control_blk_length;
1848         xcrb64.request_control_blk_addr =
1849                 compat_ptr(xcrb32.request_control_blk_addr);
1850         xcrb64.request_data_length =
1851                 xcrb32.request_data_length;
1852         xcrb64.request_data_address =
1853                 compat_ptr(xcrb32.request_data_address);
1854         xcrb64.reply_control_blk_length =
1855                 xcrb32.reply_control_blk_length;
1856         xcrb64.reply_control_blk_addr =
1857                 compat_ptr(xcrb32.reply_control_blk_addr);
1858         xcrb64.reply_data_length = xcrb32.reply_data_length;
1859         xcrb64.reply_data_addr =
1860                 compat_ptr(xcrb32.reply_data_addr);
1861         xcrb64.priority_window = xcrb32.priority_window;
1862         xcrb64.status = xcrb32.status;
1863         do {
1864                 rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64);
1865         } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1866
1867         /* on ENODEV failure: retry once again after a requested rescan */
1868         if (rc == -ENODEV && zcrypt_process_rescan())
1869                 do {
1870                         rc = _zcrypt_send_cprb(true, perms, &tr, &xcrb64);
1871                 } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX);
1872         if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX)
1873                 rc = -EIO;
1874         xcrb32.reply_control_blk_length = xcrb64.reply_control_blk_length;
1875         xcrb32.reply_data_length = xcrb64.reply_data_length;
1876         xcrb32.status = xcrb64.status;
1877         if (copy_to_user(uxcrb32, &xcrb32, sizeof(xcrb32)))
1878                 return -EFAULT;
1879         return rc;
1880 }
1881
1882 static long zcrypt_compat_ioctl(struct file *filp, unsigned int cmd,
1883                                 unsigned long arg)
1884 {
1885         int rc;
1886         struct ap_perms *perms =
1887                 (struct ap_perms *)filp->private_data;
1888
1889         rc = zcrypt_check_ioctl(perms, cmd);
1890         if (rc)
1891                 return rc;
1892
1893         if (cmd == ICARSAMODEXPO)
1894                 return trans_modexpo32(perms, filp, cmd, arg);
1895         if (cmd == ICARSACRT)
1896                 return trans_modexpo_crt32(perms, filp, cmd, arg);
1897         if (cmd == ZSECSENDCPRB)
1898                 return trans_xcrb32(perms, filp, cmd, arg);
1899         return zcrypt_unlocked_ioctl(filp, cmd, arg);
1900 }
1901 #endif
1902
1903 /*
1904  * Misc device file operations.
1905  */
1906 static const struct file_operations zcrypt_fops = {
1907         .owner          = THIS_MODULE,
1908         .read           = zcrypt_read,
1909         .write          = zcrypt_write,
1910         .unlocked_ioctl = zcrypt_unlocked_ioctl,
1911 #ifdef CONFIG_COMPAT
1912         .compat_ioctl   = zcrypt_compat_ioctl,
1913 #endif
1914         .open           = zcrypt_open,
1915         .release        = zcrypt_release,
1916         .llseek         = no_llseek,
1917 };
1918
1919 /*
1920  * Misc device.
1921  */
1922 static struct miscdevice zcrypt_misc_device = {
1923         .minor      = MISC_DYNAMIC_MINOR,
1924         .name       = "z90crypt",
1925         .fops       = &zcrypt_fops,
1926 };
1927
1928 static int zcrypt_rng_device_count;
1929 static u32 *zcrypt_rng_buffer;
1930 static int zcrypt_rng_buffer_index;
1931 static DEFINE_MUTEX(zcrypt_rng_mutex);
1932
1933 static int zcrypt_rng_data_read(struct hwrng *rng, u32 *data)
1934 {
1935         int rc;
1936
1937         /*
1938          * We don't need locking here because the RNG API guarantees serialized
1939          * read method calls.
1940          */
1941         if (zcrypt_rng_buffer_index == 0) {
1942                 rc = zcrypt_rng((char *)zcrypt_rng_buffer);
1943                 /* on ENODEV failure: retry once again after an AP bus rescan */
1944                 if (rc == -ENODEV && zcrypt_process_rescan())
1945                         rc = zcrypt_rng((char *)zcrypt_rng_buffer);
1946                 if (rc < 0)
1947                         return -EIO;
1948                 zcrypt_rng_buffer_index = rc / sizeof(*data);
1949         }
1950         *data = zcrypt_rng_buffer[--zcrypt_rng_buffer_index];
1951         return sizeof(*data);
1952 }
1953
1954 static struct hwrng zcrypt_rng_dev = {
1955         .name           = "zcrypt",
1956         .data_read      = zcrypt_rng_data_read,
1957         .quality        = 990,
1958 };
1959
1960 int zcrypt_rng_device_add(void)
1961 {
1962         int rc = 0;
1963
1964         mutex_lock(&zcrypt_rng_mutex);
1965         if (zcrypt_rng_device_count == 0) {
1966                 zcrypt_rng_buffer = (u32 *)get_zeroed_page(GFP_KERNEL);
1967                 if (!zcrypt_rng_buffer) {
1968                         rc = -ENOMEM;
1969                         goto out;
1970                 }
1971                 zcrypt_rng_buffer_index = 0;
1972                 rc = hwrng_register(&zcrypt_rng_dev);
1973                 if (rc)
1974                         goto out_free;
1975                 zcrypt_rng_device_count = 1;
1976         } else {
1977                 zcrypt_rng_device_count++;
1978         }
1979         mutex_unlock(&zcrypt_rng_mutex);
1980         return 0;
1981
1982 out_free:
1983         free_page((unsigned long)zcrypt_rng_buffer);
1984 out:
1985         mutex_unlock(&zcrypt_rng_mutex);
1986         return rc;
1987 }
1988
1989 void zcrypt_rng_device_remove(void)
1990 {
1991         mutex_lock(&zcrypt_rng_mutex);
1992         zcrypt_rng_device_count--;
1993         if (zcrypt_rng_device_count == 0) {
1994                 hwrng_unregister(&zcrypt_rng_dev);
1995                 free_page((unsigned long)zcrypt_rng_buffer);
1996         }
1997         mutex_unlock(&zcrypt_rng_mutex);
1998 }
1999
2000 /*
2001  * Wait until the zcrypt api is operational.
2002  * The AP bus scan and the binding of ap devices to device drivers is
2003  * an asynchronous job. This function waits until these initial jobs
2004  * are done and so the zcrypt api should be ready to serve crypto
2005  * requests - if there are resources available. The function uses an
2006  * internal timeout of 30s. The very first caller will either wait for
2007  * ap bus bindings complete or the timeout happens. This state will be
2008  * remembered for further callers which will only be blocked until a
2009  * decision is made (timeout or bindings complete).
2010  * On timeout -ETIME is returned, on success the return value is 0.
2011  */
2012 int zcrypt_wait_api_operational(void)
2013 {
2014         static DEFINE_MUTEX(zcrypt_wait_api_lock);
2015         static int zcrypt_wait_api_state;
2016         int rc;
2017
2018         rc = mutex_lock_interruptible(&zcrypt_wait_api_lock);
2019         if (rc)
2020                 return rc;
2021
2022         switch (zcrypt_wait_api_state) {
2023         case 0:
2024                 /* initial state, invoke wait for the ap bus complete */
2025                 rc = ap_wait_apqn_bindings_complete(
2026                         msecs_to_jiffies(ZCRYPT_WAIT_BINDINGS_COMPLETE_MS));
2027                 switch (rc) {
2028                 case 0:
2029                         /* ap bus bindings are complete */
2030                         zcrypt_wait_api_state = 1;
2031                         break;
2032                 case -EINTR:
2033                         /* interrupted, go back to caller */
2034                         break;
2035                 case -ETIME:
2036                         /* timeout */
2037                         ZCRYPT_DBF_WARN("%s ap_wait_init_apqn_bindings_complete()=ETIME\n",
2038                                         __func__);
2039                         zcrypt_wait_api_state = -ETIME;
2040                         break;
2041                 default:
2042                         /* other failure */
2043                         pr_debug("%s ap_wait_init_apqn_bindings_complete()=%d\n",
2044                                  __func__, rc);
2045                         break;
2046                 }
2047                 break;
2048         case 1:
2049                 /* a previous caller already found ap bus bindings complete */
2050                 rc = 0;
2051                 break;
2052         default:
2053                 /* a previous caller had timeout or other failure */
2054                 rc = zcrypt_wait_api_state;
2055                 break;
2056         }
2057
2058         mutex_unlock(&zcrypt_wait_api_lock);
2059
2060         return rc;
2061 }
2062 EXPORT_SYMBOL(zcrypt_wait_api_operational);
2063
2064 int __init zcrypt_debug_init(void)
2065 {
2066         zcrypt_dbf_info = debug_register("zcrypt", 2, 1,
2067                                          ZCRYPT_DBF_MAX_SPRINTF_ARGS * sizeof(long));
2068         debug_register_view(zcrypt_dbf_info, &debug_sprintf_view);
2069         debug_set_level(zcrypt_dbf_info, DBF_ERR);
2070
2071         return 0;
2072 }
2073
2074 void zcrypt_debug_exit(void)
2075 {
2076         debug_unregister(zcrypt_dbf_info);
2077 }
2078
2079 static int __init zcdn_init(void)
2080 {
2081         int rc;
2082
2083         /* create a new class 'zcrypt' */
2084         rc = class_register(&zcrypt_class);
2085         if (rc)
2086                 goto out_class_register_failed;
2087
2088         /* alloc device minor range */
2089         rc = alloc_chrdev_region(&zcrypt_devt,
2090                                  0, ZCRYPT_MAX_MINOR_NODES,
2091                                  ZCRYPT_NAME);
2092         if (rc)
2093                 goto out_alloc_chrdev_failed;
2094
2095         cdev_init(&zcrypt_cdev, &zcrypt_fops);
2096         zcrypt_cdev.owner = THIS_MODULE;
2097         rc = cdev_add(&zcrypt_cdev, zcrypt_devt, ZCRYPT_MAX_MINOR_NODES);
2098         if (rc)
2099                 goto out_cdev_add_failed;
2100
2101         /* need some class specific sysfs attributes */
2102         rc = class_create_file(&zcrypt_class, &class_attr_zcdn_create);
2103         if (rc)
2104                 goto out_class_create_file_1_failed;
2105         rc = class_create_file(&zcrypt_class, &class_attr_zcdn_destroy);
2106         if (rc)
2107                 goto out_class_create_file_2_failed;
2108
2109         return 0;
2110
2111 out_class_create_file_2_failed:
2112         class_remove_file(&zcrypt_class, &class_attr_zcdn_create);
2113 out_class_create_file_1_failed:
2114         cdev_del(&zcrypt_cdev);
2115 out_cdev_add_failed:
2116         unregister_chrdev_region(zcrypt_devt, ZCRYPT_MAX_MINOR_NODES);
2117 out_alloc_chrdev_failed:
2118         class_unregister(&zcrypt_class);
2119 out_class_register_failed:
2120         return rc;
2121 }
2122
2123 static void zcdn_exit(void)
2124 {
2125         class_remove_file(&zcrypt_class, &class_attr_zcdn_create);
2126         class_remove_file(&zcrypt_class, &class_attr_zcdn_destroy);
2127         zcdn_destroy_all();
2128         cdev_del(&zcrypt_cdev);
2129         unregister_chrdev_region(zcrypt_devt, ZCRYPT_MAX_MINOR_NODES);
2130         class_unregister(&zcrypt_class);
2131 }
2132
2133 /*
2134  * zcrypt_api_init(): Module initialization.
2135  *
2136  * The module initialization code.
2137  */
2138 int __init zcrypt_api_init(void)
2139 {
2140         int rc;
2141
2142         rc = zcrypt_debug_init();
2143         if (rc)
2144                 goto out;
2145
2146         rc = zcdn_init();
2147         if (rc)
2148                 goto out;
2149
2150         /* Register the request sprayer. */
2151         rc = misc_register(&zcrypt_misc_device);
2152         if (rc < 0)
2153                 goto out_misc_register_failed;
2154
2155         zcrypt_msgtype6_init();
2156         zcrypt_msgtype50_init();
2157
2158         return 0;
2159
2160 out_misc_register_failed:
2161         zcdn_exit();
2162         zcrypt_debug_exit();
2163 out:
2164         return rc;
2165 }
2166
2167 /*
2168  * zcrypt_api_exit(): Module termination.
2169  *
2170  * The module termination code.
2171  */
2172 void __exit zcrypt_api_exit(void)
2173 {
2174         zcdn_exit();
2175         misc_deregister(&zcrypt_misc_device);
2176         zcrypt_msgtype6_exit();
2177         zcrypt_msgtype50_exit();
2178         zcrypt_ccamisc_exit();
2179         zcrypt_ep11misc_exit();
2180         zcrypt_debug_exit();
2181 }
2182
2183 module_init(zcrypt_api_init);
2184 module_exit(zcrypt_api_exit);