drivers/nvdimm/bus.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   4  */
   5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   6 #include <linux/libnvdimm.h>
   7 #include <linux/sched/mm.h>
   8 #include <linux/vmalloc.h>
   9 #include <linux/uaccess.h>
  10 #include <linux/module.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/fcntl.h>
  13 #include <linux/async.h>
  14 #include <linux/genhd.h>
  15 #include <linux/ndctl.h>
  16 #include <linux/sched.h>
  17 #include <linux/slab.h>
  18 #include <linux/cpu.h>
  19 #include <linux/fs.h>
  20 #include <linux/io.h>
  21 #include <linux/mm.h>
  22 #include <linux/nd.h>
  23 #include "nd-core.h"
  24 #include "nd.h"
  25 #include "pfn.h"
  26
  27 int nvdimm_major;
  28 static int nvdimm_bus_major;
  29 struct class *nd_class;
  30 static DEFINE_IDA(nd_ida);
  31
  32 static int to_nd_device_type(struct device *dev)
  33 {
  34         if (is_nvdimm(dev))
  35                 return ND_DEVICE_DIMM;
  36         else if (is_memory(dev))
  37                 return ND_DEVICE_REGION_PMEM;
  38         else if (is_nd_blk(dev))
  39                 return ND_DEVICE_REGION_BLK;
  40         else if (is_nd_dax(dev))
  41                 return ND_DEVICE_DAX_PMEM;
  42         else if (is_nd_region(dev->parent))
  43                 return nd_region_to_nstype(to_nd_region(dev->parent));
  44
  45         return 0;
  46 }
  47
  48 static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  49 {
  50         return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
  51                         to_nd_device_type(dev));
  52 }
  53
  54 static struct module *to_bus_provider(struct device *dev)
  55 {
  56         /* pin bus providers while regions are enabled */
  57         if (is_nd_region(dev)) {
  58                 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  59
  60                 return nvdimm_bus->nd_desc->module;
  61         }
  62         return NULL;
  63 }
  64
  65 static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
  66 {
  67         nvdimm_bus_lock(&nvdimm_bus->dev);
  68         nvdimm_bus->probe_active++;
  69         nvdimm_bus_unlock(&nvdimm_bus->dev);
  70 }
  71
  72 static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
  73 {
  74         nvdimm_bus_lock(&nvdimm_bus->dev);
  75         if (--nvdimm_bus->probe_active == 0)
  76                 wake_up(&nvdimm_bus->wait);
  77         nvdimm_bus_unlock(&nvdimm_bus->dev);
  78 }
  79
  80 static int nvdimm_bus_probe(struct device *dev)
  81 {
  82         struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
  83         struct module *provider = to_bus_provider(dev);
  84         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  85         int rc;
  86
  87         if (!try_module_get(provider))
  88                 return -ENXIO;
  89
  90         dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
  91                         dev->driver->name, dev_name(dev));
  92
  93         nvdimm_bus_probe_start(nvdimm_bus);
  94         debug_nvdimm_lock(dev);
  95         rc = nd_drv->probe(dev);
  96         debug_nvdimm_unlock(dev);
  97
  98         if ((rc == 0 || rc == -EOPNOTSUPP) &&
  99                         dev->parent && is_nd_region(dev->parent))
 100                 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
 101         nvdimm_bus_probe_end(nvdimm_bus);
 102
 103         dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
 104                         dev_name(dev), rc);
 105
 106         if (rc != 0)
 107                 module_put(provider);
 108         return rc;
 109 }
 110
 111 static void nvdimm_bus_remove(struct device *dev)
 112 {
 113         struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
 114         struct module *provider = to_bus_provider(dev);
 115         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 116
 117         if (nd_drv->remove) {
 118                 debug_nvdimm_lock(dev);
 119                 nd_drv->remove(dev);
 120                 debug_nvdimm_unlock(dev);
 121         }
 122
 123         dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
 124                         dev_name(dev));
 125         module_put(provider);
 126 }
 127
 128 static void nvdimm_bus_shutdown(struct device *dev)
 129 {
 130         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 131         struct nd_device_driver *nd_drv = NULL;
 132
 133         if (dev->driver)
 134                 nd_drv = to_nd_device_driver(dev->driver);
 135
 136         if (nd_drv && nd_drv->shutdown) {
 137                 nd_drv->shutdown(dev);
 138                 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
 139                                 dev->driver->name, dev_name(dev));
 140         }
 141 }
 142
 143 void nd_device_notify(struct device *dev, enum nvdimm_event event)
 144 {
 145         nd_device_lock(dev);
 146         if (dev->driver) {
 147                 struct nd_device_driver *nd_drv;
 148
 149                 nd_drv = to_nd_device_driver(dev->driver);
 150                 if (nd_drv->notify)
 151                         nd_drv->notify(dev, event);
 152         }
 153         nd_device_unlock(dev);
 154 }
 155 EXPORT_SYMBOL(nd_device_notify);
 156
 157 void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
 158 {
 159         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
 160
 161         if (!nvdimm_bus)
 162                 return;
 163
 164         /* caller is responsible for holding a reference on the device */
 165         nd_device_notify(&nd_region->dev, event);
 166 }
 167 EXPORT_SYMBOL_GPL(nvdimm_region_notify);
 168
 169 struct clear_badblocks_context {
 170         resource_size_t phys, cleared;
 171 };
 172
 173 static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
 174 {
 175         struct clear_badblocks_context *ctx = data;
 176         struct nd_region *nd_region;
 177         resource_size_t ndr_end;
 178         sector_t sector;
 179
 180         /* make sure device is a region */
 181         if (!is_memory(dev))
 182                 return 0;
 183
 184         nd_region = to_nd_region(dev);
 185         ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
 186
 187         /* make sure we are in the region */
 188         if (ctx->phys < nd_region->ndr_start
 189                         || (ctx->phys + ctx->cleared) > ndr_end)
 190                 return 0;
 191
 192         sector = (ctx->phys - nd_region->ndr_start) / 512;
 193         badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
 194
 195         if (nd_region->bb_state)
 196                 sysfs_notify_dirent(nd_region->bb_state);
 197
 198         return 0;
 199 }
 200
 201 static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
 202                 phys_addr_t phys, u64 cleared)
 203 {
 204         struct clear_badblocks_context ctx = {
 205                 .phys = phys,
 206                 .cleared = cleared,
 207         };
 208
 209         device_for_each_child(&nvdimm_bus->dev, &ctx,
 210                         nvdimm_clear_badblocks_region);
 211 }
 212
 213 static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
 214                 phys_addr_t phys, u64 cleared)
 215 {
 216         if (cleared > 0)
 217                 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
 218
 219         if (cleared > 0 && cleared / 512)
 220                 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
 221 }
 222
 223 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
 224                 unsigned int len)
 225 {
 226         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 227         struct nvdimm_bus_descriptor *nd_desc;
 228         struct nd_cmd_clear_error clear_err;
 229         struct nd_cmd_ars_cap ars_cap;
 230         u32 clear_err_unit, mask;
 231         unsigned int noio_flag;
 232         int cmd_rc, rc;
 233
 234         if (!nvdimm_bus)
 235                 return -ENXIO;
 236
 237         nd_desc = nvdimm_bus->nd_desc;
 238         /*
 239          * if ndctl does not exist, it's PMEM_LEGACY and
 240          * we want to just pretend everything is handled.
 241          */
 242         if (!nd_desc->ndctl)
 243                 return len;
 244
 245         memset(&ars_cap, 0, sizeof(ars_cap));
 246         ars_cap.address = phys;
 247         ars_cap.length = len;
 248         noio_flag = memalloc_noio_save();
 249         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
 250                         sizeof(ars_cap), &cmd_rc);
 251         memalloc_noio_restore(noio_flag);
 252         if (rc < 0)
 253                 return rc;
 254         if (cmd_rc < 0)
 255                 return cmd_rc;
 256         clear_err_unit = ars_cap.clear_err_unit;
 257         if (!clear_err_unit || !is_power_of_2(clear_err_unit))
 258                 return -ENXIO;
 259
 260         mask = clear_err_unit - 1;
 261         if ((phys | len) & mask)
 262                 return -ENXIO;
 263         memset(&clear_err, 0, sizeof(clear_err));
 264         clear_err.address = phys;
 265         clear_err.length = len;
 266         noio_flag = memalloc_noio_save();
 267         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
 268                         sizeof(clear_err), &cmd_rc);
 269         memalloc_noio_restore(noio_flag);
 270         if (rc < 0)
 271                 return rc;
 272         if (cmd_rc < 0)
 273                 return cmd_rc;
 274
 275         nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
 276
 277         return clear_err.cleared;
 278 }
 279 EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
 280
 281 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
 282
 283 static struct bus_type nvdimm_bus_type = {
 284         .name = "nd",
 285         .uevent = nvdimm_bus_uevent,
 286         .match = nvdimm_bus_match,
 287         .probe = nvdimm_bus_probe,
 288         .remove = nvdimm_bus_remove,
 289         .shutdown = nvdimm_bus_shutdown,
 290 };
 291
 292 static void nvdimm_bus_release(struct device *dev)
 293 {
 294         struct nvdimm_bus *nvdimm_bus;
 295
 296         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
 297         ida_simple_remove(&nd_ida, nvdimm_bus->id);
 298         kfree(nvdimm_bus);
 299 }
 300
 301 static const struct device_type nvdimm_bus_dev_type = {
 302         .release = nvdimm_bus_release,
 303         .groups = nvdimm_bus_attribute_groups,
 304 };
 305
 306 bool is_nvdimm_bus(struct device *dev)
 307 {
 308         return dev->type == &nvdimm_bus_dev_type;
 309 }
 310
 311 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
 312 {
 313         struct device *dev;
 314
 315         for (dev = nd_dev; dev; dev = dev->parent)
 316                 if (is_nvdimm_bus(dev))
 317                         break;
 318         dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
 319         if (dev)
 320                 return to_nvdimm_bus(dev);
 321         return NULL;
 322 }
 323
 324 struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
 325 {
 326         struct nvdimm_bus *nvdimm_bus;
 327
 328         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
 329         WARN_ON(!is_nvdimm_bus(dev));
 330         return nvdimm_bus;
 331 }
 332 EXPORT_SYMBOL_GPL(to_nvdimm_bus);
 333
 334 struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
 335 {
 336         return to_nvdimm_bus(nvdimm->dev.parent);
 337 }
 338 EXPORT_SYMBOL_GPL(nvdimm_to_bus);
 339
 340 struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
 341                 struct nvdimm_bus_descriptor *nd_desc)
 342 {
 343         struct nvdimm_bus *nvdimm_bus;
 344         int rc;
 345
 346         nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
 347         if (!nvdimm_bus)
 348                 return NULL;
 349         INIT_LIST_HEAD(&nvdimm_bus->list);
 350         INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
 351         init_waitqueue_head(&nvdimm_bus->wait);
 352         nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
 353         if (nvdimm_bus->id < 0) {
 354                 kfree(nvdimm_bus);
 355                 return NULL;
 356         }
 357         mutex_init(&nvdimm_bus->reconfig_mutex);
 358         badrange_init(&nvdimm_bus->badrange);
 359         nvdimm_bus->nd_desc = nd_desc;
 360         nvdimm_bus->dev.parent = parent;
 361         nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
 362         nvdimm_bus->dev.groups = nd_desc->attr_groups;
 363         nvdimm_bus->dev.bus = &nvdimm_bus_type;
 364         nvdimm_bus->dev.of_node = nd_desc->of_node;
 365         device_initialize(&nvdimm_bus->dev);
 366         device_set_pm_not_required(&nvdimm_bus->dev);
 367         rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
 368         if (rc)
 369                 goto err;
 370
 371         rc = device_add(&nvdimm_bus->dev);
 372         if (rc) {
 373                 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
 374                 goto err;
 375         }
 376
 377         return nvdimm_bus;
 378  err:
 379         put_device(&nvdimm_bus->dev);
 380         return NULL;
 381 }
 382 EXPORT_SYMBOL_GPL(nvdimm_bus_register);
 383
 384 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
 385 {
 386         if (!nvdimm_bus)
 387                 return;
 388         device_unregister(&nvdimm_bus->dev);
 389 }
 390 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
 391
 392 static int child_unregister(struct device *dev, void *data)
 393 {
 394         /*
 395          * the singular ndctl class device per bus needs to be
 396          * "device_destroy"ed, so skip it here
 397          *
 398          * i.e. remove classless children
 399          */
 400         if (dev->class)
 401                 return 0;
 402
 403         if (is_nvdimm(dev))
 404                 nvdimm_delete(to_nvdimm(dev));
 405         else
 406                 nd_device_unregister(dev, ND_SYNC);
 407
 408         return 0;
 409 }
 410
 411 static void free_badrange_list(struct list_head *badrange_list)
 412 {
 413         struct badrange_entry *bre, *next;
 414
 415         list_for_each_entry_safe(bre, next, badrange_list, list) {
 416                 list_del(&bre->list);
 417                 kfree(bre);
 418         }
 419         list_del_init(badrange_list);
 420 }
 421
 422 static void nd_bus_remove(struct device *dev)
 423 {
 424         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 425
 426         mutex_lock(&nvdimm_bus_list_mutex);
 427         list_del_init(&nvdimm_bus->list);
 428         mutex_unlock(&nvdimm_bus_list_mutex);
 429
 430         wait_event(nvdimm_bus->wait,
 431                         atomic_read(&nvdimm_bus->ioctl_active) == 0);
 432
 433         nd_synchronize();
 434         device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
 435
 436         spin_lock(&nvdimm_bus->badrange.lock);
 437         free_badrange_list(&nvdimm_bus->badrange.list);
 438         spin_unlock(&nvdimm_bus->badrange.lock);
 439
 440         nvdimm_bus_destroy_ndctl(nvdimm_bus);
 441 }
 442
 443 static int nd_bus_probe(struct device *dev)
 444 {
 445         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 446         int rc;
 447
 448         rc = nvdimm_bus_create_ndctl(nvdimm_bus);
 449         if (rc)
 450                 return rc;
 451
 452         mutex_lock(&nvdimm_bus_list_mutex);
 453         list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
 454         mutex_unlock(&nvdimm_bus_list_mutex);
 455
 456         /* enable bus provider attributes to look up their local context */
 457         dev_set_drvdata(dev, nvdimm_bus->nd_desc);
 458
 459         return 0;
 460 }
 461
 462 static struct nd_device_driver nd_bus_driver = {
 463         .probe = nd_bus_probe,
 464         .remove = nd_bus_remove,
 465         .drv = {
 466                 .name = "nd_bus",
 467                 .suppress_bind_attrs = true,
 468                 .bus = &nvdimm_bus_type,
 469                 .owner = THIS_MODULE,
 470                 .mod_name = KBUILD_MODNAME,
 471         },
 472 };
 473
 474 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
 475 {
 476         struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
 477
 478         if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
 479                 return true;
 480
 481         return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
 482 }
 483
 484 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
 485
 486 void nd_synchronize(void)
 487 {
 488         async_synchronize_full_domain(&nd_async_domain);
 489 }
 490 EXPORT_SYMBOL_GPL(nd_synchronize);
 491
 492 static void nd_async_device_register(void *d, async_cookie_t cookie)
 493 {
 494         struct device *dev = d;
 495
 496         if (device_add(dev) != 0) {
 497                 dev_err(dev, "%s: failed\n", __func__);
 498                 put_device(dev);
 499         }
 500         put_device(dev);
 501         if (dev->parent)
 502                 put_device(dev->parent);
 503 }
 504
 505 static void nd_async_device_unregister(void *d, async_cookie_t cookie)
 506 {
 507         struct device *dev = d;
 508
 509         /* flush bus operations before delete */
 510         nvdimm_bus_lock(dev);
 511         nvdimm_bus_unlock(dev);
 512
 513         device_unregister(dev);
 514         put_device(dev);
 515 }
 516
 517 void __nd_device_register(struct device *dev)
 518 {
 519         if (!dev)
 520                 return;
 521
 522         /*
 523          * Ensure that region devices always have their NUMA node set as
 524          * early as possible. This way we are able to make certain that
 525          * any memory associated with the creation and the creation
 526          * itself of the region is associated with the correct node.
 527          */
 528         if (is_nd_region(dev))
 529                 set_dev_node(dev, to_nd_region(dev)->numa_node);
 530
 531         dev->bus = &nvdimm_bus_type;
 532         device_set_pm_not_required(dev);
 533         if (dev->parent) {
 534                 get_device(dev->parent);
 535                 if (dev_to_node(dev) == NUMA_NO_NODE)
 536                         set_dev_node(dev, dev_to_node(dev->parent));
 537         }
 538         get_device(dev);
 539
 540         async_schedule_dev_domain(nd_async_device_register, dev,
 541                                   &nd_async_domain);
 542 }
 543
 544 void nd_device_register(struct device *dev)
 545 {
 546         device_initialize(dev);
 547         __nd_device_register(dev);
 548 }
 549 EXPORT_SYMBOL(nd_device_register);
 550
 551 void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
 552 {
 553         bool killed;
 554
 555         switch (mode) {
 556         case ND_ASYNC:
 557                 /*
 558                  * In the async case this is being triggered with the
 559                  * device lock held and the unregistration work needs to
 560                  * be moved out of line iff this is thread has won the
 561                  * race to schedule the deletion.
 562                  */
 563                 if (!kill_device(dev))
 564                         return;
 565
 566                 get_device(dev);
 567                 async_schedule_domain(nd_async_device_unregister, dev,
 568                                 &nd_async_domain);
 569                 break;
 570         case ND_SYNC:
 571                 /*
 572                  * In the sync case the device is being unregistered due
 573                  * to a state change of the parent. Claim the kill state
 574                  * to synchronize against other unregistration requests,
 575                  * or otherwise let the async path handle it if the
 576                  * unregistration was already queued.
 577                  */
 578                 nd_device_lock(dev);
 579                 killed = kill_device(dev);
 580                 nd_device_unlock(dev);
 581
 582                 if (!killed)
 583                         return;
 584
 585                 nd_synchronize();
 586                 device_unregister(dev);
 587                 break;
 588         }
 589 }
 590 EXPORT_SYMBOL(nd_device_unregister);
 591
 592 /**
 593  * __nd_driver_register() - register a region or a namespace driver
 594  * @nd_drv: driver to register
 595  * @owner: automatically set by nd_driver_register() macro
 596  * @mod_name: automatically set by nd_driver_register() macro
 597  */
 598 int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
 599                 const char *mod_name)
 600 {
 601         struct device_driver *drv = &nd_drv->drv;
 602
 603         if (!nd_drv->type) {
 604                 pr_debug("driver type bitmask not set (%ps)\n",
 605                                 __builtin_return_address(0));
 606                 return -EINVAL;
 607         }
 608
 609         if (!nd_drv->probe) {
 610                 pr_debug("%s ->probe() must be specified\n", mod_name);
 611                 return -EINVAL;
 612         }
 613
 614         drv->bus = &nvdimm_bus_type;
 615         drv->owner = owner;
 616         drv->mod_name = mod_name;
 617
 618         return driver_register(drv);
 619 }
 620 EXPORT_SYMBOL(__nd_driver_register);
 621
 622 void nvdimm_check_and_set_ro(struct gendisk *disk)
 623 {
 624         struct device *dev = disk_to_dev(disk)->parent;
 625         struct nd_region *nd_region = to_nd_region(dev->parent);
 626         int disk_ro = get_disk_ro(disk);
 627
 628         /* catch the disk up with the region ro state */
 629         if (disk_ro == nd_region->ro)
 630                 return;
 631
 632         dev_info(dev, "%s read-%s, marking %s read-%s\n",
 633                  dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
 634                  disk->disk_name, nd_region->ro ? "only" : "write");
 635         set_disk_ro(disk, nd_region->ro);
 636 }
 637 EXPORT_SYMBOL(nvdimm_check_and_set_ro);
 638
 639 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
 640                 char *buf)
 641 {
 642         return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
 643                         to_nd_device_type(dev));
 644 }
 645 static DEVICE_ATTR_RO(modalias);
 646
 647 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
 648                 char *buf)
 649 {
 650         return sprintf(buf, "%s\n", dev->type->name);
 651 }
 652 static DEVICE_ATTR_RO(devtype);
 653
 654 static struct attribute *nd_device_attributes[] = {
 655         &dev_attr_modalias.attr,
 656         &dev_attr_devtype.attr,
 657         NULL,
 658 };
 659
 660 /*
 661  * nd_device_attribute_group - generic attributes for all devices on an nd bus
 662  */
 663 const struct attribute_group nd_device_attribute_group = {
 664         .attrs = nd_device_attributes,
 665 };
 666
 667 static ssize_t numa_node_show(struct device *dev,
 668                 struct device_attribute *attr, char *buf)
 669 {
 670         return sprintf(buf, "%d\n", dev_to_node(dev));
 671 }
 672 static DEVICE_ATTR_RO(numa_node);
 673
 674 static int nvdimm_dev_to_target_node(struct device *dev)
 675 {
 676         struct device *parent = dev->parent;
 677         struct nd_region *nd_region = NULL;
 678
 679         if (is_nd_region(dev))
 680                 nd_region = to_nd_region(dev);
 681         else if (parent && is_nd_region(parent))
 682                 nd_region = to_nd_region(parent);
 683
 684         if (!nd_region)
 685                 return NUMA_NO_NODE;
 686         return nd_region->target_node;
 687 }
 688
 689 static ssize_t target_node_show(struct device *dev,
 690                 struct device_attribute *attr, char *buf)
 691 {
 692         return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
 693 }
 694 static DEVICE_ATTR_RO(target_node);
 695
 696 static struct attribute *nd_numa_attributes[] = {
 697         &dev_attr_numa_node.attr,
 698         &dev_attr_target_node.attr,
 699         NULL,
 700 };
 701
 702 static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
 703                 int n)
 704 {
 705         struct device *dev = container_of(kobj, typeof(*dev), kobj);
 706
 707         if (!IS_ENABLED(CONFIG_NUMA))
 708                 return 0;
 709
 710         if (a == &dev_attr_target_node.attr &&
 711                         nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
 712                 return 0;
 713
 714         return a->mode;
 715 }
 716
 717 /*
 718  * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
 719  */
 720 const struct attribute_group nd_numa_attribute_group = {
 721         .attrs = nd_numa_attributes,
 722         .is_visible = nd_numa_attr_visible,
 723 };
 724
 725 static void ndctl_release(struct device *dev)
 726 {
 727         kfree(dev);
 728 }
 729
 730 int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
 731 {
 732         dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
 733         struct device *dev;
 734         int rc;
 735
 736         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 737         if (!dev)
 738                 return -ENOMEM;
 739         device_initialize(dev);
 740         device_set_pm_not_required(dev);
 741         dev->class = nd_class;
 742         dev->parent = &nvdimm_bus->dev;
 743         dev->devt = devt;
 744         dev->release = ndctl_release;
 745         rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id);
 746         if (rc)
 747                 goto err;
 748
 749         rc = device_add(dev);
 750         if (rc) {
 751                 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
 752                                 nvdimm_bus->id, rc);
 753                 goto err;
 754         }
 755         return 0;
 756
 757 err:
 758         put_device(dev);
 759         return rc;
 760 }
 761
 762 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
 763 {
 764         device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
 765 }
 766
 767 static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
 768         [ND_CMD_IMPLEMENTED] = { },
 769         [ND_CMD_SMART] = {
 770                 .out_num = 2,
 771                 .out_sizes = { 4, 128, },
 772         },
 773         [ND_CMD_SMART_THRESHOLD] = {
 774                 .out_num = 2,
 775                 .out_sizes = { 4, 8, },
 776         },
 777         [ND_CMD_DIMM_FLAGS] = {
 778                 .out_num = 2,
 779                 .out_sizes = { 4, 4 },
 780         },
 781         [ND_CMD_GET_CONFIG_SIZE] = {
 782                 .out_num = 3,
 783                 .out_sizes = { 4, 4, 4, },
 784         },
 785         [ND_CMD_GET_CONFIG_DATA] = {
 786                 .in_num = 2,
 787                 .in_sizes = { 4, 4, },
 788                 .out_num = 2,
 789                 .out_sizes = { 4, UINT_MAX, },
 790         },
 791         [ND_CMD_SET_CONFIG_DATA] = {
 792                 .in_num = 3,
 793                 .in_sizes = { 4, 4, UINT_MAX, },
 794                 .out_num = 1,
 795                 .out_sizes = { 4, },
 796         },
 797         [ND_CMD_VENDOR] = {
 798                 .in_num = 3,
 799                 .in_sizes = { 4, 4, UINT_MAX, },
 800                 .out_num = 3,
 801                 .out_sizes = { 4, 4, UINT_MAX, },
 802         },
 803         [ND_CMD_CALL] = {
 804                 .in_num = 2,
 805                 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
 806                 .out_num = 1,
 807                 .out_sizes = { UINT_MAX, },
 808         },
 809 };
 810
 811 const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
 812 {
 813         if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
 814                 return &__nd_cmd_dimm_descs[cmd];
 815         return NULL;
 816 }
 817 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
 818
 819 static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
 820         [ND_CMD_IMPLEMENTED] = { },
 821         [ND_CMD_ARS_CAP] = {
 822                 .in_num = 2,
 823                 .in_sizes = { 8, 8, },
 824                 .out_num = 4,
 825                 .out_sizes = { 4, 4, 4, 4, },
 826         },
 827         [ND_CMD_ARS_START] = {
 828                 .in_num = 5,
 829                 .in_sizes = { 8, 8, 2, 1, 5, },
 830                 .out_num = 2,
 831                 .out_sizes = { 4, 4, },
 832         },
 833         [ND_CMD_ARS_STATUS] = {
 834                 .out_num = 3,
 835                 .out_sizes = { 4, 4, UINT_MAX, },
 836         },
 837         [ND_CMD_CLEAR_ERROR] = {
 838                 .in_num = 2,
 839                 .in_sizes = { 8, 8, },
 840                 .out_num = 3,
 841                 .out_sizes = { 4, 4, 8, },
 842         },
 843         [ND_CMD_CALL] = {
 844                 .in_num = 2,
 845                 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
 846                 .out_num = 1,
 847                 .out_sizes = { UINT_MAX, },
 848         },
 849 };
 850
 851 const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
 852 {
 853         if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
 854                 return &__nd_cmd_bus_descs[cmd];
 855         return NULL;
 856 }
 857 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
 858
 859 u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
 860                 const struct nd_cmd_desc *desc, int idx, void *buf)
 861 {
 862         if (idx >= desc->in_num)
 863                 return UINT_MAX;
 864
 865         if (desc->in_sizes[idx] < UINT_MAX)
 866                 return desc->in_sizes[idx];
 867
 868         if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
 869                 struct nd_cmd_set_config_hdr *hdr = buf;
 870
 871                 return hdr->in_length;
 872         } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
 873                 struct nd_cmd_vendor_hdr *hdr = buf;
 874
 875                 return hdr->in_length;
 876         } else if (cmd == ND_CMD_CALL) {
 877                 struct nd_cmd_pkg *pkg = buf;
 878
 879                 return pkg->nd_size_in;
 880         }
 881
 882         return UINT_MAX;
 883 }
 884 EXPORT_SYMBOL_GPL(nd_cmd_in_size);
 885
 886 u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
 887                 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
 888                 const u32 *out_field, unsigned long remainder)
 889 {
 890         if (idx >= desc->out_num)
 891                 return UINT_MAX;
 892
 893         if (desc->out_sizes[idx] < UINT_MAX)
 894                 return desc->out_sizes[idx];
 895
 896         if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
 897                 return in_field[1];
 898         else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
 899                 return out_field[1];
 900         else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
 901                 /*
 902                  * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
 903                  * "Size of Output Buffer in bytes, including this
 904                  * field."
 905                  */
 906                 if (out_field[1] < 4)
 907                         return 0;
 908                 /*
 909                  * ACPI 6.1 is ambiguous if 'status' is included in the
 910                  * output size. If we encounter an output size that
 911                  * overshoots the remainder by 4 bytes, assume it was
 912                  * including 'status'.
 913                  */
 914                 if (out_field[1] - 4 == remainder)
 915                         return remainder;
 916                 return out_field[1] - 8;
 917         } else if (cmd == ND_CMD_CALL) {
 918                 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
 919
 920                 return pkg->nd_size_out;
 921         }
 922
 923
 924         return UINT_MAX;
 925 }
 926 EXPORT_SYMBOL_GPL(nd_cmd_out_size);
 927
 928 void wait_nvdimm_bus_probe_idle(struct device *dev)
 929 {
 930         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 931
 932         do {
 933                 if (nvdimm_bus->probe_active == 0)
 934                         break;
 935                 nvdimm_bus_unlock(dev);
 936                 nd_device_unlock(dev);
 937                 wait_event(nvdimm_bus->wait,
 938                                 nvdimm_bus->probe_active == 0);
 939                 nd_device_lock(dev);
 940                 nvdimm_bus_lock(dev);
 941         } while (true);
 942 }
 943
 944 static int nd_pmem_forget_poison_check(struct device *dev, void *data)
 945 {
 946         struct nd_cmd_clear_error *clear_err =
 947                 (struct nd_cmd_clear_error *)data;
 948         struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
 949         struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
 950         struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
 951         struct nd_namespace_common *ndns = NULL;
 952         struct nd_namespace_io *nsio;
 953         resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
 954
 955         if (nd_dax || !dev->driver)
 956                 return 0;
 957
 958         start = clear_err->address;
 959         end = clear_err->address + clear_err->cleared - 1;
 960
 961         if (nd_btt || nd_pfn || nd_dax) {
 962                 if (nd_btt)
 963                         ndns = nd_btt->ndns;
 964                 else if (nd_pfn)
 965                         ndns = nd_pfn->ndns;
 966                 else if (nd_dax)
 967                         ndns = nd_dax->nd_pfn.ndns;
 968
 969                 if (!ndns)
 970                         return 0;
 971         } else
 972                 ndns = to_ndns(dev);
 973
 974         nsio = to_nd_namespace_io(&ndns->dev);
 975         pstart = nsio->res.start + offset;
 976         pend = nsio->res.end - end_trunc;
 977
 978         if ((pstart >= start) && (pend <= end))
 979                 return -EBUSY;
 980
 981         return 0;
 982
 983 }
 984
 985 static int nd_ns_forget_poison_check(struct device *dev, void *data)
 986 {
 987         return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
 988 }
 989
 990 /* set_config requires an idle interleave set */
 991 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
 992                 struct nvdimm *nvdimm, unsigned int cmd, void *data)
 993 {
 994         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
 995
 996         /* ask the bus provider if it would like to block this request */
 997         if (nd_desc->clear_to_send) {
 998                 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
 999
1000                 if (rc)
1001                         return rc;
1002         }
1003
1004         /* require clear error to go through the pmem driver */
1005         if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
1006                 return device_for_each_child(&nvdimm_bus->dev, data,
1007                                 nd_ns_forget_poison_check);
1008
1009         if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1010                 return 0;
1011
1012         /* prevent label manipulation while the kernel owns label updates */
1013         wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
1014         if (atomic_read(&nvdimm->busy))
1015                 return -EBUSY;
1016         return 0;
1017 }
1018
1019 static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1020                 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1021 {
1022         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1023         const struct nd_cmd_desc *desc = NULL;
1024         unsigned int cmd = _IOC_NR(ioctl_cmd);
1025         struct device *dev = &nvdimm_bus->dev;
1026         void __user *p = (void __user *) arg;
1027         char *out_env = NULL, *in_env = NULL;
1028         const char *cmd_name, *dimm_name;
1029         u32 in_len = 0, out_len = 0;
1030         unsigned int func = cmd;
1031         unsigned long cmd_mask;
1032         struct nd_cmd_pkg pkg;
1033         int rc, i, cmd_rc;
1034         void *buf = NULL;
1035         u64 buf_len = 0;
1036
1037         if (nvdimm) {
1038                 desc = nd_cmd_dimm_desc(cmd);
1039                 cmd_name = nvdimm_cmd_name(cmd);
1040                 cmd_mask = nvdimm->cmd_mask;
1041                 dimm_name = dev_name(&nvdimm->dev);
1042         } else {
1043                 desc = nd_cmd_bus_desc(cmd);
1044                 cmd_name = nvdimm_bus_cmd_name(cmd);
1045                 cmd_mask = nd_desc->cmd_mask;
1046                 dimm_name = "bus";
1047         }
1048
1049         /* Validate command family support against bus declared support */
1050         if (cmd == ND_CMD_CALL) {
1051                 unsigned long *mask;
1052
1053                 if (copy_from_user(&pkg, p, sizeof(pkg)))
1054                         return -EFAULT;
1055
1056                 if (nvdimm) {
1057                         if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1058                                 return -EINVAL;
1059                         mask = &nd_desc->dimm_family_mask;
1060                 } else {
1061                         if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1062                                 return -EINVAL;
1063                         mask = &nd_desc->bus_family_mask;
1064                 }
1065
1066                 if (!test_bit(pkg.nd_family, mask))
1067                         return -EINVAL;
1068         }
1069
1070         if (!desc ||
1071             (desc->out_num + desc->in_num == 0) ||
1072             cmd > ND_CMD_CALL ||
1073             !test_bit(cmd, &cmd_mask))
1074                 return -ENOTTY;
1075
1076         /* fail write commands (when read-only) */
1077         if (read_only)
1078                 switch (cmd) {
1079                 case ND_CMD_VENDOR:
1080                 case ND_CMD_SET_CONFIG_DATA:
1081                 case ND_CMD_ARS_START:
1082                 case ND_CMD_CLEAR_ERROR:
1083                 case ND_CMD_CALL:
1084                         dev_dbg(dev, "'%s' command while read-only.\n",
1085                                         nvdimm ? nvdimm_cmd_name(cmd)
1086                                         : nvdimm_bus_cmd_name(cmd));
1087                         return -EPERM;
1088                 default:
1089                         break;
1090                 }
1091
1092         /* process an input envelope */
1093         in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1094         if (!in_env)
1095                 return -ENOMEM;
1096         for (i = 0; i < desc->in_num; i++) {
1097                 u32 in_size, copy;
1098
1099                 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1100                 if (in_size == UINT_MAX) {
1101                         dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1102                                         __func__, dimm_name, cmd_name, i);
1103                         rc = -ENXIO;
1104                         goto out;
1105                 }
1106                 if (in_len < ND_CMD_MAX_ENVELOPE)
1107                         copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1108                 else
1109                         copy = 0;
1110                 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1111                         rc = -EFAULT;
1112                         goto out;
1113                 }
1114                 in_len += in_size;
1115         }
1116
1117         if (cmd == ND_CMD_CALL) {
1118                 func = pkg.nd_command;
1119                 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1120                                 dimm_name, pkg.nd_command,
1121                                 in_len, out_len, buf_len);
1122         }
1123
1124         /* process an output envelope */
1125         out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1126         if (!out_env) {
1127                 rc = -ENOMEM;
1128                 goto out;
1129         }
1130
1131         for (i = 0; i < desc->out_num; i++) {
1132                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1133                                 (u32 *) in_env, (u32 *) out_env, 0);
1134                 u32 copy;
1135
1136                 if (out_size == UINT_MAX) {
1137                         dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1138                                         dimm_name, cmd_name, i);
1139                         rc = -EFAULT;
1140                         goto out;
1141                 }
1142                 if (out_len < ND_CMD_MAX_ENVELOPE)
1143                         copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1144                 else
1145                         copy = 0;
1146                 if (copy && copy_from_user(&out_env[out_len],
1147                                         p + in_len + out_len, copy)) {
1148                         rc = -EFAULT;
1149                         goto out;
1150                 }
1151                 out_len += out_size;
1152         }
1153
1154         buf_len = (u64) out_len + (u64) in_len;
1155         if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1156                 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1157                                 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1158                 rc = -EINVAL;
1159                 goto out;
1160         }
1161
1162         buf = vmalloc(buf_len);
1163         if (!buf) {
1164                 rc = -ENOMEM;
1165                 goto out;
1166         }
1167
1168         if (copy_from_user(buf, p, buf_len)) {
1169                 rc = -EFAULT;
1170                 goto out;
1171         }
1172
1173         nd_device_lock(dev);
1174         nvdimm_bus_lock(dev);
1175         rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
1176         if (rc)
1177                 goto out_unlock;
1178
1179         rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1180         if (rc < 0)
1181                 goto out_unlock;
1182
1183         if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1184                 struct nd_cmd_clear_error *clear_err = buf;
1185
1186                 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1187                                 clear_err->cleared);
1188         }
1189
1190         if (copy_to_user(p, buf, buf_len))
1191                 rc = -EFAULT;
1192
1193 out_unlock:
1194         nvdimm_bus_unlock(dev);
1195         nd_device_unlock(dev);
1196 out:
1197         kfree(in_env);
1198         kfree(out_env);
1199         vfree(buf);
1200         return rc;
1201 }
1202
1203 enum nd_ioctl_mode {
1204         BUS_IOCTL,
1205         DIMM_IOCTL,
1206 };
1207
1208 static int match_dimm(struct device *dev, void *data)
1209 {
1210         long id = (long) data;
1211
1212         if (is_nvdimm(dev)) {
1213                 struct nvdimm *nvdimm = to_nvdimm(dev);
1214
1215                 return nvdimm->id == id;
1216         }
1217
1218         return 0;
1219 }
1220
1221 static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1222                 enum nd_ioctl_mode mode)
1223
1224 {
1225         struct nvdimm_bus *nvdimm_bus, *found = NULL;
1226         long id = (long) file->private_data;
1227         struct nvdimm *nvdimm = NULL;
1228         int rc, ro;
1229
1230         ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1231         mutex_lock(&nvdimm_bus_list_mutex);
1232         list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1233                 if (mode == DIMM_IOCTL) {
1234                         struct device *dev;
1235
1236                         dev = device_find_child(&nvdimm_bus->dev,
1237                                         file->private_data, match_dimm);
1238                         if (!dev)
1239                                 continue;
1240                         nvdimm = to_nvdimm(dev);
1241                         found = nvdimm_bus;
1242                 } else if (nvdimm_bus->id == id) {
1243                         found = nvdimm_bus;
1244                 }
1245
1246                 if (found) {
1247                         atomic_inc(&nvdimm_bus->ioctl_active);
1248                         break;
1249                 }
1250         }
1251         mutex_unlock(&nvdimm_bus_list_mutex);
1252
1253         if (!found)
1254                 return -ENXIO;
1255
1256         nvdimm_bus = found;
1257         rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1258
1259         if (nvdimm)
1260                 put_device(&nvdimm->dev);
1261         if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1262                 wake_up(&nvdimm_bus->wait);
1263
1264         return rc;
1265 }
1266
1267 static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1268 {
1269         return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1270 }
1271
1272 static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1273 {
1274         return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1275 }
1276
1277 static int nd_open(struct inode *inode, struct file *file)
1278 {
1279         long minor = iminor(inode);
1280
1281         file->private_data = (void *) minor;
1282         return 0;
1283 }
1284
1285 static const struct file_operations nvdimm_bus_fops = {
1286         .owner = THIS_MODULE,
1287         .open = nd_open,
1288         .unlocked_ioctl = bus_ioctl,
1289         .compat_ioctl = compat_ptr_ioctl,
1290         .llseek = noop_llseek,
1291 };
1292
1293 static const struct file_operations nvdimm_fops = {
1294         .owner = THIS_MODULE,
1295         .open = nd_open,
1296         .unlocked_ioctl = dimm_ioctl,
1297         .compat_ioctl = compat_ptr_ioctl,
1298         .llseek = noop_llseek,
1299 };
1300
1301 int __init nvdimm_bus_init(void)
1302 {
1303         int rc;
1304
1305         rc = bus_register(&nvdimm_bus_type);
1306         if (rc)
1307                 return rc;
1308
1309         rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1310         if (rc < 0)
1311                 goto err_bus_chrdev;
1312         nvdimm_bus_major = rc;
1313
1314         rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1315         if (rc < 0)
1316                 goto err_dimm_chrdev;
1317         nvdimm_major = rc;
1318
1319         nd_class = class_create(THIS_MODULE, "nd");
1320         if (IS_ERR(nd_class)) {
1321                 rc = PTR_ERR(nd_class);
1322                 goto err_class;
1323         }
1324
1325         rc = driver_register(&nd_bus_driver.drv);
1326         if (rc)
1327                 goto err_nd_bus;
1328
1329         return 0;
1330
1331  err_nd_bus:
1332         class_destroy(nd_class);
1333  err_class:
1334         unregister_chrdev(nvdimm_major, "dimmctl");
1335  err_dimm_chrdev:
1336         unregister_chrdev(nvdimm_bus_major, "ndctl");
1337  err_bus_chrdev:
1338         bus_unregister(&nvdimm_bus_type);
1339
1340         return rc;
1341 }
1342
1343 void nvdimm_bus_exit(void)
1344 {
1345         driver_unregister(&nd_bus_driver.drv);
1346         class_destroy(nd_class);
1347         unregister_chrdev(nvdimm_bus_major, "ndctl");
1348         unregister_chrdev(nvdimm_major, "dimmctl");
1349         bus_unregister(&nvdimm_bus_type);
1350         ida_destroy(&nd_ida);
1351 }