Merge drm/drm-next into drm-misc-next
[linux-2.6-microblaze.git] / drivers / nvdimm / bus.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4  */
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/libnvdimm.h>
7 #include <linux/sched/mm.h>
8 #include <linux/vmalloc.h>
9 #include <linux/uaccess.h>
10 #include <linux/module.h>
11 #include <linux/blkdev.h>
12 #include <linux/fcntl.h>
13 #include <linux/async.h>
14 #include <linux/genhd.h>
15 #include <linux/ndctl.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/fs.h>
20 #include <linux/io.h>
21 #include <linux/mm.h>
22 #include <linux/nd.h>
23 #include "nd-core.h"
24 #include "nd.h"
25 #include "pfn.h"
26
27 int nvdimm_major;
28 static int nvdimm_bus_major;
29 struct class *nd_class;
30 static DEFINE_IDA(nd_ida);
31
32 static int to_nd_device_type(struct device *dev)
33 {
34         if (is_nvdimm(dev))
35                 return ND_DEVICE_DIMM;
36         else if (is_memory(dev))
37                 return ND_DEVICE_REGION_PMEM;
38         else if (is_nd_blk(dev))
39                 return ND_DEVICE_REGION_BLK;
40         else if (is_nd_dax(dev))
41                 return ND_DEVICE_DAX_PMEM;
42         else if (is_nd_region(dev->parent))
43                 return nd_region_to_nstype(to_nd_region(dev->parent));
44
45         return 0;
46 }
47
48 static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
49 {
50         return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
51                         to_nd_device_type(dev));
52 }
53
54 static struct module *to_bus_provider(struct device *dev)
55 {
56         /* pin bus providers while regions are enabled */
57         if (is_nd_region(dev)) {
58                 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
59
60                 return nvdimm_bus->nd_desc->module;
61         }
62         return NULL;
63 }
64
65 static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
66 {
67         nvdimm_bus_lock(&nvdimm_bus->dev);
68         nvdimm_bus->probe_active++;
69         nvdimm_bus_unlock(&nvdimm_bus->dev);
70 }
71
72 static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
73 {
74         nvdimm_bus_lock(&nvdimm_bus->dev);
75         if (--nvdimm_bus->probe_active == 0)
76                 wake_up(&nvdimm_bus->wait);
77         nvdimm_bus_unlock(&nvdimm_bus->dev);
78 }
79
80 static int nvdimm_bus_probe(struct device *dev)
81 {
82         struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
83         struct module *provider = to_bus_provider(dev);
84         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
85         int rc;
86
87         if (!try_module_get(provider))
88                 return -ENXIO;
89
90         dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
91                         dev->driver->name, dev_name(dev));
92
93         nvdimm_bus_probe_start(nvdimm_bus);
94         debug_nvdimm_lock(dev);
95         rc = nd_drv->probe(dev);
96         debug_nvdimm_unlock(dev);
97
98         if ((rc == 0 || rc == -EOPNOTSUPP) &&
99                         dev->parent && is_nd_region(dev->parent))
100                 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
101         nvdimm_bus_probe_end(nvdimm_bus);
102
103         dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
104                         dev_name(dev), rc);
105
106         if (rc != 0)
107                 module_put(provider);
108         return rc;
109 }
110
111 static void nvdimm_bus_remove(struct device *dev)
112 {
113         struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
114         struct module *provider = to_bus_provider(dev);
115         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
116
117         if (nd_drv->remove) {
118                 debug_nvdimm_lock(dev);
119                 nd_drv->remove(dev);
120                 debug_nvdimm_unlock(dev);
121         }
122
123         dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
124                         dev_name(dev));
125         module_put(provider);
126 }
127
128 static void nvdimm_bus_shutdown(struct device *dev)
129 {
130         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
131         struct nd_device_driver *nd_drv = NULL;
132
133         if (dev->driver)
134                 nd_drv = to_nd_device_driver(dev->driver);
135
136         if (nd_drv && nd_drv->shutdown) {
137                 nd_drv->shutdown(dev);
138                 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
139                                 dev->driver->name, dev_name(dev));
140         }
141 }
142
143 void nd_device_notify(struct device *dev, enum nvdimm_event event)
144 {
145         nd_device_lock(dev);
146         if (dev->driver) {
147                 struct nd_device_driver *nd_drv;
148
149                 nd_drv = to_nd_device_driver(dev->driver);
150                 if (nd_drv->notify)
151                         nd_drv->notify(dev, event);
152         }
153         nd_device_unlock(dev);
154 }
155 EXPORT_SYMBOL(nd_device_notify);
156
157 void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
158 {
159         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
160
161         if (!nvdimm_bus)
162                 return;
163
164         /* caller is responsible for holding a reference on the device */
165         nd_device_notify(&nd_region->dev, event);
166 }
167 EXPORT_SYMBOL_GPL(nvdimm_region_notify);
168
169 struct clear_badblocks_context {
170         resource_size_t phys, cleared;
171 };
172
173 static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
174 {
175         struct clear_badblocks_context *ctx = data;
176         struct nd_region *nd_region;
177         resource_size_t ndr_end;
178         sector_t sector;
179
180         /* make sure device is a region */
181         if (!is_memory(dev))
182                 return 0;
183
184         nd_region = to_nd_region(dev);
185         ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
186
187         /* make sure we are in the region */
188         if (ctx->phys < nd_region->ndr_start
189                         || (ctx->phys + ctx->cleared) > ndr_end)
190                 return 0;
191
192         sector = (ctx->phys - nd_region->ndr_start) / 512;
193         badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
194
195         if (nd_region->bb_state)
196                 sysfs_notify_dirent(nd_region->bb_state);
197
198         return 0;
199 }
200
201 static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
202                 phys_addr_t phys, u64 cleared)
203 {
204         struct clear_badblocks_context ctx = {
205                 .phys = phys,
206                 .cleared = cleared,
207         };
208
209         device_for_each_child(&nvdimm_bus->dev, &ctx,
210                         nvdimm_clear_badblocks_region);
211 }
212
213 static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
214                 phys_addr_t phys, u64 cleared)
215 {
216         if (cleared > 0)
217                 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
218
219         if (cleared > 0 && cleared / 512)
220                 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
221 }
222
223 long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
224                 unsigned int len)
225 {
226         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
227         struct nvdimm_bus_descriptor *nd_desc;
228         struct nd_cmd_clear_error clear_err;
229         struct nd_cmd_ars_cap ars_cap;
230         u32 clear_err_unit, mask;
231         unsigned int noio_flag;
232         int cmd_rc, rc;
233
234         if (!nvdimm_bus)
235                 return -ENXIO;
236
237         nd_desc = nvdimm_bus->nd_desc;
238         /*
239          * if ndctl does not exist, it's PMEM_LEGACY and
240          * we want to just pretend everything is handled.
241          */
242         if (!nd_desc->ndctl)
243                 return len;
244
245         memset(&ars_cap, 0, sizeof(ars_cap));
246         ars_cap.address = phys;
247         ars_cap.length = len;
248         noio_flag = memalloc_noio_save();
249         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
250                         sizeof(ars_cap), &cmd_rc);
251         memalloc_noio_restore(noio_flag);
252         if (rc < 0)
253                 return rc;
254         if (cmd_rc < 0)
255                 return cmd_rc;
256         clear_err_unit = ars_cap.clear_err_unit;
257         if (!clear_err_unit || !is_power_of_2(clear_err_unit))
258                 return -ENXIO;
259
260         mask = clear_err_unit - 1;
261         if ((phys | len) & mask)
262                 return -ENXIO;
263         memset(&clear_err, 0, sizeof(clear_err));
264         clear_err.address = phys;
265         clear_err.length = len;
266         noio_flag = memalloc_noio_save();
267         rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
268                         sizeof(clear_err), &cmd_rc);
269         memalloc_noio_restore(noio_flag);
270         if (rc < 0)
271                 return rc;
272         if (cmd_rc < 0)
273                 return cmd_rc;
274
275         nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
276
277         return clear_err.cleared;
278 }
279 EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
280
281 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
282
283 static struct bus_type nvdimm_bus_type = {
284         .name = "nd",
285         .uevent = nvdimm_bus_uevent,
286         .match = nvdimm_bus_match,
287         .probe = nvdimm_bus_probe,
288         .remove = nvdimm_bus_remove,
289         .shutdown = nvdimm_bus_shutdown,
290 };
291
292 static void nvdimm_bus_release(struct device *dev)
293 {
294         struct nvdimm_bus *nvdimm_bus;
295
296         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
297         ida_simple_remove(&nd_ida, nvdimm_bus->id);
298         kfree(nvdimm_bus);
299 }
300
301 static const struct device_type nvdimm_bus_dev_type = {
302         .release = nvdimm_bus_release,
303         .groups = nvdimm_bus_attribute_groups,
304 };
305
306 bool is_nvdimm_bus(struct device *dev)
307 {
308         return dev->type == &nvdimm_bus_dev_type;
309 }
310
311 struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
312 {
313         struct device *dev;
314
315         for (dev = nd_dev; dev; dev = dev->parent)
316                 if (is_nvdimm_bus(dev))
317                         break;
318         dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
319         if (dev)
320                 return to_nvdimm_bus(dev);
321         return NULL;
322 }
323
324 struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
325 {
326         struct nvdimm_bus *nvdimm_bus;
327
328         nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
329         WARN_ON(!is_nvdimm_bus(dev));
330         return nvdimm_bus;
331 }
332 EXPORT_SYMBOL_GPL(to_nvdimm_bus);
333
334 struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
335 {
336         return to_nvdimm_bus(nvdimm->dev.parent);
337 }
338 EXPORT_SYMBOL_GPL(nvdimm_to_bus);
339
340 struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
341                 struct nvdimm_bus_descriptor *nd_desc)
342 {
343         struct nvdimm_bus *nvdimm_bus;
344         int rc;
345
346         nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
347         if (!nvdimm_bus)
348                 return NULL;
349         INIT_LIST_HEAD(&nvdimm_bus->list);
350         INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
351         init_waitqueue_head(&nvdimm_bus->wait);
352         nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
353         if (nvdimm_bus->id < 0) {
354                 kfree(nvdimm_bus);
355                 return NULL;
356         }
357         mutex_init(&nvdimm_bus->reconfig_mutex);
358         badrange_init(&nvdimm_bus->badrange);
359         nvdimm_bus->nd_desc = nd_desc;
360         nvdimm_bus->dev.parent = parent;
361         nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
362         nvdimm_bus->dev.groups = nd_desc->attr_groups;
363         nvdimm_bus->dev.bus = &nvdimm_bus_type;
364         nvdimm_bus->dev.of_node = nd_desc->of_node;
365         device_initialize(&nvdimm_bus->dev);
366         device_set_pm_not_required(&nvdimm_bus->dev);
367         rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
368         if (rc)
369                 goto err;
370
371         rc = device_add(&nvdimm_bus->dev);
372         if (rc) {
373                 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
374                 goto err;
375         }
376
377         return nvdimm_bus;
378  err:
379         put_device(&nvdimm_bus->dev);
380         return NULL;
381 }
382 EXPORT_SYMBOL_GPL(nvdimm_bus_register);
383
384 void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
385 {
386         if (!nvdimm_bus)
387                 return;
388         device_unregister(&nvdimm_bus->dev);
389 }
390 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
391
392 static int child_unregister(struct device *dev, void *data)
393 {
394         /*
395          * the singular ndctl class device per bus needs to be
396          * "device_destroy"ed, so skip it here
397          *
398          * i.e. remove classless children
399          */
400         if (dev->class)
401                 return 0;
402
403         if (is_nvdimm(dev))
404                 nvdimm_delete(to_nvdimm(dev));
405         else
406                 nd_device_unregister(dev, ND_SYNC);
407
408         return 0;
409 }
410
411 static void free_badrange_list(struct list_head *badrange_list)
412 {
413         struct badrange_entry *bre, *next;
414
415         list_for_each_entry_safe(bre, next, badrange_list, list) {
416                 list_del(&bre->list);
417                 kfree(bre);
418         }
419         list_del_init(badrange_list);
420 }
421
422 static void nd_bus_remove(struct device *dev)
423 {
424         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
425
426         mutex_lock(&nvdimm_bus_list_mutex);
427         list_del_init(&nvdimm_bus->list);
428         mutex_unlock(&nvdimm_bus_list_mutex);
429
430         wait_event(nvdimm_bus->wait,
431                         atomic_read(&nvdimm_bus->ioctl_active) == 0);
432
433         nd_synchronize();
434         device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
435
436         spin_lock(&nvdimm_bus->badrange.lock);
437         free_badrange_list(&nvdimm_bus->badrange.list);
438         spin_unlock(&nvdimm_bus->badrange.lock);
439
440         nvdimm_bus_destroy_ndctl(nvdimm_bus);
441 }
442
443 static int nd_bus_probe(struct device *dev)
444 {
445         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
446         int rc;
447
448         rc = nvdimm_bus_create_ndctl(nvdimm_bus);
449         if (rc)
450                 return rc;
451
452         mutex_lock(&nvdimm_bus_list_mutex);
453         list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
454         mutex_unlock(&nvdimm_bus_list_mutex);
455
456         /* enable bus provider attributes to look up their local context */
457         dev_set_drvdata(dev, nvdimm_bus->nd_desc);
458
459         return 0;
460 }
461
462 static struct nd_device_driver nd_bus_driver = {
463         .probe = nd_bus_probe,
464         .remove = nd_bus_remove,
465         .drv = {
466                 .name = "nd_bus",
467                 .suppress_bind_attrs = true,
468                 .bus = &nvdimm_bus_type,
469                 .owner = THIS_MODULE,
470                 .mod_name = KBUILD_MODNAME,
471         },
472 };
473
474 static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
475 {
476         struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
477
478         if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
479                 return true;
480
481         return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
482 }
483
484 static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
485
486 void nd_synchronize(void)
487 {
488         async_synchronize_full_domain(&nd_async_domain);
489 }
490 EXPORT_SYMBOL_GPL(nd_synchronize);
491
492 static void nd_async_device_register(void *d, async_cookie_t cookie)
493 {
494         struct device *dev = d;
495
496         if (device_add(dev) != 0) {
497                 dev_err(dev, "%s: failed\n", __func__);
498                 put_device(dev);
499         }
500         put_device(dev);
501         if (dev->parent)
502                 put_device(dev->parent);
503 }
504
505 static void nd_async_device_unregister(void *d, async_cookie_t cookie)
506 {
507         struct device *dev = d;
508
509         /* flush bus operations before delete */
510         nvdimm_bus_lock(dev);
511         nvdimm_bus_unlock(dev);
512
513         device_unregister(dev);
514         put_device(dev);
515 }
516
517 void __nd_device_register(struct device *dev)
518 {
519         if (!dev)
520                 return;
521
522         /*
523          * Ensure that region devices always have their NUMA node set as
524          * early as possible. This way we are able to make certain that
525          * any memory associated with the creation and the creation
526          * itself of the region is associated with the correct node.
527          */
528         if (is_nd_region(dev))
529                 set_dev_node(dev, to_nd_region(dev)->numa_node);
530
531         dev->bus = &nvdimm_bus_type;
532         device_set_pm_not_required(dev);
533         if (dev->parent) {
534                 get_device(dev->parent);
535                 if (dev_to_node(dev) == NUMA_NO_NODE)
536                         set_dev_node(dev, dev_to_node(dev->parent));
537         }
538         get_device(dev);
539
540         async_schedule_dev_domain(nd_async_device_register, dev,
541                                   &nd_async_domain);
542 }
543
544 void nd_device_register(struct device *dev)
545 {
546         device_initialize(dev);
547         __nd_device_register(dev);
548 }
549 EXPORT_SYMBOL(nd_device_register);
550
551 void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
552 {
553         bool killed;
554
555         switch (mode) {
556         case ND_ASYNC:
557                 /*
558                  * In the async case this is being triggered with the
559                  * device lock held and the unregistration work needs to
560                  * be moved out of line iff this is thread has won the
561                  * race to schedule the deletion.
562                  */
563                 if (!kill_device(dev))
564                         return;
565
566                 get_device(dev);
567                 async_schedule_domain(nd_async_device_unregister, dev,
568                                 &nd_async_domain);
569                 break;
570         case ND_SYNC:
571                 /*
572                  * In the sync case the device is being unregistered due
573                  * to a state change of the parent. Claim the kill state
574                  * to synchronize against other unregistration requests,
575                  * or otherwise let the async path handle it if the
576                  * unregistration was already queued.
577                  */
578                 nd_device_lock(dev);
579                 killed = kill_device(dev);
580                 nd_device_unlock(dev);
581
582                 if (!killed)
583                         return;
584
585                 nd_synchronize();
586                 device_unregister(dev);
587                 break;
588         }
589 }
590 EXPORT_SYMBOL(nd_device_unregister);
591
592 /**
593  * __nd_driver_register() - register a region or a namespace driver
594  * @nd_drv: driver to register
595  * @owner: automatically set by nd_driver_register() macro
596  * @mod_name: automatically set by nd_driver_register() macro
597  */
598 int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
599                 const char *mod_name)
600 {
601         struct device_driver *drv = &nd_drv->drv;
602
603         if (!nd_drv->type) {
604                 pr_debug("driver type bitmask not set (%ps)\n",
605                                 __builtin_return_address(0));
606                 return -EINVAL;
607         }
608
609         if (!nd_drv->probe) {
610                 pr_debug("%s ->probe() must be specified\n", mod_name);
611                 return -EINVAL;
612         }
613
614         drv->bus = &nvdimm_bus_type;
615         drv->owner = owner;
616         drv->mod_name = mod_name;
617
618         return driver_register(drv);
619 }
620 EXPORT_SYMBOL(__nd_driver_register);
621
622 void nvdimm_check_and_set_ro(struct gendisk *disk)
623 {
624         struct device *dev = disk_to_dev(disk)->parent;
625         struct nd_region *nd_region = to_nd_region(dev->parent);
626         int disk_ro = get_disk_ro(disk);
627
628         /* catch the disk up with the region ro state */
629         if (disk_ro == nd_region->ro)
630                 return;
631
632         dev_info(dev, "%s read-%s, marking %s read-%s\n",
633                  dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
634                  disk->disk_name, nd_region->ro ? "only" : "write");
635         set_disk_ro(disk, nd_region->ro);
636 }
637 EXPORT_SYMBOL(nvdimm_check_and_set_ro);
638
639 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
640                 char *buf)
641 {
642         return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
643                         to_nd_device_type(dev));
644 }
645 static DEVICE_ATTR_RO(modalias);
646
647 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
648                 char *buf)
649 {
650         return sprintf(buf, "%s\n", dev->type->name);
651 }
652 static DEVICE_ATTR_RO(devtype);
653
654 static struct attribute *nd_device_attributes[] = {
655         &dev_attr_modalias.attr,
656         &dev_attr_devtype.attr,
657         NULL,
658 };
659
660 /*
661  * nd_device_attribute_group - generic attributes for all devices on an nd bus
662  */
663 const struct attribute_group nd_device_attribute_group = {
664         .attrs = nd_device_attributes,
665 };
666
667 static ssize_t numa_node_show(struct device *dev,
668                 struct device_attribute *attr, char *buf)
669 {
670         return sprintf(buf, "%d\n", dev_to_node(dev));
671 }
672 static DEVICE_ATTR_RO(numa_node);
673
674 static int nvdimm_dev_to_target_node(struct device *dev)
675 {
676         struct device *parent = dev->parent;
677         struct nd_region *nd_region = NULL;
678
679         if (is_nd_region(dev))
680                 nd_region = to_nd_region(dev);
681         else if (parent && is_nd_region(parent))
682                 nd_region = to_nd_region(parent);
683
684         if (!nd_region)
685                 return NUMA_NO_NODE;
686         return nd_region->target_node;
687 }
688
689 static ssize_t target_node_show(struct device *dev,
690                 struct device_attribute *attr, char *buf)
691 {
692         return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
693 }
694 static DEVICE_ATTR_RO(target_node);
695
696 static struct attribute *nd_numa_attributes[] = {
697         &dev_attr_numa_node.attr,
698         &dev_attr_target_node.attr,
699         NULL,
700 };
701
702 static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
703                 int n)
704 {
705         struct device *dev = container_of(kobj, typeof(*dev), kobj);
706
707         if (!IS_ENABLED(CONFIG_NUMA))
708                 return 0;
709
710         if (a == &dev_attr_target_node.attr &&
711                         nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
712                 return 0;
713
714         return a->mode;
715 }
716
717 /*
718  * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
719  */
720 const struct attribute_group nd_numa_attribute_group = {
721         .attrs = nd_numa_attributes,
722         .is_visible = nd_numa_attr_visible,
723 };
724
725 static void ndctl_release(struct device *dev)
726 {
727         kfree(dev);
728 }
729
730 int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
731 {
732         dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
733         struct device *dev;
734         int rc;
735
736         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
737         if (!dev)
738                 return -ENOMEM;
739         device_initialize(dev);
740         device_set_pm_not_required(dev);
741         dev->class = nd_class;
742         dev->parent = &nvdimm_bus->dev;
743         dev->devt = devt;
744         dev->release = ndctl_release;
745         rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id);
746         if (rc)
747                 goto err;
748
749         rc = device_add(dev);
750         if (rc) {
751                 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
752                                 nvdimm_bus->id, rc);
753                 goto err;
754         }
755         return 0;
756
757 err:
758         put_device(dev);
759         return rc;
760 }
761
762 void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
763 {
764         device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
765 }
766
767 static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
768         [ND_CMD_IMPLEMENTED] = { },
769         [ND_CMD_SMART] = {
770                 .out_num = 2,
771                 .out_sizes = { 4, 128, },
772         },
773         [ND_CMD_SMART_THRESHOLD] = {
774                 .out_num = 2,
775                 .out_sizes = { 4, 8, },
776         },
777         [ND_CMD_DIMM_FLAGS] = {
778                 .out_num = 2,
779                 .out_sizes = { 4, 4 },
780         },
781         [ND_CMD_GET_CONFIG_SIZE] = {
782                 .out_num = 3,
783                 .out_sizes = { 4, 4, 4, },
784         },
785         [ND_CMD_GET_CONFIG_DATA] = {
786                 .in_num = 2,
787                 .in_sizes = { 4, 4, },
788                 .out_num = 2,
789                 .out_sizes = { 4, UINT_MAX, },
790         },
791         [ND_CMD_SET_CONFIG_DATA] = {
792                 .in_num = 3,
793                 .in_sizes = { 4, 4, UINT_MAX, },
794                 .out_num = 1,
795                 .out_sizes = { 4, },
796         },
797         [ND_CMD_VENDOR] = {
798                 .in_num = 3,
799                 .in_sizes = { 4, 4, UINT_MAX, },
800                 .out_num = 3,
801                 .out_sizes = { 4, 4, UINT_MAX, },
802         },
803         [ND_CMD_CALL] = {
804                 .in_num = 2,
805                 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
806                 .out_num = 1,
807                 .out_sizes = { UINT_MAX, },
808         },
809 };
810
811 const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
812 {
813         if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
814                 return &__nd_cmd_dimm_descs[cmd];
815         return NULL;
816 }
817 EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
818
819 static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
820         [ND_CMD_IMPLEMENTED] = { },
821         [ND_CMD_ARS_CAP] = {
822                 .in_num = 2,
823                 .in_sizes = { 8, 8, },
824                 .out_num = 4,
825                 .out_sizes = { 4, 4, 4, 4, },
826         },
827         [ND_CMD_ARS_START] = {
828                 .in_num = 5,
829                 .in_sizes = { 8, 8, 2, 1, 5, },
830                 .out_num = 2,
831                 .out_sizes = { 4, 4, },
832         },
833         [ND_CMD_ARS_STATUS] = {
834                 .out_num = 3,
835                 .out_sizes = { 4, 4, UINT_MAX, },
836         },
837         [ND_CMD_CLEAR_ERROR] = {
838                 .in_num = 2,
839                 .in_sizes = { 8, 8, },
840                 .out_num = 3,
841                 .out_sizes = { 4, 4, 8, },
842         },
843         [ND_CMD_CALL] = {
844                 .in_num = 2,
845                 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
846                 .out_num = 1,
847                 .out_sizes = { UINT_MAX, },
848         },
849 };
850
851 const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
852 {
853         if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
854                 return &__nd_cmd_bus_descs[cmd];
855         return NULL;
856 }
857 EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
858
859 u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
860                 const struct nd_cmd_desc *desc, int idx, void *buf)
861 {
862         if (idx >= desc->in_num)
863                 return UINT_MAX;
864
865         if (desc->in_sizes[idx] < UINT_MAX)
866                 return desc->in_sizes[idx];
867
868         if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
869                 struct nd_cmd_set_config_hdr *hdr = buf;
870
871                 return hdr->in_length;
872         } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
873                 struct nd_cmd_vendor_hdr *hdr = buf;
874
875                 return hdr->in_length;
876         } else if (cmd == ND_CMD_CALL) {
877                 struct nd_cmd_pkg *pkg = buf;
878
879                 return pkg->nd_size_in;
880         }
881
882         return UINT_MAX;
883 }
884 EXPORT_SYMBOL_GPL(nd_cmd_in_size);
885
886 u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
887                 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
888                 const u32 *out_field, unsigned long remainder)
889 {
890         if (idx >= desc->out_num)
891                 return UINT_MAX;
892
893         if (desc->out_sizes[idx] < UINT_MAX)
894                 return desc->out_sizes[idx];
895
896         if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
897                 return in_field[1];
898         else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
899                 return out_field[1];
900         else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
901                 /*
902                  * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
903                  * "Size of Output Buffer in bytes, including this
904                  * field."
905                  */
906                 if (out_field[1] < 4)
907                         return 0;
908                 /*
909                  * ACPI 6.1 is ambiguous if 'status' is included in the
910                  * output size. If we encounter an output size that
911                  * overshoots the remainder by 4 bytes, assume it was
912                  * including 'status'.
913                  */
914                 if (out_field[1] - 4 == remainder)
915                         return remainder;
916                 return out_field[1] - 8;
917         } else if (cmd == ND_CMD_CALL) {
918                 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
919
920                 return pkg->nd_size_out;
921         }
922
923
924         return UINT_MAX;
925 }
926 EXPORT_SYMBOL_GPL(nd_cmd_out_size);
927
928 void wait_nvdimm_bus_probe_idle(struct device *dev)
929 {
930         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
931
932         do {
933                 if (nvdimm_bus->probe_active == 0)
934                         break;
935                 nvdimm_bus_unlock(dev);
936                 nd_device_unlock(dev);
937                 wait_event(nvdimm_bus->wait,
938                                 nvdimm_bus->probe_active == 0);
939                 nd_device_lock(dev);
940                 nvdimm_bus_lock(dev);
941         } while (true);
942 }
943
944 static int nd_pmem_forget_poison_check(struct device *dev, void *data)
945 {
946         struct nd_cmd_clear_error *clear_err =
947                 (struct nd_cmd_clear_error *)data;
948         struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
949         struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
950         struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
951         struct nd_namespace_common *ndns = NULL;
952         struct nd_namespace_io *nsio;
953         resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
954
955         if (nd_dax || !dev->driver)
956                 return 0;
957
958         start = clear_err->address;
959         end = clear_err->address + clear_err->cleared - 1;
960
961         if (nd_btt || nd_pfn || nd_dax) {
962                 if (nd_btt)
963                         ndns = nd_btt->ndns;
964                 else if (nd_pfn)
965                         ndns = nd_pfn->ndns;
966                 else if (nd_dax)
967                         ndns = nd_dax->nd_pfn.ndns;
968
969                 if (!ndns)
970                         return 0;
971         } else
972                 ndns = to_ndns(dev);
973
974         nsio = to_nd_namespace_io(&ndns->dev);
975         pstart = nsio->res.start + offset;
976         pend = nsio->res.end - end_trunc;
977
978         if ((pstart >= start) && (pend <= end))
979                 return -EBUSY;
980
981         return 0;
982
983 }
984
985 static int nd_ns_forget_poison_check(struct device *dev, void *data)
986 {
987         return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
988 }
989
990 /* set_config requires an idle interleave set */
991 static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
992                 struct nvdimm *nvdimm, unsigned int cmd, void *data)
993 {
994         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
995
996         /* ask the bus provider if it would like to block this request */
997         if (nd_desc->clear_to_send) {
998                 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
999
1000                 if (rc)
1001                         return rc;
1002         }
1003
1004         /* require clear error to go through the pmem driver */
1005         if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
1006                 return device_for_each_child(&nvdimm_bus->dev, data,
1007                                 nd_ns_forget_poison_check);
1008
1009         if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1010                 return 0;
1011
1012         /* prevent label manipulation while the kernel owns label updates */
1013         wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
1014         if (atomic_read(&nvdimm->busy))
1015                 return -EBUSY;
1016         return 0;
1017 }
1018
1019 static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1020                 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1021 {
1022         struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1023         const struct nd_cmd_desc *desc = NULL;
1024         unsigned int cmd = _IOC_NR(ioctl_cmd);
1025         struct device *dev = &nvdimm_bus->dev;
1026         void __user *p = (void __user *) arg;
1027         char *out_env = NULL, *in_env = NULL;
1028         const char *cmd_name, *dimm_name;
1029         u32 in_len = 0, out_len = 0;
1030         unsigned int func = cmd;
1031         unsigned long cmd_mask;
1032         struct nd_cmd_pkg pkg;
1033         int rc, i, cmd_rc;
1034         void *buf = NULL;
1035         u64 buf_len = 0;
1036
1037         if (nvdimm) {
1038                 desc = nd_cmd_dimm_desc(cmd);
1039                 cmd_name = nvdimm_cmd_name(cmd);
1040                 cmd_mask = nvdimm->cmd_mask;
1041                 dimm_name = dev_name(&nvdimm->dev);
1042         } else {
1043                 desc = nd_cmd_bus_desc(cmd);
1044                 cmd_name = nvdimm_bus_cmd_name(cmd);
1045                 cmd_mask = nd_desc->cmd_mask;
1046                 dimm_name = "bus";
1047         }
1048
1049         /* Validate command family support against bus declared support */
1050         if (cmd == ND_CMD_CALL) {
1051                 unsigned long *mask;
1052
1053                 if (copy_from_user(&pkg, p, sizeof(pkg)))
1054                         return -EFAULT;
1055
1056                 if (nvdimm) {
1057                         if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1058                                 return -EINVAL;
1059                         mask = &nd_desc->dimm_family_mask;
1060                 } else {
1061                         if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1062                                 return -EINVAL;
1063                         mask = &nd_desc->bus_family_mask;
1064                 }
1065
1066                 if (!test_bit(pkg.nd_family, mask))
1067                         return -EINVAL;
1068         }
1069
1070         if (!desc ||
1071             (desc->out_num + desc->in_num == 0) ||
1072             cmd > ND_CMD_CALL ||
1073             !test_bit(cmd, &cmd_mask))
1074                 return -ENOTTY;
1075
1076         /* fail write commands (when read-only) */
1077         if (read_only)
1078                 switch (cmd) {
1079                 case ND_CMD_VENDOR:
1080                 case ND_CMD_SET_CONFIG_DATA:
1081                 case ND_CMD_ARS_START:
1082                 case ND_CMD_CLEAR_ERROR:
1083                 case ND_CMD_CALL:
1084                         dev_dbg(dev, "'%s' command while read-only.\n",
1085                                         nvdimm ? nvdimm_cmd_name(cmd)
1086                                         : nvdimm_bus_cmd_name(cmd));
1087                         return -EPERM;
1088                 default:
1089                         break;
1090                 }
1091
1092         /* process an input envelope */
1093         in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1094         if (!in_env)
1095                 return -ENOMEM;
1096         for (i = 0; i < desc->in_num; i++) {
1097                 u32 in_size, copy;
1098
1099                 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1100                 if (in_size == UINT_MAX) {
1101                         dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1102                                         __func__, dimm_name, cmd_name, i);
1103                         rc = -ENXIO;
1104                         goto out;
1105                 }
1106                 if (in_len < ND_CMD_MAX_ENVELOPE)
1107                         copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1108                 else
1109                         copy = 0;
1110                 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1111                         rc = -EFAULT;
1112                         goto out;
1113                 }
1114                 in_len += in_size;
1115         }
1116
1117         if (cmd == ND_CMD_CALL) {
1118                 func = pkg.nd_command;
1119                 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1120                                 dimm_name, pkg.nd_command,
1121                                 in_len, out_len, buf_len);
1122         }
1123
1124         /* process an output envelope */
1125         out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1126         if (!out_env) {
1127                 rc = -ENOMEM;
1128                 goto out;
1129         }
1130
1131         for (i = 0; i < desc->out_num; i++) {
1132                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1133                                 (u32 *) in_env, (u32 *) out_env, 0);
1134                 u32 copy;
1135
1136                 if (out_size == UINT_MAX) {
1137                         dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1138                                         dimm_name, cmd_name, i);
1139                         rc = -EFAULT;
1140                         goto out;
1141                 }
1142                 if (out_len < ND_CMD_MAX_ENVELOPE)
1143                         copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1144                 else
1145                         copy = 0;
1146                 if (copy && copy_from_user(&out_env[out_len],
1147                                         p + in_len + out_len, copy)) {
1148                         rc = -EFAULT;
1149                         goto out;
1150                 }
1151                 out_len += out_size;
1152         }
1153
1154         buf_len = (u64) out_len + (u64) in_len;
1155         if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1156                 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1157                                 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1158                 rc = -EINVAL;
1159                 goto out;
1160         }
1161
1162         buf = vmalloc(buf_len);
1163         if (!buf) {
1164                 rc = -ENOMEM;
1165                 goto out;
1166         }
1167
1168         if (copy_from_user(buf, p, buf_len)) {
1169                 rc = -EFAULT;
1170                 goto out;
1171         }
1172
1173         nd_device_lock(dev);
1174         nvdimm_bus_lock(dev);
1175         rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
1176         if (rc)
1177                 goto out_unlock;
1178
1179         rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1180         if (rc < 0)
1181                 goto out_unlock;
1182
1183         if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1184                 struct nd_cmd_clear_error *clear_err = buf;
1185
1186                 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1187                                 clear_err->cleared);
1188         }
1189
1190         if (copy_to_user(p, buf, buf_len))
1191                 rc = -EFAULT;
1192
1193 out_unlock:
1194         nvdimm_bus_unlock(dev);
1195         nd_device_unlock(dev);
1196 out:
1197         kfree(in_env);
1198         kfree(out_env);
1199         vfree(buf);
1200         return rc;
1201 }
1202
1203 enum nd_ioctl_mode {
1204         BUS_IOCTL,
1205         DIMM_IOCTL,
1206 };
1207
1208 static int match_dimm(struct device *dev, void *data)
1209 {
1210         long id = (long) data;
1211
1212         if (is_nvdimm(dev)) {
1213                 struct nvdimm *nvdimm = to_nvdimm(dev);
1214
1215                 return nvdimm->id == id;
1216         }
1217
1218         return 0;
1219 }
1220
1221 static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1222                 enum nd_ioctl_mode mode)
1223
1224 {
1225         struct nvdimm_bus *nvdimm_bus, *found = NULL;
1226         long id = (long) file->private_data;
1227         struct nvdimm *nvdimm = NULL;
1228         int rc, ro;
1229
1230         ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1231         mutex_lock(&nvdimm_bus_list_mutex);
1232         list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1233                 if (mode == DIMM_IOCTL) {
1234                         struct device *dev;
1235
1236                         dev = device_find_child(&nvdimm_bus->dev,
1237                                         file->private_data, match_dimm);
1238                         if (!dev)
1239                                 continue;
1240                         nvdimm = to_nvdimm(dev);
1241                         found = nvdimm_bus;
1242                 } else if (nvdimm_bus->id == id) {
1243                         found = nvdimm_bus;
1244                 }
1245
1246                 if (found) {
1247                         atomic_inc(&nvdimm_bus->ioctl_active);
1248                         break;
1249                 }
1250         }
1251         mutex_unlock(&nvdimm_bus_list_mutex);
1252
1253         if (!found)
1254                 return -ENXIO;
1255
1256         nvdimm_bus = found;
1257         rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1258
1259         if (nvdimm)
1260                 put_device(&nvdimm->dev);
1261         if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1262                 wake_up(&nvdimm_bus->wait);
1263
1264         return rc;
1265 }
1266
1267 static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1268 {
1269         return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1270 }
1271
1272 static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1273 {
1274         return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1275 }
1276
1277 static int nd_open(struct inode *inode, struct file *file)
1278 {
1279         long minor = iminor(inode);
1280
1281         file->private_data = (void *) minor;
1282         return 0;
1283 }
1284
1285 static const struct file_operations nvdimm_bus_fops = {
1286         .owner = THIS_MODULE,
1287         .open = nd_open,
1288         .unlocked_ioctl = bus_ioctl,
1289         .compat_ioctl = compat_ptr_ioctl,
1290         .llseek = noop_llseek,
1291 };
1292
1293 static const struct file_operations nvdimm_fops = {
1294         .owner = THIS_MODULE,
1295         .open = nd_open,
1296         .unlocked_ioctl = dimm_ioctl,
1297         .compat_ioctl = compat_ptr_ioctl,
1298         .llseek = noop_llseek,
1299 };
1300
1301 int __init nvdimm_bus_init(void)
1302 {
1303         int rc;
1304
1305         rc = bus_register(&nvdimm_bus_type);
1306         if (rc)
1307                 return rc;
1308
1309         rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1310         if (rc < 0)
1311                 goto err_bus_chrdev;
1312         nvdimm_bus_major = rc;
1313
1314         rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1315         if (rc < 0)
1316                 goto err_dimm_chrdev;
1317         nvdimm_major = rc;
1318
1319         nd_class = class_create(THIS_MODULE, "nd");
1320         if (IS_ERR(nd_class)) {
1321                 rc = PTR_ERR(nd_class);
1322                 goto err_class;
1323         }
1324
1325         rc = driver_register(&nd_bus_driver.drv);
1326         if (rc)
1327                 goto err_nd_bus;
1328
1329         return 0;
1330
1331  err_nd_bus:
1332         class_destroy(nd_class);
1333  err_class:
1334         unregister_chrdev(nvdimm_major, "dimmctl");
1335  err_dimm_chrdev:
1336         unregister_chrdev(nvdimm_bus_major, "ndctl");
1337  err_bus_chrdev:
1338         bus_unregister(&nvdimm_bus_type);
1339
1340         return rc;
1341 }
1342
1343 void nvdimm_bus_exit(void)
1344 {
1345         driver_unregister(&nd_bus_driver.drv);
1346         class_destroy(nd_class);
1347         unregister_chrdev(nvdimm_bus_major, "ndctl");
1348         unregister_chrdev(nvdimm_major, "dimmctl");
1349         bus_unregister(&nvdimm_bus_type);
1350         ida_destroy(&nd_ida);
1351 }