drivers/dax/bus.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
   3 #include <linux/memremap.h>
   4 #include <linux/device.h>
   5 #include <linux/mutex.h>
   6 #include <linux/list.h>
   7 #include <linux/slab.h>
   8 #include <linux/dax.h>
   9 #include <linux/io.h>
  10 #include "dax-private.h"
  11 #include "bus.h"
  12
  13 static struct class *dax_class;
  14
  15 static DEFINE_MUTEX(dax_bus_lock);
  16
  17 #define DAX_NAME_LEN 30
  18 struct dax_id {
  19         struct list_head list;
  20         char dev_name[DAX_NAME_LEN];
  21 };
  22
  23 static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  24 {
  25         /*
  26          * We only ever expect to handle device-dax instances, i.e. the
  27          * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
  28          */
  29         return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
  30 }
  31
  32 static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
  33 {
  34         return container_of(drv, struct dax_device_driver, drv);
  35 }
  36
  37 static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
  38                 const char *dev_name)
  39 {
  40         struct dax_id *dax_id;
  41
  42         lockdep_assert_held(&dax_bus_lock);
  43
  44         list_for_each_entry(dax_id, &dax_drv->ids, list)
  45                 if (sysfs_streq(dax_id->dev_name, dev_name))
  46                         return dax_id;
  47         return NULL;
  48 }
  49
  50 static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
  51 {
  52         int match;
  53
  54         mutex_lock(&dax_bus_lock);
  55         match = !!__dax_match_id(dax_drv, dev_name(dev));
  56         mutex_unlock(&dax_bus_lock);
  57
  58         return match;
  59 }
  60
  61 enum id_action {
  62         ID_REMOVE,
  63         ID_ADD,
  64 };
  65
  66 static ssize_t do_id_store(struct device_driver *drv, const char *buf,
  67                 size_t count, enum id_action action)
  68 {
  69         struct dax_device_driver *dax_drv = to_dax_drv(drv);
  70         unsigned int region_id, id;
  71         char devname[DAX_NAME_LEN];
  72         struct dax_id *dax_id;
  73         ssize_t rc = count;
  74         int fields;
  75
  76         fields = sscanf(buf, "dax%d.%d", &region_id, &id);
  77         if (fields != 2)
  78                 return -EINVAL;
  79         sprintf(devname, "dax%d.%d", region_id, id);
  80         if (!sysfs_streq(buf, devname))
  81                 return -EINVAL;
  82
  83         mutex_lock(&dax_bus_lock);
  84         dax_id = __dax_match_id(dax_drv, buf);
  85         if (!dax_id) {
  86                 if (action == ID_ADD) {
  87                         dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
  88                         if (dax_id) {
  89                                 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
  90                                 list_add(&dax_id->list, &dax_drv->ids);
  91                         } else
  92                                 rc = -ENOMEM;
  93                 } else
  94                         /* nothing to remove */;
  95         } else if (action == ID_REMOVE) {
  96                 list_del(&dax_id->list);
  97                 kfree(dax_id);
  98         } else
  99                 /* dax_id already added */;
 100         mutex_unlock(&dax_bus_lock);
 101
 102         if (rc < 0)
 103                 return rc;
 104         if (action == ID_ADD)
 105                 rc = driver_attach(drv);
 106         if (rc)
 107                 return rc;
 108         return count;
 109 }
 110
 111 static ssize_t new_id_store(struct device_driver *drv, const char *buf,
 112                 size_t count)
 113 {
 114         return do_id_store(drv, buf, count, ID_ADD);
 115 }
 116 static DRIVER_ATTR_WO(new_id);
 117
 118 static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
 119                 size_t count)
 120 {
 121         return do_id_store(drv, buf, count, ID_REMOVE);
 122 }
 123 static DRIVER_ATTR_WO(remove_id);
 124
 125 static struct attribute *dax_drv_attrs[] = {
 126         &driver_attr_new_id.attr,
 127         &driver_attr_remove_id.attr,
 128         NULL,
 129 };
 130 ATTRIBUTE_GROUPS(dax_drv);
 131
 132 static int dax_bus_match(struct device *dev, struct device_driver *drv);
 133
 134 static bool is_static(struct dax_region *dax_region)
 135 {
 136         return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
 137 }
 138
 139 static u64 dev_dax_size(struct dev_dax *dev_dax)
 140 {
 141         u64 size = 0;
 142         int i;
 143
 144         device_lock_assert(&dev_dax->dev);
 145
 146         for (i = 0; i < dev_dax->nr_range; i++)
 147                 size += range_len(&dev_dax->ranges[i].range);
 148
 149         return size;
 150 }
 151
 152 static int dax_bus_probe(struct device *dev)
 153 {
 154         struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
 155         struct dev_dax *dev_dax = to_dev_dax(dev);
 156         struct dax_region *dax_region = dev_dax->region;
 157         int rc;
 158
 159         if (dev_dax_size(dev_dax) == 0 || dev_dax->id < 0)
 160                 return -ENXIO;
 161
 162         rc = dax_drv->probe(dev_dax);
 163
 164         if (rc || is_static(dax_region))
 165                 return rc;
 166
 167         /*
 168          * Track new seed creation only after successful probe of the
 169          * previous seed.
 170          */
 171         if (dax_region->seed == dev)
 172                 dax_region->seed = NULL;
 173
 174         return 0;
 175 }
 176
 177 static int dax_bus_remove(struct device *dev)
 178 {
 179         struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
 180         struct dev_dax *dev_dax = to_dev_dax(dev);
 181
 182         return dax_drv->remove(dev_dax);
 183 }
 184
 185 static struct bus_type dax_bus_type = {
 186         .name = "dax",
 187         .uevent = dax_bus_uevent,
 188         .match = dax_bus_match,
 189         .probe = dax_bus_probe,
 190         .remove = dax_bus_remove,
 191         .drv_groups = dax_drv_groups,
 192 };
 193
 194 static int dax_bus_match(struct device *dev, struct device_driver *drv)
 195 {
 196         struct dax_device_driver *dax_drv = to_dax_drv(drv);
 197
 198         /*
 199          * All but the 'device-dax' driver, which has 'match_always'
 200          * set, requires an exact id match.
 201          */
 202         if (dax_drv->match_always)
 203                 return 1;
 204
 205         return dax_match_id(dax_drv, dev);
 206 }
 207
 208 /*
 209  * Rely on the fact that drvdata is set before the attributes are
 210  * registered, and that the attributes are unregistered before drvdata
 211  * is cleared to assume that drvdata is always valid.
 212  */
 213 static ssize_t id_show(struct device *dev,
 214                 struct device_attribute *attr, char *buf)
 215 {
 216         struct dax_region *dax_region = dev_get_drvdata(dev);
 217
 218         return sprintf(buf, "%d\n", dax_region->id);
 219 }
 220 static DEVICE_ATTR_RO(id);
 221
 222 static ssize_t region_size_show(struct device *dev,
 223                 struct device_attribute *attr, char *buf)
 224 {
 225         struct dax_region *dax_region = dev_get_drvdata(dev);
 226
 227         return sprintf(buf, "%llu\n", (unsigned long long)
 228                         resource_size(&dax_region->res));
 229 }
 230 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
 231                 region_size_show, NULL);
 232
 233 static ssize_t region_align_show(struct device *dev,
 234                 struct device_attribute *attr, char *buf)
 235 {
 236         struct dax_region *dax_region = dev_get_drvdata(dev);
 237
 238         return sprintf(buf, "%u\n", dax_region->align);
 239 }
 240 static struct device_attribute dev_attr_region_align =
 241                 __ATTR(align, 0400, region_align_show, NULL);
 242
 243 #define for_each_dax_region_resource(dax_region, res) \
 244         for (res = (dax_region)->res.child; res; res = res->sibling)
 245
 246 static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
 247 {
 248         resource_size_t size = resource_size(&dax_region->res);
 249         struct resource *res;
 250
 251         device_lock_assert(dax_region->dev);
 252
 253         for_each_dax_region_resource(dax_region, res)
 254                 size -= resource_size(res);
 255         return size;
 256 }
 257
 258 static ssize_t available_size_show(struct device *dev,
 259                 struct device_attribute *attr, char *buf)
 260 {
 261         struct dax_region *dax_region = dev_get_drvdata(dev);
 262         unsigned long long size;
 263
 264         device_lock(dev);
 265         size = dax_region_avail_size(dax_region);
 266         device_unlock(dev);
 267
 268         return sprintf(buf, "%llu\n", size);
 269 }
 270 static DEVICE_ATTR_RO(available_size);
 271
 272 static ssize_t seed_show(struct device *dev,
 273                 struct device_attribute *attr, char *buf)
 274 {
 275         struct dax_region *dax_region = dev_get_drvdata(dev);
 276         struct device *seed;
 277         ssize_t rc;
 278
 279         if (is_static(dax_region))
 280                 return -EINVAL;
 281
 282         device_lock(dev);
 283         seed = dax_region->seed;
 284         rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
 285         device_unlock(dev);
 286
 287         return rc;
 288 }
 289 static DEVICE_ATTR_RO(seed);
 290
 291 static ssize_t create_show(struct device *dev,
 292                 struct device_attribute *attr, char *buf)
 293 {
 294         struct dax_region *dax_region = dev_get_drvdata(dev);
 295         struct device *youngest;
 296         ssize_t rc;
 297
 298         if (is_static(dax_region))
 299                 return -EINVAL;
 300
 301         device_lock(dev);
 302         youngest = dax_region->youngest;
 303         rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
 304         device_unlock(dev);
 305
 306         return rc;
 307 }
 308
 309 static ssize_t create_store(struct device *dev, struct device_attribute *attr,
 310                 const char *buf, size_t len)
 311 {
 312         struct dax_region *dax_region = dev_get_drvdata(dev);
 313         unsigned long long avail;
 314         ssize_t rc;
 315         int val;
 316
 317         if (is_static(dax_region))
 318                 return -EINVAL;
 319
 320         rc = kstrtoint(buf, 0, &val);
 321         if (rc)
 322                 return rc;
 323         if (val != 1)
 324                 return -EINVAL;
 325
 326         device_lock(dev);
 327         avail = dax_region_avail_size(dax_region);
 328         if (avail == 0)
 329                 rc = -ENOSPC;
 330         else {
 331                 struct dev_dax_data data = {
 332                         .dax_region = dax_region,
 333                         .size = 0,
 334                         .id = -1,
 335                 };
 336                 struct dev_dax *dev_dax = devm_create_dev_dax(&data);
 337
 338                 if (IS_ERR(dev_dax))
 339                         rc = PTR_ERR(dev_dax);
 340                 else {
 341                         /*
 342                          * In support of crafting multiple new devices
 343                          * simultaneously multiple seeds can be created,
 344                          * but only the first one that has not been
 345                          * successfully bound is tracked as the region
 346                          * seed.
 347                          */
 348                         if (!dax_region->seed)
 349                                 dax_region->seed = &dev_dax->dev;
 350                         dax_region->youngest = &dev_dax->dev;
 351                         rc = len;
 352                 }
 353         }
 354         device_unlock(dev);
 355
 356         return rc;
 357 }
 358 static DEVICE_ATTR_RW(create);
 359
 360 void kill_dev_dax(struct dev_dax *dev_dax)
 361 {
 362         struct dax_device *dax_dev = dev_dax->dax_dev;
 363         struct inode *inode = dax_inode(dax_dev);
 364
 365         kill_dax(dax_dev);
 366         unmap_mapping_range(inode->i_mapping, 0, 0, 1);
 367 }
 368 EXPORT_SYMBOL_GPL(kill_dev_dax);
 369
 370 static void free_dev_dax_ranges(struct dev_dax *dev_dax)
 371 {
 372         struct dax_region *dax_region = dev_dax->region;
 373         int i;
 374
 375         device_lock_assert(dax_region->dev);
 376         for (i = 0; i < dev_dax->nr_range; i++) {
 377                 struct range *range = &dev_dax->ranges[i].range;
 378
 379                 __release_region(&dax_region->res, range->start,
 380                                 range_len(range));
 381         }
 382         dev_dax->nr_range = 0;
 383 }
 384
 385 static void unregister_dev_dax(void *dev)
 386 {
 387         struct dev_dax *dev_dax = to_dev_dax(dev);
 388
 389         dev_dbg(dev, "%s\n", __func__);
 390
 391         kill_dev_dax(dev_dax);
 392         free_dev_dax_ranges(dev_dax);
 393         device_del(dev);
 394         put_device(dev);
 395 }
 396
 397 /* a return value >= 0 indicates this invocation invalidated the id */
 398 static int __free_dev_dax_id(struct dev_dax *dev_dax)
 399 {
 400         struct dax_region *dax_region = dev_dax->region;
 401         struct device *dev = &dev_dax->dev;
 402         int rc = dev_dax->id;
 403
 404         device_lock_assert(dev);
 405
 406         if (is_static(dax_region) || dev_dax->id < 0)
 407                 return -1;
 408         ida_free(&dax_region->ida, dev_dax->id);
 409         dev_dax->id = -1;
 410         return rc;
 411 }
 412
 413 static int free_dev_dax_id(struct dev_dax *dev_dax)
 414 {
 415         struct device *dev = &dev_dax->dev;
 416         int rc;
 417
 418         device_lock(dev);
 419         rc = __free_dev_dax_id(dev_dax);
 420         device_unlock(dev);
 421         return rc;
 422 }
 423
 424 static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
 425                 const char *buf, size_t len)
 426 {
 427         struct dax_region *dax_region = dev_get_drvdata(dev);
 428         struct dev_dax *dev_dax;
 429         struct device *victim;
 430         bool do_del = false;
 431         int rc;
 432
 433         if (is_static(dax_region))
 434                 return -EINVAL;
 435
 436         victim = device_find_child_by_name(dax_region->dev, buf);
 437         if (!victim)
 438                 return -ENXIO;
 439
 440         device_lock(dev);
 441         device_lock(victim);
 442         dev_dax = to_dev_dax(victim);
 443         if (victim->driver || dev_dax_size(dev_dax))
 444                 rc = -EBUSY;
 445         else {
 446                 /*
 447                  * Invalidate the device so it does not become active
 448                  * again, but always preserve device-id-0 so that
 449                  * /sys/bus/dax/ is guaranteed to be populated while any
 450                  * dax_region is registered.
 451                  */
 452                 if (dev_dax->id > 0) {
 453                         do_del = __free_dev_dax_id(dev_dax) >= 0;
 454                         rc = len;
 455                         if (dax_region->seed == victim)
 456                                 dax_region->seed = NULL;
 457                         if (dax_region->youngest == victim)
 458                                 dax_region->youngest = NULL;
 459                 } else
 460                         rc = -EBUSY;
 461         }
 462         device_unlock(victim);
 463
 464         /* won the race to invalidate the device, clean it up */
 465         if (do_del)
 466                 devm_release_action(dev, unregister_dev_dax, victim);
 467         device_unlock(dev);
 468         put_device(victim);
 469
 470         return rc;
 471 }
 472 static DEVICE_ATTR_WO(delete);
 473
 474 static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
 475                 int n)
 476 {
 477         struct device *dev = container_of(kobj, struct device, kobj);
 478         struct dax_region *dax_region = dev_get_drvdata(dev);
 479
 480         if (is_static(dax_region))
 481                 if (a == &dev_attr_available_size.attr
 482                                 || a == &dev_attr_create.attr
 483                                 || a == &dev_attr_seed.attr
 484                                 || a == &dev_attr_delete.attr)
 485                         return 0;
 486         return a->mode;
 487 }
 488
 489 static struct attribute *dax_region_attributes[] = {
 490         &dev_attr_available_size.attr,
 491         &dev_attr_region_size.attr,
 492         &dev_attr_region_align.attr,
 493         &dev_attr_create.attr,
 494         &dev_attr_seed.attr,
 495         &dev_attr_delete.attr,
 496         &dev_attr_id.attr,
 497         NULL,
 498 };
 499
 500 static const struct attribute_group dax_region_attribute_group = {
 501         .name = "dax_region",
 502         .attrs = dax_region_attributes,
 503         .is_visible = dax_region_visible,
 504 };
 505
 506 static const struct attribute_group *dax_region_attribute_groups[] = {
 507         &dax_region_attribute_group,
 508         NULL,
 509 };
 510
 511 static void dax_region_free(struct kref *kref)
 512 {
 513         struct dax_region *dax_region;
 514
 515         dax_region = container_of(kref, struct dax_region, kref);
 516         kfree(dax_region);
 517 }
 518
 519 void dax_region_put(struct dax_region *dax_region)
 520 {
 521         kref_put(&dax_region->kref, dax_region_free);
 522 }
 523 EXPORT_SYMBOL_GPL(dax_region_put);
 524
 525 static void dax_region_unregister(void *region)
 526 {
 527         struct dax_region *dax_region = region;
 528
 529         sysfs_remove_groups(&dax_region->dev->kobj,
 530                         dax_region_attribute_groups);
 531         dax_region_put(dax_region);
 532 }
 533
 534 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 535                 struct range *range, int target_node, unsigned int align,
 536                 unsigned long flags)
 537 {
 538         struct dax_region *dax_region;
 539
 540         /*
 541          * The DAX core assumes that it can store its private data in
 542          * parent->driver_data. This WARN is a reminder / safeguard for
 543          * developers of device-dax drivers.
 544          */
 545         if (dev_get_drvdata(parent)) {
 546                 dev_WARN(parent, "dax core failed to setup private data\n");
 547                 return NULL;
 548         }
 549
 550         if (!IS_ALIGNED(range->start, align)
 551                         || !IS_ALIGNED(range_len(range), align))
 552                 return NULL;
 553
 554         dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
 555         if (!dax_region)
 556                 return NULL;
 557
 558         dev_set_drvdata(parent, dax_region);
 559         kref_init(&dax_region->kref);
 560         dax_region->id = region_id;
 561         dax_region->align = align;
 562         dax_region->dev = parent;
 563         dax_region->target_node = target_node;
 564         ida_init(&dax_region->ida);
 565         dax_region->res = (struct resource) {
 566                 .start = range->start,
 567                 .end = range->end,
 568                 .flags = IORESOURCE_MEM | flags,
 569         };
 570
 571         if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
 572                 kfree(dax_region);
 573                 return NULL;
 574         }
 575
 576         kref_get(&dax_region->kref);
 577         if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
 578                 return NULL;
 579         return dax_region;
 580 }
 581 EXPORT_SYMBOL_GPL(alloc_dax_region);
 582
 583 static void dax_mapping_release(struct device *dev)
 584 {
 585         struct dax_mapping *mapping = to_dax_mapping(dev);
 586         struct dev_dax *dev_dax = to_dev_dax(dev->parent);
 587
 588         ida_free(&dev_dax->ida, mapping->id);
 589         kfree(mapping);
 590 }
 591
 592 static void unregister_dax_mapping(void *data)
 593 {
 594         struct device *dev = data;
 595         struct dax_mapping *mapping = to_dax_mapping(dev);
 596         struct dev_dax *dev_dax = to_dev_dax(dev->parent);
 597         struct dax_region *dax_region = dev_dax->region;
 598
 599         dev_dbg(dev, "%s\n", __func__);
 600
 601         device_lock_assert(dax_region->dev);
 602
 603         dev_dax->ranges[mapping->range_id].mapping = NULL;
 604         mapping->range_id = -1;
 605
 606         device_del(dev);
 607         put_device(dev);
 608 }
 609
 610 static struct dev_dax_range *get_dax_range(struct device *dev)
 611 {
 612         struct dax_mapping *mapping = to_dax_mapping(dev);
 613         struct dev_dax *dev_dax = to_dev_dax(dev->parent);
 614         struct dax_region *dax_region = dev_dax->region;
 615
 616         device_lock(dax_region->dev);
 617         if (mapping->range_id < 0) {
 618                 device_unlock(dax_region->dev);
 619                 return NULL;
 620         }
 621
 622         return &dev_dax->ranges[mapping->range_id];
 623 }
 624
 625 static void put_dax_range(struct dev_dax_range *dax_range)
 626 {
 627         struct dax_mapping *mapping = dax_range->mapping;
 628         struct dev_dax *dev_dax = to_dev_dax(mapping->dev.parent);
 629         struct dax_region *dax_region = dev_dax->region;
 630
 631         device_unlock(dax_region->dev);
 632 }
 633
 634 static ssize_t start_show(struct device *dev,
 635                 struct device_attribute *attr, char *buf)
 636 {
 637         struct dev_dax_range *dax_range;
 638         ssize_t rc;
 639
 640         dax_range = get_dax_range(dev);
 641         if (!dax_range)
 642                 return -ENXIO;
 643         rc = sprintf(buf, "%#llx\n", dax_range->range.start);
 644         put_dax_range(dax_range);
 645
 646         return rc;
 647 }
 648 static DEVICE_ATTR(start, 0400, start_show, NULL);
 649
 650 static ssize_t end_show(struct device *dev,
 651                 struct device_attribute *attr, char *buf)
 652 {
 653         struct dev_dax_range *dax_range;
 654         ssize_t rc;
 655
 656         dax_range = get_dax_range(dev);
 657         if (!dax_range)
 658                 return -ENXIO;
 659         rc = sprintf(buf, "%#llx\n", dax_range->range.end);
 660         put_dax_range(dax_range);
 661
 662         return rc;
 663 }
 664 static DEVICE_ATTR(end, 0400, end_show, NULL);
 665
 666 static ssize_t pgoff_show(struct device *dev,
 667                 struct device_attribute *attr, char *buf)
 668 {
 669         struct dev_dax_range *dax_range;
 670         ssize_t rc;
 671
 672         dax_range = get_dax_range(dev);
 673         if (!dax_range)
 674                 return -ENXIO;
 675         rc = sprintf(buf, "%#lx\n", dax_range->pgoff);
 676         put_dax_range(dax_range);
 677
 678         return rc;
 679 }
 680 static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
 681
 682 static struct attribute *dax_mapping_attributes[] = {
 683         &dev_attr_start.attr,
 684         &dev_attr_end.attr,
 685         &dev_attr_page_offset.attr,
 686         NULL,
 687 };
 688
 689 static const struct attribute_group dax_mapping_attribute_group = {
 690         .attrs = dax_mapping_attributes,
 691 };
 692
 693 static const struct attribute_group *dax_mapping_attribute_groups[] = {
 694         &dax_mapping_attribute_group,
 695         NULL,
 696 };
 697
 698 static struct device_type dax_mapping_type = {
 699         .release = dax_mapping_release,
 700         .groups = dax_mapping_attribute_groups,
 701 };
 702
 703 static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
 704 {
 705         struct dax_region *dax_region = dev_dax->region;
 706         struct dax_mapping *mapping;
 707         struct device *dev;
 708         int rc;
 709
 710         device_lock_assert(dax_region->dev);
 711
 712         if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
 713                                 "region disabled\n"))
 714                 return -ENXIO;
 715
 716         mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 717         if (!mapping)
 718                 return -ENOMEM;
 719         mapping->range_id = range_id;
 720         mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
 721         if (mapping->id < 0) {
 722                 kfree(mapping);
 723                 return -ENOMEM;
 724         }
 725         dev_dax->ranges[range_id].mapping = mapping;
 726         dev = &mapping->dev;
 727         device_initialize(dev);
 728         dev->parent = &dev_dax->dev;
 729         dev->type = &dax_mapping_type;
 730         dev_set_name(dev, "mapping%d", mapping->id);
 731         rc = device_add(dev);
 732         if (rc) {
 733                 put_device(dev);
 734                 return rc;
 735         }
 736
 737         rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
 738                         dev);
 739         if (rc)
 740                 return rc;
 741         return 0;
 742 }
 743
 744 static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
 745                 resource_size_t size)
 746 {
 747         struct dax_region *dax_region = dev_dax->region;
 748         struct resource *res = &dax_region->res;
 749         struct device *dev = &dev_dax->dev;
 750         struct dev_dax_range *ranges;
 751         unsigned long pgoff = 0;
 752         struct resource *alloc;
 753         int i, rc;
 754
 755         device_lock_assert(dax_region->dev);
 756
 757         /* handle the seed alloc special case */
 758         if (!size) {
 759                 if (dev_WARN_ONCE(dev, dev_dax->nr_range,
 760                                         "0-size allocation must be first\n"))
 761                         return -EBUSY;
 762                 /* nr_range == 0 is elsewhere special cased as 0-size device */
 763                 return 0;
 764         }
 765
 766         ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
 767                         * (dev_dax->nr_range + 1), GFP_KERNEL);
 768         if (!ranges)
 769                 return -ENOMEM;
 770
 771         alloc = __request_region(res, start, size, dev_name(dev), 0);
 772         if (!alloc) {
 773                 /*
 774                  * If this was an empty set of ranges nothing else
 775                  * will release @ranges, so do it now.
 776                  */
 777                 if (!dev_dax->nr_range) {
 778                         kfree(ranges);
 779                         ranges = NULL;
 780                 }
 781                 dev_dax->ranges = ranges;
 782                 return -ENOMEM;
 783         }
 784
 785         for (i = 0; i < dev_dax->nr_range; i++)
 786                 pgoff += PHYS_PFN(range_len(&ranges[i].range));
 787         dev_dax->ranges = ranges;
 788         ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
 789                 .pgoff = pgoff,
 790                 .range = {
 791                         .start = alloc->start,
 792                         .end = alloc->end,
 793                 },
 794         };
 795
 796         dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
 797                         &alloc->start, &alloc->end);
 798         /*
 799          * A dev_dax instance must be registered before mapping device
 800          * children can be added. Defer to devm_create_dev_dax() to add
 801          * the initial mapping device.
 802          */
 803         if (!device_is_registered(&dev_dax->dev))
 804                 return 0;
 805
 806         rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
 807         if (rc) {
 808                 dev_dbg(dev, "delete range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
 809                                 &alloc->start, &alloc->end);
 810                 dev_dax->nr_range--;
 811                 __release_region(res, alloc->start, resource_size(alloc));
 812                 return rc;
 813         }
 814
 815         return 0;
 816 }
 817
 818 static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
 819 {
 820         int last_range = dev_dax->nr_range - 1;
 821         struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
 822         struct dax_region *dax_region = dev_dax->region;
 823         bool is_shrink = resource_size(res) > size;
 824         struct range *range = &dax_range->range;
 825         struct device *dev = &dev_dax->dev;
 826         int rc;
 827
 828         device_lock_assert(dax_region->dev);
 829
 830         if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
 831                 return -EINVAL;
 832
 833         rc = adjust_resource(res, range->start, size);
 834         if (rc)
 835                 return rc;
 836
 837         *range = (struct range) {
 838                 .start = range->start,
 839                 .end = range->start + size - 1,
 840         };
 841
 842         dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
 843                         last_range, (unsigned long long) range->start,
 844                         (unsigned long long) range->end);
 845
 846         return 0;
 847 }
 848
 849 static ssize_t size_show(struct device *dev,
 850                 struct device_attribute *attr, char *buf)
 851 {
 852         struct dev_dax *dev_dax = to_dev_dax(dev);
 853         unsigned long long size;
 854
 855         device_lock(dev);
 856         size = dev_dax_size(dev_dax);
 857         device_unlock(dev);
 858
 859         return sprintf(buf, "%llu\n", size);
 860 }
 861
 862 static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
 863 {
 864         /*
 865          * The minimum mapping granularity for a device instance is a
 866          * single subsection, unless the arch says otherwise.
 867          */
 868         return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
 869 }
 870
 871 static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
 872 {
 873         resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
 874         struct dax_region *dax_region = dev_dax->region;
 875         struct device *dev = &dev_dax->dev;
 876         int i;
 877
 878         for (i = dev_dax->nr_range - 1; i >= 0; i--) {
 879                 struct range *range = &dev_dax->ranges[i].range;
 880                 struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
 881                 struct resource *adjust = NULL, *res;
 882                 resource_size_t shrink;
 883
 884                 shrink = min_t(u64, to_shrink, range_len(range));
 885                 if (shrink >= range_len(range)) {
 886                         devm_release_action(dax_region->dev,
 887                                         unregister_dax_mapping, &mapping->dev);
 888                         __release_region(&dax_region->res, range->start,
 889                                         range_len(range));
 890                         dev_dax->nr_range--;
 891                         dev_dbg(dev, "delete range[%d]: %#llx:%#llx\n", i,
 892                                         (unsigned long long) range->start,
 893                                         (unsigned long long) range->end);
 894                         to_shrink -= shrink;
 895                         if (!to_shrink)
 896                                 break;
 897                         continue;
 898                 }
 899
 900                 for_each_dax_region_resource(dax_region, res)
 901                         if (strcmp(res->name, dev_name(dev)) == 0
 902                                         && res->start == range->start) {
 903                                 adjust = res;
 904                                 break;
 905                         }
 906
 907                 if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
 908                                         "failed to find matching resource\n"))
 909                         return -ENXIO;
 910                 return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
 911                                 - shrink);
 912         }
 913         return 0;
 914 }
 915
 916 /*
 917  * Only allow adjustments that preserve the relative pgoff of existing
 918  * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
 919  */
 920 static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
 921 {
 922         struct dev_dax_range *last;
 923         int i;
 924
 925         if (dev_dax->nr_range == 0)
 926                 return false;
 927         if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
 928                 return false;
 929         last = &dev_dax->ranges[dev_dax->nr_range - 1];
 930         if (last->range.start != res->start || last->range.end != res->end)
 931                 return false;
 932         for (i = 0; i < dev_dax->nr_range - 1; i++) {
 933                 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
 934
 935                 if (dax_range->pgoff > last->pgoff)
 936                         return false;
 937         }
 938
 939         return true;
 940 }
 941
 942 static ssize_t dev_dax_resize(struct dax_region *dax_region,
 943                 struct dev_dax *dev_dax, resource_size_t size)
 944 {
 945         resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
 946         resource_size_t dev_size = dev_dax_size(dev_dax);
 947         struct resource *region_res = &dax_region->res;
 948         struct device *dev = &dev_dax->dev;
 949         struct resource *res, *first;
 950         resource_size_t alloc = 0;
 951         int rc;
 952
 953         if (dev->driver)
 954                 return -EBUSY;
 955         if (size == dev_size)
 956                 return 0;
 957         if (size > dev_size && size - dev_size > avail)
 958                 return -ENOSPC;
 959         if (size < dev_size)
 960                 return dev_dax_shrink(dev_dax, size);
 961
 962         to_alloc = size - dev_size;
 963         if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
 964                         "resize of %pa misaligned\n", &to_alloc))
 965                 return -ENXIO;
 966
 967         /*
 968          * Expand the device into the unused portion of the region. This
 969          * may involve adjusting the end of an existing resource, or
 970          * allocating a new resource.
 971          */
 972 retry:
 973         first = region_res->child;
 974         if (!first)
 975                 return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
 976
 977         rc = -ENOSPC;
 978         for (res = first; res; res = res->sibling) {
 979                 struct resource *next = res->sibling;
 980
 981                 /* space at the beginning of the region */
 982                 if (res == first && res->start > dax_region->res.start) {
 983                         alloc = min(res->start - dax_region->res.start, to_alloc);
 984                         rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
 985                         break;
 986                 }
 987
 988                 alloc = 0;
 989                 /* space between allocations */
 990                 if (next && next->start > res->end + 1)
 991                         alloc = min(next->start - (res->end + 1), to_alloc);
 992
 993                 /* space at the end of the region */
 994                 if (!alloc && !next && res->end < region_res->end)
 995                         alloc = min(region_res->end - res->end, to_alloc);
 996
 997                 if (!alloc)
 998                         continue;
 999
1000                 if (adjust_ok(dev_dax, res)) {
1001                         rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
1002                         break;
1003                 }
1004                 rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
1005                 break;
1006         }
1007         if (rc)
1008                 return rc;
1009         to_alloc -= alloc;
1010         if (to_alloc)
1011                 goto retry;
1012         return 0;
1013 }
1014
1015 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
1016                 const char *buf, size_t len)
1017 {
1018         ssize_t rc;
1019         unsigned long long val;
1020         struct dev_dax *dev_dax = to_dev_dax(dev);
1021         struct dax_region *dax_region = dev_dax->region;
1022
1023         rc = kstrtoull(buf, 0, &val);
1024         if (rc)
1025                 return rc;
1026
1027         if (!alloc_is_aligned(dev_dax, val)) {
1028                 dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
1029                 return -EINVAL;
1030         }
1031
1032         device_lock(dax_region->dev);
1033         if (!dax_region->dev->driver) {
1034                 device_unlock(dax_region->dev);
1035                 return -ENXIO;
1036         }
1037         device_lock(dev);
1038         rc = dev_dax_resize(dax_region, dev_dax, val);
1039         device_unlock(dev);
1040         device_unlock(dax_region->dev);
1041
1042         return rc == 0 ? len : rc;
1043 }
1044 static DEVICE_ATTR_RW(size);
1045
1046 static ssize_t range_parse(const char *opt, size_t len, struct range *range)
1047 {
1048         unsigned long long addr = 0;
1049         char *start, *end, *str;
1050         ssize_t rc = EINVAL;
1051
1052         str = kstrdup(opt, GFP_KERNEL);
1053         if (!str)
1054                 return rc;
1055
1056         end = str;
1057         start = strsep(&end, "-");
1058         if (!start || !end)
1059                 goto err;
1060
1061         rc = kstrtoull(start, 16, &addr);
1062         if (rc)
1063                 goto err;
1064         range->start = addr;
1065
1066         rc = kstrtoull(end, 16, &addr);
1067         if (rc)
1068                 goto err;
1069         range->end = addr;
1070
1071 err:
1072         kfree(str);
1073         return rc;
1074 }
1075
1076 static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
1077                 const char *buf, size_t len)
1078 {
1079         struct dev_dax *dev_dax = to_dev_dax(dev);
1080         struct dax_region *dax_region = dev_dax->region;
1081         size_t to_alloc;
1082         struct range r;
1083         ssize_t rc;
1084
1085         rc = range_parse(buf, len, &r);
1086         if (rc)
1087                 return rc;
1088
1089         rc = -ENXIO;
1090         device_lock(dax_region->dev);
1091         if (!dax_region->dev->driver) {
1092                 device_unlock(dax_region->dev);
1093                 return rc;
1094         }
1095         device_lock(dev);
1096
1097         to_alloc = range_len(&r);
1098         if (alloc_is_aligned(dev_dax, to_alloc))
1099                 rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
1100         device_unlock(dev);
1101         device_unlock(dax_region->dev);
1102
1103         return rc == 0 ? len : rc;
1104 }
1105 static DEVICE_ATTR_WO(mapping);
1106
1107 static ssize_t align_show(struct device *dev,
1108                 struct device_attribute *attr, char *buf)
1109 {
1110         struct dev_dax *dev_dax = to_dev_dax(dev);
1111
1112         return sprintf(buf, "%d\n", dev_dax->align);
1113 }
1114
1115 static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
1116 {
1117         resource_size_t dev_size = dev_dax_size(dev_dax);
1118         struct device *dev = &dev_dax->dev;
1119         int i;
1120
1121         if (dev_size > 0 && !alloc_is_aligned(dev_dax, dev_size)) {
1122                 dev_dbg(dev, "%s: align %u invalid for size %pa\n",
1123                         __func__, dev_dax->align, &dev_size);
1124                 return -EINVAL;
1125         }
1126
1127         for (i = 0; i < dev_dax->nr_range; i++) {
1128                 size_t len = range_len(&dev_dax->ranges[i].range);
1129
1130                 if (!alloc_is_aligned(dev_dax, len)) {
1131                         dev_dbg(dev, "%s: align %u invalid for range %d\n",
1132                                 __func__, dev_dax->align, i);
1133                         return -EINVAL;
1134                 }
1135         }
1136
1137         return 0;
1138 }
1139
1140 static ssize_t align_store(struct device *dev, struct device_attribute *attr,
1141                 const char *buf, size_t len)
1142 {
1143         struct dev_dax *dev_dax = to_dev_dax(dev);
1144         struct dax_region *dax_region = dev_dax->region;
1145         unsigned long val, align_save;
1146         ssize_t rc;
1147
1148         rc = kstrtoul(buf, 0, &val);
1149         if (rc)
1150                 return -ENXIO;
1151
1152         if (!dax_align_valid(val))
1153                 return -EINVAL;
1154
1155         device_lock(dax_region->dev);
1156         if (!dax_region->dev->driver) {
1157                 device_unlock(dax_region->dev);
1158                 return -ENXIO;
1159         }
1160
1161         device_lock(dev);
1162         if (dev->driver) {
1163                 rc = -EBUSY;
1164                 goto out_unlock;
1165         }
1166
1167         align_save = dev_dax->align;
1168         dev_dax->align = val;
1169         rc = dev_dax_validate_align(dev_dax);
1170         if (rc)
1171                 dev_dax->align = align_save;
1172 out_unlock:
1173         device_unlock(dev);
1174         device_unlock(dax_region->dev);
1175         return rc == 0 ? len : rc;
1176 }
1177 static DEVICE_ATTR_RW(align);
1178
1179 static int dev_dax_target_node(struct dev_dax *dev_dax)
1180 {
1181         struct dax_region *dax_region = dev_dax->region;
1182
1183         return dax_region->target_node;
1184 }
1185
1186 static ssize_t target_node_show(struct device *dev,
1187                 struct device_attribute *attr, char *buf)
1188 {
1189         struct dev_dax *dev_dax = to_dev_dax(dev);
1190
1191         return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
1192 }
1193 static DEVICE_ATTR_RO(target_node);
1194
1195 static ssize_t resource_show(struct device *dev,
1196                 struct device_attribute *attr, char *buf)
1197 {
1198         struct dev_dax *dev_dax = to_dev_dax(dev);
1199         struct dax_region *dax_region = dev_dax->region;
1200         unsigned long long start;
1201
1202         if (dev_dax->nr_range < 1)
1203                 start = dax_region->res.start;
1204         else
1205                 start = dev_dax->ranges[0].range.start;
1206
1207         return sprintf(buf, "%#llx\n", start);
1208 }
1209 static DEVICE_ATTR(resource, 0400, resource_show, NULL);
1210
1211 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
1212                 char *buf)
1213 {
1214         /*
1215          * We only ever expect to handle device-dax instances, i.e. the
1216          * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
1217          */
1218         return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
1219 }
1220 static DEVICE_ATTR_RO(modalias);
1221
1222 static ssize_t numa_node_show(struct device *dev,
1223                 struct device_attribute *attr, char *buf)
1224 {
1225         return sprintf(buf, "%d\n", dev_to_node(dev));
1226 }
1227 static DEVICE_ATTR_RO(numa_node);
1228
1229 static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
1230 {
1231         struct device *dev = container_of(kobj, struct device, kobj);
1232         struct dev_dax *dev_dax = to_dev_dax(dev);
1233         struct dax_region *dax_region = dev_dax->region;
1234
1235         if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
1236                 return 0;
1237         if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
1238                 return 0;
1239         if (a == &dev_attr_mapping.attr && is_static(dax_region))
1240                 return 0;
1241         if ((a == &dev_attr_align.attr ||
1242              a == &dev_attr_size.attr) && is_static(dax_region))
1243                 return 0444;
1244         return a->mode;
1245 }
1246
1247 static struct attribute *dev_dax_attributes[] = {
1248         &dev_attr_modalias.attr,
1249         &dev_attr_size.attr,
1250         &dev_attr_mapping.attr,
1251         &dev_attr_target_node.attr,
1252         &dev_attr_align.attr,
1253         &dev_attr_resource.attr,
1254         &dev_attr_numa_node.attr,
1255         NULL,
1256 };
1257
1258 static const struct attribute_group dev_dax_attribute_group = {
1259         .attrs = dev_dax_attributes,
1260         .is_visible = dev_dax_visible,
1261 };
1262
1263 static const struct attribute_group *dax_attribute_groups[] = {
1264         &dev_dax_attribute_group,
1265         NULL,
1266 };
1267
1268 static void dev_dax_release(struct device *dev)
1269 {
1270         struct dev_dax *dev_dax = to_dev_dax(dev);
1271         struct dax_region *dax_region = dev_dax->region;
1272         struct dax_device *dax_dev = dev_dax->dax_dev;
1273
1274         put_dax(dax_dev);
1275         free_dev_dax_id(dev_dax);
1276         dax_region_put(dax_region);
1277         kfree(dev_dax->ranges);
1278         kfree(dev_dax->pgmap);
1279         kfree(dev_dax);
1280 }
1281
1282 static const struct device_type dev_dax_type = {
1283         .release = dev_dax_release,
1284         .groups = dax_attribute_groups,
1285 };
1286
1287 struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
1288 {
1289         struct dax_region *dax_region = data->dax_region;
1290         struct device *parent = dax_region->dev;
1291         struct dax_device *dax_dev;
1292         struct dev_dax *dev_dax;
1293         struct inode *inode;
1294         struct device *dev;
1295         int rc;
1296
1297         dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
1298         if (!dev_dax)
1299                 return ERR_PTR(-ENOMEM);
1300
1301         if (is_static(dax_region)) {
1302                 if (dev_WARN_ONCE(parent, data->id < 0,
1303                                 "dynamic id specified to static region\n")) {
1304                         rc = -EINVAL;
1305                         goto err_id;
1306                 }
1307
1308                 dev_dax->id = data->id;
1309         } else {
1310                 if (dev_WARN_ONCE(parent, data->id >= 0,
1311                                 "static id specified to dynamic region\n")) {
1312                         rc = -EINVAL;
1313                         goto err_id;
1314                 }
1315
1316                 rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
1317                 if (rc < 0)
1318                         goto err_id;
1319                 dev_dax->id = rc;
1320         }
1321
1322         dev_dax->region = dax_region;
1323         dev = &dev_dax->dev;
1324         device_initialize(dev);
1325         dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
1326
1327         rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
1328         if (rc)
1329                 goto err_range;
1330
1331         if (data->pgmap) {
1332                 dev_WARN_ONCE(parent, !is_static(dax_region),
1333                         "custom dev_pagemap requires a static dax_region\n");
1334
1335                 dev_dax->pgmap = kmemdup(data->pgmap,
1336                                 sizeof(struct dev_pagemap), GFP_KERNEL);
1337                 if (!dev_dax->pgmap) {
1338                         rc = -ENOMEM;
1339                         goto err_pgmap;
1340                 }
1341         }
1342
1343         /*
1344          * No 'host' or dax_operations since there is no access to this
1345          * device outside of mmap of the resulting character device.
1346          */
1347         dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC);
1348         if (IS_ERR(dax_dev)) {
1349                 rc = PTR_ERR(dax_dev);
1350                 goto err_alloc_dax;
1351         }
1352
1353         /* a device_dax instance is dead while the driver is not attached */
1354         kill_dax(dax_dev);
1355
1356         dev_dax->dax_dev = dax_dev;
1357         dev_dax->target_node = dax_region->target_node;
1358         dev_dax->align = dax_region->align;
1359         ida_init(&dev_dax->ida);
1360         kref_get(&dax_region->kref);
1361
1362         inode = dax_inode(dax_dev);
1363         dev->devt = inode->i_rdev;
1364         if (data->subsys == DEV_DAX_BUS)
1365                 dev->bus = &dax_bus_type;
1366         else
1367                 dev->class = dax_class;
1368         dev->parent = parent;
1369         dev->type = &dev_dax_type;
1370
1371         rc = device_add(dev);
1372         if (rc) {
1373                 kill_dev_dax(dev_dax);
1374                 put_device(dev);
1375                 return ERR_PTR(rc);
1376         }
1377
1378         rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
1379         if (rc)
1380                 return ERR_PTR(rc);
1381
1382         /* register mapping device for the initial allocation range */
1383         if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
1384                 rc = devm_register_dax_mapping(dev_dax, 0);
1385                 if (rc)
1386                         return ERR_PTR(rc);
1387         }
1388
1389         return dev_dax;
1390
1391 err_alloc_dax:
1392         kfree(dev_dax->pgmap);
1393 err_pgmap:
1394         free_dev_dax_ranges(dev_dax);
1395 err_range:
1396         free_dev_dax_id(dev_dax);
1397 err_id:
1398         kfree(dev_dax);
1399
1400         return ERR_PTR(rc);
1401 }
1402 EXPORT_SYMBOL_GPL(devm_create_dev_dax);
1403
1404 static int match_always_count;
1405
1406 int __dax_driver_register(struct dax_device_driver *dax_drv,
1407                 struct module *module, const char *mod_name)
1408 {
1409         struct device_driver *drv = &dax_drv->drv;
1410         int rc = 0;
1411
1412         INIT_LIST_HEAD(&dax_drv->ids);
1413         drv->owner = module;
1414         drv->name = mod_name;
1415         drv->mod_name = mod_name;
1416         drv->bus = &dax_bus_type;
1417
1418         /* there can only be one default driver */
1419         mutex_lock(&dax_bus_lock);
1420         match_always_count += dax_drv->match_always;
1421         if (match_always_count > 1) {
1422                 match_always_count--;
1423                 WARN_ON(1);
1424                 rc = -EINVAL;
1425         }
1426         mutex_unlock(&dax_bus_lock);
1427         if (rc)
1428                 return rc;
1429         return driver_register(drv);
1430 }
1431 EXPORT_SYMBOL_GPL(__dax_driver_register);
1432
1433 void dax_driver_unregister(struct dax_device_driver *dax_drv)
1434 {
1435         struct device_driver *drv = &dax_drv->drv;
1436         struct dax_id *dax_id, *_id;
1437
1438         mutex_lock(&dax_bus_lock);
1439         match_always_count -= dax_drv->match_always;
1440         list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
1441                 list_del(&dax_id->list);
1442                 kfree(dax_id);
1443         }
1444         mutex_unlock(&dax_bus_lock);
1445         driver_unregister(drv);
1446 }
1447 EXPORT_SYMBOL_GPL(dax_driver_unregister);
1448
1449 int __init dax_bus_init(void)
1450 {
1451         int rc;
1452
1453         if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) {
1454                 dax_class = class_create(THIS_MODULE, "dax");
1455                 if (IS_ERR(dax_class))
1456                         return PTR_ERR(dax_class);
1457         }
1458
1459         rc = bus_register(&dax_bus_type);
1460         if (rc)
1461                 class_destroy(dax_class);
1462         return rc;
1463 }
1464
1465 void __exit dax_bus_exit(void)
1466 {
1467         bus_unregister(&dax_bus_type);
1468         class_destroy(dax_class);
1469 }