Merge branch 'work.poll2' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
[linux-2.6-microblaze.git] / drivers / nvdimm / region_devs.c
1 /*
2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of version 2 of the GNU General Public License as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful, but
9  * WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  */
13 #include <linux/scatterlist.h>
14 #include <linux/highmem.h>
15 #include <linux/sched.h>
16 #include <linux/slab.h>
17 #include <linux/hash.h>
18 #include <linux/sort.h>
19 #include <linux/io.h>
20 #include <linux/nd.h>
21 #include "nd-core.h"
22 #include "nd.h"
23
24 /*
25  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
26  * irrelevant.
27  */
28 #include <linux/io-64-nonatomic-hi-lo.h>
29
30 static DEFINE_IDA(region_ida);
31 static DEFINE_PER_CPU(int, flush_idx);
32
33 static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
34                 struct nd_region_data *ndrd)
35 {
36         int i, j;
37
38         dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm),
39                         nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es");
40         for (i = 0; i < (1 << ndrd->hints_shift); i++) {
41                 struct resource *res = &nvdimm->flush_wpq[i];
42                 unsigned long pfn = PHYS_PFN(res->start);
43                 void __iomem *flush_page;
44
45                 /* check if flush hints share a page */
46                 for (j = 0; j < i; j++) {
47                         struct resource *res_j = &nvdimm->flush_wpq[j];
48                         unsigned long pfn_j = PHYS_PFN(res_j->start);
49
50                         if (pfn == pfn_j)
51                                 break;
52                 }
53
54                 if (j < i)
55                         flush_page = (void __iomem *) ((unsigned long)
56                                         ndrd_get_flush_wpq(ndrd, dimm, j)
57                                         & PAGE_MASK);
58                 else
59                         flush_page = devm_nvdimm_ioremap(dev,
60                                         PFN_PHYS(pfn), PAGE_SIZE);
61                 if (!flush_page)
62                         return -ENXIO;
63                 ndrd_set_flush_wpq(ndrd, dimm, i, flush_page
64                                 + (res->start & ~PAGE_MASK));
65         }
66
67         return 0;
68 }
69
70 int nd_region_activate(struct nd_region *nd_region)
71 {
72         int i, j, num_flush = 0;
73         struct nd_region_data *ndrd;
74         struct device *dev = &nd_region->dev;
75         size_t flush_data_size = sizeof(void *);
76
77         nvdimm_bus_lock(&nd_region->dev);
78         for (i = 0; i < nd_region->ndr_mappings; i++) {
79                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
80                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
81
82                 /* at least one null hint slot per-dimm for the "no-hint" case */
83                 flush_data_size += sizeof(void *);
84                 num_flush = min_not_zero(num_flush, nvdimm->num_flush);
85                 if (!nvdimm->num_flush)
86                         continue;
87                 flush_data_size += nvdimm->num_flush * sizeof(void *);
88         }
89         nvdimm_bus_unlock(&nd_region->dev);
90
91         ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL);
92         if (!ndrd)
93                 return -ENOMEM;
94         dev_set_drvdata(dev, ndrd);
95
96         if (!num_flush)
97                 return 0;
98
99         ndrd->hints_shift = ilog2(num_flush);
100         for (i = 0; i < nd_region->ndr_mappings; i++) {
101                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
102                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
103                 int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd);
104
105                 if (rc)
106                         return rc;
107         }
108
109         /*
110          * Clear out entries that are duplicates. This should prevent the
111          * extra flushings.
112          */
113         for (i = 0; i < nd_region->ndr_mappings - 1; i++) {
114                 /* ignore if NULL already */
115                 if (!ndrd_get_flush_wpq(ndrd, i, 0))
116                         continue;
117
118                 for (j = i + 1; j < nd_region->ndr_mappings; j++)
119                         if (ndrd_get_flush_wpq(ndrd, i, 0) ==
120                             ndrd_get_flush_wpq(ndrd, j, 0))
121                                 ndrd_set_flush_wpq(ndrd, j, 0, NULL);
122         }
123
124         return 0;
125 }
126
127 static void nd_region_release(struct device *dev)
128 {
129         struct nd_region *nd_region = to_nd_region(dev);
130         u16 i;
131
132         for (i = 0; i < nd_region->ndr_mappings; i++) {
133                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
134                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
135
136                 put_device(&nvdimm->dev);
137         }
138         free_percpu(nd_region->lane);
139         ida_simple_remove(&region_ida, nd_region->id);
140         if (is_nd_blk(dev))
141                 kfree(to_nd_blk_region(dev));
142         else
143                 kfree(nd_region);
144 }
145
146 static struct device_type nd_blk_device_type = {
147         .name = "nd_blk",
148         .release = nd_region_release,
149 };
150
151 static struct device_type nd_pmem_device_type = {
152         .name = "nd_pmem",
153         .release = nd_region_release,
154 };
155
156 static struct device_type nd_volatile_device_type = {
157         .name = "nd_volatile",
158         .release = nd_region_release,
159 };
160
161 bool is_nd_pmem(struct device *dev)
162 {
163         return dev ? dev->type == &nd_pmem_device_type : false;
164 }
165
166 bool is_nd_blk(struct device *dev)
167 {
168         return dev ? dev->type == &nd_blk_device_type : false;
169 }
170
171 bool is_nd_volatile(struct device *dev)
172 {
173         return dev ? dev->type == &nd_volatile_device_type : false;
174 }
175
176 struct nd_region *to_nd_region(struct device *dev)
177 {
178         struct nd_region *nd_region = container_of(dev, struct nd_region, dev);
179
180         WARN_ON(dev->type->release != nd_region_release);
181         return nd_region;
182 }
183 EXPORT_SYMBOL_GPL(to_nd_region);
184
185 struct nd_blk_region *to_nd_blk_region(struct device *dev)
186 {
187         struct nd_region *nd_region = to_nd_region(dev);
188
189         WARN_ON(!is_nd_blk(dev));
190         return container_of(nd_region, struct nd_blk_region, nd_region);
191 }
192 EXPORT_SYMBOL_GPL(to_nd_blk_region);
193
194 void *nd_region_provider_data(struct nd_region *nd_region)
195 {
196         return nd_region->provider_data;
197 }
198 EXPORT_SYMBOL_GPL(nd_region_provider_data);
199
200 void *nd_blk_region_provider_data(struct nd_blk_region *ndbr)
201 {
202         return ndbr->blk_provider_data;
203 }
204 EXPORT_SYMBOL_GPL(nd_blk_region_provider_data);
205
206 void nd_blk_region_set_provider_data(struct nd_blk_region *ndbr, void *data)
207 {
208         ndbr->blk_provider_data = data;
209 }
210 EXPORT_SYMBOL_GPL(nd_blk_region_set_provider_data);
211
212 /**
213  * nd_region_to_nstype() - region to an integer namespace type
214  * @nd_region: region-device to interrogate
215  *
216  * This is the 'nstype' attribute of a region as well, an input to the
217  * MODALIAS for namespace devices, and bit number for a nvdimm_bus to match
218  * namespace devices with namespace drivers.
219  */
220 int nd_region_to_nstype(struct nd_region *nd_region)
221 {
222         if (is_memory(&nd_region->dev)) {
223                 u16 i, alias;
224
225                 for (i = 0, alias = 0; i < nd_region->ndr_mappings; i++) {
226                         struct nd_mapping *nd_mapping = &nd_region->mapping[i];
227                         struct nvdimm *nvdimm = nd_mapping->nvdimm;
228
229                         if (test_bit(NDD_ALIASING, &nvdimm->flags))
230                                 alias++;
231                 }
232                 if (alias)
233                         return ND_DEVICE_NAMESPACE_PMEM;
234                 else
235                         return ND_DEVICE_NAMESPACE_IO;
236         } else if (is_nd_blk(&nd_region->dev)) {
237                 return ND_DEVICE_NAMESPACE_BLK;
238         }
239
240         return 0;
241 }
242 EXPORT_SYMBOL(nd_region_to_nstype);
243
244 static ssize_t size_show(struct device *dev,
245                 struct device_attribute *attr, char *buf)
246 {
247         struct nd_region *nd_region = to_nd_region(dev);
248         unsigned long long size = 0;
249
250         if (is_memory(dev)) {
251                 size = nd_region->ndr_size;
252         } else if (nd_region->ndr_mappings == 1) {
253                 struct nd_mapping *nd_mapping = &nd_region->mapping[0];
254
255                 size = nd_mapping->size;
256         }
257
258         return sprintf(buf, "%llu\n", size);
259 }
260 static DEVICE_ATTR_RO(size);
261
262 static ssize_t deep_flush_show(struct device *dev,
263                 struct device_attribute *attr, char *buf)
264 {
265         struct nd_region *nd_region = to_nd_region(dev);
266
267         /*
268          * NOTE: in the nvdimm_has_flush() error case this attribute is
269          * not visible.
270          */
271         return sprintf(buf, "%d\n", nvdimm_has_flush(nd_region));
272 }
273
274 static ssize_t deep_flush_store(struct device *dev, struct device_attribute *attr,
275                 const char *buf, size_t len)
276 {
277         bool flush;
278         int rc = strtobool(buf, &flush);
279         struct nd_region *nd_region = to_nd_region(dev);
280
281         if (rc)
282                 return rc;
283         if (!flush)
284                 return -EINVAL;
285         nvdimm_flush(nd_region);
286
287         return len;
288 }
289 static DEVICE_ATTR_RW(deep_flush);
290
291 static ssize_t mappings_show(struct device *dev,
292                 struct device_attribute *attr, char *buf)
293 {
294         struct nd_region *nd_region = to_nd_region(dev);
295
296         return sprintf(buf, "%d\n", nd_region->ndr_mappings);
297 }
298 static DEVICE_ATTR_RO(mappings);
299
300 static ssize_t nstype_show(struct device *dev,
301                 struct device_attribute *attr, char *buf)
302 {
303         struct nd_region *nd_region = to_nd_region(dev);
304
305         return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region));
306 }
307 static DEVICE_ATTR_RO(nstype);
308
309 static ssize_t set_cookie_show(struct device *dev,
310                 struct device_attribute *attr, char *buf)
311 {
312         struct nd_region *nd_region = to_nd_region(dev);
313         struct nd_interleave_set *nd_set = nd_region->nd_set;
314         ssize_t rc = 0;
315
316         if (is_memory(dev) && nd_set)
317                 /* pass, should be precluded by region_visible */;
318         else
319                 return -ENXIO;
320
321         /*
322          * The cookie to show depends on which specification of the
323          * labels we are using. If there are not labels then default to
324          * the v1.1 namespace label cookie definition. To read all this
325          * data we need to wait for probing to settle.
326          */
327         device_lock(dev);
328         nvdimm_bus_lock(dev);
329         wait_nvdimm_bus_probe_idle(dev);
330         if (nd_region->ndr_mappings) {
331                 struct nd_mapping *nd_mapping = &nd_region->mapping[0];
332                 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
333
334                 if (ndd) {
335                         struct nd_namespace_index *nsindex;
336
337                         nsindex = to_namespace_index(ndd, ndd->ns_current);
338                         rc = sprintf(buf, "%#llx\n",
339                                         nd_region_interleave_set_cookie(nd_region,
340                                                 nsindex));
341                 }
342         }
343         nvdimm_bus_unlock(dev);
344         device_unlock(dev);
345
346         if (rc)
347                 return rc;
348         return sprintf(buf, "%#llx\n", nd_set->cookie1);
349 }
350 static DEVICE_ATTR_RO(set_cookie);
351
352 resource_size_t nd_region_available_dpa(struct nd_region *nd_region)
353 {
354         resource_size_t blk_max_overlap = 0, available, overlap;
355         int i;
356
357         WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));
358
359  retry:
360         available = 0;
361         overlap = blk_max_overlap;
362         for (i = 0; i < nd_region->ndr_mappings; i++) {
363                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
364                 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
365
366                 /* if a dimm is disabled the available capacity is zero */
367                 if (!ndd)
368                         return 0;
369
370                 if (is_memory(&nd_region->dev)) {
371                         available += nd_pmem_available_dpa(nd_region,
372                                         nd_mapping, &overlap);
373                         if (overlap > blk_max_overlap) {
374                                 blk_max_overlap = overlap;
375                                 goto retry;
376                         }
377                 } else if (is_nd_blk(&nd_region->dev))
378                         available += nd_blk_available_dpa(nd_region);
379         }
380
381         return available;
382 }
383
384 static ssize_t available_size_show(struct device *dev,
385                 struct device_attribute *attr, char *buf)
386 {
387         struct nd_region *nd_region = to_nd_region(dev);
388         unsigned long long available = 0;
389
390         /*
391          * Flush in-flight updates and grab a snapshot of the available
392          * size.  Of course, this value is potentially invalidated the
393          * memory nvdimm_bus_lock() is dropped, but that's userspace's
394          * problem to not race itself.
395          */
396         nvdimm_bus_lock(dev);
397         wait_nvdimm_bus_probe_idle(dev);
398         available = nd_region_available_dpa(nd_region);
399         nvdimm_bus_unlock(dev);
400
401         return sprintf(buf, "%llu\n", available);
402 }
403 static DEVICE_ATTR_RO(available_size);
404
405 static ssize_t init_namespaces_show(struct device *dev,
406                 struct device_attribute *attr, char *buf)
407 {
408         struct nd_region_data *ndrd = dev_get_drvdata(dev);
409         ssize_t rc;
410
411         nvdimm_bus_lock(dev);
412         if (ndrd)
413                 rc = sprintf(buf, "%d/%d\n", ndrd->ns_active, ndrd->ns_count);
414         else
415                 rc = -ENXIO;
416         nvdimm_bus_unlock(dev);
417
418         return rc;
419 }
420 static DEVICE_ATTR_RO(init_namespaces);
421
422 static ssize_t namespace_seed_show(struct device *dev,
423                 struct device_attribute *attr, char *buf)
424 {
425         struct nd_region *nd_region = to_nd_region(dev);
426         ssize_t rc;
427
428         nvdimm_bus_lock(dev);
429         if (nd_region->ns_seed)
430                 rc = sprintf(buf, "%s\n", dev_name(nd_region->ns_seed));
431         else
432                 rc = sprintf(buf, "\n");
433         nvdimm_bus_unlock(dev);
434         return rc;
435 }
436 static DEVICE_ATTR_RO(namespace_seed);
437
438 static ssize_t btt_seed_show(struct device *dev,
439                 struct device_attribute *attr, char *buf)
440 {
441         struct nd_region *nd_region = to_nd_region(dev);
442         ssize_t rc;
443
444         nvdimm_bus_lock(dev);
445         if (nd_region->btt_seed)
446                 rc = sprintf(buf, "%s\n", dev_name(nd_region->btt_seed));
447         else
448                 rc = sprintf(buf, "\n");
449         nvdimm_bus_unlock(dev);
450
451         return rc;
452 }
453 static DEVICE_ATTR_RO(btt_seed);
454
455 static ssize_t pfn_seed_show(struct device *dev,
456                 struct device_attribute *attr, char *buf)
457 {
458         struct nd_region *nd_region = to_nd_region(dev);
459         ssize_t rc;
460
461         nvdimm_bus_lock(dev);
462         if (nd_region->pfn_seed)
463                 rc = sprintf(buf, "%s\n", dev_name(nd_region->pfn_seed));
464         else
465                 rc = sprintf(buf, "\n");
466         nvdimm_bus_unlock(dev);
467
468         return rc;
469 }
470 static DEVICE_ATTR_RO(pfn_seed);
471
472 static ssize_t dax_seed_show(struct device *dev,
473                 struct device_attribute *attr, char *buf)
474 {
475         struct nd_region *nd_region = to_nd_region(dev);
476         ssize_t rc;
477
478         nvdimm_bus_lock(dev);
479         if (nd_region->dax_seed)
480                 rc = sprintf(buf, "%s\n", dev_name(nd_region->dax_seed));
481         else
482                 rc = sprintf(buf, "\n");
483         nvdimm_bus_unlock(dev);
484
485         return rc;
486 }
487 static DEVICE_ATTR_RO(dax_seed);
488
489 static ssize_t read_only_show(struct device *dev,
490                 struct device_attribute *attr, char *buf)
491 {
492         struct nd_region *nd_region = to_nd_region(dev);
493
494         return sprintf(buf, "%d\n", nd_region->ro);
495 }
496
497 static ssize_t read_only_store(struct device *dev,
498                 struct device_attribute *attr, const char *buf, size_t len)
499 {
500         bool ro;
501         int rc = strtobool(buf, &ro);
502         struct nd_region *nd_region = to_nd_region(dev);
503
504         if (rc)
505                 return rc;
506
507         nd_region->ro = ro;
508         return len;
509 }
510 static DEVICE_ATTR_RW(read_only);
511
512 static ssize_t region_badblocks_show(struct device *dev,
513                 struct device_attribute *attr, char *buf)
514 {
515         struct nd_region *nd_region = to_nd_region(dev);
516
517         return badblocks_show(&nd_region->bb, buf, 0);
518 }
519
520 static DEVICE_ATTR(badblocks, 0444, region_badblocks_show, NULL);
521
522 static ssize_t resource_show(struct device *dev,
523                 struct device_attribute *attr, char *buf)
524 {
525         struct nd_region *nd_region = to_nd_region(dev);
526
527         return sprintf(buf, "%#llx\n", nd_region->ndr_start);
528 }
529 static DEVICE_ATTR_RO(resource);
530
531 static ssize_t persistence_domain_show(struct device *dev,
532                 struct device_attribute *attr, char *buf)
533 {
534         struct nd_region *nd_region = to_nd_region(dev);
535         unsigned long flags = nd_region->flags;
536
537         return sprintf(buf, "%s%s\n",
538                         flags & BIT(ND_REGION_PERSIST_CACHE) ? "cpu_cache " : "",
539                         flags & BIT(ND_REGION_PERSIST_MEMCTRL) ? "memory_controller " : "");
540 }
541 static DEVICE_ATTR_RO(persistence_domain);
542
543 static struct attribute *nd_region_attributes[] = {
544         &dev_attr_size.attr,
545         &dev_attr_nstype.attr,
546         &dev_attr_mappings.attr,
547         &dev_attr_btt_seed.attr,
548         &dev_attr_pfn_seed.attr,
549         &dev_attr_dax_seed.attr,
550         &dev_attr_deep_flush.attr,
551         &dev_attr_read_only.attr,
552         &dev_attr_set_cookie.attr,
553         &dev_attr_available_size.attr,
554         &dev_attr_namespace_seed.attr,
555         &dev_attr_init_namespaces.attr,
556         &dev_attr_badblocks.attr,
557         &dev_attr_resource.attr,
558         &dev_attr_persistence_domain.attr,
559         NULL,
560 };
561
562 static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n)
563 {
564         struct device *dev = container_of(kobj, typeof(*dev), kobj);
565         struct nd_region *nd_region = to_nd_region(dev);
566         struct nd_interleave_set *nd_set = nd_region->nd_set;
567         int type = nd_region_to_nstype(nd_region);
568
569         if (!is_memory(dev) && a == &dev_attr_pfn_seed.attr)
570                 return 0;
571
572         if (!is_memory(dev) && a == &dev_attr_dax_seed.attr)
573                 return 0;
574
575         if (!is_nd_pmem(dev) && a == &dev_attr_badblocks.attr)
576                 return 0;
577
578         if (a == &dev_attr_resource.attr) {
579                 if (is_nd_pmem(dev))
580                         return 0400;
581                 else
582                         return 0;
583         }
584
585         if (a == &dev_attr_deep_flush.attr) {
586                 int has_flush = nvdimm_has_flush(nd_region);
587
588                 if (has_flush == 1)
589                         return a->mode;
590                 else if (has_flush == 0)
591                         return 0444;
592                 else
593                         return 0;
594         }
595
596         if (a != &dev_attr_set_cookie.attr
597                         && a != &dev_attr_available_size.attr)
598                 return a->mode;
599
600         if ((type == ND_DEVICE_NAMESPACE_PMEM
601                                 || type == ND_DEVICE_NAMESPACE_BLK)
602                         && a == &dev_attr_available_size.attr)
603                 return a->mode;
604         else if (is_memory(dev) && nd_set)
605                 return a->mode;
606
607         return 0;
608 }
609
610 struct attribute_group nd_region_attribute_group = {
611         .attrs = nd_region_attributes,
612         .is_visible = region_visible,
613 };
614 EXPORT_SYMBOL_GPL(nd_region_attribute_group);
615
616 u64 nd_region_interleave_set_cookie(struct nd_region *nd_region,
617                 struct nd_namespace_index *nsindex)
618 {
619         struct nd_interleave_set *nd_set = nd_region->nd_set;
620
621         if (!nd_set)
622                 return 0;
623
624         if (nsindex && __le16_to_cpu(nsindex->major) == 1
625                         && __le16_to_cpu(nsindex->minor) == 1)
626                 return nd_set->cookie1;
627         return nd_set->cookie2;
628 }
629
630 u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region)
631 {
632         struct nd_interleave_set *nd_set = nd_region->nd_set;
633
634         if (nd_set)
635                 return nd_set->altcookie;
636         return 0;
637 }
638
639 void nd_mapping_free_labels(struct nd_mapping *nd_mapping)
640 {
641         struct nd_label_ent *label_ent, *e;
642
643         lockdep_assert_held(&nd_mapping->lock);
644         list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) {
645                 list_del(&label_ent->list);
646                 kfree(label_ent);
647         }
648 }
649
650 /*
651  * Upon successful probe/remove, take/release a reference on the
652  * associated interleave set (if present), and plant new btt + namespace
653  * seeds.  Also, on the removal of a BLK region, notify the provider to
654  * disable the region.
655  */
656 static void nd_region_notify_driver_action(struct nvdimm_bus *nvdimm_bus,
657                 struct device *dev, bool probe)
658 {
659         struct nd_region *nd_region;
660
661         if (!probe && is_nd_region(dev)) {
662                 int i;
663
664                 nd_region = to_nd_region(dev);
665                 for (i = 0; i < nd_region->ndr_mappings; i++) {
666                         struct nd_mapping *nd_mapping = &nd_region->mapping[i];
667                         struct nvdimm_drvdata *ndd = nd_mapping->ndd;
668                         struct nvdimm *nvdimm = nd_mapping->nvdimm;
669
670                         mutex_lock(&nd_mapping->lock);
671                         nd_mapping_free_labels(nd_mapping);
672                         mutex_unlock(&nd_mapping->lock);
673
674                         put_ndd(ndd);
675                         nd_mapping->ndd = NULL;
676                         if (ndd)
677                                 atomic_dec(&nvdimm->busy);
678                 }
679         }
680         if (dev->parent && is_nd_region(dev->parent) && probe) {
681                 nd_region = to_nd_region(dev->parent);
682                 nvdimm_bus_lock(dev);
683                 if (nd_region->ns_seed == dev)
684                         nd_region_create_ns_seed(nd_region);
685                 nvdimm_bus_unlock(dev);
686         }
687         if (is_nd_btt(dev) && probe) {
688                 struct nd_btt *nd_btt = to_nd_btt(dev);
689
690                 nd_region = to_nd_region(dev->parent);
691                 nvdimm_bus_lock(dev);
692                 if (nd_region->btt_seed == dev)
693                         nd_region_create_btt_seed(nd_region);
694                 if (nd_region->ns_seed == &nd_btt->ndns->dev)
695                         nd_region_create_ns_seed(nd_region);
696                 nvdimm_bus_unlock(dev);
697         }
698         if (is_nd_pfn(dev) && probe) {
699                 struct nd_pfn *nd_pfn = to_nd_pfn(dev);
700
701                 nd_region = to_nd_region(dev->parent);
702                 nvdimm_bus_lock(dev);
703                 if (nd_region->pfn_seed == dev)
704                         nd_region_create_pfn_seed(nd_region);
705                 if (nd_region->ns_seed == &nd_pfn->ndns->dev)
706                         nd_region_create_ns_seed(nd_region);
707                 nvdimm_bus_unlock(dev);
708         }
709         if (is_nd_dax(dev) && probe) {
710                 struct nd_dax *nd_dax = to_nd_dax(dev);
711
712                 nd_region = to_nd_region(dev->parent);
713                 nvdimm_bus_lock(dev);
714                 if (nd_region->dax_seed == dev)
715                         nd_region_create_dax_seed(nd_region);
716                 if (nd_region->ns_seed == &nd_dax->nd_pfn.ndns->dev)
717                         nd_region_create_ns_seed(nd_region);
718                 nvdimm_bus_unlock(dev);
719         }
720 }
721
722 void nd_region_probe_success(struct nvdimm_bus *nvdimm_bus, struct device *dev)
723 {
724         nd_region_notify_driver_action(nvdimm_bus, dev, true);
725 }
726
727 void nd_region_disable(struct nvdimm_bus *nvdimm_bus, struct device *dev)
728 {
729         nd_region_notify_driver_action(nvdimm_bus, dev, false);
730 }
731
732 static ssize_t mappingN(struct device *dev, char *buf, int n)
733 {
734         struct nd_region *nd_region = to_nd_region(dev);
735         struct nd_mapping *nd_mapping;
736         struct nvdimm *nvdimm;
737
738         if (n >= nd_region->ndr_mappings)
739                 return -ENXIO;
740         nd_mapping = &nd_region->mapping[n];
741         nvdimm = nd_mapping->nvdimm;
742
743         return sprintf(buf, "%s,%llu,%llu,%d\n", dev_name(&nvdimm->dev),
744                         nd_mapping->start, nd_mapping->size,
745                         nd_mapping->position);
746 }
747
748 #define REGION_MAPPING(idx) \
749 static ssize_t mapping##idx##_show(struct device *dev,          \
750                 struct device_attribute *attr, char *buf)       \
751 {                                                               \
752         return mappingN(dev, buf, idx);                         \
753 }                                                               \
754 static DEVICE_ATTR_RO(mapping##idx)
755
756 /*
757  * 32 should be enough for a while, even in the presence of socket
758  * interleave a 32-way interleave set is a degenerate case.
759  */
760 REGION_MAPPING(0);
761 REGION_MAPPING(1);
762 REGION_MAPPING(2);
763 REGION_MAPPING(3);
764 REGION_MAPPING(4);
765 REGION_MAPPING(5);
766 REGION_MAPPING(6);
767 REGION_MAPPING(7);
768 REGION_MAPPING(8);
769 REGION_MAPPING(9);
770 REGION_MAPPING(10);
771 REGION_MAPPING(11);
772 REGION_MAPPING(12);
773 REGION_MAPPING(13);
774 REGION_MAPPING(14);
775 REGION_MAPPING(15);
776 REGION_MAPPING(16);
777 REGION_MAPPING(17);
778 REGION_MAPPING(18);
779 REGION_MAPPING(19);
780 REGION_MAPPING(20);
781 REGION_MAPPING(21);
782 REGION_MAPPING(22);
783 REGION_MAPPING(23);
784 REGION_MAPPING(24);
785 REGION_MAPPING(25);
786 REGION_MAPPING(26);
787 REGION_MAPPING(27);
788 REGION_MAPPING(28);
789 REGION_MAPPING(29);
790 REGION_MAPPING(30);
791 REGION_MAPPING(31);
792
793 static umode_t mapping_visible(struct kobject *kobj, struct attribute *a, int n)
794 {
795         struct device *dev = container_of(kobj, struct device, kobj);
796         struct nd_region *nd_region = to_nd_region(dev);
797
798         if (n < nd_region->ndr_mappings)
799                 return a->mode;
800         return 0;
801 }
802
803 static struct attribute *mapping_attributes[] = {
804         &dev_attr_mapping0.attr,
805         &dev_attr_mapping1.attr,
806         &dev_attr_mapping2.attr,
807         &dev_attr_mapping3.attr,
808         &dev_attr_mapping4.attr,
809         &dev_attr_mapping5.attr,
810         &dev_attr_mapping6.attr,
811         &dev_attr_mapping7.attr,
812         &dev_attr_mapping8.attr,
813         &dev_attr_mapping9.attr,
814         &dev_attr_mapping10.attr,
815         &dev_attr_mapping11.attr,
816         &dev_attr_mapping12.attr,
817         &dev_attr_mapping13.attr,
818         &dev_attr_mapping14.attr,
819         &dev_attr_mapping15.attr,
820         &dev_attr_mapping16.attr,
821         &dev_attr_mapping17.attr,
822         &dev_attr_mapping18.attr,
823         &dev_attr_mapping19.attr,
824         &dev_attr_mapping20.attr,
825         &dev_attr_mapping21.attr,
826         &dev_attr_mapping22.attr,
827         &dev_attr_mapping23.attr,
828         &dev_attr_mapping24.attr,
829         &dev_attr_mapping25.attr,
830         &dev_attr_mapping26.attr,
831         &dev_attr_mapping27.attr,
832         &dev_attr_mapping28.attr,
833         &dev_attr_mapping29.attr,
834         &dev_attr_mapping30.attr,
835         &dev_attr_mapping31.attr,
836         NULL,
837 };
838
839 struct attribute_group nd_mapping_attribute_group = {
840         .is_visible = mapping_visible,
841         .attrs = mapping_attributes,
842 };
843 EXPORT_SYMBOL_GPL(nd_mapping_attribute_group);
844
845 int nd_blk_region_init(struct nd_region *nd_region)
846 {
847         struct device *dev = &nd_region->dev;
848         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
849
850         if (!is_nd_blk(dev))
851                 return 0;
852
853         if (nd_region->ndr_mappings < 1) {
854                 dev_dbg(dev, "invalid BLK region\n");
855                 return -ENXIO;
856         }
857
858         return to_nd_blk_region(dev)->enable(nvdimm_bus, dev);
859 }
860
861 /**
862  * nd_region_acquire_lane - allocate and lock a lane
863  * @nd_region: region id and number of lanes possible
864  *
865  * A lane correlates to a BLK-data-window and/or a log slot in the BTT.
866  * We optimize for the common case where there are 256 lanes, one
867  * per-cpu.  For larger systems we need to lock to share lanes.  For now
868  * this implementation assumes the cost of maintaining an allocator for
869  * free lanes is on the order of the lock hold time, so it implements a
870  * static lane = cpu % num_lanes mapping.
871  *
872  * In the case of a BTT instance on top of a BLK namespace a lane may be
873  * acquired recursively.  We lock on the first instance.
874  *
875  * In the case of a BTT instance on top of PMEM, we only acquire a lane
876  * for the BTT metadata updates.
877  */
878 unsigned int nd_region_acquire_lane(struct nd_region *nd_region)
879 {
880         unsigned int cpu, lane;
881
882         cpu = get_cpu();
883         if (nd_region->num_lanes < nr_cpu_ids) {
884                 struct nd_percpu_lane *ndl_lock, *ndl_count;
885
886                 lane = cpu % nd_region->num_lanes;
887                 ndl_count = per_cpu_ptr(nd_region->lane, cpu);
888                 ndl_lock = per_cpu_ptr(nd_region->lane, lane);
889                 if (ndl_count->count++ == 0)
890                         spin_lock(&ndl_lock->lock);
891         } else
892                 lane = cpu;
893
894         return lane;
895 }
896 EXPORT_SYMBOL(nd_region_acquire_lane);
897
898 void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane)
899 {
900         if (nd_region->num_lanes < nr_cpu_ids) {
901                 unsigned int cpu = get_cpu();
902                 struct nd_percpu_lane *ndl_lock, *ndl_count;
903
904                 ndl_count = per_cpu_ptr(nd_region->lane, cpu);
905                 ndl_lock = per_cpu_ptr(nd_region->lane, lane);
906                 if (--ndl_count->count == 0)
907                         spin_unlock(&ndl_lock->lock);
908                 put_cpu();
909         }
910         put_cpu();
911 }
912 EXPORT_SYMBOL(nd_region_release_lane);
913
914 static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
915                 struct nd_region_desc *ndr_desc, struct device_type *dev_type,
916                 const char *caller)
917 {
918         struct nd_region *nd_region;
919         struct device *dev;
920         void *region_buf;
921         unsigned int i;
922         int ro = 0;
923
924         for (i = 0; i < ndr_desc->num_mappings; i++) {
925                 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
926                 struct nvdimm *nvdimm = mapping->nvdimm;
927
928                 if ((mapping->start | mapping->size) % SZ_4K) {
929                         dev_err(&nvdimm_bus->dev, "%s: %s mapping%d is not 4K aligned\n",
930                                         caller, dev_name(&nvdimm->dev), i);
931
932                         return NULL;
933                 }
934
935                 if (test_bit(NDD_UNARMED, &nvdimm->flags))
936                         ro = 1;
937         }
938
939         if (dev_type == &nd_blk_device_type) {
940                 struct nd_blk_region_desc *ndbr_desc;
941                 struct nd_blk_region *ndbr;
942
943                 ndbr_desc = to_blk_region_desc(ndr_desc);
944                 ndbr = kzalloc(sizeof(*ndbr) + sizeof(struct nd_mapping)
945                                 * ndr_desc->num_mappings,
946                                 GFP_KERNEL);
947                 if (ndbr) {
948                         nd_region = &ndbr->nd_region;
949                         ndbr->enable = ndbr_desc->enable;
950                         ndbr->do_io = ndbr_desc->do_io;
951                 }
952                 region_buf = ndbr;
953         } else {
954                 nd_region = kzalloc(sizeof(struct nd_region)
955                                 + sizeof(struct nd_mapping)
956                                 * ndr_desc->num_mappings,
957                                 GFP_KERNEL);
958                 region_buf = nd_region;
959         }
960
961         if (!region_buf)
962                 return NULL;
963         nd_region->id = ida_simple_get(&region_ida, 0, 0, GFP_KERNEL);
964         if (nd_region->id < 0)
965                 goto err_id;
966
967         nd_region->lane = alloc_percpu(struct nd_percpu_lane);
968         if (!nd_region->lane)
969                 goto err_percpu;
970
971         for (i = 0; i < nr_cpu_ids; i++) {
972                 struct nd_percpu_lane *ndl;
973
974                 ndl = per_cpu_ptr(nd_region->lane, i);
975                 spin_lock_init(&ndl->lock);
976                 ndl->count = 0;
977         }
978
979         for (i = 0; i < ndr_desc->num_mappings; i++) {
980                 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i];
981                 struct nvdimm *nvdimm = mapping->nvdimm;
982
983                 nd_region->mapping[i].nvdimm = nvdimm;
984                 nd_region->mapping[i].start = mapping->start;
985                 nd_region->mapping[i].size = mapping->size;
986                 nd_region->mapping[i].position = mapping->position;
987                 INIT_LIST_HEAD(&nd_region->mapping[i].labels);
988                 mutex_init(&nd_region->mapping[i].lock);
989
990                 get_device(&nvdimm->dev);
991         }
992         nd_region->ndr_mappings = ndr_desc->num_mappings;
993         nd_region->provider_data = ndr_desc->provider_data;
994         nd_region->nd_set = ndr_desc->nd_set;
995         nd_region->num_lanes = ndr_desc->num_lanes;
996         nd_region->flags = ndr_desc->flags;
997         nd_region->ro = ro;
998         nd_region->numa_node = ndr_desc->numa_node;
999         ida_init(&nd_region->ns_ida);
1000         ida_init(&nd_region->btt_ida);
1001         ida_init(&nd_region->pfn_ida);
1002         ida_init(&nd_region->dax_ida);
1003         dev = &nd_region->dev;
1004         dev_set_name(dev, "region%d", nd_region->id);
1005         dev->parent = &nvdimm_bus->dev;
1006         dev->type = dev_type;
1007         dev->groups = ndr_desc->attr_groups;
1008         nd_region->ndr_size = resource_size(ndr_desc->res);
1009         nd_region->ndr_start = ndr_desc->res->start;
1010         nd_device_register(dev);
1011
1012         return nd_region;
1013
1014  err_percpu:
1015         ida_simple_remove(&region_ida, nd_region->id);
1016  err_id:
1017         kfree(region_buf);
1018         return NULL;
1019 }
1020
1021 struct nd_region *nvdimm_pmem_region_create(struct nvdimm_bus *nvdimm_bus,
1022                 struct nd_region_desc *ndr_desc)
1023 {
1024         ndr_desc->num_lanes = ND_MAX_LANES;
1025         return nd_region_create(nvdimm_bus, ndr_desc, &nd_pmem_device_type,
1026                         __func__);
1027 }
1028 EXPORT_SYMBOL_GPL(nvdimm_pmem_region_create);
1029
1030 struct nd_region *nvdimm_blk_region_create(struct nvdimm_bus *nvdimm_bus,
1031                 struct nd_region_desc *ndr_desc)
1032 {
1033         if (ndr_desc->num_mappings > 1)
1034                 return NULL;
1035         ndr_desc->num_lanes = min(ndr_desc->num_lanes, ND_MAX_LANES);
1036         return nd_region_create(nvdimm_bus, ndr_desc, &nd_blk_device_type,
1037                         __func__);
1038 }
1039 EXPORT_SYMBOL_GPL(nvdimm_blk_region_create);
1040
1041 struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus,
1042                 struct nd_region_desc *ndr_desc)
1043 {
1044         ndr_desc->num_lanes = ND_MAX_LANES;
1045         return nd_region_create(nvdimm_bus, ndr_desc, &nd_volatile_device_type,
1046                         __func__);
1047 }
1048 EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
1049
1050 /**
1051  * nvdimm_flush - flush any posted write queues between the cpu and pmem media
1052  * @nd_region: blk or interleaved pmem region
1053  */
1054 void nvdimm_flush(struct nd_region *nd_region)
1055 {
1056         struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
1057         int i, idx;
1058
1059         /*
1060          * Try to encourage some diversity in flush hint addresses
1061          * across cpus assuming a limited number of flush hints.
1062          */
1063         idx = this_cpu_read(flush_idx);
1064         idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8));
1065
1066         /*
1067          * The first wmb() is needed to 'sfence' all previous writes
1068          * such that they are architecturally visible for the platform
1069          * buffer flush.  Note that we've already arranged for pmem
1070          * writes to avoid the cache via memcpy_flushcache().  The final
1071          * wmb() ensures ordering for the NVDIMM flush write.
1072          */
1073         wmb();
1074         for (i = 0; i < nd_region->ndr_mappings; i++)
1075                 if (ndrd_get_flush_wpq(ndrd, i, 0))
1076                         writeq(1, ndrd_get_flush_wpq(ndrd, i, idx));
1077         wmb();
1078 }
1079 EXPORT_SYMBOL_GPL(nvdimm_flush);
1080
1081 /**
1082  * nvdimm_has_flush - determine write flushing requirements
1083  * @nd_region: blk or interleaved pmem region
1084  *
1085  * Returns 1 if writes require flushing
1086  * Returns 0 if writes do not require flushing
1087  * Returns -ENXIO if flushing capability can not be determined
1088  */
1089 int nvdimm_has_flush(struct nd_region *nd_region)
1090 {
1091         int i;
1092
1093         /* no nvdimm or pmem api == flushing capability unknown */
1094         if (nd_region->ndr_mappings == 0
1095                         || !IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API))
1096                 return -ENXIO;
1097
1098         for (i = 0; i < nd_region->ndr_mappings; i++) {
1099                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
1100                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
1101
1102                 /* flush hints present / available */
1103                 if (nvdimm->num_flush)
1104                         return 1;
1105         }
1106
1107         /*
1108          * The platform defines dimm devices without hints, assume
1109          * platform persistence mechanism like ADR
1110          */
1111         return 0;
1112 }
1113 EXPORT_SYMBOL_GPL(nvdimm_has_flush);
1114
1115 int nvdimm_has_cache(struct nd_region *nd_region)
1116 {
1117         return is_nd_pmem(&nd_region->dev);
1118 }
1119 EXPORT_SYMBOL_GPL(nvdimm_has_cache);
1120
1121 void __exit nd_region_devs_exit(void)
1122 {
1123         ida_destroy(&region_ida);
1124 }