drivers/s390/block/dcssblk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * dcssblk.c -- the S/390 block driver for dcss memory
   4  *
   5  * Authors: Carsten Otte, Stefan Weinhuber, Gerald Schaefer
   6  */
   7
   8 #define KMSG_COMPONENT "dcssblk"
   9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/module.h>
  12 #include <linux/moduleparam.h>
  13 #include <linux/ctype.h>
  14 #include <linux/errno.h>
  15 #include <linux/init.h>
  16 #include <linux/slab.h>
  17 #include <linux/blkdev.h>
  18 #include <linux/completion.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/pfn_t.h>
  22 #include <linux/uio.h>
  23 #include <linux/dax.h>
  24 #include <asm/extmem.h>
  25 #include <asm/io.h>
  26
  27 #define DCSSBLK_NAME "dcssblk"
  28 #define DCSSBLK_MINORS_PER_DISK 1
  29 #define DCSSBLK_PARM_LEN 400
  30 #define DCSS_BUS_ID_SIZE 20
  31
  32 static int dcssblk_open(struct block_device *bdev, fmode_t mode);
  33 static void dcssblk_release(struct gendisk *disk, fmode_t mode);
  34 static blk_qc_t dcssblk_make_request(struct request_queue *q,
  35                                                 struct bio *bio);
  36 static long dcssblk_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
  37                 long nr_pages, void **kaddr, pfn_t *pfn);
  38
  39 static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0";
  40
  41 static int dcssblk_major;
  42 static const struct block_device_operations dcssblk_devops = {
  43         .owner          = THIS_MODULE,
  44         .open           = dcssblk_open,
  45         .release        = dcssblk_release,
  46 };
  47
  48 static size_t dcssblk_dax_copy_from_iter(struct dax_device *dax_dev,
  49                 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
  50 {
  51         return copy_from_iter(addr, bytes, i);
  52 }
  53
  54 static size_t dcssblk_dax_copy_to_iter(struct dax_device *dax_dev,
  55                 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
  56 {
  57         return copy_to_iter(addr, bytes, i);
  58 }
  59
  60 static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev,
  61                                        pgoff_t pgoff, size_t nr_pages)
  62 {
  63         long rc;
  64         void *kaddr;
  65
  66         rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL);
  67         if (rc < 0)
  68                 return rc;
  69         memset(kaddr, 0, nr_pages << PAGE_SHIFT);
  70         dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT);
  71         return 0;
  72 }
  73
  74 static const struct dax_operations dcssblk_dax_ops = {
  75         .direct_access = dcssblk_dax_direct_access,
  76         .dax_supported = generic_fsdax_supported,
  77         .copy_from_iter = dcssblk_dax_copy_from_iter,
  78         .copy_to_iter = dcssblk_dax_copy_to_iter,
  79         .zero_page_range = dcssblk_dax_zero_page_range,
  80 };
  81
  82 struct dcssblk_dev_info {
  83         struct list_head lh;
  84         struct device dev;
  85         char segment_name[DCSS_BUS_ID_SIZE];
  86         atomic_t use_count;
  87         struct gendisk *gd;
  88         unsigned long start;
  89         unsigned long end;
  90         int segment_type;
  91         unsigned char save_pending;
  92         unsigned char is_shared;
  93         struct request_queue *dcssblk_queue;
  94         int num_of_segments;
  95         struct list_head seg_list;
  96         struct dax_device *dax_dev;
  97 };
  98
  99 struct segment_info {
 100         struct list_head lh;
 101         char segment_name[DCSS_BUS_ID_SIZE];
 102         unsigned long start;
 103         unsigned long end;
 104         int segment_type;
 105 };
 106
 107 static ssize_t dcssblk_add_store(struct device * dev, struct device_attribute *attr, const char * buf,
 108                                   size_t count);
 109 static ssize_t dcssblk_remove_store(struct device * dev, struct device_attribute *attr, const char * buf,
 110                                   size_t count);
 111
 112 static DEVICE_ATTR(add, S_IWUSR, NULL, dcssblk_add_store);
 113 static DEVICE_ATTR(remove, S_IWUSR, NULL, dcssblk_remove_store);
 114
 115 static struct device *dcssblk_root_dev;
 116
 117 static LIST_HEAD(dcssblk_devices);
 118 static struct rw_semaphore dcssblk_devices_sem;
 119
 120 /*
 121  * release function for segment device.
 122  */
 123 static void
 124 dcssblk_release_segment(struct device *dev)
 125 {
 126         struct dcssblk_dev_info *dev_info;
 127         struct segment_info *entry, *temp;
 128
 129         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 130         list_for_each_entry_safe(entry, temp, &dev_info->seg_list, lh) {
 131                 list_del(&entry->lh);
 132                 kfree(entry);
 133         }
 134         kfree(dev_info);
 135         module_put(THIS_MODULE);
 136 }
 137
 138 /*
 139  * get a minor number. needs to be called with
 140  * down_write(&dcssblk_devices_sem) and the
 141  * device needs to be enqueued before the semaphore is
 142  * freed.
 143  */
 144 static int
 145 dcssblk_assign_free_minor(struct dcssblk_dev_info *dev_info)
 146 {
 147         int minor, found;
 148         struct dcssblk_dev_info *entry;
 149
 150         if (dev_info == NULL)
 151                 return -EINVAL;
 152         for (minor = 0; minor < (1<<MINORBITS); minor++) {
 153                 found = 0;
 154                 // test if minor available
 155                 list_for_each_entry(entry, &dcssblk_devices, lh)
 156                         if (minor == entry->gd->first_minor)
 157                                 found++;
 158                 if (!found) break; // got unused minor
 159         }
 160         if (found)
 161                 return -EBUSY;
 162         dev_info->gd->first_minor = minor;
 163         return 0;
 164 }
 165
 166 /*
 167  * get the struct dcssblk_dev_info from dcssblk_devices
 168  * for the given name.
 169  * down_read(&dcssblk_devices_sem) must be held.
 170  */
 171 static struct dcssblk_dev_info *
 172 dcssblk_get_device_by_name(char *name)
 173 {
 174         struct dcssblk_dev_info *entry;
 175
 176         list_for_each_entry(entry, &dcssblk_devices, lh) {
 177                 if (!strcmp(name, entry->segment_name)) {
 178                         return entry;
 179                 }
 180         }
 181         return NULL;
 182 }
 183
 184 /*
 185  * get the struct segment_info from seg_list
 186  * for the given name.
 187  * down_read(&dcssblk_devices_sem) must be held.
 188  */
 189 static struct segment_info *
 190 dcssblk_get_segment_by_name(char *name)
 191 {
 192         struct dcssblk_dev_info *dev_info;
 193         struct segment_info *entry;
 194
 195         list_for_each_entry(dev_info, &dcssblk_devices, lh) {
 196                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 197                         if (!strcmp(name, entry->segment_name))
 198                                 return entry;
 199                 }
 200         }
 201         return NULL;
 202 }
 203
 204 /*
 205  * get the highest address of the multi-segment block.
 206  */
 207 static unsigned long
 208 dcssblk_find_highest_addr(struct dcssblk_dev_info *dev_info)
 209 {
 210         unsigned long highest_addr;
 211         struct segment_info *entry;
 212
 213         highest_addr = 0;
 214         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 215                 if (highest_addr < entry->end)
 216                         highest_addr = entry->end;
 217         }
 218         return highest_addr;
 219 }
 220
 221 /*
 222  * get the lowest address of the multi-segment block.
 223  */
 224 static unsigned long
 225 dcssblk_find_lowest_addr(struct dcssblk_dev_info *dev_info)
 226 {
 227         int set_first;
 228         unsigned long lowest_addr;
 229         struct segment_info *entry;
 230
 231         set_first = 0;
 232         lowest_addr = 0;
 233         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 234                 if (set_first == 0) {
 235                         lowest_addr = entry->start;
 236                         set_first = 1;
 237                 } else {
 238                         if (lowest_addr > entry->start)
 239                                 lowest_addr = entry->start;
 240                 }
 241         }
 242         return lowest_addr;
 243 }
 244
 245 /*
 246  * Check continuity of segments.
 247  */
 248 static int
 249 dcssblk_is_continuous(struct dcssblk_dev_info *dev_info)
 250 {
 251         int i, j, rc;
 252         struct segment_info *sort_list, *entry, temp;
 253
 254         if (dev_info->num_of_segments <= 1)
 255                 return 0;
 256
 257         sort_list = kcalloc(dev_info->num_of_segments,
 258                             sizeof(struct segment_info),
 259                             GFP_KERNEL);
 260         if (sort_list == NULL)
 261                 return -ENOMEM;
 262         i = 0;
 263         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 264                 memcpy(&sort_list[i], entry, sizeof(struct segment_info));
 265                 i++;
 266         }
 267
 268         /* sort segments */
 269         for (i = 0; i < dev_info->num_of_segments; i++)
 270                 for (j = 0; j < dev_info->num_of_segments; j++)
 271                         if (sort_list[j].start > sort_list[i].start) {
 272                                 memcpy(&temp, &sort_list[i],
 273                                         sizeof(struct segment_info));
 274                                 memcpy(&sort_list[i], &sort_list[j],
 275                                         sizeof(struct segment_info));
 276                                 memcpy(&sort_list[j], &temp,
 277                                         sizeof(struct segment_info));
 278                         }
 279
 280         /* check continuity */
 281         for (i = 0; i < dev_info->num_of_segments - 1; i++) {
 282                 if ((sort_list[i].end + 1) != sort_list[i+1].start) {
 283                         pr_err("Adjacent DCSSs %s and %s are not "
 284                                "contiguous\n", sort_list[i].segment_name,
 285                                sort_list[i+1].segment_name);
 286                         rc = -EINVAL;
 287                         goto out;
 288                 }
 289                 /* EN and EW are allowed in a block device */
 290                 if (sort_list[i].segment_type != sort_list[i+1].segment_type) {
 291                         if (!(sort_list[i].segment_type & SEGMENT_EXCLUSIVE) ||
 292                                 (sort_list[i].segment_type == SEG_TYPE_ER) ||
 293                                 !(sort_list[i+1].segment_type &
 294                                 SEGMENT_EXCLUSIVE) ||
 295                                 (sort_list[i+1].segment_type == SEG_TYPE_ER)) {
 296                                 pr_err("DCSS %s and DCSS %s have "
 297                                        "incompatible types\n",
 298                                        sort_list[i].segment_name,
 299                                        sort_list[i+1].segment_name);
 300                                 rc = -EINVAL;
 301                                 goto out;
 302                         }
 303                 }
 304         }
 305         rc = 0;
 306 out:
 307         kfree(sort_list);
 308         return rc;
 309 }
 310
 311 /*
 312  * Load a segment
 313  */
 314 static int
 315 dcssblk_load_segment(char *name, struct segment_info **seg_info)
 316 {
 317         int rc;
 318
 319         /* already loaded? */
 320         down_read(&dcssblk_devices_sem);
 321         *seg_info = dcssblk_get_segment_by_name(name);
 322         up_read(&dcssblk_devices_sem);
 323         if (*seg_info != NULL)
 324                 return -EEXIST;
 325
 326         /* get a struct segment_info */
 327         *seg_info = kzalloc(sizeof(struct segment_info), GFP_KERNEL);
 328         if (*seg_info == NULL)
 329                 return -ENOMEM;
 330
 331         strcpy((*seg_info)->segment_name, name);
 332
 333         /* load the segment */
 334         rc = segment_load(name, SEGMENT_SHARED,
 335                         &(*seg_info)->start, &(*seg_info)->end);
 336         if (rc < 0) {
 337                 segment_warning(rc, (*seg_info)->segment_name);
 338                 kfree(*seg_info);
 339         } else {
 340                 INIT_LIST_HEAD(&(*seg_info)->lh);
 341                 (*seg_info)->segment_type = rc;
 342         }
 343         return rc;
 344 }
 345
 346 /*
 347  * device attribute for switching shared/nonshared (exclusive)
 348  * operation (show + store)
 349  */
 350 static ssize_t
 351 dcssblk_shared_show(struct device *dev, struct device_attribute *attr, char *buf)
 352 {
 353         struct dcssblk_dev_info *dev_info;
 354
 355         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 356         return sprintf(buf, dev_info->is_shared ? "1\n" : "0\n");
 357 }
 358
 359 static ssize_t
 360 dcssblk_shared_store(struct device *dev, struct device_attribute *attr, const char *inbuf, size_t count)
 361 {
 362         struct dcssblk_dev_info *dev_info;
 363         struct segment_info *entry, *temp;
 364         int rc;
 365
 366         if ((count > 1) && (inbuf[1] != '\n') && (inbuf[1] != '\0'))
 367                 return -EINVAL;
 368         down_write(&dcssblk_devices_sem);
 369         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 370         if (atomic_read(&dev_info->use_count)) {
 371                 rc = -EBUSY;
 372                 goto out;
 373         }
 374         if (inbuf[0] == '1') {
 375                 /* reload segments in shared mode */
 376                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 377                         rc = segment_modify_shared(entry->segment_name,
 378                                                 SEGMENT_SHARED);
 379                         if (rc < 0) {
 380                                 BUG_ON(rc == -EINVAL);
 381                                 if (rc != -EAGAIN)
 382                                         goto removeseg;
 383                         }
 384                 }
 385                 dev_info->is_shared = 1;
 386                 switch (dev_info->segment_type) {
 387                 case SEG_TYPE_SR:
 388                 case SEG_TYPE_ER:
 389                 case SEG_TYPE_SC:
 390                         set_disk_ro(dev_info->gd, 1);
 391                 }
 392         } else if (inbuf[0] == '0') {
 393                 /* reload segments in exclusive mode */
 394                 if (dev_info->segment_type == SEG_TYPE_SC) {
 395                         pr_err("DCSS %s is of type SC and cannot be "
 396                                "loaded as exclusive-writable\n",
 397                                dev_info->segment_name);
 398                         rc = -EINVAL;
 399                         goto out;
 400                 }
 401                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 402                         rc = segment_modify_shared(entry->segment_name,
 403                                                    SEGMENT_EXCLUSIVE);
 404                         if (rc < 0) {
 405                                 BUG_ON(rc == -EINVAL);
 406                                 if (rc != -EAGAIN)
 407                                         goto removeseg;
 408                         }
 409                 }
 410                 dev_info->is_shared = 0;
 411                 set_disk_ro(dev_info->gd, 0);
 412         } else {
 413                 rc = -EINVAL;
 414                 goto out;
 415         }
 416         rc = count;
 417         goto out;
 418
 419 removeseg:
 420         pr_err("DCSS device %s is removed after a failed access mode "
 421                "change\n", dev_info->segment_name);
 422         temp = entry;
 423         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 424                 if (entry != temp)
 425                         segment_unload(entry->segment_name);
 426         }
 427         list_del(&dev_info->lh);
 428
 429         kill_dax(dev_info->dax_dev);
 430         put_dax(dev_info->dax_dev);
 431         del_gendisk(dev_info->gd);
 432         blk_cleanup_queue(dev_info->dcssblk_queue);
 433         dev_info->gd->queue = NULL;
 434         put_disk(dev_info->gd);
 435         up_write(&dcssblk_devices_sem);
 436
 437         if (device_remove_file_self(dev, attr)) {
 438                 device_unregister(dev);
 439                 put_device(dev);
 440         }
 441         return rc;
 442 out:
 443         up_write(&dcssblk_devices_sem);
 444         return rc;
 445 }
 446 static DEVICE_ATTR(shared, S_IWUSR | S_IRUSR, dcssblk_shared_show,
 447                    dcssblk_shared_store);
 448
 449 /*
 450  * device attribute for save operation on current copy
 451  * of the segment. If the segment is busy, saving will
 452  * become pending until it gets released, which can be
 453  * undone by storing a non-true value to this entry.
 454  * (show + store)
 455  */
 456 static ssize_t
 457 dcssblk_save_show(struct device *dev, struct device_attribute *attr, char *buf)
 458 {
 459         struct dcssblk_dev_info *dev_info;
 460
 461         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 462         return sprintf(buf, dev_info->save_pending ? "1\n" : "0\n");
 463 }
 464
 465 static ssize_t
 466 dcssblk_save_store(struct device *dev, struct device_attribute *attr, const char *inbuf, size_t count)
 467 {
 468         struct dcssblk_dev_info *dev_info;
 469         struct segment_info *entry;
 470
 471         if ((count > 1) && (inbuf[1] != '\n') && (inbuf[1] != '\0'))
 472                 return -EINVAL;
 473         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 474
 475         down_write(&dcssblk_devices_sem);
 476         if (inbuf[0] == '1') {
 477                 if (atomic_read(&dev_info->use_count) == 0) {
 478                         // device is idle => we save immediately
 479                         pr_info("All DCSSs that map to device %s are "
 480                                 "saved\n", dev_info->segment_name);
 481                         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 482                                 if (entry->segment_type == SEG_TYPE_EN ||
 483                                     entry->segment_type == SEG_TYPE_SN)
 484                                         pr_warn("DCSS %s is of type SN or EN"
 485                                                 " and cannot be saved\n",
 486                                                 entry->segment_name);
 487                                 else
 488                                         segment_save(entry->segment_name);
 489                         }
 490                 }  else {
 491                         // device is busy => we save it when it becomes
 492                         // idle in dcssblk_release
 493                         pr_info("Device %s is in use, its DCSSs will be "
 494                                 "saved when it becomes idle\n",
 495                                 dev_info->segment_name);
 496                         dev_info->save_pending = 1;
 497                 }
 498         } else if (inbuf[0] == '0') {
 499                 if (dev_info->save_pending) {
 500                         // device is busy & the user wants to undo his save
 501                         // request
 502                         dev_info->save_pending = 0;
 503                         pr_info("A pending save request for device %s "
 504                                 "has been canceled\n",
 505                                 dev_info->segment_name);
 506                 }
 507         } else {
 508                 up_write(&dcssblk_devices_sem);
 509                 return -EINVAL;
 510         }
 511         up_write(&dcssblk_devices_sem);
 512         return count;
 513 }
 514 static DEVICE_ATTR(save, S_IWUSR | S_IRUSR, dcssblk_save_show,
 515                    dcssblk_save_store);
 516
 517 /*
 518  * device attribute for showing all segments in a device
 519  */
 520 static ssize_t
 521 dcssblk_seglist_show(struct device *dev, struct device_attribute *attr,
 522                 char *buf)
 523 {
 524         int i;
 525
 526         struct dcssblk_dev_info *dev_info;
 527         struct segment_info *entry;
 528
 529         down_read(&dcssblk_devices_sem);
 530         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 531         i = 0;
 532         buf[0] = '\0';
 533         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 534                 strcpy(&buf[i], entry->segment_name);
 535                 i += strlen(entry->segment_name);
 536                 buf[i] = '\n';
 537                 i++;
 538         }
 539         up_read(&dcssblk_devices_sem);
 540         return i;
 541 }
 542 static DEVICE_ATTR(seglist, S_IRUSR, dcssblk_seglist_show, NULL);
 543
 544 static struct attribute *dcssblk_dev_attrs[] = {
 545         &dev_attr_shared.attr,
 546         &dev_attr_save.attr,
 547         &dev_attr_seglist.attr,
 548         NULL,
 549 };
 550 static struct attribute_group dcssblk_dev_attr_group = {
 551         .attrs = dcssblk_dev_attrs,
 552 };
 553 static const struct attribute_group *dcssblk_dev_attr_groups[] = {
 554         &dcssblk_dev_attr_group,
 555         NULL,
 556 };
 557
 558 /*
 559  * device attribute for adding devices
 560  */
 561 static ssize_t
 562 dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
 563 {
 564         int rc, i, j, num_of_segments;
 565         struct dcssblk_dev_info *dev_info;
 566         struct segment_info *seg_info, *temp;
 567         char *local_buf;
 568         unsigned long seg_byte_size;
 569
 570         dev_info = NULL;
 571         seg_info = NULL;
 572         if (dev != dcssblk_root_dev) {
 573                 rc = -EINVAL;
 574                 goto out_nobuf;
 575         }
 576         if ((count < 1) || (buf[0] == '\0') || (buf[0] == '\n')) {
 577                 rc = -ENAMETOOLONG;
 578                 goto out_nobuf;
 579         }
 580
 581         local_buf = kmalloc(count + 1, GFP_KERNEL);
 582         if (local_buf == NULL) {
 583                 rc = -ENOMEM;
 584                 goto out_nobuf;
 585         }
 586
 587         /*
 588          * parse input
 589          */
 590         num_of_segments = 0;
 591         for (i = 0; (i < count && (buf[i] != '\0') && (buf[i] != '\n')); i++) {
 592                 for (j = i; j < count &&
 593                         (buf[j] != ':') &&
 594                         (buf[j] != '\0') &&
 595                         (buf[j] != '\n'); j++) {
 596                         local_buf[j-i] = toupper(buf[j]);
 597                 }
 598                 local_buf[j-i] = '\0';
 599                 if (((j - i) == 0) || ((j - i) > 8)) {
 600                         rc = -ENAMETOOLONG;
 601                         goto seg_list_del;
 602                 }
 603
 604                 rc = dcssblk_load_segment(local_buf, &seg_info);
 605                 if (rc < 0)
 606                         goto seg_list_del;
 607                 /*
 608                  * get a struct dcssblk_dev_info
 609                  */
 610                 if (num_of_segments == 0) {
 611                         dev_info = kzalloc(sizeof(struct dcssblk_dev_info),
 612                                         GFP_KERNEL);
 613                         if (dev_info == NULL) {
 614                                 rc = -ENOMEM;
 615                                 goto out;
 616                         }
 617                         strcpy(dev_info->segment_name, local_buf);
 618                         dev_info->segment_type = seg_info->segment_type;
 619                         INIT_LIST_HEAD(&dev_info->seg_list);
 620                 }
 621                 list_add_tail(&seg_info->lh, &dev_info->seg_list);
 622                 num_of_segments++;
 623                 i = j;
 624
 625                 if ((buf[j] == '\0') || (buf[j] == '\n'))
 626                         break;
 627         }
 628
 629         /* no trailing colon at the end of the input */
 630         if ((i > 0) && (buf[i-1] == ':')) {
 631                 rc = -ENAMETOOLONG;
 632                 goto seg_list_del;
 633         }
 634         strlcpy(local_buf, buf, i + 1);
 635         dev_info->num_of_segments = num_of_segments;
 636         rc = dcssblk_is_continuous(dev_info);
 637         if (rc < 0)
 638                 goto seg_list_del;
 639
 640         dev_info->start = dcssblk_find_lowest_addr(dev_info);
 641         dev_info->end = dcssblk_find_highest_addr(dev_info);
 642
 643         dev_set_name(&dev_info->dev, "%s", dev_info->segment_name);
 644         dev_info->dev.release = dcssblk_release_segment;
 645         dev_info->dev.groups = dcssblk_dev_attr_groups;
 646         INIT_LIST_HEAD(&dev_info->lh);
 647         dev_info->gd = alloc_disk(DCSSBLK_MINORS_PER_DISK);
 648         if (dev_info->gd == NULL) {
 649                 rc = -ENOMEM;
 650                 goto seg_list_del;
 651         }
 652         dev_info->gd->major = dcssblk_major;
 653         dev_info->gd->fops = &dcssblk_devops;
 654         dev_info->dcssblk_queue = blk_alloc_queue(GFP_KERNEL);
 655         dev_info->gd->queue = dev_info->dcssblk_queue;
 656         dev_info->gd->private_data = dev_info;
 657         blk_queue_make_request(dev_info->dcssblk_queue, dcssblk_make_request);
 658         blk_queue_logical_block_size(dev_info->dcssblk_queue, 4096);
 659         blk_queue_flag_set(QUEUE_FLAG_DAX, dev_info->dcssblk_queue);
 660
 661         seg_byte_size = (dev_info->end - dev_info->start + 1);
 662         set_capacity(dev_info->gd, seg_byte_size >> 9); // size in sectors
 663         pr_info("Loaded %s with total size %lu bytes and capacity %lu "
 664                 "sectors\n", local_buf, seg_byte_size, seg_byte_size >> 9);
 665
 666         dev_info->save_pending = 0;
 667         dev_info->is_shared = 1;
 668         dev_info->dev.parent = dcssblk_root_dev;
 669
 670         /*
 671          *get minor, add to list
 672          */
 673         down_write(&dcssblk_devices_sem);
 674         if (dcssblk_get_segment_by_name(local_buf)) {
 675                 rc = -EEXIST;
 676                 goto release_gd;
 677         }
 678         rc = dcssblk_assign_free_minor(dev_info);
 679         if (rc)
 680                 goto release_gd;
 681         sprintf(dev_info->gd->disk_name, "dcssblk%d",
 682                 dev_info->gd->first_minor);
 683         list_add_tail(&dev_info->lh, &dcssblk_devices);
 684
 685         if (!try_module_get(THIS_MODULE)) {
 686                 rc = -ENODEV;
 687                 goto dev_list_del;
 688         }
 689         /*
 690          * register the device
 691          */
 692         rc = device_register(&dev_info->dev);
 693         if (rc)
 694                 goto put_dev;
 695
 696         dev_info->dax_dev = alloc_dax(dev_info, dev_info->gd->disk_name,
 697                         &dcssblk_dax_ops, DAXDEV_F_SYNC);
 698         if (IS_ERR(dev_info->dax_dev)) {
 699                 rc = PTR_ERR(dev_info->dax_dev);
 700                 dev_info->dax_dev = NULL;
 701                 goto put_dev;
 702         }
 703
 704         get_device(&dev_info->dev);
 705         device_add_disk(&dev_info->dev, dev_info->gd, NULL);
 706
 707         switch (dev_info->segment_type) {
 708                 case SEG_TYPE_SR:
 709                 case SEG_TYPE_ER:
 710                 case SEG_TYPE_SC:
 711                         set_disk_ro(dev_info->gd,1);
 712                         break;
 713                 default:
 714                         set_disk_ro(dev_info->gd,0);
 715                         break;
 716         }
 717         up_write(&dcssblk_devices_sem);
 718         rc = count;
 719         goto out;
 720
 721 put_dev:
 722         list_del(&dev_info->lh);
 723         blk_cleanup_queue(dev_info->dcssblk_queue);
 724         dev_info->gd->queue = NULL;
 725         put_disk(dev_info->gd);
 726         list_for_each_entry(seg_info, &dev_info->seg_list, lh) {
 727                 segment_unload(seg_info->segment_name);
 728         }
 729         put_device(&dev_info->dev);
 730         up_write(&dcssblk_devices_sem);
 731         goto out;
 732 dev_list_del:
 733         list_del(&dev_info->lh);
 734 release_gd:
 735         blk_cleanup_queue(dev_info->dcssblk_queue);
 736         dev_info->gd->queue = NULL;
 737         put_disk(dev_info->gd);
 738         up_write(&dcssblk_devices_sem);
 739 seg_list_del:
 740         if (dev_info == NULL)
 741                 goto out;
 742         list_for_each_entry_safe(seg_info, temp, &dev_info->seg_list, lh) {
 743                 list_del(&seg_info->lh);
 744                 segment_unload(seg_info->segment_name);
 745                 kfree(seg_info);
 746         }
 747         kfree(dev_info);
 748 out:
 749         kfree(local_buf);
 750 out_nobuf:
 751         return rc;
 752 }
 753
 754 /*
 755  * device attribute for removing devices
 756  */
 757 static ssize_t
 758 dcssblk_remove_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
 759 {
 760         struct dcssblk_dev_info *dev_info;
 761         struct segment_info *entry;
 762         int rc, i;
 763         char *local_buf;
 764
 765         if (dev != dcssblk_root_dev) {
 766                 return -EINVAL;
 767         }
 768         local_buf = kmalloc(count + 1, GFP_KERNEL);
 769         if (local_buf == NULL) {
 770                 return -ENOMEM;
 771         }
 772         /*
 773          * parse input
 774          */
 775         for (i = 0; (i < count && (*(buf+i)!='\0') && (*(buf+i)!='\n')); i++) {
 776                 local_buf[i] = toupper(buf[i]);
 777         }
 778         local_buf[i] = '\0';
 779         if ((i == 0) || (i > 8)) {
 780                 rc = -ENAMETOOLONG;
 781                 goto out_buf;
 782         }
 783
 784         down_write(&dcssblk_devices_sem);
 785         dev_info = dcssblk_get_device_by_name(local_buf);
 786         if (dev_info == NULL) {
 787                 up_write(&dcssblk_devices_sem);
 788                 pr_warn("Device %s cannot be removed because it is not a known device\n",
 789                         local_buf);
 790                 rc = -ENODEV;
 791                 goto out_buf;
 792         }
 793         if (atomic_read(&dev_info->use_count) != 0) {
 794                 up_write(&dcssblk_devices_sem);
 795                 pr_warn("Device %s cannot be removed while it is in use\n",
 796                         local_buf);
 797                 rc = -EBUSY;
 798                 goto out_buf;
 799         }
 800
 801         list_del(&dev_info->lh);
 802         kill_dax(dev_info->dax_dev);
 803         put_dax(dev_info->dax_dev);
 804         del_gendisk(dev_info->gd);
 805         blk_cleanup_queue(dev_info->dcssblk_queue);
 806         dev_info->gd->queue = NULL;
 807         put_disk(dev_info->gd);
 808
 809         /* unload all related segments */
 810         list_for_each_entry(entry, &dev_info->seg_list, lh)
 811                 segment_unload(entry->segment_name);
 812
 813         up_write(&dcssblk_devices_sem);
 814
 815         device_unregister(&dev_info->dev);
 816         put_device(&dev_info->dev);
 817
 818         rc = count;
 819 out_buf:
 820         kfree(local_buf);
 821         return rc;
 822 }
 823
 824 static int
 825 dcssblk_open(struct block_device *bdev, fmode_t mode)
 826 {
 827         struct dcssblk_dev_info *dev_info;
 828         int rc;
 829
 830         dev_info = bdev->bd_disk->private_data;
 831         if (NULL == dev_info) {
 832                 rc = -ENODEV;
 833                 goto out;
 834         }
 835         atomic_inc(&dev_info->use_count);
 836         bdev->bd_block_size = 4096;
 837         rc = 0;
 838 out:
 839         return rc;
 840 }
 841
 842 static void
 843 dcssblk_release(struct gendisk *disk, fmode_t mode)
 844 {
 845         struct dcssblk_dev_info *dev_info = disk->private_data;
 846         struct segment_info *entry;
 847
 848         if (!dev_info) {
 849                 WARN_ON(1);
 850                 return;
 851         }
 852         down_write(&dcssblk_devices_sem);
 853         if (atomic_dec_and_test(&dev_info->use_count)
 854             && (dev_info->save_pending)) {
 855                 pr_info("Device %s has become idle and is being saved "
 856                         "now\n", dev_info->segment_name);
 857                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 858                         if (entry->segment_type == SEG_TYPE_EN ||
 859                             entry->segment_type == SEG_TYPE_SN)
 860                                 pr_warn("DCSS %s is of type SN or EN and cannot"
 861                                         " be saved\n", entry->segment_name);
 862                         else
 863                                 segment_save(entry->segment_name);
 864                 }
 865                 dev_info->save_pending = 0;
 866         }
 867         up_write(&dcssblk_devices_sem);
 868 }
 869
 870 static blk_qc_t
 871 dcssblk_make_request(struct request_queue *q, struct bio *bio)
 872 {
 873         struct dcssblk_dev_info *dev_info;
 874         struct bio_vec bvec;
 875         struct bvec_iter iter;
 876         unsigned long index;
 877         unsigned long page_addr;
 878         unsigned long source_addr;
 879         unsigned long bytes_done;
 880
 881         blk_queue_split(q, &bio);
 882
 883         bytes_done = 0;
 884         dev_info = bio->bi_disk->private_data;
 885         if (dev_info == NULL)
 886                 goto fail;
 887         if ((bio->bi_iter.bi_sector & 7) != 0 ||
 888             (bio->bi_iter.bi_size & 4095) != 0)
 889                 /* Request is not page-aligned. */
 890                 goto fail;
 891         if (bio_end_sector(bio) > get_capacity(bio->bi_disk)) {
 892                 /* Request beyond end of DCSS segment. */
 893                 goto fail;
 894         }
 895         /* verify data transfer direction */
 896         if (dev_info->is_shared) {
 897                 switch (dev_info->segment_type) {
 898                 case SEG_TYPE_SR:
 899                 case SEG_TYPE_ER:
 900                 case SEG_TYPE_SC:
 901                         /* cannot write to these segments */
 902                         if (bio_data_dir(bio) == WRITE) {
 903                                 pr_warn("Writing to %s failed because it is a read-only device\n",
 904                                         dev_name(&dev_info->dev));
 905                                 goto fail;
 906                         }
 907                 }
 908         }
 909
 910         index = (bio->bi_iter.bi_sector >> 3);
 911         bio_for_each_segment(bvec, bio, iter) {
 912                 page_addr = (unsigned long)
 913                         page_address(bvec.bv_page) + bvec.bv_offset;
 914                 source_addr = dev_info->start + (index<<12) + bytes_done;
 915                 if (unlikely((page_addr & 4095) != 0) || (bvec.bv_len & 4095) != 0)
 916                         // More paranoia.
 917                         goto fail;
 918                 if (bio_data_dir(bio) == READ) {
 919                         memcpy((void*)page_addr, (void*)source_addr,
 920                                 bvec.bv_len);
 921                 } else {
 922                         memcpy((void*)source_addr, (void*)page_addr,
 923                                 bvec.bv_len);
 924                 }
 925                 bytes_done += bvec.bv_len;
 926         }
 927         bio_endio(bio);
 928         return BLK_QC_T_NONE;
 929 fail:
 930         bio_io_error(bio);
 931         return BLK_QC_T_NONE;
 932 }
 933
 934 static long
 935 __dcssblk_direct_access(struct dcssblk_dev_info *dev_info, pgoff_t pgoff,
 936                 long nr_pages, void **kaddr, pfn_t *pfn)
 937 {
 938         resource_size_t offset = pgoff * PAGE_SIZE;
 939         unsigned long dev_sz;
 940
 941         dev_sz = dev_info->end - dev_info->start + 1;
 942         if (kaddr)
 943                 *kaddr = (void *) dev_info->start + offset;
 944         if (pfn)
 945                 *pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset),
 946                                 PFN_DEV|PFN_SPECIAL);
 947
 948         return (dev_sz - offset) / PAGE_SIZE;
 949 }
 950
 951 static long
 952 dcssblk_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
 953                 long nr_pages, void **kaddr, pfn_t *pfn)
 954 {
 955         struct dcssblk_dev_info *dev_info = dax_get_private(dax_dev);
 956
 957         return __dcssblk_direct_access(dev_info, pgoff, nr_pages, kaddr, pfn);
 958 }
 959
 960 static void
 961 dcssblk_check_params(void)
 962 {
 963         int rc, i, j, k;
 964         char buf[DCSSBLK_PARM_LEN + 1];
 965         struct dcssblk_dev_info *dev_info;
 966
 967         for (i = 0; (i < DCSSBLK_PARM_LEN) && (dcssblk_segments[i] != '\0');
 968              i++) {
 969                 for (j = i; (j < DCSSBLK_PARM_LEN) &&
 970                             (dcssblk_segments[j] != ',')  &&
 971                             (dcssblk_segments[j] != '\0') &&
 972                             (dcssblk_segments[j] != '('); j++)
 973                 {
 974                         buf[j-i] = dcssblk_segments[j];
 975                 }
 976                 buf[j-i] = '\0';
 977                 rc = dcssblk_add_store(dcssblk_root_dev, NULL, buf, j-i);
 978                 if ((rc >= 0) && (dcssblk_segments[j] == '(')) {
 979                         for (k = 0; (buf[k] != ':') && (buf[k] != '\0'); k++)
 980                                 buf[k] = toupper(buf[k]);
 981                         buf[k] = '\0';
 982                         if (!strncmp(&dcssblk_segments[j], "(local)", 7)) {
 983                                 down_read(&dcssblk_devices_sem);
 984                                 dev_info = dcssblk_get_device_by_name(buf);
 985                                 up_read(&dcssblk_devices_sem);
 986                                 if (dev_info)
 987                                         dcssblk_shared_store(&dev_info->dev,
 988                                                              NULL, "0\n", 2);
 989                         }
 990                 }
 991                 while ((dcssblk_segments[j] != ',') &&
 992                        (dcssblk_segments[j] != '\0'))
 993                 {
 994                         j++;
 995                 }
 996                 if (dcssblk_segments[j] == '\0')
 997                         break;
 998                 i = j;
 999         }
1000 }
1001
1002 /*
1003  * Suspend / Resume
1004  */
1005 static int dcssblk_freeze(struct device *dev)
1006 {
1007         struct dcssblk_dev_info *dev_info;
1008         int rc = 0;
1009
1010         list_for_each_entry(dev_info, &dcssblk_devices, lh) {
1011                 switch (dev_info->segment_type) {
1012                         case SEG_TYPE_SR:
1013                         case SEG_TYPE_ER:
1014                         case SEG_TYPE_SC:
1015                                 if (!dev_info->is_shared)
1016                                         rc = -EINVAL;
1017                                 break;
1018                         default:
1019                                 rc = -EINVAL;
1020                                 break;
1021                 }
1022                 if (rc)
1023                         break;
1024         }
1025         if (rc)
1026                 pr_err("Suspending the system failed because DCSS device %s "
1027                        "is writable\n",
1028                        dev_info->segment_name);
1029         return rc;
1030 }
1031
1032 static int dcssblk_restore(struct device *dev)
1033 {
1034         struct dcssblk_dev_info *dev_info;
1035         struct segment_info *entry;
1036         unsigned long start, end;
1037         int rc = 0;
1038
1039         list_for_each_entry(dev_info, &dcssblk_devices, lh) {
1040                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
1041                         segment_unload(entry->segment_name);
1042                         rc = segment_load(entry->segment_name, SEGMENT_SHARED,
1043                                           &start, &end);
1044                         if (rc < 0) {
1045 // TODO in_use check ?
1046                                 segment_warning(rc, entry->segment_name);
1047                                 goto out_panic;
1048                         }
1049                         if (start != entry->start || end != entry->end) {
1050                                 pr_err("The address range of DCSS %s changed "
1051                                        "while the system was suspended\n",
1052                                        entry->segment_name);
1053                                 goto out_panic;
1054                         }
1055                 }
1056         }
1057         return 0;
1058 out_panic:
1059         panic("fatal dcssblk resume error\n");
1060 }
1061
1062 static int dcssblk_thaw(struct device *dev)
1063 {
1064         return 0;
1065 }
1066
1067 static const struct dev_pm_ops dcssblk_pm_ops = {
1068         .freeze         = dcssblk_freeze,
1069         .thaw           = dcssblk_thaw,
1070         .restore        = dcssblk_restore,
1071 };
1072
1073 static struct platform_driver dcssblk_pdrv = {
1074         .driver = {
1075                 .name   = "dcssblk",
1076                 .pm     = &dcssblk_pm_ops,
1077         },
1078 };
1079
1080 static struct platform_device *dcssblk_pdev;
1081
1082
1083 /*
1084  * The init/exit functions.
1085  */
1086 static void __exit
1087 dcssblk_exit(void)
1088 {
1089         platform_device_unregister(dcssblk_pdev);
1090         platform_driver_unregister(&dcssblk_pdrv);
1091         root_device_unregister(dcssblk_root_dev);
1092         unregister_blkdev(dcssblk_major, DCSSBLK_NAME);
1093 }
1094
1095 static int __init
1096 dcssblk_init(void)
1097 {
1098         int rc;
1099
1100         rc = platform_driver_register(&dcssblk_pdrv);
1101         if (rc)
1102                 return rc;
1103
1104         dcssblk_pdev = platform_device_register_simple("dcssblk", -1, NULL,
1105                                                         0);
1106         if (IS_ERR(dcssblk_pdev)) {
1107                 rc = PTR_ERR(dcssblk_pdev);
1108                 goto out_pdrv;
1109         }
1110
1111         dcssblk_root_dev = root_device_register("dcssblk");
1112         if (IS_ERR(dcssblk_root_dev)) {
1113                 rc = PTR_ERR(dcssblk_root_dev);
1114                 goto out_pdev;
1115         }
1116         rc = device_create_file(dcssblk_root_dev, &dev_attr_add);
1117         if (rc)
1118                 goto out_root;
1119         rc = device_create_file(dcssblk_root_dev, &dev_attr_remove);
1120         if (rc)
1121                 goto out_root;
1122         rc = register_blkdev(0, DCSSBLK_NAME);
1123         if (rc < 0)
1124                 goto out_root;
1125         dcssblk_major = rc;
1126         init_rwsem(&dcssblk_devices_sem);
1127
1128         dcssblk_check_params();
1129         return 0;
1130
1131 out_root:
1132         root_device_unregister(dcssblk_root_dev);
1133 out_pdev:
1134         platform_device_unregister(dcssblk_pdev);
1135 out_pdrv:
1136         platform_driver_unregister(&dcssblk_pdrv);
1137         return rc;
1138 }
1139
1140 module_init(dcssblk_init);
1141 module_exit(dcssblk_exit);
1142
1143 module_param_string(segments, dcssblk_segments, DCSSBLK_PARM_LEN, 0444);
1144 MODULE_PARM_DESC(segments, "Name of DCSS segment(s) to be loaded, "
1145                  "comma-separated list, names in each set separated "
1146                  "by commas are separated by colons, each set contains "
1147                  "names of contiguous segments and each name max. 8 chars.\n"
1148                  "Adding \"(local)\" to the end of each set equals echoing 0 "
1149                  "to /sys/devices/dcssblk/<device name>/shared after loading "
1150                  "the contiguous segments - \n"
1151                  "e.g. segments=\"mydcss1,mydcss2:mydcss3,mydcss4(local)\"");
1152
1153 MODULE_LICENSE("GPL");