drivers/s390/block/dcssblk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * dcssblk.c -- the S/390 block driver for dcss memory
   4  *
   5  * Authors: Carsten Otte, Stefan Weinhuber, Gerald Schaefer
   6  */
   7
   8 #define KMSG_COMPONENT "dcssblk"
   9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/module.h>
  12 #include <linux/moduleparam.h>
  13 #include <linux/ctype.h>
  14 #include <linux/errno.h>
  15 #include <linux/init.h>
  16 #include <linux/slab.h>
  17 #include <linux/blkdev.h>
  18 #include <linux/completion.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/pfn_t.h>
  21 #include <linux/uio.h>
  22 #include <linux/dax.h>
  23 #include <asm/extmem.h>
  24 #include <asm/io.h>
  25
  26 #define DCSSBLK_NAME "dcssblk"
  27 #define DCSSBLK_MINORS_PER_DISK 1
  28 #define DCSSBLK_PARM_LEN 400
  29 #define DCSS_BUS_ID_SIZE 20
  30
  31 static int dcssblk_open(struct block_device *bdev, fmode_t mode);
  32 static void dcssblk_release(struct gendisk *disk, fmode_t mode);
  33 static blk_qc_t dcssblk_submit_bio(struct bio *bio);
  34 static long dcssblk_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
  35                 long nr_pages, void **kaddr, pfn_t *pfn);
  36
  37 static char dcssblk_segments[DCSSBLK_PARM_LEN] = "\0";
  38
  39 static int dcssblk_major;
  40 static const struct block_device_operations dcssblk_devops = {
  41         .owner          = THIS_MODULE,
  42         .submit_bio     = dcssblk_submit_bio,
  43         .open           = dcssblk_open,
  44         .release        = dcssblk_release,
  45 };
  46
  47 static size_t dcssblk_dax_copy_from_iter(struct dax_device *dax_dev,
  48                 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
  49 {
  50         return copy_from_iter(addr, bytes, i);
  51 }
  52
  53 static size_t dcssblk_dax_copy_to_iter(struct dax_device *dax_dev,
  54                 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
  55 {
  56         return copy_to_iter(addr, bytes, i);
  57 }
  58
  59 static int dcssblk_dax_zero_page_range(struct dax_device *dax_dev,
  60                                        pgoff_t pgoff, size_t nr_pages)
  61 {
  62         long rc;
  63         void *kaddr;
  64
  65         rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL);
  66         if (rc < 0)
  67                 return rc;
  68         memset(kaddr, 0, nr_pages << PAGE_SHIFT);
  69         dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT);
  70         return 0;
  71 }
  72
  73 static const struct dax_operations dcssblk_dax_ops = {
  74         .direct_access = dcssblk_dax_direct_access,
  75         .dax_supported = generic_fsdax_supported,
  76         .copy_from_iter = dcssblk_dax_copy_from_iter,
  77         .copy_to_iter = dcssblk_dax_copy_to_iter,
  78         .zero_page_range = dcssblk_dax_zero_page_range,
  79 };
  80
  81 struct dcssblk_dev_info {
  82         struct list_head lh;
  83         struct device dev;
  84         char segment_name[DCSS_BUS_ID_SIZE];
  85         atomic_t use_count;
  86         struct gendisk *gd;
  87         unsigned long start;
  88         unsigned long end;
  89         int segment_type;
  90         unsigned char save_pending;
  91         unsigned char is_shared;
  92         int num_of_segments;
  93         struct list_head seg_list;
  94         struct dax_device *dax_dev;
  95 };
  96
  97 struct segment_info {
  98         struct list_head lh;
  99         char segment_name[DCSS_BUS_ID_SIZE];
 100         unsigned long start;
 101         unsigned long end;
 102         int segment_type;
 103 };
 104
 105 static ssize_t dcssblk_add_store(struct device * dev, struct device_attribute *attr, const char * buf,
 106                                   size_t count);
 107 static ssize_t dcssblk_remove_store(struct device * dev, struct device_attribute *attr, const char * buf,
 108                                   size_t count);
 109
 110 static DEVICE_ATTR(add, S_IWUSR, NULL, dcssblk_add_store);
 111 static DEVICE_ATTR(remove, S_IWUSR, NULL, dcssblk_remove_store);
 112
 113 static struct device *dcssblk_root_dev;
 114
 115 static LIST_HEAD(dcssblk_devices);
 116 static struct rw_semaphore dcssblk_devices_sem;
 117
 118 /*
 119  * release function for segment device.
 120  */
 121 static void
 122 dcssblk_release_segment(struct device *dev)
 123 {
 124         struct dcssblk_dev_info *dev_info;
 125         struct segment_info *entry, *temp;
 126
 127         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 128         list_for_each_entry_safe(entry, temp, &dev_info->seg_list, lh) {
 129                 list_del(&entry->lh);
 130                 kfree(entry);
 131         }
 132         kfree(dev_info);
 133         module_put(THIS_MODULE);
 134 }
 135
 136 /*
 137  * get a minor number. needs to be called with
 138  * down_write(&dcssblk_devices_sem) and the
 139  * device needs to be enqueued before the semaphore is
 140  * freed.
 141  */
 142 static int
 143 dcssblk_assign_free_minor(struct dcssblk_dev_info *dev_info)
 144 {
 145         int minor, found;
 146         struct dcssblk_dev_info *entry;
 147
 148         if (dev_info == NULL)
 149                 return -EINVAL;
 150         for (minor = 0; minor < (1<<MINORBITS); minor++) {
 151                 found = 0;
 152                 // test if minor available
 153                 list_for_each_entry(entry, &dcssblk_devices, lh)
 154                         if (minor == entry->gd->first_minor)
 155                                 found++;
 156                 if (!found) break; // got unused minor
 157         }
 158         if (found)
 159                 return -EBUSY;
 160         dev_info->gd->first_minor = minor;
 161         return 0;
 162 }
 163
 164 /*
 165  * get the struct dcssblk_dev_info from dcssblk_devices
 166  * for the given name.
 167  * down_read(&dcssblk_devices_sem) must be held.
 168  */
 169 static struct dcssblk_dev_info *
 170 dcssblk_get_device_by_name(char *name)
 171 {
 172         struct dcssblk_dev_info *entry;
 173
 174         list_for_each_entry(entry, &dcssblk_devices, lh) {
 175                 if (!strcmp(name, entry->segment_name)) {
 176                         return entry;
 177                 }
 178         }
 179         return NULL;
 180 }
 181
 182 /*
 183  * get the struct segment_info from seg_list
 184  * for the given name.
 185  * down_read(&dcssblk_devices_sem) must be held.
 186  */
 187 static struct segment_info *
 188 dcssblk_get_segment_by_name(char *name)
 189 {
 190         struct dcssblk_dev_info *dev_info;
 191         struct segment_info *entry;
 192
 193         list_for_each_entry(dev_info, &dcssblk_devices, lh) {
 194                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 195                         if (!strcmp(name, entry->segment_name))
 196                                 return entry;
 197                 }
 198         }
 199         return NULL;
 200 }
 201
 202 /*
 203  * get the highest address of the multi-segment block.
 204  */
 205 static unsigned long
 206 dcssblk_find_highest_addr(struct dcssblk_dev_info *dev_info)
 207 {
 208         unsigned long highest_addr;
 209         struct segment_info *entry;
 210
 211         highest_addr = 0;
 212         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 213                 if (highest_addr < entry->end)
 214                         highest_addr = entry->end;
 215         }
 216         return highest_addr;
 217 }
 218
 219 /*
 220  * get the lowest address of the multi-segment block.
 221  */
 222 static unsigned long
 223 dcssblk_find_lowest_addr(struct dcssblk_dev_info *dev_info)
 224 {
 225         int set_first;
 226         unsigned long lowest_addr;
 227         struct segment_info *entry;
 228
 229         set_first = 0;
 230         lowest_addr = 0;
 231         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 232                 if (set_first == 0) {
 233                         lowest_addr = entry->start;
 234                         set_first = 1;
 235                 } else {
 236                         if (lowest_addr > entry->start)
 237                                 lowest_addr = entry->start;
 238                 }
 239         }
 240         return lowest_addr;
 241 }
 242
 243 /*
 244  * Check continuity of segments.
 245  */
 246 static int
 247 dcssblk_is_continuous(struct dcssblk_dev_info *dev_info)
 248 {
 249         int i, j, rc;
 250         struct segment_info *sort_list, *entry, temp;
 251
 252         if (dev_info->num_of_segments <= 1)
 253                 return 0;
 254
 255         sort_list = kcalloc(dev_info->num_of_segments,
 256                             sizeof(struct segment_info),
 257                             GFP_KERNEL);
 258         if (sort_list == NULL)
 259                 return -ENOMEM;
 260         i = 0;
 261         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 262                 memcpy(&sort_list[i], entry, sizeof(struct segment_info));
 263                 i++;
 264         }
 265
 266         /* sort segments */
 267         for (i = 0; i < dev_info->num_of_segments; i++)
 268                 for (j = 0; j < dev_info->num_of_segments; j++)
 269                         if (sort_list[j].start > sort_list[i].start) {
 270                                 memcpy(&temp, &sort_list[i],
 271                                         sizeof(struct segment_info));
 272                                 memcpy(&sort_list[i], &sort_list[j],
 273                                         sizeof(struct segment_info));
 274                                 memcpy(&sort_list[j], &temp,
 275                                         sizeof(struct segment_info));
 276                         }
 277
 278         /* check continuity */
 279         for (i = 0; i < dev_info->num_of_segments - 1; i++) {
 280                 if ((sort_list[i].end + 1) != sort_list[i+1].start) {
 281                         pr_err("Adjacent DCSSs %s and %s are not "
 282                                "contiguous\n", sort_list[i].segment_name,
 283                                sort_list[i+1].segment_name);
 284                         rc = -EINVAL;
 285                         goto out;
 286                 }
 287                 /* EN and EW are allowed in a block device */
 288                 if (sort_list[i].segment_type != sort_list[i+1].segment_type) {
 289                         if (!(sort_list[i].segment_type & SEGMENT_EXCLUSIVE) ||
 290                                 (sort_list[i].segment_type == SEG_TYPE_ER) ||
 291                                 !(sort_list[i+1].segment_type &
 292                                 SEGMENT_EXCLUSIVE) ||
 293                                 (sort_list[i+1].segment_type == SEG_TYPE_ER)) {
 294                                 pr_err("DCSS %s and DCSS %s have "
 295                                        "incompatible types\n",
 296                                        sort_list[i].segment_name,
 297                                        sort_list[i+1].segment_name);
 298                                 rc = -EINVAL;
 299                                 goto out;
 300                         }
 301                 }
 302         }
 303         rc = 0;
 304 out:
 305         kfree(sort_list);
 306         return rc;
 307 }
 308
 309 /*
 310  * Load a segment
 311  */
 312 static int
 313 dcssblk_load_segment(char *name, struct segment_info **seg_info)
 314 {
 315         int rc;
 316
 317         /* already loaded? */
 318         down_read(&dcssblk_devices_sem);
 319         *seg_info = dcssblk_get_segment_by_name(name);
 320         up_read(&dcssblk_devices_sem);
 321         if (*seg_info != NULL)
 322                 return -EEXIST;
 323
 324         /* get a struct segment_info */
 325         *seg_info = kzalloc(sizeof(struct segment_info), GFP_KERNEL);
 326         if (*seg_info == NULL)
 327                 return -ENOMEM;
 328
 329         strcpy((*seg_info)->segment_name, name);
 330
 331         /* load the segment */
 332         rc = segment_load(name, SEGMENT_SHARED,
 333                         &(*seg_info)->start, &(*seg_info)->end);
 334         if (rc < 0) {
 335                 segment_warning(rc, (*seg_info)->segment_name);
 336                 kfree(*seg_info);
 337         } else {
 338                 INIT_LIST_HEAD(&(*seg_info)->lh);
 339                 (*seg_info)->segment_type = rc;
 340         }
 341         return rc;
 342 }
 343
 344 /*
 345  * device attribute for switching shared/nonshared (exclusive)
 346  * operation (show + store)
 347  */
 348 static ssize_t
 349 dcssblk_shared_show(struct device *dev, struct device_attribute *attr, char *buf)
 350 {
 351         struct dcssblk_dev_info *dev_info;
 352
 353         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 354         return sprintf(buf, dev_info->is_shared ? "1\n" : "0\n");
 355 }
 356
 357 static ssize_t
 358 dcssblk_shared_store(struct device *dev, struct device_attribute *attr, const char *inbuf, size_t count)
 359 {
 360         struct dcssblk_dev_info *dev_info;
 361         struct segment_info *entry, *temp;
 362         int rc;
 363
 364         if ((count > 1) && (inbuf[1] != '\n') && (inbuf[1] != '\0'))
 365                 return -EINVAL;
 366         down_write(&dcssblk_devices_sem);
 367         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 368         if (atomic_read(&dev_info->use_count)) {
 369                 rc = -EBUSY;
 370                 goto out;
 371         }
 372         if (inbuf[0] == '1') {
 373                 /* reload segments in shared mode */
 374                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 375                         rc = segment_modify_shared(entry->segment_name,
 376                                                 SEGMENT_SHARED);
 377                         if (rc < 0) {
 378                                 BUG_ON(rc == -EINVAL);
 379                                 if (rc != -EAGAIN)
 380                                         goto removeseg;
 381                         }
 382                 }
 383                 dev_info->is_shared = 1;
 384                 switch (dev_info->segment_type) {
 385                 case SEG_TYPE_SR:
 386                 case SEG_TYPE_ER:
 387                 case SEG_TYPE_SC:
 388                         set_disk_ro(dev_info->gd, 1);
 389                 }
 390         } else if (inbuf[0] == '0') {
 391                 /* reload segments in exclusive mode */
 392                 if (dev_info->segment_type == SEG_TYPE_SC) {
 393                         pr_err("DCSS %s is of type SC and cannot be "
 394                                "loaded as exclusive-writable\n",
 395                                dev_info->segment_name);
 396                         rc = -EINVAL;
 397                         goto out;
 398                 }
 399                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 400                         rc = segment_modify_shared(entry->segment_name,
 401                                                    SEGMENT_EXCLUSIVE);
 402                         if (rc < 0) {
 403                                 BUG_ON(rc == -EINVAL);
 404                                 if (rc != -EAGAIN)
 405                                         goto removeseg;
 406                         }
 407                 }
 408                 dev_info->is_shared = 0;
 409                 set_disk_ro(dev_info->gd, 0);
 410         } else {
 411                 rc = -EINVAL;
 412                 goto out;
 413         }
 414         rc = count;
 415         goto out;
 416
 417 removeseg:
 418         pr_err("DCSS device %s is removed after a failed access mode "
 419                "change\n", dev_info->segment_name);
 420         temp = entry;
 421         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 422                 if (entry != temp)
 423                         segment_unload(entry->segment_name);
 424         }
 425         list_del(&dev_info->lh);
 426
 427         kill_dax(dev_info->dax_dev);
 428         put_dax(dev_info->dax_dev);
 429         del_gendisk(dev_info->gd);
 430         blk_cleanup_disk(dev_info->gd);
 431         up_write(&dcssblk_devices_sem);
 432
 433         if (device_remove_file_self(dev, attr)) {
 434                 device_unregister(dev);
 435                 put_device(dev);
 436         }
 437         return rc;
 438 out:
 439         up_write(&dcssblk_devices_sem);
 440         return rc;
 441 }
 442 static DEVICE_ATTR(shared, S_IWUSR | S_IRUSR, dcssblk_shared_show,
 443                    dcssblk_shared_store);
 444
 445 /*
 446  * device attribute for save operation on current copy
 447  * of the segment. If the segment is busy, saving will
 448  * become pending until it gets released, which can be
 449  * undone by storing a non-true value to this entry.
 450  * (show + store)
 451  */
 452 static ssize_t
 453 dcssblk_save_show(struct device *dev, struct device_attribute *attr, char *buf)
 454 {
 455         struct dcssblk_dev_info *dev_info;
 456
 457         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 458         return sprintf(buf, dev_info->save_pending ? "1\n" : "0\n");
 459 }
 460
 461 static ssize_t
 462 dcssblk_save_store(struct device *dev, struct device_attribute *attr, const char *inbuf, size_t count)
 463 {
 464         struct dcssblk_dev_info *dev_info;
 465         struct segment_info *entry;
 466
 467         if ((count > 1) && (inbuf[1] != '\n') && (inbuf[1] != '\0'))
 468                 return -EINVAL;
 469         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 470
 471         down_write(&dcssblk_devices_sem);
 472         if (inbuf[0] == '1') {
 473                 if (atomic_read(&dev_info->use_count) == 0) {
 474                         // device is idle => we save immediately
 475                         pr_info("All DCSSs that map to device %s are "
 476                                 "saved\n", dev_info->segment_name);
 477                         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 478                                 if (entry->segment_type == SEG_TYPE_EN ||
 479                                     entry->segment_type == SEG_TYPE_SN)
 480                                         pr_warn("DCSS %s is of type SN or EN"
 481                                                 " and cannot be saved\n",
 482                                                 entry->segment_name);
 483                                 else
 484                                         segment_save(entry->segment_name);
 485                         }
 486                 }  else {
 487                         // device is busy => we save it when it becomes
 488                         // idle in dcssblk_release
 489                         pr_info("Device %s is in use, its DCSSs will be "
 490                                 "saved when it becomes idle\n",
 491                                 dev_info->segment_name);
 492                         dev_info->save_pending = 1;
 493                 }
 494         } else if (inbuf[0] == '0') {
 495                 if (dev_info->save_pending) {
 496                         // device is busy & the user wants to undo his save
 497                         // request
 498                         dev_info->save_pending = 0;
 499                         pr_info("A pending save request for device %s "
 500                                 "has been canceled\n",
 501                                 dev_info->segment_name);
 502                 }
 503         } else {
 504                 up_write(&dcssblk_devices_sem);
 505                 return -EINVAL;
 506         }
 507         up_write(&dcssblk_devices_sem);
 508         return count;
 509 }
 510 static DEVICE_ATTR(save, S_IWUSR | S_IRUSR, dcssblk_save_show,
 511                    dcssblk_save_store);
 512
 513 /*
 514  * device attribute for showing all segments in a device
 515  */
 516 static ssize_t
 517 dcssblk_seglist_show(struct device *dev, struct device_attribute *attr,
 518                 char *buf)
 519 {
 520         int i;
 521
 522         struct dcssblk_dev_info *dev_info;
 523         struct segment_info *entry;
 524
 525         down_read(&dcssblk_devices_sem);
 526         dev_info = container_of(dev, struct dcssblk_dev_info, dev);
 527         i = 0;
 528         buf[0] = '\0';
 529         list_for_each_entry(entry, &dev_info->seg_list, lh) {
 530                 strcpy(&buf[i], entry->segment_name);
 531                 i += strlen(entry->segment_name);
 532                 buf[i] = '\n';
 533                 i++;
 534         }
 535         up_read(&dcssblk_devices_sem);
 536         return i;
 537 }
 538 static DEVICE_ATTR(seglist, S_IRUSR, dcssblk_seglist_show, NULL);
 539
 540 static struct attribute *dcssblk_dev_attrs[] = {
 541         &dev_attr_shared.attr,
 542         &dev_attr_save.attr,
 543         &dev_attr_seglist.attr,
 544         NULL,
 545 };
 546 static struct attribute_group dcssblk_dev_attr_group = {
 547         .attrs = dcssblk_dev_attrs,
 548 };
 549 static const struct attribute_group *dcssblk_dev_attr_groups[] = {
 550         &dcssblk_dev_attr_group,
 551         NULL,
 552 };
 553
 554 /*
 555  * device attribute for adding devices
 556  */
 557 static ssize_t
 558 dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
 559 {
 560         int rc, i, j, num_of_segments;
 561         struct dcssblk_dev_info *dev_info;
 562         struct segment_info *seg_info, *temp;
 563         char *local_buf;
 564         unsigned long seg_byte_size;
 565
 566         dev_info = NULL;
 567         seg_info = NULL;
 568         if (dev != dcssblk_root_dev) {
 569                 rc = -EINVAL;
 570                 goto out_nobuf;
 571         }
 572         if ((count < 1) || (buf[0] == '\0') || (buf[0] == '\n')) {
 573                 rc = -ENAMETOOLONG;
 574                 goto out_nobuf;
 575         }
 576
 577         local_buf = kmalloc(count + 1, GFP_KERNEL);
 578         if (local_buf == NULL) {
 579                 rc = -ENOMEM;
 580                 goto out_nobuf;
 581         }
 582
 583         /*
 584          * parse input
 585          */
 586         num_of_segments = 0;
 587         for (i = 0; (i < count && (buf[i] != '\0') && (buf[i] != '\n')); i++) {
 588                 for (j = i; j < count &&
 589                         (buf[j] != ':') &&
 590                         (buf[j] != '\0') &&
 591                         (buf[j] != '\n'); j++) {
 592                         local_buf[j-i] = toupper(buf[j]);
 593                 }
 594                 local_buf[j-i] = '\0';
 595                 if (((j - i) == 0) || ((j - i) > 8)) {
 596                         rc = -ENAMETOOLONG;
 597                         goto seg_list_del;
 598                 }
 599
 600                 rc = dcssblk_load_segment(local_buf, &seg_info);
 601                 if (rc < 0)
 602                         goto seg_list_del;
 603                 /*
 604                  * get a struct dcssblk_dev_info
 605                  */
 606                 if (num_of_segments == 0) {
 607                         dev_info = kzalloc(sizeof(struct dcssblk_dev_info),
 608                                         GFP_KERNEL);
 609                         if (dev_info == NULL) {
 610                                 rc = -ENOMEM;
 611                                 goto out;
 612                         }
 613                         strcpy(dev_info->segment_name, local_buf);
 614                         dev_info->segment_type = seg_info->segment_type;
 615                         INIT_LIST_HEAD(&dev_info->seg_list);
 616                 }
 617                 list_add_tail(&seg_info->lh, &dev_info->seg_list);
 618                 num_of_segments++;
 619                 i = j;
 620
 621                 if ((buf[j] == '\0') || (buf[j] == '\n'))
 622                         break;
 623         }
 624
 625         /* no trailing colon at the end of the input */
 626         if ((i > 0) && (buf[i-1] == ':')) {
 627                 rc = -ENAMETOOLONG;
 628                 goto seg_list_del;
 629         }
 630         strlcpy(local_buf, buf, i + 1);
 631         dev_info->num_of_segments = num_of_segments;
 632         rc = dcssblk_is_continuous(dev_info);
 633         if (rc < 0)
 634                 goto seg_list_del;
 635
 636         dev_info->start = dcssblk_find_lowest_addr(dev_info);
 637         dev_info->end = dcssblk_find_highest_addr(dev_info);
 638
 639         dev_set_name(&dev_info->dev, "%s", dev_info->segment_name);
 640         dev_info->dev.release = dcssblk_release_segment;
 641         dev_info->dev.groups = dcssblk_dev_attr_groups;
 642         INIT_LIST_HEAD(&dev_info->lh);
 643         dev_info->gd = blk_alloc_disk(NUMA_NO_NODE);
 644         if (dev_info->gd == NULL) {
 645                 rc = -ENOMEM;
 646                 goto seg_list_del;
 647         }
 648         dev_info->gd->major = dcssblk_major;
 649         dev_info->gd->minors = DCSSBLK_MINORS_PER_DISK;
 650         dev_info->gd->fops = &dcssblk_devops;
 651         dev_info->gd->private_data = dev_info;
 652         blk_queue_logical_block_size(dev_info->gd->queue, 4096);
 653         blk_queue_flag_set(QUEUE_FLAG_DAX, dev_info->gd->queue);
 654
 655         seg_byte_size = (dev_info->end - dev_info->start + 1);
 656         set_capacity(dev_info->gd, seg_byte_size >> 9); // size in sectors
 657         pr_info("Loaded %s with total size %lu bytes and capacity %lu "
 658                 "sectors\n", local_buf, seg_byte_size, seg_byte_size >> 9);
 659
 660         dev_info->save_pending = 0;
 661         dev_info->is_shared = 1;
 662         dev_info->dev.parent = dcssblk_root_dev;
 663
 664         /*
 665          *get minor, add to list
 666          */
 667         down_write(&dcssblk_devices_sem);
 668         if (dcssblk_get_segment_by_name(local_buf)) {
 669                 rc = -EEXIST;
 670                 goto release_gd;
 671         }
 672         rc = dcssblk_assign_free_minor(dev_info);
 673         if (rc)
 674                 goto release_gd;
 675         sprintf(dev_info->gd->disk_name, "dcssblk%d",
 676                 dev_info->gd->first_minor);
 677         list_add_tail(&dev_info->lh, &dcssblk_devices);
 678
 679         if (!try_module_get(THIS_MODULE)) {
 680                 rc = -ENODEV;
 681                 goto dev_list_del;
 682         }
 683         /*
 684          * register the device
 685          */
 686         rc = device_register(&dev_info->dev);
 687         if (rc)
 688                 goto put_dev;
 689
 690         dev_info->dax_dev = alloc_dax(dev_info, dev_info->gd->disk_name,
 691                         &dcssblk_dax_ops, DAXDEV_F_SYNC);
 692         if (IS_ERR(dev_info->dax_dev)) {
 693                 rc = PTR_ERR(dev_info->dax_dev);
 694                 dev_info->dax_dev = NULL;
 695                 goto put_dev;
 696         }
 697
 698         get_device(&dev_info->dev);
 699         device_add_disk(&dev_info->dev, dev_info->gd, NULL);
 700
 701         switch (dev_info->segment_type) {
 702                 case SEG_TYPE_SR:
 703                 case SEG_TYPE_ER:
 704                 case SEG_TYPE_SC:
 705                         set_disk_ro(dev_info->gd,1);
 706                         break;
 707                 default:
 708                         set_disk_ro(dev_info->gd,0);
 709                         break;
 710         }
 711         up_write(&dcssblk_devices_sem);
 712         rc = count;
 713         goto out;
 714
 715 put_dev:
 716         list_del(&dev_info->lh);
 717         blk_cleanup_disk(dev_info->gd);
 718         list_for_each_entry(seg_info, &dev_info->seg_list, lh) {
 719                 segment_unload(seg_info->segment_name);
 720         }
 721         put_device(&dev_info->dev);
 722         up_write(&dcssblk_devices_sem);
 723         goto out;
 724 dev_list_del:
 725         list_del(&dev_info->lh);
 726 release_gd:
 727         blk_cleanup_disk(dev_info->gd);
 728         up_write(&dcssblk_devices_sem);
 729 seg_list_del:
 730         if (dev_info == NULL)
 731                 goto out;
 732         list_for_each_entry_safe(seg_info, temp, &dev_info->seg_list, lh) {
 733                 list_del(&seg_info->lh);
 734                 segment_unload(seg_info->segment_name);
 735                 kfree(seg_info);
 736         }
 737         kfree(dev_info);
 738 out:
 739         kfree(local_buf);
 740 out_nobuf:
 741         return rc;
 742 }
 743
 744 /*
 745  * device attribute for removing devices
 746  */
 747 static ssize_t
 748 dcssblk_remove_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
 749 {
 750         struct dcssblk_dev_info *dev_info;
 751         struct segment_info *entry;
 752         int rc, i;
 753         char *local_buf;
 754
 755         if (dev != dcssblk_root_dev) {
 756                 return -EINVAL;
 757         }
 758         local_buf = kmalloc(count + 1, GFP_KERNEL);
 759         if (local_buf == NULL) {
 760                 return -ENOMEM;
 761         }
 762         /*
 763          * parse input
 764          */
 765         for (i = 0; (i < count && (*(buf+i)!='\0') && (*(buf+i)!='\n')); i++) {
 766                 local_buf[i] = toupper(buf[i]);
 767         }
 768         local_buf[i] = '\0';
 769         if ((i == 0) || (i > 8)) {
 770                 rc = -ENAMETOOLONG;
 771                 goto out_buf;
 772         }
 773
 774         down_write(&dcssblk_devices_sem);
 775         dev_info = dcssblk_get_device_by_name(local_buf);
 776         if (dev_info == NULL) {
 777                 up_write(&dcssblk_devices_sem);
 778                 pr_warn("Device %s cannot be removed because it is not a known device\n",
 779                         local_buf);
 780                 rc = -ENODEV;
 781                 goto out_buf;
 782         }
 783         if (atomic_read(&dev_info->use_count) != 0) {
 784                 up_write(&dcssblk_devices_sem);
 785                 pr_warn("Device %s cannot be removed while it is in use\n",
 786                         local_buf);
 787                 rc = -EBUSY;
 788                 goto out_buf;
 789         }
 790
 791         list_del(&dev_info->lh);
 792         kill_dax(dev_info->dax_dev);
 793         put_dax(dev_info->dax_dev);
 794         del_gendisk(dev_info->gd);
 795         blk_cleanup_disk(dev_info->gd);
 796
 797         /* unload all related segments */
 798         list_for_each_entry(entry, &dev_info->seg_list, lh)
 799                 segment_unload(entry->segment_name);
 800
 801         up_write(&dcssblk_devices_sem);
 802
 803         device_unregister(&dev_info->dev);
 804         put_device(&dev_info->dev);
 805
 806         rc = count;
 807 out_buf:
 808         kfree(local_buf);
 809         return rc;
 810 }
 811
 812 static int
 813 dcssblk_open(struct block_device *bdev, fmode_t mode)
 814 {
 815         struct dcssblk_dev_info *dev_info;
 816         int rc;
 817
 818         dev_info = bdev->bd_disk->private_data;
 819         if (NULL == dev_info) {
 820                 rc = -ENODEV;
 821                 goto out;
 822         }
 823         atomic_inc(&dev_info->use_count);
 824         rc = 0;
 825 out:
 826         return rc;
 827 }
 828
 829 static void
 830 dcssblk_release(struct gendisk *disk, fmode_t mode)
 831 {
 832         struct dcssblk_dev_info *dev_info = disk->private_data;
 833         struct segment_info *entry;
 834
 835         if (!dev_info) {
 836                 WARN_ON(1);
 837                 return;
 838         }
 839         down_write(&dcssblk_devices_sem);
 840         if (atomic_dec_and_test(&dev_info->use_count)
 841             && (dev_info->save_pending)) {
 842                 pr_info("Device %s has become idle and is being saved "
 843                         "now\n", dev_info->segment_name);
 844                 list_for_each_entry(entry, &dev_info->seg_list, lh) {
 845                         if (entry->segment_type == SEG_TYPE_EN ||
 846                             entry->segment_type == SEG_TYPE_SN)
 847                                 pr_warn("DCSS %s is of type SN or EN and cannot"
 848                                         " be saved\n", entry->segment_name);
 849                         else
 850                                 segment_save(entry->segment_name);
 851                 }
 852                 dev_info->save_pending = 0;
 853         }
 854         up_write(&dcssblk_devices_sem);
 855 }
 856
 857 static blk_qc_t
 858 dcssblk_submit_bio(struct bio *bio)
 859 {
 860         struct dcssblk_dev_info *dev_info;
 861         struct bio_vec bvec;
 862         struct bvec_iter iter;
 863         unsigned long index;
 864         unsigned long page_addr;
 865         unsigned long source_addr;
 866         unsigned long bytes_done;
 867
 868         blk_queue_split(&bio);
 869
 870         bytes_done = 0;
 871         dev_info = bio->bi_bdev->bd_disk->private_data;
 872         if (dev_info == NULL)
 873                 goto fail;
 874         if ((bio->bi_iter.bi_sector & 7) != 0 ||
 875             (bio->bi_iter.bi_size & 4095) != 0)
 876                 /* Request is not page-aligned. */
 877                 goto fail;
 878         /* verify data transfer direction */
 879         if (dev_info->is_shared) {
 880                 switch (dev_info->segment_type) {
 881                 case SEG_TYPE_SR:
 882                 case SEG_TYPE_ER:
 883                 case SEG_TYPE_SC:
 884                         /* cannot write to these segments */
 885                         if (bio_data_dir(bio) == WRITE) {
 886                                 pr_warn("Writing to %s failed because it is a read-only device\n",
 887                                         dev_name(&dev_info->dev));
 888                                 goto fail;
 889                         }
 890                 }
 891         }
 892
 893         index = (bio->bi_iter.bi_sector >> 3);
 894         bio_for_each_segment(bvec, bio, iter) {
 895                 page_addr = (unsigned long)bvec_virt(&bvec);
 896                 source_addr = dev_info->start + (index<<12) + bytes_done;
 897                 if (unlikely((page_addr & 4095) != 0) || (bvec.bv_len & 4095) != 0)
 898                         // More paranoia.
 899                         goto fail;
 900                 if (bio_data_dir(bio) == READ) {
 901                         memcpy((void*)page_addr, (void*)source_addr,
 902                                 bvec.bv_len);
 903                 } else {
 904                         memcpy((void*)source_addr, (void*)page_addr,
 905                                 bvec.bv_len);
 906                 }
 907                 bytes_done += bvec.bv_len;
 908         }
 909         bio_endio(bio);
 910         return BLK_QC_T_NONE;
 911 fail:
 912         bio_io_error(bio);
 913         return BLK_QC_T_NONE;
 914 }
 915
 916 static long
 917 __dcssblk_direct_access(struct dcssblk_dev_info *dev_info, pgoff_t pgoff,
 918                 long nr_pages, void **kaddr, pfn_t *pfn)
 919 {
 920         resource_size_t offset = pgoff * PAGE_SIZE;
 921         unsigned long dev_sz;
 922
 923         dev_sz = dev_info->end - dev_info->start + 1;
 924         if (kaddr)
 925                 *kaddr = (void *) dev_info->start + offset;
 926         if (pfn)
 927                 *pfn = __pfn_to_pfn_t(PFN_DOWN(dev_info->start + offset),
 928                                 PFN_DEV|PFN_SPECIAL);
 929
 930         return (dev_sz - offset) / PAGE_SIZE;
 931 }
 932
 933 static long
 934 dcssblk_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
 935                 long nr_pages, void **kaddr, pfn_t *pfn)
 936 {
 937         struct dcssblk_dev_info *dev_info = dax_get_private(dax_dev);
 938
 939         return __dcssblk_direct_access(dev_info, pgoff, nr_pages, kaddr, pfn);
 940 }
 941
 942 static void
 943 dcssblk_check_params(void)
 944 {
 945         int rc, i, j, k;
 946         char buf[DCSSBLK_PARM_LEN + 1];
 947         struct dcssblk_dev_info *dev_info;
 948
 949         for (i = 0; (i < DCSSBLK_PARM_LEN) && (dcssblk_segments[i] != '\0');
 950              i++) {
 951                 for (j = i; (j < DCSSBLK_PARM_LEN) &&
 952                             (dcssblk_segments[j] != ',')  &&
 953                             (dcssblk_segments[j] != '\0') &&
 954                             (dcssblk_segments[j] != '('); j++)
 955                 {
 956                         buf[j-i] = dcssblk_segments[j];
 957                 }
 958                 buf[j-i] = '\0';
 959                 rc = dcssblk_add_store(dcssblk_root_dev, NULL, buf, j-i);
 960                 if ((rc >= 0) && (dcssblk_segments[j] == '(')) {
 961                         for (k = 0; (buf[k] != ':') && (buf[k] != '\0'); k++)
 962                                 buf[k] = toupper(buf[k]);
 963                         buf[k] = '\0';
 964                         if (!strncmp(&dcssblk_segments[j], "(local)", 7)) {
 965                                 down_read(&dcssblk_devices_sem);
 966                                 dev_info = dcssblk_get_device_by_name(buf);
 967                                 up_read(&dcssblk_devices_sem);
 968                                 if (dev_info)
 969                                         dcssblk_shared_store(&dev_info->dev,
 970                                                              NULL, "0\n", 2);
 971                         }
 972                 }
 973                 while ((dcssblk_segments[j] != ',') &&
 974                        (dcssblk_segments[j] != '\0'))
 975                 {
 976                         j++;
 977                 }
 978                 if (dcssblk_segments[j] == '\0')
 979                         break;
 980                 i = j;
 981         }
 982 }
 983
 984 /*
 985  * The init/exit functions.
 986  */
 987 static void __exit
 988 dcssblk_exit(void)
 989 {
 990         root_device_unregister(dcssblk_root_dev);
 991         unregister_blkdev(dcssblk_major, DCSSBLK_NAME);
 992 }
 993
 994 static int __init
 995 dcssblk_init(void)
 996 {
 997         int rc;
 998
 999         dcssblk_root_dev = root_device_register("dcssblk");
1000         if (IS_ERR(dcssblk_root_dev))
1001                 return PTR_ERR(dcssblk_root_dev);
1002         rc = device_create_file(dcssblk_root_dev, &dev_attr_add);
1003         if (rc)
1004                 goto out_root;
1005         rc = device_create_file(dcssblk_root_dev, &dev_attr_remove);
1006         if (rc)
1007                 goto out_root;
1008         rc = register_blkdev(0, DCSSBLK_NAME);
1009         if (rc < 0)
1010                 goto out_root;
1011         dcssblk_major = rc;
1012         init_rwsem(&dcssblk_devices_sem);
1013
1014         dcssblk_check_params();
1015         return 0;
1016
1017 out_root:
1018         root_device_unregister(dcssblk_root_dev);
1019
1020         return rc;
1021 }
1022
1023 module_init(dcssblk_init);
1024 module_exit(dcssblk_exit);
1025
1026 module_param_string(segments, dcssblk_segments, DCSSBLK_PARM_LEN, 0444);
1027 MODULE_PARM_DESC(segments, "Name of DCSS segment(s) to be loaded, "
1028                  "comma-separated list, names in each set separated "
1029                  "by commas are separated by colons, each set contains "
1030                  "names of contiguous segments and each name max. 8 chars.\n"
1031                  "Adding \"(local)\" to the end of each set equals echoing 0 "
1032                  "to /sys/devices/dcssblk/<device name>/shared after loading "
1033                  "the contiguous segments - \n"
1034                  "e.g. segments=\"mydcss1,mydcss2:mydcss3,mydcss4(local)\"");
1035
1036 MODULE_LICENSE("GPL");