drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-rq.h"
  11 #include "dm-bio-record.h"
  12 #include "dm-path-selector.h"
  13 #include "dm-uevent.h"
  14
  15 #include <linux/blkdev.h>
  16 #include <linux/ctype.h>
  17 #include <linux/init.h>
  18 #include <linux/mempool.h>
  19 #include <linux/module.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/slab.h>
  22 #include <linux/time.h>
  23 #include <linux/timer.h>
  24 #include <linux/workqueue.h>
  25 #include <linux/delay.h>
  26 #include <scsi/scsi_dh.h>
  27 #include <linux/atomic.h>
  28 #include <linux/blk-mq.h>
  29
  30 #define DM_MSG_PREFIX "multipath"
  31 #define DM_PG_INIT_DELAY_MSECS 2000
  32 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
  33 #define QUEUE_IF_NO_PATH_TIMEOUT_DEFAULT 0
  34
  35 static unsigned long queue_if_no_path_timeout_secs = QUEUE_IF_NO_PATH_TIMEOUT_DEFAULT;
  36
  37 /* Path properties */
  38 struct pgpath {
  39         struct list_head list;
  40
  41         struct priority_group *pg;      /* Owning PG */
  42         unsigned fail_count;            /* Cumulative failure count */
  43
  44         struct dm_path path;
  45         struct delayed_work activate_path;
  46
  47         bool is_active:1;               /* Path status */
  48 };
  49
  50 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  51
  52 /*
  53  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  54  * Each has a path selector which controls which path gets used.
  55  */
  56 struct priority_group {
  57         struct list_head list;
  58
  59         struct multipath *m;            /* Owning multipath instance */
  60         struct path_selector ps;
  61
  62         unsigned pg_num;                /* Reference number */
  63         unsigned nr_pgpaths;            /* Number of paths in PG */
  64         struct list_head pgpaths;
  65
  66         bool bypassed:1;                /* Temporarily bypass this PG? */
  67 };
  68
  69 /* Multipath context */
  70 struct multipath {
  71         unsigned long flags;            /* Multipath state flags */
  72
  73         spinlock_t lock;
  74         enum dm_queue_mode queue_mode;
  75
  76         struct pgpath *current_pgpath;
  77         struct priority_group *current_pg;
  78         struct priority_group *next_pg; /* Switch to this PG if set */
  79
  80         atomic_t nr_valid_paths;        /* Total number of usable paths */
  81         unsigned nr_priority_groups;
  82         struct list_head priority_groups;
  83
  84         const char *hw_handler_name;
  85         char *hw_handler_params;
  86         wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  87         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  88         unsigned pg_init_delay_msecs;   /* Number of msecs before pg_init retry */
  89         atomic_t pg_init_in_progress;   /* Only one pg_init allowed at once */
  90         atomic_t pg_init_count;         /* Number of times pg_init called */
  91
  92         struct mutex work_mutex;
  93         struct work_struct trigger_event;
  94         struct dm_target *ti;
  95
  96         struct work_struct process_queued_bios;
  97         struct bio_list queued_bios;
  98
  99         struct timer_list nopath_timer; /* Timeout for queue_if_no_path */
 100 };
 101
 102 /*
 103  * Context information attached to each io we process.
 104  */
 105 struct dm_mpath_io {
 106         struct pgpath *pgpath;
 107         size_t nr_bytes;
 108 };
 109
 110 typedef int (*action_fn) (struct pgpath *pgpath);
 111
 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 113 static void trigger_event(struct work_struct *work);
 114 static void activate_or_offline_path(struct pgpath *pgpath);
 115 static void activate_path_work(struct work_struct *work);
 116 static void process_queued_bios(struct work_struct *work);
 117 static void queue_if_no_path_timeout_work(struct timer_list *t);
 118
 119 /*-----------------------------------------------
 120  * Multipath state flags.
 121  *-----------------------------------------------*/
 122
 123 #define MPATHF_QUEUE_IO 0                       /* Must we queue all I/O? */
 124 #define MPATHF_QUEUE_IF_NO_PATH 1               /* Queue I/O if last path fails? */
 125 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2         /* Saved state during suspension */
 126 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3     /* If there's already a hw_handler present, don't change it. */
 127 #define MPATHF_PG_INIT_DISABLED 4               /* pg_init is not currently allowed */
 128 #define MPATHF_PG_INIT_REQUIRED 5               /* pg_init needs calling? */
 129 #define MPATHF_PG_INIT_DELAY_RETRY 6            /* Delay pg_init retry? */
 130
 131 /*-----------------------------------------------
 132  * Allocation routines
 133  *-----------------------------------------------*/
 134
 135 static struct pgpath *alloc_pgpath(void)
 136 {
 137         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 138
 139         if (!pgpath)
 140                 return NULL;
 141
 142         pgpath->is_active = true;
 143
 144         return pgpath;
 145 }
 146
 147 static void free_pgpath(struct pgpath *pgpath)
 148 {
 149         kfree(pgpath);
 150 }
 151
 152 static struct priority_group *alloc_priority_group(void)
 153 {
 154         struct priority_group *pg;
 155
 156         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 157
 158         if (pg)
 159                 INIT_LIST_HEAD(&pg->pgpaths);
 160
 161         return pg;
 162 }
 163
 164 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 165 {
 166         struct pgpath *pgpath, *tmp;
 167
 168         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 169                 list_del(&pgpath->list);
 170                 dm_put_device(ti, pgpath->path.dev);
 171                 free_pgpath(pgpath);
 172         }
 173 }
 174
 175 static void free_priority_group(struct priority_group *pg,
 176                                 struct dm_target *ti)
 177 {
 178         struct path_selector *ps = &pg->ps;
 179
 180         if (ps->type) {
 181                 ps->type->destroy(ps);
 182                 dm_put_path_selector(ps->type);
 183         }
 184
 185         free_pgpaths(&pg->pgpaths, ti);
 186         kfree(pg);
 187 }
 188
 189 static struct multipath *alloc_multipath(struct dm_target *ti)
 190 {
 191         struct multipath *m;
 192
 193         m = kzalloc(sizeof(*m), GFP_KERNEL);
 194         if (m) {
 195                 INIT_LIST_HEAD(&m->priority_groups);
 196                 spin_lock_init(&m->lock);
 197                 atomic_set(&m->nr_valid_paths, 0);
 198                 INIT_WORK(&m->trigger_event, trigger_event);
 199                 mutex_init(&m->work_mutex);
 200
 201                 m->queue_mode = DM_TYPE_NONE;
 202
 203                 m->ti = ti;
 204                 ti->private = m;
 205
 206                 timer_setup(&m->nopath_timer, queue_if_no_path_timeout_work, 0);
 207         }
 208
 209         return m;
 210 }
 211
 212 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 213 {
 214         if (m->queue_mode == DM_TYPE_NONE) {
 215                 m->queue_mode = DM_TYPE_REQUEST_BASED;
 216         } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
 217                 INIT_WORK(&m->process_queued_bios, process_queued_bios);
 218                 /*
 219                  * bio-based doesn't support any direct scsi_dh management;
 220                  * it just discovers if a scsi_dh is attached.
 221                  */
 222                 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
 223         }
 224
 225         dm_table_set_type(ti->table, m->queue_mode);
 226
 227         /*
 228          * Init fields that are only used when a scsi_dh is attached
 229          * - must do this unconditionally (really doesn't hurt non-SCSI uses)
 230          */
 231         set_bit(MPATHF_QUEUE_IO, &m->flags);
 232         atomic_set(&m->pg_init_in_progress, 0);
 233         atomic_set(&m->pg_init_count, 0);
 234         m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 235         init_waitqueue_head(&m->pg_init_wait);
 236
 237         return 0;
 238 }
 239
 240 static void free_multipath(struct multipath *m)
 241 {
 242         struct priority_group *pg, *tmp;
 243
 244         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 245                 list_del(&pg->list);
 246                 free_priority_group(pg, m->ti);
 247         }
 248
 249         kfree(m->hw_handler_name);
 250         kfree(m->hw_handler_params);
 251         mutex_destroy(&m->work_mutex);
 252         kfree(m);
 253 }
 254
 255 static struct dm_mpath_io *get_mpio(union map_info *info)
 256 {
 257         return info->ptr;
 258 }
 259
 260 static size_t multipath_per_bio_data_size(void)
 261 {
 262         return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
 263 }
 264
 265 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
 266 {
 267         return dm_per_bio_data(bio, multipath_per_bio_data_size());
 268 }
 269
 270 static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
 271 {
 272         /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
 273         void *bio_details = mpio + 1;
 274         return bio_details;
 275 }
 276
 277 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
 278 {
 279         struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 280         struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
 281
 282         mpio->nr_bytes = bio->bi_iter.bi_size;
 283         mpio->pgpath = NULL;
 284         *mpio_p = mpio;
 285
 286         dm_bio_record(bio_details, bio);
 287 }
 288
 289 /*-----------------------------------------------
 290  * Path selection
 291  *-----------------------------------------------*/
 292
 293 static int __pg_init_all_paths(struct multipath *m)
 294 {
 295         struct pgpath *pgpath;
 296         unsigned long pg_init_delay = 0;
 297
 298         lockdep_assert_held(&m->lock);
 299
 300         if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
 301                 return 0;
 302
 303         atomic_inc(&m->pg_init_count);
 304         clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 305
 306         /* Check here to reset pg_init_required */
 307         if (!m->current_pg)
 308                 return 0;
 309
 310         if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
 311                 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
 312                                                  m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
 313         list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
 314                 /* Skip failed paths */
 315                 if (!pgpath->is_active)
 316                         continue;
 317                 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
 318                                        pg_init_delay))
 319                         atomic_inc(&m->pg_init_in_progress);
 320         }
 321         return atomic_read(&m->pg_init_in_progress);
 322 }
 323
 324 static int pg_init_all_paths(struct multipath *m)
 325 {
 326         int ret;
 327         unsigned long flags;
 328
 329         spin_lock_irqsave(&m->lock, flags);
 330         ret = __pg_init_all_paths(m);
 331         spin_unlock_irqrestore(&m->lock, flags);
 332
 333         return ret;
 334 }
 335
 336 static void __switch_pg(struct multipath *m, struct priority_group *pg)
 337 {
 338         lockdep_assert_held(&m->lock);
 339
 340         m->current_pg = pg;
 341
 342         /* Must we initialise the PG first, and queue I/O till it's ready? */
 343         if (m->hw_handler_name) {
 344                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 345                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 346         } else {
 347                 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 348                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 349         }
 350
 351         atomic_set(&m->pg_init_count, 0);
 352 }
 353
 354 static struct pgpath *choose_path_in_pg(struct multipath *m,
 355                                         struct priority_group *pg,
 356                                         size_t nr_bytes)
 357 {
 358         unsigned long flags;
 359         struct dm_path *path;
 360         struct pgpath *pgpath;
 361
 362         path = pg->ps.type->select_path(&pg->ps, nr_bytes);
 363         if (!path)
 364                 return ERR_PTR(-ENXIO);
 365
 366         pgpath = path_to_pgpath(path);
 367
 368         if (unlikely(READ_ONCE(m->current_pg) != pg)) {
 369                 /* Only update current_pgpath if pg changed */
 370                 spin_lock_irqsave(&m->lock, flags);
 371                 m->current_pgpath = pgpath;
 372                 __switch_pg(m, pg);
 373                 spin_unlock_irqrestore(&m->lock, flags);
 374         }
 375
 376         return pgpath;
 377 }
 378
 379 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
 380 {
 381         unsigned long flags;
 382         struct priority_group *pg;
 383         struct pgpath *pgpath;
 384         unsigned bypassed = 1;
 385
 386         if (!atomic_read(&m->nr_valid_paths)) {
 387                 spin_lock_irqsave(&m->lock, flags);
 388                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 389                 spin_unlock_irqrestore(&m->lock, flags);
 390                 goto failed;
 391         }
 392
 393         /* Were we instructed to switch PG? */
 394         if (READ_ONCE(m->next_pg)) {
 395                 spin_lock_irqsave(&m->lock, flags);
 396                 pg = m->next_pg;
 397                 if (!pg) {
 398                         spin_unlock_irqrestore(&m->lock, flags);
 399                         goto check_current_pg;
 400                 }
 401                 m->next_pg = NULL;
 402                 spin_unlock_irqrestore(&m->lock, flags);
 403                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 404                 if (!IS_ERR_OR_NULL(pgpath))
 405                         return pgpath;
 406         }
 407
 408         /* Don't change PG until it has no remaining paths */
 409 check_current_pg:
 410         pg = READ_ONCE(m->current_pg);
 411         if (pg) {
 412                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 413                 if (!IS_ERR_OR_NULL(pgpath))
 414                         return pgpath;
 415         }
 416
 417         /*
 418          * Loop through priority groups until we find a valid path.
 419          * First time we skip PGs marked 'bypassed'.
 420          * Second time we only try the ones we skipped, but set
 421          * pg_init_delay_retry so we do not hammer controllers.
 422          */
 423         do {
 424                 list_for_each_entry(pg, &m->priority_groups, list) {
 425                         if (pg->bypassed == !!bypassed)
 426                                 continue;
 427                         pgpath = choose_path_in_pg(m, pg, nr_bytes);
 428                         if (!IS_ERR_OR_NULL(pgpath)) {
 429                                 if (!bypassed) {
 430                                         spin_lock_irqsave(&m->lock, flags);
 431                                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
 432                                         spin_unlock_irqrestore(&m->lock, flags);
 433                                 }
 434                                 return pgpath;
 435                         }
 436                 }
 437         } while (bypassed--);
 438
 439 failed:
 440         spin_lock_irqsave(&m->lock, flags);
 441         m->current_pgpath = NULL;
 442         m->current_pg = NULL;
 443         spin_unlock_irqrestore(&m->lock, flags);
 444
 445         return NULL;
 446 }
 447
 448 /*
 449  * dm_report_EIO() is a macro instead of a function to make pr_debug_ratelimited()
 450  * report the function name and line number of the function from which
 451  * it has been invoked.
 452  */
 453 #define dm_report_EIO(m)                                                \
 454 do {                                                                    \
 455         struct mapped_device *md = dm_table_get_md((m)->ti->table);     \
 456                                                                         \
 457         DMDEBUG_LIMIT("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d", \
 458                       dm_device_name(md),                               \
 459                       test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags),   \
 460                       test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \
 461                       dm_noflush_suspending((m)->ti));                  \
 462 } while (0)
 463
 464 /*
 465  * Check whether bios must be queued in the device-mapper core rather
 466  * than here in the target.
 467  */
 468 static bool __must_push_back(struct multipath *m)
 469 {
 470         return dm_noflush_suspending(m->ti);
 471 }
 472
 473 static bool must_push_back_rq(struct multipath *m)
 474 {
 475         return test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) || __must_push_back(m);
 476 }
 477
 478 /*
 479  * Map cloned requests (request-based multipath)
 480  */
 481 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 482                                    union map_info *map_context,
 483                                    struct request **__clone)
 484 {
 485         struct multipath *m = ti->private;
 486         size_t nr_bytes = blk_rq_bytes(rq);
 487         struct pgpath *pgpath;
 488         struct block_device *bdev;
 489         struct dm_mpath_io *mpio = get_mpio(map_context);
 490         struct request_queue *q;
 491         struct request *clone;
 492
 493         /* Do we need to select a new pgpath? */
 494         pgpath = READ_ONCE(m->current_pgpath);
 495         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 496                 pgpath = choose_pgpath(m, nr_bytes);
 497
 498         if (!pgpath) {
 499                 if (must_push_back_rq(m))
 500                         return DM_MAPIO_DELAY_REQUEUE;
 501                 dm_report_EIO(m);       /* Failed */
 502                 return DM_MAPIO_KILL;
 503         } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 504                    test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
 505                 pg_init_all_paths(m);
 506                 return DM_MAPIO_DELAY_REQUEUE;
 507         }
 508
 509         mpio->pgpath = pgpath;
 510         mpio->nr_bytes = nr_bytes;
 511
 512         bdev = pgpath->path.dev->bdev;
 513         q = bdev_get_queue(bdev);
 514         clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE,
 515                         BLK_MQ_REQ_NOWAIT);
 516         if (IS_ERR(clone)) {
 517                 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 518                 if (blk_queue_dying(q)) {
 519                         atomic_inc(&m->pg_init_in_progress);
 520                         activate_or_offline_path(pgpath);
 521                         return DM_MAPIO_DELAY_REQUEUE;
 522                 }
 523
 524                 /*
 525                  * blk-mq's SCHED_RESTART can cover this requeue, so we
 526                  * needn't deal with it by DELAY_REQUEUE. More importantly,
 527                  * we have to return DM_MAPIO_REQUEUE so that blk-mq can
 528                  * get the queue busy feedback (via BLK_STS_RESOURCE),
 529                  * otherwise I/O merging can suffer.
 530                  */
 531                 return DM_MAPIO_REQUEUE;
 532         }
 533         clone->bio = clone->biotail = NULL;
 534         clone->rq_disk = bdev->bd_disk;
 535         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 536         *__clone = clone;
 537
 538         if (pgpath->pg->ps.type->start_io)
 539                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 540                                               &pgpath->path,
 541                                               nr_bytes);
 542         return DM_MAPIO_REMAPPED;
 543 }
 544
 545 static void multipath_release_clone(struct request *clone,
 546                                     union map_info *map_context)
 547 {
 548         if (unlikely(map_context)) {
 549                 /*
 550                  * non-NULL map_context means caller is still map
 551                  * method; must undo multipath_clone_and_map()
 552                  */
 553                 struct dm_mpath_io *mpio = get_mpio(map_context);
 554                 struct pgpath *pgpath = mpio->pgpath;
 555
 556                 if (pgpath && pgpath->pg->ps.type->end_io)
 557                         pgpath->pg->ps.type->end_io(&pgpath->pg->ps,
 558                                                     &pgpath->path,
 559                                                     mpio->nr_bytes,
 560                                                     clone->io_start_time_ns);
 561         }
 562
 563         blk_put_request(clone);
 564 }
 565
 566 /*
 567  * Map cloned bios (bio-based multipath)
 568  */
 569
 570 static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
 571 {
 572         struct pgpath *pgpath;
 573         unsigned long flags;
 574         bool queue_io;
 575
 576         /* Do we need to select a new pgpath? */
 577         pgpath = READ_ONCE(m->current_pgpath);
 578         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 579                 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
 580
 581         /* MPATHF_QUEUE_IO might have been cleared by choose_pgpath. */
 582         queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
 583
 584         if ((pgpath && queue_io) ||
 585             (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
 586                 /* Queue for the daemon to resubmit */
 587                 spin_lock_irqsave(&m->lock, flags);
 588                 bio_list_add(&m->queued_bios, bio);
 589                 spin_unlock_irqrestore(&m->lock, flags);
 590
 591                 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
 592                 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 593                         pg_init_all_paths(m);
 594                 else if (!queue_io)
 595                         queue_work(kmultipathd, &m->process_queued_bios);
 596
 597                 return ERR_PTR(-EAGAIN);
 598         }
 599
 600         return pgpath;
 601 }
 602
 603 static int __multipath_map_bio(struct multipath *m, struct bio *bio,
 604                                struct dm_mpath_io *mpio)
 605 {
 606         struct pgpath *pgpath = __map_bio(m, bio);
 607
 608         if (IS_ERR(pgpath))
 609                 return DM_MAPIO_SUBMITTED;
 610
 611         if (!pgpath) {
 612                 if (__must_push_back(m))
 613                         return DM_MAPIO_REQUEUE;
 614                 dm_report_EIO(m);
 615                 return DM_MAPIO_KILL;
 616         }
 617
 618         mpio->pgpath = pgpath;
 619
 620         bio->bi_status = 0;
 621         bio_set_dev(bio, pgpath->path.dev->bdev);
 622         bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 623
 624         if (pgpath->pg->ps.type->start_io)
 625                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 626                                               &pgpath->path,
 627                                               mpio->nr_bytes);
 628         return DM_MAPIO_REMAPPED;
 629 }
 630
 631 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 632 {
 633         struct multipath *m = ti->private;
 634         struct dm_mpath_io *mpio = NULL;
 635
 636         multipath_init_per_bio_data(bio, &mpio);
 637         return __multipath_map_bio(m, bio, mpio);
 638 }
 639
 640 static void process_queued_io_list(struct multipath *m)
 641 {
 642         if (m->queue_mode == DM_TYPE_REQUEST_BASED)
 643                 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
 644         else if (m->queue_mode == DM_TYPE_BIO_BASED)
 645                 queue_work(kmultipathd, &m->process_queued_bios);
 646 }
 647
 648 static void process_queued_bios(struct work_struct *work)
 649 {
 650         int r;
 651         unsigned long flags;
 652         struct bio *bio;
 653         struct bio_list bios;
 654         struct blk_plug plug;
 655         struct multipath *m =
 656                 container_of(work, struct multipath, process_queued_bios);
 657
 658         bio_list_init(&bios);
 659
 660         spin_lock_irqsave(&m->lock, flags);
 661
 662         if (bio_list_empty(&m->queued_bios)) {
 663                 spin_unlock_irqrestore(&m->lock, flags);
 664                 return;
 665         }
 666
 667         bio_list_merge(&bios, &m->queued_bios);
 668         bio_list_init(&m->queued_bios);
 669
 670         spin_unlock_irqrestore(&m->lock, flags);
 671
 672         blk_start_plug(&plug);
 673         while ((bio = bio_list_pop(&bios))) {
 674                 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 675                 dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
 676                 r = __multipath_map_bio(m, bio, mpio);
 677                 switch (r) {
 678                 case DM_MAPIO_KILL:
 679                         bio->bi_status = BLK_STS_IOERR;
 680                         bio_endio(bio);
 681                         break;
 682                 case DM_MAPIO_REQUEUE:
 683                         bio->bi_status = BLK_STS_DM_REQUEUE;
 684                         bio_endio(bio);
 685                         break;
 686                 case DM_MAPIO_REMAPPED:
 687                         generic_make_request(bio);
 688                         break;
 689                 case DM_MAPIO_SUBMITTED:
 690                         break;
 691                 default:
 692                         WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
 693                 }
 694         }
 695         blk_finish_plug(&plug);
 696 }
 697
 698 /*
 699  * If we run out of usable paths, should we queue I/O or error it?
 700  */
 701 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
 702                             bool save_old_value, const char *caller)
 703 {
 704         unsigned long flags;
 705         bool queue_if_no_path_bit, saved_queue_if_no_path_bit;
 706         const char *dm_dev_name = dm_device_name(dm_table_get_md(m->ti->table));
 707
 708         DMDEBUG("%s: %s caller=%s queue_if_no_path=%d save_old_value=%d",
 709                 dm_dev_name, __func__, caller, queue_if_no_path, save_old_value);
 710
 711         spin_lock_irqsave(&m->lock, flags);
 712
 713         queue_if_no_path_bit = test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
 714         saved_queue_if_no_path_bit = test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
 715
 716         if (save_old_value) {
 717                 if (unlikely(!queue_if_no_path_bit && saved_queue_if_no_path_bit)) {
 718                         DMERR("%s: QIFNP disabled but saved as enabled, saving again loses state, not saving!",
 719                               dm_dev_name);
 720                 } else
 721                         assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path_bit);
 722         } else if (!queue_if_no_path && saved_queue_if_no_path_bit) {
 723                 /* due to "fail_if_no_path" message, need to honor it. */
 724                 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
 725         }
 726         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
 727
 728         DMDEBUG("%s: after %s changes; QIFNP = %d; SQIFNP = %d; DNFS = %d",
 729                 dm_dev_name, __func__,
 730                 test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags),
 731                 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags),
 732                 dm_noflush_suspending(m->ti));
 733
 734         spin_unlock_irqrestore(&m->lock, flags);
 735
 736         if (!queue_if_no_path) {
 737                 dm_table_run_md_queue_async(m->ti->table);
 738                 process_queued_io_list(m);
 739         }
 740
 741         return 0;
 742 }
 743
 744 /*
 745  * If the queue_if_no_path timeout fires, turn off queue_if_no_path and
 746  * process any queued I/O.
 747  */
 748 static void queue_if_no_path_timeout_work(struct timer_list *t)
 749 {
 750         struct multipath *m = from_timer(m, t, nopath_timer);
 751         struct mapped_device *md = dm_table_get_md(m->ti->table);
 752
 753         DMWARN("queue_if_no_path timeout on %s, failing queued IO", dm_device_name(md));
 754         queue_if_no_path(m, false, false, __func__);
 755 }
 756
 757 /*
 758  * Enable the queue_if_no_path timeout if necessary.
 759  * Called with m->lock held.
 760  */
 761 static void enable_nopath_timeout(struct multipath *m)
 762 {
 763         unsigned long queue_if_no_path_timeout =
 764                 READ_ONCE(queue_if_no_path_timeout_secs) * HZ;
 765
 766         lockdep_assert_held(&m->lock);
 767
 768         if (queue_if_no_path_timeout > 0 &&
 769             atomic_read(&m->nr_valid_paths) == 0 &&
 770             test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
 771                 mod_timer(&m->nopath_timer,
 772                           jiffies + queue_if_no_path_timeout);
 773         }
 774 }
 775
 776 static void disable_nopath_timeout(struct multipath *m)
 777 {
 778         del_timer_sync(&m->nopath_timer);
 779 }
 780
 781 /*
 782  * An event is triggered whenever a path is taken out of use.
 783  * Includes path failure and PG bypass.
 784  */
 785 static void trigger_event(struct work_struct *work)
 786 {
 787         struct multipath *m =
 788                 container_of(work, struct multipath, trigger_event);
 789
 790         dm_table_event(m->ti->table);
 791 }
 792
 793 /*-----------------------------------------------------------------
 794  * Constructor/argument parsing:
 795  * <#multipath feature args> [<arg>]*
 796  * <#hw_handler args> [hw_handler [<arg>]*]
 797  * <#priority groups>
 798  * <initial priority group>
 799  *     [<selector> <#selector args> [<arg>]*
 800  *      <#paths> <#per-path selector args>
 801  *         [<path> [<arg>]* ]+ ]+
 802  *---------------------------------------------------------------*/
 803 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 804                                struct dm_target *ti)
 805 {
 806         int r;
 807         struct path_selector_type *pst;
 808         unsigned ps_argc;
 809
 810         static const struct dm_arg _args[] = {
 811                 {0, 1024, "invalid number of path selector args"},
 812         };
 813
 814         pst = dm_get_path_selector(dm_shift_arg(as));
 815         if (!pst) {
 816                 ti->error = "unknown path selector type";
 817                 return -EINVAL;
 818         }
 819
 820         r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
 821         if (r) {
 822                 dm_put_path_selector(pst);
 823                 return -EINVAL;
 824         }
 825
 826         r = pst->create(&pg->ps, ps_argc, as->argv);
 827         if (r) {
 828                 dm_put_path_selector(pst);
 829                 ti->error = "path selector constructor failed";
 830                 return r;
 831         }
 832
 833         pg->ps.type = pst;
 834         dm_consume_args(as, ps_argc);
 835
 836         return 0;
 837 }
 838
 839 static int setup_scsi_dh(struct block_device *bdev, struct multipath *m,
 840                          const char **attached_handler_name, char **error)
 841 {
 842         struct request_queue *q = bdev_get_queue(bdev);
 843         int r;
 844
 845         if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
 846 retain:
 847                 if (*attached_handler_name) {
 848                         /*
 849                          * Clear any hw_handler_params associated with a
 850                          * handler that isn't already attached.
 851                          */
 852                         if (m->hw_handler_name && strcmp(*attached_handler_name, m->hw_handler_name)) {
 853                                 kfree(m->hw_handler_params);
 854                                 m->hw_handler_params = NULL;
 855                         }
 856
 857                         /*
 858                          * Reset hw_handler_name to match the attached handler
 859                          *
 860                          * NB. This modifies the table line to show the actual
 861                          * handler instead of the original table passed in.
 862                          */
 863                         kfree(m->hw_handler_name);
 864                         m->hw_handler_name = *attached_handler_name;
 865                         *attached_handler_name = NULL;
 866                 }
 867         }
 868
 869         if (m->hw_handler_name) {
 870                 r = scsi_dh_attach(q, m->hw_handler_name);
 871                 if (r == -EBUSY) {
 872                         char b[BDEVNAME_SIZE];
 873
 874                         printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
 875                                bdevname(bdev, b));
 876                         goto retain;
 877                 }
 878                 if (r < 0) {
 879                         *error = "error attaching hardware handler";
 880                         return r;
 881                 }
 882
 883                 if (m->hw_handler_params) {
 884                         r = scsi_dh_set_params(q, m->hw_handler_params);
 885                         if (r < 0) {
 886                                 *error = "unable to set hardware handler parameters";
 887                                 return r;
 888                         }
 889                 }
 890         }
 891
 892         return 0;
 893 }
 894
 895 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
 896                                  struct dm_target *ti)
 897 {
 898         int r;
 899         struct pgpath *p;
 900         struct multipath *m = ti->private;
 901         struct request_queue *q;
 902         const char *attached_handler_name = NULL;
 903
 904         /* we need at least a path arg */
 905         if (as->argc < 1) {
 906                 ti->error = "no device given";
 907                 return ERR_PTR(-EINVAL);
 908         }
 909
 910         p = alloc_pgpath();
 911         if (!p)
 912                 return ERR_PTR(-ENOMEM);
 913
 914         r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
 915                           &p->path.dev);
 916         if (r) {
 917                 ti->error = "error getting device";
 918                 goto bad;
 919         }
 920
 921         q = bdev_get_queue(p->path.dev->bdev);
 922         attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 923         if (attached_handler_name || m->hw_handler_name) {
 924                 INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
 925                 r = setup_scsi_dh(p->path.dev->bdev, m, &attached_handler_name, &ti->error);
 926                 kfree(attached_handler_name);
 927                 if (r) {
 928                         dm_put_device(ti, p->path.dev);
 929                         goto bad;
 930                 }
 931         }
 932
 933         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 934         if (r) {
 935                 dm_put_device(ti, p->path.dev);
 936                 goto bad;
 937         }
 938
 939         return p;
 940  bad:
 941         free_pgpath(p);
 942         return ERR_PTR(r);
 943 }
 944
 945 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 946                                                    struct multipath *m)
 947 {
 948         static const struct dm_arg _args[] = {
 949                 {1, 1024, "invalid number of paths"},
 950                 {0, 1024, "invalid number of selector args"}
 951         };
 952
 953         int r;
 954         unsigned i, nr_selector_args, nr_args;
 955         struct priority_group *pg;
 956         struct dm_target *ti = m->ti;
 957
 958         if (as->argc < 2) {
 959                 as->argc = 0;
 960                 ti->error = "not enough priority group arguments";
 961                 return ERR_PTR(-EINVAL);
 962         }
 963
 964         pg = alloc_priority_group();
 965         if (!pg) {
 966                 ti->error = "couldn't allocate priority group";
 967                 return ERR_PTR(-ENOMEM);
 968         }
 969         pg->m = m;
 970
 971         r = parse_path_selector(as, pg, ti);
 972         if (r)
 973                 goto bad;
 974
 975         /*
 976          * read the paths
 977          */
 978         r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
 979         if (r)
 980                 goto bad;
 981
 982         r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
 983         if (r)
 984                 goto bad;
 985
 986         nr_args = 1 + nr_selector_args;
 987         for (i = 0; i < pg->nr_pgpaths; i++) {
 988                 struct pgpath *pgpath;
 989                 struct dm_arg_set path_args;
 990
 991                 if (as->argc < nr_args) {
 992                         ti->error = "not enough path parameters";
 993                         r = -EINVAL;
 994                         goto bad;
 995                 }
 996
 997                 path_args.argc = nr_args;
 998                 path_args.argv = as->argv;
 999
1000                 pgpath = parse_path(&path_args, &pg->ps, ti);
1001                 if (IS_ERR(pgpath)) {
1002                         r = PTR_ERR(pgpath);
1003                         goto bad;
1004                 }
1005
1006                 pgpath->pg = pg;
1007                 list_add_tail(&pgpath->list, &pg->pgpaths);
1008                 dm_consume_args(as, nr_args);
1009         }
1010
1011         return pg;
1012
1013  bad:
1014         free_priority_group(pg, ti);
1015         return ERR_PTR(r);
1016 }
1017
1018 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
1019 {
1020         unsigned hw_argc;
1021         int ret;
1022         struct dm_target *ti = m->ti;
1023
1024         static const struct dm_arg _args[] = {
1025                 {0, 1024, "invalid number of hardware handler args"},
1026         };
1027
1028         if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
1029                 return -EINVAL;
1030
1031         if (!hw_argc)
1032                 return 0;
1033
1034         if (m->queue_mode == DM_TYPE_BIO_BASED) {
1035                 dm_consume_args(as, hw_argc);
1036                 DMERR("bio-based multipath doesn't allow hardware handler args");
1037                 return 0;
1038         }
1039
1040         m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
1041         if (!m->hw_handler_name)
1042                 return -EINVAL;
1043
1044         if (hw_argc > 1) {
1045                 char *p;
1046                 int i, j, len = 4;
1047
1048                 for (i = 0; i <= hw_argc - 2; i++)
1049                         len += strlen(as->argv[i]) + 1;
1050                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
1051                 if (!p) {
1052                         ti->error = "memory allocation failed";
1053                         ret = -ENOMEM;
1054                         goto fail;
1055                 }
1056                 j = sprintf(p, "%d", hw_argc - 1);
1057                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
1058                         j = sprintf(p, "%s", as->argv[i]);
1059         }
1060         dm_consume_args(as, hw_argc - 1);
1061
1062         return 0;
1063 fail:
1064         kfree(m->hw_handler_name);
1065         m->hw_handler_name = NULL;
1066         return ret;
1067 }
1068
1069 static int parse_features(struct dm_arg_set *as, struct multipath *m)
1070 {
1071         int r;
1072         unsigned argc;
1073         struct dm_target *ti = m->ti;
1074         const char *arg_name;
1075
1076         static const struct dm_arg _args[] = {
1077                 {0, 8, "invalid number of feature args"},
1078                 {1, 50, "pg_init_retries must be between 1 and 50"},
1079                 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
1080         };
1081
1082         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1083         if (r)
1084                 return -EINVAL;
1085
1086         if (!argc)
1087                 return 0;
1088
1089         do {
1090                 arg_name = dm_shift_arg(as);
1091                 argc--;
1092
1093                 if (!strcasecmp(arg_name, "queue_if_no_path")) {
1094                         r = queue_if_no_path(m, true, false, __func__);
1095                         continue;
1096                 }
1097
1098                 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
1099                         set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
1100                         continue;
1101                 }
1102
1103                 if (!strcasecmp(arg_name, "pg_init_retries") &&
1104                     (argc >= 1)) {
1105                         r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
1106                         argc--;
1107                         continue;
1108                 }
1109
1110                 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
1111                     (argc >= 1)) {
1112                         r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
1113                         argc--;
1114                         continue;
1115                 }
1116
1117                 if (!strcasecmp(arg_name, "queue_mode") &&
1118                     (argc >= 1)) {
1119                         const char *queue_mode_name = dm_shift_arg(as);
1120
1121                         if (!strcasecmp(queue_mode_name, "bio"))
1122                                 m->queue_mode = DM_TYPE_BIO_BASED;
1123                         else if (!strcasecmp(queue_mode_name, "rq") ||
1124                                  !strcasecmp(queue_mode_name, "mq"))
1125                                 m->queue_mode = DM_TYPE_REQUEST_BASED;
1126                         else {
1127                                 ti->error = "Unknown 'queue_mode' requested";
1128                                 r = -EINVAL;
1129                         }
1130                         argc--;
1131                         continue;
1132                 }
1133
1134                 ti->error = "Unrecognised multipath feature request";
1135                 r = -EINVAL;
1136         } while (argc && !r);
1137
1138         return r;
1139 }
1140
1141 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1142 {
1143         /* target arguments */
1144         static const struct dm_arg _args[] = {
1145                 {0, 1024, "invalid number of priority groups"},
1146                 {0, 1024, "invalid initial priority group number"},
1147         };
1148
1149         int r;
1150         struct multipath *m;
1151         struct dm_arg_set as;
1152         unsigned pg_count = 0;
1153         unsigned next_pg_num;
1154         unsigned long flags;
1155
1156         as.argc = argc;
1157         as.argv = argv;
1158
1159         m = alloc_multipath(ti);
1160         if (!m) {
1161                 ti->error = "can't allocate multipath";
1162                 return -EINVAL;
1163         }
1164
1165         r = parse_features(&as, m);
1166         if (r)
1167                 goto bad;
1168
1169         r = alloc_multipath_stage2(ti, m);
1170         if (r)
1171                 goto bad;
1172
1173         r = parse_hw_handler(&as, m);
1174         if (r)
1175                 goto bad;
1176
1177         r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1178         if (r)
1179                 goto bad;
1180
1181         r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1182         if (r)
1183                 goto bad;
1184
1185         if ((!m->nr_priority_groups && next_pg_num) ||
1186             (m->nr_priority_groups && !next_pg_num)) {
1187                 ti->error = "invalid initial priority group";
1188                 r = -EINVAL;
1189                 goto bad;
1190         }
1191
1192         /* parse the priority groups */
1193         while (as.argc) {
1194                 struct priority_group *pg;
1195                 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1196
1197                 pg = parse_priority_group(&as, m);
1198                 if (IS_ERR(pg)) {
1199                         r = PTR_ERR(pg);
1200                         goto bad;
1201                 }
1202
1203                 nr_valid_paths += pg->nr_pgpaths;
1204                 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1205
1206                 list_add_tail(&pg->list, &m->priority_groups);
1207                 pg_count++;
1208                 pg->pg_num = pg_count;
1209                 if (!--next_pg_num)
1210                         m->next_pg = pg;
1211         }
1212
1213         if (pg_count != m->nr_priority_groups) {
1214                 ti->error = "priority group count mismatch";
1215                 r = -EINVAL;
1216                 goto bad;
1217         }
1218
1219         spin_lock_irqsave(&m->lock, flags);
1220         enable_nopath_timeout(m);
1221         spin_unlock_irqrestore(&m->lock, flags);
1222
1223         ti->num_flush_bios = 1;
1224         ti->num_discard_bios = 1;
1225         ti->num_write_same_bios = 1;
1226         ti->num_write_zeroes_bios = 1;
1227         if (m->queue_mode == DM_TYPE_BIO_BASED)
1228                 ti->per_io_data_size = multipath_per_bio_data_size();
1229         else
1230                 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1231
1232         return 0;
1233
1234  bad:
1235         free_multipath(m);
1236         return r;
1237 }
1238
1239 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1240 {
1241         DEFINE_WAIT(wait);
1242
1243         while (1) {
1244                 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1245
1246                 if (!atomic_read(&m->pg_init_in_progress))
1247                         break;
1248
1249                 io_schedule();
1250         }
1251         finish_wait(&m->pg_init_wait, &wait);
1252 }
1253
1254 static void flush_multipath_work(struct multipath *m)
1255 {
1256         if (m->hw_handler_name) {
1257                 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1258                 smp_mb__after_atomic();
1259
1260                 if (atomic_read(&m->pg_init_in_progress))
1261                         flush_workqueue(kmpath_handlerd);
1262                 multipath_wait_for_pg_init_completion(m);
1263
1264                 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1265                 smp_mb__after_atomic();
1266         }
1267
1268         if (m->queue_mode == DM_TYPE_BIO_BASED)
1269                 flush_work(&m->process_queued_bios);
1270         flush_work(&m->trigger_event);
1271 }
1272
1273 static void multipath_dtr(struct dm_target *ti)
1274 {
1275         struct multipath *m = ti->private;
1276
1277         disable_nopath_timeout(m);
1278         flush_multipath_work(m);
1279         free_multipath(m);
1280 }
1281
1282 /*
1283  * Take a path out of use.
1284  */
1285 static int fail_path(struct pgpath *pgpath)
1286 {
1287         unsigned long flags;
1288         struct multipath *m = pgpath->pg->m;
1289
1290         spin_lock_irqsave(&m->lock, flags);
1291
1292         if (!pgpath->is_active)
1293                 goto out;
1294
1295         DMWARN("%s: Failing path %s.",
1296                dm_device_name(dm_table_get_md(m->ti->table)),
1297                pgpath->path.dev->name);
1298
1299         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1300         pgpath->is_active = false;
1301         pgpath->fail_count++;
1302
1303         atomic_dec(&m->nr_valid_paths);
1304
1305         if (pgpath == m->current_pgpath)
1306                 m->current_pgpath = NULL;
1307
1308         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1309                        pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1310
1311         schedule_work(&m->trigger_event);
1312
1313         enable_nopath_timeout(m);
1314
1315 out:
1316         spin_unlock_irqrestore(&m->lock, flags);
1317
1318         return 0;
1319 }
1320
1321 /*
1322  * Reinstate a previously-failed path
1323  */
1324 static int reinstate_path(struct pgpath *pgpath)
1325 {
1326         int r = 0, run_queue = 0;
1327         unsigned long flags;
1328         struct multipath *m = pgpath->pg->m;
1329         unsigned nr_valid_paths;
1330
1331         spin_lock_irqsave(&m->lock, flags);
1332
1333         if (pgpath->is_active)
1334                 goto out;
1335
1336         DMWARN("%s: Reinstating path %s.",
1337                dm_device_name(dm_table_get_md(m->ti->table)),
1338                pgpath->path.dev->name);
1339
1340         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1341         if (r)
1342                 goto out;
1343
1344         pgpath->is_active = true;
1345
1346         nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1347         if (nr_valid_paths == 1) {
1348                 m->current_pgpath = NULL;
1349                 run_queue = 1;
1350         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1351                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1352                         atomic_inc(&m->pg_init_in_progress);
1353         }
1354
1355         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1356                        pgpath->path.dev->name, nr_valid_paths);
1357
1358         schedule_work(&m->trigger_event);
1359
1360 out:
1361         spin_unlock_irqrestore(&m->lock, flags);
1362         if (run_queue) {
1363                 dm_table_run_md_queue_async(m->ti->table);
1364                 process_queued_io_list(m);
1365         }
1366
1367         if (pgpath->is_active)
1368                 disable_nopath_timeout(m);
1369
1370         return r;
1371 }
1372
1373 /*
1374  * Fail or reinstate all paths that match the provided struct dm_dev.
1375  */
1376 static int action_dev(struct multipath *m, struct dm_dev *dev,
1377                       action_fn action)
1378 {
1379         int r = -EINVAL;
1380         struct pgpath *pgpath;
1381         struct priority_group *pg;
1382
1383         list_for_each_entry(pg, &m->priority_groups, list) {
1384                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1385                         if (pgpath->path.dev == dev)
1386                                 r = action(pgpath);
1387                 }
1388         }
1389
1390         return r;
1391 }
1392
1393 /*
1394  * Temporarily try to avoid having to use the specified PG
1395  */
1396 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1397                       bool bypassed)
1398 {
1399         unsigned long flags;
1400
1401         spin_lock_irqsave(&m->lock, flags);
1402
1403         pg->bypassed = bypassed;
1404         m->current_pgpath = NULL;
1405         m->current_pg = NULL;
1406
1407         spin_unlock_irqrestore(&m->lock, flags);
1408
1409         schedule_work(&m->trigger_event);
1410 }
1411
1412 /*
1413  * Switch to using the specified PG from the next I/O that gets mapped
1414  */
1415 static int switch_pg_num(struct multipath *m, const char *pgstr)
1416 {
1417         struct priority_group *pg;
1418         unsigned pgnum;
1419         unsigned long flags;
1420         char dummy;
1421
1422         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1423             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1424                 DMWARN("invalid PG number supplied to switch_pg_num");
1425                 return -EINVAL;
1426         }
1427
1428         spin_lock_irqsave(&m->lock, flags);
1429         list_for_each_entry(pg, &m->priority_groups, list) {
1430                 pg->bypassed = false;
1431                 if (--pgnum)
1432                         continue;
1433
1434                 m->current_pgpath = NULL;
1435                 m->current_pg = NULL;
1436                 m->next_pg = pg;
1437         }
1438         spin_unlock_irqrestore(&m->lock, flags);
1439
1440         schedule_work(&m->trigger_event);
1441         return 0;
1442 }
1443
1444 /*
1445  * Set/clear bypassed status of a PG.
1446  * PGs are numbered upwards from 1 in the order they were declared.
1447  */
1448 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1449 {
1450         struct priority_group *pg;
1451         unsigned pgnum;
1452         char dummy;
1453
1454         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1455             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1456                 DMWARN("invalid PG number supplied to bypass_pg");
1457                 return -EINVAL;
1458         }
1459
1460         list_for_each_entry(pg, &m->priority_groups, list) {
1461                 if (!--pgnum)
1462                         break;
1463         }
1464
1465         bypass_pg(m, pg, bypassed);
1466         return 0;
1467 }
1468
1469 /*
1470  * Should we retry pg_init immediately?
1471  */
1472 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1473 {
1474         unsigned long flags;
1475         bool limit_reached = false;
1476
1477         spin_lock_irqsave(&m->lock, flags);
1478
1479         if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1480             !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1481                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1482         else
1483                 limit_reached = true;
1484
1485         spin_unlock_irqrestore(&m->lock, flags);
1486
1487         return limit_reached;
1488 }
1489
1490 static void pg_init_done(void *data, int errors)
1491 {
1492         struct pgpath *pgpath = data;
1493         struct priority_group *pg = pgpath->pg;
1494         struct multipath *m = pg->m;
1495         unsigned long flags;
1496         bool delay_retry = false;
1497
1498         /* device or driver problems */
1499         switch (errors) {
1500         case SCSI_DH_OK:
1501                 break;
1502         case SCSI_DH_NOSYS:
1503                 if (!m->hw_handler_name) {
1504                         errors = 0;
1505                         break;
1506                 }
1507                 DMERR("Could not failover the device: Handler scsi_dh_%s "
1508                       "Error %d.", m->hw_handler_name, errors);
1509                 /*
1510                  * Fail path for now, so we do not ping pong
1511                  */
1512                 fail_path(pgpath);
1513                 break;
1514         case SCSI_DH_DEV_TEMP_BUSY:
1515                 /*
1516                  * Probably doing something like FW upgrade on the
1517                  * controller so try the other pg.
1518                  */
1519                 bypass_pg(m, pg, true);
1520                 break;
1521         case SCSI_DH_RETRY:
1522                 /* Wait before retrying. */
1523                 delay_retry = true;
1524                 /* fall through */
1525         case SCSI_DH_IMM_RETRY:
1526         case SCSI_DH_RES_TEMP_UNAVAIL:
1527                 if (pg_init_limit_reached(m, pgpath))
1528                         fail_path(pgpath);
1529                 errors = 0;
1530                 break;
1531         case SCSI_DH_DEV_OFFLINED:
1532         default:
1533                 /*
1534                  * We probably do not want to fail the path for a device
1535                  * error, but this is what the old dm did. In future
1536                  * patches we can do more advanced handling.
1537                  */
1538                 fail_path(pgpath);
1539         }
1540
1541         spin_lock_irqsave(&m->lock, flags);
1542         if (errors) {
1543                 if (pgpath == m->current_pgpath) {
1544                         DMERR("Could not failover device. Error %d.", errors);
1545                         m->current_pgpath = NULL;
1546                         m->current_pg = NULL;
1547                 }
1548         } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1549                 pg->bypassed = false;
1550
1551         if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1552                 /* Activations of other paths are still on going */
1553                 goto out;
1554
1555         if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1556                 if (delay_retry)
1557                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1558                 else
1559                         clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1560
1561                 if (__pg_init_all_paths(m))
1562                         goto out;
1563         }
1564         clear_bit(MPATHF_QUEUE_IO, &m->flags);
1565
1566         process_queued_io_list(m);
1567
1568         /*
1569          * Wake up any thread waiting to suspend.
1570          */
1571         wake_up(&m->pg_init_wait);
1572
1573 out:
1574         spin_unlock_irqrestore(&m->lock, flags);
1575 }
1576
1577 static void activate_or_offline_path(struct pgpath *pgpath)
1578 {
1579         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1580
1581         if (pgpath->is_active && !blk_queue_dying(q))
1582                 scsi_dh_activate(q, pg_init_done, pgpath);
1583         else
1584                 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1585 }
1586
1587 static void activate_path_work(struct work_struct *work)
1588 {
1589         struct pgpath *pgpath =
1590                 container_of(work, struct pgpath, activate_path.work);
1591
1592         activate_or_offline_path(pgpath);
1593 }
1594
1595 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1596                             blk_status_t error, union map_info *map_context)
1597 {
1598         struct dm_mpath_io *mpio = get_mpio(map_context);
1599         struct pgpath *pgpath = mpio->pgpath;
1600         int r = DM_ENDIO_DONE;
1601
1602         /*
1603          * We don't queue any clone request inside the multipath target
1604          * during end I/O handling, since those clone requests don't have
1605          * bio clones.  If we queue them inside the multipath target,
1606          * we need to make bio clones, that requires memory allocation.
1607          * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1608          *  don't have bio clones.)
1609          * Instead of queueing the clone request here, we queue the original
1610          * request into dm core, which will remake a clone request and
1611          * clone bios for it and resubmit it later.
1612          */
1613         if (error && blk_path_error(error)) {
1614                 struct multipath *m = ti->private;
1615
1616                 if (error == BLK_STS_RESOURCE)
1617                         r = DM_ENDIO_DELAY_REQUEUE;
1618                 else
1619                         r = DM_ENDIO_REQUEUE;
1620
1621                 if (pgpath)
1622                         fail_path(pgpath);
1623
1624                 if (!atomic_read(&m->nr_valid_paths)) {
1625                         unsigned long flags;
1626                         spin_lock_irqsave(&m->lock, flags);
1627                         if (!must_push_back_rq(m)) {
1628                                 if (error == BLK_STS_IOERR)
1629                                         dm_report_EIO(m);
1630                                 /* complete with the original error */
1631                                 r = DM_ENDIO_DONE;
1632                         }
1633                         spin_unlock_irqrestore(&m->lock, flags);
1634                 }
1635         }
1636
1637         if (pgpath) {
1638                 struct path_selector *ps = &pgpath->pg->ps;
1639
1640                 if (ps->type->end_io)
1641                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes,
1642                                          clone->io_start_time_ns);
1643         }
1644
1645         return r;
1646 }
1647
1648 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1649                                 blk_status_t *error)
1650 {
1651         struct multipath *m = ti->private;
1652         struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1653         struct pgpath *pgpath = mpio->pgpath;
1654         unsigned long flags;
1655         int r = DM_ENDIO_DONE;
1656
1657         if (!*error || !blk_path_error(*error))
1658                 goto done;
1659
1660         if (pgpath)
1661                 fail_path(pgpath);
1662
1663         if (!atomic_read(&m->nr_valid_paths)) {
1664                 spin_lock_irqsave(&m->lock, flags);
1665                 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1666                         if (__must_push_back(m)) {
1667                                 r = DM_ENDIO_REQUEUE;
1668                         } else {
1669                                 dm_report_EIO(m);
1670                                 *error = BLK_STS_IOERR;
1671                         }
1672                         spin_unlock_irqrestore(&m->lock, flags);
1673                         goto done;
1674                 }
1675                 spin_unlock_irqrestore(&m->lock, flags);
1676         }
1677
1678         spin_lock_irqsave(&m->lock, flags);
1679         bio_list_add(&m->queued_bios, clone);
1680         if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1681                 queue_work(kmultipathd, &m->process_queued_bios);
1682         spin_unlock_irqrestore(&m->lock, flags);
1683
1684         r = DM_ENDIO_INCOMPLETE;
1685 done:
1686         if (pgpath) {
1687                 struct path_selector *ps = &pgpath->pg->ps;
1688
1689                 if (ps->type->end_io)
1690                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes,
1691                                          dm_start_time_ns_from_clone(clone));
1692         }
1693
1694         return r;
1695 }
1696
1697 /*
1698  * Suspend with flush can't complete until all the I/O is processed
1699  * so if the last path fails we must error any remaining I/O.
1700  * - Note that if the freeze_bdev fails while suspending, the
1701  *   queue_if_no_path state is lost - userspace should reset it.
1702  * Otherwise, during noflush suspend, queue_if_no_path will not change.
1703  */
1704 static void multipath_presuspend(struct dm_target *ti)
1705 {
1706         struct multipath *m = ti->private;
1707
1708         /* FIXME: bio-based shouldn't need to always disable queue_if_no_path */
1709         if (m->queue_mode == DM_TYPE_BIO_BASED || !dm_noflush_suspending(m->ti))
1710                 queue_if_no_path(m, false, true, __func__);
1711 }
1712
1713 static void multipath_postsuspend(struct dm_target *ti)
1714 {
1715         struct multipath *m = ti->private;
1716
1717         mutex_lock(&m->work_mutex);
1718         flush_multipath_work(m);
1719         mutex_unlock(&m->work_mutex);
1720 }
1721
1722 /*
1723  * Restore the queue_if_no_path setting.
1724  */
1725 static void multipath_resume(struct dm_target *ti)
1726 {
1727         struct multipath *m = ti->private;
1728         unsigned long flags;
1729
1730         spin_lock_irqsave(&m->lock, flags);
1731         if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) {
1732                 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
1733                 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
1734         }
1735
1736         DMDEBUG("%s: %s finished; QIFNP = %d; SQIFNP = %d",
1737                 dm_device_name(dm_table_get_md(m->ti->table)), __func__,
1738                 test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags),
1739                 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
1740
1741         spin_unlock_irqrestore(&m->lock, flags);
1742 }
1743
1744 /*
1745  * Info output has the following format:
1746  * num_multipath_feature_args [multipath_feature_args]*
1747  * num_handler_status_args [handler_status_args]*
1748  * num_groups init_group_number
1749  *            [A|D|E num_ps_status_args [ps_status_args]*
1750  *             num_paths num_selector_args
1751  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1752  *
1753  * Table output has the following format (identical to the constructor string):
1754  * num_feature_args [features_args]*
1755  * num_handler_args hw_handler [hw_handler_args]*
1756  * num_groups init_group_number
1757  *     [priority selector-name num_ps_args [ps_args]*
1758  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1759  */
1760 static void multipath_status(struct dm_target *ti, status_type_t type,
1761                              unsigned status_flags, char *result, unsigned maxlen)
1762 {
1763         int sz = 0;
1764         unsigned long flags;
1765         struct multipath *m = ti->private;
1766         struct priority_group *pg;
1767         struct pgpath *p;
1768         unsigned pg_num;
1769         char state;
1770
1771         spin_lock_irqsave(&m->lock, flags);
1772
1773         /* Features */
1774         if (type == STATUSTYPE_INFO)
1775                 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1776                        atomic_read(&m->pg_init_count));
1777         else {
1778                 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1779                               (m->pg_init_retries > 0) * 2 +
1780                               (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1781                               test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1782                               (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1783
1784                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1785                         DMEMIT("queue_if_no_path ");
1786                 if (m->pg_init_retries)
1787                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1788                 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1789                         DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1790                 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1791                         DMEMIT("retain_attached_hw_handler ");
1792                 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1793                         switch(m->queue_mode) {
1794                         case DM_TYPE_BIO_BASED:
1795                                 DMEMIT("queue_mode bio ");
1796                                 break;
1797                         default:
1798                                 WARN_ON_ONCE(true);
1799                                 break;
1800                         }
1801                 }
1802         }
1803
1804         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1805                 DMEMIT("0 ");
1806         else
1807                 DMEMIT("1 %s ", m->hw_handler_name);
1808
1809         DMEMIT("%u ", m->nr_priority_groups);
1810
1811         if (m->next_pg)
1812                 pg_num = m->next_pg->pg_num;
1813         else if (m->current_pg)
1814                 pg_num = m->current_pg->pg_num;
1815         else
1816                 pg_num = (m->nr_priority_groups ? 1 : 0);
1817
1818         DMEMIT("%u ", pg_num);
1819
1820         switch (type) {
1821         case STATUSTYPE_INFO:
1822                 list_for_each_entry(pg, &m->priority_groups, list) {
1823                         if (pg->bypassed)
1824                                 state = 'D';    /* Disabled */
1825                         else if (pg == m->current_pg)
1826                                 state = 'A';    /* Currently Active */
1827                         else
1828                                 state = 'E';    /* Enabled */
1829
1830                         DMEMIT("%c ", state);
1831
1832                         if (pg->ps.type->status)
1833                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1834                                                           result + sz,
1835                                                           maxlen - sz);
1836                         else
1837                                 DMEMIT("0 ");
1838
1839                         DMEMIT("%u %u ", pg->nr_pgpaths,
1840                                pg->ps.type->info_args);
1841
1842                         list_for_each_entry(p, &pg->pgpaths, list) {
1843                                 DMEMIT("%s %s %u ", p->path.dev->name,
1844                                        p->is_active ? "A" : "F",
1845                                        p->fail_count);
1846                                 if (pg->ps.type->status)
1847                                         sz += pg->ps.type->status(&pg->ps,
1848                                               &p->path, type, result + sz,
1849                                               maxlen - sz);
1850                         }
1851                 }
1852                 break;
1853
1854         case STATUSTYPE_TABLE:
1855                 list_for_each_entry(pg, &m->priority_groups, list) {
1856                         DMEMIT("%s ", pg->ps.type->name);
1857
1858                         if (pg->ps.type->status)
1859                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1860                                                           result + sz,
1861                                                           maxlen - sz);
1862                         else
1863                                 DMEMIT("0 ");
1864
1865                         DMEMIT("%u %u ", pg->nr_pgpaths,
1866                                pg->ps.type->table_args);
1867
1868                         list_for_each_entry(p, &pg->pgpaths, list) {
1869                                 DMEMIT("%s ", p->path.dev->name);
1870                                 if (pg->ps.type->status)
1871                                         sz += pg->ps.type->status(&pg->ps,
1872                                               &p->path, type, result + sz,
1873                                               maxlen - sz);
1874                         }
1875                 }
1876                 break;
1877         }
1878
1879         spin_unlock_irqrestore(&m->lock, flags);
1880 }
1881
1882 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv,
1883                              char *result, unsigned maxlen)
1884 {
1885         int r = -EINVAL;
1886         struct dm_dev *dev;
1887         struct multipath *m = ti->private;
1888         action_fn action;
1889         unsigned long flags;
1890
1891         mutex_lock(&m->work_mutex);
1892
1893         if (dm_suspended(ti)) {
1894                 r = -EBUSY;
1895                 goto out;
1896         }
1897
1898         if (argc == 1) {
1899                 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1900                         r = queue_if_no_path(m, true, false, __func__);
1901                         spin_lock_irqsave(&m->lock, flags);
1902                         enable_nopath_timeout(m);
1903                         spin_unlock_irqrestore(&m->lock, flags);
1904                         goto out;
1905                 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1906                         r = queue_if_no_path(m, false, false, __func__);
1907                         disable_nopath_timeout(m);
1908                         goto out;
1909                 }
1910         }
1911
1912         if (argc != 2) {
1913                 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1914                 goto out;
1915         }
1916
1917         if (!strcasecmp(argv[0], "disable_group")) {
1918                 r = bypass_pg_num(m, argv[1], true);
1919                 goto out;
1920         } else if (!strcasecmp(argv[0], "enable_group")) {
1921                 r = bypass_pg_num(m, argv[1], false);
1922                 goto out;
1923         } else if (!strcasecmp(argv[0], "switch_group")) {
1924                 r = switch_pg_num(m, argv[1]);
1925                 goto out;
1926         } else if (!strcasecmp(argv[0], "reinstate_path"))
1927                 action = reinstate_path;
1928         else if (!strcasecmp(argv[0], "fail_path"))
1929                 action = fail_path;
1930         else {
1931                 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1932                 goto out;
1933         }
1934
1935         r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1936         if (r) {
1937                 DMWARN("message: error getting device %s",
1938                        argv[1]);
1939                 goto out;
1940         }
1941
1942         r = action_dev(m, dev, action);
1943
1944         dm_put_device(ti, dev);
1945
1946 out:
1947         mutex_unlock(&m->work_mutex);
1948         return r;
1949 }
1950
1951 static int multipath_prepare_ioctl(struct dm_target *ti,
1952                                    struct block_device **bdev)
1953 {
1954         struct multipath *m = ti->private;
1955         struct pgpath *current_pgpath;
1956         unsigned long flags;
1957         int r;
1958
1959         current_pgpath = READ_ONCE(m->current_pgpath);
1960         if (!current_pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
1961                 current_pgpath = choose_pgpath(m, 0);
1962
1963         if (current_pgpath) {
1964                 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1965                         *bdev = current_pgpath->path.dev->bdev;
1966                         r = 0;
1967                 } else {
1968                         /* pg_init has not started or completed */
1969                         r = -ENOTCONN;
1970                 }
1971         } else {
1972                 /* No path is available */
1973                 r = -EIO;
1974                 spin_lock_irqsave(&m->lock, flags);
1975                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1976                         r = -ENOTCONN;
1977                 spin_unlock_irqrestore(&m->lock, flags);
1978         }
1979
1980         if (r == -ENOTCONN) {
1981                 if (!READ_ONCE(m->current_pg)) {
1982                         /* Path status changed, redo selection */
1983                         (void) choose_pgpath(m, 0);
1984                 }
1985                 spin_lock_irqsave(&m->lock, flags);
1986                 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1987                         (void) __pg_init_all_paths(m);
1988                 spin_unlock_irqrestore(&m->lock, flags);
1989                 dm_table_run_md_queue_async(m->ti->table);
1990                 process_queued_io_list(m);
1991         }
1992
1993         /*
1994          * Only pass ioctls through if the device sizes match exactly.
1995          */
1996         if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1997                 return 1;
1998         return r;
1999 }
2000
2001 static int multipath_iterate_devices(struct dm_target *ti,
2002                                      iterate_devices_callout_fn fn, void *data)
2003 {
2004         struct multipath *m = ti->private;
2005         struct priority_group *pg;
2006         struct pgpath *p;
2007         int ret = 0;
2008
2009         list_for_each_entry(pg, &m->priority_groups, list) {
2010                 list_for_each_entry(p, &pg->pgpaths, list) {
2011                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
2012                         if (ret)
2013                                 goto out;
2014                 }
2015         }
2016
2017 out:
2018         return ret;
2019 }
2020
2021 static int pgpath_busy(struct pgpath *pgpath)
2022 {
2023         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
2024
2025         return blk_lld_busy(q);
2026 }
2027
2028 /*
2029  * We return "busy", only when we can map I/Os but underlying devices
2030  * are busy (so even if we map I/Os now, the I/Os will wait on
2031  * the underlying queue).
2032  * In other words, if we want to kill I/Os or queue them inside us
2033  * due to map unavailability, we don't return "busy".  Otherwise,
2034  * dm core won't give us the I/Os and we can't do what we want.
2035  */
2036 static int multipath_busy(struct dm_target *ti)
2037 {
2038         bool busy = false, has_active = false;
2039         struct multipath *m = ti->private;
2040         struct priority_group *pg, *next_pg;
2041         struct pgpath *pgpath;
2042
2043         /* pg_init in progress */
2044         if (atomic_read(&m->pg_init_in_progress))
2045                 return true;
2046
2047         /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
2048         if (!atomic_read(&m->nr_valid_paths)) {
2049                 unsigned long flags;
2050                 spin_lock_irqsave(&m->lock, flags);
2051                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
2052                         spin_unlock_irqrestore(&m->lock, flags);
2053                         return (m->queue_mode != DM_TYPE_REQUEST_BASED);
2054                 }
2055                 spin_unlock_irqrestore(&m->lock, flags);
2056         }
2057
2058         /* Guess which priority_group will be used at next mapping time */
2059         pg = READ_ONCE(m->current_pg);
2060         next_pg = READ_ONCE(m->next_pg);
2061         if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg))
2062                 pg = next_pg;
2063
2064         if (!pg) {
2065                 /*
2066                  * We don't know which pg will be used at next mapping time.
2067                  * We don't call choose_pgpath() here to avoid to trigger
2068                  * pg_init just by busy checking.
2069                  * So we don't know whether underlying devices we will be using
2070                  * at next mapping time are busy or not. Just try mapping.
2071                  */
2072                 return busy;
2073         }
2074
2075         /*
2076          * If there is one non-busy active path at least, the path selector
2077          * will be able to select it. So we consider such a pg as not busy.
2078          */
2079         busy = true;
2080         list_for_each_entry(pgpath, &pg->pgpaths, list) {
2081                 if (pgpath->is_active) {
2082                         has_active = true;
2083                         if (!pgpath_busy(pgpath)) {
2084                                 busy = false;
2085                                 break;
2086                         }
2087                 }
2088         }
2089
2090         if (!has_active) {
2091                 /*
2092                  * No active path in this pg, so this pg won't be used and
2093                  * the current_pg will be changed at next mapping time.
2094                  * We need to try mapping to determine it.
2095                  */
2096                 busy = false;
2097         }
2098
2099         return busy;
2100 }
2101
2102 /*-----------------------------------------------------------------
2103  * Module setup
2104  *---------------------------------------------------------------*/
2105 static struct target_type multipath_target = {
2106         .name = "multipath",
2107         .version = {1, 14, 0},
2108         .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE |
2109                     DM_TARGET_PASSES_INTEGRITY,
2110         .module = THIS_MODULE,
2111         .ctr = multipath_ctr,
2112         .dtr = multipath_dtr,
2113         .clone_and_map_rq = multipath_clone_and_map,
2114         .release_clone_rq = multipath_release_clone,
2115         .rq_end_io = multipath_end_io,
2116         .map = multipath_map_bio,
2117         .end_io = multipath_end_io_bio,
2118         .presuspend = multipath_presuspend,
2119         .postsuspend = multipath_postsuspend,
2120         .resume = multipath_resume,
2121         .status = multipath_status,
2122         .message = multipath_message,
2123         .prepare_ioctl = multipath_prepare_ioctl,
2124         .iterate_devices = multipath_iterate_devices,
2125         .busy = multipath_busy,
2126 };
2127
2128 static int __init dm_multipath_init(void)
2129 {
2130         int r;
2131
2132         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2133         if (!kmultipathd) {
2134                 DMERR("failed to create workqueue kmpathd");
2135                 r = -ENOMEM;
2136                 goto bad_alloc_kmultipathd;
2137         }
2138
2139         /*
2140          * A separate workqueue is used to handle the device handlers
2141          * to avoid overloading existing workqueue. Overloading the
2142          * old workqueue would also create a bottleneck in the
2143          * path of the storage hardware device activation.
2144          */
2145         kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2146                                                   WQ_MEM_RECLAIM);
2147         if (!kmpath_handlerd) {
2148                 DMERR("failed to create workqueue kmpath_handlerd");
2149                 r = -ENOMEM;
2150                 goto bad_alloc_kmpath_handlerd;
2151         }
2152
2153         r = dm_register_target(&multipath_target);
2154         if (r < 0) {
2155                 DMERR("request-based register failed %d", r);
2156                 r = -EINVAL;
2157                 goto bad_register_target;
2158         }
2159
2160         return 0;
2161
2162 bad_register_target:
2163         destroy_workqueue(kmpath_handlerd);
2164 bad_alloc_kmpath_handlerd:
2165         destroy_workqueue(kmultipathd);
2166 bad_alloc_kmultipathd:
2167         return r;
2168 }
2169
2170 static void __exit dm_multipath_exit(void)
2171 {
2172         destroy_workqueue(kmpath_handlerd);
2173         destroy_workqueue(kmultipathd);
2174
2175         dm_unregister_target(&multipath_target);
2176 }
2177
2178 module_init(dm_multipath_init);
2179 module_exit(dm_multipath_exit);
2180
2181 module_param_named(queue_if_no_path_timeout_secs,
2182                    queue_if_no_path_timeout_secs, ulong, S_IRUGO | S_IWUSR);
2183 MODULE_PARM_DESC(queue_if_no_path_timeout_secs, "No available paths queue IO timeout in seconds");
2184
2185 MODULE_DESCRIPTION(DM_NAME " multipath target");
2186 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2187 MODULE_LICENSE("GPL");