drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-rq.h"
  11 #include "dm-bio-record.h"
  12 #include "dm-path-selector.h"
  13 #include "dm-uevent.h"
  14
  15 #include <linux/blkdev.h>
  16 #include <linux/ctype.h>
  17 #include <linux/init.h>
  18 #include <linux/mempool.h>
  19 #include <linux/module.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/slab.h>
  22 #include <linux/time.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/delay.h>
  25 #include <scsi/scsi_dh.h>
  26 #include <linux/atomic.h>
  27 #include <linux/blk-mq.h>
  28
  29 #define DM_MSG_PREFIX "multipath"
  30 #define DM_PG_INIT_DELAY_MSECS 2000
  31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
  32
  33 /* Path properties */
  34 struct pgpath {
  35         struct list_head list;
  36
  37         struct priority_group *pg;      /* Owning PG */
  38         unsigned fail_count;            /* Cumulative failure count */
  39
  40         struct dm_path path;
  41         struct delayed_work activate_path;
  42
  43         bool is_active:1;               /* Path status */
  44 };
  45
  46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  47
  48 /*
  49  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  50  * Each has a path selector which controls which path gets used.
  51  */
  52 struct priority_group {
  53         struct list_head list;
  54
  55         struct multipath *m;            /* Owning multipath instance */
  56         struct path_selector ps;
  57
  58         unsigned pg_num;                /* Reference number */
  59         unsigned nr_pgpaths;            /* Number of paths in PG */
  60         struct list_head pgpaths;
  61
  62         bool bypassed:1;                /* Temporarily bypass this PG? */
  63 };
  64
  65 /* Multipath context */
  66 struct multipath {
  67         unsigned long flags;            /* Multipath state flags */
  68
  69         spinlock_t lock;
  70         enum dm_queue_mode queue_mode;
  71
  72         struct pgpath *current_pgpath;
  73         struct priority_group *current_pg;
  74         struct priority_group *next_pg; /* Switch to this PG if set */
  75
  76         atomic_t nr_valid_paths;        /* Total number of usable paths */
  77         unsigned nr_priority_groups;
  78         struct list_head priority_groups;
  79
  80         const char *hw_handler_name;
  81         char *hw_handler_params;
  82         wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  83         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  84         unsigned pg_init_delay_msecs;   /* Number of msecs before pg_init retry */
  85         atomic_t pg_init_in_progress;   /* Only one pg_init allowed at once */
  86         atomic_t pg_init_count;         /* Number of times pg_init called */
  87
  88         struct mutex work_mutex;
  89         struct work_struct trigger_event;
  90         struct dm_target *ti;
  91
  92         struct work_struct process_queued_bios;
  93         struct bio_list queued_bios;
  94 };
  95
  96 /*
  97  * Context information attached to each io we process.
  98  */
  99 struct dm_mpath_io {
 100         struct pgpath *pgpath;
 101         size_t nr_bytes;
 102 };
 103
 104 typedef int (*action_fn) (struct pgpath *pgpath);
 105
 106 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 107 static void trigger_event(struct work_struct *work);
 108 static void activate_or_offline_path(struct pgpath *pgpath);
 109 static void activate_path_work(struct work_struct *work);
 110 static void process_queued_bios(struct work_struct *work);
 111
 112 /*-----------------------------------------------
 113  * Multipath state flags.
 114  *-----------------------------------------------*/
 115
 116 #define MPATHF_QUEUE_IO 0                       /* Must we queue all I/O? */
 117 #define MPATHF_QUEUE_IF_NO_PATH 1               /* Queue I/O if last path fails? */
 118 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2         /* Saved state during suspension */
 119 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3     /* If there's already a hw_handler present, don't change it. */
 120 #define MPATHF_PG_INIT_DISABLED 4               /* pg_init is not currently allowed */
 121 #define MPATHF_PG_INIT_REQUIRED 5               /* pg_init needs calling? */
 122 #define MPATHF_PG_INIT_DELAY_RETRY 6            /* Delay pg_init retry? */
 123
 124 /*-----------------------------------------------
 125  * Allocation routines
 126  *-----------------------------------------------*/
 127
 128 static struct pgpath *alloc_pgpath(void)
 129 {
 130         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 131
 132         if (!pgpath)
 133                 return NULL;
 134
 135         pgpath->is_active = true;
 136
 137         return pgpath;
 138 }
 139
 140 static void free_pgpath(struct pgpath *pgpath)
 141 {
 142         kfree(pgpath);
 143 }
 144
 145 static struct priority_group *alloc_priority_group(void)
 146 {
 147         struct priority_group *pg;
 148
 149         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 150
 151         if (pg)
 152                 INIT_LIST_HEAD(&pg->pgpaths);
 153
 154         return pg;
 155 }
 156
 157 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 158 {
 159         struct pgpath *pgpath, *tmp;
 160
 161         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 162                 list_del(&pgpath->list);
 163                 dm_put_device(ti, pgpath->path.dev);
 164                 free_pgpath(pgpath);
 165         }
 166 }
 167
 168 static void free_priority_group(struct priority_group *pg,
 169                                 struct dm_target *ti)
 170 {
 171         struct path_selector *ps = &pg->ps;
 172
 173         if (ps->type) {
 174                 ps->type->destroy(ps);
 175                 dm_put_path_selector(ps->type);
 176         }
 177
 178         free_pgpaths(&pg->pgpaths, ti);
 179         kfree(pg);
 180 }
 181
 182 static struct multipath *alloc_multipath(struct dm_target *ti)
 183 {
 184         struct multipath *m;
 185
 186         m = kzalloc(sizeof(*m), GFP_KERNEL);
 187         if (m) {
 188                 INIT_LIST_HEAD(&m->priority_groups);
 189                 spin_lock_init(&m->lock);
 190                 atomic_set(&m->nr_valid_paths, 0);
 191                 INIT_WORK(&m->trigger_event, trigger_event);
 192                 mutex_init(&m->work_mutex);
 193
 194                 m->queue_mode = DM_TYPE_NONE;
 195
 196                 m->ti = ti;
 197                 ti->private = m;
 198         }
 199
 200         return m;
 201 }
 202
 203 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 204 {
 205         if (m->queue_mode == DM_TYPE_NONE) {
 206                 /*
 207                  * Default to request-based.
 208                  */
 209                 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
 210                         m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
 211                 else
 212                         m->queue_mode = DM_TYPE_REQUEST_BASED;
 213
 214         } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
 215                 INIT_WORK(&m->process_queued_bios, process_queued_bios);
 216                 /*
 217                  * bio-based doesn't support any direct scsi_dh management;
 218                  * it just discovers if a scsi_dh is attached.
 219                  */
 220                 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
 221         }
 222
 223         dm_table_set_type(ti->table, m->queue_mode);
 224
 225         /*
 226          * Init fields that are only used when a scsi_dh is attached
 227          * - must do this unconditionally (really doesn't hurt non-SCSI uses)
 228          */
 229         set_bit(MPATHF_QUEUE_IO, &m->flags);
 230         atomic_set(&m->pg_init_in_progress, 0);
 231         atomic_set(&m->pg_init_count, 0);
 232         m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 233         init_waitqueue_head(&m->pg_init_wait);
 234
 235         return 0;
 236 }
 237
 238 static void free_multipath(struct multipath *m)
 239 {
 240         struct priority_group *pg, *tmp;
 241
 242         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 243                 list_del(&pg->list);
 244                 free_priority_group(pg, m->ti);
 245         }
 246
 247         kfree(m->hw_handler_name);
 248         kfree(m->hw_handler_params);
 249         mutex_destroy(&m->work_mutex);
 250         kfree(m);
 251 }
 252
 253 static struct dm_mpath_io *get_mpio(union map_info *info)
 254 {
 255         return info->ptr;
 256 }
 257
 258 static size_t multipath_per_bio_data_size(void)
 259 {
 260         return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
 261 }
 262
 263 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
 264 {
 265         return dm_per_bio_data(bio, multipath_per_bio_data_size());
 266 }
 267
 268 static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
 269 {
 270         /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
 271         void *bio_details = mpio + 1;
 272         return bio_details;
 273 }
 274
 275 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
 276 {
 277         struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 278         struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
 279
 280         mpio->nr_bytes = bio->bi_iter.bi_size;
 281         mpio->pgpath = NULL;
 282         *mpio_p = mpio;
 283
 284         dm_bio_record(bio_details, bio);
 285 }
 286
 287 /*-----------------------------------------------
 288  * Path selection
 289  *-----------------------------------------------*/
 290
 291 static int __pg_init_all_paths(struct multipath *m)
 292 {
 293         struct pgpath *pgpath;
 294         unsigned long pg_init_delay = 0;
 295
 296         lockdep_assert_held(&m->lock);
 297
 298         if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
 299                 return 0;
 300
 301         atomic_inc(&m->pg_init_count);
 302         clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 303
 304         /* Check here to reset pg_init_required */
 305         if (!m->current_pg)
 306                 return 0;
 307
 308         if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
 309                 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
 310                                                  m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
 311         list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
 312                 /* Skip failed paths */
 313                 if (!pgpath->is_active)
 314                         continue;
 315                 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
 316                                        pg_init_delay))
 317                         atomic_inc(&m->pg_init_in_progress);
 318         }
 319         return atomic_read(&m->pg_init_in_progress);
 320 }
 321
 322 static int pg_init_all_paths(struct multipath *m)
 323 {
 324         int ret;
 325         unsigned long flags;
 326
 327         spin_lock_irqsave(&m->lock, flags);
 328         ret = __pg_init_all_paths(m);
 329         spin_unlock_irqrestore(&m->lock, flags);
 330
 331         return ret;
 332 }
 333
 334 static void __switch_pg(struct multipath *m, struct priority_group *pg)
 335 {
 336         m->current_pg = pg;
 337
 338         /* Must we initialise the PG first, and queue I/O till it's ready? */
 339         if (m->hw_handler_name) {
 340                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 341                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 342         } else {
 343                 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 344                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 345         }
 346
 347         atomic_set(&m->pg_init_count, 0);
 348 }
 349
 350 static struct pgpath *choose_path_in_pg(struct multipath *m,
 351                                         struct priority_group *pg,
 352                                         size_t nr_bytes)
 353 {
 354         unsigned long flags;
 355         struct dm_path *path;
 356         struct pgpath *pgpath;
 357
 358         path = pg->ps.type->select_path(&pg->ps, nr_bytes);
 359         if (!path)
 360                 return ERR_PTR(-ENXIO);
 361
 362         pgpath = path_to_pgpath(path);
 363
 364         if (unlikely(READ_ONCE(m->current_pg) != pg)) {
 365                 /* Only update current_pgpath if pg changed */
 366                 spin_lock_irqsave(&m->lock, flags);
 367                 m->current_pgpath = pgpath;
 368                 __switch_pg(m, pg);
 369                 spin_unlock_irqrestore(&m->lock, flags);
 370         }
 371
 372         return pgpath;
 373 }
 374
 375 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
 376 {
 377         unsigned long flags;
 378         struct priority_group *pg;
 379         struct pgpath *pgpath;
 380         unsigned bypassed = 1;
 381
 382         if (!atomic_read(&m->nr_valid_paths)) {
 383                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 384                 goto failed;
 385         }
 386
 387         /* Were we instructed to switch PG? */
 388         if (READ_ONCE(m->next_pg)) {
 389                 spin_lock_irqsave(&m->lock, flags);
 390                 pg = m->next_pg;
 391                 if (!pg) {
 392                         spin_unlock_irqrestore(&m->lock, flags);
 393                         goto check_current_pg;
 394                 }
 395                 m->next_pg = NULL;
 396                 spin_unlock_irqrestore(&m->lock, flags);
 397                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 398                 if (!IS_ERR_OR_NULL(pgpath))
 399                         return pgpath;
 400         }
 401
 402         /* Don't change PG until it has no remaining paths */
 403 check_current_pg:
 404         pg = READ_ONCE(m->current_pg);
 405         if (pg) {
 406                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 407                 if (!IS_ERR_OR_NULL(pgpath))
 408                         return pgpath;
 409         }
 410
 411         /*
 412          * Loop through priority groups until we find a valid path.
 413          * First time we skip PGs marked 'bypassed'.
 414          * Second time we only try the ones we skipped, but set
 415          * pg_init_delay_retry so we do not hammer controllers.
 416          */
 417         do {
 418                 list_for_each_entry(pg, &m->priority_groups, list) {
 419                         if (pg->bypassed == !!bypassed)
 420                                 continue;
 421                         pgpath = choose_path_in_pg(m, pg, nr_bytes);
 422                         if (!IS_ERR_OR_NULL(pgpath)) {
 423                                 if (!bypassed)
 424                                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
 425                                 return pgpath;
 426                         }
 427                 }
 428         } while (bypassed--);
 429
 430 failed:
 431         spin_lock_irqsave(&m->lock, flags);
 432         m->current_pgpath = NULL;
 433         m->current_pg = NULL;
 434         spin_unlock_irqrestore(&m->lock, flags);
 435
 436         return NULL;
 437 }
 438
 439 /*
 440  * dm_report_EIO() is a macro instead of a function to make pr_debug()
 441  * report the function name and line number of the function from which
 442  * it has been invoked.
 443  */
 444 #define dm_report_EIO(m)                                                \
 445 do {                                                                    \
 446         struct mapped_device *md = dm_table_get_md((m)->ti->table);     \
 447                                                                         \
 448         pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \
 449                  dm_device_name(md),                                    \
 450                  test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags),        \
 451                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags),  \
 452                  dm_noflush_suspending((m)->ti));                       \
 453 } while (0)
 454
 455 /*
 456  * Check whether bios must be queued in the device-mapper core rather
 457  * than here in the target.
 458  *
 459  * If MPATHF_QUEUE_IF_NO_PATH and MPATHF_SAVED_QUEUE_IF_NO_PATH hold
 460  * the same value then we are not between multipath_presuspend()
 461  * and multipath_resume() calls and we have no need to check
 462  * for the DMF_NOFLUSH_SUSPENDING flag.
 463  */
 464 static bool __must_push_back(struct multipath *m, unsigned long flags)
 465 {
 466         return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) !=
 467                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &flags)) &&
 468                 dm_noflush_suspending(m->ti));
 469 }
 470
 471 /*
 472  * Following functions use READ_ONCE to get atomic access to
 473  * all m->flags to avoid taking spinlock
 474  */
 475 static bool must_push_back_rq(struct multipath *m)
 476 {
 477         unsigned long flags = READ_ONCE(m->flags);
 478         return test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) || __must_push_back(m, flags);
 479 }
 480
 481 static bool must_push_back_bio(struct multipath *m)
 482 {
 483         unsigned long flags = READ_ONCE(m->flags);
 484         return __must_push_back(m, flags);
 485 }
 486
 487 /*
 488  * Map cloned requests (request-based multipath)
 489  */
 490 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 491                                    union map_info *map_context,
 492                                    struct request **__clone)
 493 {
 494         struct multipath *m = ti->private;
 495         size_t nr_bytes = blk_rq_bytes(rq);
 496         struct pgpath *pgpath;
 497         struct block_device *bdev;
 498         struct dm_mpath_io *mpio = get_mpio(map_context);
 499         struct request_queue *q;
 500         struct request *clone;
 501
 502         /* Do we need to select a new pgpath? */
 503         pgpath = READ_ONCE(m->current_pgpath);
 504         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 505                 pgpath = choose_pgpath(m, nr_bytes);
 506
 507         if (!pgpath) {
 508                 if (must_push_back_rq(m))
 509                         return DM_MAPIO_DELAY_REQUEUE;
 510                 dm_report_EIO(m);       /* Failed */
 511                 return DM_MAPIO_KILL;
 512         } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 513                    test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
 514                 pg_init_all_paths(m);
 515                 return DM_MAPIO_DELAY_REQUEUE;
 516         }
 517
 518         mpio->pgpath = pgpath;
 519         mpio->nr_bytes = nr_bytes;
 520
 521         bdev = pgpath->path.dev->bdev;
 522         q = bdev_get_queue(bdev);
 523         clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE,
 524                         BLK_MQ_REQ_NOWAIT);
 525         if (IS_ERR(clone)) {
 526                 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 527                 if (blk_queue_dying(q)) {
 528                         atomic_inc(&m->pg_init_in_progress);
 529                         activate_or_offline_path(pgpath);
 530                         return DM_MAPIO_DELAY_REQUEUE;
 531                 }
 532
 533                 /*
 534                  * blk-mq's SCHED_RESTART can cover this requeue, so we
 535                  * needn't deal with it by DELAY_REQUEUE. More importantly,
 536                  * we have to return DM_MAPIO_REQUEUE so that blk-mq can
 537                  * get the queue busy feedback (via BLK_STS_RESOURCE),
 538                  * otherwise I/O merging can suffer.
 539                  */
 540                 if (q->mq_ops)
 541                         return DM_MAPIO_REQUEUE;
 542                 else
 543                         return DM_MAPIO_DELAY_REQUEUE;
 544         }
 545         clone->bio = clone->biotail = NULL;
 546         clone->rq_disk = bdev->bd_disk;
 547         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 548         *__clone = clone;
 549
 550         if (pgpath->pg->ps.type->start_io)
 551                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 552                                               &pgpath->path,
 553                                               nr_bytes);
 554         return DM_MAPIO_REMAPPED;
 555 }
 556
 557 static void multipath_release_clone(struct request *clone)
 558 {
 559         blk_put_request(clone);
 560 }
 561
 562 /*
 563  * Map cloned bios (bio-based multipath)
 564  */
 565
 566 static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
 567 {
 568         struct pgpath *pgpath;
 569         unsigned long flags;
 570         bool queue_io;
 571
 572         /* Do we need to select a new pgpath? */
 573         pgpath = READ_ONCE(m->current_pgpath);
 574         queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
 575         if (!pgpath || !queue_io)
 576                 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
 577
 578         if ((pgpath && queue_io) ||
 579             (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
 580                 /* Queue for the daemon to resubmit */
 581                 spin_lock_irqsave(&m->lock, flags);
 582                 bio_list_add(&m->queued_bios, bio);
 583                 spin_unlock_irqrestore(&m->lock, flags);
 584
 585                 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
 586                 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 587                         pg_init_all_paths(m);
 588                 else if (!queue_io)
 589                         queue_work(kmultipathd, &m->process_queued_bios);
 590
 591                 return ERR_PTR(-EAGAIN);
 592         }
 593
 594         return pgpath;
 595 }
 596
 597 static struct pgpath *__map_bio_fast(struct multipath *m, struct bio *bio)
 598 {
 599         struct pgpath *pgpath;
 600         unsigned long flags;
 601
 602         /* Do we need to select a new pgpath? */
 603         /*
 604          * FIXME: currently only switching path if no path (due to failure, etc)
 605          * - which negates the point of using a path selector
 606          */
 607         pgpath = READ_ONCE(m->current_pgpath);
 608         if (!pgpath)
 609                 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
 610
 611         if (!pgpath) {
 612                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
 613                         /* Queue for the daemon to resubmit */
 614                         spin_lock_irqsave(&m->lock, flags);
 615                         bio_list_add(&m->queued_bios, bio);
 616                         spin_unlock_irqrestore(&m->lock, flags);
 617                         queue_work(kmultipathd, &m->process_queued_bios);
 618
 619                         return ERR_PTR(-EAGAIN);
 620                 }
 621                 return NULL;
 622         }
 623
 624         return pgpath;
 625 }
 626
 627 static int __multipath_map_bio(struct multipath *m, struct bio *bio,
 628                                struct dm_mpath_io *mpio)
 629 {
 630         struct pgpath *pgpath;
 631
 632         if (!m->hw_handler_name)
 633                 pgpath = __map_bio_fast(m, bio);
 634         else
 635                 pgpath = __map_bio(m, bio);
 636
 637         if (IS_ERR(pgpath))
 638                 return DM_MAPIO_SUBMITTED;
 639
 640         if (!pgpath) {
 641                 if (must_push_back_bio(m))
 642                         return DM_MAPIO_REQUEUE;
 643                 dm_report_EIO(m);
 644                 return DM_MAPIO_KILL;
 645         }
 646
 647         mpio->pgpath = pgpath;
 648
 649         bio->bi_status = 0;
 650         bio_set_dev(bio, pgpath->path.dev->bdev);
 651         bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 652
 653         if (pgpath->pg->ps.type->start_io)
 654                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 655                                               &pgpath->path,
 656                                               mpio->nr_bytes);
 657         return DM_MAPIO_REMAPPED;
 658 }
 659
 660 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 661 {
 662         struct multipath *m = ti->private;
 663         struct dm_mpath_io *mpio = NULL;
 664
 665         multipath_init_per_bio_data(bio, &mpio);
 666         return __multipath_map_bio(m, bio, mpio);
 667 }
 668
 669 static void process_queued_io_list(struct multipath *m)
 670 {
 671         if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
 672                 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
 673         else if (m->queue_mode == DM_TYPE_BIO_BASED)
 674                 queue_work(kmultipathd, &m->process_queued_bios);
 675 }
 676
 677 static void process_queued_bios(struct work_struct *work)
 678 {
 679         int r;
 680         unsigned long flags;
 681         struct bio *bio;
 682         struct bio_list bios;
 683         struct blk_plug plug;
 684         struct multipath *m =
 685                 container_of(work, struct multipath, process_queued_bios);
 686
 687         bio_list_init(&bios);
 688
 689         spin_lock_irqsave(&m->lock, flags);
 690
 691         if (bio_list_empty(&m->queued_bios)) {
 692                 spin_unlock_irqrestore(&m->lock, flags);
 693                 return;
 694         }
 695
 696         bio_list_merge(&bios, &m->queued_bios);
 697         bio_list_init(&m->queued_bios);
 698
 699         spin_unlock_irqrestore(&m->lock, flags);
 700
 701         blk_start_plug(&plug);
 702         while ((bio = bio_list_pop(&bios))) {
 703                 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 704                 dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
 705                 r = __multipath_map_bio(m, bio, mpio);
 706                 switch (r) {
 707                 case DM_MAPIO_KILL:
 708                         bio->bi_status = BLK_STS_IOERR;
 709                         bio_endio(bio);
 710                         break;
 711                 case DM_MAPIO_REQUEUE:
 712                         bio->bi_status = BLK_STS_DM_REQUEUE;
 713                         bio_endio(bio);
 714                         break;
 715                 case DM_MAPIO_REMAPPED:
 716                         generic_make_request(bio);
 717                         break;
 718                 case DM_MAPIO_SUBMITTED:
 719                         break;
 720                 default:
 721                         WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
 722                 }
 723         }
 724         blk_finish_plug(&plug);
 725 }
 726
 727 /*
 728  * If we run out of usable paths, should we queue I/O or error it?
 729  */
 730 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
 731                             bool save_old_value)
 732 {
 733         unsigned long flags;
 734
 735         spin_lock_irqsave(&m->lock, flags);
 736         assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags,
 737                    (save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) ||
 738                    (!save_old_value && queue_if_no_path));
 739         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
 740         spin_unlock_irqrestore(&m->lock, flags);
 741
 742         if (!queue_if_no_path) {
 743                 dm_table_run_md_queue_async(m->ti->table);
 744                 process_queued_io_list(m);
 745         }
 746
 747         return 0;
 748 }
 749
 750 /*
 751  * An event is triggered whenever a path is taken out of use.
 752  * Includes path failure and PG bypass.
 753  */
 754 static void trigger_event(struct work_struct *work)
 755 {
 756         struct multipath *m =
 757                 container_of(work, struct multipath, trigger_event);
 758
 759         dm_table_event(m->ti->table);
 760 }
 761
 762 /*-----------------------------------------------------------------
 763  * Constructor/argument parsing:
 764  * <#multipath feature args> [<arg>]*
 765  * <#hw_handler args> [hw_handler [<arg>]*]
 766  * <#priority groups>
 767  * <initial priority group>
 768  *     [<selector> <#selector args> [<arg>]*
 769  *      <#paths> <#per-path selector args>
 770  *         [<path> [<arg>]* ]+ ]+
 771  *---------------------------------------------------------------*/
 772 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 773                                struct dm_target *ti)
 774 {
 775         int r;
 776         struct path_selector_type *pst;
 777         unsigned ps_argc;
 778
 779         static const struct dm_arg _args[] = {
 780                 {0, 1024, "invalid number of path selector args"},
 781         };
 782
 783         pst = dm_get_path_selector(dm_shift_arg(as));
 784         if (!pst) {
 785                 ti->error = "unknown path selector type";
 786                 return -EINVAL;
 787         }
 788
 789         r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
 790         if (r) {
 791                 dm_put_path_selector(pst);
 792                 return -EINVAL;
 793         }
 794
 795         r = pst->create(&pg->ps, ps_argc, as->argv);
 796         if (r) {
 797                 dm_put_path_selector(pst);
 798                 ti->error = "path selector constructor failed";
 799                 return r;
 800         }
 801
 802         pg->ps.type = pst;
 803         dm_consume_args(as, ps_argc);
 804
 805         return 0;
 806 }
 807
 808 static int setup_scsi_dh(struct block_device *bdev, struct multipath *m,
 809                          const char *attached_handler_name, char **error)
 810 {
 811         struct request_queue *q = bdev_get_queue(bdev);
 812         int r;
 813
 814         if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
 815 retain:
 816                 if (attached_handler_name) {
 817                         /*
 818                          * Clear any hw_handler_params associated with a
 819                          * handler that isn't already attached.
 820                          */
 821                         if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) {
 822                                 kfree(m->hw_handler_params);
 823                                 m->hw_handler_params = NULL;
 824                         }
 825
 826                         /*
 827                          * Reset hw_handler_name to match the attached handler
 828                          *
 829                          * NB. This modifies the table line to show the actual
 830                          * handler instead of the original table passed in.
 831                          */
 832                         kfree(m->hw_handler_name);
 833                         m->hw_handler_name = attached_handler_name;
 834                 }
 835         }
 836
 837         if (m->hw_handler_name) {
 838                 r = scsi_dh_attach(q, m->hw_handler_name);
 839                 if (r == -EBUSY) {
 840                         char b[BDEVNAME_SIZE];
 841
 842                         printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
 843                                bdevname(bdev, b));
 844                         goto retain;
 845                 }
 846                 if (r < 0) {
 847                         *error = "error attaching hardware handler";
 848                         return r;
 849                 }
 850
 851                 if (m->hw_handler_params) {
 852                         r = scsi_dh_set_params(q, m->hw_handler_params);
 853                         if (r < 0) {
 854                                 *error = "unable to set hardware handler parameters";
 855                                 return r;
 856                         }
 857                 }
 858         }
 859
 860         return 0;
 861 }
 862
 863 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
 864                                  struct dm_target *ti)
 865 {
 866         int r;
 867         struct pgpath *p;
 868         struct multipath *m = ti->private;
 869         struct request_queue *q;
 870         const char *attached_handler_name;
 871
 872         /* we need at least a path arg */
 873         if (as->argc < 1) {
 874                 ti->error = "no device given";
 875                 return ERR_PTR(-EINVAL);
 876         }
 877
 878         p = alloc_pgpath();
 879         if (!p)
 880                 return ERR_PTR(-ENOMEM);
 881
 882         r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
 883                           &p->path.dev);
 884         if (r) {
 885                 ti->error = "error getting device";
 886                 goto bad;
 887         }
 888
 889         q = bdev_get_queue(p->path.dev->bdev);
 890         attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 891         if (attached_handler_name || m->hw_handler_name) {
 892                 INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
 893                 r = setup_scsi_dh(p->path.dev->bdev, m, attached_handler_name, &ti->error);
 894                 if (r) {
 895                         dm_put_device(ti, p->path.dev);
 896                         goto bad;
 897                 }
 898         }
 899
 900         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 901         if (r) {
 902                 dm_put_device(ti, p->path.dev);
 903                 goto bad;
 904         }
 905
 906         return p;
 907  bad:
 908         free_pgpath(p);
 909         return ERR_PTR(r);
 910 }
 911
 912 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 913                                                    struct multipath *m)
 914 {
 915         static const struct dm_arg _args[] = {
 916                 {1, 1024, "invalid number of paths"},
 917                 {0, 1024, "invalid number of selector args"}
 918         };
 919
 920         int r;
 921         unsigned i, nr_selector_args, nr_args;
 922         struct priority_group *pg;
 923         struct dm_target *ti = m->ti;
 924
 925         if (as->argc < 2) {
 926                 as->argc = 0;
 927                 ti->error = "not enough priority group arguments";
 928                 return ERR_PTR(-EINVAL);
 929         }
 930
 931         pg = alloc_priority_group();
 932         if (!pg) {
 933                 ti->error = "couldn't allocate priority group";
 934                 return ERR_PTR(-ENOMEM);
 935         }
 936         pg->m = m;
 937
 938         r = parse_path_selector(as, pg, ti);
 939         if (r)
 940                 goto bad;
 941
 942         /*
 943          * read the paths
 944          */
 945         r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
 946         if (r)
 947                 goto bad;
 948
 949         r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
 950         if (r)
 951                 goto bad;
 952
 953         nr_args = 1 + nr_selector_args;
 954         for (i = 0; i < pg->nr_pgpaths; i++) {
 955                 struct pgpath *pgpath;
 956                 struct dm_arg_set path_args;
 957
 958                 if (as->argc < nr_args) {
 959                         ti->error = "not enough path parameters";
 960                         r = -EINVAL;
 961                         goto bad;
 962                 }
 963
 964                 path_args.argc = nr_args;
 965                 path_args.argv = as->argv;
 966
 967                 pgpath = parse_path(&path_args, &pg->ps, ti);
 968                 if (IS_ERR(pgpath)) {
 969                         r = PTR_ERR(pgpath);
 970                         goto bad;
 971                 }
 972
 973                 pgpath->pg = pg;
 974                 list_add_tail(&pgpath->list, &pg->pgpaths);
 975                 dm_consume_args(as, nr_args);
 976         }
 977
 978         return pg;
 979
 980  bad:
 981         free_priority_group(pg, ti);
 982         return ERR_PTR(r);
 983 }
 984
 985 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 986 {
 987         unsigned hw_argc;
 988         int ret;
 989         struct dm_target *ti = m->ti;
 990
 991         static const struct dm_arg _args[] = {
 992                 {0, 1024, "invalid number of hardware handler args"},
 993         };
 994
 995         if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
 996                 return -EINVAL;
 997
 998         if (!hw_argc)
 999                 return 0;
1000
1001         if (m->queue_mode == DM_TYPE_BIO_BASED) {
1002                 dm_consume_args(as, hw_argc);
1003                 DMERR("bio-based multipath doesn't allow hardware handler args");
1004                 return 0;
1005         }
1006
1007         m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
1008         if (!m->hw_handler_name)
1009                 return -EINVAL;
1010
1011         if (hw_argc > 1) {
1012                 char *p;
1013                 int i, j, len = 4;
1014
1015                 for (i = 0; i <= hw_argc - 2; i++)
1016                         len += strlen(as->argv[i]) + 1;
1017                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
1018                 if (!p) {
1019                         ti->error = "memory allocation failed";
1020                         ret = -ENOMEM;
1021                         goto fail;
1022                 }
1023                 j = sprintf(p, "%d", hw_argc - 1);
1024                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
1025                         j = sprintf(p, "%s", as->argv[i]);
1026         }
1027         dm_consume_args(as, hw_argc - 1);
1028
1029         return 0;
1030 fail:
1031         kfree(m->hw_handler_name);
1032         m->hw_handler_name = NULL;
1033         return ret;
1034 }
1035
1036 static int parse_features(struct dm_arg_set *as, struct multipath *m)
1037 {
1038         int r;
1039         unsigned argc;
1040         struct dm_target *ti = m->ti;
1041         const char *arg_name;
1042
1043         static const struct dm_arg _args[] = {
1044                 {0, 8, "invalid number of feature args"},
1045                 {1, 50, "pg_init_retries must be between 1 and 50"},
1046                 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
1047         };
1048
1049         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1050         if (r)
1051                 return -EINVAL;
1052
1053         if (!argc)
1054                 return 0;
1055
1056         do {
1057                 arg_name = dm_shift_arg(as);
1058                 argc--;
1059
1060                 if (!strcasecmp(arg_name, "queue_if_no_path")) {
1061                         r = queue_if_no_path(m, true, false);
1062                         continue;
1063                 }
1064
1065                 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
1066                         set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
1067                         continue;
1068                 }
1069
1070                 if (!strcasecmp(arg_name, "pg_init_retries") &&
1071                     (argc >= 1)) {
1072                         r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
1073                         argc--;
1074                         continue;
1075                 }
1076
1077                 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
1078                     (argc >= 1)) {
1079                         r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
1080                         argc--;
1081                         continue;
1082                 }
1083
1084                 if (!strcasecmp(arg_name, "queue_mode") &&
1085                     (argc >= 1)) {
1086                         const char *queue_mode_name = dm_shift_arg(as);
1087
1088                         if (!strcasecmp(queue_mode_name, "bio"))
1089                                 m->queue_mode = DM_TYPE_BIO_BASED;
1090                         else if (!strcasecmp(queue_mode_name, "rq"))
1091                                 m->queue_mode = DM_TYPE_REQUEST_BASED;
1092                         else if (!strcasecmp(queue_mode_name, "mq"))
1093                                 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1094                         else {
1095                                 ti->error = "Unknown 'queue_mode' requested";
1096                                 r = -EINVAL;
1097                         }
1098                         argc--;
1099                         continue;
1100                 }
1101
1102                 ti->error = "Unrecognised multipath feature request";
1103                 r = -EINVAL;
1104         } while (argc && !r);
1105
1106         return r;
1107 }
1108
1109 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1110 {
1111         /* target arguments */
1112         static const struct dm_arg _args[] = {
1113                 {0, 1024, "invalid number of priority groups"},
1114                 {0, 1024, "invalid initial priority group number"},
1115         };
1116
1117         int r;
1118         struct multipath *m;
1119         struct dm_arg_set as;
1120         unsigned pg_count = 0;
1121         unsigned next_pg_num;
1122
1123         as.argc = argc;
1124         as.argv = argv;
1125
1126         m = alloc_multipath(ti);
1127         if (!m) {
1128                 ti->error = "can't allocate multipath";
1129                 return -EINVAL;
1130         }
1131
1132         r = parse_features(&as, m);
1133         if (r)
1134                 goto bad;
1135
1136         r = alloc_multipath_stage2(ti, m);
1137         if (r)
1138                 goto bad;
1139
1140         r = parse_hw_handler(&as, m);
1141         if (r)
1142                 goto bad;
1143
1144         r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1145         if (r)
1146                 goto bad;
1147
1148         r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1149         if (r)
1150                 goto bad;
1151
1152         if ((!m->nr_priority_groups && next_pg_num) ||
1153             (m->nr_priority_groups && !next_pg_num)) {
1154                 ti->error = "invalid initial priority group";
1155                 r = -EINVAL;
1156                 goto bad;
1157         }
1158
1159         /* parse the priority groups */
1160         while (as.argc) {
1161                 struct priority_group *pg;
1162                 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1163
1164                 pg = parse_priority_group(&as, m);
1165                 if (IS_ERR(pg)) {
1166                         r = PTR_ERR(pg);
1167                         goto bad;
1168                 }
1169
1170                 nr_valid_paths += pg->nr_pgpaths;
1171                 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1172
1173                 list_add_tail(&pg->list, &m->priority_groups);
1174                 pg_count++;
1175                 pg->pg_num = pg_count;
1176                 if (!--next_pg_num)
1177                         m->next_pg = pg;
1178         }
1179
1180         if (pg_count != m->nr_priority_groups) {
1181                 ti->error = "priority group count mismatch";
1182                 r = -EINVAL;
1183                 goto bad;
1184         }
1185
1186         ti->num_flush_bios = 1;
1187         ti->num_discard_bios = 1;
1188         ti->num_write_same_bios = 1;
1189         ti->num_write_zeroes_bios = 1;
1190         if (m->queue_mode == DM_TYPE_BIO_BASED)
1191                 ti->per_io_data_size = multipath_per_bio_data_size();
1192         else
1193                 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1194
1195         return 0;
1196
1197  bad:
1198         free_multipath(m);
1199         return r;
1200 }
1201
1202 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1203 {
1204         DEFINE_WAIT(wait);
1205
1206         while (1) {
1207                 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1208
1209                 if (!atomic_read(&m->pg_init_in_progress))
1210                         break;
1211
1212                 io_schedule();
1213         }
1214         finish_wait(&m->pg_init_wait, &wait);
1215 }
1216
1217 static void flush_multipath_work(struct multipath *m)
1218 {
1219         if (m->hw_handler_name) {
1220                 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1221                 smp_mb__after_atomic();
1222
1223                 flush_workqueue(kmpath_handlerd);
1224                 multipath_wait_for_pg_init_completion(m);
1225
1226                 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1227                 smp_mb__after_atomic();
1228         }
1229
1230         flush_workqueue(kmultipathd);
1231         flush_work(&m->trigger_event);
1232 }
1233
1234 static void multipath_dtr(struct dm_target *ti)
1235 {
1236         struct multipath *m = ti->private;
1237
1238         flush_multipath_work(m);
1239         free_multipath(m);
1240 }
1241
1242 /*
1243  * Take a path out of use.
1244  */
1245 static int fail_path(struct pgpath *pgpath)
1246 {
1247         unsigned long flags;
1248         struct multipath *m = pgpath->pg->m;
1249
1250         spin_lock_irqsave(&m->lock, flags);
1251
1252         if (!pgpath->is_active)
1253                 goto out;
1254
1255         DMWARN("Failing path %s.", pgpath->path.dev->name);
1256
1257         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1258         pgpath->is_active = false;
1259         pgpath->fail_count++;
1260
1261         atomic_dec(&m->nr_valid_paths);
1262
1263         if (pgpath == m->current_pgpath)
1264                 m->current_pgpath = NULL;
1265
1266         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1267                        pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1268
1269         schedule_work(&m->trigger_event);
1270
1271 out:
1272         spin_unlock_irqrestore(&m->lock, flags);
1273
1274         return 0;
1275 }
1276
1277 /*
1278  * Reinstate a previously-failed path
1279  */
1280 static int reinstate_path(struct pgpath *pgpath)
1281 {
1282         int r = 0, run_queue = 0;
1283         unsigned long flags;
1284         struct multipath *m = pgpath->pg->m;
1285         unsigned nr_valid_paths;
1286
1287         spin_lock_irqsave(&m->lock, flags);
1288
1289         if (pgpath->is_active)
1290                 goto out;
1291
1292         DMWARN("Reinstating path %s.", pgpath->path.dev->name);
1293
1294         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1295         if (r)
1296                 goto out;
1297
1298         pgpath->is_active = true;
1299
1300         nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1301         if (nr_valid_paths == 1) {
1302                 m->current_pgpath = NULL;
1303                 run_queue = 1;
1304         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1305                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1306                         atomic_inc(&m->pg_init_in_progress);
1307         }
1308
1309         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1310                        pgpath->path.dev->name, nr_valid_paths);
1311
1312         schedule_work(&m->trigger_event);
1313
1314 out:
1315         spin_unlock_irqrestore(&m->lock, flags);
1316         if (run_queue) {
1317                 dm_table_run_md_queue_async(m->ti->table);
1318                 process_queued_io_list(m);
1319         }
1320
1321         return r;
1322 }
1323
1324 /*
1325  * Fail or reinstate all paths that match the provided struct dm_dev.
1326  */
1327 static int action_dev(struct multipath *m, struct dm_dev *dev,
1328                       action_fn action)
1329 {
1330         int r = -EINVAL;
1331         struct pgpath *pgpath;
1332         struct priority_group *pg;
1333
1334         list_for_each_entry(pg, &m->priority_groups, list) {
1335                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1336                         if (pgpath->path.dev == dev)
1337                                 r = action(pgpath);
1338                 }
1339         }
1340
1341         return r;
1342 }
1343
1344 /*
1345  * Temporarily try to avoid having to use the specified PG
1346  */
1347 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1348                       bool bypassed)
1349 {
1350         unsigned long flags;
1351
1352         spin_lock_irqsave(&m->lock, flags);
1353
1354         pg->bypassed = bypassed;
1355         m->current_pgpath = NULL;
1356         m->current_pg = NULL;
1357
1358         spin_unlock_irqrestore(&m->lock, flags);
1359
1360         schedule_work(&m->trigger_event);
1361 }
1362
1363 /*
1364  * Switch to using the specified PG from the next I/O that gets mapped
1365  */
1366 static int switch_pg_num(struct multipath *m, const char *pgstr)
1367 {
1368         struct priority_group *pg;
1369         unsigned pgnum;
1370         unsigned long flags;
1371         char dummy;
1372
1373         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1374             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1375                 DMWARN("invalid PG number supplied to switch_pg_num");
1376                 return -EINVAL;
1377         }
1378
1379         spin_lock_irqsave(&m->lock, flags);
1380         list_for_each_entry(pg, &m->priority_groups, list) {
1381                 pg->bypassed = false;
1382                 if (--pgnum)
1383                         continue;
1384
1385                 m->current_pgpath = NULL;
1386                 m->current_pg = NULL;
1387                 m->next_pg = pg;
1388         }
1389         spin_unlock_irqrestore(&m->lock, flags);
1390
1391         schedule_work(&m->trigger_event);
1392         return 0;
1393 }
1394
1395 /*
1396  * Set/clear bypassed status of a PG.
1397  * PGs are numbered upwards from 1 in the order they were declared.
1398  */
1399 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1400 {
1401         struct priority_group *pg;
1402         unsigned pgnum;
1403         char dummy;
1404
1405         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1406             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1407                 DMWARN("invalid PG number supplied to bypass_pg");
1408                 return -EINVAL;
1409         }
1410
1411         list_for_each_entry(pg, &m->priority_groups, list) {
1412                 if (!--pgnum)
1413                         break;
1414         }
1415
1416         bypass_pg(m, pg, bypassed);
1417         return 0;
1418 }
1419
1420 /*
1421  * Should we retry pg_init immediately?
1422  */
1423 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1424 {
1425         unsigned long flags;
1426         bool limit_reached = false;
1427
1428         spin_lock_irqsave(&m->lock, flags);
1429
1430         if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1431             !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1432                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1433         else
1434                 limit_reached = true;
1435
1436         spin_unlock_irqrestore(&m->lock, flags);
1437
1438         return limit_reached;
1439 }
1440
1441 static void pg_init_done(void *data, int errors)
1442 {
1443         struct pgpath *pgpath = data;
1444         struct priority_group *pg = pgpath->pg;
1445         struct multipath *m = pg->m;
1446         unsigned long flags;
1447         bool delay_retry = false;
1448
1449         /* device or driver problems */
1450         switch (errors) {
1451         case SCSI_DH_OK:
1452                 break;
1453         case SCSI_DH_NOSYS:
1454                 if (!m->hw_handler_name) {
1455                         errors = 0;
1456                         break;
1457                 }
1458                 DMERR("Could not failover the device: Handler scsi_dh_%s "
1459                       "Error %d.", m->hw_handler_name, errors);
1460                 /*
1461                  * Fail path for now, so we do not ping pong
1462                  */
1463                 fail_path(pgpath);
1464                 break;
1465         case SCSI_DH_DEV_TEMP_BUSY:
1466                 /*
1467                  * Probably doing something like FW upgrade on the
1468                  * controller so try the other pg.
1469                  */
1470                 bypass_pg(m, pg, true);
1471                 break;
1472         case SCSI_DH_RETRY:
1473                 /* Wait before retrying. */
1474                 delay_retry = 1;
1475                 /* fall through */
1476         case SCSI_DH_IMM_RETRY:
1477         case SCSI_DH_RES_TEMP_UNAVAIL:
1478                 if (pg_init_limit_reached(m, pgpath))
1479                         fail_path(pgpath);
1480                 errors = 0;
1481                 break;
1482         case SCSI_DH_DEV_OFFLINED:
1483         default:
1484                 /*
1485                  * We probably do not want to fail the path for a device
1486                  * error, but this is what the old dm did. In future
1487                  * patches we can do more advanced handling.
1488                  */
1489                 fail_path(pgpath);
1490         }
1491
1492         spin_lock_irqsave(&m->lock, flags);
1493         if (errors) {
1494                 if (pgpath == m->current_pgpath) {
1495                         DMERR("Could not failover device. Error %d.", errors);
1496                         m->current_pgpath = NULL;
1497                         m->current_pg = NULL;
1498                 }
1499         } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1500                 pg->bypassed = false;
1501
1502         if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1503                 /* Activations of other paths are still on going */
1504                 goto out;
1505
1506         if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1507                 if (delay_retry)
1508                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1509                 else
1510                         clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1511
1512                 if (__pg_init_all_paths(m))
1513                         goto out;
1514         }
1515         clear_bit(MPATHF_QUEUE_IO, &m->flags);
1516
1517         process_queued_io_list(m);
1518
1519         /*
1520          * Wake up any thread waiting to suspend.
1521          */
1522         wake_up(&m->pg_init_wait);
1523
1524 out:
1525         spin_unlock_irqrestore(&m->lock, flags);
1526 }
1527
1528 static void activate_or_offline_path(struct pgpath *pgpath)
1529 {
1530         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1531
1532         if (pgpath->is_active && !blk_queue_dying(q))
1533                 scsi_dh_activate(q, pg_init_done, pgpath);
1534         else
1535                 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1536 }
1537
1538 static void activate_path_work(struct work_struct *work)
1539 {
1540         struct pgpath *pgpath =
1541                 container_of(work, struct pgpath, activate_path.work);
1542
1543         activate_or_offline_path(pgpath);
1544 }
1545
1546 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1547                             blk_status_t error, union map_info *map_context)
1548 {
1549         struct dm_mpath_io *mpio = get_mpio(map_context);
1550         struct pgpath *pgpath = mpio->pgpath;
1551         int r = DM_ENDIO_DONE;
1552
1553         /*
1554          * We don't queue any clone request inside the multipath target
1555          * during end I/O handling, since those clone requests don't have
1556          * bio clones.  If we queue them inside the multipath target,
1557          * we need to make bio clones, that requires memory allocation.
1558          * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1559          *  don't have bio clones.)
1560          * Instead of queueing the clone request here, we queue the original
1561          * request into dm core, which will remake a clone request and
1562          * clone bios for it and resubmit it later.
1563          */
1564         if (error && blk_path_error(error)) {
1565                 struct multipath *m = ti->private;
1566
1567                 if (error == BLK_STS_RESOURCE)
1568                         r = DM_ENDIO_DELAY_REQUEUE;
1569                 else
1570                         r = DM_ENDIO_REQUEUE;
1571
1572                 if (pgpath)
1573                         fail_path(pgpath);
1574
1575                 if (atomic_read(&m->nr_valid_paths) == 0 &&
1576                     !must_push_back_rq(m)) {
1577                         if (error == BLK_STS_IOERR)
1578                                 dm_report_EIO(m);
1579                         /* complete with the original error */
1580                         r = DM_ENDIO_DONE;
1581                 }
1582         }
1583
1584         if (pgpath) {
1585                 struct path_selector *ps = &pgpath->pg->ps;
1586
1587                 if (ps->type->end_io)
1588                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1589         }
1590
1591         return r;
1592 }
1593
1594 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1595                                 blk_status_t *error)
1596 {
1597         struct multipath *m = ti->private;
1598         struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1599         struct pgpath *pgpath = mpio->pgpath;
1600         unsigned long flags;
1601         int r = DM_ENDIO_DONE;
1602
1603         if (!*error || !blk_path_error(*error))
1604                 goto done;
1605
1606         if (pgpath)
1607                 fail_path(pgpath);
1608
1609         if (atomic_read(&m->nr_valid_paths) == 0 &&
1610             !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1611                 if (must_push_back_bio(m)) {
1612                         r = DM_ENDIO_REQUEUE;
1613                 } else {
1614                         dm_report_EIO(m);
1615                         *error = BLK_STS_IOERR;
1616                 }
1617                 goto done;
1618         }
1619
1620         spin_lock_irqsave(&m->lock, flags);
1621         bio_list_add(&m->queued_bios, clone);
1622         spin_unlock_irqrestore(&m->lock, flags);
1623         if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1624                 queue_work(kmultipathd, &m->process_queued_bios);
1625
1626         r = DM_ENDIO_INCOMPLETE;
1627 done:
1628         if (pgpath) {
1629                 struct path_selector *ps = &pgpath->pg->ps;
1630
1631                 if (ps->type->end_io)
1632                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1633         }
1634
1635         return r;
1636 }
1637
1638 /*
1639  * Suspend can't complete until all the I/O is processed so if
1640  * the last path fails we must error any remaining I/O.
1641  * Note that if the freeze_bdev fails while suspending, the
1642  * queue_if_no_path state is lost - userspace should reset it.
1643  */
1644 static void multipath_presuspend(struct dm_target *ti)
1645 {
1646         struct multipath *m = ti->private;
1647
1648         queue_if_no_path(m, false, true);
1649 }
1650
1651 static void multipath_postsuspend(struct dm_target *ti)
1652 {
1653         struct multipath *m = ti->private;
1654
1655         mutex_lock(&m->work_mutex);
1656         flush_multipath_work(m);
1657         mutex_unlock(&m->work_mutex);
1658 }
1659
1660 /*
1661  * Restore the queue_if_no_path setting.
1662  */
1663 static void multipath_resume(struct dm_target *ti)
1664 {
1665         struct multipath *m = ti->private;
1666         unsigned long flags;
1667
1668         spin_lock_irqsave(&m->lock, flags);
1669         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags,
1670                    test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
1671         spin_unlock_irqrestore(&m->lock, flags);
1672 }
1673
1674 /*
1675  * Info output has the following format:
1676  * num_multipath_feature_args [multipath_feature_args]*
1677  * num_handler_status_args [handler_status_args]*
1678  * num_groups init_group_number
1679  *            [A|D|E num_ps_status_args [ps_status_args]*
1680  *             num_paths num_selector_args
1681  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1682  *
1683  * Table output has the following format (identical to the constructor string):
1684  * num_feature_args [features_args]*
1685  * num_handler_args hw_handler [hw_handler_args]*
1686  * num_groups init_group_number
1687  *     [priority selector-name num_ps_args [ps_args]*
1688  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1689  */
1690 static void multipath_status(struct dm_target *ti, status_type_t type,
1691                              unsigned status_flags, char *result, unsigned maxlen)
1692 {
1693         int sz = 0;
1694         unsigned long flags;
1695         struct multipath *m = ti->private;
1696         struct priority_group *pg;
1697         struct pgpath *p;
1698         unsigned pg_num;
1699         char state;
1700
1701         spin_lock_irqsave(&m->lock, flags);
1702
1703         /* Features */
1704         if (type == STATUSTYPE_INFO)
1705                 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1706                        atomic_read(&m->pg_init_count));
1707         else {
1708                 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1709                               (m->pg_init_retries > 0) * 2 +
1710                               (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1711                               test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1712                               (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1713
1714                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1715                         DMEMIT("queue_if_no_path ");
1716                 if (m->pg_init_retries)
1717                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1718                 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1719                         DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1720                 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1721                         DMEMIT("retain_attached_hw_handler ");
1722                 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1723                         switch(m->queue_mode) {
1724                         case DM_TYPE_BIO_BASED:
1725                                 DMEMIT("queue_mode bio ");
1726                                 break;
1727                         case DM_TYPE_MQ_REQUEST_BASED:
1728                                 DMEMIT("queue_mode mq ");
1729                                 break;
1730                         default:
1731                                 WARN_ON_ONCE(true);
1732                                 break;
1733                         }
1734                 }
1735         }
1736
1737         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1738                 DMEMIT("0 ");
1739         else
1740                 DMEMIT("1 %s ", m->hw_handler_name);
1741
1742         DMEMIT("%u ", m->nr_priority_groups);
1743
1744         if (m->next_pg)
1745                 pg_num = m->next_pg->pg_num;
1746         else if (m->current_pg)
1747                 pg_num = m->current_pg->pg_num;
1748         else
1749                 pg_num = (m->nr_priority_groups ? 1 : 0);
1750
1751         DMEMIT("%u ", pg_num);
1752
1753         switch (type) {
1754         case STATUSTYPE_INFO:
1755                 list_for_each_entry(pg, &m->priority_groups, list) {
1756                         if (pg->bypassed)
1757                                 state = 'D';    /* Disabled */
1758                         else if (pg == m->current_pg)
1759                                 state = 'A';    /* Currently Active */
1760                         else
1761                                 state = 'E';    /* Enabled */
1762
1763                         DMEMIT("%c ", state);
1764
1765                         if (pg->ps.type->status)
1766                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1767                                                           result + sz,
1768                                                           maxlen - sz);
1769                         else
1770                                 DMEMIT("0 ");
1771
1772                         DMEMIT("%u %u ", pg->nr_pgpaths,
1773                                pg->ps.type->info_args);
1774
1775                         list_for_each_entry(p, &pg->pgpaths, list) {
1776                                 DMEMIT("%s %s %u ", p->path.dev->name,
1777                                        p->is_active ? "A" : "F",
1778                                        p->fail_count);
1779                                 if (pg->ps.type->status)
1780                                         sz += pg->ps.type->status(&pg->ps,
1781                                               &p->path, type, result + sz,
1782                                               maxlen - sz);
1783                         }
1784                 }
1785                 break;
1786
1787         case STATUSTYPE_TABLE:
1788                 list_for_each_entry(pg, &m->priority_groups, list) {
1789                         DMEMIT("%s ", pg->ps.type->name);
1790
1791                         if (pg->ps.type->status)
1792                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1793                                                           result + sz,
1794                                                           maxlen - sz);
1795                         else
1796                                 DMEMIT("0 ");
1797
1798                         DMEMIT("%u %u ", pg->nr_pgpaths,
1799                                pg->ps.type->table_args);
1800
1801                         list_for_each_entry(p, &pg->pgpaths, list) {
1802                                 DMEMIT("%s ", p->path.dev->name);
1803                                 if (pg->ps.type->status)
1804                                         sz += pg->ps.type->status(&pg->ps,
1805                                               &p->path, type, result + sz,
1806                                               maxlen - sz);
1807                         }
1808                 }
1809                 break;
1810         }
1811
1812         spin_unlock_irqrestore(&m->lock, flags);
1813 }
1814
1815 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv,
1816                              char *result, unsigned maxlen)
1817 {
1818         int r = -EINVAL;
1819         struct dm_dev *dev;
1820         struct multipath *m = ti->private;
1821         action_fn action;
1822
1823         mutex_lock(&m->work_mutex);
1824
1825         if (dm_suspended(ti)) {
1826                 r = -EBUSY;
1827                 goto out;
1828         }
1829
1830         if (argc == 1) {
1831                 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1832                         r = queue_if_no_path(m, true, false);
1833                         goto out;
1834                 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1835                         r = queue_if_no_path(m, false, false);
1836                         goto out;
1837                 }
1838         }
1839
1840         if (argc != 2) {
1841                 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1842                 goto out;
1843         }
1844
1845         if (!strcasecmp(argv[0], "disable_group")) {
1846                 r = bypass_pg_num(m, argv[1], true);
1847                 goto out;
1848         } else if (!strcasecmp(argv[0], "enable_group")) {
1849                 r = bypass_pg_num(m, argv[1], false);
1850                 goto out;
1851         } else if (!strcasecmp(argv[0], "switch_group")) {
1852                 r = switch_pg_num(m, argv[1]);
1853                 goto out;
1854         } else if (!strcasecmp(argv[0], "reinstate_path"))
1855                 action = reinstate_path;
1856         else if (!strcasecmp(argv[0], "fail_path"))
1857                 action = fail_path;
1858         else {
1859                 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1860                 goto out;
1861         }
1862
1863         r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1864         if (r) {
1865                 DMWARN("message: error getting device %s",
1866                        argv[1]);
1867                 goto out;
1868         }
1869
1870         r = action_dev(m, dev, action);
1871
1872         dm_put_device(ti, dev);
1873
1874 out:
1875         mutex_unlock(&m->work_mutex);
1876         return r;
1877 }
1878
1879 static int multipath_prepare_ioctl(struct dm_target *ti,
1880                                    struct block_device **bdev)
1881 {
1882         struct multipath *m = ti->private;
1883         struct pgpath *current_pgpath;
1884         int r;
1885
1886         current_pgpath = READ_ONCE(m->current_pgpath);
1887         if (!current_pgpath)
1888                 current_pgpath = choose_pgpath(m, 0);
1889
1890         if (current_pgpath) {
1891                 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1892                         *bdev = current_pgpath->path.dev->bdev;
1893                         r = 0;
1894                 } else {
1895                         /* pg_init has not started or completed */
1896                         r = -ENOTCONN;
1897                 }
1898         } else {
1899                 /* No path is available */
1900                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1901                         r = -ENOTCONN;
1902                 else
1903                         r = -EIO;
1904         }
1905
1906         if (r == -ENOTCONN) {
1907                 if (!READ_ONCE(m->current_pg)) {
1908                         /* Path status changed, redo selection */
1909                         (void) choose_pgpath(m, 0);
1910                 }
1911                 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1912                         pg_init_all_paths(m);
1913                 dm_table_run_md_queue_async(m->ti->table);
1914                 process_queued_io_list(m);
1915         }
1916
1917         /*
1918          * Only pass ioctls through if the device sizes match exactly.
1919          */
1920         if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1921                 return 1;
1922         return r;
1923 }
1924
1925 static int multipath_iterate_devices(struct dm_target *ti,
1926                                      iterate_devices_callout_fn fn, void *data)
1927 {
1928         struct multipath *m = ti->private;
1929         struct priority_group *pg;
1930         struct pgpath *p;
1931         int ret = 0;
1932
1933         list_for_each_entry(pg, &m->priority_groups, list) {
1934                 list_for_each_entry(p, &pg->pgpaths, list) {
1935                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1936                         if (ret)
1937                                 goto out;
1938                 }
1939         }
1940
1941 out:
1942         return ret;
1943 }
1944
1945 static int pgpath_busy(struct pgpath *pgpath)
1946 {
1947         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1948
1949         return blk_lld_busy(q);
1950 }
1951
1952 /*
1953  * We return "busy", only when we can map I/Os but underlying devices
1954  * are busy (so even if we map I/Os now, the I/Os will wait on
1955  * the underlying queue).
1956  * In other words, if we want to kill I/Os or queue them inside us
1957  * due to map unavailability, we don't return "busy".  Otherwise,
1958  * dm core won't give us the I/Os and we can't do what we want.
1959  */
1960 static int multipath_busy(struct dm_target *ti)
1961 {
1962         bool busy = false, has_active = false;
1963         struct multipath *m = ti->private;
1964         struct priority_group *pg, *next_pg;
1965         struct pgpath *pgpath;
1966
1967         /* pg_init in progress */
1968         if (atomic_read(&m->pg_init_in_progress))
1969                 return true;
1970
1971         /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
1972         if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1973                 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
1974
1975         /* Guess which priority_group will be used at next mapping time */
1976         pg = READ_ONCE(m->current_pg);
1977         next_pg = READ_ONCE(m->next_pg);
1978         if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg))
1979                 pg = next_pg;
1980
1981         if (!pg) {
1982                 /*
1983                  * We don't know which pg will be used at next mapping time.
1984                  * We don't call choose_pgpath() here to avoid to trigger
1985                  * pg_init just by busy checking.
1986                  * So we don't know whether underlying devices we will be using
1987                  * at next mapping time are busy or not. Just try mapping.
1988                  */
1989                 return busy;
1990         }
1991
1992         /*
1993          * If there is one non-busy active path at least, the path selector
1994          * will be able to select it. So we consider such a pg as not busy.
1995          */
1996         busy = true;
1997         list_for_each_entry(pgpath, &pg->pgpaths, list) {
1998                 if (pgpath->is_active) {
1999                         has_active = true;
2000                         if (!pgpath_busy(pgpath)) {
2001                                 busy = false;
2002                                 break;
2003                         }
2004                 }
2005         }
2006
2007         if (!has_active) {
2008                 /*
2009                  * No active path in this pg, so this pg won't be used and
2010                  * the current_pg will be changed at next mapping time.
2011                  * We need to try mapping to determine it.
2012                  */
2013                 busy = false;
2014         }
2015
2016         return busy;
2017 }
2018
2019 /*-----------------------------------------------------------------
2020  * Module setup
2021  *---------------------------------------------------------------*/
2022 static struct target_type multipath_target = {
2023         .name = "multipath",
2024         .version = {1, 13, 0},
2025         .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE |
2026                     DM_TARGET_PASSES_INTEGRITY,
2027         .module = THIS_MODULE,
2028         .ctr = multipath_ctr,
2029         .dtr = multipath_dtr,
2030         .clone_and_map_rq = multipath_clone_and_map,
2031         .release_clone_rq = multipath_release_clone,
2032         .rq_end_io = multipath_end_io,
2033         .map = multipath_map_bio,
2034         .end_io = multipath_end_io_bio,
2035         .presuspend = multipath_presuspend,
2036         .postsuspend = multipath_postsuspend,
2037         .resume = multipath_resume,
2038         .status = multipath_status,
2039         .message = multipath_message,
2040         .prepare_ioctl = multipath_prepare_ioctl,
2041         .iterate_devices = multipath_iterate_devices,
2042         .busy = multipath_busy,
2043 };
2044
2045 static int __init dm_multipath_init(void)
2046 {
2047         int r;
2048
2049         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2050         if (!kmultipathd) {
2051                 DMERR("failed to create workqueue kmpathd");
2052                 r = -ENOMEM;
2053                 goto bad_alloc_kmultipathd;
2054         }
2055
2056         /*
2057          * A separate workqueue is used to handle the device handlers
2058          * to avoid overloading existing workqueue. Overloading the
2059          * old workqueue would also create a bottleneck in the
2060          * path of the storage hardware device activation.
2061          */
2062         kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2063                                                   WQ_MEM_RECLAIM);
2064         if (!kmpath_handlerd) {
2065                 DMERR("failed to create workqueue kmpath_handlerd");
2066                 r = -ENOMEM;
2067                 goto bad_alloc_kmpath_handlerd;
2068         }
2069
2070         r = dm_register_target(&multipath_target);
2071         if (r < 0) {
2072                 DMERR("request-based register failed %d", r);
2073                 r = -EINVAL;
2074                 goto bad_register_target;
2075         }
2076
2077         return 0;
2078
2079 bad_register_target:
2080         destroy_workqueue(kmpath_handlerd);
2081 bad_alloc_kmpath_handlerd:
2082         destroy_workqueue(kmultipathd);
2083 bad_alloc_kmultipathd:
2084         return r;
2085 }
2086
2087 static void __exit dm_multipath_exit(void)
2088 {
2089         destroy_workqueue(kmpath_handlerd);
2090         destroy_workqueue(kmultipathd);
2091
2092         dm_unregister_target(&multipath_target);
2093 }
2094
2095 module_init(dm_multipath_init);
2096 module_exit(dm_multipath_exit);
2097
2098 MODULE_DESCRIPTION(DM_NAME " multipath target");
2099 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2100 MODULE_LICENSE("GPL");