drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-path-selector.h"
  11 #include "dm-uevent.h"
  12
  13 #include <linux/ctype.h>
  14 #include <linux/init.h>
  15 #include <linux/mempool.h>
  16 #include <linux/module.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/slab.h>
  19 #include <linux/time.h>
  20 #include <linux/workqueue.h>
  21 #include <scsi/scsi_dh.h>
  22 #include <asm/atomic.h>
  23
  24 #define DM_MSG_PREFIX "multipath"
  25 #define MESG_STR(x) x, sizeof(x)
  26
  27 /* Path properties */
  28 struct pgpath {
  29         struct list_head list;
  30
  31         struct priority_group *pg;      /* Owning PG */
  32         unsigned is_active;             /* Path status */
  33         unsigned fail_count;            /* Cumulative failure count */
  34
  35         struct dm_path path;
  36         struct work_struct deactivate_path;
  37         struct work_struct activate_path;
  38 };
  39
  40 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  41
  42 /*
  43  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  44  * Each has a path selector which controls which path gets used.
  45  */
  46 struct priority_group {
  47         struct list_head list;
  48
  49         struct multipath *m;            /* Owning multipath instance */
  50         struct path_selector ps;
  51
  52         unsigned pg_num;                /* Reference number */
  53         unsigned bypassed;              /* Temporarily bypass this PG? */
  54
  55         unsigned nr_pgpaths;            /* Number of paths in PG */
  56         struct list_head pgpaths;
  57 };
  58
  59 /* Multipath context */
  60 struct multipath {
  61         struct list_head list;
  62         struct dm_target *ti;
  63
  64         spinlock_t lock;
  65
  66         const char *hw_handler_name;
  67         char *hw_handler_params;
  68         unsigned nr_priority_groups;
  69         struct list_head priority_groups;
  70         unsigned pg_init_required;      /* pg_init needs calling? */
  71         unsigned pg_init_in_progress;   /* Only one pg_init allowed at once */
  72
  73         unsigned nr_valid_paths;        /* Total number of usable paths */
  74         struct pgpath *current_pgpath;
  75         struct priority_group *current_pg;
  76         struct priority_group *next_pg; /* Switch to this PG if set */
  77         unsigned repeat_count;          /* I/Os left before calling PS again */
  78
  79         unsigned queue_io;              /* Must we queue all I/O? */
  80         unsigned queue_if_no_path;      /* Queue I/O if last path fails? */
  81         unsigned saved_queue_if_no_path;/* Saved state during suspension */
  82         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  83         unsigned pg_init_count;         /* Number of times pg_init called */
  84
  85         struct work_struct process_queued_ios;
  86         struct list_head queued_ios;
  87         unsigned queue_size;
  88
  89         struct work_struct trigger_event;
  90
  91         /*
  92          * We must use a mempool of dm_mpath_io structs so that we
  93          * can resubmit bios on error.
  94          */
  95         mempool_t *mpio_pool;
  96 };
  97
  98 /*
  99  * Context information attached to each bio we process.
 100  */
 101 struct dm_mpath_io {
 102         struct pgpath *pgpath;
 103         size_t nr_bytes;
 104 };
 105
 106 typedef int (*action_fn) (struct pgpath *pgpath);
 107
 108 #define MIN_IOS 256     /* Mempool size */
 109
 110 static struct kmem_cache *_mpio_cache;
 111
 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 113 static void process_queued_ios(struct work_struct *work);
 114 static void trigger_event(struct work_struct *work);
 115 static void activate_path(struct work_struct *work);
 116 static void deactivate_path(struct work_struct *work);
 117
 118
 119 /*-----------------------------------------------
 120  * Allocation routines
 121  *-----------------------------------------------*/
 122
 123 static struct pgpath *alloc_pgpath(void)
 124 {
 125         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 126
 127         if (pgpath) {
 128                 pgpath->is_active = 1;
 129                 INIT_WORK(&pgpath->deactivate_path, deactivate_path);
 130                 INIT_WORK(&pgpath->activate_path, activate_path);
 131         }
 132
 133         return pgpath;
 134 }
 135
 136 static void free_pgpath(struct pgpath *pgpath)
 137 {
 138         kfree(pgpath);
 139 }
 140
 141 static void deactivate_path(struct work_struct *work)
 142 {
 143         struct pgpath *pgpath =
 144                 container_of(work, struct pgpath, deactivate_path);
 145
 146         blk_abort_queue(pgpath->path.dev->bdev->bd_disk->queue);
 147 }
 148
 149 static struct priority_group *alloc_priority_group(void)
 150 {
 151         struct priority_group *pg;
 152
 153         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 154
 155         if (pg)
 156                 INIT_LIST_HEAD(&pg->pgpaths);
 157
 158         return pg;
 159 }
 160
 161 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 162 {
 163         struct pgpath *pgpath, *tmp;
 164         struct multipath *m = ti->private;
 165
 166         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 167                 list_del(&pgpath->list);
 168                 if (m->hw_handler_name)
 169                         scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev));
 170                 dm_put_device(ti, pgpath->path.dev);
 171                 free_pgpath(pgpath);
 172         }
 173 }
 174
 175 static void free_priority_group(struct priority_group *pg,
 176                                 struct dm_target *ti)
 177 {
 178         struct path_selector *ps = &pg->ps;
 179
 180         if (ps->type) {
 181                 ps->type->destroy(ps);
 182                 dm_put_path_selector(ps->type);
 183         }
 184
 185         free_pgpaths(&pg->pgpaths, ti);
 186         kfree(pg);
 187 }
 188
 189 static struct multipath *alloc_multipath(struct dm_target *ti)
 190 {
 191         struct multipath *m;
 192
 193         m = kzalloc(sizeof(*m), GFP_KERNEL);
 194         if (m) {
 195                 INIT_LIST_HEAD(&m->priority_groups);
 196                 INIT_LIST_HEAD(&m->queued_ios);
 197                 spin_lock_init(&m->lock);
 198                 m->queue_io = 1;
 199                 INIT_WORK(&m->process_queued_ios, process_queued_ios);
 200                 INIT_WORK(&m->trigger_event, trigger_event);
 201                 m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
 202                 if (!m->mpio_pool) {
 203                         kfree(m);
 204                         return NULL;
 205                 }
 206                 m->ti = ti;
 207                 ti->private = m;
 208         }
 209
 210         return m;
 211 }
 212
 213 static void free_multipath(struct multipath *m)
 214 {
 215         struct priority_group *pg, *tmp;
 216
 217         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 218                 list_del(&pg->list);
 219                 free_priority_group(pg, m->ti);
 220         }
 221
 222         kfree(m->hw_handler_name);
 223         kfree(m->hw_handler_params);
 224         mempool_destroy(m->mpio_pool);
 225         kfree(m);
 226 }
 227
 228
 229 /*-----------------------------------------------
 230  * Path selection
 231  *-----------------------------------------------*/
 232
 233 static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
 234 {
 235         m->current_pg = pgpath->pg;
 236
 237         /* Must we initialise the PG first, and queue I/O till it's ready? */
 238         if (m->hw_handler_name) {
 239                 m->pg_init_required = 1;
 240                 m->queue_io = 1;
 241         } else {
 242                 m->pg_init_required = 0;
 243                 m->queue_io = 0;
 244         }
 245
 246         m->pg_init_count = 0;
 247 }
 248
 249 static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
 250                                size_t nr_bytes)
 251 {
 252         struct dm_path *path;
 253
 254         path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
 255         if (!path)
 256                 return -ENXIO;
 257
 258         m->current_pgpath = path_to_pgpath(path);
 259
 260         if (m->current_pg != pg)
 261                 __switch_pg(m, m->current_pgpath);
 262
 263         return 0;
 264 }
 265
 266 static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
 267 {
 268         struct priority_group *pg;
 269         unsigned bypassed = 1;
 270
 271         if (!m->nr_valid_paths)
 272                 goto failed;
 273
 274         /* Were we instructed to switch PG? */
 275         if (m->next_pg) {
 276                 pg = m->next_pg;
 277                 m->next_pg = NULL;
 278                 if (!__choose_path_in_pg(m, pg, nr_bytes))
 279                         return;
 280         }
 281
 282         /* Don't change PG until it has no remaining paths */
 283         if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
 284                 return;
 285
 286         /*
 287          * Loop through priority groups until we find a valid path.
 288          * First time we skip PGs marked 'bypassed'.
 289          * Second time we only try the ones we skipped.
 290          */
 291         do {
 292                 list_for_each_entry(pg, &m->priority_groups, list) {
 293                         if (pg->bypassed == bypassed)
 294                                 continue;
 295                         if (!__choose_path_in_pg(m, pg, nr_bytes))
 296                                 return;
 297                 }
 298         } while (bypassed--);
 299
 300 failed:
 301         m->current_pgpath = NULL;
 302         m->current_pg = NULL;
 303 }
 304
 305 /*
 306  * Check whether bios must be queued in the device-mapper core rather
 307  * than here in the target.
 308  *
 309  * m->lock must be held on entry.
 310  *
 311  * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
 312  * same value then we are not between multipath_presuspend()
 313  * and multipath_resume() calls and we have no need to check
 314  * for the DMF_NOFLUSH_SUSPENDING flag.
 315  */
 316 static int __must_push_back(struct multipath *m)
 317 {
 318         return (m->queue_if_no_path != m->saved_queue_if_no_path &&
 319                 dm_noflush_suspending(m->ti));
 320 }
 321
 322 static int map_io(struct multipath *m, struct request *clone,
 323                   struct dm_mpath_io *mpio, unsigned was_queued)
 324 {
 325         int r = DM_MAPIO_REMAPPED;
 326         size_t nr_bytes = blk_rq_bytes(clone);
 327         unsigned long flags;
 328         struct pgpath *pgpath;
 329         struct block_device *bdev;
 330
 331         spin_lock_irqsave(&m->lock, flags);
 332
 333         /* Do we need to select a new pgpath? */
 334         if (!m->current_pgpath ||
 335             (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
 336                 __choose_pgpath(m, nr_bytes);
 337
 338         pgpath = m->current_pgpath;
 339
 340         if (was_queued)
 341                 m->queue_size--;
 342
 343         if ((pgpath && m->queue_io) ||
 344             (!pgpath && m->queue_if_no_path)) {
 345                 /* Queue for the daemon to resubmit */
 346                 list_add_tail(&clone->queuelist, &m->queued_ios);
 347                 m->queue_size++;
 348                 if ((m->pg_init_required && !m->pg_init_in_progress) ||
 349                     !m->queue_io)
 350                         queue_work(kmultipathd, &m->process_queued_ios);
 351                 pgpath = NULL;
 352                 r = DM_MAPIO_SUBMITTED;
 353         } else if (pgpath) {
 354                 bdev = pgpath->path.dev->bdev;
 355                 clone->q = bdev_get_queue(bdev);
 356                 clone->rq_disk = bdev->bd_disk;
 357         } else if (__must_push_back(m))
 358                 r = DM_MAPIO_REQUEUE;
 359         else
 360                 r = -EIO;       /* Failed */
 361
 362         mpio->pgpath = pgpath;
 363         mpio->nr_bytes = nr_bytes;
 364
 365         if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
 366                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
 367                                               nr_bytes);
 368
 369         spin_unlock_irqrestore(&m->lock, flags);
 370
 371         return r;
 372 }
 373
 374 /*
 375  * If we run out of usable paths, should we queue I/O or error it?
 376  */
 377 static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
 378                             unsigned save_old_value)
 379 {
 380         unsigned long flags;
 381
 382         spin_lock_irqsave(&m->lock, flags);
 383
 384         if (save_old_value)
 385                 m->saved_queue_if_no_path = m->queue_if_no_path;
 386         else
 387                 m->saved_queue_if_no_path = queue_if_no_path;
 388         m->queue_if_no_path = queue_if_no_path;
 389         if (!m->queue_if_no_path && m->queue_size)
 390                 queue_work(kmultipathd, &m->process_queued_ios);
 391
 392         spin_unlock_irqrestore(&m->lock, flags);
 393
 394         return 0;
 395 }
 396
 397 /*-----------------------------------------------------------------
 398  * The multipath daemon is responsible for resubmitting queued ios.
 399  *---------------------------------------------------------------*/
 400
 401 static void dispatch_queued_ios(struct multipath *m)
 402 {
 403         int r;
 404         unsigned long flags;
 405         struct dm_mpath_io *mpio;
 406         union map_info *info;
 407         struct request *clone, *n;
 408         LIST_HEAD(cl);
 409
 410         spin_lock_irqsave(&m->lock, flags);
 411         list_splice_init(&m->queued_ios, &cl);
 412         spin_unlock_irqrestore(&m->lock, flags);
 413
 414         list_for_each_entry_safe(clone, n, &cl, queuelist) {
 415                 list_del_init(&clone->queuelist);
 416
 417                 info = dm_get_rq_mapinfo(clone);
 418                 mpio = info->ptr;
 419
 420                 r = map_io(m, clone, mpio, 1);
 421                 if (r < 0) {
 422                         mempool_free(mpio, m->mpio_pool);
 423                         dm_kill_unmapped_request(clone, r);
 424                 } else if (r == DM_MAPIO_REMAPPED)
 425                         dm_dispatch_request(clone);
 426                 else if (r == DM_MAPIO_REQUEUE) {
 427                         mempool_free(mpio, m->mpio_pool);
 428                         dm_requeue_unmapped_request(clone);
 429                 }
 430         }
 431 }
 432
 433 static void process_queued_ios(struct work_struct *work)
 434 {
 435         struct multipath *m =
 436                 container_of(work, struct multipath, process_queued_ios);
 437         struct pgpath *pgpath = NULL, *tmp;
 438         unsigned must_queue = 1;
 439         unsigned long flags;
 440
 441         spin_lock_irqsave(&m->lock, flags);
 442
 443         if (!m->queue_size)
 444                 goto out;
 445
 446         if (!m->current_pgpath)
 447                 __choose_pgpath(m, 0);
 448
 449         pgpath = m->current_pgpath;
 450
 451         if ((pgpath && !m->queue_io) ||
 452             (!pgpath && !m->queue_if_no_path))
 453                 must_queue = 0;
 454
 455         if (m->pg_init_required && !m->pg_init_in_progress && pgpath) {
 456                 m->pg_init_count++;
 457                 m->pg_init_required = 0;
 458                 list_for_each_entry(tmp, &pgpath->pg->pgpaths, list) {
 459                         if (queue_work(kmpath_handlerd, &tmp->activate_path))
 460                                 m->pg_init_in_progress++;
 461                 }
 462         }
 463 out:
 464         spin_unlock_irqrestore(&m->lock, flags);
 465         if (!must_queue)
 466                 dispatch_queued_ios(m);
 467 }
 468
 469 /*
 470  * An event is triggered whenever a path is taken out of use.
 471  * Includes path failure and PG bypass.
 472  */
 473 static void trigger_event(struct work_struct *work)
 474 {
 475         struct multipath *m =
 476                 container_of(work, struct multipath, trigger_event);
 477
 478         dm_table_event(m->ti->table);
 479 }
 480
 481 /*-----------------------------------------------------------------
 482  * Constructor/argument parsing:
 483  * <#multipath feature args> [<arg>]*
 484  * <#hw_handler args> [hw_handler [<arg>]*]
 485  * <#priority groups>
 486  * <initial priority group>
 487  *     [<selector> <#selector args> [<arg>]*
 488  *      <#paths> <#per-path selector args>
 489  *         [<path> [<arg>]* ]+ ]+
 490  *---------------------------------------------------------------*/
 491 struct param {
 492         unsigned min;
 493         unsigned max;
 494         char *error;
 495 };
 496
 497 static int read_param(struct param *param, char *str, unsigned *v, char **error)
 498 {
 499         if (!str ||
 500             (sscanf(str, "%u", v) != 1) ||
 501             (*v < param->min) ||
 502             (*v > param->max)) {
 503                 *error = param->error;
 504                 return -EINVAL;
 505         }
 506
 507         return 0;
 508 }
 509
 510 struct arg_set {
 511         unsigned argc;
 512         char **argv;
 513 };
 514
 515 static char *shift(struct arg_set *as)
 516 {
 517         char *r;
 518
 519         if (as->argc) {
 520                 as->argc--;
 521                 r = *as->argv;
 522                 as->argv++;
 523                 return r;
 524         }
 525
 526         return NULL;
 527 }
 528
 529 static void consume(struct arg_set *as, unsigned n)
 530 {
 531         BUG_ON (as->argc < n);
 532         as->argc -= n;
 533         as->argv += n;
 534 }
 535
 536 static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
 537                                struct dm_target *ti)
 538 {
 539         int r;
 540         struct path_selector_type *pst;
 541         unsigned ps_argc;
 542
 543         static struct param _params[] = {
 544                 {0, 1024, "invalid number of path selector args"},
 545         };
 546
 547         pst = dm_get_path_selector(shift(as));
 548         if (!pst) {
 549                 ti->error = "unknown path selector type";
 550                 return -EINVAL;
 551         }
 552
 553         r = read_param(_params, shift(as), &ps_argc, &ti->error);
 554         if (r) {
 555                 dm_put_path_selector(pst);
 556                 return -EINVAL;
 557         }
 558
 559         if (ps_argc > as->argc) {
 560                 dm_put_path_selector(pst);
 561                 ti->error = "not enough arguments for path selector";
 562                 return -EINVAL;
 563         }
 564
 565         r = pst->create(&pg->ps, ps_argc, as->argv);
 566         if (r) {
 567                 dm_put_path_selector(pst);
 568                 ti->error = "path selector constructor failed";
 569                 return r;
 570         }
 571
 572         pg->ps.type = pst;
 573         consume(as, ps_argc);
 574
 575         return 0;
 576 }
 577
 578 static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
 579                                struct dm_target *ti)
 580 {
 581         int r;
 582         struct pgpath *p;
 583         struct multipath *m = ti->private;
 584
 585         /* we need at least a path arg */
 586         if (as->argc < 1) {
 587                 ti->error = "no device given";
 588                 return ERR_PTR(-EINVAL);
 589         }
 590
 591         p = alloc_pgpath();
 592         if (!p)
 593                 return ERR_PTR(-ENOMEM);
 594
 595         r = dm_get_device(ti, shift(as), ti->begin, ti->len,
 596                           dm_table_get_mode(ti->table), &p->path.dev);
 597         if (r) {
 598                 ti->error = "error getting device";
 599                 goto bad;
 600         }
 601
 602         if (m->hw_handler_name) {
 603                 struct request_queue *q = bdev_get_queue(p->path.dev->bdev);
 604
 605                 r = scsi_dh_attach(q, m->hw_handler_name);
 606                 if (r == -EBUSY) {
 607                         /*
 608                          * Already attached to different hw_handler,
 609                          * try to reattach with correct one.
 610                          */
 611                         scsi_dh_detach(q);
 612                         r = scsi_dh_attach(q, m->hw_handler_name);
 613                 }
 614
 615                 if (r < 0) {
 616                         ti->error = "error attaching hardware handler";
 617                         dm_put_device(ti, p->path.dev);
 618                         goto bad;
 619                 }
 620
 621                 if (m->hw_handler_params) {
 622                         r = scsi_dh_set_params(q, m->hw_handler_params);
 623                         if (r < 0) {
 624                                 ti->error = "unable to set hardware "
 625                                                         "handler parameters";
 626                                 scsi_dh_detach(q);
 627                                 dm_put_device(ti, p->path.dev);
 628                                 goto bad;
 629                         }
 630                 }
 631         }
 632
 633         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 634         if (r) {
 635                 dm_put_device(ti, p->path.dev);
 636                 goto bad;
 637         }
 638
 639         return p;
 640
 641  bad:
 642         free_pgpath(p);
 643         return ERR_PTR(r);
 644 }
 645
 646 static struct priority_group *parse_priority_group(struct arg_set *as,
 647                                                    struct multipath *m)
 648 {
 649         static struct param _params[] = {
 650                 {1, 1024, "invalid number of paths"},
 651                 {0, 1024, "invalid number of selector args"}
 652         };
 653
 654         int r;
 655         unsigned i, nr_selector_args, nr_params;
 656         struct priority_group *pg;
 657         struct dm_target *ti = m->ti;
 658
 659         if (as->argc < 2) {
 660                 as->argc = 0;
 661                 ti->error = "not enough priority group arguments";
 662                 return ERR_PTR(-EINVAL);
 663         }
 664
 665         pg = alloc_priority_group();
 666         if (!pg) {
 667                 ti->error = "couldn't allocate priority group";
 668                 return ERR_PTR(-ENOMEM);
 669         }
 670         pg->m = m;
 671
 672         r = parse_path_selector(as, pg, ti);
 673         if (r)
 674                 goto bad;
 675
 676         /*
 677          * read the paths
 678          */
 679         r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error);
 680         if (r)
 681                 goto bad;
 682
 683         r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error);
 684         if (r)
 685                 goto bad;
 686
 687         nr_params = 1 + nr_selector_args;
 688         for (i = 0; i < pg->nr_pgpaths; i++) {
 689                 struct pgpath *pgpath;
 690                 struct arg_set path_args;
 691
 692                 if (as->argc < nr_params) {
 693                         ti->error = "not enough path parameters";
 694                         goto bad;
 695                 }
 696
 697                 path_args.argc = nr_params;
 698                 path_args.argv = as->argv;
 699
 700                 pgpath = parse_path(&path_args, &pg->ps, ti);
 701                 if (IS_ERR(pgpath)) {
 702                         r = PTR_ERR(pgpath);
 703                         goto bad;
 704                 }
 705
 706                 pgpath->pg = pg;
 707                 list_add_tail(&pgpath->list, &pg->pgpaths);
 708                 consume(as, nr_params);
 709         }
 710
 711         return pg;
 712
 713  bad:
 714         free_priority_group(pg, ti);
 715         return ERR_PTR(r);
 716 }
 717
 718 static int parse_hw_handler(struct arg_set *as, struct multipath *m)
 719 {
 720         unsigned hw_argc;
 721         int ret;
 722         struct dm_target *ti = m->ti;
 723
 724         static struct param _params[] = {
 725                 {0, 1024, "invalid number of hardware handler args"},
 726         };
 727
 728         if (read_param(_params, shift(as), &hw_argc, &ti->error))
 729                 return -EINVAL;
 730
 731         if (!hw_argc)
 732                 return 0;
 733
 734         if (hw_argc > as->argc) {
 735                 ti->error = "not enough arguments for hardware handler";
 736                 return -EINVAL;
 737         }
 738
 739         m->hw_handler_name = kstrdup(shift(as), GFP_KERNEL);
 740         request_module("scsi_dh_%s", m->hw_handler_name);
 741         if (scsi_dh_handler_exist(m->hw_handler_name) == 0) {
 742                 ti->error = "unknown hardware handler type";
 743                 ret = -EINVAL;
 744                 goto fail;
 745         }
 746
 747         if (hw_argc > 1) {
 748                 char *p;
 749                 int i, j, len = 4;
 750
 751                 for (i = 0; i <= hw_argc - 2; i++)
 752                         len += strlen(as->argv[i]) + 1;
 753                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
 754                 if (!p) {
 755                         ti->error = "memory allocation failed";
 756                         ret = -ENOMEM;
 757                         goto fail;
 758                 }
 759                 j = sprintf(p, "%d", hw_argc - 1);
 760                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
 761                         j = sprintf(p, "%s", as->argv[i]);
 762         }
 763         consume(as, hw_argc - 1);
 764
 765         return 0;
 766 fail:
 767         kfree(m->hw_handler_name);
 768         m->hw_handler_name = NULL;
 769         return ret;
 770 }
 771
 772 static int parse_features(struct arg_set *as, struct multipath *m)
 773 {
 774         int r;
 775         unsigned argc;
 776         struct dm_target *ti = m->ti;
 777         const char *param_name;
 778
 779         static struct param _params[] = {
 780                 {0, 3, "invalid number of feature args"},
 781                 {1, 50, "pg_init_retries must be between 1 and 50"},
 782         };
 783
 784         r = read_param(_params, shift(as), &argc, &ti->error);
 785         if (r)
 786                 return -EINVAL;
 787
 788         if (!argc)
 789                 return 0;
 790
 791         do {
 792                 param_name = shift(as);
 793                 argc--;
 794
 795                 if (!strnicmp(param_name, MESG_STR("queue_if_no_path"))) {
 796                         r = queue_if_no_path(m, 1, 0);
 797                         continue;
 798                 }
 799
 800                 if (!strnicmp(param_name, MESG_STR("pg_init_retries")) &&
 801                     (argc >= 1)) {
 802                         r = read_param(_params + 1, shift(as),
 803                                        &m->pg_init_retries, &ti->error);
 804                         argc--;
 805                         continue;
 806                 }
 807
 808                 ti->error = "Unrecognised multipath feature request";
 809                 r = -EINVAL;
 810         } while (argc && !r);
 811
 812         return r;
 813 }
 814
 815 static int multipath_ctr(struct dm_target *ti, unsigned int argc,
 816                          char **argv)
 817 {
 818         /* target parameters */
 819         static struct param _params[] = {
 820                 {1, 1024, "invalid number of priority groups"},
 821                 {1, 1024, "invalid initial priority group number"},
 822         };
 823
 824         int r;
 825         struct multipath *m;
 826         struct arg_set as;
 827         unsigned pg_count = 0;
 828         unsigned next_pg_num;
 829
 830         as.argc = argc;
 831         as.argv = argv;
 832
 833         m = alloc_multipath(ti);
 834         if (!m) {
 835                 ti->error = "can't allocate multipath";
 836                 return -EINVAL;
 837         }
 838
 839         r = parse_features(&as, m);
 840         if (r)
 841                 goto bad;
 842
 843         r = parse_hw_handler(&as, m);
 844         if (r)
 845                 goto bad;
 846
 847         r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error);
 848         if (r)
 849                 goto bad;
 850
 851         r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error);
 852         if (r)
 853                 goto bad;
 854
 855         /* parse the priority groups */
 856         while (as.argc) {
 857                 struct priority_group *pg;
 858
 859                 pg = parse_priority_group(&as, m);
 860                 if (IS_ERR(pg)) {
 861                         r = PTR_ERR(pg);
 862                         goto bad;
 863                 }
 864
 865                 m->nr_valid_paths += pg->nr_pgpaths;
 866                 list_add_tail(&pg->list, &m->priority_groups);
 867                 pg_count++;
 868                 pg->pg_num = pg_count;
 869                 if (!--next_pg_num)
 870                         m->next_pg = pg;
 871         }
 872
 873         if (pg_count != m->nr_priority_groups) {
 874                 ti->error = "priority group count mismatch";
 875                 r = -EINVAL;
 876                 goto bad;
 877         }
 878
 879         ti->num_flush_requests = 1;
 880
 881         return 0;
 882
 883  bad:
 884         free_multipath(m);
 885         return r;
 886 }
 887
 888 static void flush_multipath_work(void)
 889 {
 890         flush_workqueue(kmpath_handlerd);
 891         flush_workqueue(kmultipathd);
 892         flush_scheduled_work();
 893 }
 894
 895 static void multipath_dtr(struct dm_target *ti)
 896 {
 897         struct multipath *m = ti->private;
 898
 899         flush_multipath_work();
 900         free_multipath(m);
 901 }
 902
 903 /*
 904  * Map cloned requests
 905  */
 906 static int multipath_map(struct dm_target *ti, struct request *clone,
 907                          union map_info *map_context)
 908 {
 909         int r;
 910         struct dm_mpath_io *mpio;
 911         struct multipath *m = (struct multipath *) ti->private;
 912
 913         mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
 914         if (!mpio)
 915                 /* ENOMEM, requeue */
 916                 return DM_MAPIO_REQUEUE;
 917         memset(mpio, 0, sizeof(*mpio));
 918
 919         map_context->ptr = mpio;
 920         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 921         r = map_io(m, clone, mpio, 0);
 922         if (r < 0 || r == DM_MAPIO_REQUEUE)
 923                 mempool_free(mpio, m->mpio_pool);
 924
 925         return r;
 926 }
 927
 928 /*
 929  * Take a path out of use.
 930  */
 931 static int fail_path(struct pgpath *pgpath)
 932 {
 933         unsigned long flags;
 934         struct multipath *m = pgpath->pg->m;
 935
 936         spin_lock_irqsave(&m->lock, flags);
 937
 938         if (!pgpath->is_active)
 939                 goto out;
 940
 941         DMWARN("Failing path %s.", pgpath->path.dev->name);
 942
 943         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
 944         pgpath->is_active = 0;
 945         pgpath->fail_count++;
 946
 947         m->nr_valid_paths--;
 948
 949         if (pgpath == m->current_pgpath)
 950                 m->current_pgpath = NULL;
 951
 952         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
 953                       pgpath->path.dev->name, m->nr_valid_paths);
 954
 955         schedule_work(&m->trigger_event);
 956         queue_work(kmultipathd, &pgpath->deactivate_path);
 957
 958 out:
 959         spin_unlock_irqrestore(&m->lock, flags);
 960
 961         return 0;
 962 }
 963
 964 /*
 965  * Reinstate a previously-failed path
 966  */
 967 static int reinstate_path(struct pgpath *pgpath)
 968 {
 969         int r = 0;
 970         unsigned long flags;
 971         struct multipath *m = pgpath->pg->m;
 972
 973         spin_lock_irqsave(&m->lock, flags);
 974
 975         if (pgpath->is_active)
 976                 goto out;
 977
 978         if (!pgpath->pg->ps.type->reinstate_path) {
 979                 DMWARN("Reinstate path not supported by path selector %s",
 980                        pgpath->pg->ps.type->name);
 981                 r = -EINVAL;
 982                 goto out;
 983         }
 984
 985         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
 986         if (r)
 987                 goto out;
 988
 989         pgpath->is_active = 1;
 990
 991         if (!m->nr_valid_paths++ && m->queue_size) {
 992                 m->current_pgpath = NULL;
 993                 queue_work(kmultipathd, &m->process_queued_ios);
 994         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
 995                 if (queue_work(kmpath_handlerd, &pgpath->activate_path))
 996                         m->pg_init_in_progress++;
 997         }
 998
 999         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1000                       pgpath->path.dev->name, m->nr_valid_paths);
1001
1002         schedule_work(&m->trigger_event);
1003
1004 out:
1005         spin_unlock_irqrestore(&m->lock, flags);
1006
1007         return r;
1008 }
1009
1010 /*
1011  * Fail or reinstate all paths that match the provided struct dm_dev.
1012  */
1013 static int action_dev(struct multipath *m, struct dm_dev *dev,
1014                       action_fn action)
1015 {
1016         int r = 0;
1017         struct pgpath *pgpath;
1018         struct priority_group *pg;
1019
1020         list_for_each_entry(pg, &m->priority_groups, list) {
1021                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1022                         if (pgpath->path.dev == dev)
1023                                 r = action(pgpath);
1024                 }
1025         }
1026
1027         return r;
1028 }
1029
1030 /*
1031  * Temporarily try to avoid having to use the specified PG
1032  */
1033 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1034                       int bypassed)
1035 {
1036         unsigned long flags;
1037
1038         spin_lock_irqsave(&m->lock, flags);
1039
1040         pg->bypassed = bypassed;
1041         m->current_pgpath = NULL;
1042         m->current_pg = NULL;
1043
1044         spin_unlock_irqrestore(&m->lock, flags);
1045
1046         schedule_work(&m->trigger_event);
1047 }
1048
1049 /*
1050  * Switch to using the specified PG from the next I/O that gets mapped
1051  */
1052 static int switch_pg_num(struct multipath *m, const char *pgstr)
1053 {
1054         struct priority_group *pg;
1055         unsigned pgnum;
1056         unsigned long flags;
1057
1058         if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
1059             (pgnum > m->nr_priority_groups)) {
1060                 DMWARN("invalid PG number supplied to switch_pg_num");
1061                 return -EINVAL;
1062         }
1063
1064         spin_lock_irqsave(&m->lock, flags);
1065         list_for_each_entry(pg, &m->priority_groups, list) {
1066                 pg->bypassed = 0;
1067                 if (--pgnum)
1068                         continue;
1069
1070                 m->current_pgpath = NULL;
1071                 m->current_pg = NULL;
1072                 m->next_pg = pg;
1073         }
1074         spin_unlock_irqrestore(&m->lock, flags);
1075
1076         schedule_work(&m->trigger_event);
1077         return 0;
1078 }
1079
1080 /*
1081  * Set/clear bypassed status of a PG.
1082  * PGs are numbered upwards from 1 in the order they were declared.
1083  */
1084 static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
1085 {
1086         struct priority_group *pg;
1087         unsigned pgnum;
1088
1089         if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
1090             (pgnum > m->nr_priority_groups)) {
1091                 DMWARN("invalid PG number supplied to bypass_pg");
1092                 return -EINVAL;
1093         }
1094
1095         list_for_each_entry(pg, &m->priority_groups, list) {
1096                 if (!--pgnum)
1097                         break;
1098         }
1099
1100         bypass_pg(m, pg, bypassed);
1101         return 0;
1102 }
1103
1104 /*
1105  * Should we retry pg_init immediately?
1106  */
1107 static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1108 {
1109         unsigned long flags;
1110         int limit_reached = 0;
1111
1112         spin_lock_irqsave(&m->lock, flags);
1113
1114         if (m->pg_init_count <= m->pg_init_retries)
1115                 m->pg_init_required = 1;
1116         else
1117                 limit_reached = 1;
1118
1119         spin_unlock_irqrestore(&m->lock, flags);
1120
1121         return limit_reached;
1122 }
1123
1124 static void pg_init_done(void *data, int errors)
1125 {
1126         struct dm_path *path = data;
1127         struct pgpath *pgpath = path_to_pgpath(path);
1128         struct priority_group *pg = pgpath->pg;
1129         struct multipath *m = pg->m;
1130         unsigned long flags;
1131
1132         /* device or driver problems */
1133         switch (errors) {
1134         case SCSI_DH_OK:
1135                 break;
1136         case SCSI_DH_NOSYS:
1137                 if (!m->hw_handler_name) {
1138                         errors = 0;
1139                         break;
1140                 }
1141                 DMERR("Cannot failover device because scsi_dh_%s was not "
1142                       "loaded.", m->hw_handler_name);
1143                 /*
1144                  * Fail path for now, so we do not ping pong
1145                  */
1146                 fail_path(pgpath);
1147                 break;
1148         case SCSI_DH_DEV_TEMP_BUSY:
1149                 /*
1150                  * Probably doing something like FW upgrade on the
1151                  * controller so try the other pg.
1152                  */
1153                 bypass_pg(m, pg, 1);
1154                 break;
1155         /* TODO: For SCSI_DH_RETRY we should wait a couple seconds */
1156         case SCSI_DH_RETRY:
1157         case SCSI_DH_IMM_RETRY:
1158         case SCSI_DH_RES_TEMP_UNAVAIL:
1159                 if (pg_init_limit_reached(m, pgpath))
1160                         fail_path(pgpath);
1161                 errors = 0;
1162                 break;
1163         default:
1164                 /*
1165                  * We probably do not want to fail the path for a device
1166                  * error, but this is what the old dm did. In future
1167                  * patches we can do more advanced handling.
1168                  */
1169                 fail_path(pgpath);
1170         }
1171
1172         spin_lock_irqsave(&m->lock, flags);
1173         if (errors) {
1174                 if (pgpath == m->current_pgpath) {
1175                         DMERR("Could not failover device. Error %d.", errors);
1176                         m->current_pgpath = NULL;
1177                         m->current_pg = NULL;
1178                 }
1179         } else if (!m->pg_init_required) {
1180                 m->queue_io = 0;
1181                 pg->bypassed = 0;
1182         }
1183
1184         m->pg_init_in_progress--;
1185         if (!m->pg_init_in_progress)
1186                 queue_work(kmultipathd, &m->process_queued_ios);
1187         spin_unlock_irqrestore(&m->lock, flags);
1188 }
1189
1190 static void activate_path(struct work_struct *work)
1191 {
1192         struct pgpath *pgpath =
1193                 container_of(work, struct pgpath, activate_path);
1194
1195         scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1196                                 pg_init_done, &pgpath->path);
1197 }
1198
1199 /*
1200  * end_io handling
1201  */
1202 static int do_end_io(struct multipath *m, struct request *clone,
1203                      int error, struct dm_mpath_io *mpio)
1204 {
1205         /*
1206          * We don't queue any clone request inside the multipath target
1207          * during end I/O handling, since those clone requests don't have
1208          * bio clones.  If we queue them inside the multipath target,
1209          * we need to make bio clones, that requires memory allocation.
1210          * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1211          *  don't have bio clones.)
1212          * Instead of queueing the clone request here, we queue the original
1213          * request into dm core, which will remake a clone request and
1214          * clone bios for it and resubmit it later.
1215          */
1216         int r = DM_ENDIO_REQUEUE;
1217         unsigned long flags;
1218
1219         if (!error && !clone->errors)
1220                 return 0;       /* I/O complete */
1221
1222         if (error == -EOPNOTSUPP)
1223                 return error;
1224
1225         if (mpio->pgpath)
1226                 fail_path(mpio->pgpath);
1227
1228         spin_lock_irqsave(&m->lock, flags);
1229         if (!m->nr_valid_paths && !m->queue_if_no_path && !__must_push_back(m))
1230                 r = -EIO;
1231         spin_unlock_irqrestore(&m->lock, flags);
1232
1233         return r;
1234 }
1235
1236 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1237                             int error, union map_info *map_context)
1238 {
1239         struct multipath *m = ti->private;
1240         struct dm_mpath_io *mpio = map_context->ptr;
1241         struct pgpath *pgpath = mpio->pgpath;
1242         struct path_selector *ps;
1243         int r;
1244
1245         r  = do_end_io(m, clone, error, mpio);
1246         if (pgpath) {
1247                 ps = &pgpath->pg->ps;
1248                 if (ps->type->end_io)
1249                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1250         }
1251         mempool_free(mpio, m->mpio_pool);
1252
1253         return r;
1254 }
1255
1256 /*
1257  * Suspend can't complete until all the I/O is processed so if
1258  * the last path fails we must error any remaining I/O.
1259  * Note that if the freeze_bdev fails while suspending, the
1260  * queue_if_no_path state is lost - userspace should reset it.
1261  */
1262 static void multipath_presuspend(struct dm_target *ti)
1263 {
1264         struct multipath *m = (struct multipath *) ti->private;
1265
1266         queue_if_no_path(m, 0, 1);
1267 }
1268
1269 static void multipath_postsuspend(struct dm_target *ti)
1270 {
1271         flush_multipath_work();
1272 }
1273
1274 /*
1275  * Restore the queue_if_no_path setting.
1276  */
1277 static void multipath_resume(struct dm_target *ti)
1278 {
1279         struct multipath *m = (struct multipath *) ti->private;
1280         unsigned long flags;
1281
1282         spin_lock_irqsave(&m->lock, flags);
1283         m->queue_if_no_path = m->saved_queue_if_no_path;
1284         spin_unlock_irqrestore(&m->lock, flags);
1285 }
1286
1287 /*
1288  * Info output has the following format:
1289  * num_multipath_feature_args [multipath_feature_args]*
1290  * num_handler_status_args [handler_status_args]*
1291  * num_groups init_group_number
1292  *            [A|D|E num_ps_status_args [ps_status_args]*
1293  *             num_paths num_selector_args
1294  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1295  *
1296  * Table output has the following format (identical to the constructor string):
1297  * num_feature_args [features_args]*
1298  * num_handler_args hw_handler [hw_handler_args]*
1299  * num_groups init_group_number
1300  *     [priority selector-name num_ps_args [ps_args]*
1301  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1302  */
1303 static int multipath_status(struct dm_target *ti, status_type_t type,
1304                             char *result, unsigned int maxlen)
1305 {
1306         int sz = 0;
1307         unsigned long flags;
1308         struct multipath *m = (struct multipath *) ti->private;
1309         struct priority_group *pg;
1310         struct pgpath *p;
1311         unsigned pg_num;
1312         char state;
1313
1314         spin_lock_irqsave(&m->lock, flags);
1315
1316         /* Features */
1317         if (type == STATUSTYPE_INFO)
1318                 DMEMIT("2 %u %u ", m->queue_size, m->pg_init_count);
1319         else {
1320                 DMEMIT("%u ", m->queue_if_no_path +
1321                               (m->pg_init_retries > 0) * 2);
1322                 if (m->queue_if_no_path)
1323                         DMEMIT("queue_if_no_path ");
1324                 if (m->pg_init_retries)
1325                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1326         }
1327
1328         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1329                 DMEMIT("0 ");
1330         else
1331                 DMEMIT("1 %s ", m->hw_handler_name);
1332
1333         DMEMIT("%u ", m->nr_priority_groups);
1334
1335         if (m->next_pg)
1336                 pg_num = m->next_pg->pg_num;
1337         else if (m->current_pg)
1338                 pg_num = m->current_pg->pg_num;
1339         else
1340                         pg_num = 1;
1341
1342         DMEMIT("%u ", pg_num);
1343
1344         switch (type) {
1345         case STATUSTYPE_INFO:
1346                 list_for_each_entry(pg, &m->priority_groups, list) {
1347                         if (pg->bypassed)
1348                                 state = 'D';    /* Disabled */
1349                         else if (pg == m->current_pg)
1350                                 state = 'A';    /* Currently Active */
1351                         else
1352                                 state = 'E';    /* Enabled */
1353
1354                         DMEMIT("%c ", state);
1355
1356                         if (pg->ps.type->status)
1357                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1358                                                           result + sz,
1359                                                           maxlen - sz);
1360                         else
1361                                 DMEMIT("0 ");
1362
1363                         DMEMIT("%u %u ", pg->nr_pgpaths,
1364                                pg->ps.type->info_args);
1365
1366                         list_for_each_entry(p, &pg->pgpaths, list) {
1367                                 DMEMIT("%s %s %u ", p->path.dev->name,
1368                                        p->is_active ? "A" : "F",
1369                                        p->fail_count);
1370                                 if (pg->ps.type->status)
1371                                         sz += pg->ps.type->status(&pg->ps,
1372                                               &p->path, type, result + sz,
1373                                               maxlen - sz);
1374                         }
1375                 }
1376                 break;
1377
1378         case STATUSTYPE_TABLE:
1379                 list_for_each_entry(pg, &m->priority_groups, list) {
1380                         DMEMIT("%s ", pg->ps.type->name);
1381
1382                         if (pg->ps.type->status)
1383                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1384                                                           result + sz,
1385                                                           maxlen - sz);
1386                         else
1387                                 DMEMIT("0 ");
1388
1389                         DMEMIT("%u %u ", pg->nr_pgpaths,
1390                                pg->ps.type->table_args);
1391
1392                         list_for_each_entry(p, &pg->pgpaths, list) {
1393                                 DMEMIT("%s ", p->path.dev->name);
1394                                 if (pg->ps.type->status)
1395                                         sz += pg->ps.type->status(&pg->ps,
1396                                               &p->path, type, result + sz,
1397                                               maxlen - sz);
1398                         }
1399                 }
1400                 break;
1401         }
1402
1403         spin_unlock_irqrestore(&m->lock, flags);
1404
1405         return 0;
1406 }
1407
1408 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1409 {
1410         int r;
1411         struct dm_dev *dev;
1412         struct multipath *m = (struct multipath *) ti->private;
1413         action_fn action;
1414
1415         if (argc == 1) {
1416                 if (!strnicmp(argv[0], MESG_STR("queue_if_no_path")))
1417                         return queue_if_no_path(m, 1, 0);
1418                 else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path")))
1419                         return queue_if_no_path(m, 0, 0);
1420         }
1421
1422         if (argc != 2)
1423                 goto error;
1424
1425         if (!strnicmp(argv[0], MESG_STR("disable_group")))
1426                 return bypass_pg_num(m, argv[1], 1);
1427         else if (!strnicmp(argv[0], MESG_STR("enable_group")))
1428                 return bypass_pg_num(m, argv[1], 0);
1429         else if (!strnicmp(argv[0], MESG_STR("switch_group")))
1430                 return switch_pg_num(m, argv[1]);
1431         else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
1432                 action = reinstate_path;
1433         else if (!strnicmp(argv[0], MESG_STR("fail_path")))
1434                 action = fail_path;
1435         else
1436                 goto error;
1437
1438         r = dm_get_device(ti, argv[1], ti->begin, ti->len,
1439                           dm_table_get_mode(ti->table), &dev);
1440         if (r) {
1441                 DMWARN("message: error getting device %s",
1442                        argv[1]);
1443                 return -EINVAL;
1444         }
1445
1446         r = action_dev(m, dev, action);
1447
1448         dm_put_device(ti, dev);
1449
1450         return r;
1451
1452 error:
1453         DMWARN("Unrecognised multipath message received.");
1454         return -EINVAL;
1455 }
1456
1457 static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
1458                            unsigned long arg)
1459 {
1460         struct multipath *m = (struct multipath *) ti->private;
1461         struct block_device *bdev = NULL;
1462         fmode_t mode = 0;
1463         unsigned long flags;
1464         int r = 0;
1465
1466         spin_lock_irqsave(&m->lock, flags);
1467
1468         if (!m->current_pgpath)
1469                 __choose_pgpath(m, 0);
1470
1471         if (m->current_pgpath) {
1472                 bdev = m->current_pgpath->path.dev->bdev;
1473                 mode = m->current_pgpath->path.dev->mode;
1474         }
1475
1476         if (m->queue_io)
1477                 r = -EAGAIN;
1478         else if (!bdev)
1479                 r = -EIO;
1480
1481         spin_unlock_irqrestore(&m->lock, flags);
1482
1483         return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
1484 }
1485
1486 static int multipath_iterate_devices(struct dm_target *ti,
1487                                      iterate_devices_callout_fn fn, void *data)
1488 {
1489         struct multipath *m = ti->private;
1490         struct priority_group *pg;
1491         struct pgpath *p;
1492         int ret = 0;
1493
1494         list_for_each_entry(pg, &m->priority_groups, list) {
1495                 list_for_each_entry(p, &pg->pgpaths, list) {
1496                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1497                         if (ret)
1498                                 goto out;
1499                 }
1500         }
1501
1502 out:
1503         return ret;
1504 }
1505
1506 static int __pgpath_busy(struct pgpath *pgpath)
1507 {
1508         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1509
1510         return dm_underlying_device_busy(q);
1511 }
1512
1513 /*
1514  * We return "busy", only when we can map I/Os but underlying devices
1515  * are busy (so even if we map I/Os now, the I/Os will wait on
1516  * the underlying queue).
1517  * In other words, if we want to kill I/Os or queue them inside us
1518  * due to map unavailability, we don't return "busy".  Otherwise,
1519  * dm core won't give us the I/Os and we can't do what we want.
1520  */
1521 static int multipath_busy(struct dm_target *ti)
1522 {
1523         int busy = 0, has_active = 0;
1524         struct multipath *m = ti->private;
1525         struct priority_group *pg;
1526         struct pgpath *pgpath;
1527         unsigned long flags;
1528
1529         spin_lock_irqsave(&m->lock, flags);
1530
1531         /* Guess which priority_group will be used at next mapping time */
1532         if (unlikely(!m->current_pgpath && m->next_pg))
1533                 pg = m->next_pg;
1534         else if (likely(m->current_pg))
1535                 pg = m->current_pg;
1536         else
1537                 /*
1538                  * We don't know which pg will be used at next mapping time.
1539                  * We don't call __choose_pgpath() here to avoid to trigger
1540                  * pg_init just by busy checking.
1541                  * So we don't know whether underlying devices we will be using
1542                  * at next mapping time are busy or not. Just try mapping.
1543                  */
1544                 goto out;
1545
1546         /*
1547          * If there is one non-busy active path at least, the path selector
1548          * will be able to select it. So we consider such a pg as not busy.
1549          */
1550         busy = 1;
1551         list_for_each_entry(pgpath, &pg->pgpaths, list)
1552                 if (pgpath->is_active) {
1553                         has_active = 1;
1554
1555                         if (!__pgpath_busy(pgpath)) {
1556                                 busy = 0;
1557                                 break;
1558                         }
1559                 }
1560
1561         if (!has_active)
1562                 /*
1563                  * No active path in this pg, so this pg won't be used and
1564                  * the current_pg will be changed at next mapping time.
1565                  * We need to try mapping to determine it.
1566                  */
1567                 busy = 0;
1568
1569 out:
1570         spin_unlock_irqrestore(&m->lock, flags);
1571
1572         return busy;
1573 }
1574
1575 /*-----------------------------------------------------------------
1576  * Module setup
1577  *---------------------------------------------------------------*/
1578 static struct target_type multipath_target = {
1579         .name = "multipath",
1580         .version = {1, 1, 1},
1581         .module = THIS_MODULE,
1582         .ctr = multipath_ctr,
1583         .dtr = multipath_dtr,
1584         .map_rq = multipath_map,
1585         .rq_end_io = multipath_end_io,
1586         .presuspend = multipath_presuspend,
1587         .postsuspend = multipath_postsuspend,
1588         .resume = multipath_resume,
1589         .status = multipath_status,
1590         .message = multipath_message,
1591         .ioctl  = multipath_ioctl,
1592         .iterate_devices = multipath_iterate_devices,
1593         .busy = multipath_busy,
1594 };
1595
1596 static int __init dm_multipath_init(void)
1597 {
1598         int r;
1599
1600         /* allocate a slab for the dm_ios */
1601         _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
1602         if (!_mpio_cache)
1603                 return -ENOMEM;
1604
1605         r = dm_register_target(&multipath_target);
1606         if (r < 0) {
1607                 DMERR("register failed %d", r);
1608                 kmem_cache_destroy(_mpio_cache);
1609                 return -EINVAL;
1610         }
1611
1612         kmultipathd = create_workqueue("kmpathd");
1613         if (!kmultipathd) {
1614                 DMERR("failed to create workqueue kmpathd");
1615                 dm_unregister_target(&multipath_target);
1616                 kmem_cache_destroy(_mpio_cache);
1617                 return -ENOMEM;
1618         }
1619
1620         /*
1621          * A separate workqueue is used to handle the device handlers
1622          * to avoid overloading existing workqueue. Overloading the
1623          * old workqueue would also create a bottleneck in the
1624          * path of the storage hardware device activation.
1625          */
1626         kmpath_handlerd = create_singlethread_workqueue("kmpath_handlerd");
1627         if (!kmpath_handlerd) {
1628                 DMERR("failed to create workqueue kmpath_handlerd");
1629                 destroy_workqueue(kmultipathd);
1630                 dm_unregister_target(&multipath_target);
1631                 kmem_cache_destroy(_mpio_cache);
1632                 return -ENOMEM;
1633         }
1634
1635         DMINFO("version %u.%u.%u loaded",
1636                multipath_target.version[0], multipath_target.version[1],
1637                multipath_target.version[2]);
1638
1639         return r;
1640 }
1641
1642 static void __exit dm_multipath_exit(void)
1643 {
1644         destroy_workqueue(kmpath_handlerd);
1645         destroy_workqueue(kmultipathd);
1646
1647         dm_unregister_target(&multipath_target);
1648         kmem_cache_destroy(_mpio_cache);
1649 }
1650
1651 module_init(dm_multipath_init);
1652 module_exit(dm_multipath_exit);
1653
1654 MODULE_DESCRIPTION(DM_NAME " multipath target");
1655 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1656 MODULE_LICENSE("GPL");