drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-path-selector.h"
  11 #include "dm-uevent.h"
  12
  13 #include <linux/ctype.h>
  14 #include <linux/init.h>
  15 #include <linux/mempool.h>
  16 #include <linux/module.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/slab.h>
  19 #include <linux/time.h>
  20 #include <linux/workqueue.h>
  21 #include <scsi/scsi_dh.h>
  22 #include <linux/atomic.h>
  23
  24 #define DM_MSG_PREFIX "multipath"
  25 #define DM_PG_INIT_DELAY_MSECS 2000
  26 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
  27
  28 /* Path properties */
  29 struct pgpath {
  30         struct list_head list;
  31
  32         struct priority_group *pg;      /* Owning PG */
  33         unsigned is_active;             /* Path status */
  34         unsigned fail_count;            /* Cumulative failure count */
  35
  36         struct dm_path path;
  37         struct delayed_work activate_path;
  38 };
  39
  40 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  41
  42 /*
  43  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  44  * Each has a path selector which controls which path gets used.
  45  */
  46 struct priority_group {
  47         struct list_head list;
  48
  49         struct multipath *m;            /* Owning multipath instance */
  50         struct path_selector ps;
  51
  52         unsigned pg_num;                /* Reference number */
  53         unsigned bypassed;              /* Temporarily bypass this PG? */
  54
  55         unsigned nr_pgpaths;            /* Number of paths in PG */
  56         struct list_head pgpaths;
  57 };
  58
  59 #define FEATURE_NO_PARTITIONS 1
  60
  61 /* Multipath context */
  62 struct multipath {
  63         struct list_head list;
  64         struct dm_target *ti;
  65
  66         spinlock_t lock;
  67
  68         const char *hw_handler_name;
  69         char *hw_handler_params;
  70
  71         unsigned nr_priority_groups;
  72         struct list_head priority_groups;
  73
  74         wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  75
  76         unsigned pg_init_required;      /* pg_init needs calling? */
  77         unsigned pg_init_in_progress;   /* Only one pg_init allowed at once */
  78         unsigned pg_init_delay_retry;   /* Delay pg_init retry? */
  79
  80         unsigned nr_valid_paths;        /* Total number of usable paths */
  81         struct pgpath *current_pgpath;
  82         struct priority_group *current_pg;
  83         struct priority_group *next_pg; /* Switch to this PG if set */
  84         unsigned repeat_count;          /* I/Os left before calling PS again */
  85
  86         unsigned queue_io;              /* Must we queue all I/O? */
  87         unsigned queue_if_no_path;      /* Queue I/O if last path fails? */
  88         unsigned saved_queue_if_no_path;/* Saved state during suspension */
  89         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  90         unsigned pg_init_count;         /* Number of times pg_init called */
  91         unsigned pg_init_delay_msecs;   /* Number of msecs before pg_init retry */
  92         unsigned features;              /* Additional selected features */
  93
  94         struct work_struct process_queued_ios;
  95         struct list_head queued_ios;
  96         unsigned queue_size;
  97
  98         struct work_struct trigger_event;
  99
 100         /*
 101          * We must use a mempool of dm_mpath_io structs so that we
 102          * can resubmit bios on error.
 103          */
 104         mempool_t *mpio_pool;
 105
 106         struct mutex work_mutex;
 107 };
 108
 109 /*
 110  * Context information attached to each bio we process.
 111  */
 112 struct dm_mpath_io {
 113         struct pgpath *pgpath;
 114         size_t nr_bytes;
 115 };
 116
 117 typedef int (*action_fn) (struct pgpath *pgpath);
 118
 119 #define MIN_IOS 256     /* Mempool size */
 120
 121 static struct kmem_cache *_mpio_cache;
 122
 123 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 124 static void process_queued_ios(struct work_struct *work);
 125 static void trigger_event(struct work_struct *work);
 126 static void activate_path(struct work_struct *work);
 127
 128
 129 /*-----------------------------------------------
 130  * Allocation routines
 131  *-----------------------------------------------*/
 132
 133 static struct pgpath *alloc_pgpath(void)
 134 {
 135         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 136
 137         if (pgpath) {
 138                 pgpath->is_active = 1;
 139                 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
 140         }
 141
 142         return pgpath;
 143 }
 144
 145 static void free_pgpath(struct pgpath *pgpath)
 146 {
 147         kfree(pgpath);
 148 }
 149
 150 static struct priority_group *alloc_priority_group(void)
 151 {
 152         struct priority_group *pg;
 153
 154         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 155
 156         if (pg)
 157                 INIT_LIST_HEAD(&pg->pgpaths);
 158
 159         return pg;
 160 }
 161
 162 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 163 {
 164         struct pgpath *pgpath, *tmp;
 165
 166         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 167                 list_del(&pgpath->list);
 168                 dm_put_device(ti, pgpath->path.dev);
 169                 free_pgpath(pgpath);
 170         }
 171 }
 172
 173 static void free_priority_group(struct priority_group *pg,
 174                                 struct dm_target *ti)
 175 {
 176         struct path_selector *ps = &pg->ps;
 177
 178         if (ps->type) {
 179                 ps->type->destroy(ps);
 180                 dm_put_path_selector(ps->type);
 181         }
 182
 183         free_pgpaths(&pg->pgpaths, ti);
 184         kfree(pg);
 185 }
 186
 187 static struct multipath *alloc_multipath(struct dm_target *ti)
 188 {
 189         struct multipath *m;
 190
 191         m = kzalloc(sizeof(*m), GFP_KERNEL);
 192         if (m) {
 193                 INIT_LIST_HEAD(&m->priority_groups);
 194                 INIT_LIST_HEAD(&m->queued_ios);
 195                 spin_lock_init(&m->lock);
 196                 m->queue_io = 1;
 197                 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 198                 INIT_WORK(&m->process_queued_ios, process_queued_ios);
 199                 INIT_WORK(&m->trigger_event, trigger_event);
 200                 init_waitqueue_head(&m->pg_init_wait);
 201                 mutex_init(&m->work_mutex);
 202                 m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
 203                 if (!m->mpio_pool) {
 204                         kfree(m);
 205                         return NULL;
 206                 }
 207                 m->ti = ti;
 208                 ti->private = m;
 209         }
 210
 211         return m;
 212 }
 213
 214 static void free_multipath(struct multipath *m)
 215 {
 216         struct priority_group *pg, *tmp;
 217
 218         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 219                 list_del(&pg->list);
 220                 free_priority_group(pg, m->ti);
 221         }
 222
 223         kfree(m->hw_handler_name);
 224         kfree(m->hw_handler_params);
 225         mempool_destroy(m->mpio_pool);
 226         kfree(m);
 227 }
 228
 229 static int set_mapinfo(struct multipath *m, union map_info *info)
 230 {
 231         struct dm_mpath_io *mpio;
 232
 233         mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
 234         if (!mpio)
 235                 return -ENOMEM;
 236
 237         memset(mpio, 0, sizeof(*mpio));
 238         info->ptr = mpio;
 239
 240         return 0;
 241 }
 242
 243 static void clear_mapinfo(struct multipath *m, union map_info *info)
 244 {
 245         struct dm_mpath_io *mpio = info->ptr;
 246
 247         info->ptr = NULL;
 248         mempool_free(mpio, m->mpio_pool);
 249 }
 250
 251 /*-----------------------------------------------
 252  * Path selection
 253  *-----------------------------------------------*/
 254
 255 static void __pg_init_all_paths(struct multipath *m)
 256 {
 257         struct pgpath *pgpath;
 258         unsigned long pg_init_delay = 0;
 259
 260         m->pg_init_count++;
 261         m->pg_init_required = 0;
 262         if (m->pg_init_delay_retry)
 263                 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
 264                                                  m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
 265         list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
 266                 /* Skip failed paths */
 267                 if (!pgpath->is_active)
 268                         continue;
 269                 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
 270                                        pg_init_delay))
 271                         m->pg_init_in_progress++;
 272         }
 273 }
 274
 275 static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
 276 {
 277         m->current_pg = pgpath->pg;
 278
 279         /* Must we initialise the PG first, and queue I/O till it's ready? */
 280         if (m->hw_handler_name) {
 281                 m->pg_init_required = 1;
 282                 m->queue_io = 1;
 283         } else {
 284                 m->pg_init_required = 0;
 285                 m->queue_io = 0;
 286         }
 287
 288         m->pg_init_count = 0;
 289 }
 290
 291 static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
 292                                size_t nr_bytes)
 293 {
 294         struct dm_path *path;
 295
 296         path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
 297         if (!path)
 298                 return -ENXIO;
 299
 300         m->current_pgpath = path_to_pgpath(path);
 301
 302         if (!m->current_pgpath->path.dev) {
 303                 m->current_pgpath = NULL;
 304                 return -ENODEV;
 305         }
 306
 307         if (m->current_pg != pg)
 308                 __switch_pg(m, m->current_pgpath);
 309
 310         return 0;
 311 }
 312
 313 static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
 314 {
 315         struct priority_group *pg;
 316         unsigned bypassed = 1;
 317
 318         if (!m->nr_valid_paths)
 319                 goto failed;
 320
 321         /* Were we instructed to switch PG? */
 322         if (m->next_pg) {
 323                 pg = m->next_pg;
 324                 m->next_pg = NULL;
 325                 if (!__choose_path_in_pg(m, pg, nr_bytes))
 326                         return;
 327         }
 328
 329         /* Don't change PG until it has no remaining paths */
 330         if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
 331                 return;
 332
 333         /*
 334          * Loop through priority groups until we find a valid path.
 335          * First time we skip PGs marked 'bypassed'.
 336          * Second time we only try the ones we skipped.
 337          */
 338         do {
 339                 list_for_each_entry(pg, &m->priority_groups, list) {
 340                         if (pg->bypassed == bypassed)
 341                                 continue;
 342                         if (!__choose_path_in_pg(m, pg, nr_bytes))
 343                                 return;
 344                 }
 345         } while (bypassed--);
 346
 347 failed:
 348         m->current_pgpath = NULL;
 349         m->current_pg = NULL;
 350 }
 351
 352 /*
 353  * Check whether bios must be queued in the device-mapper core rather
 354  * than here in the target.
 355  *
 356  * m->lock must be held on entry.
 357  *
 358  * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
 359  * same value then we are not between multipath_presuspend()
 360  * and multipath_resume() calls and we have no need to check
 361  * for the DMF_NOFLUSH_SUSPENDING flag.
 362  */
 363 static int __must_push_back(struct multipath *m)
 364 {
 365         return (m->queue_if_no_path != m->saved_queue_if_no_path &&
 366                 dm_noflush_suspending(m->ti));
 367 }
 368
 369 static int map_io(struct multipath *m, struct request *clone,
 370                   union map_info *map_context, unsigned was_queued)
 371 {
 372         int r = DM_MAPIO_REMAPPED;
 373         size_t nr_bytes = blk_rq_bytes(clone);
 374         unsigned long flags;
 375         struct pgpath *pgpath;
 376         struct block_device *bdev;
 377         struct dm_mpath_io *mpio = map_context->ptr;
 378
 379         spin_lock_irqsave(&m->lock, flags);
 380
 381         /* Do we need to select a new pgpath? */
 382         if (!m->current_pgpath ||
 383             (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
 384                 __choose_pgpath(m, nr_bytes);
 385
 386         pgpath = m->current_pgpath;
 387
 388         if (was_queued)
 389                 m->queue_size--;
 390
 391         if ((pgpath && m->queue_io) ||
 392             (!pgpath && m->queue_if_no_path)) {
 393                 /* Queue for the daemon to resubmit */
 394                 list_add_tail(&clone->queuelist, &m->queued_ios);
 395                 m->queue_size++;
 396                 if ((m->pg_init_required && !m->pg_init_in_progress) ||
 397                     !m->queue_io)
 398                         queue_work(kmultipathd, &m->process_queued_ios);
 399                 pgpath = NULL;
 400                 r = DM_MAPIO_SUBMITTED;
 401         } else if (pgpath) {
 402                 bdev = pgpath->path.dev->bdev;
 403                 clone->q = bdev_get_queue(bdev);
 404                 clone->rq_disk = bdev->bd_disk;
 405         } else if (__must_push_back(m))
 406                 r = DM_MAPIO_REQUEUE;
 407         else
 408                 r = -EIO;       /* Failed */
 409
 410         mpio->pgpath = pgpath;
 411         mpio->nr_bytes = nr_bytes;
 412
 413         if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
 414                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
 415                                               nr_bytes);
 416
 417         spin_unlock_irqrestore(&m->lock, flags);
 418
 419         return r;
 420 }
 421
 422 /*
 423  * If we run out of usable paths, should we queue I/O or error it?
 424  */
 425 static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
 426                             unsigned save_old_value)
 427 {
 428         unsigned long flags;
 429
 430         spin_lock_irqsave(&m->lock, flags);
 431
 432         if (save_old_value)
 433                 m->saved_queue_if_no_path = m->queue_if_no_path;
 434         else
 435                 m->saved_queue_if_no_path = queue_if_no_path;
 436         m->queue_if_no_path = queue_if_no_path;
 437         if (!m->queue_if_no_path && m->queue_size)
 438                 queue_work(kmultipathd, &m->process_queued_ios);
 439
 440         spin_unlock_irqrestore(&m->lock, flags);
 441
 442         return 0;
 443 }
 444
 445 /*-----------------------------------------------------------------
 446  * The multipath daemon is responsible for resubmitting queued ios.
 447  *---------------------------------------------------------------*/
 448
 449 static void dispatch_queued_ios(struct multipath *m)
 450 {
 451         int r;
 452         unsigned long flags;
 453         union map_info *info;
 454         struct request *clone, *n;
 455         LIST_HEAD(cl);
 456
 457         spin_lock_irqsave(&m->lock, flags);
 458         list_splice_init(&m->queued_ios, &cl);
 459         spin_unlock_irqrestore(&m->lock, flags);
 460
 461         list_for_each_entry_safe(clone, n, &cl, queuelist) {
 462                 list_del_init(&clone->queuelist);
 463
 464                 info = dm_get_rq_mapinfo(clone);
 465
 466                 r = map_io(m, clone, info, 1);
 467                 if (r < 0) {
 468                         clear_mapinfo(m, info);
 469                         dm_kill_unmapped_request(clone, r);
 470                 } else if (r == DM_MAPIO_REMAPPED)
 471                         dm_dispatch_request(clone);
 472                 else if (r == DM_MAPIO_REQUEUE) {
 473                         clear_mapinfo(m, info);
 474                         dm_requeue_unmapped_request(clone);
 475                 }
 476         }
 477 }
 478
 479 static void process_queued_ios(struct work_struct *work)
 480 {
 481         struct multipath *m =
 482                 container_of(work, struct multipath, process_queued_ios);
 483         struct pgpath *pgpath = NULL;
 484         unsigned must_queue = 1;
 485         unsigned long flags;
 486
 487         spin_lock_irqsave(&m->lock, flags);
 488
 489         if (!m->queue_size)
 490                 goto out;
 491
 492         if (!m->current_pgpath)
 493                 __choose_pgpath(m, 0);
 494
 495         pgpath = m->current_pgpath;
 496
 497         if ((pgpath && !m->queue_io) ||
 498             (!pgpath && !m->queue_if_no_path))
 499                 must_queue = 0;
 500
 501         if (m->pg_init_required && !m->pg_init_in_progress && pgpath)
 502                 __pg_init_all_paths(m);
 503
 504 out:
 505         spin_unlock_irqrestore(&m->lock, flags);
 506         if (!must_queue)
 507                 dispatch_queued_ios(m);
 508 }
 509
 510 /*
 511  * An event is triggered whenever a path is taken out of use.
 512  * Includes path failure and PG bypass.
 513  */
 514 static void trigger_event(struct work_struct *work)
 515 {
 516         struct multipath *m =
 517                 container_of(work, struct multipath, trigger_event);
 518
 519         dm_table_event(m->ti->table);
 520 }
 521
 522 /*-----------------------------------------------------------------
 523  * Constructor/argument parsing:
 524  * <#multipath feature args> [<arg>]*
 525  * <#hw_handler args> [hw_handler [<arg>]*]
 526  * <#priority groups>
 527  * <initial priority group>
 528  *     [<selector> <#selector args> [<arg>]*
 529  *      <#paths> <#per-path selector args>
 530  *         [<path> [<arg>]* ]+ ]+
 531  *---------------------------------------------------------------*/
 532 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 533                                struct dm_target *ti)
 534 {
 535         int r;
 536         struct path_selector_type *pst;
 537         unsigned ps_argc;
 538
 539         static struct dm_arg _args[] = {
 540                 {0, 1024, "invalid number of path selector args"},
 541         };
 542
 543         pst = dm_get_path_selector(dm_shift_arg(as));
 544         if (!pst) {
 545                 ti->error = "unknown path selector type";
 546                 return -EINVAL;
 547         }
 548
 549         r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
 550         if (r) {
 551                 dm_put_path_selector(pst);
 552                 return -EINVAL;
 553         }
 554
 555         r = pst->create(&pg->ps, ps_argc, as->argv);
 556         if (r) {
 557                 dm_put_path_selector(pst);
 558                 ti->error = "path selector constructor failed";
 559                 return r;
 560         }
 561
 562         pg->ps.type = pst;
 563         dm_consume_args(as, ps_argc);
 564
 565         return 0;
 566 }
 567
 568 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
 569                                struct dm_target *ti)
 570 {
 571         int r;
 572         struct pgpath *p;
 573         char *path;
 574         struct multipath *m = ti->private;
 575
 576         /* we need at least a path arg */
 577         if (as->argc < 1) {
 578                 ti->error = "no device given";
 579                 return ERR_PTR(-EINVAL);
 580         }
 581
 582         p = alloc_pgpath();
 583         if (!p)
 584                 return ERR_PTR(-ENOMEM);
 585
 586         path = dm_shift_arg(as);
 587         r = dm_get_device(ti, path, dm_table_get_mode(ti->table),
 588                           &p->path.dev);
 589         if (r) {
 590                 unsigned major, minor;
 591
 592                 /* Try to add a failed device */
 593                 if (r == -ENXIO && sscanf(path, "%u:%u", &major, &minor) == 2) {
 594                         dev_t dev;
 595
 596                         /* Extract the major/minor numbers */
 597                         dev = MKDEV(major, minor);
 598                         if (MAJOR(dev) != major || MINOR(dev) != minor) {
 599                                 /* Nice try, didn't work */
 600                                 DMWARN("Invalid device path %s", path);
 601                                 ti->error = "error converting devnum";
 602                                 goto bad;
 603                         }
 604                         DMWARN("adding disabled device %d:%d", major, minor);
 605                         p->path.dev = NULL;
 606                         format_dev_t(p->path.pdev, dev);
 607                         p->is_active = 0;
 608                 } else {
 609                         ti->error = "error getting device";
 610                         goto bad;
 611                 }
 612         } else {
 613                 memcpy(p->path.pdev, p->path.dev->name, 16);
 614         }
 615
 616         if (p->path.dev) {
 617                 struct request_queue *q = bdev_get_queue(p->path.dev->bdev);
 618
 619                 if (m->hw_handler_name) {
 620                         r = scsi_dh_attach(q, m->hw_handler_name);
 621                         if (r == -EBUSY) {
 622                                 /*
 623                                  * Already attached to different hw_handler,
 624                                  * try to reattach with correct one.
 625                                  */
 626                                 scsi_dh_detach(q);
 627                                 r = scsi_dh_attach(q, m->hw_handler_name);
 628                         }
 629                         if (r < 0) {
 630                                 ti->error = "error attaching hardware handler";
 631                                 dm_put_device(ti, p->path.dev);
 632                                 goto bad;
 633                         }
 634                 } else {
 635                         /* Play safe and detach hardware handler */
 636                         scsi_dh_detach(q);
 637                 }
 638
 639                 if (m->hw_handler_params) {
 640                         r = scsi_dh_set_params(q, m->hw_handler_params);
 641                         if (r < 0) {
 642                                 ti->error = "unable to set hardware "
 643                                                         "handler parameters";
 644                                 scsi_dh_detach(q);
 645                                 dm_put_device(ti, p->path.dev);
 646                                 goto bad;
 647                         }
 648                 }
 649         }
 650
 651         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 652         if (r) {
 653                 dm_put_device(ti, p->path.dev);
 654                 goto bad;
 655         }
 656
 657         if (!p->is_active) {
 658                 ps->type->fail_path(ps, &p->path);
 659                 p->fail_count++;
 660                 m->nr_valid_paths--;
 661         }
 662         return p;
 663
 664  bad:
 665         free_pgpath(p);
 666         return ERR_PTR(r);
 667 }
 668
 669 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 670                                                    struct multipath *m)
 671 {
 672         static struct dm_arg _args[] = {
 673                 {1, 1024, "invalid number of paths"},
 674                 {0, 1024, "invalid number of selector args"}
 675         };
 676
 677         int r;
 678         unsigned i, nr_selector_args, nr_args;
 679         struct priority_group *pg;
 680         struct dm_target *ti = m->ti;
 681
 682         if (as->argc < 2) {
 683                 as->argc = 0;
 684                 ti->error = "not enough priority group arguments";
 685                 return ERR_PTR(-EINVAL);
 686         }
 687
 688         pg = alloc_priority_group();
 689         if (!pg) {
 690                 ti->error = "couldn't allocate priority group";
 691                 return ERR_PTR(-ENOMEM);
 692         }
 693         pg->m = m;
 694
 695         r = parse_path_selector(as, pg, ti);
 696         if (r)
 697                 goto bad;
 698
 699         /*
 700          * read the paths
 701          */
 702         r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
 703         if (r)
 704                 goto bad;
 705
 706         r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
 707         if (r)
 708                 goto bad;
 709
 710         nr_args = 1 + nr_selector_args;
 711         for (i = 0; i < pg->nr_pgpaths; i++) {
 712                 struct pgpath *pgpath;
 713                 struct dm_arg_set path_args;
 714
 715                 if (as->argc < nr_args) {
 716                         ti->error = "not enough path parameters";
 717                         r = -EINVAL;
 718                         goto bad;
 719                 }
 720
 721                 path_args.argc = nr_args;
 722                 path_args.argv = as->argv;
 723
 724                 pgpath = parse_path(&path_args, &pg->ps, ti);
 725                 if (IS_ERR(pgpath)) {
 726                         r = PTR_ERR(pgpath);
 727                         goto bad;
 728                 }
 729
 730                 pgpath->pg = pg;
 731                 list_add_tail(&pgpath->list, &pg->pgpaths);
 732                 dm_consume_args(as, nr_args);
 733         }
 734
 735         return pg;
 736
 737  bad:
 738         free_priority_group(pg, ti);
 739         return ERR_PTR(r);
 740 }
 741
 742 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 743 {
 744         unsigned hw_argc;
 745         int ret;
 746         struct dm_target *ti = m->ti;
 747
 748         static struct dm_arg _args[] = {
 749                 {0, 1024, "invalid number of hardware handler args"},
 750         };
 751
 752         if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
 753                 return -EINVAL;
 754
 755         if (!hw_argc)
 756                 return 0;
 757
 758         m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
 759         if (!try_then_request_module(scsi_dh_handler_exist(m->hw_handler_name),
 760                                      "scsi_dh_%s", m->hw_handler_name)) {
 761                 ti->error = "unknown hardware handler type";
 762                 ret = -EINVAL;
 763                 goto fail;
 764         }
 765
 766         if (hw_argc > 1) {
 767                 char *p;
 768                 int i, j, len = 4;
 769
 770                 for (i = 0; i <= hw_argc - 2; i++)
 771                         len += strlen(as->argv[i]) + 1;
 772                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
 773                 if (!p) {
 774                         ti->error = "memory allocation failed";
 775                         ret = -ENOMEM;
 776                         goto fail;
 777                 }
 778                 j = sprintf(p, "%d", hw_argc - 1);
 779                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
 780                         j = sprintf(p, "%s", as->argv[i]);
 781         }
 782         dm_consume_args(as, hw_argc - 1);
 783
 784         return 0;
 785 fail:
 786         kfree(m->hw_handler_name);
 787         m->hw_handler_name = NULL;
 788         return ret;
 789 }
 790
 791 static int parse_features(struct dm_arg_set *as, struct multipath *m)
 792 {
 793         int r;
 794         unsigned argc;
 795         struct dm_target *ti = m->ti;
 796         const char *arg_name;
 797
 798         static struct dm_arg _args[] = {
 799                 {0, 5, "invalid number of feature args"},
 800                 {1, 50, "pg_init_retries must be between 1 and 50"},
 801                 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
 802         };
 803
 804         r = dm_read_arg_group(_args, as, &argc, &ti->error);
 805         if (r)
 806                 return -EINVAL;
 807
 808         if (!argc)
 809                 return 0;
 810
 811         do {
 812                 arg_name = dm_shift_arg(as);
 813                 argc--;
 814
 815                 if (!strcasecmp(arg_name, "queue_if_no_path")) {
 816                         r = queue_if_no_path(m, 1, 0);
 817                         continue;
 818                 }
 819
 820                 if (!strcasecmp(arg_name, "no_partitions")) {
 821                         m->features |= FEATURE_NO_PARTITIONS;
 822                         continue;
 823                 }
 824                 if (!strcasecmp(arg_name, "pg_init_retries") &&
 825                     (argc >= 1)) {
 826                         r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
 827                         argc--;
 828                         continue;
 829                 }
 830
 831                 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
 832                     (argc >= 1)) {
 833                         r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
 834                         argc--;
 835                         continue;
 836                 }
 837
 838                 ti->error = "Unrecognised multipath feature request";
 839                 r = -EINVAL;
 840         } while (argc && !r);
 841
 842         return r;
 843 }
 844
 845 static int multipath_ctr(struct dm_target *ti, unsigned int argc,
 846                          char **argv)
 847 {
 848         /* target arguments */
 849         static struct dm_arg _args[] = {
 850                 {0, 1024, "invalid number of priority groups"},
 851                 {0, 1024, "invalid initial priority group number"},
 852         };
 853
 854         int r;
 855         struct multipath *m;
 856         struct dm_arg_set as;
 857         unsigned pg_count = 0;
 858         unsigned next_pg_num;
 859
 860         as.argc = argc;
 861         as.argv = argv;
 862
 863         m = alloc_multipath(ti);
 864         if (!m) {
 865                 ti->error = "can't allocate multipath";
 866                 return -EINVAL;
 867         }
 868
 869         r = parse_features(&as, m);
 870         if (r)
 871                 goto bad;
 872
 873         r = parse_hw_handler(&as, m);
 874         if (r)
 875                 goto bad;
 876
 877         r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
 878         if (r)
 879                 goto bad;
 880
 881         r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
 882         if (r)
 883                 goto bad;
 884
 885         if ((!m->nr_priority_groups && next_pg_num) ||
 886             (m->nr_priority_groups && !next_pg_num)) {
 887                 ti->error = "invalid initial priority group";
 888                 r = -EINVAL;
 889                 goto bad;
 890         }
 891
 892         /* parse the priority groups */
 893         while (as.argc) {
 894                 struct priority_group *pg;
 895
 896                 pg = parse_priority_group(&as, m);
 897                 if (IS_ERR(pg)) {
 898                         r = PTR_ERR(pg);
 899                         goto bad;
 900                 }
 901
 902                 m->nr_valid_paths += pg->nr_pgpaths;
 903                 list_add_tail(&pg->list, &m->priority_groups);
 904                 pg_count++;
 905                 pg->pg_num = pg_count;
 906                 if (!--next_pg_num)
 907                         m->next_pg = pg;
 908         }
 909
 910         if (pg_count != m->nr_priority_groups) {
 911                 ti->error = "priority group count mismatch";
 912                 r = -EINVAL;
 913                 goto bad;
 914         }
 915
 916         ti->num_flush_requests = 1;
 917         ti->num_discard_requests = 1;
 918
 919         return 0;
 920
 921  bad:
 922         free_multipath(m);
 923         return r;
 924 }
 925
 926 static void multipath_wait_for_pg_init_completion(struct multipath *m)
 927 {
 928         DECLARE_WAITQUEUE(wait, current);
 929         unsigned long flags;
 930
 931         add_wait_queue(&m->pg_init_wait, &wait);
 932
 933         while (1) {
 934                 set_current_state(TASK_UNINTERRUPTIBLE);
 935
 936                 spin_lock_irqsave(&m->lock, flags);
 937                 if (!m->pg_init_in_progress) {
 938                         spin_unlock_irqrestore(&m->lock, flags);
 939                         break;
 940                 }
 941                 spin_unlock_irqrestore(&m->lock, flags);
 942
 943                 io_schedule();
 944         }
 945         set_current_state(TASK_RUNNING);
 946
 947         remove_wait_queue(&m->pg_init_wait, &wait);
 948 }
 949
 950 static void flush_multipath_work(struct multipath *m)
 951 {
 952         flush_workqueue(kmpath_handlerd);
 953         multipath_wait_for_pg_init_completion(m);
 954         flush_workqueue(kmultipathd);
 955         flush_work_sync(&m->trigger_event);
 956 }
 957
 958 static void multipath_dtr(struct dm_target *ti)
 959 {
 960         struct multipath *m = ti->private;
 961
 962         flush_multipath_work(m);
 963         free_multipath(m);
 964 }
 965
 966 /*
 967  * Map cloned requests
 968  */
 969 static int multipath_map(struct dm_target *ti, struct request *clone,
 970                          union map_info *map_context)
 971 {
 972         int r;
 973         struct multipath *m = (struct multipath *) ti->private;
 974
 975         if (set_mapinfo(m, map_context) < 0)
 976                 /* ENOMEM, requeue */
 977                 return DM_MAPIO_REQUEUE;
 978
 979         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 980         r = map_io(m, clone, map_context, 0);
 981         if (r < 0 || r == DM_MAPIO_REQUEUE)
 982                 clear_mapinfo(m, map_context);
 983
 984         return r;
 985 }
 986
 987 /*
 988  * Take a path out of use.
 989  */
 990 static int fail_path(struct pgpath *pgpath)
 991 {
 992         unsigned long flags;
 993         struct multipath *m = pgpath->pg->m;
 994
 995         spin_lock_irqsave(&m->lock, flags);
 996
 997         if (!pgpath->is_active)
 998                 goto out;
 999
1000         DMWARN("Failing path %s.", pgpath->path.pdev);
1001
1002         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1003         pgpath->is_active = 0;
1004         pgpath->fail_count++;
1005
1006         m->nr_valid_paths--;
1007
1008         if (pgpath == m->current_pgpath)
1009                 m->current_pgpath = NULL;
1010
1011         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1012                        pgpath->path.pdev, m->nr_valid_paths);
1013
1014         schedule_work(&m->trigger_event);
1015
1016 out:
1017         spin_unlock_irqrestore(&m->lock, flags);
1018
1019         return 0;
1020 }
1021
1022 /*
1023  * Reinstate a previously-failed path
1024  */
1025 static int reinstate_path(struct pgpath *pgpath)
1026 {
1027         int r = 0;
1028         unsigned long flags;
1029         struct multipath *m = pgpath->pg->m;
1030
1031         spin_lock_irqsave(&m->lock, flags);
1032
1033         if (pgpath->is_active)
1034                 goto out;
1035
1036         if (!pgpath->path.dev) {
1037                 DMWARN("Cannot reinstate disabled path %s", pgpath->path.pdev);
1038                 r = -ENODEV;
1039                 goto out;
1040         }
1041
1042         if (!pgpath->pg->ps.type->reinstate_path) {
1043                 DMWARN("Reinstate path not supported by path selector %s",
1044                        pgpath->pg->ps.type->name);
1045                 r = -EINVAL;
1046                 goto out;
1047         }
1048
1049         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1050         if (r)
1051                 goto out;
1052
1053         pgpath->is_active = 1;
1054
1055         if (!m->nr_valid_paths++ && m->queue_size) {
1056                 m->current_pgpath = NULL;
1057                 queue_work(kmultipathd, &m->process_queued_ios);
1058         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1059                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1060                         m->pg_init_in_progress++;
1061         }
1062
1063         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1064                        pgpath->path.pdev, m->nr_valid_paths);
1065
1066         schedule_work(&m->trigger_event);
1067
1068 out:
1069         spin_unlock_irqrestore(&m->lock, flags);
1070
1071         return r;
1072 }
1073
1074 /*
1075  * Fail or reinstate all paths that match the provided struct dm_dev.
1076  */
1077 static int action_dev(struct multipath *m, struct dm_dev *dev,
1078                       action_fn action)
1079 {
1080         int r = -EINVAL;
1081         struct pgpath *pgpath;
1082         struct priority_group *pg;
1083
1084         if (!dev)
1085                 return 0;
1086
1087         list_for_each_entry(pg, &m->priority_groups, list) {
1088                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1089                         if (pgpath->path.dev == dev)
1090                                 r = action(pgpath);
1091                 }
1092         }
1093
1094         return r;
1095 }
1096
1097 /*
1098  * Temporarily try to avoid having to use the specified PG
1099  */
1100 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1101                       int bypassed)
1102 {
1103         unsigned long flags;
1104
1105         spin_lock_irqsave(&m->lock, flags);
1106
1107         pg->bypassed = bypassed;
1108         m->current_pgpath = NULL;
1109         m->current_pg = NULL;
1110
1111         spin_unlock_irqrestore(&m->lock, flags);
1112
1113         schedule_work(&m->trigger_event);
1114 }
1115
1116 /*
1117  * Switch to using the specified PG from the next I/O that gets mapped
1118  */
1119 static int switch_pg_num(struct multipath *m, const char *pgstr)
1120 {
1121         struct priority_group *pg;
1122         unsigned pgnum;
1123         unsigned long flags;
1124         char dummy;
1125
1126         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1127             (pgnum > m->nr_priority_groups)) {
1128                 DMWARN("invalid PG number supplied to switch_pg_num");
1129                 return -EINVAL;
1130         }
1131
1132         spin_lock_irqsave(&m->lock, flags);
1133         list_for_each_entry(pg, &m->priority_groups, list) {
1134                 pg->bypassed = 0;
1135                 if (--pgnum)
1136                         continue;
1137
1138                 m->current_pgpath = NULL;
1139                 m->current_pg = NULL;
1140                 m->next_pg = pg;
1141         }
1142         spin_unlock_irqrestore(&m->lock, flags);
1143
1144         schedule_work(&m->trigger_event);
1145         return 0;
1146 }
1147
1148 /*
1149  * Set/clear bypassed status of a PG.
1150  * PGs are numbered upwards from 1 in the order they were declared.
1151  */
1152 static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
1153 {
1154         struct priority_group *pg;
1155         unsigned pgnum;
1156         char dummy;
1157
1158         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1159             (pgnum > m->nr_priority_groups)) {
1160                 DMWARN("invalid PG number supplied to bypass_pg");
1161                 return -EINVAL;
1162         }
1163
1164         list_for_each_entry(pg, &m->priority_groups, list) {
1165                 if (!--pgnum)
1166                         break;
1167         }
1168
1169         bypass_pg(m, pg, bypassed);
1170         return 0;
1171 }
1172
1173 /*
1174  * Should we retry pg_init immediately?
1175  */
1176 static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1177 {
1178         unsigned long flags;
1179         int limit_reached = 0;
1180
1181         spin_lock_irqsave(&m->lock, flags);
1182
1183         if (m->pg_init_count <= m->pg_init_retries)
1184                 m->pg_init_required = 1;
1185         else
1186                 limit_reached = 1;
1187
1188         spin_unlock_irqrestore(&m->lock, flags);
1189
1190         return limit_reached;
1191 }
1192
1193 static void pg_init_done(void *data, int errors)
1194 {
1195         struct pgpath *pgpath = data;
1196         struct priority_group *pg = pgpath->pg;
1197         struct multipath *m = pg->m;
1198         unsigned long flags;
1199         unsigned delay_retry = 0;
1200
1201         /* device or driver problems */
1202         switch (errors) {
1203         case SCSI_DH_OK:
1204                 break;
1205         case SCSI_DH_NOSYS:
1206                 if (!m->hw_handler_name) {
1207                         errors = 0;
1208                         break;
1209                 }
1210                 DMERR("Count not failover device %s: Handler scsi_dh_%s "
1211                       "was not loaded.", pgpath->path.dev->name,
1212                       m->hw_handler_name);
1213                 /*
1214                  * Fail path for now, so we do not ping pong
1215                  */
1216                 fail_path(pgpath);
1217                 break;
1218         case SCSI_DH_DEV_TEMP_BUSY:
1219                 /*
1220                  * Probably doing something like FW upgrade on the
1221                  * controller so try the other pg.
1222                  */
1223                 bypass_pg(m, pg, 1);
1224                 break;
1225         case SCSI_DH_DEV_OFFLINED:
1226                 DMWARN("Device %s offlined.", pgpath->path.dev->name);
1227                 errors = 0;
1228                 break;
1229         case SCSI_DH_RETRY:
1230                 /* Wait before retrying. */
1231                 delay_retry = 1;
1232         case SCSI_DH_IMM_RETRY:
1233         case SCSI_DH_RES_TEMP_UNAVAIL:
1234                 if (pg_init_limit_reached(m, pgpath))
1235                         fail_path(pgpath);
1236                 errors = 0;
1237                 break;
1238         default:
1239                 /*
1240                  * We probably do not want to fail the path for a device
1241                  * error, but this is what the old dm did. In future
1242                  * patches we can do more advanced handling.
1243                  */
1244                 fail_path(pgpath);
1245         }
1246
1247         spin_lock_irqsave(&m->lock, flags);
1248         if (errors) {
1249                 if (pgpath == m->current_pgpath) {
1250                         DMERR("Could not failover device %s, error %d.",
1251                               pgpath->path.dev->name, errors);
1252                         m->current_pgpath = NULL;
1253                         m->current_pg = NULL;
1254                 }
1255         } else if (!m->pg_init_required)
1256                 pg->bypassed = 0;
1257
1258         if (--m->pg_init_in_progress)
1259                 /* Activations of other paths are still on going */
1260                 goto out;
1261
1262         if (!m->pg_init_required)
1263                 m->queue_io = 0;
1264
1265         m->pg_init_delay_retry = delay_retry;
1266         queue_work(kmultipathd, &m->process_queued_ios);
1267
1268         /*
1269          * Wake up any thread waiting to suspend.
1270          */
1271         wake_up(&m->pg_init_wait);
1272
1273 out:
1274         spin_unlock_irqrestore(&m->lock, flags);
1275 }
1276
1277 static void activate_path(struct work_struct *work)
1278 {
1279         struct pgpath *pgpath =
1280                 container_of(work, struct pgpath, activate_path.work);
1281
1282         if (pgpath->path.dev)
1283                 scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1284                                  pg_init_done, pgpath);
1285 }
1286
1287 /*
1288  * end_io handling
1289  */
1290 static int do_end_io(struct multipath *m, struct request *clone,
1291                      int error, struct dm_mpath_io *mpio)
1292 {
1293         /*
1294          * We don't queue any clone request inside the multipath target
1295          * during end I/O handling, since those clone requests don't have
1296          * bio clones.  If we queue them inside the multipath target,
1297          * we need to make bio clones, that requires memory allocation.
1298          * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
1299          *  don't have bio clones.)
1300          * Instead of queueing the clone request here, we queue the original
1301          * request into dm core, which will remake a clone request and
1302          * clone bios for it and resubmit it later.
1303          */
1304         int r = DM_ENDIO_REQUEUE;
1305         unsigned long flags;
1306
1307         if (!error && !clone->errors)
1308                 return 0;       /* I/O complete */
1309
1310         if (error == -EOPNOTSUPP || error == -EREMOTEIO || error == -EILSEQ)
1311                 return error;
1312
1313         if (mpio->pgpath)
1314                 fail_path(mpio->pgpath);
1315
1316         spin_lock_irqsave(&m->lock, flags);
1317         if (!m->nr_valid_paths) {
1318                 if (!m->queue_if_no_path) {
1319                         if (!__must_push_back(m))
1320                                 r = -EIO;
1321                 } else {
1322                         if (error == -EBADE)
1323                                 r = error;
1324                 }
1325         }
1326         spin_unlock_irqrestore(&m->lock, flags);
1327
1328         return r;
1329 }
1330
1331 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1332                             int error, union map_info *map_context)
1333 {
1334         struct multipath *m = ti->private;
1335         struct dm_mpath_io *mpio = map_context->ptr;
1336         struct pgpath *pgpath = mpio->pgpath;
1337         struct path_selector *ps;
1338         int r;
1339
1340         BUG_ON(!mpio);
1341
1342         r  = do_end_io(m, clone, error, mpio);
1343         if (pgpath) {
1344                 ps = &pgpath->pg->ps;
1345                 if (ps->type->end_io)
1346                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1347         }
1348         clear_mapinfo(m, map_context);
1349
1350         return r;
1351 }
1352
1353 /*
1354  * Suspend can't complete until all the I/O is processed so if
1355  * the last path fails we must error any remaining I/O.
1356  * Note that if the freeze_bdev fails while suspending, the
1357  * queue_if_no_path state is lost - userspace should reset it.
1358  */
1359 static void multipath_presuspend(struct dm_target *ti)
1360 {
1361         struct multipath *m = (struct multipath *) ti->private;
1362
1363         queue_if_no_path(m, 0, 1);
1364 }
1365
1366 static void multipath_postsuspend(struct dm_target *ti)
1367 {
1368         struct multipath *m = ti->private;
1369
1370         mutex_lock(&m->work_mutex);
1371         flush_multipath_work(m);
1372         mutex_unlock(&m->work_mutex);
1373 }
1374
1375 /*
1376  * Restore the queue_if_no_path setting.
1377  */
1378 static void multipath_resume(struct dm_target *ti)
1379 {
1380         struct multipath *m = (struct multipath *) ti->private;
1381         unsigned long flags;
1382
1383         spin_lock_irqsave(&m->lock, flags);
1384         m->queue_if_no_path = m->saved_queue_if_no_path;
1385         spin_unlock_irqrestore(&m->lock, flags);
1386 }
1387
1388 /*
1389  * Info output has the following format:
1390  * num_multipath_feature_args [multipath_feature_args]*
1391  * num_handler_status_args [handler_status_args]*
1392  * num_groups init_group_number
1393  *            [A|D|E num_ps_status_args [ps_status_args]*
1394  *             num_paths num_selector_args
1395  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1396  *
1397  * Table output has the following format (identical to the constructor string):
1398  * num_feature_args [features_args]*
1399  * num_handler_args hw_handler [hw_handler_args]*
1400  * num_groups init_group_number
1401  *     [priority selector-name num_ps_args [ps_args]*
1402  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1403  */
1404 static int multipath_status(struct dm_target *ti, status_type_t type,
1405                             char *result, unsigned int maxlen)
1406 {
1407         int sz = 0;
1408         unsigned long flags;
1409         struct multipath *m = (struct multipath *) ti->private;
1410         struct priority_group *pg;
1411         struct pgpath *p;
1412         unsigned pg_num;
1413         char state;
1414
1415         spin_lock_irqsave(&m->lock, flags);
1416
1417         /* Features */
1418         if (type == STATUSTYPE_INFO)
1419                 DMEMIT("2 %u %u ", m->queue_size, m->pg_init_count);
1420         else {
1421                 DMEMIT("%u ", m->queue_if_no_path +
1422                               (m->pg_init_retries > 0) * 2 +
1423                               (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1424                               (m->features & FEATURE_NO_PARTITIONS));
1425                 if (m->queue_if_no_path)
1426                         DMEMIT("queue_if_no_path ");
1427                 if (m->pg_init_retries)
1428                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1429                 if (m->features & FEATURE_NO_PARTITIONS)
1430                         DMEMIT("no_partitions ");
1431                 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1432                         DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1433         }
1434
1435         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1436                 DMEMIT("0 ");
1437         else
1438                 DMEMIT("1 %s ", m->hw_handler_name);
1439
1440         DMEMIT("%u ", m->nr_priority_groups);
1441
1442         if (m->next_pg)
1443                 pg_num = m->next_pg->pg_num;
1444         else if (m->current_pg)
1445                 pg_num = m->current_pg->pg_num;
1446         else
1447                 pg_num = (m->nr_priority_groups ? 1 : 0);
1448
1449         DMEMIT("%u ", pg_num);
1450
1451         switch (type) {
1452         case STATUSTYPE_INFO:
1453                 list_for_each_entry(pg, &m->priority_groups, list) {
1454                         if (pg->bypassed)
1455                                 state = 'D';    /* Disabled */
1456                         else if (pg == m->current_pg)
1457                                 state = 'A';    /* Currently Active */
1458                         else
1459                                 state = 'E';    /* Enabled */
1460
1461                         DMEMIT("%c ", state);
1462
1463                         if (pg->ps.type->status)
1464                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1465                                                           result + sz,
1466                                                           maxlen - sz);
1467                         else
1468                                 DMEMIT("0 ");
1469
1470                         DMEMIT("%u %u ", pg->nr_pgpaths,
1471                                pg->ps.type->info_args);
1472
1473                         list_for_each_entry(p, &pg->pgpaths, list) {
1474                                 DMEMIT("%s %s %u ", p->path.pdev,
1475                                        p->is_active ? "A" : "F",
1476                                        p->fail_count);
1477                                 if (pg->ps.type->status)
1478                                         sz += pg->ps.type->status(&pg->ps,
1479                                               &p->path, type, result + sz,
1480                                               maxlen - sz);
1481                         }
1482                 }
1483                 break;
1484
1485         case STATUSTYPE_TABLE:
1486                 list_for_each_entry(pg, &m->priority_groups, list) {
1487                         DMEMIT("%s ", pg->ps.type->name);
1488
1489                         if (pg->ps.type->status)
1490                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1491                                                           result + sz,
1492                                                           maxlen - sz);
1493                         else
1494                                 DMEMIT("0 ");
1495
1496                         DMEMIT("%u %u ", pg->nr_pgpaths,
1497                                pg->ps.type->table_args);
1498
1499                         list_for_each_entry(p, &pg->pgpaths, list) {
1500                                 DMEMIT("%s ", p->path.pdev);
1501                                 if (pg->ps.type->status)
1502                                         sz += pg->ps.type->status(&pg->ps,
1503                                               &p->path, type, result + sz,
1504                                               maxlen - sz);
1505                         }
1506                 }
1507                 break;
1508         }
1509
1510         spin_unlock_irqrestore(&m->lock, flags);
1511
1512         return 0;
1513 }
1514
1515 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1516 {
1517         int r = -EINVAL;
1518         struct dm_dev *dev;
1519         struct multipath *m = (struct multipath *) ti->private;
1520         action_fn action;
1521
1522         mutex_lock(&m->work_mutex);
1523
1524         if (dm_suspended(ti)) {
1525                 r = -EBUSY;
1526                 goto out;
1527         }
1528
1529         if (argc == 1) {
1530                 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1531                         r = queue_if_no_path(m, 1, 0);
1532                         goto out;
1533                 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1534                         r = queue_if_no_path(m, 0, 0);
1535                         goto out;
1536                 }
1537         }
1538
1539         if (argc != 2) {
1540                 DMWARN("Unrecognised multipath message received.");
1541                 goto out;
1542         }
1543
1544         if (!strcasecmp(argv[0], "disable_group")) {
1545                 r = bypass_pg_num(m, argv[1], 1);
1546                 goto out;
1547         } else if (!strcasecmp(argv[0], "enable_group")) {
1548                 r = bypass_pg_num(m, argv[1], 0);
1549                 goto out;
1550         } else if (!strcasecmp(argv[0], "switch_group")) {
1551                 r = switch_pg_num(m, argv[1]);
1552                 goto out;
1553         } else if (!strcasecmp(argv[0], "reinstate_path"))
1554                 action = reinstate_path;
1555         else if (!strcasecmp(argv[0], "fail_path"))
1556                 action = fail_path;
1557         else {
1558                 DMWARN("Unrecognised multipath message received.");
1559                 goto out;
1560         }
1561
1562         r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1563         if (r) {
1564                 DMWARN("message: error getting device %s",
1565                        argv[1]);
1566                 goto out;
1567         }
1568
1569         r = action_dev(m, dev, action);
1570
1571         dm_put_device(ti, dev);
1572
1573 out:
1574         mutex_unlock(&m->work_mutex);
1575         return r;
1576 }
1577
1578 static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
1579                            unsigned long arg)
1580 {
1581         struct multipath *m = (struct multipath *) ti->private;
1582         struct block_device *bdev = NULL;
1583         fmode_t mode = 0;
1584         unsigned long flags;
1585         int r = 0;
1586
1587         spin_lock_irqsave(&m->lock, flags);
1588
1589         if (!m->current_pgpath)
1590                 __choose_pgpath(m, 0);
1591
1592         if (m->current_pgpath && m->current_pgpath->path.dev) {
1593                 bdev = m->current_pgpath->path.dev->bdev;
1594                 mode = m->current_pgpath->path.dev->mode;
1595         }
1596
1597         if (m->queue_io)
1598                 r = -EAGAIN;
1599         else if (!bdev)
1600                 r = -EIO;
1601
1602         spin_unlock_irqrestore(&m->lock, flags);
1603
1604         /*
1605          * Only pass ioctls through if the device sizes match exactly.
1606          */
1607         if (!r && ti->len != i_size_read(bdev->bd_inode) >> SECTOR_SHIFT)
1608                 r = scsi_verify_blk_ioctl(NULL, cmd);
1609
1610         return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
1611 }
1612
1613 static int multipath_iterate_devices(struct dm_target *ti,
1614                                      iterate_devices_callout_fn fn, void *data)
1615 {
1616         struct multipath *m = ti->private;
1617         struct priority_group *pg;
1618         struct pgpath *p;
1619         int ret = 0;
1620
1621         list_for_each_entry(pg, &m->priority_groups, list) {
1622                 list_for_each_entry(p, &pg->pgpaths, list) {
1623                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1624                         if (ret)
1625                                 goto out;
1626                 }
1627         }
1628
1629 out:
1630         return ret;
1631 }
1632
1633 static int __pgpath_busy(struct pgpath *pgpath)
1634 {
1635         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1636
1637         return dm_underlying_device_busy(q);
1638 }
1639
1640 /*
1641  * We return "busy", only when we can map I/Os but underlying devices
1642  * are busy (so even if we map I/Os now, the I/Os will wait on
1643  * the underlying queue).
1644  * In other words, if we want to kill I/Os or queue them inside us
1645  * due to map unavailability, we don't return "busy".  Otherwise,
1646  * dm core won't give us the I/Os and we can't do what we want.
1647  */
1648 static int multipath_busy(struct dm_target *ti)
1649 {
1650         int busy = 0, has_active = 0;
1651         struct multipath *m = ti->private;
1652         struct priority_group *pg;
1653         struct pgpath *pgpath;
1654         unsigned long flags;
1655
1656         spin_lock_irqsave(&m->lock, flags);
1657
1658         /* Guess which priority_group will be used at next mapping time */
1659         if (unlikely(!m->current_pgpath && m->next_pg))
1660                 pg = m->next_pg;
1661         else if (likely(m->current_pg))
1662                 pg = m->current_pg;
1663         else
1664                 /*
1665                  * We don't know which pg will be used at next mapping time.
1666                  * We don't call __choose_pgpath() here to avoid to trigger
1667                  * pg_init just by busy checking.
1668                  * So we don't know whether underlying devices we will be using
1669                  * at next mapping time are busy or not. Just try mapping.
1670                  */
1671                 goto out;
1672
1673         /*
1674          * If there is one non-busy active path at least, the path selector
1675          * will be able to select it. So we consider such a pg as not busy.
1676          */
1677         busy = 1;
1678         list_for_each_entry(pgpath, &pg->pgpaths, list)
1679                 if (pgpath->is_active) {
1680                         has_active = 1;
1681
1682                         if (!__pgpath_busy(pgpath)) {
1683                                 busy = 0;
1684                                 break;
1685                         }
1686                 }
1687
1688         if (!has_active)
1689                 /*
1690                  * No active path in this pg, so this pg won't be used and
1691                  * the current_pg will be changed at next mapping time.
1692                  * We need to try mapping to determine it.
1693                  */
1694                 busy = 0;
1695
1696 out:
1697         spin_unlock_irqrestore(&m->lock, flags);
1698
1699         return busy;
1700 }
1701
1702 /*-----------------------------------------------------------------
1703  * Module setup
1704  *---------------------------------------------------------------*/
1705 static struct target_type multipath_target = {
1706         .name = "multipath",
1707         .version = {1, 3, 0},
1708         .module = THIS_MODULE,
1709         .ctr = multipath_ctr,
1710         .dtr = multipath_dtr,
1711         .map_rq = multipath_map,
1712         .rq_end_io = multipath_end_io,
1713         .presuspend = multipath_presuspend,
1714         .postsuspend = multipath_postsuspend,
1715         .resume = multipath_resume,
1716         .status = multipath_status,
1717         .message = multipath_message,
1718         .ioctl  = multipath_ioctl,
1719         .iterate_devices = multipath_iterate_devices,
1720         .busy = multipath_busy,
1721 };
1722
1723 static int __init dm_multipath_init(void)
1724 {
1725         int r;
1726
1727         /* allocate a slab for the dm_ios */
1728         _mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
1729         if (!_mpio_cache)
1730                 return -ENOMEM;
1731
1732         r = dm_register_target(&multipath_target);
1733         if (r < 0) {
1734                 DMERR("register failed %d", r);
1735                 kmem_cache_destroy(_mpio_cache);
1736                 return -EINVAL;
1737         }
1738
1739         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
1740         if (!kmultipathd) {
1741                 DMERR("failed to create workqueue kmpathd");
1742                 dm_unregister_target(&multipath_target);
1743                 kmem_cache_destroy(_mpio_cache);
1744                 return -ENOMEM;
1745         }
1746
1747         /*
1748          * A separate workqueue is used to handle the device handlers
1749          * to avoid overloading existing workqueue. Overloading the
1750          * old workqueue would also create a bottleneck in the
1751          * path of the storage hardware device activation.
1752          */
1753         kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
1754                                                   WQ_MEM_RECLAIM);
1755         if (!kmpath_handlerd) {
1756                 DMERR("failed to create workqueue kmpath_handlerd");
1757                 destroy_workqueue(kmultipathd);
1758                 dm_unregister_target(&multipath_target);
1759                 kmem_cache_destroy(_mpio_cache);
1760                 return -ENOMEM;
1761         }
1762
1763         DMINFO("version %u.%u.%u loaded",
1764                multipath_target.version[0], multipath_target.version[1],
1765                multipath_target.version[2]);
1766
1767         return r;
1768 }
1769
1770 static void __exit dm_multipath_exit(void)
1771 {
1772         destroy_workqueue(kmpath_handlerd);
1773         destroy_workqueue(kmultipathd);
1774
1775         dm_unregister_target(&multipath_target);
1776         kmem_cache_destroy(_mpio_cache);
1777 }
1778
1779 module_init(dm_multipath_init);
1780 module_exit(dm_multipath_exit);
1781
1782 MODULE_DESCRIPTION(DM_NAME " multipath target");
1783 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
1784 MODULE_LICENSE("GPL");