drivers/xen/core/evtchn.c

   1 /******************************************************************************
   2  * evtchn.c
   3  *
   4  * Communication via Xen event channels.
   5  *
   6  * Copyright (c) 2002-2005, K A Fraser
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License version 2
  10  * as published by the Free Software Foundation; or, when distributed
  11  * separately from the Linux kernel or incorporated into other
  12  * software packages, subject to the following license:
  13  *
  14  * Permission is hereby granted, free of charge, to any person obtaining a copy
  15  * of this source file (the "Software"), to deal in the Software without
  16  * restriction, including without limitation the rights to use, copy, modify,
  17  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  18  * and to permit persons to whom the Software is furnished to do so, subject to
  19  * the following conditions:
  20  *
  21  * The above copyright notice and this permission notice shall be included in
  22  * all copies or substantial portions of the Software.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  25  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  26  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  27  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  28  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  29  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  30  * IN THE SOFTWARE.
  31  */
  32
  33 #include <linux/module.h>
  34 #include <linux/slab.h>
  35 #include <linux/irq.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/sched.h>
  38 #include <linux/kernel_stat.h>
  39 #include <linux/ftrace.h>
  40 #include <linux/atomic.h>
  41 #include <asm/barrier.h>
  42 #include <asm/ptrace.h>
  43 #include <xen/evtchn.h>
  44 #include <xen/interface/event_channel.h>
  45 #include <xen/interface/physdev.h>
  46 #include <asm/hypervisor.h>
  47 #include <linux/mc146818rtc.h> /* RTC_IRQ */
  48 #include "../../../kernel/irq/internals.h" /* IRQS_AUTODETECT, IRQS_PENDING */
  49
  50 /*
  51  * This lock protects updates to the following mapping and reference-count
  52  * arrays. The lock does not need to be acquired to read the mapping tables.
  53  */
  54 static DEFINE_SPINLOCK(irq_mapping_update_lock);
  55
  56 /* IRQ <-> event-channel mappings. */
  57 static int evtchn_to_irq[NR_EVENT_CHANNELS] = {
  58         [0 ...  NR_EVENT_CHANNELS-1] = -1 };
  59
  60 #if defined(CONFIG_SMP) && defined(CONFIG_X86)
  61 static struct percpu_irqaction {
  62         struct irqaction action; /* must be first */
  63         struct percpu_irqaction *next;
  64         cpumask_var_t cpus;
  65 } *virq_actions[NR_VIRQS];
  66 /* IRQ <-> VIRQ mapping. */
  67 static DECLARE_BITMAP(virq_per_cpu, NR_VIRQS) __read_mostly;
  68 static DEFINE_PER_CPU_READ_MOSTLY(int[NR_VIRQS], virq_to_evtchn);
  69 #define BUG_IF_VIRQ_PER_CPU(irq_cfg) \
  70         BUG_ON(type_from_irq_cfg(irq_cfg) == IRQT_VIRQ \
  71                && test_bit(index_from_irq_cfg(irq_cfg), virq_per_cpu))
  72 #else
  73 #define BUG_IF_VIRQ_PER_CPU(irq_cfg) ((void)0)
  74 #define PER_CPU_VIRQ_IRQ
  75 #endif
  76
  77 /* IRQ <-> IPI mapping. */
  78 #if defined(CONFIG_SMP) && defined(CONFIG_X86)
  79 static int __read_mostly ipi_irq = -1;
  80 DEFINE_PER_CPU(DECLARE_BITMAP(, NR_IPIS), ipi_pending);
  81 static DEFINE_PER_CPU_READ_MOSTLY(evtchn_port_t, ipi_evtchn);
  82 #else
  83 #define PER_CPU_IPI_IRQ
  84 #endif
  85 #if !defined(CONFIG_SMP) || !defined(PER_CPU_IPI_IRQ)
  86 #define BUG_IF_IPI(irq_cfg) BUG_ON(type_from_irq_cfg(irq_cfg) == IRQT_IPI)
  87 #else
  88 #define BUG_IF_IPI(irq_cfg) ((void)0)
  89 #endif
  90
  91 /* Binding types. */
  92 enum {
  93         IRQT_UNBOUND,
  94         IRQT_PIRQ,
  95         IRQT_VIRQ,
  96         IRQT_IPI,
  97         IRQT_LOCAL_PORT,
  98         IRQT_CALLER_PORT,
  99         _IRQT_COUNT
 100 };
 101
 102 #define _IRQT_BITS 4
 103 #define _EVTCHN_BITS 12
 104 #define _INDEX_BITS (32 - _IRQT_BITS - _EVTCHN_BITS)
 105
 106 /* Convenient shorthand for packed representation of an unbound IRQ. */
 107 #define IRQ_UNBOUND     (IRQT_UNBOUND << (32 - _IRQT_BITS))
 108
 109 static struct irq_cfg _irq_cfg[] = {
 110         [0 ...
 111 #ifdef CONFIG_SPARSE_IRQ
 112                BUILD_BUG_ON_ZERO(PIRQ_BASE) + NR_IRQS_LEGACY
 113 #else
 114                NR_IRQS
 115 #endif
 116                        - 1].info = IRQ_UNBOUND
 117 };
 118
 119 static inline struct irq_cfg *__pure irq_cfg(unsigned int irq)
 120 {
 121 #ifdef CONFIG_SPARSE_IRQ
 122         return irq_get_chip_data(irq);
 123 #else
 124         return irq < NR_IRQS ? _irq_cfg + irq : NULL;
 125 #endif
 126 }
 127
 128 static inline struct irq_cfg *__pure irq_data_cfg(struct irq_data *data)
 129 {
 130         return irq_data_get_irq_chip_data(data);
 131 }
 132
 133 /* Constructor for packed IRQ information. */
 134 static inline u32 mk_irq_info(u32 type, u32 index, u32 evtchn)
 135 {
 136         BUILD_BUG_ON(_IRQT_COUNT > (1U << _IRQT_BITS));
 137
 138         BUILD_BUG_ON(NR_PIRQS > (1U << _INDEX_BITS));
 139         BUILD_BUG_ON(NR_VIRQS > (1U << _INDEX_BITS));
 140 #if defined(PER_CPU_IPI_IRQ) && defined(NR_IPIS)
 141         BUILD_BUG_ON(NR_IPIS > (1U << _INDEX_BITS));
 142 #endif
 143         BUG_ON(index >> _INDEX_BITS);
 144
 145         BUILD_BUG_ON(NR_EVENT_CHANNELS > (1U << _EVTCHN_BITS));
 146
 147         return ((type << (32 - _IRQT_BITS)) | (index << _EVTCHN_BITS) | evtchn);
 148 }
 149
 150 /*
 151  * Accessors for packed IRQ information.
 152  */
 153
 154 static inline unsigned int index_from_irq_cfg(const struct irq_cfg *cfg)
 155 {
 156         return (cfg->info >> _EVTCHN_BITS) & ((1U << _INDEX_BITS) - 1);
 157 }
 158
 159 static inline unsigned int index_from_irq(int irq)
 160 {
 161         const struct irq_cfg *cfg = irq_cfg(irq);
 162
 163         return cfg ? index_from_irq_cfg(cfg) : 0;
 164 }
 165
 166 static inline unsigned int type_from_irq_cfg(const struct irq_cfg *cfg)
 167 {
 168         return cfg->info >> (32 - _IRQT_BITS);
 169 }
 170
 171 static inline unsigned int type_from_irq(int irq)
 172 {
 173         const struct irq_cfg *cfg = irq_cfg(irq);
 174
 175         return cfg ? type_from_irq_cfg(cfg) : IRQT_UNBOUND;
 176 }
 177
 178 static inline unsigned int evtchn_from_per_cpu_irq(const struct irq_cfg *cfg,
 179                                                    unsigned int cpu)
 180 {
 181         switch (type_from_irq_cfg(cfg)) {
 182 #ifndef PER_CPU_VIRQ_IRQ
 183         case IRQT_VIRQ:
 184                 return per_cpu(virq_to_evtchn, cpu)[index_from_irq_cfg(cfg)];
 185 #endif
 186 #ifndef PER_CPU_IPI_IRQ
 187         case IRQT_IPI:
 188                 return per_cpu(ipi_evtchn, cpu);
 189 #endif
 190         }
 191         BUG();
 192         return 0;
 193 }
 194
 195 static inline unsigned int evtchn_from_irq_cfg(const struct irq_cfg *cfg)
 196 {
 197         switch (type_from_irq_cfg(cfg)) {
 198 #ifndef PER_CPU_VIRQ_IRQ
 199         case IRQT_VIRQ:
 200 #endif
 201 #ifndef PER_CPU_IPI_IRQ
 202         case IRQT_IPI:
 203 #endif
 204                 return evtchn_from_per_cpu_irq(cfg, smp_processor_id());
 205         }
 206         return cfg->info & ((1U << _EVTCHN_BITS) - 1);
 207 }
 208
 209 static inline unsigned int evtchn_from_irq_data(struct irq_data *data)
 210 {
 211         const struct irq_cfg *cfg = irq_data_cfg(data);
 212
 213         return cfg ? evtchn_from_irq_cfg(cfg) : 0;
 214 }
 215
 216 static inline unsigned int evtchn_from_irq(int irq)
 217 {
 218         struct irq_data *data = irq_get_irq_data(irq);
 219
 220         return data ? evtchn_from_irq_data(data) : 0;
 221 }
 222
 223 unsigned int irq_from_evtchn(unsigned int port)
 224 {
 225         return evtchn_to_irq[port];
 226 }
 227 EXPORT_SYMBOL_GPL(irq_from_evtchn);
 228
 229 /* IRQ <-> VIRQ mapping. */
 230 DEFINE_PER_CPU(int[NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
 231
 232 #if defined(CONFIG_SMP) && defined(PER_CPU_IPI_IRQ)
 233 /* IRQ <-> IPI mapping. */
 234 #ifndef NR_IPIS
 235 #define NR_IPIS 1
 236 #endif
 237 DEFINE_PER_CPU(int[NR_IPIS], ipi_to_irq) = {[0 ... NR_IPIS-1] = -1};
 238 #endif
 239
 240 #ifdef CONFIG_SMP
 241
 242 #if CONFIG_NR_CPUS <= 256
 243 static u8 cpu_evtchn[NR_EVENT_CHANNELS];
 244 #else
 245 static u16 cpu_evtchn[NR_EVENT_CHANNELS];
 246 #endif
 247 static DEFINE_PER_CPU(unsigned long[BITS_TO_LONGS(NR_EVENT_CHANNELS)],
 248                       cpu_evtchn_mask);
 249
 250 static inline unsigned long active_evtchns(unsigned int idx)
 251 {
 252         shared_info_t *sh = HYPERVISOR_shared_info;
 253
 254         return (sh->evtchn_pending[idx] &
 255                 percpu_read(cpu_evtchn_mask[idx]) &
 256                 ~sh->evtchn_mask[idx]);
 257 }
 258
 259 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
 260 {
 261         shared_info_t *s = HYPERVISOR_shared_info;
 262         int irq = evtchn_to_irq[chn];
 263
 264         BUG_ON(!test_bit(chn, s->evtchn_mask));
 265
 266         if (irq != -1) {
 267                 struct irq_data *data = irq_get_irq_data(irq);
 268
 269                 if (!irqd_is_per_cpu(data))
 270                         cpumask_copy(data->affinity, cpumask_of(cpu));
 271                 else
 272                         cpumask_set_cpu(cpu, data->affinity);
 273         }
 274
 275         clear_bit(chn, per_cpu(cpu_evtchn_mask, cpu_evtchn[chn]));
 276         set_bit(chn, per_cpu(cpu_evtchn_mask, cpu));
 277         cpu_evtchn[chn] = cpu;
 278 }
 279
 280 static void init_evtchn_cpu_bindings(void)
 281 {
 282         int i;
 283
 284         /* By default all event channels notify CPU#0. */
 285         for (i = 0; i < nr_irqs; i++) {
 286                 struct irq_data *data = irq_get_irq_data(i);
 287
 288                 if (data)
 289                         cpumask_copy(data->affinity, cpumask_of(0));
 290         }
 291
 292         memset(cpu_evtchn, 0, sizeof(cpu_evtchn));
 293         for_each_possible_cpu(i)
 294                 memset(per_cpu(cpu_evtchn_mask, i), -!i,
 295                        sizeof(per_cpu(cpu_evtchn_mask, i)));
 296 }
 297
 298 static inline unsigned int cpu_from_evtchn(unsigned int evtchn)
 299 {
 300         return cpu_evtchn[evtchn];
 301 }
 302
 303 #else
 304
 305 static inline unsigned long active_evtchns(unsigned int idx)
 306 {
 307         shared_info_t *sh = HYPERVISOR_shared_info;
 308
 309         return (sh->evtchn_pending[idx] & ~sh->evtchn_mask[idx]);
 310 }
 311
 312 static void bind_evtchn_to_cpu(unsigned int chn, unsigned int cpu)
 313 {
 314 }
 315
 316 static void init_evtchn_cpu_bindings(void)
 317 {
 318 }
 319
 320 static inline unsigned int cpu_from_evtchn(unsigned int evtchn)
 321 {
 322         return 0;
 323 }
 324
 325 #endif
 326
 327 #ifdef CONFIG_X86
 328 void __init xen_init_IRQ(void);
 329 void __init init_IRQ(void)
 330 {
 331         irq_ctx_init(0);
 332         xen_init_IRQ();
 333 }
 334 #include <asm/idle.h>
 335 #endif
 336
 337 /* Xen will never allocate port zero for any purpose. */
 338 #define VALID_EVTCHN(chn)       ((chn) != 0)
 339
 340 /*
 341  * Force a proper event-channel callback from Xen after clearing the
 342  * callback mask. We do this in a very simple manner, by making a call
 343  * down into Xen. The pending flag will be checked by Xen on return.
 344  */
 345 void force_evtchn_callback(void)
 346 {
 347         VOID(HYPERVISOR_xen_version(0, NULL));
 348 }
 349 /* Not a GPL symbol: used in ubiquitous macros, so too restrictive. */
 350 EXPORT_SYMBOL(force_evtchn_callback);
 351
 352 #define UPC_INACTIVE 0
 353 #define UPC_ACTIVE 1
 354 #define UPC_NESTED_LATCH 2
 355 #define UPC_RESTART (UPC_ACTIVE|UPC_NESTED_LATCH)
 356 static DEFINE_PER_CPU(unsigned int, upcall_state);
 357 static DEFINE_PER_CPU(unsigned int, current_l1i);
 358 static DEFINE_PER_CPU(unsigned int, current_l2i);
 359
 360 #ifndef vcpu_info_xchg
 361 #define vcpu_info_xchg(fld, val) xchg(&current_vcpu_info()->fld, val)
 362 #endif
 363
 364 /* NB. Interrupts are disabled on entry. */
 365 asmlinkage void __irq_entry evtchn_do_upcall(struct pt_regs *regs)
 366 {
 367         unsigned long       l1, l2;
 368         unsigned long       masked_l1, masked_l2;
 369         unsigned int        l1i, l2i, start_l1i, start_l2i, port, i;
 370         int                 irq;
 371         struct pt_regs     *old_regs;
 372
 373         /* Nested invocations bail immediately. */
 374         if (unlikely(__this_cpu_cmpxchg(upcall_state, UPC_INACTIVE,
 375                                         UPC_ACTIVE) != UPC_INACTIVE)) {
 376                 __this_cpu_or(upcall_state, UPC_NESTED_LATCH);
 377                 /* Avoid a callback storm when we reenable delivery. */
 378                 vcpu_info_write(evtchn_upcall_pending, 0);
 379                 return;
 380         }
 381
 382         old_regs = set_irq_regs(regs);
 383         xen_spin_irq_enter();
 384         irq_enter();
 385         exit_idle();
 386
 387         do {
 388                 vcpu_info_write(evtchn_upcall_pending, 0);
 389
 390 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
 391                 /* Clear master flag /before/ clearing selector flag. */
 392                 wmb();
 393 #else
 394                 barrier();
 395 #endif
 396
 397 #ifndef CONFIG_NO_HZ
 398                 /*
 399                  * Handle timer interrupts before all others, so that all
 400                  * hardirq handlers see an up-to-date system time even if we
 401                  * have just woken from a long idle period.
 402                  */
 403 #ifdef PER_CPU_VIRQ_IRQ
 404                 if ((irq = percpu_read(virq_to_irq[VIRQ_TIMER])) != -1) {
 405                         port = evtchn_from_irq(irq);
 406 #else
 407                 port = __this_cpu_read(virq_to_evtchn[VIRQ_TIMER]);
 408                 if (VALID_EVTCHN(port)) {
 409 #endif
 410                         l1i = port / BITS_PER_LONG;
 411                         l2i = port % BITS_PER_LONG;
 412                         if (active_evtchns(l1i) & (1ul<<l2i)) {
 413                                 mask_evtchn(port);
 414                                 clear_evtchn(port);
 415 #ifndef PER_CPU_VIRQ_IRQ
 416                                 irq = evtchn_to_irq[port];
 417                                 BUG_ON(irq == -1);
 418 #endif
 419                                 if (!handle_irq(irq, regs))
 420                                         BUG();
 421                         }
 422                 }
 423 #endif /* CONFIG_NO_HZ */
 424
 425                 l1 = vcpu_info_xchg(evtchn_pending_sel, 0);
 426
 427                 start_l1i = l1i = percpu_read(current_l1i);
 428                 start_l2i = percpu_read(current_l2i);
 429
 430                 for (i = 0; l1 != 0; i++) {
 431                         masked_l1 = l1 & ((~0UL) << l1i);
 432                         /* If we masked out all events, wrap to beginning. */
 433                         if (masked_l1 == 0) {
 434                                 l1i = l2i = 0;
 435                                 continue;
 436                         }
 437                         l1i = __ffs(masked_l1);
 438
 439                         l2 = active_evtchns(l1i);
 440                         l2i = 0; /* usually scan entire word from start */
 441                         if (l1i == start_l1i) {
 442                                 /* We scan the starting word in two parts. */
 443                                 if (i == 0)
 444                                         /* 1st time: start in the middle */
 445                                         l2i = start_l2i;
 446                                 else
 447                                         /* 2nd time: mask bits done already */
 448                                         l2 &= (1ul << start_l2i) - 1;
 449                         }
 450
 451                         do {
 452                                 bool handled = false;
 453
 454                                 masked_l2 = l2 & ((~0UL) << l2i);
 455                                 if (masked_l2 == 0)
 456                                         break;
 457                                 l2i = __ffs(masked_l2);
 458
 459                                 /* process port */
 460                                 port = (l1i * BITS_PER_LONG) + l2i;
 461                                 mask_evtchn(port);
 462                                 if ((irq = evtchn_to_irq[port]) != -1) {
 463 #ifndef PER_CPU_IPI_IRQ
 464                                         if (port != __this_cpu_read(ipi_evtchn))
 465 #endif
 466                                                 clear_evtchn(port);
 467                                         handled = handle_irq(irq, regs);
 468                                 }
 469                                 if (!handled && printk_ratelimit())
 470                                         pr_emerg("No handler for irq %d"
 471                                                  " (port %u)\n",
 472                                                  irq, port);
 473
 474                                 l2i = (l2i + 1) % BITS_PER_LONG;
 475
 476                                 /* Next caller starts at last processed + 1 */
 477                                 percpu_write(current_l1i,
 478                                         l2i ? l1i : (l1i + 1) % BITS_PER_LONG);
 479                                 percpu_write(current_l2i, l2i);
 480
 481                         } while (l2i != 0);
 482
 483                         /* Scan start_l1i twice; all others once. */
 484                         if ((l1i != start_l1i) || (i != 0))
 485                                 l1 &= ~(1UL << l1i);
 486
 487                         l1i = (l1i + 1) % BITS_PER_LONG;
 488                 }
 489
 490                 /* If there were nested callbacks then we have more to do. */
 491         } while (unlikely(__this_cpu_cmpxchg(upcall_state, UPC_RESTART,
 492                                              UPC_ACTIVE) == UPC_RESTART));
 493
 494         __this_cpu_write(upcall_state, UPC_INACTIVE);
 495         irq_exit();
 496         xen_spin_irq_exit();
 497         set_irq_regs(old_regs);
 498 }
 499
 500 static int find_unbound_irq(unsigned int node, struct irq_cfg **pcfg,
 501                             struct irq_chip *chip, bool percpu)
 502 {
 503         static int warned;
 504         int irq;
 505
 506         for (irq = DYNIRQ_BASE; irq < nr_irqs; irq++) {
 507                 struct irq_cfg *cfg = alloc_irq_and_cfg_at(irq, node);
 508                 struct irq_data *data = irq_get_irq_data(irq);
 509
 510                 if (unlikely(!cfg))
 511                         return -ENOMEM;
 512                 if (data->chip != &no_irq_chip &&
 513                     data->chip != chip)
 514                         continue;
 515
 516                 if (!cfg->bindcount) {
 517                         irq_flow_handler_t handle;
 518                         const char *name;
 519
 520                         *pcfg = cfg;
 521                         irq_set_noprobe(irq);
 522                         if (!percpu) {
 523                                 handle = handle_fasteoi_irq;
 524                                 name = "fasteoi";
 525                         } else {
 526                                 handle = handle_percpu_irq;
 527                                 name = "percpu";
 528                         }
 529                         irq_set_chip_and_handler_name(irq, chip,
 530                                                       handle, name);
 531                         return irq;
 532                 }
 533         }
 534
 535         if (!warned) {
 536                 warned = 1;
 537                 pr_warning("No available IRQ to bind to: "
 538                            "increase NR_DYNIRQS.\n");
 539         }
 540
 541         return -ENOSPC;
 542 }
 543
 544 static struct irq_chip dynirq_chip;
 545
 546 static int bind_caller_port_to_irq(unsigned int caller_port)
 547 {
 548         struct irq_cfg *cfg;
 549         int irq;
 550
 551         spin_lock(&irq_mapping_update_lock);
 552
 553         if ((irq = evtchn_to_irq[caller_port]) == -1) {
 554                 if ((irq = find_unbound_irq(numa_node_id(), &cfg,
 555                                             &dynirq_chip, false)) < 0)
 556                         goto out;
 557
 558                 evtchn_to_irq[caller_port] = irq;
 559                 cfg->info = mk_irq_info(IRQT_CALLER_PORT, 0, caller_port);
 560         } else
 561                 cfg = irq_cfg(irq);
 562
 563         cfg->bindcount++;
 564
 565  out:
 566         spin_unlock(&irq_mapping_update_lock);
 567         return irq;
 568 }
 569
 570 static int bind_local_port_to_irq(unsigned int local_port)
 571 {
 572         struct irq_cfg *cfg;
 573         int irq;
 574
 575         spin_lock(&irq_mapping_update_lock);
 576
 577         BUG_ON(evtchn_to_irq[local_port] != -1);
 578
 579         if ((irq = find_unbound_irq(numa_node_id(), &cfg, &dynirq_chip,
 580                                     false)) < 0) {
 581                 if (close_evtchn(local_port))
 582                         BUG();
 583                 goto out;
 584         }
 585
 586         evtchn_to_irq[local_port] = irq;
 587         cfg->info = mk_irq_info(IRQT_LOCAL_PORT, 0, local_port);
 588         cfg->bindcount++;
 589
 590  out:
 591         spin_unlock(&irq_mapping_update_lock);
 592         return irq;
 593 }
 594
 595 static int bind_listening_port_to_irq(unsigned int remote_domain)
 596 {
 597         struct evtchn_alloc_unbound alloc_unbound;
 598         int err;
 599
 600         alloc_unbound.dom        = DOMID_SELF;
 601         alloc_unbound.remote_dom = remote_domain;
 602
 603         err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
 604                                           &alloc_unbound);
 605
 606         return err ? : bind_local_port_to_irq(alloc_unbound.port);
 607 }
 608
 609 static int bind_interdomain_evtchn_to_irq(unsigned int remote_domain,
 610                                           unsigned int remote_port)
 611 {
 612         struct evtchn_bind_interdomain bind_interdomain;
 613         int err;
 614
 615         bind_interdomain.remote_dom  = remote_domain;
 616         bind_interdomain.remote_port = remote_port;
 617
 618         err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
 619                                           &bind_interdomain);
 620
 621         return err ? : bind_local_port_to_irq(bind_interdomain.local_port);
 622 }
 623
 624 static int bind_virq_to_irq(unsigned int virq, unsigned int cpu)
 625 {
 626         struct evtchn_bind_virq bind_virq;
 627         struct irq_cfg *cfg;
 628         int evtchn, irq;
 629
 630         spin_lock(&irq_mapping_update_lock);
 631
 632         if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) {
 633                 if ((irq = find_unbound_irq(cpu_to_node(cpu), &cfg,
 634                                             &dynirq_chip, false)) < 0)
 635                         goto out;
 636
 637                 bind_virq.virq = virq;
 638                 bind_virq.vcpu = cpu;
 639                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
 640                                                 &bind_virq) != 0)
 641                         BUG();
 642                 evtchn = bind_virq.port;
 643
 644                 evtchn_to_irq[evtchn] = irq;
 645 #ifndef PER_CPU_VIRQ_IRQ
 646                 {
 647                         unsigned int cpu;
 648
 649                         for_each_possible_cpu(cpu)
 650                                 per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
 651                 }
 652 #endif
 653                 cfg->info = mk_irq_info(IRQT_VIRQ, virq, evtchn);
 654
 655                 per_cpu(virq_to_irq, cpu)[virq] = irq;
 656
 657                 bind_evtchn_to_cpu(evtchn, cpu);
 658         } else
 659                 cfg = irq_cfg(irq);
 660
 661         cfg->bindcount++;
 662
 663  out:
 664         spin_unlock(&irq_mapping_update_lock);
 665         return irq;
 666 }
 667
 668 #if defined(CONFIG_SMP) && defined(PER_CPU_IPI_IRQ)
 669 static int bind_ipi_to_irq(unsigned int ipi, unsigned int cpu)
 670 {
 671         struct evtchn_bind_ipi bind_ipi;
 672         struct irq_cfg *cfg;
 673         int evtchn, irq;
 674
 675         spin_lock(&irq_mapping_update_lock);
 676
 677         if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1) {
 678                 if ((irq = find_unbound_irq(cpu_to_node(cpu), &cfg,
 679                                             &dynirq_chip, false)) < 0)
 680                         goto out;
 681
 682                 bind_ipi.vcpu = cpu;
 683                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
 684                                                 &bind_ipi) != 0)
 685                         BUG();
 686                 evtchn = bind_ipi.port;
 687
 688                 evtchn_to_irq[evtchn] = irq;
 689                 cfg->info = mk_irq_info(IRQT_IPI, ipi, evtchn);
 690
 691                 per_cpu(ipi_to_irq, cpu)[ipi] = irq;
 692
 693                 bind_evtchn_to_cpu(evtchn, cpu);
 694         } else
 695                 cfg = irq_cfg(irq);
 696
 697         cfg->bindcount++;
 698
 699  out:
 700         spin_unlock(&irq_mapping_update_lock);
 701         return irq;
 702 }
 703 #endif
 704
 705 static void unbind_from_irq(unsigned int irq)
 706 {
 707         struct irq_cfg *cfg = irq_cfg(irq);
 708         unsigned int evtchn = evtchn_from_irq_cfg(cfg);
 709
 710         BUG_IF_VIRQ_PER_CPU(cfg);
 711         BUG_IF_IPI(cfg);
 712
 713         spin_lock(&irq_mapping_update_lock);
 714
 715         if (!--cfg->bindcount && VALID_EVTCHN(evtchn)) {
 716                 if ((type_from_irq_cfg(cfg) != IRQT_CALLER_PORT) &&
 717                     close_evtchn(evtchn))
 718                         BUG();
 719
 720                 switch (type_from_irq_cfg(cfg)) {
 721                 case IRQT_VIRQ:
 722                         per_cpu(virq_to_irq, cpu_from_evtchn(evtchn))
 723                                 [index_from_irq_cfg(cfg)] = -1;
 724 #ifndef PER_CPU_VIRQ_IRQ
 725                         {
 726                                 unsigned int cpu;
 727
 728                                 for_each_possible_cpu(cpu)
 729                                         per_cpu(virq_to_evtchn, cpu)
 730                                                 [index_from_irq_cfg(cfg)] = 0;
 731                         }
 732 #endif
 733                         break;
 734 #if defined(CONFIG_SMP) && defined(PER_CPU_IPI_IRQ)
 735                 case IRQT_IPI:
 736                         per_cpu(ipi_to_irq, cpu_from_evtchn(evtchn))
 737                                 [index_from_irq_cfg(cfg)] = -1;
 738                         break;
 739 #endif
 740                 default:
 741                         break;
 742                 }
 743
 744                 /* Closed ports are implicitly re-bound to VCPU0. */
 745                 bind_evtchn_to_cpu(evtchn, 0);
 746
 747                 evtchn_to_irq[evtchn] = -1;
 748                 cfg->info = IRQ_UNBOUND;
 749
 750                 dynamic_irq_cleanup(irq);
 751         }
 752
 753         spin_unlock(&irq_mapping_update_lock);
 754 }
 755
 756 #if !defined(PER_CPU_IPI_IRQ) || !defined(PER_CPU_VIRQ_IRQ)
 757 static inline struct percpu_irqaction *alloc_percpu_irqaction(gfp_t gfp)
 758 {
 759         struct percpu_irqaction *new = kzalloc(sizeof(*new), GFP_ATOMIC);
 760
 761         if (new && !zalloc_cpumask_var(&new->cpus, gfp)) {
 762                 kfree(new);
 763                 new = NULL;
 764         }
 765         return new;
 766 }
 767
 768 static inline void free_percpu_irqaction(struct percpu_irqaction *action)
 769 {
 770         if (!action)
 771                 return;
 772         free_cpumask_var(action->cpus);
 773         kfree(action);
 774 }
 775
 776 void unbind_from_per_cpu_irq(unsigned int irq, unsigned int cpu,
 777                              struct irqaction *action)
 778 {
 779         struct evtchn_close close;
 780         struct irq_data *data = irq_get_irq_data(irq);
 781         struct irq_cfg *cfg = irq_data_cfg(data);
 782         unsigned int evtchn = evtchn_from_per_cpu_irq(cfg, cpu);
 783         struct percpu_irqaction *free_action = NULL;
 784
 785         spin_lock(&irq_mapping_update_lock);
 786
 787         if (VALID_EVTCHN(evtchn)) {
 788                 mask_evtchn(evtchn);
 789
 790                 BUG_ON(cfg->bindcount <= 1);
 791                 cfg->bindcount--;
 792
 793 #ifndef PER_CPU_VIRQ_IRQ
 794                 if (type_from_irq_cfg(cfg) == IRQT_VIRQ) {
 795                         unsigned int virq = index_from_irq_cfg(cfg);
 796                         struct percpu_irqaction *cur, *prev = NULL;
 797
 798                         cur = virq_actions[virq];
 799                         while (cur) {
 800                                 if (cur->action.dev_id == action) {
 801                                         cpumask_clear_cpu(cpu, cur->cpus);
 802                                         if (cpumask_empty(cur->cpus)) {
 803                                                 WARN_ON(free_action);
 804                                                 if (prev)
 805                                                         prev->next = cur->next;
 806                                                 else
 807                                                         virq_actions[virq]
 808                                                                 = cur->next;
 809                                                 free_action = cur;
 810                                         }
 811                                 } else if (cpumask_test_cpu(cpu, cur->cpus))
 812                                         evtchn = 0;
 813                                 cur = (prev = cur)->next;
 814                         }
 815                         if (!VALID_EVTCHN(evtchn))
 816                                 goto done;
 817                 }
 818 #endif
 819
 820                 cpumask_clear_cpu(cpu, data->affinity);
 821
 822                 close.port = evtchn;
 823                 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
 824                         BUG();
 825
 826                 switch (type_from_irq_cfg(cfg)) {
 827 #ifndef PER_CPU_VIRQ_IRQ
 828                 case IRQT_VIRQ:
 829                         per_cpu(virq_to_evtchn, cpu)
 830                                 [index_from_irq_cfg(cfg)] = 0;
 831                         break;
 832 #endif
 833 #ifndef PER_CPU_IPI_IRQ
 834                 case IRQT_IPI:
 835                         per_cpu(ipi_evtchn, cpu) = 0;
 836                         break;
 837 #endif
 838                 default:
 839                         BUG();
 840                         break;
 841                 }
 842
 843                 /* Closed ports are implicitly re-bound to VCPU0. */
 844                 bind_evtchn_to_cpu(evtchn, 0);
 845
 846                 evtchn_to_irq[evtchn] = -1;
 847         }
 848
 849 #ifndef PER_CPU_VIRQ_IRQ
 850 done:
 851 #endif
 852         spin_unlock(&irq_mapping_update_lock);
 853
 854         if (free_action) {
 855                 cpumask_t *cpus = free_action->cpus;
 856
 857                 free_irq(irq, free_action->action.dev_id);
 858                 free_cpumask_var(cpus);
 859         }
 860 }
 861 EXPORT_SYMBOL_GPL(unbind_from_per_cpu_irq);
 862 #endif /* !PER_CPU_IPI_IRQ || !PER_CPU_VIRQ_IRQ */
 863
 864 int bind_caller_port_to_irqhandler(
 865         unsigned int caller_port,
 866         irq_handler_t handler,
 867         unsigned long irqflags,
 868         const char *devname,
 869         void *dev_id)
 870 {
 871         int irq, retval;
 872
 873         irq = bind_caller_port_to_irq(caller_port);
 874         if (irq < 0)
 875                 return irq;
 876
 877         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 878         if (retval != 0) {
 879                 unbind_from_irq(irq);
 880                 return retval;
 881         }
 882
 883         return irq;
 884 }
 885 EXPORT_SYMBOL_GPL(bind_caller_port_to_irqhandler);
 886
 887 int bind_listening_port_to_irqhandler(
 888         unsigned int remote_domain,
 889         irq_handler_t handler,
 890         unsigned long irqflags,
 891         const char *devname,
 892         void *dev_id)
 893 {
 894         int irq, retval;
 895
 896         irq = bind_listening_port_to_irq(remote_domain);
 897         if (irq < 0)
 898                 return irq;
 899
 900         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 901         if (retval != 0) {
 902                 unbind_from_irq(irq);
 903                 return retval;
 904         }
 905
 906         return irq;
 907 }
 908 EXPORT_SYMBOL_GPL(bind_listening_port_to_irqhandler);
 909
 910 int bind_interdomain_evtchn_to_irqhandler(
 911         unsigned int remote_domain,
 912         unsigned int remote_port,
 913         irq_handler_t handler,
 914         unsigned long irqflags,
 915         const char *devname,
 916         void *dev_id)
 917 {
 918         int irq, retval;
 919
 920         irq = bind_interdomain_evtchn_to_irq(remote_domain, remote_port);
 921         if (irq < 0)
 922                 return irq;
 923
 924         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 925         if (retval != 0) {
 926                 unbind_from_irq(irq);
 927                 return retval;
 928         }
 929
 930         return irq;
 931 }
 932 EXPORT_SYMBOL_GPL(bind_interdomain_evtchn_to_irqhandler);
 933
 934 int bind_virq_to_irqhandler(
 935         unsigned int virq,
 936         unsigned int cpu,
 937         irq_handler_t handler,
 938         unsigned long irqflags,
 939         const char *devname,
 940         void *dev_id)
 941 {
 942         int irq, retval;
 943
 944 #ifndef PER_CPU_VIRQ_IRQ
 945         BUG_ON(test_bit(virq, virq_per_cpu));
 946 #endif
 947
 948         irq = bind_virq_to_irq(virq, cpu);
 949         if (irq < 0)
 950                 return irq;
 951
 952         retval = request_irq(irq, handler, irqflags, devname, dev_id);
 953         if (retval != 0) {
 954                 unbind_from_irq(irq);
 955                 return retval;
 956         }
 957
 958         return irq;
 959 }
 960 EXPORT_SYMBOL_GPL(bind_virq_to_irqhandler);
 961
 962 #ifdef CONFIG_SMP
 963 #ifndef PER_CPU_VIRQ_IRQ
 964 int bind_virq_to_irqaction(
 965         unsigned int virq,
 966         unsigned int cpu,
 967         struct irqaction *action)
 968 {
 969         struct evtchn_bind_virq bind_virq;
 970         struct irq_cfg *cfg;
 971         unsigned int evtchn;
 972         int irq, retval = 0;
 973         struct percpu_irqaction *cur = NULL, *new;
 974
 975         BUG_ON(!test_bit(virq, virq_per_cpu));
 976
 977         if (action->dev_id)
 978                 return -EINVAL;
 979
 980         new = alloc_percpu_irqaction(GFP_ATOMIC);
 981         if (new) {
 982                 new->action = *action;
 983                 new->action.dev_id = action;
 984         }
 985
 986         spin_lock(&irq_mapping_update_lock);
 987
 988         for (cur = virq_actions[virq]; cur; cur = cur->next)
 989                 if (cur->action.dev_id == action)
 990                         break;
 991         if (!cur) {
 992                 if (!new) {
 993                         spin_unlock(&irq_mapping_update_lock);
 994                         return -ENOMEM;
 995                 }
 996                 new->next = virq_actions[virq];
 997                 virq_actions[virq] = cur = new;
 998                 new = NULL;
 999                 retval = 1;
1000         }
1001         cpumask_set_cpu(cpu, cur->cpus);
1002         action = &cur->action;
1003
1004         if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1) {
1005                 unsigned int nr;
1006
1007                 BUG_ON(!retval);
1008
1009                 if ((irq = find_unbound_irq(cpu_to_node(cpu), &cfg,
1010                                             &dynirq_chip, true)) < 0) {
1011                         virq_actions[virq] = cur->next;
1012                         spin_unlock(&irq_mapping_update_lock);
1013                         free_percpu_irqaction(new);
1014                         return irq;
1015                 }
1016
1017                 /* Extra reference so count will never drop to zero. */
1018                 cfg->bindcount++;
1019
1020                 for_each_possible_cpu(nr)
1021                         per_cpu(virq_to_irq, nr)[virq] = irq;
1022                 cfg->info = mk_irq_info(IRQT_VIRQ, virq, 0);
1023         } else
1024                 cfg = irq_cfg(irq);
1025
1026         evtchn = per_cpu(virq_to_evtchn, cpu)[virq];
1027         if (!VALID_EVTCHN(evtchn)) {
1028                 bind_virq.virq = virq;
1029                 bind_virq.vcpu = cpu;
1030                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1031                                                 &bind_virq) != 0)
1032                         BUG();
1033                 evtchn = bind_virq.port;
1034                 evtchn_to_irq[evtchn] = irq;
1035                 per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
1036
1037                 bind_evtchn_to_cpu(evtchn, cpu);
1038         }
1039
1040         cfg->bindcount++;
1041
1042         spin_unlock(&irq_mapping_update_lock);
1043
1044         free_percpu_irqaction(new);
1045
1046         if (retval == 0) {
1047                 unsigned long flags;
1048
1049                 local_irq_save(flags);
1050                 unmask_evtchn(evtchn);
1051                 local_irq_restore(flags);
1052         } else {
1053                 action->flags |= IRQF_PERCPU;
1054                 retval = setup_irq(irq, action);
1055                 if (retval) {
1056                         unbind_from_per_cpu_irq(irq, cpu, action);
1057                         BUG_ON(retval > 0);
1058                         irq = retval;
1059                 }
1060         }
1061
1062         return irq;
1063 }
1064 EXPORT_SYMBOL_GPL(bind_virq_to_irqaction);
1065 #endif
1066
1067 #ifdef PER_CPU_IPI_IRQ
1068 int bind_ipi_to_irqhandler(
1069         unsigned int ipi,
1070         unsigned int cpu,
1071         irq_handler_t handler,
1072         unsigned long irqflags,
1073         const char *devname,
1074         void *dev_id)
1075 {
1076         int irq, retval;
1077
1078         irq = bind_ipi_to_irq(ipi, cpu);
1079         if (irq < 0)
1080                 return irq;
1081
1082         retval = request_irq(irq, handler, irqflags | IRQF_NO_SUSPEND,
1083                              devname, dev_id);
1084         if (retval != 0) {
1085                 unbind_from_irq(irq);
1086                 return retval;
1087         }
1088
1089         return irq;
1090 }
1091 #else
1092 int __cpuinit bind_ipi_to_irqaction(
1093         unsigned int cpu,
1094         struct irqaction *action)
1095 {
1096         struct evtchn_bind_ipi bind_ipi;
1097         struct irq_cfg *cfg;
1098         unsigned int evtchn;
1099         int retval = 0;
1100
1101         spin_lock(&irq_mapping_update_lock);
1102
1103         if (VALID_EVTCHN(per_cpu(ipi_evtchn, cpu))) {
1104                 spin_unlock(&irq_mapping_update_lock);
1105                 return -EBUSY;
1106         }
1107
1108         if (ipi_irq < 0) {
1109                 if ((ipi_irq = find_unbound_irq(cpu_to_node(cpu), &cfg,
1110                                                 &dynirq_chip, true)) < 0) {
1111                         spin_unlock(&irq_mapping_update_lock);
1112                         return ipi_irq;
1113                 }
1114
1115                 /* Extra reference so count will never drop to zero. */
1116                 cfg->bindcount++;
1117
1118                 cfg->info = mk_irq_info(IRQT_IPI, 0, 0);
1119                 retval = 1;
1120         } else
1121                 cfg = irq_cfg(ipi_irq);
1122
1123         bind_ipi.vcpu = cpu;
1124         if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi))
1125                 BUG();
1126
1127         evtchn = bind_ipi.port;
1128         evtchn_to_irq[evtchn] = ipi_irq;
1129         per_cpu(ipi_evtchn, cpu) = evtchn;
1130
1131         bind_evtchn_to_cpu(evtchn, cpu);
1132
1133         cfg->bindcount++;
1134
1135         spin_unlock(&irq_mapping_update_lock);
1136
1137         if (retval == 0) {
1138                 unsigned long flags;
1139
1140                 local_irq_save(flags);
1141                 unmask_evtchn(evtchn);
1142                 local_irq_restore(flags);
1143         } else {
1144                 action->flags |= IRQF_PERCPU | IRQF_NO_SUSPEND;
1145                 retval = setup_irq(ipi_irq, action);
1146                 if (retval) {
1147                         unbind_from_per_cpu_irq(ipi_irq, cpu, NULL);
1148                         BUG_ON(retval > 0);
1149                         ipi_irq = retval;
1150                 }
1151         }
1152
1153         return ipi_irq;
1154 }
1155 #endif /* PER_CPU_IPI_IRQ */
1156 #endif /* CONFIG_SMP */
1157
1158 void unbind_from_irqhandler(unsigned int irq, void *dev_id)
1159 {
1160         free_irq(irq, dev_id);
1161         unbind_from_irq(irq);
1162 }
1163 EXPORT_SYMBOL_GPL(unbind_from_irqhandler);
1164
1165 #ifdef CONFIG_SMP
1166 static int set_affinity_irq(struct irq_data *data,
1167                             const struct cpumask *dest, bool force)
1168 {
1169         const struct irq_cfg *cfg = irq_data_cfg(data);
1170         unsigned int port = evtchn_from_irq_cfg(cfg);
1171         unsigned int cpu = cpumask_any(dest);
1172         struct evtchn_bind_vcpu ebv = { .port = port, .vcpu = cpu };
1173         bool masked;
1174         int rc;
1175
1176         BUG_IF_VIRQ_PER_CPU(cfg);
1177         BUG_IF_IPI(cfg);
1178
1179         if (!VALID_EVTCHN(port))
1180                 return -ENXIO;
1181
1182         masked = test_and_set_evtchn_mask(port);
1183         rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &ebv);
1184         if (rc == 0) {
1185                 bind_evtchn_to_cpu(port, cpu);
1186                 rc = evtchn_to_irq[port] != -1 ? IRQ_SET_MASK_OK_NOCOPY
1187                                                : IRQ_SET_MASK_OK;
1188         }
1189         if (!masked)
1190                 unmask_evtchn(port);
1191
1192         return rc;
1193 }
1194 #endif
1195
1196 int resend_irq_on_evtchn(struct irq_data *data)
1197 {
1198         unsigned int evtchn = evtchn_from_irq_data(data);
1199         bool masked;
1200
1201         if (!VALID_EVTCHN(evtchn))
1202                 return 1;
1203
1204         masked = test_and_set_evtchn_mask(evtchn);
1205         set_evtchn(evtchn);
1206         if (!masked)
1207                 unmask_evtchn(evtchn);
1208
1209         return 1;
1210 }
1211
1212 /*
1213  * Interface to generic handling in irq.c
1214  */
1215
1216 static void unmask_dynirq(struct irq_data *data)
1217 {
1218         unsigned int evtchn = evtchn_from_irq_data(data);
1219
1220         if (VALID_EVTCHN(evtchn))
1221                 unmask_evtchn(evtchn);
1222 }
1223
1224 static void mask_dynirq(struct irq_data *data)
1225 {
1226         unsigned int evtchn = evtchn_from_irq_data(data);
1227
1228         if (VALID_EVTCHN(evtchn))
1229                 mask_evtchn(evtchn);
1230 }
1231
1232 static unsigned int startup_dynirq(struct irq_data *data)
1233 {
1234         unmask_dynirq(data);
1235         return 0;
1236 }
1237
1238 #define shutdown_dynirq mask_dynirq
1239
1240 static void end_dynirq(struct irq_data *data)
1241 {
1242         if (!irqd_irq_disabled(data)) {
1243                 irq_move_masked_irq(data);
1244                 unmask_dynirq(data);
1245         }
1246 }
1247
1248 static struct irq_chip dynirq_chip = {
1249         .name             = "Dynamic",
1250         .irq_startup      = startup_dynirq,
1251         .irq_shutdown     = shutdown_dynirq,
1252         .irq_enable       = unmask_dynirq,
1253         .irq_disable      = mask_dynirq,
1254         .irq_mask         = mask_dynirq,
1255         .irq_unmask       = unmask_dynirq,
1256         .irq_eoi          = end_dynirq,
1257 #ifdef CONFIG_SMP
1258         .irq_set_affinity = set_affinity_irq,
1259 #endif
1260         .irq_retrigger    = resend_irq_on_evtchn,
1261 };
1262
1263 /* Bitmap indicating which PIRQs require Xen to be notified on unmask. */
1264 static bool pirq_eoi_does_unmask;
1265 static unsigned long *pirq_needs_eoi;
1266 static DECLARE_BITMAP(probing_pirq, NR_PIRQS);
1267
1268 static void pirq_unmask_and_notify(unsigned int evtchn, unsigned int irq)
1269 {
1270         struct physdev_eoi eoi = { .irq = evtchn_get_xen_pirq(irq) };
1271
1272         if (pirq_eoi_does_unmask) {
1273                 if (test_bit(eoi.irq, pirq_needs_eoi))
1274                         VOID(HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi));
1275                 else
1276                         unmask_evtchn(evtchn);
1277         } else if (test_bit(irq - PIRQ_BASE, pirq_needs_eoi)) {
1278                 if (smp_processor_id() != cpu_from_evtchn(evtchn)) {
1279                         struct evtchn_unmask unmask = { .port = evtchn };
1280                         struct multicall_entry mcl[2];
1281
1282                         mcl[0].op = __HYPERVISOR_event_channel_op;
1283                         mcl[0].args[0] = EVTCHNOP_unmask;
1284                         mcl[0].args[1] = (unsigned long)&unmask;
1285                         mcl[1].op = __HYPERVISOR_physdev_op;
1286                         mcl[1].args[0] = PHYSDEVOP_eoi;
1287                         mcl[1].args[1] = (unsigned long)&eoi;
1288
1289                         if (HYPERVISOR_multicall(mcl, 2))
1290                                 BUG();
1291                 } else {
1292                         unmask_evtchn(evtchn);
1293                         VOID(HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi));
1294                 }
1295         } else
1296                 unmask_evtchn(evtchn);
1297 }
1298
1299 static inline void pirq_query_unmask(int irq)
1300 {
1301         struct physdev_irq_status_query irq_status;
1302
1303         if (pirq_eoi_does_unmask)
1304                 return;
1305         irq_status.irq = evtchn_get_xen_pirq(irq);
1306         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1307                 irq_status.flags = 0;
1308         clear_bit(irq - PIRQ_BASE, pirq_needs_eoi);
1309         if (irq_status.flags & XENIRQSTAT_needs_eoi)
1310                 set_bit(irq - PIRQ_BASE, pirq_needs_eoi);
1311 }
1312
1313 static int set_type_pirq(struct irq_data *data, unsigned int type)
1314 {
1315         if (type != IRQ_TYPE_PROBE)
1316                 return -EINVAL;
1317         set_bit(data->irq - PIRQ_BASE, probing_pirq);
1318         return 0;
1319 }
1320
1321 static void enable_pirq(struct irq_data *data)
1322 {
1323         struct evtchn_bind_pirq bind_pirq;
1324         struct irq_cfg *cfg = irq_data_cfg(data);
1325         unsigned int evtchn = evtchn_from_irq_cfg(cfg);
1326         unsigned int irq = data->irq, pirq = irq - PIRQ_BASE;
1327
1328         if (VALID_EVTCHN(evtchn)) {
1329                 if (pirq < nr_pirqs)
1330                         clear_bit(pirq, probing_pirq);
1331                 goto out;
1332         }
1333
1334         bind_pirq.pirq = evtchn_get_xen_pirq(irq);
1335         /* NB. We are happy to share unless we are probing. */
1336         bind_pirq.flags = (pirq < nr_pirqs
1337                            && test_and_clear_bit(pirq, probing_pirq))
1338                           || (irq_to_desc(irq)->istate & IRQS_AUTODETECT)
1339                           ? 0 : BIND_PIRQ__WILL_SHARE;
1340         if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq) != 0) {
1341                 if (bind_pirq.flags)
1342                         pr_info("Failed to obtain physical IRQ %d\n", irq);
1343                 return;
1344         }
1345         evtchn = bind_pirq.port;
1346
1347         pirq_query_unmask(irq);
1348
1349         evtchn_to_irq[evtchn] = irq;
1350         bind_evtchn_to_cpu(evtchn, 0);
1351         cfg->info = mk_irq_info(IRQT_PIRQ, bind_pirq.pirq, evtchn);
1352
1353  out:
1354         pirq_unmask_and_notify(evtchn, irq);
1355 }
1356
1357 #define disable_pirq mask_pirq
1358
1359 static unsigned int startup_pirq(struct irq_data *data)
1360 {
1361         enable_pirq(data);
1362         return 0;
1363 }
1364
1365 static void shutdown_pirq(struct irq_data *data)
1366 {
1367         struct irq_cfg *cfg = irq_data_cfg(data);
1368         unsigned int evtchn = evtchn_from_irq_cfg(cfg);
1369
1370         if (!VALID_EVTCHN(evtchn))
1371                 return;
1372
1373         mask_evtchn(evtchn);
1374
1375         if (close_evtchn(evtchn))
1376                 BUG();
1377
1378         bind_evtchn_to_cpu(evtchn, 0);
1379         evtchn_to_irq[evtchn] = -1;
1380         cfg->info = mk_irq_info(IRQT_PIRQ, index_from_irq_cfg(cfg), 0);
1381 }
1382
1383 static void unmask_pirq(struct irq_data *data)
1384 {
1385         unsigned int evtchn = evtchn_from_irq_data(data);
1386
1387         if (VALID_EVTCHN(evtchn))
1388                 pirq_unmask_and_notify(evtchn, data->irq);
1389 }
1390
1391 #define mask_pirq mask_dynirq
1392
1393 static void end_pirq(struct irq_data *data)
1394 {
1395         bool disabled = irqd_irq_disabled(data);
1396
1397         if (disabled && (irq_to_desc(data->irq)->istate & IRQS_PENDING))
1398                 shutdown_pirq(data);
1399         else {
1400                 if (!disabled)
1401                         irq_move_masked_irq(data);
1402                 unmask_pirq(data);
1403         }
1404 }
1405
1406 static struct irq_chip pirq_chip = {
1407         .name             = "Phys",
1408         .irq_startup      = startup_pirq,
1409         .irq_shutdown     = shutdown_pirq,
1410         .irq_enable       = enable_pirq,
1411         .irq_disable      = disable_pirq,
1412         .irq_mask         = mask_pirq,
1413         .irq_unmask       = unmask_pirq,
1414         .irq_eoi          = end_pirq,
1415         .irq_set_type     = set_type_pirq,
1416 #ifdef CONFIG_SMP
1417         .irq_set_affinity = set_affinity_irq,
1418 #endif
1419         .irq_retrigger    = resend_irq_on_evtchn,
1420 };
1421
1422 int irq_ignore_unhandled(unsigned int irq)
1423 {
1424         struct physdev_irq_status_query irq_status = { .irq = irq };
1425
1426         if (!is_running_on_xen() || irq >= nr_pirqs)
1427                 return 0;
1428
1429         if (HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status))
1430                 return 0;
1431         return !!(irq_status.flags & XENIRQSTAT_shared);
1432 }
1433
1434 #if defined(CONFIG_SMP) && !defined(PER_CPU_IPI_IRQ)
1435 void notify_remote_via_ipi(unsigned int ipi, unsigned int cpu)
1436 {
1437         unsigned int evtchn = per_cpu(ipi_evtchn, cpu);
1438
1439 #ifdef NMI_VECTOR
1440         if (ipi == NMI_VECTOR) {
1441                 int rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, cpu, NULL);
1442
1443                 if (rc)
1444                         pr_warn_once("Unable (%d) to send NMI to CPU#%u\n",
1445                                      rc, cpu);
1446                 return;
1447         }
1448 #endif
1449
1450         if (VALID_EVTCHN(evtchn)
1451             && !test_and_set_bit(ipi, per_cpu(ipi_pending, cpu))
1452             && !test_evtchn(evtchn))
1453                 notify_remote_via_evtchn(evtchn);
1454 }
1455
1456 void clear_ipi_evtchn(void)
1457 {
1458         unsigned int evtchn = this_cpu_read(ipi_evtchn);
1459
1460         BUG_ON(!VALID_EVTCHN(evtchn));
1461         clear_evtchn(evtchn);
1462 }
1463 #endif
1464
1465 void notify_remote_via_irq(int irq)
1466 {
1467         const struct irq_cfg *cfg = irq_cfg(irq);
1468         unsigned int evtchn;
1469
1470         if (WARN_ON_ONCE(!cfg))
1471                 return;
1472         BUG_ON(type_from_irq_cfg(cfg) == IRQT_VIRQ);
1473         BUG_IF_IPI(cfg);
1474
1475         evtchn = evtchn_from_irq_cfg(cfg);
1476         if (VALID_EVTCHN(evtchn))
1477                 notify_remote_via_evtchn(evtchn);
1478 }
1479 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
1480
1481 #if defined(CONFIG_XEN_BACKEND) || defined(CONFIG_XEN_BACKEND_MODULE)
1482 int multi_notify_remote_via_irq(multicall_entry_t *mcl, int irq)
1483 {
1484         const struct irq_cfg *cfg = irq_cfg(irq);
1485         unsigned int evtchn;
1486
1487         if (WARN_ON_ONCE(!cfg))
1488                 return -EINVAL;
1489         BUG_ON(type_from_irq_cfg(cfg) == IRQT_VIRQ);
1490         BUG_IF_IPI(cfg);
1491
1492         evtchn = evtchn_from_irq_cfg(cfg);
1493         if (!VALID_EVTCHN(evtchn))
1494                 return -EINVAL;
1495
1496         multi_notify_remote_via_evtchn(mcl, evtchn);
1497         return 0;
1498 }
1499 EXPORT_SYMBOL_GPL(multi_notify_remote_via_irq);
1500 #endif
1501
1502 int irq_to_evtchn_port(int irq)
1503 {
1504         const struct irq_cfg *cfg = irq_cfg(irq);
1505
1506         if (!cfg)
1507                 return 0;
1508         BUG_IF_VIRQ_PER_CPU(cfg);
1509         BUG_IF_IPI(cfg);
1510         return evtchn_from_irq_cfg(cfg);
1511 }
1512 EXPORT_SYMBOL_GPL(irq_to_evtchn_port);
1513
1514 void mask_evtchn(int port)
1515 {
1516         shared_info_t *s = HYPERVISOR_shared_info;
1517         sync_set_bit(port, s->evtchn_mask);
1518 }
1519 EXPORT_SYMBOL_GPL(mask_evtchn);
1520
1521 void unmask_evtchn(int port)
1522 {
1523         shared_info_t *s = HYPERVISOR_shared_info;
1524         unsigned int cpu = smp_processor_id();
1525
1526         BUG_ON(!irqs_disabled());
1527
1528         /* Slow path (hypercall) if this is a non-local port. */
1529         if (unlikely(cpu != cpu_from_evtchn(port))) {
1530                 struct evtchn_unmask unmask = { .port = port };
1531                 VOID(HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask));
1532                 return;
1533         }
1534
1535         sync_clear_bit(port, s->evtchn_mask);
1536
1537         /* Did we miss an interrupt 'edge'? Re-fire if so. */
1538         if (sync_test_bit(port, s->evtchn_pending)) {
1539                 vcpu_info_t *v = current_vcpu_info();
1540
1541                 if (!sync_test_and_set_bit(port / BITS_PER_LONG,
1542                                            &v->evtchn_pending_sel))
1543                         v->evtchn_upcall_pending = 1;
1544         }
1545 }
1546 EXPORT_SYMBOL_GPL(unmask_evtchn);
1547
1548 void disable_all_local_evtchn(void)
1549 {
1550         unsigned i, cpu = smp_processor_id();
1551         shared_info_t *s = HYPERVISOR_shared_info;
1552
1553         for (i = 0; i < NR_EVENT_CHANNELS; ++i)
1554                 if (cpu_from_evtchn(i) == cpu)
1555                         sync_set_bit(i, &s->evtchn_mask[0]);
1556 }
1557
1558 /* Test an irq's pending state. */
1559 int xen_test_irq_pending(int irq)
1560 {
1561         unsigned int evtchn = evtchn_from_irq(irq);
1562
1563         return VALID_EVTCHN(evtchn) && test_evtchn(evtchn);
1564 }
1565
1566 #ifdef CONFIG_PM_SLEEP
1567 #include <linux/syscore_ops.h>
1568
1569 static void restore_cpu_virqs(unsigned int cpu)
1570 {
1571         struct evtchn_bind_virq bind_virq;
1572         int virq, irq, evtchn;
1573
1574         for (virq = 0; virq < NR_VIRQS; virq++) {
1575                 if ((irq = per_cpu(virq_to_irq, cpu)[virq]) == -1)
1576                         continue;
1577
1578 #ifndef PER_CPU_VIRQ_IRQ
1579                 if (test_bit(virq, virq_per_cpu)
1580                     && !VALID_EVTCHN(per_cpu(virq_to_evtchn, cpu)[virq]))
1581                         continue;
1582 #endif
1583
1584                 BUG_ON(irq_cfg(irq)->info != mk_irq_info(IRQT_VIRQ, virq, 0));
1585
1586                 /* Get a new binding from Xen. */
1587                 bind_virq.virq = virq;
1588                 bind_virq.vcpu = cpu;
1589                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
1590                                                 &bind_virq) != 0)
1591                         BUG();
1592                 evtchn = bind_virq.port;
1593
1594                 /* Record the new mapping. */
1595                 evtchn_to_irq[evtchn] = irq;
1596 #ifdef PER_CPU_VIRQ_IRQ
1597                 irq_cfg(irq)->info = mk_irq_info(IRQT_VIRQ, virq, evtchn);
1598 #else
1599                 if (test_bit(virq, virq_per_cpu))
1600                         per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
1601                 else {
1602                         unsigned int cpu;
1603
1604                         irq_cfg(irq)->info = mk_irq_info(IRQT_VIRQ, virq,
1605                                                          evtchn);
1606                         for_each_possible_cpu(cpu)
1607                                 per_cpu(virq_to_evtchn, cpu)[virq] = evtchn;
1608                 }
1609 #endif
1610                 bind_evtchn_to_cpu(evtchn, cpu);
1611
1612                 /* Ready for use. */
1613                 unmask_evtchn(evtchn);
1614         }
1615 }
1616
1617 static void restore_cpu_ipis(unsigned int cpu)
1618 {
1619 #ifdef CONFIG_SMP
1620         struct evtchn_bind_ipi bind_ipi;
1621         struct irq_data *data;
1622         unsigned int evtchn;
1623 #ifdef PER_CPU_IPI_IRQ
1624         int ipi, irq;
1625
1626         for (ipi = 0; ipi < NR_IPIS; ipi++) {
1627                 if ((irq = per_cpu(ipi_to_irq, cpu)[ipi]) == -1)
1628                         continue;
1629 #else
1630 #define ipi 0
1631 #define irq ipi_irq
1632                 if (irq == -1
1633                     || !VALID_EVTCHN(per_cpu(ipi_evtchn, cpu)))
1634                         return;
1635 #endif
1636
1637                 data = irq_get_irq_data(irq);
1638                 BUG_ON(irq_data_cfg(data)->info != mk_irq_info(IRQT_IPI, ipi, 0));
1639
1640                 /* Get a new binding from Xen. */
1641                 bind_ipi.vcpu = cpu;
1642                 if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
1643                                                 &bind_ipi) != 0)
1644                         BUG();
1645                 evtchn = bind_ipi.port;
1646
1647                 /* Record the new mapping. */
1648                 evtchn_to_irq[evtchn] = irq;
1649 #ifdef PER_CPU_IPI_IRQ
1650                 irq_data_cfg(data)->info = mk_irq_info(IRQT_IPI, ipi, evtchn);
1651 #else
1652                 per_cpu(ipi_evtchn, cpu) = evtchn;
1653 #endif
1654                 bind_evtchn_to_cpu(evtchn, cpu);
1655
1656                 /* Ready for use. */
1657                 if (!irqd_irq_disabled(data))
1658                         unmask_evtchn(evtchn);
1659 #ifdef PER_CPU_IPI_IRQ
1660         }
1661 #else
1662 #undef irq
1663 #undef ipi
1664 #endif
1665 #endif /* CONFIG_SMP */
1666 }
1667
1668 static void evtchn_resume(void)
1669 {
1670         unsigned int cpu, irq, evtchn;
1671         struct evtchn_status status;
1672
1673         /* Avoid doing anything in the 'suspend cancelled' case. */
1674         status.dom = DOMID_SELF;
1675 #ifdef PER_CPU_VIRQ_IRQ
1676         status.port = evtchn_from_irq(__this_cpu_read(virq_to_irq[VIRQ_TIMER]));
1677 #else
1678         status.port = __this_cpu_read(virq_to_evtchn[VIRQ_TIMER]);
1679 #endif
1680         if (HYPERVISOR_event_channel_op(EVTCHNOP_status, &status))
1681                 BUG();
1682         if (status.status == EVTCHNSTAT_virq
1683             && status.vcpu == smp_processor_id()
1684             && status.u.virq == VIRQ_TIMER)
1685                 return;
1686
1687         init_evtchn_cpu_bindings();
1688
1689         if (pirq_eoi_does_unmask) {
1690                 struct physdev_pirq_eoi_gmfn eoi_gmfn;
1691
1692                 eoi_gmfn.gmfn = virt_to_machine(pirq_needs_eoi) >> PAGE_SHIFT;
1693                 if (HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn, &eoi_gmfn))
1694                         BUG();
1695         }
1696
1697         /* New event-channel space is not 'live' yet. */
1698         for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1699                 mask_evtchn(evtchn);
1700
1701         /* No IRQ <-> event-channel mappings. */
1702         for (irq = 0; irq < nr_irqs; irq++) {
1703                 struct irq_cfg *cfg = irq_cfg(irq);
1704
1705                 if (!cfg)
1706                         continue;
1707
1708                 /* Check that no PIRQs are still bound. */
1709 #ifdef CONFIG_SPARSE_IRQ
1710                 if (irq < PIRQ_BASE || irq >= PIRQ_BASE + nr_pirqs)
1711                         BUG_ON(type_from_irq_cfg(cfg) == IRQT_PIRQ);
1712                 else
1713 #endif
1714                         BUG_ON(cfg->info != IRQ_UNBOUND);
1715
1716                 cfg->info &= ~((1U << _EVTCHN_BITS) - 1);
1717         }
1718         for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++)
1719                 evtchn_to_irq[evtchn] = -1;
1720
1721         for_each_possible_cpu(cpu) {
1722                 restore_cpu_virqs(cpu);
1723                 restore_cpu_ipis(cpu);
1724         }
1725 }
1726
1727 static struct syscore_ops evtchn_syscore_ops = {
1728         .resume = evtchn_resume,
1729 };
1730
1731 static int __init evtchn_register(void)
1732 {
1733         if (!is_initial_xendomain())
1734                 register_syscore_ops(&evtchn_syscore_ops);
1735         return 0;
1736 }
1737 core_initcall(evtchn_register);
1738 #endif
1739
1740 int __init arch_early_irq_init(void)
1741 {
1742         unsigned int i;
1743
1744         for (i = 0; i < ARRAY_SIZE(_irq_cfg); i++)
1745                 irq_set_chip_data(i, _irq_cfg + i);
1746
1747         return 0;
1748 }
1749
1750 struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node)
1751 {
1752         int res = irq_alloc_desc_at(at, node);
1753         struct irq_cfg *cfg = NULL;
1754
1755         if (res < 0) {
1756                 if (res != -EEXIST)
1757                         return NULL;
1758                 cfg = irq_get_chip_data(at);
1759                 if (cfg)
1760                         return cfg;
1761         }
1762
1763 #ifdef CONFIG_SPARSE_IRQ
1764 #ifdef CONFIG_SMP
1765         /* By default all event channels notify CPU#0. */
1766         cpumask_copy(irq_get_irq_data(at)->affinity, cpumask_of(0));
1767 #endif
1768
1769         cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1770         if (cfg)
1771                 irq_set_chip_data(at, cfg);
1772         else
1773                 irq_free_desc(at);
1774
1775         return cfg;
1776 #else
1777         return irq_cfg(at);
1778 #endif
1779 }
1780
1781 #ifdef CONFIG_SPARSE_IRQ
1782 #ifdef CONFIG_X86_IO_APIC
1783 #include <asm/io_apic.h>
1784 #endif
1785
1786 int nr_pirqs = NR_PIRQS;
1787 EXPORT_SYMBOL_GPL(nr_pirqs);
1788
1789 int __init arch_probe_nr_irqs(void)
1790 {
1791         int nr = 64 + CONFIG_XEN_NR_GUEST_DEVICES, nr_irqs_gsi;
1792
1793         if (is_initial_xendomain()) {
1794                 nr_irqs_gsi = NR_IRQS_LEGACY;
1795 #ifdef CONFIG_X86_IO_APIC
1796                 nr_irqs_gsi += gsi_top;
1797 #endif
1798 #ifdef CONFIG_PCI_MSI
1799                 nr += max(nr_irqs_gsi * 16, nr_cpu_ids * 8);
1800 #endif
1801         } else {
1802                 nr_irqs_gsi = NR_VECTORS;
1803 #ifdef CONFIG_PCI_MSI
1804                 nr += max(NR_IRQS_LEGACY * 16, nr_cpu_ids * 8);
1805 #endif
1806         }
1807
1808         if (nr_pirqs > nr_irqs_gsi)
1809                 nr_pirqs = nr_irqs_gsi;
1810         if (nr > min_t(int, NR_DYNIRQS, NR_EVENT_CHANNELS))
1811                 nr = min_t(int, NR_DYNIRQS, NR_EVENT_CHANNELS);
1812         nr_irqs = min_t(int, nr_pirqs + nr, PAGE_SIZE * 8);
1813
1814         printk(KERN_DEBUG "nr_pirqs: %d\n", nr_pirqs);
1815
1816         return ARRAY_SIZE(_irq_cfg);
1817 }
1818 #endif
1819
1820 #if defined(CONFIG_X86_IO_APIC)
1821 int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1822 {
1823         struct physdev_irq irq_op;
1824
1825         if (irq < PIRQ_BASE || irq - PIRQ_BASE >= nr_pirqs)
1826                 return -EINVAL;
1827
1828         if (cfg->vector)
1829                 return 0;
1830
1831         irq_op.irq = irq;
1832         if (HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &irq_op))
1833                 return -ENOSPC;
1834
1835         cfg->vector = irq_op.vector;
1836
1837         return 0;
1838 }
1839 #define identity_mapped_irq(irq) (!IO_APIC_IRQ((irq) - PIRQ_BASE))
1840 #elif defined(CONFIG_X86)
1841 #define identity_mapped_irq(irq) (((irq) - PIRQ_BASE) < NR_IRQS_LEGACY)
1842 #else
1843 #define identity_mapped_irq(irq) (1)
1844 #endif
1845
1846 void evtchn_register_pirq(int irq)
1847 {
1848         struct irq_cfg *cfg = irq_cfg(irq);
1849
1850         BUG_ON(irq < PIRQ_BASE || irq - PIRQ_BASE >= nr_pirqs);
1851         if (identity_mapped_irq(irq) || type_from_irq_cfg(cfg) != IRQT_UNBOUND)
1852                 return;
1853         cfg->info = mk_irq_info(IRQT_PIRQ, irq, 0);
1854         irq_set_chip_and_handler_name(irq, &pirq_chip, handle_fasteoi_irq,
1855                                       "fasteoi");
1856 }
1857
1858 #ifdef CONFIG_PCI_MSI
1859 int evtchn_map_pirq(int irq, int xen_pirq)
1860 {
1861         if (irq < 0) {
1862 #ifdef CONFIG_SPARSE_IRQ
1863                 struct irq_cfg *cfg;
1864
1865                 spin_lock(&irq_mapping_update_lock);
1866                 irq = find_unbound_irq(numa_node_id(), &cfg, &pirq_chip,
1867                                        false);
1868                 if (irq >= 0) {
1869                         BUG_ON(type_from_irq_cfg(cfg) != IRQT_UNBOUND);
1870                         cfg->bindcount++;
1871                         cfg->info = mk_irq_info(IRQT_PIRQ, xen_pirq, 0);
1872                 }
1873                 spin_unlock(&irq_mapping_update_lock);
1874                 if (irq < 0)
1875                         return irq;
1876         } else if (irq >= PIRQ_BASE && irq < PIRQ_BASE + nr_pirqs) {
1877                 WARN_ONCE(1, "Non-MSI IRQ#%d (Xen %d)\n", irq, xen_pirq);
1878                 return -EINVAL;
1879 #else
1880                 static DEFINE_SPINLOCK(irq_alloc_lock);
1881
1882                 irq = PIRQ_BASE + nr_pirqs - 1;
1883                 spin_lock(&irq_alloc_lock);
1884                 do {
1885                         struct irq_cfg *cfg;
1886
1887                         if (identity_mapped_irq(irq))
1888                                 continue;
1889                         cfg = alloc_irq_and_cfg_at(irq, numa_node_id());
1890                         if (unlikely(!cfg)) {
1891                                 spin_unlock(&irq_alloc_lock);
1892                                 return -ENOMEM;
1893                         }
1894                         if (!index_from_irq_cfg(cfg)) {
1895                                 BUG_ON(type_from_irq_cfg(cfg) != IRQT_UNBOUND);
1896                                 cfg->info = mk_irq_info(IRQT_PIRQ,
1897                                                         xen_pirq, 0);
1898                                 break;
1899                         }
1900                 } while (--irq >= PIRQ_BASE);
1901                 spin_unlock(&irq_alloc_lock);
1902                 if (irq < PIRQ_BASE)
1903                         return -ENOSPC;
1904                 irq_set_chip_and_handler_name(irq, &pirq_chip,
1905                                               handle_fasteoi_irq, "fasteoi");
1906 #endif
1907         } else if (!xen_pirq) {
1908                 struct irq_cfg *cfg = irq_cfg(irq);
1909
1910                 if (!cfg || unlikely(type_from_irq_cfg(cfg) != IRQT_PIRQ))
1911                         return -EINVAL;
1912                 /*
1913                  * dynamic_irq_cleanup(irq) would seem to be the correct thing
1914                  * here, but cannot be used as we get here also during shutdown
1915                  * when a driver didn't free_irq() its MSI(-X) IRQ(s), which
1916                  * then causes a warning in dynamic_irq_cleanup().
1917                  */
1918                 irq_set_chip_and_handler(irq, NULL, NULL);
1919                 cfg->info = IRQ_UNBOUND;
1920 #ifdef CONFIG_SPARSE_IRQ
1921                 cfg->bindcount--;
1922 #endif
1923                 return 0;
1924         } else if (type_from_irq(irq) != IRQT_PIRQ
1925                    || index_from_irq(irq) != xen_pirq) {
1926                 pr_err("IRQ#%d is already mapped to %d:%u - "
1927                        "cannot map to PIRQ#%u\n",
1928                        irq, type_from_irq(irq), index_from_irq(irq), xen_pirq);
1929                 return -EINVAL;
1930         }
1931         return index_from_irq(irq) ? irq : -EINVAL;
1932 }
1933 #endif
1934
1935 int evtchn_get_xen_pirq(int irq)
1936 {
1937         struct irq_cfg *cfg = irq_cfg(irq);
1938
1939         if (identity_mapped_irq(irq))
1940                 return irq;
1941         BUG_ON(type_from_irq_cfg(cfg) != IRQT_PIRQ);
1942         return index_from_irq_cfg(cfg);
1943 }
1944
1945 void __init xen_init_IRQ(void)
1946 {
1947         unsigned int i;
1948         struct physdev_pirq_eoi_gmfn eoi_gmfn;
1949
1950 #ifndef PER_CPU_VIRQ_IRQ
1951         __set_bit(VIRQ_TIMER, virq_per_cpu);
1952         __set_bit(VIRQ_DEBUG, virq_per_cpu);
1953         __set_bit(VIRQ_XENOPROF, virq_per_cpu);
1954 #ifdef CONFIG_IA64
1955         __set_bit(VIRQ_ITC, virq_per_cpu);
1956 #endif
1957 #endif
1958
1959         init_evtchn_cpu_bindings();
1960
1961 #ifdef CONFIG_SPARSE_IRQ
1962         i = nr_irqs;
1963 #else
1964         i = nr_pirqs;
1965 #endif
1966         i = get_order(sizeof(unsigned long) * BITS_TO_LONGS(i));
1967         pirq_needs_eoi = (void *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, i);
1968         BUILD_BUG_ON(NR_PIRQS > PAGE_SIZE * 8);
1969         eoi_gmfn.gmfn = virt_to_machine(pirq_needs_eoi) >> PAGE_SHIFT;
1970         if (HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn, &eoi_gmfn) == 0)
1971                 pirq_eoi_does_unmask = true;
1972
1973         /* No event channels are 'live' right now. */
1974         for (i = 0; i < NR_EVENT_CHANNELS; i++)
1975                 mask_evtchn(i);
1976
1977 #ifndef CONFIG_SPARSE_IRQ
1978         for (i = DYNIRQ_BASE; i < (DYNIRQ_BASE + NR_DYNIRQS); i++) {
1979                 irq_set_noprobe(i);
1980                 irq_set_chip_and_handler_name(i, &dynirq_chip,
1981                                               handle_fasteoi_irq, "fasteoi");
1982         }
1983
1984         for (i = PIRQ_BASE; i < (PIRQ_BASE + nr_pirqs); i++) {
1985 #else
1986         for (i = PIRQ_BASE; i < (PIRQ_BASE + NR_IRQS_LEGACY); i++) {
1987 #endif
1988                 if (!identity_mapped_irq(i))
1989                         continue;
1990
1991 #ifdef RTC_IRQ
1992                 /* If not domain 0, force our RTC driver to fail its probe. */
1993                 if (i - PIRQ_BASE == RTC_IRQ && !is_initial_xendomain())
1994                         continue;
1995 #endif
1996
1997                 irq_set_chip_and_handler_name(i, &pirq_chip,
1998                                               handle_fasteoi_irq, "fasteoi");
1999         }
2000 }