drivers/xen/core/spinlock.c

   1 /*
   2  *      Xen spinlock functions
   3  *
   4  *      See arch/x86/xen/smp.c for copyright and credits for derived
   5  *      portions of this file.
   6  */
   7 #define XEN_SPINLOCK_SOURCE
   8 #include <linux/spinlock_types.h>
   9
  10 #ifdef TICKET_SHIFT
  11
  12 #include <linux/export.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <asm/hardirq.h>
  16 #include <xen/clock.h>
  17 #include <xen/evtchn.h>
  18
  19 struct spinning {
  20         arch_spinlock_t *lock;
  21         unsigned int ticket;
  22 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
  23         unsigned int irq_count;
  24 #endif
  25         struct spinning *prev;
  26 };
  27 static DEFINE_PER_CPU(struct spinning *, _spinning);
  28 static DEFINE_PER_CPU_READ_MOSTLY(evtchn_port_t, poll_evtchn);
  29 /*
  30  * Protect removal of objects: Addition can be done lockless, and even
  31  * removal itself doesn't need protection - what needs to be prevented is
  32  * removed objects going out of scope (as they're allocated on the stack).
  33  */
  34 struct rm_seq {
  35         unsigned int idx;
  36 #define SEQ_REMOVE_BIAS (1 << !!CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING)
  37         atomic_t ctr[2];
  38 };
  39 static DEFINE_PER_CPU(struct rm_seq, rm_seq);
  40
  41 int __cpuinit xen_spinlock_init(unsigned int cpu)
  42 {
  43         struct evtchn_bind_ipi bind_ipi;
  44         int rc;
  45
  46         setup_runstate_area(cpu);
  47
  48         WARN_ON(per_cpu(poll_evtchn, cpu));
  49         bind_ipi.vcpu = cpu;
  50         rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
  51         if (!rc)
  52                 per_cpu(poll_evtchn, cpu) = bind_ipi.port;
  53         else
  54                 pr_warning("No spinlock poll event channel for CPU#%u (%d)\n",
  55                            cpu, rc);
  56
  57         return rc;
  58 }
  59
  60 void __cpuinit xen_spinlock_cleanup(unsigned int cpu)
  61 {
  62         struct evtchn_close close;
  63
  64         close.port = per_cpu(poll_evtchn, cpu);
  65         per_cpu(poll_evtchn, cpu) = 0;
  66         WARN_ON(HYPERVISOR_event_channel_op(EVTCHNOP_close, &close));
  67 }
  68
  69 #ifdef CONFIG_PM_SLEEP
  70 #include <linux/syscore_ops.h>
  71
  72 static void __cpuinit spinlock_resume(void)
  73 {
  74         unsigned int cpu;
  75
  76         for_each_online_cpu(cpu) {
  77                 per_cpu(poll_evtchn, cpu) = 0;
  78                 xen_spinlock_init(cpu);
  79         }
  80 }
  81
  82 static struct syscore_ops __cpuinitdata spinlock_syscore_ops = {
  83         .resume = spinlock_resume
  84 };
  85
  86 static int __init spinlock_register(void)
  87 {
  88         if (!is_initial_xendomain())
  89                 register_syscore_ops(&spinlock_syscore_ops);
  90         return 0;
  91 }
  92 core_initcall(spinlock_register);
  93 #endif
  94
  95 static inline void sequence(unsigned int bias)
  96 {
  97         unsigned int rm_idx = percpu_read(rm_seq.idx);
  98
  99         smp_wmb();
 100         percpu_write(rm_seq.idx, (rm_idx + bias) ^ (SEQ_REMOVE_BIAS / 2));
 101         smp_mb();
 102         rm_idx &= 1;
 103         while (percpu_read(rm_seq.ctr[rm_idx].counter))
 104                 cpu_relax();
 105 }
 106
 107 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 108 static DEFINE_PER_CPU(unsigned int, _irq_count);
 109
 110 static __ticket_t spin_adjust(struct spinning *spinning,
 111                               const arch_spinlock_t *lock,
 112                               __ticket_t ticket)
 113 {
 114         for (; spinning; spinning = spinning->prev) {
 115                 unsigned int old = spinning->ticket;
 116
 117                 if (spinning->lock != lock)
 118                         continue;
 119                 while (likely(old + 1)) {
 120                         unsigned int cur;
 121
 122 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING > 1
 123                         ticket = spin_adjust(spinning->prev, lock, ticket);
 124 #endif
 125                         cur = cmpxchg(&spinning->ticket, old, ticket);
 126                         if (cur == old)
 127                                 return cur;
 128                         old = cur;
 129                 }
 130 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING == 1
 131                 break;
 132 #endif
 133         }
 134         return ticket;
 135 }
 136
 137 struct __raw_tickets xen_spin_adjust(const arch_spinlock_t *lock,
 138                                      struct __raw_tickets token)
 139 {
 140         token.tail = spin_adjust(percpu_read(_spinning), lock, token.tail);
 141         token.head = ACCESS_ONCE(lock->tickets.head);
 142         return token;
 143 }
 144
 145 static unsigned int ticket_drop(struct spinning *spinning,
 146                                 unsigned int ticket, unsigned int cpu)
 147 {
 148         arch_spinlock_t *lock = spinning->lock;
 149
 150         if (cmpxchg(&spinning->ticket, ticket, -1) != ticket)
 151                 return -1;
 152         lock->owner = cpu;
 153         __add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
 154         ticket = (__ticket_t)(ticket + 1);
 155         return ticket != lock->tickets.tail ? ticket : -1;
 156 }
 157
 158 static unsigned int ticket_get(arch_spinlock_t *lock, struct spinning *prev)
 159 {
 160         struct __raw_tickets token = xadd(&lock->tickets,
 161                                           (struct __raw_tickets){ .tail = 1 });
 162
 163         return token.head == token.tail ? token.tail
 164                                         : spin_adjust(prev, lock, token.tail);
 165 }
 166
 167 void xen_spin_irq_enter(void)
 168 {
 169         struct spinning *spinning = percpu_read(_spinning);
 170         unsigned int cpu = raw_smp_processor_id();
 171
 172         percpu_inc(_irq_count);
 173         smp_mb();
 174         for (; spinning; spinning = spinning->prev) {
 175                 arch_spinlock_t *lock = spinning->lock;
 176
 177                 /*
 178                  * Return the ticket if we now own the lock. While just being
 179                  * desirable generally (to reduce latency on spinning CPUs),
 180                  * this is essential in the case where interrupts get
 181                  * re-enabled in xen_spin_wait().
 182                  * Try to get a new ticket right away (to reduce latency after
 183                  * the current lock was released), but don't acquire the lock.
 184                  */
 185                 while (lock->tickets.head == spinning->ticket) {
 186                         unsigned int ticket = ticket_drop(spinning,
 187                                                           spinning->ticket,
 188                                                           cpu);
 189
 190                         if (!(ticket + 1))
 191                                 break;
 192                         xen_spin_kick(lock, ticket);
 193                         spinning->ticket = ticket_get(lock, spinning->prev);
 194                         smp_mb();
 195                 }
 196         }
 197 }
 198
 199 void xen_spin_irq_exit(void)
 200 {
 201         struct spinning *spinning = percpu_read(_spinning);
 202         unsigned int cpu = raw_smp_processor_id();
 203         /*
 204          * Despite its counterpart being first in xen_spin_irq_enter() (to make
 205          * xen_spin_kick() properly handle locks that get owned after their
 206          * tickets were obtained there), it can validly be done first here:
 207          * We're guaranteed to see another invocation of xen_spin_irq_enter()
 208          * if any of the tickets need to be dropped again.
 209          */
 210         unsigned int irq_count = this_cpu_dec_return(_irq_count);
 211
 212         /*
 213          * Make sure all xen_spin_kick() instances which may still have seen
 214          * our old IRQ count exit their critical region (so that we won't fail
 215          * to re-obtain a ticket if ticket_drop() completes only after our
 216          * ticket check below).
 217          */
 218         sequence(0);
 219
 220         /*
 221          * Obtain new tickets for (or acquire) all those locks at the IRQ
 222          * nesting level we are about to return to where above we avoided
 223          * acquiring them.
 224          */
 225         for (; spinning; spinning = spinning->prev) {
 226                 arch_spinlock_t *lock = spinning->lock;
 227
 228                 if (spinning->irq_count < irq_count)
 229                         break;
 230                 if (spinning->ticket + 1)
 231                         continue;
 232                 spinning->ticket = ticket_get(lock, spinning->prev);
 233                 if (ACCESS_ONCE(lock->tickets.head) == spinning->ticket)
 234                         lock->owner = cpu;
 235         }
 236 }
 237 #endif
 238
 239 unsigned int xen_spin_wait(arch_spinlock_t *lock, struct __raw_tickets *ptok,
 240                            unsigned int flags)
 241 {
 242         unsigned int cpu = raw_smp_processor_id();
 243         typeof(vcpu_info(0)->evtchn_upcall_mask) upcall_mask
 244                 = arch_local_save_flags();
 245         struct spinning spinning;
 246
 247         /* If kicker interrupt not initialized yet, just spin. */
 248         if (unlikely(!cpu_online(cpu))
 249             || unlikely(!this_cpu_read(poll_evtchn)))
 250                 return UINT_MAX;
 251
 252         /* announce we're spinning */
 253         spinning.ticket = ptok->tail;
 254         spinning.lock = lock;
 255         spinning.prev = percpu_read(_spinning);
 256 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 257         spinning.irq_count = UINT_MAX;
 258         if (upcall_mask > flags) {
 259                 const struct spinning *other;
 260                 int nesting = CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING;
 261
 262                 for (other = spinning.prev; other; other = other->prev)
 263                         if (other->lock == lock && !--nesting) {
 264                                 flags = upcall_mask;
 265                                 break;
 266                         }
 267         }
 268         arch_local_irq_disable();
 269 #endif
 270         smp_wmb();
 271         percpu_write(_spinning, &spinning);
 272
 273         for (;;) {
 274                 clear_evtchn(percpu_read(poll_evtchn));
 275
 276                 /*
 277                  * Check again to make sure it didn't become free while
 278                  * we weren't looking.
 279                  */
 280                 if (lock->tickets.head == spinning.ticket) {
 281                         /*
 282                          * If we interrupted another spinlock while it was
 283                          * blocking, make sure it doesn't block (again)
 284                          * without rechecking the lock.
 285                          */
 286                         if (spinning.prev)
 287                                 set_evtchn(percpu_read(poll_evtchn));
 288                         break;
 289                 }
 290
 291 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 292                 if (upcall_mask > flags) {
 293                         spinning.irq_count = percpu_read(_irq_count);
 294                         smp_wmb();
 295                         arch_local_irq_restore(flags);
 296                 }
 297 #endif
 298
 299                 if (!test_evtchn(percpu_read(poll_evtchn)) &&
 300                     HYPERVISOR_poll_no_timeout(&__get_cpu_var(poll_evtchn), 1))
 301                         BUG();
 302
 303 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 304                 arch_local_irq_disable();
 305                 smp_wmb();
 306                 spinning.irq_count = UINT_MAX;
 307 #endif
 308
 309                 if (test_evtchn(percpu_read(poll_evtchn))) {
 310                         inc_irq_stat(irq_lock_count);
 311                         break;
 312                 }
 313         }
 314
 315         /*
 316          * Leave the event pending so that any interrupted blocker will
 317          * re-check.
 318          */
 319
 320         /* announce we're done */
 321         percpu_write(_spinning, spinning.prev);
 322         if (!CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING)
 323                 arch_local_irq_disable();
 324         sequence(SEQ_REMOVE_BIAS);
 325         arch_local_irq_restore(upcall_mask);
 326         smp_rmb();
 327         if (lock->tickets.head == spinning.ticket) {
 328                 lock->owner = cpu;
 329                 return 0;
 330         }
 331         BUG_ON(CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING && !(spinning.ticket + 1));
 332         ptok->head = lock->tickets.head;
 333         ptok->tail = spinning.ticket;
 334
 335         return 1 << 10;
 336 }
 337
 338 void xen_spin_kick(const arch_spinlock_t *lock, unsigned int ticket)
 339 {
 340         unsigned int cpu = raw_smp_processor_id(), anchor = cpu;
 341
 342         if (unlikely(!cpu_online(cpu)))
 343                 cpu = -1, anchor = nr_cpu_ids;
 344
 345         while ((cpu = cpumask_next(cpu, cpu_online_mask)) != anchor) {
 346                 unsigned int flags;
 347                 atomic_t *rm_ctr;
 348                 struct spinning *spinning;
 349
 350                 if (cpu >= nr_cpu_ids) {
 351                         if (anchor == nr_cpu_ids)
 352                                 return;
 353                         cpu = cpumask_first(cpu_online_mask);
 354                         if (cpu == anchor)
 355                                 return;
 356                 }
 357
 358                 flags = arch_local_irq_save();
 359                 for (;;) {
 360                         unsigned int rm_idx = per_cpu(rm_seq.idx, cpu);
 361
 362                         rm_ctr = per_cpu(rm_seq.ctr, cpu) + (rm_idx & 1);
 363                         atomic_inc(rm_ctr);
 364 #ifdef CONFIG_X86 /* atomic ops are full CPU barriers */
 365                         barrier();
 366 #else
 367                         smp_mb();
 368 #endif
 369                         spinning = per_cpu(_spinning, cpu);
 370                         smp_rmb();
 371                         if ((rm_idx ^ per_cpu(rm_seq.idx, cpu))
 372                             < SEQ_REMOVE_BIAS)
 373                                 break;
 374                         atomic_dec(rm_ctr);
 375                         if (!vcpu_running(cpu))
 376                                 HYPERVISOR_yield();
 377                 }
 378
 379                 for (; spinning; spinning = spinning->prev)
 380                         if (spinning->lock == lock &&
 381                             spinning->ticket == ticket) {
 382 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 383                                 ticket = spinning->irq_count
 384                                          < per_cpu(_irq_count, cpu)
 385                                          ? ticket_drop(spinning, ticket, cpu) : -2;
 386 #endif
 387                                 break;
 388                         }
 389
 390                 atomic_dec(rm_ctr);
 391                 arch_local_irq_restore(flags);
 392
 393                 if (unlikely(spinning)) {
 394 #if CONFIG_XEN_SPINLOCK_ACQUIRE_NESTING
 395                         if (!(ticket + 1))
 396                                 return;
 397                         if (ticket + 2) {
 398                                 cpu = anchor < nr_cpu_ids ? anchor : -1;
 399                                 continue;
 400                         }
 401 #endif
 402                         notify_remote_via_evtchn(per_cpu(poll_evtchn, cpu));
 403                         return;
 404                 }
 405         }
 406 }
 407 EXPORT_SYMBOL(xen_spin_kick);
 408
 409 #endif /* TICKET_SHIFT */