kernel/spinlock.c

   1 /*
   2  * Copyright (2004) Linus Torvalds
   3  *
   4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
   5  *
   6  * Copyright (2004, 2005) Ingo Molnar
   7  *
   8  * This file contains the spinlock/rwlock implementations for the
   9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
  10  *
  11  * Note that some architectures have special knowledge about the
  12  * stack frames of these functions in their profile_pc. If you
  13  * change anything significant here that could change the stack
  14  * frame contact the architecture maintainers.
  15  */
  16
  17 #include <linux/linkage.h>
  18 #include <linux/preempt.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/debug_locks.h>
  22 #include <linux/module.h>
  23
  24 /*
  25  * If lockdep is enabled then we use the non-preemption spin-ops
  26  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
  27  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
  28  */
  29 #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
  30 /*
  31  * The __lock_function inlines are taken from
  32  * include/linux/spinlock_api_smp.h
  33  */
  34 #else
  35 /*
  36  * We build the __lock_function inlines here. They are too large for
  37  * inlining all over the place, but here is only one user per function
  38  * which embedds them into the calling _lock_function below.
  39  *
  40  * This could be a long-held lock. We both prepare to spin for a long
  41  * time (making _this_ CPU preemptable if possible), and we also signal
  42  * towards that other CPU that it should break the lock ASAP.
  43  */
  44 #define BUILD_LOCK_OPS(op, locktype)                                    \
  45 void __lockfunc __##op##_lock(locktype##_t *lock)                       \
  46 {                                                                       \
  47         for (;;) {                                                      \
  48                 preempt_disable();                                      \
  49                 if (likely(_raw_##op##_trylock(lock)))                  \
  50                         break;                                          \
  51                 preempt_enable();                                       \
  52                                                                         \
  53                 if (!(lock)->break_lock)                                \
  54                         (lock)->break_lock = 1;                         \
  55                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
  56                         _raw_##op##_relax(&lock->raw_lock);             \
  57         }                                                               \
  58         (lock)->break_lock = 0;                                         \
  59 }                                                                       \
  60                                                                         \
  61 unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock)      \
  62 {                                                                       \
  63         unsigned long flags;                                            \
  64                                                                         \
  65         for (;;) {                                                      \
  66                 preempt_disable();                                      \
  67                 local_irq_save(flags);                                  \
  68                 if (likely(_raw_##op##_trylock(lock)))                  \
  69                         break;                                          \
  70                 local_irq_restore(flags);                               \
  71                 preempt_enable();                                       \
  72                                                                         \
  73                 if (!(lock)->break_lock)                                \
  74                         (lock)->break_lock = 1;                         \
  75                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
  76                         _raw_##op##_relax(&lock->raw_lock);             \
  77         }                                                               \
  78         (lock)->break_lock = 0;                                         \
  79         return flags;                                                   \
  80 }                                                                       \
  81                                                                         \
  82 void __lockfunc __##op##_lock_irq(locktype##_t *lock)                   \
  83 {                                                                       \
  84         _##op##_lock_irqsave(lock);                                     \
  85 }                                                                       \
  86                                                                         \
  87 void __lockfunc __##op##_lock_bh(locktype##_t *lock)                    \
  88 {                                                                       \
  89         unsigned long flags;                                            \
  90                                                                         \
  91         /*                                                      */      \
  92         /* Careful: we must exclude softirqs too, hence the     */      \
  93         /* irq-disabling. We use the generic preemption-aware   */      \
  94         /* function:                                            */      \
  95         /**/                                                            \
  96         flags = _##op##_lock_irqsave(lock);                             \
  97         local_bh_disable();                                             \
  98         local_irq_restore(flags);                                       \
  99 }                                                                       \
 100
 101 /*
 102  * Build preemption-friendly versions of the following
 103  * lock-spinning functions:
 104  *
 105  *         __[spin|read|write]_lock()
 106  *         __[spin|read|write]_lock_irq()
 107  *         __[spin|read|write]_lock_irqsave()
 108  *         __[spin|read|write]_lock_bh()
 109  */
 110 BUILD_LOCK_OPS(spin, spinlock);
 111 BUILD_LOCK_OPS(read, rwlock);
 112 BUILD_LOCK_OPS(write, rwlock);
 113
 114 #endif
 115
 116 #ifndef CONFIG_INLINE_SPIN_TRYLOCK
 117 int __lockfunc _spin_trylock(spinlock_t *lock)
 118 {
 119         return __spin_trylock(lock);
 120 }
 121 EXPORT_SYMBOL(_spin_trylock);
 122 #endif
 123
 124 #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH
 125 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
 126 {
 127         return __spin_trylock_bh(lock);
 128 }
 129 EXPORT_SYMBOL(_spin_trylock_bh);
 130 #endif
 131
 132 #ifndef CONFIG_INLINE_SPIN_LOCK
 133 void __lockfunc _spin_lock(spinlock_t *lock)
 134 {
 135         __spin_lock(lock);
 136 }
 137 EXPORT_SYMBOL(_spin_lock);
 138 #endif
 139
 140 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
 141 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
 142 {
 143         return __spin_lock_irqsave(lock);
 144 }
 145 EXPORT_SYMBOL(_spin_lock_irqsave);
 146 #endif
 147
 148 #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
 149 void __lockfunc _spin_lock_irq(spinlock_t *lock)
 150 {
 151         __spin_lock_irq(lock);
 152 }
 153 EXPORT_SYMBOL(_spin_lock_irq);
 154 #endif
 155
 156 #ifndef CONFIG_INLINE_SPIN_LOCK_BH
 157 void __lockfunc _spin_lock_bh(spinlock_t *lock)
 158 {
 159         __spin_lock_bh(lock);
 160 }
 161 EXPORT_SYMBOL(_spin_lock_bh);
 162 #endif
 163
 164 #ifndef CONFIG_INLINE_SPIN_UNLOCK
 165 void __lockfunc _spin_unlock(spinlock_t *lock)
 166 {
 167         __spin_unlock(lock);
 168 }
 169 EXPORT_SYMBOL(_spin_unlock);
 170 #endif
 171
 172 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
 173 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
 174 {
 175         __spin_unlock_irqrestore(lock, flags);
 176 }
 177 EXPORT_SYMBOL(_spin_unlock_irqrestore);
 178 #endif
 179
 180 #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
 181 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
 182 {
 183         __spin_unlock_irq(lock);
 184 }
 185 EXPORT_SYMBOL(_spin_unlock_irq);
 186 #endif
 187
 188 #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
 189 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
 190 {
 191         __spin_unlock_bh(lock);
 192 }
 193 EXPORT_SYMBOL(_spin_unlock_bh);
 194 #endif
 195
 196 #ifndef CONFIG_INLINE_READ_TRYLOCK
 197 int __lockfunc _read_trylock(rwlock_t *lock)
 198 {
 199         return __read_trylock(lock);
 200 }
 201 EXPORT_SYMBOL(_read_trylock);
 202 #endif
 203
 204 #ifndef CONFIG_INLINE_READ_LOCK
 205 void __lockfunc _read_lock(rwlock_t *lock)
 206 {
 207         __read_lock(lock);
 208 }
 209 EXPORT_SYMBOL(_read_lock);
 210 #endif
 211
 212 #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE
 213 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
 214 {
 215         return __read_lock_irqsave(lock);
 216 }
 217 EXPORT_SYMBOL(_read_lock_irqsave);
 218 #endif
 219
 220 #ifndef CONFIG_INLINE_READ_LOCK_IRQ
 221 void __lockfunc _read_lock_irq(rwlock_t *lock)
 222 {
 223         __read_lock_irq(lock);
 224 }
 225 EXPORT_SYMBOL(_read_lock_irq);
 226 #endif
 227
 228 #ifndef CONFIG_INLINE_READ_LOCK_BH
 229 void __lockfunc _read_lock_bh(rwlock_t *lock)
 230 {
 231         __read_lock_bh(lock);
 232 }
 233 EXPORT_SYMBOL(_read_lock_bh);
 234 #endif
 235
 236 #ifndef CONFIG_INLINE_READ_UNLOCK
 237 void __lockfunc _read_unlock(rwlock_t *lock)
 238 {
 239         __read_unlock(lock);
 240 }
 241 EXPORT_SYMBOL(_read_unlock);
 242 #endif
 243
 244 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE
 245 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 246 {
 247         __read_unlock_irqrestore(lock, flags);
 248 }
 249 EXPORT_SYMBOL(_read_unlock_irqrestore);
 250 #endif
 251
 252 #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ
 253 void __lockfunc _read_unlock_irq(rwlock_t *lock)
 254 {
 255         __read_unlock_irq(lock);
 256 }
 257 EXPORT_SYMBOL(_read_unlock_irq);
 258 #endif
 259
 260 #ifndef CONFIG_INLINE_READ_UNLOCK_BH
 261 void __lockfunc _read_unlock_bh(rwlock_t *lock)
 262 {
 263         __read_unlock_bh(lock);
 264 }
 265 EXPORT_SYMBOL(_read_unlock_bh);
 266 #endif
 267
 268 #ifndef CONFIG_INLINE_WRITE_TRYLOCK
 269 int __lockfunc _write_trylock(rwlock_t *lock)
 270 {
 271         return __write_trylock(lock);
 272 }
 273 EXPORT_SYMBOL(_write_trylock);
 274 #endif
 275
 276 #ifndef CONFIG_INLINE_WRITE_LOCK
 277 void __lockfunc _write_lock(rwlock_t *lock)
 278 {
 279         __write_lock(lock);
 280 }
 281 EXPORT_SYMBOL(_write_lock);
 282 #endif
 283
 284 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE
 285 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
 286 {
 287         return __write_lock_irqsave(lock);
 288 }
 289 EXPORT_SYMBOL(_write_lock_irqsave);
 290 #endif
 291
 292 #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ
 293 void __lockfunc _write_lock_irq(rwlock_t *lock)
 294 {
 295         __write_lock_irq(lock);
 296 }
 297 EXPORT_SYMBOL(_write_lock_irq);
 298 #endif
 299
 300 #ifndef CONFIG_INLINE_WRITE_LOCK_BH
 301 void __lockfunc _write_lock_bh(rwlock_t *lock)
 302 {
 303         __write_lock_bh(lock);
 304 }
 305 EXPORT_SYMBOL(_write_lock_bh);
 306 #endif
 307
 308 #ifndef CONFIG_INLINE_WRITE_UNLOCK
 309 void __lockfunc _write_unlock(rwlock_t *lock)
 310 {
 311         __write_unlock(lock);
 312 }
 313 EXPORT_SYMBOL(_write_unlock);
 314 #endif
 315
 316 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE
 317 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 318 {
 319         __write_unlock_irqrestore(lock, flags);
 320 }
 321 EXPORT_SYMBOL(_write_unlock_irqrestore);
 322 #endif
 323
 324 #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ
 325 void __lockfunc _write_unlock_irq(rwlock_t *lock)
 326 {
 327         __write_unlock_irq(lock);
 328 }
 329 EXPORT_SYMBOL(_write_unlock_irq);
 330 #endif
 331
 332 #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH
 333 void __lockfunc _write_unlock_bh(rwlock_t *lock)
 334 {
 335         __write_unlock_bh(lock);
 336 }
 337 EXPORT_SYMBOL(_write_unlock_bh);
 338 #endif
 339
 340 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 341
 342 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
 343 {
 344         preempt_disable();
 345         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 346         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 347 }
 348 EXPORT_SYMBOL(_spin_lock_nested);
 349
 350 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock,
 351                                                    int subclass)
 352 {
 353         unsigned long flags;
 354
 355         local_irq_save(flags);
 356         preempt_disable();
 357         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
 358         LOCK_CONTENDED_FLAGS(lock, _raw_spin_trylock, _raw_spin_lock,
 359                                 _raw_spin_lock_flags, &flags);
 360         return flags;
 361 }
 362 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
 363
 364 void __lockfunc _spin_lock_nest_lock(spinlock_t *lock,
 365                                      struct lockdep_map *nest_lock)
 366 {
 367         preempt_disable();
 368         spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
 369         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
 370 }
 371 EXPORT_SYMBOL(_spin_lock_nest_lock);
 372
 373 #endif
 374
 375 notrace int in_lock_functions(unsigned long addr)
 376 {
 377         /* Linker adds these: start and end of __lockfunc functions */
 378         extern char __lock_text_start[], __lock_text_end[];
 379
 380         return addr >= (unsigned long)__lock_text_start
 381         && addr < (unsigned long)__lock_text_end;
 382 }
 383 EXPORT_SYMBOL(in_lock_functions);