kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel_stat.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/notifier.h>
  13 #include <linux/percpu.h>
  14 #include <linux/init.h>
  15 #include <linux/mm.h>
  16 #include <linux/notifier.h>
  17
  18 /*
  19    - No shared variables, all the data are CPU local.
  20    - If a softirq needs serialization, let it serialize itself
  21      by its own spinlocks.
  22    - Even if softirq is serialized, only local cpu is marked for
  23      execution. Hence, we get something sort of weak cpu binding.
  24      Though it is still not clear, will it result in better locality
  25      or will not.
  26
  27    Examples:
  28    - NET RX softirq. It is multithreaded and does not require
  29      any global serialization.
  30    - NET TX softirq. It kicks software netdevice queues, hence
  31      it is logically serialized per device, but this serialization
  32      is invisible to common code.
  33    - Tasklets: serialized wrt itself.
  34  */
  35
  36 /* No separate irq_stat for ia64, it is part of PSA */
  37 #if !defined(CONFIG_IA64)
  38 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  39 #endif /* CONFIG_IA64 */
  40
  41 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
  42
  43 /*
  44  * we cannot loop indefinitely here to avoid userspace starvation,
  45  * but we also don't want to introduce a worst case 1/HZ latency
  46  * to the pending events, so lets the scheduler to balance
  47  * the softirq load for us.
  48  */
  49 static inline void wakeup_softirqd(unsigned cpu)
  50 {
  51         struct task_struct * tsk = ksoftirqd_task(cpu);
  52
  53         if (tsk && tsk->state != TASK_RUNNING)
  54                 wake_up_process(tsk);
  55 }
  56
  57 asmlinkage void do_softirq(void)
  58 {
  59         __u32 pending;
  60         unsigned long flags;
  61         __u32 mask;
  62
  63         if (in_interrupt())
  64                 return;
  65
  66         local_irq_save(flags);
  67
  68         pending = local_softirq_pending();
  69
  70         if (pending) {
  71                 struct softirq_action *h;
  72
  73                 mask = ~pending;
  74                 local_bh_disable();
  75 restart:
  76                 /* Reset the pending bitmask before enabling irqs */
  77                 local_softirq_pending() = 0;
  78
  79                 local_irq_enable();
  80
  81                 h = softirq_vec;
  82
  83                 do {
  84                         if (pending & 1)
  85                                 h->action(h);
  86                         h++;
  87                         pending >>= 1;
  88                 } while (pending);
  89
  90                 local_irq_disable();
  91
  92                 pending = local_softirq_pending();
  93                 if (pending & mask) {
  94                         mask &= ~pending;
  95                         goto restart;
  96                 }
  97                 if (pending)
  98                         wakeup_softirqd(smp_processor_id());
  99                 __local_bh_enable();
 100         }
 101
 102         local_irq_restore(flags);
 103 }
 104
 105 void local_bh_enable(void)
 106 {
 107         __local_bh_enable();
 108         WARN_ON(irqs_disabled());
 109         if (unlikely(!in_interrupt() &&
 110                      local_softirq_pending()))
 111                 invoke_softirq();
 112         preempt_check_resched();
 113 }
 114 EXPORT_SYMBOL(local_bh_enable);
 115
 116 /*
 117  * This function must run with irqs disabled!
 118  */
 119 inline void cpu_raise_softirq(unsigned int cpu, unsigned int nr)
 120 {
 121         __cpu_raise_softirq(cpu, nr);
 122
 123         /*
 124          * If we're in an interrupt or softirq, we're done
 125          * (this also catches softirq-disabled code). We will
 126          * actually run the softirq once we return from
 127          * the irq or softirq.
 128          *
 129          * Otherwise we wake up ksoftirqd to make sure we
 130          * schedule the softirq soon.
 131          */
 132         if (!in_interrupt())
 133                 wakeup_softirqd(cpu);
 134 }
 135
 136 void raise_softirq(unsigned int nr)
 137 {
 138         unsigned long flags;
 139
 140         local_irq_save(flags);
 141         cpu_raise_softirq(smp_processor_id(), nr);
 142         local_irq_restore(flags);
 143 }
 144
 145 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
 146 {
 147         softirq_vec[nr].data = data;
 148         softirq_vec[nr].action = action;
 149 }
 150
 151
 152 /* Tasklets */
 153 struct tasklet_head
 154 {
 155         struct tasklet_struct *list;
 156 };
 157
 158 /* Some compilers disobey section attribute on statics when not
 159    initialized -- RR */
 160 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
 161 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 162
 163 void __tasklet_schedule(struct tasklet_struct *t)
 164 {
 165         unsigned long flags;
 166
 167         local_irq_save(flags);
 168         t->next = __get_cpu_var(tasklet_vec).list;
 169         __get_cpu_var(tasklet_vec).list = t;
 170         cpu_raise_softirq(smp_processor_id(), TASKLET_SOFTIRQ);
 171         local_irq_restore(flags);
 172 }
 173
 174 void __tasklet_hi_schedule(struct tasklet_struct *t)
 175 {
 176         unsigned long flags;
 177
 178         local_irq_save(flags);
 179         t->next = __get_cpu_var(tasklet_hi_vec).list;
 180         __get_cpu_var(tasklet_hi_vec).list = t;
 181         cpu_raise_softirq(smp_processor_id(), HI_SOFTIRQ);
 182         local_irq_restore(flags);
 183 }
 184
 185 static void tasklet_action(struct softirq_action *a)
 186 {
 187         struct tasklet_struct *list;
 188
 189         local_irq_disable();
 190         list = __get_cpu_var(tasklet_vec).list;
 191         __get_cpu_var(tasklet_vec).list = NULL;
 192         local_irq_enable();
 193
 194         while (list) {
 195                 struct tasklet_struct *t = list;
 196
 197                 list = list->next;
 198
 199                 if (tasklet_trylock(t)) {
 200                         if (!atomic_read(&t->count)) {
 201                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 202                                         BUG();
 203                                 t->func(t->data);
 204                                 tasklet_unlock(t);
 205                                 continue;
 206                         }
 207                         tasklet_unlock(t);
 208                 }
 209
 210                 local_irq_disable();
 211                 t->next = __get_cpu_var(tasklet_vec).list;
 212                 __get_cpu_var(tasklet_vec).list = t;
 213                 __cpu_raise_softirq(smp_processor_id(), TASKLET_SOFTIRQ);
 214                 local_irq_enable();
 215         }
 216 }
 217
 218 static void tasklet_hi_action(struct softirq_action *a)
 219 {
 220         struct tasklet_struct *list;
 221
 222         local_irq_disable();
 223         list = __get_cpu_var(tasklet_hi_vec).list;
 224         __get_cpu_var(tasklet_hi_vec).list = NULL;
 225         local_irq_enable();
 226
 227         while (list) {
 228                 struct tasklet_struct *t = list;
 229
 230                 list = list->next;
 231
 232                 if (tasklet_trylock(t)) {
 233                         if (!atomic_read(&t->count)) {
 234                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 235                                         BUG();
 236                                 t->func(t->data);
 237                                 tasklet_unlock(t);
 238                                 continue;
 239                         }
 240                         tasklet_unlock(t);
 241                 }
 242
 243                 local_irq_disable();
 244                 t->next = __get_cpu_var(tasklet_hi_vec).list;
 245                 __get_cpu_var(tasklet_hi_vec).list = t;
 246                 __cpu_raise_softirq(smp_processor_id(), HI_SOFTIRQ);
 247                 local_irq_enable();
 248         }
 249 }
 250
 251
 252 void tasklet_init(struct tasklet_struct *t,
 253                   void (*func)(unsigned long), unsigned long data)
 254 {
 255         t->next = NULL;
 256         t->state = 0;
 257         atomic_set(&t->count, 0);
 258         t->func = func;
 259         t->data = data;
 260 }
 261
 262 void tasklet_kill(struct tasklet_struct *t)
 263 {
 264         if (in_interrupt())
 265                 printk("Attempt to kill tasklet from interrupt\n");
 266
 267         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 268                 do
 269                         yield();
 270                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
 271         }
 272         tasklet_unlock_wait(t);
 273         clear_bit(TASKLET_STATE_SCHED, &t->state);
 274 }
 275
 276
 277 static void tasklet_init_cpu(int cpu)
 278 {
 279         per_cpu(tasklet_vec, cpu).list = NULL;
 280         per_cpu(tasklet_hi_vec, cpu).list = NULL;
 281 }
 282
 283 static int tasklet_cpu_notify(struct notifier_block *self,
 284                                 unsigned long action, void *hcpu)
 285 {
 286         long cpu = (long)hcpu;
 287         switch(action) {
 288         case CPU_UP_PREPARE:
 289                 tasklet_init_cpu(cpu);
 290                 break;
 291         default:
 292                 break;
 293         }
 294         return 0;
 295 }
 296
 297 static struct notifier_block tasklet_nb = {
 298         .notifier_call  = tasklet_cpu_notify,
 299         .next           = NULL,
 300 };
 301
 302 void __init softirq_init(void)
 303 {
 304         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
 305         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 306         tasklet_cpu_notify(&tasklet_nb, (unsigned long)CPU_UP_PREPARE,
 307                                 (void *)(long)smp_processor_id());
 308         register_cpu_notifier(&tasklet_nb);
 309 }
 310
 311 static int ksoftirqd(void * __bind_cpu)
 312 {
 313         int cpu = (int) (long) __bind_cpu;
 314
 315         daemonize("ksoftirqd/%d", cpu);
 316         set_user_nice(current, 19);
 317         current->flags |= PF_IOTHREAD;
 318
 319         /* Migrate to the right CPU */
 320         set_cpus_allowed(current, 1UL << cpu);
 321         if (smp_processor_id() != cpu)
 322                 BUG();
 323
 324         __set_current_state(TASK_INTERRUPTIBLE);
 325         mb();
 326
 327         local_ksoftirqd_task() = current;
 328
 329         for (;;) {
 330                 if (!local_softirq_pending())
 331                         schedule();
 332
 333                 __set_current_state(TASK_RUNNING);
 334
 335                 while (local_softirq_pending()) {
 336                         do_softirq();
 337                         cond_resched();
 338                 }
 339
 340                 __set_current_state(TASK_INTERRUPTIBLE);
 341         }
 342 }
 343
 344 static int __devinit cpu_callback(struct notifier_block *nfb,
 345                                   unsigned long action,
 346                                   void *hcpu)
 347 {
 348         int hotcpu = (unsigned long)hcpu;
 349
 350         if (action == CPU_ONLINE) {
 351                 if (kernel_thread(ksoftirqd, hcpu, CLONE_KERNEL) < 0) {
 352                         printk("ksoftirqd for %i failed\n", hotcpu);
 353                         return NOTIFY_BAD;
 354                 }
 355
 356                 while (!ksoftirqd_task(hotcpu))
 357                         yield();
 358         }
 359         return NOTIFY_OK;
 360 }
 361
 362 static struct notifier_block __devinitdata cpu_nfb = {
 363         .notifier_call = cpu_callback
 364 };
 365
 366 __init int spawn_ksoftirqd(void)
 367 {
 368         cpu_callback(&cpu_nfb, CPU_ONLINE, (void *)(long)smp_processor_id());
 369         register_cpu_notifier(&cpu_nfb);
 370         return 0;
 371 }