nohz: Fix stale jiffies update in tick_nohz_restart()

[linux-flexiantxendom0.git] / kernel / time / tick-sched.c

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 5f171f0..c923640 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -19,7 +19,6 @@
 #include <linux/percpu.h>
 #include <linux/profile.h>
 #include <linux/sched.h>
-#include <linux/tick.h>
 #include <linux/module.h>
 
 #include <asm/irq_regs.h>
@@ -140,7 +139,6 @@ static void tick_nohz_update_jiffies(ktime_t now)
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
        unsigned long flags;
 
-       cpumask_clear_cpu(cpu, nohz_cpu_mask);
        ts->idle_waketime = now;
 
        local_irq_save(flags);
@@ -154,15 +152,16 @@ static void tick_nohz_update_jiffies(ktime_t now)
  * Updates the per cpu time idle statistics counters
  */
 static void
-update_ts_time_stats(struct tick_sched *ts, ktime_t now, u64 *last_update_time)
+update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
 {
        ktime_t delta;
 
        if (ts->idle_active) {
                delta = ktime_sub(now, ts->idle_entrytime);
-               ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
-               if (nr_iowait_cpu() > 0)
+               if (nr_iowait_cpu(cpu) > 0)
                        ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
+               else
+                       ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
                ts->idle_entrytime = now;
        }
 
@@ -175,19 +174,19 @@ static void tick_nohz_stop_idle(int cpu, ktime_t now)
 {
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 
-       update_ts_time_stats(ts, now, NULL);
+       update_ts_time_stats(cpu, ts, now, NULL);
        ts->idle_active = 0;
 
        sched_clock_idle_wakeup_event(0);
 }
 
-static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
+static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
 {
        ktime_t now;
 
        now = ktime_get();
 
-       update_ts_time_stats(ts, now, NULL);
+       update_ts_time_stats(cpu, ts, now, NULL);
 
        ts->idle_entrytime = now;
        ts->idle_active = 1;
@@ -198,11 +197,11 @@ static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
 /**
  * get_cpu_idle_time_us - get the total idle time of a cpu
  * @cpu: CPU number to query
- * @last_update_time: variable to store update time in
+ * @last_update_time: variable to store update time in. Do not update
+ * counters if NULL.
  *
  * Return the cummulative idle time (since boot) for a given
- * CPU, in microseconds. The idle time returned includes
- * the iowait time (unlike what "top" and co report).
+ * CPU, in microseconds.
  *
  * This time is measured via accounting rather than sampling,
  * and is as accurate as ktime_get() is.
@@ -212,20 +211,35 @@ static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 {
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+       ktime_t now, idle;
 
        if (!tick_nohz_enabled)
                return -1;
 
-       update_ts_time_stats(ts, ktime_get(), last_update_time);
+       now = ktime_get();
+       if (last_update_time) {
+               update_ts_time_stats(cpu, ts, now, last_update_time);
+               idle = ts->idle_sleeptime;
+       } else {
+               if (ts->idle_active && !nr_iowait_cpu(cpu)) {
+                       ktime_t delta = ktime_sub(now, ts->idle_entrytime);
+
+                       idle = ktime_add(ts->idle_sleeptime, delta);
+               } else {
+                       idle = ts->idle_sleeptime;
+               }
+       }
+
+       return ktime_to_us(idle);
 
-       return ktime_to_us(ts->idle_sleeptime);
 }
 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 
-/*
+/**
  * get_cpu_iowait_time_us - get the total iowait time of a cpu
  * @cpu: CPU number to query
- * @last_update_time: variable to store update time in
+ * @last_update_time: variable to store update time in. Do not update
+ * counters if NULL.
  *
  * Return the cummulative iowait time (since boot) for a given
  * CPU, in microseconds.
@@ -238,13 +252,26 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 {
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+       ktime_t now, iowait;
 
        if (!tick_nohz_enabled)
                return -1;
 
-       update_ts_time_stats(ts, ktime_get(), last_update_time);
+       now = ktime_get();
+       if (last_update_time) {
+               update_ts_time_stats(cpu, ts, now, last_update_time);
+               iowait = ts->iowait_sleeptime;
+       } else {
+               if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
+                       ktime_t delta = ktime_sub(now, ts->idle_entrytime);
+
+                       iowait = ktime_add(ts->iowait_sleeptime, delta);
+               } else {
+                       iowait = ts->iowait_sleeptime;
+               }
+       }
 
-       return ktime_to_us(ts->iowait_sleeptime);
+       return ktime_to_us(iowait);
 }
 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 
@@ -284,7 +311,7 @@ void tick_nohz_stop_sched_tick(int inidle)
         */
        ts->inidle = 1;
 
-       now = tick_nohz_start_idle(ts);
+       now = tick_nohz_start_idle(cpu, ts);
 
        /*
         * If this cpu is offline and it is the one which updates
@@ -315,9 +342,6 @@ void tick_nohz_stop_sched_tick(int inidle)
                goto end;
        }
 
-       if (nohz_ratelimit(cpu))
-               goto end;
-
        ts->idle_calls++;
        /* Read jiffies and the time when jiffies were updated last */
        do {
@@ -393,9 +417,6 @@ void tick_nohz_stop_sched_tick(int inidle)
                else
                        expires.tv64 = KTIME_MAX;
 
-               if (delta_jiffies > 1)
-                       cpumask_set_cpu(cpu, nohz_cpu_mask);
-
                /* Skip reprogram of event if its not changed */
                if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
                        goto out;
@@ -445,7 +466,6 @@ void tick_nohz_stop_sched_tick(int inidle)
                 * softirq.
                 */
                tick_do_update_jiffies64(ktime_get());
-               cpumask_clear_cpu(cpu, nohz_cpu_mask);
        }
        raise_softirq_irqoff(TIMER_SOFTIRQ);
 out:
@@ -488,9 +508,9 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
                                hrtimer_get_expires(&ts->sched_timer), 0))
                                break;
                }
-               /* Update jiffies and reread time */
-               tick_do_update_jiffies64(now);
+               /* Reread time and update jiffies */
                now = ktime_get();
+               tick_do_update_jiffies64(now);
        }
 }
 
@@ -528,7 +548,6 @@ void tick_nohz_restart_sched_tick(void)
        /* Update jiffies first */
        select_nohz_load_balancer(0);
        tick_do_update_jiffies64(now);
-       cpumask_clear_cpu(cpu, nohz_cpu_mask);
 
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
        /*
@@ -644,9 +663,6 @@ static void tick_nohz_switch_to_nohz(void)
                next = ktime_add(next, tick_period);
        }
        local_irq_enable();
-
-       printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n",
-              smp_processor_id());
 }
 
 /*
@@ -777,7 +793,6 @@ void tick_setup_sched_timer(void)
 {
        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
        ktime_t now = ktime_get();
-       u64 offset;
 
        /*
         * Emulate tick processing via per-CPU hrtimers:
@@ -787,10 +802,6 @@ void tick_setup_sched_timer(void)
 
        /* Get the next period (per cpu) */
        hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
-       offset = ktime_to_ns(tick_period) >> 1;
-       do_div(offset, num_possible_cpus());
-       offset *= smp_processor_id();
-       hrtimer_add_expires_ns(&ts->sched_timer, offset);
 
        for (;;) {
                hrtimer_forward(&ts->sched_timer, now, tick_period);