sched/cputime: Increment kcpustat directly on irqtime account

commit a499a5a14dbd1d0315a96fc62a8798059325e9e6 upstream.

The irqtime is accounted is nsecs and stored in
cpu_irq_time.hardirq_time and cpu_irq_time.softirq_time. Once the
accumulated amount reaches a new jiffy, this one gets accounted to the
kcpustat.

This was necessary when kcpustat was stored in cputime_t, which could at
worst have jiffies granularity. But now kcpustat is stored in nsecs
so this whole discretization game with temporary irqtime storage has
become unnecessary.

We can now directly account the irqtime to the kcpustat.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Stanislaw Gruszka <sgruszka@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Wanpeng Li <wanpeng.li@hotmail.com>
Link: http://lkml.kernel.org/r/1485832191-26889-17-git-send-email-fweisbec@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Ivan Delalande <colona@arista.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index c07f0cfea07e09b94638fc47a890180e4eceba70..2c2060c998969094ca440c5e75c7144ddc7fb681 100644 (file)
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -44,6 +44,7 @@ void disable_sched_clock_irqtime(void)
 void irqtime_account_irq(struct task_struct *curr)
 {
        struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
+       u64 *cpustat = kcpustat_this_cpu->cpustat;
        s64 delta;
        int cpu;
 
@@ -61,49 +62,35 @@ void irqtime_account_irq(struct task_struct *curr)
         * in that case, so as not to confuse scheduler with a special task
         * that do not consume any time, but still wants to run.
         */
-       if (hardirq_count())
-               irqtime->hardirq_time += delta;
-       else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
-               irqtime->softirq_time += delta;
+       if (hardirq_count()) {
+               cpustat[CPUTIME_IRQ] += delta;
+               irqtime->tick_delta += delta;
+       } else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) {
+               cpustat[CPUTIME_SOFTIRQ] += delta;
+               irqtime->tick_delta += delta;
+       }
 
        u64_stats_update_end(&irqtime->sync);
 }
 EXPORT_SYMBOL_GPL(irqtime_account_irq);
 
-static cputime_t irqtime_account_update(u64 irqtime, int idx, cputime_t maxtime)
+static cputime_t irqtime_tick_accounted(cputime_t maxtime)
 {
-       u64 *cpustat = kcpustat_this_cpu->cpustat;
-       cputime_t irq_cputime;
-
-       irq_cputime = nsecs_to_cputime64(irqtime - cpustat[idx]);
-       irq_cputime = min(irq_cputime, maxtime);
-       cpustat[idx] += cputime_to_nsecs(irq_cputime);
-
-       return irq_cputime;
-}
+       struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
+       cputime_t delta;
 
-static cputime_t irqtime_account_hi_update(cputime_t maxtime)
-{
-       return irqtime_account_update(__this_cpu_read(cpu_irqtime.hardirq_time),
-                                     CPUTIME_IRQ, maxtime);
-}
+       delta = nsecs_to_cputime(irqtime->tick_delta);
+       delta = min(delta, maxtime);
+       irqtime->tick_delta -= cputime_to_nsecs(delta);
 
-static cputime_t irqtime_account_si_update(cputime_t maxtime)
-{
-       return irqtime_account_update(__this_cpu_read(cpu_irqtime.softirq_time),
-                                     CPUTIME_SOFTIRQ, maxtime);
+       return delta;
 }
 
 #else /* CONFIG_IRQ_TIME_ACCOUNTING */
 
 #define sched_clock_irqtime    (0)
 
-static cputime_t irqtime_account_hi_update(cputime_t dummy)
-{
-       return 0;
-}
-
-static cputime_t irqtime_account_si_update(cputime_t dummy)
+static cputime_t irqtime_tick_accounted(cputime_t dummy)
 {
        return 0;
 }
@@ -290,10 +277,7 @@ static inline cputime_t account_other_time(cputime_t max)
        accounted = steal_account_process_time(max);
 
        if (accounted < max)
-               accounted += irqtime_account_hi_update(max - accounted);
-
-       if (accounted < max)
-               accounted += irqtime_account_si_update(max - accounted);
+               accounted += irqtime_tick_accounted(max - accounted);
 
        return accounted;
 }

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index f564a1d2c9d5323ff5c4a446e9560e32e42de7f0..187c556196affa52756f2828ce70d37bb79916ca 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4,6 +4,7 @@
 #include <linux/sched/rt.h>
 #include <linux/u64_stats_sync.h>
 #include <linux/sched/deadline.h>
+#include <linux/kernel_stat.h>
 #include <linux/binfmts.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
@@ -1742,8 +1743,7 @@ static inline void nohz_balance_exit_idle(unsigned int cpu) { }
 
 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
 struct irqtime {
-       u64                     hardirq_time;
-       u64                     softirq_time;
+       u64                     tick_delta;
        u64                     irq_start_time;
        struct u64_stats_sync   sync;
 };
@@ -1753,12 +1753,13 @@ DECLARE_PER_CPU(struct irqtime, cpu_irqtime);
 static inline u64 irq_time_read(int cpu)
 {
        struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu);
+       u64 *cpustat = kcpustat_cpu(cpu).cpustat;
        unsigned int seq;
        u64 total;
 
        do {
                seq = __u64_stats_fetch_begin(&irqtime->sync);
-               total = irqtime->softirq_time + irqtime->hardirq_time;
+               total = cpustat[CPUTIME_SOFTIRQ] + cpustat[CPUTIME_IRQ];
        } while (__u64_stats_fetch_retry(&irqtime->sync, seq));
 
        return total;