include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit...
[linux-3.10.git] / arch / x86 / xen / spinlock.c
1 /*
2  * Split spinlock implementation out into its own file, so it can be
3  * compiled in a FTRACE-compatible way.
4  */
5 #include <linux/kernel_stat.h>
6 #include <linux/spinlock.h>
7 #include <linux/debugfs.h>
8 #include <linux/log2.h>
9 #include <linux/gfp.h>
10
11 #include <asm/paravirt.h>
12
13 #include <xen/interface/xen.h>
14 #include <xen/events.h>
15
16 #include "xen-ops.h"
17 #include "debugfs.h"
18
19 #ifdef CONFIG_XEN_DEBUG_FS
20 static struct xen_spinlock_stats
21 {
22         u64 taken;
23         u32 taken_slow;
24         u32 taken_slow_nested;
25         u32 taken_slow_pickup;
26         u32 taken_slow_spurious;
27         u32 taken_slow_irqenable;
28
29         u64 released;
30         u32 released_slow;
31         u32 released_slow_kicked;
32
33 #define HISTO_BUCKETS   30
34         u32 histo_spin_total[HISTO_BUCKETS+1];
35         u32 histo_spin_spinning[HISTO_BUCKETS+1];
36         u32 histo_spin_blocked[HISTO_BUCKETS+1];
37
38         u64 time_total;
39         u64 time_spinning;
40         u64 time_blocked;
41 } spinlock_stats;
42
43 static u8 zero_stats;
44
45 static unsigned lock_timeout = 1 << 10;
46 #define TIMEOUT lock_timeout
47
48 static inline void check_zero(void)
49 {
50         if (unlikely(zero_stats)) {
51                 memset(&spinlock_stats, 0, sizeof(spinlock_stats));
52                 zero_stats = 0;
53         }
54 }
55
56 #define ADD_STATS(elem, val)                    \
57         do { check_zero(); spinlock_stats.elem += (val); } while(0)
58
59 static inline u64 spin_time_start(void)
60 {
61         return xen_clocksource_read();
62 }
63
64 static void __spin_time_accum(u64 delta, u32 *array)
65 {
66         unsigned index = ilog2(delta);
67
68         check_zero();
69
70         if (index < HISTO_BUCKETS)
71                 array[index]++;
72         else
73                 array[HISTO_BUCKETS]++;
74 }
75
76 static inline void spin_time_accum_spinning(u64 start)
77 {
78         u32 delta = xen_clocksource_read() - start;
79
80         __spin_time_accum(delta, spinlock_stats.histo_spin_spinning);
81         spinlock_stats.time_spinning += delta;
82 }
83
84 static inline void spin_time_accum_total(u64 start)
85 {
86         u32 delta = xen_clocksource_read() - start;
87
88         __spin_time_accum(delta, spinlock_stats.histo_spin_total);
89         spinlock_stats.time_total += delta;
90 }
91
92 static inline void spin_time_accum_blocked(u64 start)
93 {
94         u32 delta = xen_clocksource_read() - start;
95
96         __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
97         spinlock_stats.time_blocked += delta;
98 }
99 #else  /* !CONFIG_XEN_DEBUG_FS */
100 #define TIMEOUT                 (1 << 10)
101 #define ADD_STATS(elem, val)    do { (void)(val); } while(0)
102
103 static inline u64 spin_time_start(void)
104 {
105         return 0;
106 }
107
108 static inline void spin_time_accum_total(u64 start)
109 {
110 }
111 static inline void spin_time_accum_spinning(u64 start)
112 {
113 }
114 static inline void spin_time_accum_blocked(u64 start)
115 {
116 }
117 #endif  /* CONFIG_XEN_DEBUG_FS */
118
119 struct xen_spinlock {
120         unsigned char lock;             /* 0 -> free; 1 -> locked */
121         unsigned short spinners;        /* count of waiting cpus */
122 };
123
124 static int xen_spin_is_locked(struct arch_spinlock *lock)
125 {
126         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
127
128         return xl->lock != 0;
129 }
130
131 static int xen_spin_is_contended(struct arch_spinlock *lock)
132 {
133         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
134
135         /* Not strictly true; this is only the count of contended
136            lock-takers entering the slow path. */
137         return xl->spinners != 0;
138 }
139
140 static int xen_spin_trylock(struct arch_spinlock *lock)
141 {
142         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
143         u8 old = 1;
144
145         asm("xchgb %b0,%1"
146             : "+q" (old), "+m" (xl->lock) : : "memory");
147
148         return old == 0;
149 }
150
151 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
152 static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners);
153
154 /*
155  * Mark a cpu as interested in a lock.  Returns the CPU's previous
156  * lock of interest, in case we got preempted by an interrupt.
157  */
158 static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl)
159 {
160         struct xen_spinlock *prev;
161
162         prev = __get_cpu_var(lock_spinners);
163         __get_cpu_var(lock_spinners) = xl;
164
165         wmb();                  /* set lock of interest before count */
166
167         asm(LOCK_PREFIX " incw %0"
168             : "+m" (xl->spinners) : : "memory");
169
170         return prev;
171 }
172
173 /*
174  * Mark a cpu as no longer interested in a lock.  Restores previous
175  * lock of interest (NULL for none).
176  */
177 static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev)
178 {
179         asm(LOCK_PREFIX " decw %0"
180             : "+m" (xl->spinners) : : "memory");
181         wmb();                  /* decrement count before restoring lock */
182         __get_cpu_var(lock_spinners) = prev;
183 }
184
185 static noinline int xen_spin_lock_slow(struct arch_spinlock *lock, bool irq_enable)
186 {
187         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
188         struct xen_spinlock *prev;
189         int irq = __get_cpu_var(lock_kicker_irq);
190         int ret;
191         u64 start;
192
193         /* If kicker interrupts not initialized yet, just spin */
194         if (irq == -1)
195                 return 0;
196
197         start = spin_time_start();
198
199         /* announce we're spinning */
200         prev = spinning_lock(xl);
201
202         ADD_STATS(taken_slow, 1);
203         ADD_STATS(taken_slow_nested, prev != NULL);
204
205         do {
206                 unsigned long flags;
207
208                 /* clear pending */
209                 xen_clear_irq_pending(irq);
210
211                 /* check again make sure it didn't become free while
212                    we weren't looking  */
213                 ret = xen_spin_trylock(lock);
214                 if (ret) {
215                         ADD_STATS(taken_slow_pickup, 1);
216
217                         /*
218                          * If we interrupted another spinlock while it
219                          * was blocking, make sure it doesn't block
220                          * without rechecking the lock.
221                          */
222                         if (prev != NULL)
223                                 xen_set_irq_pending(irq);
224                         goto out;
225                 }
226
227                 flags = __raw_local_save_flags();
228                 if (irq_enable) {
229                         ADD_STATS(taken_slow_irqenable, 1);
230                         raw_local_irq_enable();
231                 }
232
233                 /*
234                  * Block until irq becomes pending.  If we're
235                  * interrupted at this point (after the trylock but
236                  * before entering the block), then the nested lock
237                  * handler guarantees that the irq will be left
238                  * pending if there's any chance the lock became free;
239                  * xen_poll_irq() returns immediately if the irq is
240                  * pending.
241                  */
242                 xen_poll_irq(irq);
243
244                 raw_local_irq_restore(flags);
245
246                 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
247         } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */
248
249         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
250
251 out:
252         unspinning_lock(xl, prev);
253         spin_time_accum_blocked(start);
254
255         return ret;
256 }
257
258 static inline void __xen_spin_lock(struct arch_spinlock *lock, bool irq_enable)
259 {
260         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
261         unsigned timeout;
262         u8 oldval;
263         u64 start_spin;
264
265         ADD_STATS(taken, 1);
266
267         start_spin = spin_time_start();
268
269         do {
270                 u64 start_spin_fast = spin_time_start();
271
272                 timeout = TIMEOUT;
273
274                 asm("1: xchgb %1,%0\n"
275                     "   testb %1,%1\n"
276                     "   jz 3f\n"
277                     "2: rep;nop\n"
278                     "   cmpb $0,%0\n"
279                     "   je 1b\n"
280                     "   dec %2\n"
281                     "   jnz 2b\n"
282                     "3:\n"
283                     : "+m" (xl->lock), "=q" (oldval), "+r" (timeout)
284                     : "1" (1)
285                     : "memory");
286
287                 spin_time_accum_spinning(start_spin_fast);
288
289         } while (unlikely(oldval != 0 &&
290                           (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable))));
291
292         spin_time_accum_total(start_spin);
293 }
294
295 static void xen_spin_lock(struct arch_spinlock *lock)
296 {
297         __xen_spin_lock(lock, false);
298 }
299
300 static void xen_spin_lock_flags(struct arch_spinlock *lock, unsigned long flags)
301 {
302         __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags));
303 }
304
305 static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
306 {
307         int cpu;
308
309         ADD_STATS(released_slow, 1);
310
311         for_each_online_cpu(cpu) {
312                 /* XXX should mix up next cpu selection */
313                 if (per_cpu(lock_spinners, cpu) == xl) {
314                         ADD_STATS(released_slow_kicked, 1);
315                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
316                         break;
317                 }
318         }
319 }
320
321 static void xen_spin_unlock(struct arch_spinlock *lock)
322 {
323         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
324
325         ADD_STATS(released, 1);
326
327         smp_wmb();              /* make sure no writes get moved after unlock */
328         xl->lock = 0;           /* release lock */
329
330         /*
331          * Make sure unlock happens before checking for waiting
332          * spinners.  We need a strong barrier to enforce the
333          * write-read ordering to different memory locations, as the
334          * CPU makes no implied guarantees about their ordering.
335          */
336         mb();
337
338         if (unlikely(xl->spinners))
339                 xen_spin_unlock_slow(xl);
340 }
341
342 static irqreturn_t dummy_handler(int irq, void *dev_id)
343 {
344         BUG();
345         return IRQ_HANDLED;
346 }
347
348 void __cpuinit xen_init_lock_cpu(int cpu)
349 {
350         int irq;
351         const char *name;
352
353         name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
354         irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
355                                      cpu,
356                                      dummy_handler,
357                                      IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
358                                      name,
359                                      NULL);
360
361         if (irq >= 0) {
362                 disable_irq(irq); /* make sure it's never delivered */
363                 per_cpu(lock_kicker_irq, cpu) = irq;
364         }
365
366         printk("cpu %d spinlock event irq %d\n", cpu, irq);
367 }
368
369 void xen_uninit_lock_cpu(int cpu)
370 {
371         unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
372 }
373
374 void __init xen_init_spinlocks(void)
375 {
376         pv_lock_ops.spin_is_locked = xen_spin_is_locked;
377         pv_lock_ops.spin_is_contended = xen_spin_is_contended;
378         pv_lock_ops.spin_lock = xen_spin_lock;
379         pv_lock_ops.spin_lock_flags = xen_spin_lock_flags;
380         pv_lock_ops.spin_trylock = xen_spin_trylock;
381         pv_lock_ops.spin_unlock = xen_spin_unlock;
382 }
383
384 #ifdef CONFIG_XEN_DEBUG_FS
385
386 static struct dentry *d_spin_debug;
387
388 static int __init xen_spinlock_debugfs(void)
389 {
390         struct dentry *d_xen = xen_init_debugfs();
391
392         if (d_xen == NULL)
393                 return -ENOMEM;
394
395         d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
396
397         debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
398
399         debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout);
400
401         debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken);
402         debugfs_create_u32("taken_slow", 0444, d_spin_debug,
403                            &spinlock_stats.taken_slow);
404         debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug,
405                            &spinlock_stats.taken_slow_nested);
406         debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
407                            &spinlock_stats.taken_slow_pickup);
408         debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
409                            &spinlock_stats.taken_slow_spurious);
410         debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug,
411                            &spinlock_stats.taken_slow_irqenable);
412
413         debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released);
414         debugfs_create_u32("released_slow", 0444, d_spin_debug,
415                            &spinlock_stats.released_slow);
416         debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
417                            &spinlock_stats.released_slow_kicked);
418
419         debugfs_create_u64("time_spinning", 0444, d_spin_debug,
420                            &spinlock_stats.time_spinning);
421         debugfs_create_u64("time_blocked", 0444, d_spin_debug,
422                            &spinlock_stats.time_blocked);
423         debugfs_create_u64("time_total", 0444, d_spin_debug,
424                            &spinlock_stats.time_total);
425
426         xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug,
427                                      spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1);
428         xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug,
429                                      spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1);
430         xen_debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
431                                      spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
432
433         return 0;
434 }
435 fs_initcall(xen_spinlock_debugfs);
436
437 #endif  /* CONFIG_XEN_DEBUG_FS */