sparseirq: remove some debug print out
[linux-2.6.git] / kernel / rtmutex.c
1 /*
2  * RT-Mutexes: simple blocking mutual exclusion locks with PI support
3  *
4  * started by Ingo Molnar and Thomas Gleixner.
5  *
6  *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7  *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
8  *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
9  *  Copyright (C) 2006 Esben Nielsen
10  *
11  *  See Documentation/rt-mutex-design.txt for details.
12  */
13 #include <linux/spinlock.h>
14 #include <linux/module.h>
15 #include <linux/sched.h>
16 #include <linux/timer.h>
17
18 #include "rtmutex_common.h"
19
20 /*
21  * lock->owner state tracking:
22  *
23  * lock->owner holds the task_struct pointer of the owner. Bit 0 and 1
24  * are used to keep track of the "owner is pending" and "lock has
25  * waiters" state.
26  *
27  * owner        bit1    bit0
28  * NULL         0       0       lock is free (fast acquire possible)
29  * NULL         0       1       invalid state
30  * NULL         1       0       Transitional State*
31  * NULL         1       1       invalid state
32  * taskpointer  0       0       lock is held (fast release possible)
33  * taskpointer  0       1       task is pending owner
34  * taskpointer  1       0       lock is held and has waiters
35  * taskpointer  1       1       task is pending owner and lock has more waiters
36  *
37  * Pending ownership is assigned to the top (highest priority)
38  * waiter of the lock, when the lock is released. The thread is woken
39  * up and can now take the lock. Until the lock is taken (bit 0
40  * cleared) a competing higher priority thread can steal the lock
41  * which puts the woken up thread back on the waiters list.
42  *
43  * The fast atomic compare exchange based acquire and release is only
44  * possible when bit 0 and 1 of lock->owner are 0.
45  *
46  * (*) There's a small time where the owner can be NULL and the
47  * "lock has waiters" bit is set.  This can happen when grabbing the lock.
48  * To prevent a cmpxchg of the owner releasing the lock, we need to set this
49  * bit before looking at the lock, hence the reason this is a transitional
50  * state.
51  */
52
53 static void
54 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner,
55                    unsigned long mask)
56 {
57         unsigned long val = (unsigned long)owner | mask;
58
59         if (rt_mutex_has_waiters(lock))
60                 val |= RT_MUTEX_HAS_WAITERS;
61
62         lock->owner = (struct task_struct *)val;
63 }
64
65 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
66 {
67         lock->owner = (struct task_struct *)
68                         ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
69 }
70
71 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
72 {
73         if (!rt_mutex_has_waiters(lock))
74                 clear_rt_mutex_waiters(lock);
75 }
76
77 /*
78  * We can speed up the acquire/release, if the architecture
79  * supports cmpxchg and if there's no debugging state to be set up
80  */
81 #if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
82 # define rt_mutex_cmpxchg(l,c,n)        (cmpxchg(&l->owner, c, n) == c)
83 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
84 {
85         unsigned long owner, *p = (unsigned long *) &lock->owner;
86
87         do {
88                 owner = *p;
89         } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
90 }
91 #else
92 # define rt_mutex_cmpxchg(l,c,n)        (0)
93 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
94 {
95         lock->owner = (struct task_struct *)
96                         ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
97 }
98 #endif
99
100 /*
101  * Calculate task priority from the waiter list priority
102  *
103  * Return task->normal_prio when the waiter list is empty or when
104  * the waiter is not allowed to do priority boosting
105  */
106 int rt_mutex_getprio(struct task_struct *task)
107 {
108         if (likely(!task_has_pi_waiters(task)))
109                 return task->normal_prio;
110
111         return min(task_top_pi_waiter(task)->pi_list_entry.prio,
112                    task->normal_prio);
113 }
114
115 /*
116  * Adjust the priority of a task, after its pi_waiters got modified.
117  *
118  * This can be both boosting and unboosting. task->pi_lock must be held.
119  */
120 static void __rt_mutex_adjust_prio(struct task_struct *task)
121 {
122         int prio = rt_mutex_getprio(task);
123
124         if (task->prio != prio)
125                 rt_mutex_setprio(task, prio);
126 }
127
128 /*
129  * Adjust task priority (undo boosting). Called from the exit path of
130  * rt_mutex_slowunlock() and rt_mutex_slowlock().
131  *
132  * (Note: We do this outside of the protection of lock->wait_lock to
133  * allow the lock to be taken while or before we readjust the priority
134  * of task. We do not use the spin_xx_mutex() variants here as we are
135  * outside of the debug path.)
136  */
137 static void rt_mutex_adjust_prio(struct task_struct *task)
138 {
139         unsigned long flags;
140
141         spin_lock_irqsave(&task->pi_lock, flags);
142         __rt_mutex_adjust_prio(task);
143         spin_unlock_irqrestore(&task->pi_lock, flags);
144 }
145
146 /*
147  * Max number of times we'll walk the boosting chain:
148  */
149 int max_lock_depth = 1024;
150
151 /*
152  * Adjust the priority chain. Also used for deadlock detection.
153  * Decreases task's usage by one - may thus free the task.
154  * Returns 0 or -EDEADLK.
155  */
156 static int rt_mutex_adjust_prio_chain(struct task_struct *task,
157                                       int deadlock_detect,
158                                       struct rt_mutex *orig_lock,
159                                       struct rt_mutex_waiter *orig_waiter,
160                                       struct task_struct *top_task)
161 {
162         struct rt_mutex *lock;
163         struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
164         int detect_deadlock, ret = 0, depth = 0;
165         unsigned long flags;
166
167         detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
168                                                          deadlock_detect);
169
170         /*
171          * The (de)boosting is a step by step approach with a lot of
172          * pitfalls. We want this to be preemptible and we want hold a
173          * maximum of two locks per step. So we have to check
174          * carefully whether things change under us.
175          */
176  again:
177         if (++depth > max_lock_depth) {
178                 static int prev_max;
179
180                 /*
181                  * Print this only once. If the admin changes the limit,
182                  * print a new message when reaching the limit again.
183                  */
184                 if (prev_max != max_lock_depth) {
185                         prev_max = max_lock_depth;
186                         printk(KERN_WARNING "Maximum lock depth %d reached "
187                                "task: %s (%d)\n", max_lock_depth,
188                                top_task->comm, task_pid_nr(top_task));
189                 }
190                 put_task_struct(task);
191
192                 return deadlock_detect ? -EDEADLK : 0;
193         }
194  retry:
195         /*
196          * Task can not go away as we did a get_task() before !
197          */
198         spin_lock_irqsave(&task->pi_lock, flags);
199
200         waiter = task->pi_blocked_on;
201         /*
202          * Check whether the end of the boosting chain has been
203          * reached or the state of the chain has changed while we
204          * dropped the locks.
205          */
206         if (!waiter || !waiter->task)
207                 goto out_unlock_pi;
208
209         /*
210          * Check the orig_waiter state. After we dropped the locks,
211          * the previous owner of the lock might have released the lock
212          * and made us the pending owner:
213          */
214         if (orig_waiter && !orig_waiter->task)
215                 goto out_unlock_pi;
216
217         /*
218          * Drop out, when the task has no waiters. Note,
219          * top_waiter can be NULL, when we are in the deboosting
220          * mode!
221          */
222         if (top_waiter && (!task_has_pi_waiters(task) ||
223                            top_waiter != task_top_pi_waiter(task)))
224                 goto out_unlock_pi;
225
226         /*
227          * When deadlock detection is off then we check, if further
228          * priority adjustment is necessary.
229          */
230         if (!detect_deadlock && waiter->list_entry.prio == task->prio)
231                 goto out_unlock_pi;
232
233         lock = waiter->lock;
234         if (!spin_trylock(&lock->wait_lock)) {
235                 spin_unlock_irqrestore(&task->pi_lock, flags);
236                 cpu_relax();
237                 goto retry;
238         }
239
240         /* Deadlock detection */
241         if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
242                 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
243                 spin_unlock(&lock->wait_lock);
244                 ret = deadlock_detect ? -EDEADLK : 0;
245                 goto out_unlock_pi;
246         }
247
248         top_waiter = rt_mutex_top_waiter(lock);
249
250         /* Requeue the waiter */
251         plist_del(&waiter->list_entry, &lock->wait_list);
252         waiter->list_entry.prio = task->prio;
253         plist_add(&waiter->list_entry, &lock->wait_list);
254
255         /* Release the task */
256         spin_unlock_irqrestore(&task->pi_lock, flags);
257         put_task_struct(task);
258
259         /* Grab the next task */
260         task = rt_mutex_owner(lock);
261         get_task_struct(task);
262         spin_lock_irqsave(&task->pi_lock, flags);
263
264         if (waiter == rt_mutex_top_waiter(lock)) {
265                 /* Boost the owner */
266                 plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
267                 waiter->pi_list_entry.prio = waiter->list_entry.prio;
268                 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
269                 __rt_mutex_adjust_prio(task);
270
271         } else if (top_waiter == waiter) {
272                 /* Deboost the owner */
273                 plist_del(&waiter->pi_list_entry, &task->pi_waiters);
274                 waiter = rt_mutex_top_waiter(lock);
275                 waiter->pi_list_entry.prio = waiter->list_entry.prio;
276                 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
277                 __rt_mutex_adjust_prio(task);
278         }
279
280         spin_unlock_irqrestore(&task->pi_lock, flags);
281
282         top_waiter = rt_mutex_top_waiter(lock);
283         spin_unlock(&lock->wait_lock);
284
285         if (!detect_deadlock && waiter != top_waiter)
286                 goto out_put_task;
287
288         goto again;
289
290  out_unlock_pi:
291         spin_unlock_irqrestore(&task->pi_lock, flags);
292  out_put_task:
293         put_task_struct(task);
294
295         return ret;
296 }
297
298 /*
299  * Optimization: check if we can steal the lock from the
300  * assigned pending owner [which might not have taken the
301  * lock yet]:
302  */
303 static inline int try_to_steal_lock(struct rt_mutex *lock)
304 {
305         struct task_struct *pendowner = rt_mutex_owner(lock);
306         struct rt_mutex_waiter *next;
307         unsigned long flags;
308
309         if (!rt_mutex_owner_pending(lock))
310                 return 0;
311
312         if (pendowner == current)
313                 return 1;
314
315         spin_lock_irqsave(&pendowner->pi_lock, flags);
316         if (current->prio >= pendowner->prio) {
317                 spin_unlock_irqrestore(&pendowner->pi_lock, flags);
318                 return 0;
319         }
320
321         /*
322          * Check if a waiter is enqueued on the pending owners
323          * pi_waiters list. Remove it and readjust pending owners
324          * priority.
325          */
326         if (likely(!rt_mutex_has_waiters(lock))) {
327                 spin_unlock_irqrestore(&pendowner->pi_lock, flags);
328                 return 1;
329         }
330
331         /* No chain handling, pending owner is not blocked on anything: */
332         next = rt_mutex_top_waiter(lock);
333         plist_del(&next->pi_list_entry, &pendowner->pi_waiters);
334         __rt_mutex_adjust_prio(pendowner);
335         spin_unlock_irqrestore(&pendowner->pi_lock, flags);
336
337         /*
338          * We are going to steal the lock and a waiter was
339          * enqueued on the pending owners pi_waiters queue. So
340          * we have to enqueue this waiter into
341          * current->pi_waiters list. This covers the case,
342          * where current is boosted because it holds another
343          * lock and gets unboosted because the booster is
344          * interrupted, so we would delay a waiter with higher
345          * priority as current->normal_prio.
346          *
347          * Note: in the rare case of a SCHED_OTHER task changing
348          * its priority and thus stealing the lock, next->task
349          * might be current:
350          */
351         if (likely(next->task != current)) {
352                 spin_lock_irqsave(&current->pi_lock, flags);
353                 plist_add(&next->pi_list_entry, &current->pi_waiters);
354                 __rt_mutex_adjust_prio(current);
355                 spin_unlock_irqrestore(&current->pi_lock, flags);
356         }
357         return 1;
358 }
359
360 /*
361  * Try to take an rt-mutex
362  *
363  * This fails
364  * - when the lock has a real owner
365  * - when a different pending owner exists and has higher priority than current
366  *
367  * Must be called with lock->wait_lock held.
368  */
369 static int try_to_take_rt_mutex(struct rt_mutex *lock)
370 {
371         /*
372          * We have to be careful here if the atomic speedups are
373          * enabled, such that, when
374          *  - no other waiter is on the lock
375          *  - the lock has been released since we did the cmpxchg
376          * the lock can be released or taken while we are doing the
377          * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
378          *
379          * The atomic acquire/release aware variant of
380          * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
381          * the WAITERS bit, the atomic release / acquire can not
382          * happen anymore and lock->wait_lock protects us from the
383          * non-atomic case.
384          *
385          * Note, that this might set lock->owner =
386          * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
387          * any more. This is fixed up when we take the ownership.
388          * This is the transitional state explained at the top of this file.
389          */
390         mark_rt_mutex_waiters(lock);
391
392         if (rt_mutex_owner(lock) && !try_to_steal_lock(lock))
393                 return 0;
394
395         /* We got the lock. */
396         debug_rt_mutex_lock(lock);
397
398         rt_mutex_set_owner(lock, current, 0);
399
400         rt_mutex_deadlock_account_lock(lock, current);
401
402         return 1;
403 }
404
405 /*
406  * Task blocks on lock.
407  *
408  * Prepare waiter and propagate pi chain
409  *
410  * This must be called with lock->wait_lock held.
411  */
412 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
413                                    struct rt_mutex_waiter *waiter,
414                                    int detect_deadlock)
415 {
416         struct task_struct *owner = rt_mutex_owner(lock);
417         struct rt_mutex_waiter *top_waiter = waiter;
418         unsigned long flags;
419         int chain_walk = 0, res;
420
421         spin_lock_irqsave(&current->pi_lock, flags);
422         __rt_mutex_adjust_prio(current);
423         waiter->task = current;
424         waiter->lock = lock;
425         plist_node_init(&waiter->list_entry, current->prio);
426         plist_node_init(&waiter->pi_list_entry, current->prio);
427
428         /* Get the top priority waiter on the lock */
429         if (rt_mutex_has_waiters(lock))
430                 top_waiter = rt_mutex_top_waiter(lock);
431         plist_add(&waiter->list_entry, &lock->wait_list);
432
433         current->pi_blocked_on = waiter;
434
435         spin_unlock_irqrestore(&current->pi_lock, flags);
436
437         if (waiter == rt_mutex_top_waiter(lock)) {
438                 spin_lock_irqsave(&owner->pi_lock, flags);
439                 plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
440                 plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
441
442                 __rt_mutex_adjust_prio(owner);
443                 if (owner->pi_blocked_on)
444                         chain_walk = 1;
445                 spin_unlock_irqrestore(&owner->pi_lock, flags);
446         }
447         else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
448                 chain_walk = 1;
449
450         if (!chain_walk)
451                 return 0;
452
453         /*
454          * The owner can't disappear while holding a lock,
455          * so the owner struct is protected by wait_lock.
456          * Gets dropped in rt_mutex_adjust_prio_chain()!
457          */
458         get_task_struct(owner);
459
460         spin_unlock(&lock->wait_lock);
461
462         res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
463                                          current);
464
465         spin_lock(&lock->wait_lock);
466
467         return res;
468 }
469
470 /*
471  * Wake up the next waiter on the lock.
472  *
473  * Remove the top waiter from the current tasks waiter list and from
474  * the lock waiter list. Set it as pending owner. Then wake it up.
475  *
476  * Called with lock->wait_lock held.
477  */
478 static void wakeup_next_waiter(struct rt_mutex *lock)
479 {
480         struct rt_mutex_waiter *waiter;
481         struct task_struct *pendowner;
482         unsigned long flags;
483
484         spin_lock_irqsave(&current->pi_lock, flags);
485
486         waiter = rt_mutex_top_waiter(lock);
487         plist_del(&waiter->list_entry, &lock->wait_list);
488
489         /*
490          * Remove it from current->pi_waiters. We do not adjust a
491          * possible priority boost right now. We execute wakeup in the
492          * boosted mode and go back to normal after releasing
493          * lock->wait_lock.
494          */
495         plist_del(&waiter->pi_list_entry, &current->pi_waiters);
496         pendowner = waiter->task;
497         waiter->task = NULL;
498
499         rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING);
500
501         spin_unlock_irqrestore(&current->pi_lock, flags);
502
503         /*
504          * Clear the pi_blocked_on variable and enqueue a possible
505          * waiter into the pi_waiters list of the pending owner. This
506          * prevents that in case the pending owner gets unboosted a
507          * waiter with higher priority than pending-owner->normal_prio
508          * is blocked on the unboosted (pending) owner.
509          */
510         spin_lock_irqsave(&pendowner->pi_lock, flags);
511
512         WARN_ON(!pendowner->pi_blocked_on);
513         WARN_ON(pendowner->pi_blocked_on != waiter);
514         WARN_ON(pendowner->pi_blocked_on->lock != lock);
515
516         pendowner->pi_blocked_on = NULL;
517
518         if (rt_mutex_has_waiters(lock)) {
519                 struct rt_mutex_waiter *next;
520
521                 next = rt_mutex_top_waiter(lock);
522                 plist_add(&next->pi_list_entry, &pendowner->pi_waiters);
523         }
524         spin_unlock_irqrestore(&pendowner->pi_lock, flags);
525
526         wake_up_process(pendowner);
527 }
528
529 /*
530  * Remove a waiter from a lock
531  *
532  * Must be called with lock->wait_lock held
533  */
534 static void remove_waiter(struct rt_mutex *lock,
535                           struct rt_mutex_waiter *waiter)
536 {
537         int first = (waiter == rt_mutex_top_waiter(lock));
538         struct task_struct *owner = rt_mutex_owner(lock);
539         unsigned long flags;
540         int chain_walk = 0;
541
542         spin_lock_irqsave(&current->pi_lock, flags);
543         plist_del(&waiter->list_entry, &lock->wait_list);
544         waiter->task = NULL;
545         current->pi_blocked_on = NULL;
546         spin_unlock_irqrestore(&current->pi_lock, flags);
547
548         if (first && owner != current) {
549
550                 spin_lock_irqsave(&owner->pi_lock, flags);
551
552                 plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
553
554                 if (rt_mutex_has_waiters(lock)) {
555                         struct rt_mutex_waiter *next;
556
557                         next = rt_mutex_top_waiter(lock);
558                         plist_add(&next->pi_list_entry, &owner->pi_waiters);
559                 }
560                 __rt_mutex_adjust_prio(owner);
561
562                 if (owner->pi_blocked_on)
563                         chain_walk = 1;
564
565                 spin_unlock_irqrestore(&owner->pi_lock, flags);
566         }
567
568         WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
569
570         if (!chain_walk)
571                 return;
572
573         /* gets dropped in rt_mutex_adjust_prio_chain()! */
574         get_task_struct(owner);
575
576         spin_unlock(&lock->wait_lock);
577
578         rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
579
580         spin_lock(&lock->wait_lock);
581 }
582
583 /*
584  * Recheck the pi chain, in case we got a priority setting
585  *
586  * Called from sched_setscheduler
587  */
588 void rt_mutex_adjust_pi(struct task_struct *task)
589 {
590         struct rt_mutex_waiter *waiter;
591         unsigned long flags;
592
593         spin_lock_irqsave(&task->pi_lock, flags);
594
595         waiter = task->pi_blocked_on;
596         if (!waiter || waiter->list_entry.prio == task->prio) {
597                 spin_unlock_irqrestore(&task->pi_lock, flags);
598                 return;
599         }
600
601         spin_unlock_irqrestore(&task->pi_lock, flags);
602
603         /* gets dropped in rt_mutex_adjust_prio_chain()! */
604         get_task_struct(task);
605         rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
606 }
607
608 /*
609  * Slow path lock function:
610  */
611 static int __sched
612 rt_mutex_slowlock(struct rt_mutex *lock, int state,
613                   struct hrtimer_sleeper *timeout,
614                   int detect_deadlock)
615 {
616         struct rt_mutex_waiter waiter;
617         int ret = 0;
618
619         debug_rt_mutex_init_waiter(&waiter);
620         waiter.task = NULL;
621
622         spin_lock(&lock->wait_lock);
623
624         /* Try to acquire the lock again: */
625         if (try_to_take_rt_mutex(lock)) {
626                 spin_unlock(&lock->wait_lock);
627                 return 0;
628         }
629
630         set_current_state(state);
631
632         /* Setup the timer, when timeout != NULL */
633         if (unlikely(timeout)) {
634                 hrtimer_start(&timeout->timer, timeout->timer.expires,
635                               HRTIMER_MODE_ABS);
636                 if (!hrtimer_active(&timeout->timer))
637                         timeout->task = NULL;
638         }
639
640         for (;;) {
641                 /* Try to acquire the lock: */
642                 if (try_to_take_rt_mutex(lock))
643                         break;
644
645                 /*
646                  * TASK_INTERRUPTIBLE checks for signals and
647                  * timeout. Ignored otherwise.
648                  */
649                 if (unlikely(state == TASK_INTERRUPTIBLE)) {
650                         /* Signal pending? */
651                         if (signal_pending(current))
652                                 ret = -EINTR;
653                         if (timeout && !timeout->task)
654                                 ret = -ETIMEDOUT;
655                         if (ret)
656                                 break;
657                 }
658
659                 /*
660                  * waiter.task is NULL the first time we come here and
661                  * when we have been woken up by the previous owner
662                  * but the lock got stolen by a higher prio task.
663                  */
664                 if (!waiter.task) {
665                         ret = task_blocks_on_rt_mutex(lock, &waiter,
666                                                       detect_deadlock);
667                         /*
668                          * If we got woken up by the owner then start loop
669                          * all over without going into schedule to try
670                          * to get the lock now:
671                          */
672                         if (unlikely(!waiter.task)) {
673                                 /*
674                                  * Reset the return value. We might
675                                  * have returned with -EDEADLK and the
676                                  * owner released the lock while we
677                                  * were walking the pi chain.
678                                  */
679                                 ret = 0;
680                                 continue;
681                         }
682                         if (unlikely(ret))
683                                 break;
684                 }
685
686                 spin_unlock(&lock->wait_lock);
687
688                 debug_rt_mutex_print_deadlock(&waiter);
689
690                 if (waiter.task)
691                         schedule_rt_mutex(lock);
692
693                 spin_lock(&lock->wait_lock);
694                 set_current_state(state);
695         }
696
697         set_current_state(TASK_RUNNING);
698
699         if (unlikely(waiter.task))
700                 remove_waiter(lock, &waiter);
701
702         /*
703          * try_to_take_rt_mutex() sets the waiter bit
704          * unconditionally. We might have to fix that up.
705          */
706         fixup_rt_mutex_waiters(lock);
707
708         spin_unlock(&lock->wait_lock);
709
710         /* Remove pending timer: */
711         if (unlikely(timeout))
712                 hrtimer_cancel(&timeout->timer);
713
714         /*
715          * Readjust priority, when we did not get the lock. We might
716          * have been the pending owner and boosted. Since we did not
717          * take the lock, the PI boost has to go.
718          */
719         if (unlikely(ret))
720                 rt_mutex_adjust_prio(current);
721
722         debug_rt_mutex_free_waiter(&waiter);
723
724         return ret;
725 }
726
727 /*
728  * Slow path try-lock function:
729  */
730 static inline int
731 rt_mutex_slowtrylock(struct rt_mutex *lock)
732 {
733         int ret = 0;
734
735         spin_lock(&lock->wait_lock);
736
737         if (likely(rt_mutex_owner(lock) != current)) {
738
739                 ret = try_to_take_rt_mutex(lock);
740                 /*
741                  * try_to_take_rt_mutex() sets the lock waiters
742                  * bit unconditionally. Clean this up.
743                  */
744                 fixup_rt_mutex_waiters(lock);
745         }
746
747         spin_unlock(&lock->wait_lock);
748
749         return ret;
750 }
751
752 /*
753  * Slow path to release a rt-mutex:
754  */
755 static void __sched
756 rt_mutex_slowunlock(struct rt_mutex *lock)
757 {
758         spin_lock(&lock->wait_lock);
759
760         debug_rt_mutex_unlock(lock);
761
762         rt_mutex_deadlock_account_unlock(current);
763
764         if (!rt_mutex_has_waiters(lock)) {
765                 lock->owner = NULL;
766                 spin_unlock(&lock->wait_lock);
767                 return;
768         }
769
770         wakeup_next_waiter(lock);
771
772         spin_unlock(&lock->wait_lock);
773
774         /* Undo pi boosting if necessary: */
775         rt_mutex_adjust_prio(current);
776 }
777
778 /*
779  * debug aware fast / slowpath lock,trylock,unlock
780  *
781  * The atomic acquire/release ops are compiled away, when either the
782  * architecture does not support cmpxchg or when debugging is enabled.
783  */
784 static inline int
785 rt_mutex_fastlock(struct rt_mutex *lock, int state,
786                   int detect_deadlock,
787                   int (*slowfn)(struct rt_mutex *lock, int state,
788                                 struct hrtimer_sleeper *timeout,
789                                 int detect_deadlock))
790 {
791         if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
792                 rt_mutex_deadlock_account_lock(lock, current);
793                 return 0;
794         } else
795                 return slowfn(lock, state, NULL, detect_deadlock);
796 }
797
798 static inline int
799 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
800                         struct hrtimer_sleeper *timeout, int detect_deadlock,
801                         int (*slowfn)(struct rt_mutex *lock, int state,
802                                       struct hrtimer_sleeper *timeout,
803                                       int detect_deadlock))
804 {
805         if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
806                 rt_mutex_deadlock_account_lock(lock, current);
807                 return 0;
808         } else
809                 return slowfn(lock, state, timeout, detect_deadlock);
810 }
811
812 static inline int
813 rt_mutex_fasttrylock(struct rt_mutex *lock,
814                      int (*slowfn)(struct rt_mutex *lock))
815 {
816         if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
817                 rt_mutex_deadlock_account_lock(lock, current);
818                 return 1;
819         }
820         return slowfn(lock);
821 }
822
823 static inline void
824 rt_mutex_fastunlock(struct rt_mutex *lock,
825                     void (*slowfn)(struct rt_mutex *lock))
826 {
827         if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
828                 rt_mutex_deadlock_account_unlock(current);
829         else
830                 slowfn(lock);
831 }
832
833 /**
834  * rt_mutex_lock - lock a rt_mutex
835  *
836  * @lock: the rt_mutex to be locked
837  */
838 void __sched rt_mutex_lock(struct rt_mutex *lock)
839 {
840         might_sleep();
841
842         rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
843 }
844 EXPORT_SYMBOL_GPL(rt_mutex_lock);
845
846 /**
847  * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
848  *
849  * @lock:               the rt_mutex to be locked
850  * @detect_deadlock:    deadlock detection on/off
851  *
852  * Returns:
853  *  0           on success
854  * -EINTR       when interrupted by a signal
855  * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
856  */
857 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
858                                                  int detect_deadlock)
859 {
860         might_sleep();
861
862         return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
863                                  detect_deadlock, rt_mutex_slowlock);
864 }
865 EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
866
867 /**
868  * rt_mutex_lock_interruptible_ktime - lock a rt_mutex interruptible
869  *                                     the timeout structure is provided
870  *                                     by the caller
871  *
872  * @lock:               the rt_mutex to be locked
873  * @timeout:            timeout structure or NULL (no timeout)
874  * @detect_deadlock:    deadlock detection on/off
875  *
876  * Returns:
877  *  0           on success
878  * -EINTR       when interrupted by a signal
879  * -ETIMEOUT    when the timeout expired
880  * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
881  */
882 int
883 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
884                     int detect_deadlock)
885 {
886         might_sleep();
887
888         return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
889                                        detect_deadlock, rt_mutex_slowlock);
890 }
891 EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
892
893 /**
894  * rt_mutex_trylock - try to lock a rt_mutex
895  *
896  * @lock:       the rt_mutex to be locked
897  *
898  * Returns 1 on success and 0 on contention
899  */
900 int __sched rt_mutex_trylock(struct rt_mutex *lock)
901 {
902         return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
903 }
904 EXPORT_SYMBOL_GPL(rt_mutex_trylock);
905
906 /**
907  * rt_mutex_unlock - unlock a rt_mutex
908  *
909  * @lock: the rt_mutex to be unlocked
910  */
911 void __sched rt_mutex_unlock(struct rt_mutex *lock)
912 {
913         rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
914 }
915 EXPORT_SYMBOL_GPL(rt_mutex_unlock);
916
917 /***
918  * rt_mutex_destroy - mark a mutex unusable
919  * @lock: the mutex to be destroyed
920  *
921  * This function marks the mutex uninitialized, and any subsequent
922  * use of the mutex is forbidden. The mutex must not be locked when
923  * this function is called.
924  */
925 void rt_mutex_destroy(struct rt_mutex *lock)
926 {
927         WARN_ON(rt_mutex_is_locked(lock));
928 #ifdef CONFIG_DEBUG_RT_MUTEXES
929         lock->magic = NULL;
930 #endif
931 }
932
933 EXPORT_SYMBOL_GPL(rt_mutex_destroy);
934
935 /**
936  * __rt_mutex_init - initialize the rt lock
937  *
938  * @lock: the rt lock to be initialized
939  *
940  * Initialize the rt lock to unlocked state.
941  *
942  * Initializing of a locked rt lock is not allowed
943  */
944 void __rt_mutex_init(struct rt_mutex *lock, const char *name)
945 {
946         lock->owner = NULL;
947         spin_lock_init(&lock->wait_lock);
948         plist_head_init(&lock->wait_list, &lock->wait_lock);
949
950         debug_rt_mutex_init(lock, name);
951 }
952 EXPORT_SYMBOL_GPL(__rt_mutex_init);
953
954 /**
955  * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
956  *                              proxy owner
957  *
958  * @lock:       the rt_mutex to be locked
959  * @proxy_owner:the task to set as owner
960  *
961  * No locking. Caller has to do serializing itself
962  * Special API call for PI-futex support
963  */
964 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
965                                 struct task_struct *proxy_owner)
966 {
967         __rt_mutex_init(lock, NULL);
968         debug_rt_mutex_proxy_lock(lock, proxy_owner);
969         rt_mutex_set_owner(lock, proxy_owner, 0);
970         rt_mutex_deadlock_account_lock(lock, proxy_owner);
971 }
972
973 /**
974  * rt_mutex_proxy_unlock - release a lock on behalf of owner
975  *
976  * @lock:       the rt_mutex to be locked
977  *
978  * No locking. Caller has to do serializing itself
979  * Special API call for PI-futex support
980  */
981 void rt_mutex_proxy_unlock(struct rt_mutex *lock,
982                            struct task_struct *proxy_owner)
983 {
984         debug_rt_mutex_proxy_unlock(lock);
985         rt_mutex_set_owner(lock, NULL, 0);
986         rt_mutex_deadlock_account_unlock(proxy_owner);
987 }
988
989 /**
990  * rt_mutex_next_owner - return the next owner of the lock
991  *
992  * @lock: the rt lock query
993  *
994  * Returns the next owner of the lock or NULL
995  *
996  * Caller has to serialize against other accessors to the lock
997  * itself.
998  *
999  * Special API call for PI-futex support
1000  */
1001 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
1002 {
1003         if (!rt_mutex_has_waiters(lock))
1004                 return NULL;
1005
1006         return rt_mutex_top_waiter(lock)->task;
1007 }