perf: net_dropmonitor: Fix symbol-relative addresses
[linux-3.10.git] / kernel / rtmutex.c
1 /*
2  * RT-Mutexes: simple blocking mutual exclusion locks with PI support
3  *
4  * started by Ingo Molnar and Thomas Gleixner.
5  *
6  *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7  *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
8  *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
9  *  Copyright (C) 2006 Esben Nielsen
10  *
11  *  See Documentation/rt-mutex-design.txt for details.
12  */
13 #include <linux/spinlock.h>
14 #include <linux/export.h>
15 #include <linux/sched.h>
16 #include <linux/sched/rt.h>
17 #include <linux/timer.h>
18
19 #include "rtmutex_common.h"
20
21 /*
22  * lock->owner state tracking:
23  *
24  * lock->owner holds the task_struct pointer of the owner. Bit 0
25  * is used to keep track of the "lock has waiters" state.
26  *
27  * owner        bit0
28  * NULL         0       lock is free (fast acquire possible)
29  * NULL         1       lock is free and has waiters and the top waiter
30  *                              is going to take the lock*
31  * taskpointer  0       lock is held (fast release possible)
32  * taskpointer  1       lock is held and has waiters**
33  *
34  * The fast atomic compare exchange based acquire and release is only
35  * possible when bit 0 of lock->owner is 0.
36  *
37  * (*) It also can be a transitional state when grabbing the lock
38  * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
39  * we need to set the bit0 before looking at the lock, and the owner may be
40  * NULL in this small time, hence this can be a transitional state.
41  *
42  * (**) There is a small time when bit 0 is set but there are no
43  * waiters. This can happen when grabbing the lock in the slow path.
44  * To prevent a cmpxchg of the owner releasing the lock, we need to
45  * set this bit before looking at the lock.
46  */
47
48 static void
49 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
50 {
51         unsigned long val = (unsigned long)owner;
52
53         if (rt_mutex_has_waiters(lock))
54                 val |= RT_MUTEX_HAS_WAITERS;
55
56         lock->owner = (struct task_struct *)val;
57 }
58
59 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
60 {
61         lock->owner = (struct task_struct *)
62                         ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
63 }
64
65 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
66 {
67         if (!rt_mutex_has_waiters(lock))
68                 clear_rt_mutex_waiters(lock);
69 }
70
71 /*
72  * We can speed up the acquire/release, if the architecture
73  * supports cmpxchg and if there's no debugging state to be set up
74  */
75 #if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
76 # define rt_mutex_cmpxchg(l,c,n)        (cmpxchg(&l->owner, c, n) == c)
77 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
78 {
79         unsigned long owner, *p = (unsigned long *) &lock->owner;
80
81         do {
82                 owner = *p;
83         } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
84 }
85 #else
86 # define rt_mutex_cmpxchg(l,c,n)        (0)
87 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
88 {
89         lock->owner = (struct task_struct *)
90                         ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
91 }
92 #endif
93
94 /*
95  * Calculate task priority from the waiter list priority
96  *
97  * Return task->normal_prio when the waiter list is empty or when
98  * the waiter is not allowed to do priority boosting
99  */
100 int rt_mutex_getprio(struct task_struct *task)
101 {
102         if (likely(!task_has_pi_waiters(task)))
103                 return task->normal_prio;
104
105         return min(task_top_pi_waiter(task)->pi_list_entry.prio,
106                    task->normal_prio);
107 }
108
109 /*
110  * Adjust the priority of a task, after its pi_waiters got modified.
111  *
112  * This can be both boosting and unboosting. task->pi_lock must be held.
113  */
114 static void __rt_mutex_adjust_prio(struct task_struct *task)
115 {
116         int prio = rt_mutex_getprio(task);
117
118         if (task->prio != prio)
119                 rt_mutex_setprio(task, prio);
120 }
121
122 /*
123  * Adjust task priority (undo boosting). Called from the exit path of
124  * rt_mutex_slowunlock() and rt_mutex_slowlock().
125  *
126  * (Note: We do this outside of the protection of lock->wait_lock to
127  * allow the lock to be taken while or before we readjust the priority
128  * of task. We do not use the spin_xx_mutex() variants here as we are
129  * outside of the debug path.)
130  */
131 static void rt_mutex_adjust_prio(struct task_struct *task)
132 {
133         unsigned long flags;
134
135         raw_spin_lock_irqsave(&task->pi_lock, flags);
136         __rt_mutex_adjust_prio(task);
137         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
138 }
139
140 /*
141  * Max number of times we'll walk the boosting chain:
142  */
143 int max_lock_depth = 1024;
144
145 /*
146  * Adjust the priority chain. Also used for deadlock detection.
147  * Decreases task's usage by one - may thus free the task.
148  * Returns 0 or -EDEADLK.
149  */
150 static int rt_mutex_adjust_prio_chain(struct task_struct *task,
151                                       int deadlock_detect,
152                                       struct rt_mutex *orig_lock,
153                                       struct rt_mutex_waiter *orig_waiter,
154                                       struct task_struct *top_task)
155 {
156         struct rt_mutex *lock;
157         struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
158         int detect_deadlock, ret = 0, depth = 0;
159         unsigned long flags;
160
161         detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
162                                                          deadlock_detect);
163
164         /*
165          * The (de)boosting is a step by step approach with a lot of
166          * pitfalls. We want this to be preemptible and we want hold a
167          * maximum of two locks per step. So we have to check
168          * carefully whether things change under us.
169          */
170  again:
171         if (++depth > max_lock_depth) {
172                 static int prev_max;
173
174                 /*
175                  * Print this only once. If the admin changes the limit,
176                  * print a new message when reaching the limit again.
177                  */
178                 if (prev_max != max_lock_depth) {
179                         prev_max = max_lock_depth;
180                         printk(KERN_WARNING "Maximum lock depth %d reached "
181                                "task: %s (%d)\n", max_lock_depth,
182                                top_task->comm, task_pid_nr(top_task));
183                 }
184                 put_task_struct(task);
185
186                 return deadlock_detect ? -EDEADLK : 0;
187         }
188  retry:
189         /*
190          * Task can not go away as we did a get_task() before !
191          */
192         raw_spin_lock_irqsave(&task->pi_lock, flags);
193
194         waiter = task->pi_blocked_on;
195         /*
196          * Check whether the end of the boosting chain has been
197          * reached or the state of the chain has changed while we
198          * dropped the locks.
199          */
200         if (!waiter)
201                 goto out_unlock_pi;
202
203         /*
204          * Check the orig_waiter state. After we dropped the locks,
205          * the previous owner of the lock might have released the lock.
206          */
207         if (orig_waiter && !rt_mutex_owner(orig_lock))
208                 goto out_unlock_pi;
209
210         /*
211          * Drop out, when the task has no waiters. Note,
212          * top_waiter can be NULL, when we are in the deboosting
213          * mode!
214          */
215         if (top_waiter && (!task_has_pi_waiters(task) ||
216                            top_waiter != task_top_pi_waiter(task)))
217                 goto out_unlock_pi;
218
219         /*
220          * When deadlock detection is off then we check, if further
221          * priority adjustment is necessary.
222          */
223         if (!detect_deadlock && waiter->list_entry.prio == task->prio)
224                 goto out_unlock_pi;
225
226         lock = waiter->lock;
227         if (!raw_spin_trylock(&lock->wait_lock)) {
228                 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
229                 cpu_relax();
230                 goto retry;
231         }
232
233         /* Deadlock detection */
234         if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
235                 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
236                 raw_spin_unlock(&lock->wait_lock);
237                 ret = deadlock_detect ? -EDEADLK : 0;
238                 goto out_unlock_pi;
239         }
240
241         top_waiter = rt_mutex_top_waiter(lock);
242
243         /* Requeue the waiter */
244         plist_del(&waiter->list_entry, &lock->wait_list);
245         waiter->list_entry.prio = task->prio;
246         plist_add(&waiter->list_entry, &lock->wait_list);
247
248         /* Release the task */
249         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
250         if (!rt_mutex_owner(lock)) {
251                 /*
252                  * If the requeue above changed the top waiter, then we need
253                  * to wake the new top waiter up to try to get the lock.
254                  */
255
256                 if (top_waiter != rt_mutex_top_waiter(lock))
257                         wake_up_process(rt_mutex_top_waiter(lock)->task);
258                 raw_spin_unlock(&lock->wait_lock);
259                 goto out_put_task;
260         }
261         put_task_struct(task);
262
263         /* Grab the next task */
264         task = rt_mutex_owner(lock);
265         get_task_struct(task);
266         raw_spin_lock_irqsave(&task->pi_lock, flags);
267
268         if (waiter == rt_mutex_top_waiter(lock)) {
269                 /* Boost the owner */
270                 plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
271                 waiter->pi_list_entry.prio = waiter->list_entry.prio;
272                 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
273                 __rt_mutex_adjust_prio(task);
274
275         } else if (top_waiter == waiter) {
276                 /* Deboost the owner */
277                 plist_del(&waiter->pi_list_entry, &task->pi_waiters);
278                 waiter = rt_mutex_top_waiter(lock);
279                 waiter->pi_list_entry.prio = waiter->list_entry.prio;
280                 plist_add(&waiter->pi_list_entry, &task->pi_waiters);
281                 __rt_mutex_adjust_prio(task);
282         }
283
284         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
285
286         top_waiter = rt_mutex_top_waiter(lock);
287         raw_spin_unlock(&lock->wait_lock);
288
289         if (!detect_deadlock && waiter != top_waiter)
290                 goto out_put_task;
291
292         goto again;
293
294  out_unlock_pi:
295         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
296  out_put_task:
297         put_task_struct(task);
298
299         return ret;
300 }
301
302 /*
303  * Try to take an rt-mutex
304  *
305  * Must be called with lock->wait_lock held.
306  *
307  * @lock:   the lock to be acquired.
308  * @task:   the task which wants to acquire the lock
309  * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
310  */
311 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
312                 struct rt_mutex_waiter *waiter)
313 {
314         /*
315          * We have to be careful here if the atomic speedups are
316          * enabled, such that, when
317          *  - no other waiter is on the lock
318          *  - the lock has been released since we did the cmpxchg
319          * the lock can be released or taken while we are doing the
320          * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
321          *
322          * The atomic acquire/release aware variant of
323          * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
324          * the WAITERS bit, the atomic release / acquire can not
325          * happen anymore and lock->wait_lock protects us from the
326          * non-atomic case.
327          *
328          * Note, that this might set lock->owner =
329          * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
330          * any more. This is fixed up when we take the ownership.
331          * This is the transitional state explained at the top of this file.
332          */
333         mark_rt_mutex_waiters(lock);
334
335         if (rt_mutex_owner(lock))
336                 return 0;
337
338         /*
339          * It will get the lock because of one of these conditions:
340          * 1) there is no waiter
341          * 2) higher priority than waiters
342          * 3) it is top waiter
343          */
344         if (rt_mutex_has_waiters(lock)) {
345                 if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
346                         if (!waiter || waiter != rt_mutex_top_waiter(lock))
347                                 return 0;
348                 }
349         }
350
351         if (waiter || rt_mutex_has_waiters(lock)) {
352                 unsigned long flags;
353                 struct rt_mutex_waiter *top;
354
355                 raw_spin_lock_irqsave(&task->pi_lock, flags);
356
357                 /* remove the queued waiter. */
358                 if (waiter) {
359                         plist_del(&waiter->list_entry, &lock->wait_list);
360                         task->pi_blocked_on = NULL;
361                 }
362
363                 /*
364                  * We have to enqueue the top waiter(if it exists) into
365                  * task->pi_waiters list.
366                  */
367                 if (rt_mutex_has_waiters(lock)) {
368                         top = rt_mutex_top_waiter(lock);
369                         top->pi_list_entry.prio = top->list_entry.prio;
370                         plist_add(&top->pi_list_entry, &task->pi_waiters);
371                 }
372                 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
373         }
374
375         /* We got the lock. */
376         debug_rt_mutex_lock(lock);
377
378         rt_mutex_set_owner(lock, task);
379
380         rt_mutex_deadlock_account_lock(lock, task);
381
382         return 1;
383 }
384
385 /*
386  * Task blocks on lock.
387  *
388  * Prepare waiter and propagate pi chain
389  *
390  * This must be called with lock->wait_lock held.
391  */
392 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
393                                    struct rt_mutex_waiter *waiter,
394                                    struct task_struct *task,
395                                    int detect_deadlock)
396 {
397         struct task_struct *owner = rt_mutex_owner(lock);
398         struct rt_mutex_waiter *top_waiter = waiter;
399         unsigned long flags;
400         int chain_walk = 0, res;
401
402         raw_spin_lock_irqsave(&task->pi_lock, flags);
403         __rt_mutex_adjust_prio(task);
404         waiter->task = task;
405         waiter->lock = lock;
406         plist_node_init(&waiter->list_entry, task->prio);
407         plist_node_init(&waiter->pi_list_entry, task->prio);
408
409         /* Get the top priority waiter on the lock */
410         if (rt_mutex_has_waiters(lock))
411                 top_waiter = rt_mutex_top_waiter(lock);
412         plist_add(&waiter->list_entry, &lock->wait_list);
413
414         task->pi_blocked_on = waiter;
415
416         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
417
418         if (!owner)
419                 return 0;
420
421         if (waiter == rt_mutex_top_waiter(lock)) {
422                 raw_spin_lock_irqsave(&owner->pi_lock, flags);
423                 plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
424                 plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
425
426                 __rt_mutex_adjust_prio(owner);
427                 if (owner->pi_blocked_on)
428                         chain_walk = 1;
429                 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
430         }
431         else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
432                 chain_walk = 1;
433
434         if (!chain_walk)
435                 return 0;
436
437         /*
438          * The owner can't disappear while holding a lock,
439          * so the owner struct is protected by wait_lock.
440          * Gets dropped in rt_mutex_adjust_prio_chain()!
441          */
442         get_task_struct(owner);
443
444         raw_spin_unlock(&lock->wait_lock);
445
446         res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
447                                          task);
448
449         raw_spin_lock(&lock->wait_lock);
450
451         return res;
452 }
453
454 /*
455  * Wake up the next waiter on the lock.
456  *
457  * Remove the top waiter from the current tasks waiter list and wake it up.
458  *
459  * Called with lock->wait_lock held.
460  */
461 static void wakeup_next_waiter(struct rt_mutex *lock)
462 {
463         struct rt_mutex_waiter *waiter;
464         unsigned long flags;
465
466         raw_spin_lock_irqsave(&current->pi_lock, flags);
467
468         waiter = rt_mutex_top_waiter(lock);
469
470         /*
471          * Remove it from current->pi_waiters. We do not adjust a
472          * possible priority boost right now. We execute wakeup in the
473          * boosted mode and go back to normal after releasing
474          * lock->wait_lock.
475          */
476         plist_del(&waiter->pi_list_entry, &current->pi_waiters);
477
478         rt_mutex_set_owner(lock, NULL);
479
480         raw_spin_unlock_irqrestore(&current->pi_lock, flags);
481
482         wake_up_process(waiter->task);
483 }
484
485 /*
486  * Remove a waiter from a lock and give up
487  *
488  * Must be called with lock->wait_lock held and
489  * have just failed to try_to_take_rt_mutex().
490  */
491 static void remove_waiter(struct rt_mutex *lock,
492                           struct rt_mutex_waiter *waiter)
493 {
494         int first = (waiter == rt_mutex_top_waiter(lock));
495         struct task_struct *owner = rt_mutex_owner(lock);
496         unsigned long flags;
497         int chain_walk = 0;
498
499         raw_spin_lock_irqsave(&current->pi_lock, flags);
500         plist_del(&waiter->list_entry, &lock->wait_list);
501         current->pi_blocked_on = NULL;
502         raw_spin_unlock_irqrestore(&current->pi_lock, flags);
503
504         if (!owner)
505                 return;
506
507         if (first) {
508
509                 raw_spin_lock_irqsave(&owner->pi_lock, flags);
510
511                 plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
512
513                 if (rt_mutex_has_waiters(lock)) {
514                         struct rt_mutex_waiter *next;
515
516                         next = rt_mutex_top_waiter(lock);
517                         plist_add(&next->pi_list_entry, &owner->pi_waiters);
518                 }
519                 __rt_mutex_adjust_prio(owner);
520
521                 if (owner->pi_blocked_on)
522                         chain_walk = 1;
523
524                 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
525         }
526
527         WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
528
529         if (!chain_walk)
530                 return;
531
532         /* gets dropped in rt_mutex_adjust_prio_chain()! */
533         get_task_struct(owner);
534
535         raw_spin_unlock(&lock->wait_lock);
536
537         rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
538
539         raw_spin_lock(&lock->wait_lock);
540 }
541
542 /*
543  * Recheck the pi chain, in case we got a priority setting
544  *
545  * Called from sched_setscheduler
546  */
547 void rt_mutex_adjust_pi(struct task_struct *task)
548 {
549         struct rt_mutex_waiter *waiter;
550         unsigned long flags;
551
552         raw_spin_lock_irqsave(&task->pi_lock, flags);
553
554         waiter = task->pi_blocked_on;
555         if (!waiter || waiter->list_entry.prio == task->prio) {
556                 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
557                 return;
558         }
559
560         raw_spin_unlock_irqrestore(&task->pi_lock, flags);
561
562         /* gets dropped in rt_mutex_adjust_prio_chain()! */
563         get_task_struct(task);
564         rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
565 }
566
567 /**
568  * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
569  * @lock:                the rt_mutex to take
570  * @state:               the state the task should block in (TASK_INTERRUPTIBLE
571  *                       or TASK_UNINTERRUPTIBLE)
572  * @timeout:             the pre-initialized and started timer, or NULL for none
573  * @waiter:              the pre-initialized rt_mutex_waiter
574  *
575  * lock->wait_lock must be held by the caller.
576  */
577 static int __sched
578 __rt_mutex_slowlock(struct rt_mutex *lock, int state,
579                     struct hrtimer_sleeper *timeout,
580                     struct rt_mutex_waiter *waiter)
581 {
582         int ret = 0;
583
584         for (;;) {
585                 /* Try to acquire the lock: */
586                 if (try_to_take_rt_mutex(lock, current, waiter))
587                         break;
588
589                 /*
590                  * TASK_INTERRUPTIBLE checks for signals and
591                  * timeout. Ignored otherwise.
592                  */
593                 if (unlikely(state == TASK_INTERRUPTIBLE)) {
594                         /* Signal pending? */
595                         if (signal_pending(current))
596                                 ret = -EINTR;
597                         if (timeout && !timeout->task)
598                                 ret = -ETIMEDOUT;
599                         if (ret)
600                                 break;
601                 }
602
603                 raw_spin_unlock(&lock->wait_lock);
604
605                 debug_rt_mutex_print_deadlock(waiter);
606
607                 schedule_rt_mutex(lock);
608
609                 raw_spin_lock(&lock->wait_lock);
610                 set_current_state(state);
611         }
612
613         return ret;
614 }
615
616 /*
617  * Slow path lock function:
618  */
619 static int __sched
620 rt_mutex_slowlock(struct rt_mutex *lock, int state,
621                   struct hrtimer_sleeper *timeout,
622                   int detect_deadlock)
623 {
624         struct rt_mutex_waiter waiter;
625         int ret = 0;
626
627         debug_rt_mutex_init_waiter(&waiter);
628
629         raw_spin_lock(&lock->wait_lock);
630
631         /* Try to acquire the lock again: */
632         if (try_to_take_rt_mutex(lock, current, NULL)) {
633                 raw_spin_unlock(&lock->wait_lock);
634                 return 0;
635         }
636
637         set_current_state(state);
638
639         /* Setup the timer, when timeout != NULL */
640         if (unlikely(timeout)) {
641                 hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
642                 if (!hrtimer_active(&timeout->timer))
643                         timeout->task = NULL;
644         }
645
646         ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
647
648         if (likely(!ret))
649                 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
650
651         set_current_state(TASK_RUNNING);
652
653         if (unlikely(ret))
654                 remove_waiter(lock, &waiter);
655
656         /*
657          * try_to_take_rt_mutex() sets the waiter bit
658          * unconditionally. We might have to fix that up.
659          */
660         fixup_rt_mutex_waiters(lock);
661
662         raw_spin_unlock(&lock->wait_lock);
663
664         /* Remove pending timer: */
665         if (unlikely(timeout))
666                 hrtimer_cancel(&timeout->timer);
667
668         debug_rt_mutex_free_waiter(&waiter);
669
670         return ret;
671 }
672
673 /*
674  * Slow path try-lock function:
675  */
676 static inline int
677 rt_mutex_slowtrylock(struct rt_mutex *lock)
678 {
679         int ret = 0;
680
681         raw_spin_lock(&lock->wait_lock);
682
683         if (likely(rt_mutex_owner(lock) != current)) {
684
685                 ret = try_to_take_rt_mutex(lock, current, NULL);
686                 /*
687                  * try_to_take_rt_mutex() sets the lock waiters
688                  * bit unconditionally. Clean this up.
689                  */
690                 fixup_rt_mutex_waiters(lock);
691         }
692
693         raw_spin_unlock(&lock->wait_lock);
694
695         return ret;
696 }
697
698 /*
699  * Slow path to release a rt-mutex:
700  */
701 static void __sched
702 rt_mutex_slowunlock(struct rt_mutex *lock)
703 {
704         raw_spin_lock(&lock->wait_lock);
705
706         debug_rt_mutex_unlock(lock);
707
708         rt_mutex_deadlock_account_unlock(current);
709
710         if (!rt_mutex_has_waiters(lock)) {
711                 lock->owner = NULL;
712                 raw_spin_unlock(&lock->wait_lock);
713                 return;
714         }
715
716         wakeup_next_waiter(lock);
717
718         raw_spin_unlock(&lock->wait_lock);
719
720         /* Undo pi boosting if necessary: */
721         rt_mutex_adjust_prio(current);
722 }
723
724 /*
725  * debug aware fast / slowpath lock,trylock,unlock
726  *
727  * The atomic acquire/release ops are compiled away, when either the
728  * architecture does not support cmpxchg or when debugging is enabled.
729  */
730 static inline int
731 rt_mutex_fastlock(struct rt_mutex *lock, int state,
732                   int detect_deadlock,
733                   int (*slowfn)(struct rt_mutex *lock, int state,
734                                 struct hrtimer_sleeper *timeout,
735                                 int detect_deadlock))
736 {
737         if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
738                 rt_mutex_deadlock_account_lock(lock, current);
739                 return 0;
740         } else
741                 return slowfn(lock, state, NULL, detect_deadlock);
742 }
743
744 static inline int
745 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
746                         struct hrtimer_sleeper *timeout, int detect_deadlock,
747                         int (*slowfn)(struct rt_mutex *lock, int state,
748                                       struct hrtimer_sleeper *timeout,
749                                       int detect_deadlock))
750 {
751         if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
752                 rt_mutex_deadlock_account_lock(lock, current);
753                 return 0;
754         } else
755                 return slowfn(lock, state, timeout, detect_deadlock);
756 }
757
758 static inline int
759 rt_mutex_fasttrylock(struct rt_mutex *lock,
760                      int (*slowfn)(struct rt_mutex *lock))
761 {
762         if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
763                 rt_mutex_deadlock_account_lock(lock, current);
764                 return 1;
765         }
766         return slowfn(lock);
767 }
768
769 static inline void
770 rt_mutex_fastunlock(struct rt_mutex *lock,
771                     void (*slowfn)(struct rt_mutex *lock))
772 {
773         if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
774                 rt_mutex_deadlock_account_unlock(current);
775         else
776                 slowfn(lock);
777 }
778
779 /**
780  * rt_mutex_lock - lock a rt_mutex
781  *
782  * @lock: the rt_mutex to be locked
783  */
784 void __sched rt_mutex_lock(struct rt_mutex *lock)
785 {
786         might_sleep();
787
788         rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
789 }
790 EXPORT_SYMBOL_GPL(rt_mutex_lock);
791
792 /**
793  * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
794  *
795  * @lock:               the rt_mutex to be locked
796  * @detect_deadlock:    deadlock detection on/off
797  *
798  * Returns:
799  *  0           on success
800  * -EINTR       when interrupted by a signal
801  * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
802  */
803 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
804                                                  int detect_deadlock)
805 {
806         might_sleep();
807
808         return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
809                                  detect_deadlock, rt_mutex_slowlock);
810 }
811 EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
812
813 /**
814  * rt_mutex_timed_lock - lock a rt_mutex interruptible
815  *                      the timeout structure is provided
816  *                      by the caller
817  *
818  * @lock:               the rt_mutex to be locked
819  * @timeout:            timeout structure or NULL (no timeout)
820  * @detect_deadlock:    deadlock detection on/off
821  *
822  * Returns:
823  *  0           on success
824  * -EINTR       when interrupted by a signal
825  * -ETIMEDOUT   when the timeout expired
826  * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
827  */
828 int
829 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
830                     int detect_deadlock)
831 {
832         might_sleep();
833
834         return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
835                                        detect_deadlock, rt_mutex_slowlock);
836 }
837 EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
838
839 /**
840  * rt_mutex_trylock - try to lock a rt_mutex
841  *
842  * @lock:       the rt_mutex to be locked
843  *
844  * Returns 1 on success and 0 on contention
845  */
846 int __sched rt_mutex_trylock(struct rt_mutex *lock)
847 {
848         return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
849 }
850 EXPORT_SYMBOL_GPL(rt_mutex_trylock);
851
852 /**
853  * rt_mutex_unlock - unlock a rt_mutex
854  *
855  * @lock: the rt_mutex to be unlocked
856  */
857 void __sched rt_mutex_unlock(struct rt_mutex *lock)
858 {
859         rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
860 }
861 EXPORT_SYMBOL_GPL(rt_mutex_unlock);
862
863 /**
864  * rt_mutex_destroy - mark a mutex unusable
865  * @lock: the mutex to be destroyed
866  *
867  * This function marks the mutex uninitialized, and any subsequent
868  * use of the mutex is forbidden. The mutex must not be locked when
869  * this function is called.
870  */
871 void rt_mutex_destroy(struct rt_mutex *lock)
872 {
873         WARN_ON(rt_mutex_is_locked(lock));
874 #ifdef CONFIG_DEBUG_RT_MUTEXES
875         lock->magic = NULL;
876 #endif
877 }
878
879 EXPORT_SYMBOL_GPL(rt_mutex_destroy);
880
881 /**
882  * __rt_mutex_init - initialize the rt lock
883  *
884  * @lock: the rt lock to be initialized
885  *
886  * Initialize the rt lock to unlocked state.
887  *
888  * Initializing of a locked rt lock is not allowed
889  */
890 void __rt_mutex_init(struct rt_mutex *lock, const char *name)
891 {
892         lock->owner = NULL;
893         raw_spin_lock_init(&lock->wait_lock);
894         plist_head_init(&lock->wait_list);
895
896         debug_rt_mutex_init(lock, name);
897 }
898 EXPORT_SYMBOL_GPL(__rt_mutex_init);
899
900 /**
901  * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
902  *                              proxy owner
903  *
904  * @lock:       the rt_mutex to be locked
905  * @proxy_owner:the task to set as owner
906  *
907  * No locking. Caller has to do serializing itself
908  * Special API call for PI-futex support
909  */
910 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
911                                 struct task_struct *proxy_owner)
912 {
913         __rt_mutex_init(lock, NULL);
914         debug_rt_mutex_proxy_lock(lock, proxy_owner);
915         rt_mutex_set_owner(lock, proxy_owner);
916         rt_mutex_deadlock_account_lock(lock, proxy_owner);
917 }
918
919 /**
920  * rt_mutex_proxy_unlock - release a lock on behalf of owner
921  *
922  * @lock:       the rt_mutex to be locked
923  *
924  * No locking. Caller has to do serializing itself
925  * Special API call for PI-futex support
926  */
927 void rt_mutex_proxy_unlock(struct rt_mutex *lock,
928                            struct task_struct *proxy_owner)
929 {
930         debug_rt_mutex_proxy_unlock(lock);
931         rt_mutex_set_owner(lock, NULL);
932         rt_mutex_deadlock_account_unlock(proxy_owner);
933 }
934
935 /**
936  * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
937  * @lock:               the rt_mutex to take
938  * @waiter:             the pre-initialized rt_mutex_waiter
939  * @task:               the task to prepare
940  * @detect_deadlock:    perform deadlock detection (1) or not (0)
941  *
942  * Returns:
943  *  0 - task blocked on lock
944  *  1 - acquired the lock for task, caller should wake it up
945  * <0 - error
946  *
947  * Special API call for FUTEX_REQUEUE_PI support.
948  */
949 int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
950                               struct rt_mutex_waiter *waiter,
951                               struct task_struct *task, int detect_deadlock)
952 {
953         int ret;
954
955         raw_spin_lock(&lock->wait_lock);
956
957         if (try_to_take_rt_mutex(lock, task, NULL)) {
958                 raw_spin_unlock(&lock->wait_lock);
959                 return 1;
960         }
961
962         ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
963
964         if (ret && !rt_mutex_owner(lock)) {
965                 /*
966                  * Reset the return value. We might have
967                  * returned with -EDEADLK and the owner
968                  * released the lock while we were walking the
969                  * pi chain.  Let the waiter sort it out.
970                  */
971                 ret = 0;
972         }
973
974         if (unlikely(ret))
975                 remove_waiter(lock, waiter);
976
977         raw_spin_unlock(&lock->wait_lock);
978
979         debug_rt_mutex_print_deadlock(waiter);
980
981         return ret;
982 }
983
984 /**
985  * rt_mutex_next_owner - return the next owner of the lock
986  *
987  * @lock: the rt lock query
988  *
989  * Returns the next owner of the lock or NULL
990  *
991  * Caller has to serialize against other accessors to the lock
992  * itself.
993  *
994  * Special API call for PI-futex support
995  */
996 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
997 {
998         if (!rt_mutex_has_waiters(lock))
999                 return NULL;
1000
1001         return rt_mutex_top_waiter(lock)->task;
1002 }
1003
1004 /**
1005  * rt_mutex_finish_proxy_lock() - Complete lock acquisition
1006  * @lock:               the rt_mutex we were woken on
1007  * @to:                 the timeout, null if none. hrtimer should already have
1008  *                      been started.
1009  * @waiter:             the pre-initialized rt_mutex_waiter
1010  * @detect_deadlock:    perform deadlock detection (1) or not (0)
1011  *
1012  * Complete the lock acquisition started our behalf by another thread.
1013  *
1014  * Returns:
1015  *  0 - success
1016  * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
1017  *
1018  * Special API call for PI-futex requeue support
1019  */
1020 int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1021                                struct hrtimer_sleeper *to,
1022                                struct rt_mutex_waiter *waiter,
1023                                int detect_deadlock)
1024 {
1025         int ret;
1026
1027         raw_spin_lock(&lock->wait_lock);
1028
1029         set_current_state(TASK_INTERRUPTIBLE);
1030
1031         ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1032
1033         set_current_state(TASK_RUNNING);
1034
1035         if (unlikely(ret))
1036                 remove_waiter(lock, waiter);
1037
1038         /*
1039          * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1040          * have to fix that up.
1041          */
1042         fixup_rt_mutex_waiters(lock);
1043
1044         raw_spin_unlock(&lock->wait_lock);
1045
1046         return ret;
1047 }