[PATCH] sched: ->task_new cleanup
[linux-2.6.git] / kernel / sched_rt.c
1 /*
2  * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
3  * policies)
4  */
5
6 /*
7  * Update the current task's runtime statistics. Skip current tasks that
8  * are not in our scheduling class.
9  */
10 static inline void update_curr_rt(struct rq *rq, u64 now)
11 {
12         struct task_struct *curr = rq->curr;
13         u64 delta_exec;
14
15         if (!task_has_rt_policy(curr))
16                 return;
17
18         delta_exec = now - curr->se.exec_start;
19         if (unlikely((s64)delta_exec < 0))
20                 delta_exec = 0;
21         if (unlikely(delta_exec > curr->se.exec_max))
22                 curr->se.exec_max = delta_exec;
23
24         curr->se.sum_exec_runtime += delta_exec;
25         curr->se.exec_start = now;
26 }
27
28 static void
29 enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
30 {
31         struct rt_prio_array *array = &rq->rt.active;
32
33         list_add_tail(&p->run_list, array->queue + p->prio);
34         __set_bit(p->prio, array->bitmap);
35 }
36
37 /*
38  * Adding/removing a task to/from a priority array:
39  */
40 static void
41 dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep, u64 now)
42 {
43         struct rt_prio_array *array = &rq->rt.active;
44
45         update_curr_rt(rq, now);
46
47         list_del(&p->run_list);
48         if (list_empty(array->queue + p->prio))
49                 __clear_bit(p->prio, array->bitmap);
50 }
51
52 /*
53  * Put task to the end of the run list without the overhead of dequeue
54  * followed by enqueue.
55  */
56 static void requeue_task_rt(struct rq *rq, struct task_struct *p)
57 {
58         struct rt_prio_array *array = &rq->rt.active;
59
60         list_move_tail(&p->run_list, array->queue + p->prio);
61 }
62
63 static void
64 yield_task_rt(struct rq *rq, struct task_struct *p)
65 {
66         requeue_task_rt(rq, p);
67 }
68
69 /*
70  * Preempt the current task with a newly woken task if needed:
71  */
72 static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
73 {
74         if (p->prio < rq->curr->prio)
75                 resched_task(rq->curr);
76 }
77
78 static struct task_struct *pick_next_task_rt(struct rq *rq, u64 now)
79 {
80         struct rt_prio_array *array = &rq->rt.active;
81         struct task_struct *next;
82         struct list_head *queue;
83         int idx;
84
85         idx = sched_find_first_bit(array->bitmap);
86         if (idx >= MAX_RT_PRIO)
87                 return NULL;
88
89         queue = array->queue + idx;
90         next = list_entry(queue->next, struct task_struct, run_list);
91
92         next->se.exec_start = now;
93
94         return next;
95 }
96
97 static void put_prev_task_rt(struct rq *rq, struct task_struct *p, u64 now)
98 {
99         update_curr_rt(rq, now);
100         p->se.exec_start = 0;
101 }
102
103 /*
104  * Load-balancing iterator. Note: while the runqueue stays locked
105  * during the whole iteration, the current task might be
106  * dequeued so the iterator has to be dequeue-safe. Here we
107  * achieve that by always pre-iterating before returning
108  * the current task:
109  */
110 static struct task_struct *load_balance_start_rt(void *arg)
111 {
112         struct rq *rq = arg;
113         struct rt_prio_array *array = &rq->rt.active;
114         struct list_head *head, *curr;
115         struct task_struct *p;
116         int idx;
117
118         idx = sched_find_first_bit(array->bitmap);
119         if (idx >= MAX_RT_PRIO)
120                 return NULL;
121
122         head = array->queue + idx;
123         curr = head->prev;
124
125         p = list_entry(curr, struct task_struct, run_list);
126
127         curr = curr->prev;
128
129         rq->rt.rt_load_balance_idx = idx;
130         rq->rt.rt_load_balance_head = head;
131         rq->rt.rt_load_balance_curr = curr;
132
133         return p;
134 }
135
136 static struct task_struct *load_balance_next_rt(void *arg)
137 {
138         struct rq *rq = arg;
139         struct rt_prio_array *array = &rq->rt.active;
140         struct list_head *head, *curr;
141         struct task_struct *p;
142         int idx;
143
144         idx = rq->rt.rt_load_balance_idx;
145         head = rq->rt.rt_load_balance_head;
146         curr = rq->rt.rt_load_balance_curr;
147
148         /*
149          * If we arrived back to the head again then
150          * iterate to the next queue (if any):
151          */
152         if (unlikely(head == curr)) {
153                 int next_idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx+1);
154
155                 if (next_idx >= MAX_RT_PRIO)
156                         return NULL;
157
158                 idx = next_idx;
159                 head = array->queue + idx;
160                 curr = head->prev;
161
162                 rq->rt.rt_load_balance_idx = idx;
163                 rq->rt.rt_load_balance_head = head;
164         }
165
166         p = list_entry(curr, struct task_struct, run_list);
167
168         curr = curr->prev;
169
170         rq->rt.rt_load_balance_curr = curr;
171
172         return p;
173 }
174
175 static int
176 load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
177                         unsigned long max_nr_move, unsigned long max_load_move,
178                         struct sched_domain *sd, enum cpu_idle_type idle,
179                         int *all_pinned, unsigned long *load_moved)
180 {
181         int this_best_prio, best_prio, best_prio_seen = 0;
182         int nr_moved;
183         struct rq_iterator rt_rq_iterator;
184
185         best_prio = sched_find_first_bit(busiest->rt.active.bitmap);
186         this_best_prio = sched_find_first_bit(this_rq->rt.active.bitmap);
187
188         /*
189          * Enable handling of the case where there is more than one task
190          * with the best priority.   If the current running task is one
191          * of those with prio==best_prio we know it won't be moved
192          * and therefore it's safe to override the skip (based on load)
193          * of any task we find with that prio.
194          */
195         if (busiest->curr->prio == best_prio)
196                 best_prio_seen = 1;
197
198         rt_rq_iterator.start = load_balance_start_rt;
199         rt_rq_iterator.next = load_balance_next_rt;
200         /* pass 'busiest' rq argument into
201          * load_balance_[start|next]_rt iterators
202          */
203         rt_rq_iterator.arg = busiest;
204
205         nr_moved = balance_tasks(this_rq, this_cpu, busiest, max_nr_move,
206                         max_load_move, sd, idle, all_pinned, load_moved,
207                         this_best_prio, best_prio, best_prio_seen,
208                         &rt_rq_iterator);
209
210         return nr_moved;
211 }
212
213 static void task_tick_rt(struct rq *rq, struct task_struct *p)
214 {
215         /*
216          * RR tasks need a special form of timeslice management.
217          * FIFO tasks have no timeslices.
218          */
219         if (p->policy != SCHED_RR)
220                 return;
221
222         if (--p->time_slice)
223                 return;
224
225         p->time_slice = static_prio_timeslice(p->static_prio);
226         set_tsk_need_resched(p);
227
228         /* put it at the end of the queue: */
229         requeue_task_rt(rq, p);
230 }
231
232 static struct sched_class rt_sched_class __read_mostly = {
233         .enqueue_task           = enqueue_task_rt,
234         .dequeue_task           = dequeue_task_rt,
235         .yield_task             = yield_task_rt,
236
237         .check_preempt_curr     = check_preempt_curr_rt,
238
239         .pick_next_task         = pick_next_task_rt,
240         .put_prev_task          = put_prev_task_rt,
241
242         .load_balance           = load_balance_rt,
243
244         .task_tick              = task_tick_rt,
245 };