8acd09820f72dba67ae89e108660550dbdd0ffb6
[linux-2.6.git] / drivers / video / tegra / host / nvhost_intr.c
1 /*
2  * drivers/video/tegra/host/nvhost_intr.c
3  *
4  * Tegra Graphics Host Interrupt Management
5  *
6  * Copyright (c) 2010, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include "nvhost_intr.h"
24 #include "dev.h"
25 #include <linux/interrupt.h>
26 #include <linux/slab.h>
27 #include <linux/irq.h>
28
29 #define intr_to_dev(x) container_of(x, struct nvhost_master, intr)
30
31
32 /*** HW host sync management ***/
33
34 void init_host_sync(void __iomem *sync_regs)
35 {
36         /* disable the ip_busy_timeout. this prevents write drops, etc.
37          * there's no real way to recover from a hung client anyway.
38          */
39         writel(0, sync_regs + HOST1X_SYNC_IP_BUSY_TIMEOUT);
40
41         /* increase the auto-ack timout to the maximum value. 2d will hang
42          * otherwise on ap20.
43          */
44         writel(0xff, sync_regs + HOST1X_SYNC_CTXSW_TIMEOUT_CFG);
45 }
46
47 void set_host_clocks_per_microsecond(void __iomem *sync_regs, u32 cpm)
48 {
49         /* write microsecond clock register */
50         writel(cpm, sync_regs + HOST1X_SYNC_USEC_CLK);
51 }
52
53 static void set_syncpt_threshold(void __iomem *sync_regs, u32 id, u32 thresh)
54 {
55         thresh &= 0xffff;
56         writel(thresh, sync_regs + (HOST1X_SYNC_SYNCPT_INT_THRESH_0 + id * 4));
57 }
58
59 static void enable_syncpt_interrupt(void __iomem *sync_regs, u32 id)
60 {
61         writel(BIT(id), sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0);
62 }
63
64 void disable_all_syncpt_interrupts(void __iomem *sync_regs)
65 {
66         /* disable interrupts for both cpu's */
67         writel(0, sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE);
68
69         /* clear status for both cpu's */
70         writel(0xfffffffful, sync_regs +
71                 HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS);
72         writel(0xfffffffful, sync_regs +
73                 HOST1X_SYNC_SYNCPT_THRESH_CPU1_INT_STATUS);
74 }
75
76
77 /*** Wait list management ***/
78
79 struct nvhost_waitlist {
80         struct list_head list;
81         struct kref refcount;
82         u32 thresh;
83         enum nvhost_intr_action action;
84         atomic_t state;
85         void *data;
86         int count;
87 };
88
89 enum waitlist_state
90 {
91         WLS_PENDING,
92         WLS_REMOVED,
93         WLS_CANCELLED,
94         WLS_HANDLED
95 };
96
97 static void waiter_release(struct kref *kref)
98 {
99         kfree(container_of(kref, struct nvhost_waitlist, refcount));
100 }
101
102 /**
103  * add a waiter to a waiter queue, sorted by threshold
104  * returns true if it was added at the head of the queue
105  */
106 static bool add_waiter_to_queue(struct nvhost_waitlist *waiter,
107                                 struct list_head *queue)
108 {
109         struct nvhost_waitlist *pos;
110         u32 thresh = waiter->thresh;
111
112         list_for_each_entry_reverse(pos, queue, list)
113                 if ((s32)(pos->thresh - thresh) <= 0) {
114                         list_add(&waiter->list, &pos->list);
115                         return false;
116                 }
117
118         list_add(&waiter->list, queue);
119         return true;
120 }
121
122 /**
123  * run through a waiter queue for a single sync point ID
124  * and gather all completed waiters into lists by actions
125  */
126 static void remove_completed_waiters(struct list_head *head, u32 sync,
127                         struct list_head completed[NVHOST_INTR_ACTION_COUNT])
128 {
129         struct list_head *dest;
130         struct nvhost_waitlist *waiter, *next, *prev;
131
132         list_for_each_entry_safe(waiter, next, head, list) {
133                 if ((s32)(waiter->thresh - sync) > 0)
134                         break;
135
136                 dest = completed + waiter->action;
137
138                 /* consolidate submit cleanups */
139                 if (waiter->action == NVHOST_INTR_ACTION_SUBMIT_COMPLETE
140                         && !list_empty(dest)) {
141                         prev = list_entry(dest->prev,
142                                         struct nvhost_waitlist, list);
143                         if (prev->data == waiter->data) {
144                                 prev->count++;
145                                 dest = NULL;
146                         }
147                 }
148
149                 /* PENDING->REMOVED or CANCELLED->HANDLED */
150                 if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) {
151                         list_del(&waiter->list);
152                         kref_put(&waiter->refcount, waiter_release);
153                 } else {
154                         list_move_tail(&waiter->list, dest);
155                 }
156         }
157 }
158
159 void reset_threshold_interrupt(struct list_head *head,
160                 unsigned int id, void __iomem *sync_regs)
161 {
162         u32 thresh = list_first_entry(head,
163                                 struct nvhost_waitlist, list)->thresh;
164
165         set_syncpt_threshold(sync_regs, id, thresh);
166         enable_syncpt_interrupt(sync_regs, id);
167 }
168
169
170 static void action_submit_complete(struct nvhost_waitlist *waiter)
171 {
172         struct nvhost_channel *channel = waiter->data;
173         int nr_completed = waiter->count;
174
175         nvhost_cdma_update(&channel->cdma);
176         nvhost_module_idle_mult(&channel->mod, nr_completed);
177 }
178
179 static void action_ctxsave(struct nvhost_waitlist *waiter)
180 {
181         struct nvhost_hwctx *hwctx = waiter->data;
182         struct nvhost_channel *channel = hwctx->channel;
183
184         channel->ctxhandler.save_service(hwctx);
185         channel->ctxhandler.put(hwctx);
186 }
187
188 static void action_wakeup(struct nvhost_waitlist *waiter)
189 {
190         wait_queue_head_t *wq = waiter->data;
191
192         wake_up(wq);
193 }
194
195 static void action_wakeup_interruptible(struct nvhost_waitlist *waiter)
196 {
197         wait_queue_head_t *wq = waiter->data;
198
199         wake_up_interruptible(wq);
200 }
201
202 typedef void (*action_handler)(struct nvhost_waitlist *waiter);
203
204 static action_handler action_handlers[NVHOST_INTR_ACTION_COUNT] = {
205         action_submit_complete,
206         action_ctxsave,
207         action_wakeup,
208         action_wakeup_interruptible,
209 };
210
211 static void run_handlers(struct list_head completed[NVHOST_INTR_ACTION_COUNT])
212 {
213         struct list_head *head = completed;
214         int i;
215
216         for (i = 0; i < NVHOST_INTR_ACTION_COUNT; ++i, ++head) {
217                 action_handler handler = action_handlers[i];
218                 struct nvhost_waitlist *waiter, *next;
219
220                 list_for_each_entry_safe(waiter, next, head, list) {
221                         list_del(&waiter->list);
222                         handler(waiter);
223                         WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) != WLS_REMOVED);
224                         kref_put(&waiter->refcount, waiter_release);
225                 }
226         }
227 }
228
229 /**
230  * Remove & handle all waiters that have completed for the given syncpt
231  */
232 int process_wait_list(struct nvhost_intr_syncpt *syncpt,
233                 u32 threshold, void __iomem *sync_regs)
234 {
235         struct list_head completed[NVHOST_INTR_ACTION_COUNT];
236         unsigned int i;
237         int empty;
238
239         for (i = 0; i < NVHOST_INTR_ACTION_COUNT; ++i)
240                 INIT_LIST_HEAD(completed + i);
241
242         spin_lock(&syncpt->lock);
243
244         remove_completed_waiters(&syncpt->wait_head, threshold, completed);
245
246         empty = list_empty(&syncpt->wait_head);
247         if (!empty)
248                 reset_threshold_interrupt(&syncpt->wait_head,
249                                         syncpt->id, sync_regs);
250
251         spin_unlock(&syncpt->lock);
252
253         run_handlers(completed);
254
255         return empty;
256 }
257
258
259 /*** host syncpt interrupt service functions ***/
260
261 /**
262  * Sync point threshold interrupt service function
263  * Handles sync point threshold triggers, in interrupt context
264  */
265 static irqreturn_t syncpt_thresh_isr(int irq, void *dev_id)
266 {
267         struct nvhost_intr_syncpt *syncpt = dev_id;
268         unsigned int id = syncpt->id;
269         struct nvhost_intr *intr = container_of(syncpt, struct nvhost_intr,
270                                                 syncpt[id]);
271         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
272
273         writel(BIT(id),
274                 sync_regs + HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE);
275         writel(BIT(id),
276                 sync_regs + HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS);
277
278         return IRQ_WAKE_THREAD;
279 }
280
281 /**
282  * Sync point threshold interrupt service thread function
283  * Handles sync point threshold triggers, in thread context
284  */
285 static irqreturn_t syncpt_thresh_fn(int irq, void *dev_id)
286 {
287         struct nvhost_intr_syncpt *syncpt = dev_id;
288         unsigned int id = syncpt->id;
289         struct nvhost_intr *intr = container_of(syncpt, struct nvhost_intr,
290                                                 syncpt[id]);
291         struct nvhost_master *dev = intr_to_dev(intr);
292
293         (void)process_wait_list(syncpt,
294                         nvhost_syncpt_update_min(&dev->syncpt, id),
295                         dev->sync_aperture);
296
297         return IRQ_HANDLED;
298 }
299
300 /**
301  * lazily request a syncpt's irq
302  */
303 static int request_syncpt_irq(struct nvhost_intr_syncpt *syncpt)
304 {
305         int err;
306
307         if (syncpt->irq_requested)
308                 return 0;
309
310         err = request_threaded_irq(syncpt->irq,
311                                 syncpt_thresh_isr, syncpt_thresh_fn,
312                                 0, syncpt->thresh_irq_name, syncpt);
313         if (err)
314                 return err;
315
316         syncpt->irq_requested = 1;
317         return 0;
318 }
319
320 /**
321  * free a syncpt's irq. syncpt interrupt should be disabled first.
322  */
323 static void free_syncpt_irq(struct nvhost_intr_syncpt *syncpt)
324 {
325         if (syncpt->irq_requested) {
326                 free_irq(syncpt->irq, syncpt);
327                 syncpt->irq_requested = 0;
328         }
329 }
330
331
332 /*** host general interrupt service functions ***/
333
334 /**
335  * Host general interrupt service function
336  * Handles read / write failures
337  */
338 static irqreturn_t host1x_isr(int irq, void *dev_id)
339 {
340         struct nvhost_intr *intr = dev_id;
341         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
342         u32 stat;
343         u32 ext_stat;
344         u32 addr;
345
346         stat = readl(sync_regs + HOST1X_SYNC_HINTSTATUS);
347         ext_stat = readl(sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
348
349         if (nvhost_sync_hintstatus_ext_ip_read_int(ext_stat)) {
350                 addr = readl(sync_regs + HOST1X_SYNC_IP_READ_TIMEOUT_ADDR);
351                 pr_err("Host read timeout at address %x\n", addr);
352         }
353
354         if (nvhost_sync_hintstatus_ext_ip_write_int(ext_stat)) {
355                 addr = readl(sync_regs + HOST1X_SYNC_IP_WRITE_TIMEOUT_ADDR);
356                 pr_err("Host write timeout at address %x\n", addr);
357         }
358
359         writel(ext_stat, sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
360         writel(stat, sync_regs + HOST1X_SYNC_HINTSTATUS);
361
362         return IRQ_HANDLED;
363 }
364
365 static int request_host_general_irq(struct nvhost_intr *intr)
366 {
367         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
368         int err;
369
370         if (intr->host_general_irq_requested)
371                 return 0;
372
373         /* master disable for general (not syncpt) host interrupts */
374         writel(0, sync_regs + HOST1X_SYNC_INTMASK);
375
376         /* clear status & extstatus */
377         writel(0xfffffffful, sync_regs + HOST1X_SYNC_HINTSTATUS_EXT);
378         writel(0xfffffffful, sync_regs + HOST1X_SYNC_HINTSTATUS);
379
380         err = request_irq(intr->host_general_irq, host1x_isr, 0,
381                         "host_status", intr);
382         if (err)
383                 return err;
384
385         /* enable extra interrupt sources IP_READ_INT and IP_WRITE_INT */
386         writel(BIT(30) | BIT(31), sync_regs + HOST1X_SYNC_HINTMASK_EXT);
387
388         /* enable extra interrupt sources */
389         writel(BIT(31), sync_regs + HOST1X_SYNC_HINTMASK);
390
391         /* enable host module interrupt to CPU0 */
392         writel(BIT(0), sync_regs + HOST1X_SYNC_INTC0MASK);
393
394         /* master enable for general (not syncpt) host interrupts */
395         writel(BIT(0), sync_regs + HOST1X_SYNC_INTMASK);
396
397         intr->host_general_irq_requested = true;
398
399         return err;
400 }
401
402 static void free_host_general_irq(struct nvhost_intr *intr)
403 {
404         if (intr->host_general_irq_requested) {
405                 void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
406
407                 /* master disable for general (not syncpt) host interrupts */
408                 writel(0, sync_regs + HOST1X_SYNC_INTMASK);
409
410                 free_irq(intr->host_general_irq, intr);
411                 intr->host_general_irq_requested = false;
412         }
413 }
414
415
416 /*** Main API ***/
417
418 int nvhost_intr_add_action(struct nvhost_intr *intr, u32 id, u32 thresh,
419                         enum nvhost_intr_action action, void *data,
420                         void **ref)
421 {
422         struct nvhost_waitlist *waiter;
423         struct nvhost_intr_syncpt *syncpt;
424         void __iomem *sync_regs;
425         int queue_was_empty;
426         int err;
427
428         /* create and initialize a new waiter */
429         waiter = kmalloc(sizeof(*waiter), GFP_KERNEL);
430         if (!waiter)
431                 return -ENOMEM;
432         INIT_LIST_HEAD(&waiter->list);
433         kref_init(&waiter->refcount);
434         if (ref)
435                 kref_get(&waiter->refcount);
436         waiter->thresh = thresh;
437         waiter->action = action;
438         atomic_set(&waiter->state, WLS_PENDING);
439         waiter->data = data;
440         waiter->count = 1;
441
442         BUG_ON(id >= NV_HOST1X_SYNCPT_NB_PTS);
443         syncpt = intr->syncpt + id;
444         sync_regs = intr_to_dev(intr)->sync_aperture;
445
446         spin_lock(&syncpt->lock);
447
448         /* lazily request irq for this sync point */
449         if (!syncpt->irq_requested) {
450                 spin_unlock(&syncpt->lock);
451
452                 mutex_lock(&intr->mutex);
453                 err = request_syncpt_irq(syncpt);
454                 mutex_unlock(&intr->mutex);
455
456                 if (err) {
457                         kfree(waiter);
458                         return err;
459                 }
460
461                 spin_lock(&syncpt->lock);
462         }
463
464         queue_was_empty = list_empty(&syncpt->wait_head);
465
466         if (add_waiter_to_queue(waiter, &syncpt->wait_head)) {
467                 /* added at head of list - new threshold value */
468                 set_syncpt_threshold(sync_regs, id, thresh);
469
470                 /* added as first waiter - enable interrupt */
471                 if (queue_was_empty)
472                         enable_syncpt_interrupt(sync_regs, id);
473         }
474
475         spin_unlock(&syncpt->lock);
476
477         if (ref)
478                 *ref = waiter;
479         return 0;
480 }
481
482 void nvhost_intr_put_ref(struct nvhost_intr *intr, void *ref)
483 {
484         struct nvhost_waitlist *waiter = ref;
485
486         while (atomic_cmpxchg(&waiter->state,
487                                 WLS_PENDING, WLS_CANCELLED) == WLS_REMOVED)
488                 schedule();
489
490         kref_put(&waiter->refcount, waiter_release);
491 }
492
493
494 /*** Init & shutdown ***/
495
496 int nvhost_intr_init(struct nvhost_intr *intr, u32 irq_gen, u32 irq_sync)
497 {
498         unsigned int id;
499         struct nvhost_intr_syncpt *syncpt;
500
501         mutex_init(&intr->mutex);
502         intr->host_general_irq = irq_gen;
503         intr->host_general_irq_requested = false;
504
505         for (id = 0, syncpt = intr->syncpt;
506              id < NV_HOST1X_SYNCPT_NB_PTS;
507              ++id, ++syncpt) {
508                 syncpt->id = id;
509                 syncpt->irq = irq_sync + id;
510                 syncpt->irq_requested = 0;
511                 spin_lock_init(&syncpt->lock);
512                 INIT_LIST_HEAD(&syncpt->wait_head);
513                 snprintf(syncpt->thresh_irq_name,
514                         sizeof(syncpt->thresh_irq_name),
515                         "host_sp_%02d", id);
516         }
517
518         return 0;
519 }
520
521 void nvhost_intr_deinit(struct nvhost_intr *intr)
522 {
523         nvhost_intr_stop(intr);
524 }
525
526 void nvhost_intr_start(struct nvhost_intr *intr, u32 hz)
527 {
528         struct nvhost_master *dev = intr_to_dev(intr);
529         void __iomem *sync_regs = dev->sync_aperture;
530
531         mutex_lock(&intr->mutex);
532
533         init_host_sync(sync_regs);
534         set_host_clocks_per_microsecond(sync_regs, (hz + 1000000 - 1)/1000000);
535
536         request_host_general_irq(intr);
537
538         mutex_unlock(&intr->mutex);
539 }
540
541 void nvhost_intr_stop(struct nvhost_intr *intr)
542 {
543         void __iomem *sync_regs = intr_to_dev(intr)->sync_aperture;
544         unsigned int id;
545         struct nvhost_intr_syncpt *syncpt;
546
547         mutex_lock(&intr->mutex);
548
549         disable_all_syncpt_interrupts(sync_regs);
550
551         for (id = 0, syncpt = intr->syncpt;
552              id < NV_HOST1X_SYNCPT_NB_PTS;
553              ++id, ++syncpt) {
554                 struct nvhost_waitlist *waiter, *next;
555                 list_for_each_entry_safe(waiter, next, &syncpt->wait_head, list) {
556                         if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED)
557                                 == WLS_CANCELLED) {
558                                 list_del(&waiter->list);
559                                 kref_put(&waiter->refcount, waiter_release);
560                         }
561                 }
562
563                 if(!list_empty(&syncpt->wait_head)) {  // output diagnostics
564                         printk("%s id=%d\n",__func__,id);
565                         BUG_ON(1);
566                 }
567
568                 free_syncpt_irq(syncpt);
569         }
570
571         free_host_general_irq(intr);
572
573         mutex_unlock(&intr->mutex);
574 }