Merge branch 'samsung/exynos5' into next/soc2
[linux-2.6.git] / arch / arm / mach-tegra / dma.c
1 /*
2  * arch/arm/mach-tegra/dma.c
3  *
4  * System DMA driver for NVIDIA Tegra SoCs
5  *
6  * Copyright (c) 2008-2009, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/io.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/err.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <linux/clk.h>
31 #include <mach/dma.h>
32 #include <mach/irqs.h>
33 #include <mach/iomap.h>
34 #include <mach/suspend.h>
35
36 #include "apbio.h"
37
38 #define APB_DMA_GEN                             0x000
39 #define GEN_ENABLE                              (1<<31)
40
41 #define APB_DMA_CNTRL                           0x010
42
43 #define APB_DMA_IRQ_MASK                        0x01c
44
45 #define APB_DMA_IRQ_MASK_SET                    0x020
46
47 #define APB_DMA_CHAN_CSR                        0x000
48 #define CSR_ENB                                 (1<<31)
49 #define CSR_IE_EOC                              (1<<30)
50 #define CSR_HOLD                                (1<<29)
51 #define CSR_DIR                                 (1<<28)
52 #define CSR_ONCE                                (1<<27)
53 #define CSR_FLOW                                (1<<21)
54 #define CSR_REQ_SEL_SHIFT                       16
55 #define CSR_WCOUNT_SHIFT                        2
56 #define CSR_WCOUNT_MASK                         0xFFFC
57
58 #define APB_DMA_CHAN_STA                                0x004
59 #define STA_BUSY                                (1<<31)
60 #define STA_ISE_EOC                             (1<<30)
61 #define STA_HALT                                (1<<29)
62 #define STA_PING_PONG                           (1<<28)
63 #define STA_COUNT_SHIFT                         2
64 #define STA_COUNT_MASK                          0xFFFC
65
66 #define APB_DMA_CHAN_AHB_PTR                            0x010
67
68 #define APB_DMA_CHAN_AHB_SEQ                            0x014
69 #define AHB_SEQ_INTR_ENB                        (1<<31)
70 #define AHB_SEQ_BUS_WIDTH_SHIFT                 28
71 #define AHB_SEQ_BUS_WIDTH_MASK                  (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
72 #define AHB_SEQ_BUS_WIDTH_8                     (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
73 #define AHB_SEQ_BUS_WIDTH_16                    (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
74 #define AHB_SEQ_BUS_WIDTH_32                    (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
75 #define AHB_SEQ_BUS_WIDTH_64                    (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
76 #define AHB_SEQ_BUS_WIDTH_128                   (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
77 #define AHB_SEQ_DATA_SWAP                       (1<<27)
78 #define AHB_SEQ_BURST_MASK                      (0x7<<24)
79 #define AHB_SEQ_BURST_1                         (4<<24)
80 #define AHB_SEQ_BURST_4                         (5<<24)
81 #define AHB_SEQ_BURST_8                         (6<<24)
82 #define AHB_SEQ_DBL_BUF                         (1<<19)
83 #define AHB_SEQ_WRAP_SHIFT                      16
84 #define AHB_SEQ_WRAP_MASK                       (0x7<<AHB_SEQ_WRAP_SHIFT)
85
86 #define APB_DMA_CHAN_APB_PTR                            0x018
87
88 #define APB_DMA_CHAN_APB_SEQ                            0x01c
89 #define APB_SEQ_BUS_WIDTH_SHIFT                 28
90 #define APB_SEQ_BUS_WIDTH_MASK                  (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
91 #define APB_SEQ_BUS_WIDTH_8                     (0<<APB_SEQ_BUS_WIDTH_SHIFT)
92 #define APB_SEQ_BUS_WIDTH_16                    (1<<APB_SEQ_BUS_WIDTH_SHIFT)
93 #define APB_SEQ_BUS_WIDTH_32                    (2<<APB_SEQ_BUS_WIDTH_SHIFT)
94 #define APB_SEQ_BUS_WIDTH_64                    (3<<APB_SEQ_BUS_WIDTH_SHIFT)
95 #define APB_SEQ_BUS_WIDTH_128                   (4<<APB_SEQ_BUS_WIDTH_SHIFT)
96 #define APB_SEQ_DATA_SWAP                       (1<<27)
97 #define APB_SEQ_WRAP_SHIFT                      16
98 #define APB_SEQ_WRAP_MASK                       (0x7<<APB_SEQ_WRAP_SHIFT)
99
100 #define TEGRA_SYSTEM_DMA_CH_NR                  16
101 #define TEGRA_SYSTEM_DMA_AVP_CH_NUM             4
102 #define TEGRA_SYSTEM_DMA_CH_MIN                 0
103 #define TEGRA_SYSTEM_DMA_CH_MAX \
104         (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
105
106 #define NV_DMA_MAX_TRASFER_SIZE 0x10000
107
108 static const unsigned int ahb_addr_wrap_table[8] = {
109         0, 32, 64, 128, 256, 512, 1024, 2048
110 };
111
112 static const unsigned int apb_addr_wrap_table[8] = {
113         0, 1, 2, 4, 8, 16, 32, 64
114 };
115
116 static const unsigned int bus_width_table[5] = {
117         8, 16, 32, 64, 128
118 };
119
120 #define TEGRA_DMA_NAME_SIZE 16
121 struct tegra_dma_channel {
122         struct list_head        list;
123         int                     id;
124         spinlock_t              lock;
125         char                    name[TEGRA_DMA_NAME_SIZE];
126         void  __iomem           *addr;
127         int                     mode;
128         int                     irq;
129         int                     req_transfer_count;
130 };
131
132 #define  NV_DMA_MAX_CHANNELS  32
133
134 static bool tegra_dma_initialized;
135 static DEFINE_MUTEX(tegra_dma_lock);
136 static DEFINE_SPINLOCK(enable_lock);
137
138 static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
139 static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
140
141 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
142         struct tegra_dma_req *req);
143 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
144         struct tegra_dma_req *req);
145 static void tegra_dma_stop(struct tegra_dma_channel *ch);
146
147 void tegra_dma_flush(struct tegra_dma_channel *ch)
148 {
149 }
150 EXPORT_SYMBOL(tegra_dma_flush);
151
152 void tegra_dma_dequeue(struct tegra_dma_channel *ch)
153 {
154         struct tegra_dma_req *req;
155
156         if (tegra_dma_is_empty(ch))
157                 return;
158
159         req = list_entry(ch->list.next, typeof(*req), node);
160
161         tegra_dma_dequeue_req(ch, req);
162         return;
163 }
164
165 static void tegra_dma_stop(struct tegra_dma_channel *ch)
166 {
167         u32 csr;
168         u32 status;
169
170         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
171         csr &= ~CSR_IE_EOC;
172         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
173
174         csr &= ~CSR_ENB;
175         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
176
177         status = readl(ch->addr + APB_DMA_CHAN_STA);
178         if (status & STA_ISE_EOC)
179                 writel(status, ch->addr + APB_DMA_CHAN_STA);
180 }
181
182 static int tegra_dma_cancel(struct tegra_dma_channel *ch)
183 {
184         unsigned long irq_flags;
185
186         spin_lock_irqsave(&ch->lock, irq_flags);
187         while (!list_empty(&ch->list))
188                 list_del(ch->list.next);
189
190         tegra_dma_stop(ch);
191
192         spin_unlock_irqrestore(&ch->lock, irq_flags);
193         return 0;
194 }
195
196 static unsigned int get_channel_status(struct tegra_dma_channel *ch,
197                         struct tegra_dma_req *req, bool is_stop_dma)
198 {
199         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
200         unsigned int status;
201
202         if (is_stop_dma) {
203                 /*
204                  * STOP the DMA and get the transfer count.
205                  * Getting the transfer count is tricky.
206                  *  - Globally disable DMA on all channels
207                  *  - Read the channel's status register to know the number
208                  *    of pending bytes to be transfered.
209                  *  - Stop the dma channel
210                  *  - Globally re-enable DMA to resume other transfers
211                  */
212                 spin_lock(&enable_lock);
213                 writel(0, addr + APB_DMA_GEN);
214                 udelay(20);
215                 status = readl(ch->addr + APB_DMA_CHAN_STA);
216                 tegra_dma_stop(ch);
217                 writel(GEN_ENABLE, addr + APB_DMA_GEN);
218                 spin_unlock(&enable_lock);
219                 if (status & STA_ISE_EOC) {
220                         pr_err("Got Dma Int here clearing");
221                         writel(status, ch->addr + APB_DMA_CHAN_STA);
222                 }
223                 req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
224         } else {
225                 status = readl(ch->addr + APB_DMA_CHAN_STA);
226         }
227         return status;
228 }
229
230 /* should be called with the channel lock held */
231 static unsigned int dma_active_count(struct tegra_dma_channel *ch,
232         struct tegra_dma_req *req, unsigned int status)
233 {
234         unsigned int to_transfer;
235         unsigned int req_transfer_count;
236         unsigned int bytes_transferred;
237
238         to_transfer = ((status & STA_COUNT_MASK) >> STA_COUNT_SHIFT) + 1;
239         req_transfer_count = ch->req_transfer_count + 1;
240         bytes_transferred = req_transfer_count;
241         if (status & STA_BUSY)
242                 bytes_transferred -= to_transfer;
243         /*
244          * In continuous transfer mode, DMA only tracks the count of the
245          * half DMA buffer. So, if the DMA already finished half the DMA
246          * then add the half buffer to the completed count.
247          */
248         if (ch->mode & TEGRA_DMA_MODE_CONTINOUS) {
249                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
250                         bytes_transferred += req_transfer_count;
251                 if (status & STA_ISE_EOC)
252                         bytes_transferred += req_transfer_count;
253         }
254         bytes_transferred *= 4;
255         return bytes_transferred;
256 }
257
258 int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
259         struct tegra_dma_req *_req)
260 {
261         unsigned int status;
262         struct tegra_dma_req *req = NULL;
263         int found = 0;
264         unsigned long irq_flags;
265         int stop = 0;
266
267         spin_lock_irqsave(&ch->lock, irq_flags);
268
269         if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
270                 stop = 1;
271
272         list_for_each_entry(req, &ch->list, node) {
273                 if (req == _req) {
274                         list_del(&req->node);
275                         found = 1;
276                         break;
277                 }
278         }
279         if (!found) {
280                 spin_unlock_irqrestore(&ch->lock, irq_flags);
281                 return 0;
282         }
283
284         if (!stop)
285                 goto skip_stop_dma;
286
287         status = get_channel_status(ch, req, true);
288         req->bytes_transferred = dma_active_count(ch, req, status);
289
290         if (!list_empty(&ch->list)) {
291                 /* if the list is not empty, queue the next request */
292                 struct tegra_dma_req *next_req;
293                 next_req = list_entry(ch->list.next,
294                         typeof(*next_req), node);
295                 tegra_dma_update_hw(ch, next_req);
296         }
297
298 skip_stop_dma:
299         req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
300
301         spin_unlock_irqrestore(&ch->lock, irq_flags);
302
303         /* Callback should be called without any lock */
304         req->complete(req);
305         return 0;
306 }
307 EXPORT_SYMBOL(tegra_dma_dequeue_req);
308
309 bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
310 {
311         unsigned long irq_flags;
312         bool is_empty;
313
314         spin_lock_irqsave(&ch->lock, irq_flags);
315         if (list_empty(&ch->list))
316                 is_empty = true;
317         else
318                 is_empty = false;
319         spin_unlock_irqrestore(&ch->lock, irq_flags);
320         return is_empty;
321 }
322 EXPORT_SYMBOL(tegra_dma_is_empty);
323
324 bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
325         struct tegra_dma_req *_req)
326 {
327         unsigned long irq_flags;
328         struct tegra_dma_req *req;
329
330         spin_lock_irqsave(&ch->lock, irq_flags);
331         list_for_each_entry(req, &ch->list, node) {
332                 if (req == _req) {
333                         spin_unlock_irqrestore(&ch->lock, irq_flags);
334                         return true;
335                 }
336         }
337         spin_unlock_irqrestore(&ch->lock, irq_flags);
338         return false;
339 }
340 EXPORT_SYMBOL(tegra_dma_is_req_inflight);
341
342 int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
343         struct tegra_dma_req *req)
344 {
345         unsigned long irq_flags;
346         struct tegra_dma_req *_req;
347         int start_dma = 0;
348
349         if (req->size > NV_DMA_MAX_TRASFER_SIZE ||
350                 req->source_addr & 0x3 || req->dest_addr & 0x3) {
351                 pr_err("Invalid DMA request for channel %d\n", ch->id);
352                 return -EINVAL;
353         }
354
355         spin_lock_irqsave(&ch->lock, irq_flags);
356
357         list_for_each_entry(_req, &ch->list, node) {
358                 if (req == _req) {
359                     spin_unlock_irqrestore(&ch->lock, irq_flags);
360                     return -EEXIST;
361                 }
362         }
363
364         req->bytes_transferred = 0;
365         req->status = 0;
366         req->buffer_status = 0;
367         if (list_empty(&ch->list))
368                 start_dma = 1;
369
370         list_add_tail(&req->node, &ch->list);
371
372         if (start_dma)
373                 tegra_dma_update_hw(ch, req);
374
375         spin_unlock_irqrestore(&ch->lock, irq_flags);
376
377         return 0;
378 }
379 EXPORT_SYMBOL(tegra_dma_enqueue_req);
380
381 struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
382 {
383         int channel;
384         struct tegra_dma_channel *ch = NULL;
385
386         if (!tegra_dma_initialized)
387                 return NULL;
388
389         mutex_lock(&tegra_dma_lock);
390
391         /* first channel is the shared channel */
392         if (mode & TEGRA_DMA_SHARED) {
393                 channel = TEGRA_SYSTEM_DMA_CH_MIN;
394         } else {
395                 channel = find_first_zero_bit(channel_usage,
396                         ARRAY_SIZE(dma_channels));
397                 if (channel >= ARRAY_SIZE(dma_channels))
398                         goto out;
399         }
400         __set_bit(channel, channel_usage);
401         ch = &dma_channels[channel];
402         ch->mode = mode;
403
404 out:
405         mutex_unlock(&tegra_dma_lock);
406         return ch;
407 }
408 EXPORT_SYMBOL(tegra_dma_allocate_channel);
409
410 void tegra_dma_free_channel(struct tegra_dma_channel *ch)
411 {
412         if (ch->mode & TEGRA_DMA_SHARED)
413                 return;
414         tegra_dma_cancel(ch);
415         mutex_lock(&tegra_dma_lock);
416         __clear_bit(ch->id, channel_usage);
417         mutex_unlock(&tegra_dma_lock);
418 }
419 EXPORT_SYMBOL(tegra_dma_free_channel);
420
421 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
422         struct tegra_dma_req *req)
423 {
424         u32 apb_ptr;
425         u32 ahb_ptr;
426
427         if (req->to_memory) {
428                 apb_ptr = req->source_addr;
429                 ahb_ptr = req->dest_addr;
430         } else {
431                 apb_ptr = req->dest_addr;
432                 ahb_ptr = req->source_addr;
433         }
434         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
435         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
436
437         req->status = TEGRA_DMA_REQ_INFLIGHT;
438         return;
439 }
440
441 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
442         struct tegra_dma_req *req)
443 {
444         int ahb_addr_wrap;
445         int apb_addr_wrap;
446         int ahb_bus_width;
447         int apb_bus_width;
448         int index;
449
450         u32 ahb_seq;
451         u32 apb_seq;
452         u32 ahb_ptr;
453         u32 apb_ptr;
454         u32 csr;
455
456         csr = CSR_IE_EOC | CSR_FLOW;
457         ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1;
458         apb_seq = 0;
459
460         csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
461
462         /* One shot mode is always single buffered,
463          * continuous mode is always double buffered
464          * */
465         if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
466                 csr |= CSR_ONCE;
467                 ch->req_transfer_count = (req->size >> 2) - 1;
468         } else {
469                 ahb_seq |= AHB_SEQ_DBL_BUF;
470
471                 /* In double buffered mode, we set the size to half the
472                  * requested size and interrupt when half the buffer
473                  * is full */
474                 ch->req_transfer_count = (req->size >> 3) - 1;
475         }
476
477         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
478
479         if (req->to_memory) {
480                 apb_ptr = req->source_addr;
481                 ahb_ptr = req->dest_addr;
482
483                 apb_addr_wrap = req->source_wrap;
484                 ahb_addr_wrap = req->dest_wrap;
485                 apb_bus_width = req->source_bus_width;
486                 ahb_bus_width = req->dest_bus_width;
487
488         } else {
489                 csr |= CSR_DIR;
490                 apb_ptr = req->dest_addr;
491                 ahb_ptr = req->source_addr;
492
493                 apb_addr_wrap = req->dest_wrap;
494                 ahb_addr_wrap = req->source_wrap;
495                 apb_bus_width = req->dest_bus_width;
496                 ahb_bus_width = req->source_bus_width;
497         }
498
499         apb_addr_wrap >>= 2;
500         ahb_addr_wrap >>= 2;
501
502         /* set address wrap for APB size */
503         index = 0;
504         do  {
505                 if (apb_addr_wrap_table[index] == apb_addr_wrap)
506                         break;
507                 index++;
508         } while (index < ARRAY_SIZE(apb_addr_wrap_table));
509         BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
510         apb_seq |= index << APB_SEQ_WRAP_SHIFT;
511
512         /* set address wrap for AHB size */
513         index = 0;
514         do  {
515                 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
516                         break;
517                 index++;
518         } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
519         BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
520         ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
521
522         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
523                 if (bus_width_table[index] == ahb_bus_width)
524                         break;
525         }
526         BUG_ON(index == ARRAY_SIZE(bus_width_table));
527         ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
528
529         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
530                 if (bus_width_table[index] == apb_bus_width)
531                         break;
532         }
533         BUG_ON(index == ARRAY_SIZE(bus_width_table));
534         apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
535
536         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
537         writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
538         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
539         writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
540         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
541
542         csr |= CSR_ENB;
543         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
544
545         req->status = TEGRA_DMA_REQ_INFLIGHT;
546 }
547
548 static void handle_oneshot_dma(struct tegra_dma_channel *ch)
549 {
550         struct tegra_dma_req *req;
551         unsigned long irq_flags;
552
553         spin_lock_irqsave(&ch->lock, irq_flags);
554         if (list_empty(&ch->list)) {
555                 spin_unlock_irqrestore(&ch->lock, irq_flags);
556                 return;
557         }
558
559         req = list_entry(ch->list.next, typeof(*req), node);
560         if (req) {
561                 int bytes_transferred;
562
563                 bytes_transferred = ch->req_transfer_count;
564                 bytes_transferred += 1;
565                 bytes_transferred <<= 2;
566
567                 list_del(&req->node);
568                 req->bytes_transferred = bytes_transferred;
569                 req->status = TEGRA_DMA_REQ_SUCCESS;
570
571                 spin_unlock_irqrestore(&ch->lock, irq_flags);
572                 /* Callback should be called without any lock */
573                 pr_debug("%s: transferred %d bytes\n", __func__,
574                         req->bytes_transferred);
575                 req->complete(req);
576                 spin_lock_irqsave(&ch->lock, irq_flags);
577         }
578
579         if (!list_empty(&ch->list)) {
580                 req = list_entry(ch->list.next, typeof(*req), node);
581                 /* the complete function we just called may have enqueued
582                    another req, in which case dma has already started */
583                 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
584                         tegra_dma_update_hw(ch, req);
585         }
586         spin_unlock_irqrestore(&ch->lock, irq_flags);
587 }
588
589 static void handle_continuous_dma(struct tegra_dma_channel *ch)
590 {
591         struct tegra_dma_req *req;
592         unsigned long irq_flags;
593
594         spin_lock_irqsave(&ch->lock, irq_flags);
595         if (list_empty(&ch->list)) {
596                 spin_unlock_irqrestore(&ch->lock, irq_flags);
597                 return;
598         }
599
600         req = list_entry(ch->list.next, typeof(*req), node);
601         if (req) {
602                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
603                         bool is_dma_ping_complete;
604                         is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
605                                                 & STA_PING_PONG) ? true : false;
606                         if (req->to_memory)
607                                 is_dma_ping_complete = !is_dma_ping_complete;
608                         /* Out of sync - Release current buffer */
609                         if (!is_dma_ping_complete) {
610                                 int bytes_transferred;
611
612                                 bytes_transferred = ch->req_transfer_count;
613                                 bytes_transferred += 1;
614                                 bytes_transferred <<= 3;
615                                 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
616                                 req->bytes_transferred = bytes_transferred;
617                                 req->status = TEGRA_DMA_REQ_SUCCESS;
618                                 tegra_dma_stop(ch);
619
620                                 if (!list_is_last(&req->node, &ch->list)) {
621                                         struct tegra_dma_req *next_req;
622
623                                         next_req = list_entry(req->node.next,
624                                                 typeof(*next_req), node);
625                                         tegra_dma_update_hw(ch, next_req);
626                                 }
627
628                                 list_del(&req->node);
629
630                                 /* DMA lock is NOT held when callbak is called */
631                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
632                                 req->complete(req);
633                                 return;
634                         }
635                         /* Load the next request into the hardware, if available
636                          * */
637                         if (!list_is_last(&req->node, &ch->list)) {
638                                 struct tegra_dma_req *next_req;
639
640                                 next_req = list_entry(req->node.next,
641                                         typeof(*next_req), node);
642                                 tegra_dma_update_hw_partial(ch, next_req);
643                         }
644                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
645                         req->status = TEGRA_DMA_REQ_SUCCESS;
646                         /* DMA lock is NOT held when callback is called */
647                         spin_unlock_irqrestore(&ch->lock, irq_flags);
648                         if (likely(req->threshold))
649                                 req->threshold(req);
650                         return;
651
652                 } else if (req->buffer_status ==
653                         TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
654                         /* Callback when the buffer is completely full (i.e on
655                          * the second  interrupt */
656                         int bytes_transferred;
657
658                         bytes_transferred = ch->req_transfer_count;
659                         bytes_transferred += 1;
660                         bytes_transferred <<= 3;
661
662                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
663                         req->bytes_transferred = bytes_transferred;
664                         req->status = TEGRA_DMA_REQ_SUCCESS;
665                         list_del(&req->node);
666
667                         /* DMA lock is NOT held when callbak is called */
668                         spin_unlock_irqrestore(&ch->lock, irq_flags);
669                         req->complete(req);
670                         return;
671
672                 } else {
673                         BUG();
674                 }
675         }
676         spin_unlock_irqrestore(&ch->lock, irq_flags);
677 }
678
679 static irqreturn_t dma_isr(int irq, void *data)
680 {
681         struct tegra_dma_channel *ch = data;
682         unsigned long status;
683
684         status = readl(ch->addr + APB_DMA_CHAN_STA);
685         if (status & STA_ISE_EOC)
686                 writel(status, ch->addr + APB_DMA_CHAN_STA);
687         else {
688                 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
689                 return IRQ_HANDLED;
690         }
691         return IRQ_WAKE_THREAD;
692 }
693
694 static irqreturn_t dma_thread_fn(int irq, void *data)
695 {
696         struct tegra_dma_channel *ch = data;
697
698         if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
699                 handle_oneshot_dma(ch);
700         else
701                 handle_continuous_dma(ch);
702
703
704         return IRQ_HANDLED;
705 }
706
707 int __init tegra_dma_init(void)
708 {
709         int ret = 0;
710         int i;
711         unsigned int irq;
712         void __iomem *addr;
713         struct clk *c;
714
715         bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
716
717         c = clk_get_sys("tegra-dma", NULL);
718         if (IS_ERR(c)) {
719                 pr_err("Unable to get clock for APB DMA\n");
720                 ret = PTR_ERR(c);
721                 goto fail;
722         }
723         ret = clk_enable(c);
724         if (ret != 0) {
725                 pr_err("Unable to enable clock for APB DMA\n");
726                 goto fail;
727         }
728
729         addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
730         writel(GEN_ENABLE, addr + APB_DMA_GEN);
731         writel(0, addr + APB_DMA_CNTRL);
732         writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
733                addr + APB_DMA_IRQ_MASK_SET);
734
735         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
736                 struct tegra_dma_channel *ch = &dma_channels[i];
737
738                 ch->id = i;
739                 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
740
741                 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
742                         TEGRA_APB_DMA_CH0_SIZE * i);
743
744                 spin_lock_init(&ch->lock);
745                 INIT_LIST_HEAD(&ch->list);
746
747                 irq = INT_APB_DMA_CH0 + i;
748                 ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0,
749                         dma_channels[i].name, ch);
750                 if (ret) {
751                         pr_err("Failed to register IRQ %d for DMA %d\n",
752                                 irq, i);
753                         goto fail;
754                 }
755                 ch->irq = irq;
756
757                 __clear_bit(i, channel_usage);
758         }
759         /* mark the shared channel allocated */
760         __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
761
762         tegra_dma_initialized = true;
763
764         return 0;
765 fail:
766         writel(0, addr + APB_DMA_GEN);
767         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
768                 struct tegra_dma_channel *ch = &dma_channels[i];
769                 if (ch->irq)
770                         free_irq(ch->irq, ch);
771         }
772         return ret;
773 }
774 postcore_initcall(tegra_dma_init);
775
776 #ifdef CONFIG_PM
777 static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
778
779 void tegra_dma_suspend(void)
780 {
781         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
782         u32 *ctx = apb_dma;
783         int i;
784
785         *ctx++ = readl(addr + APB_DMA_GEN);
786         *ctx++ = readl(addr + APB_DMA_CNTRL);
787         *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
788
789         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
790                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
791                                   TEGRA_APB_DMA_CH0_SIZE * i);
792
793                 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
794                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
795                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
796                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
797                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
798         }
799 }
800
801 void tegra_dma_resume(void)
802 {
803         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
804         u32 *ctx = apb_dma;
805         int i;
806
807         writel(*ctx++, addr + APB_DMA_GEN);
808         writel(*ctx++, addr + APB_DMA_CNTRL);
809         writel(*ctx++, addr + APB_DMA_IRQ_MASK);
810
811         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
812                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
813                                   TEGRA_APB_DMA_CH0_SIZE * i);
814
815                 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
816                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
817                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
818                 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
819                 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
820         }
821 }
822
823 #endif