DTV: dma: fixed dma burst size issue DTV xmit
[linux-2.6.git] / arch / arm / mach-tegra / dma.c
1 /*
2  * arch/arm/mach-tegra/dma.c
3  *
4  * System DMA driver for NVIDIA Tegra SoCs
5  *
6  * Copyright (c) 2008-2011, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/io.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/err.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <linux/clk.h>
31 #include <linux/syscore_ops.h>
32 #include <mach/dma.h>
33 #include <mach/irqs.h>
34 #include <mach/iomap.h>
35 #include <mach/clk.h>
36
37 #define APB_DMA_GEN                             0x000
38 #define GEN_ENABLE                              (1<<31)
39
40 #define APB_DMA_CNTRL                           0x010
41
42 #define APB_DMA_IRQ_MASK                        0x01c
43
44 #define APB_DMA_IRQ_MASK_SET                    0x020
45
46 #define APB_DMA_CHAN_CSR                        0x000
47 #define CSR_ENB                                 (1<<31)
48 #define CSR_IE_EOC                              (1<<30)
49 #define CSR_HOLD                                (1<<29)
50 #define CSR_DIR                                 (1<<28)
51 #define CSR_ONCE                                (1<<27)
52 #define CSR_FLOW                                (1<<21)
53 #define CSR_REQ_SEL_SHIFT                       16
54 #define CSR_REQ_SEL_MASK                        (0x1F<<CSR_REQ_SEL_SHIFT)
55 #define CSR_WCOUNT_SHIFT                        2
56 #define CSR_WCOUNT_MASK                         0xFFFC
57
58 #define APB_DMA_CHAN_STA                                0x004
59 #define STA_BUSY                                (1<<31)
60 #define STA_ISE_EOC                             (1<<30)
61 #define STA_HALT                                (1<<29)
62 #define STA_PING_PONG                           (1<<28)
63 #define STA_COUNT_SHIFT                         2
64 #define STA_COUNT_MASK                          0xFFFC
65
66 #define APB_DMA_CHAN_AHB_PTR                            0x010
67
68 #define APB_DMA_CHAN_AHB_SEQ                            0x014
69 #define AHB_SEQ_INTR_ENB                        (1<<31)
70 #define AHB_SEQ_BUS_WIDTH_SHIFT                 28
71 #define AHB_SEQ_BUS_WIDTH_MASK                  (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
72 #define AHB_SEQ_BUS_WIDTH_8                     (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
73 #define AHB_SEQ_BUS_WIDTH_16                    (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
74 #define AHB_SEQ_BUS_WIDTH_32                    (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
75 #define AHB_SEQ_BUS_WIDTH_64                    (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
76 #define AHB_SEQ_BUS_WIDTH_128                   (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
77 #define AHB_SEQ_DATA_SWAP                       (1<<27)
78 #define AHB_SEQ_BURST_MASK                      (0x7<<24)
79 #define AHB_SEQ_BURST_1                         (4<<24)
80 #define AHB_SEQ_BURST_4                         (5<<24)
81 #define AHB_SEQ_BURST_8                         (6<<24)
82 #define AHB_SEQ_DBL_BUF                         (1<<19)
83 #define AHB_SEQ_WRAP_SHIFT                      16
84 #define AHB_SEQ_WRAP_MASK                       (0x7<<AHB_SEQ_WRAP_SHIFT)
85
86 #define APB_DMA_CHAN_APB_PTR                            0x018
87
88 #define APB_DMA_CHAN_APB_SEQ                            0x01c
89 #define APB_SEQ_BUS_WIDTH_SHIFT                 28
90 #define APB_SEQ_BUS_WIDTH_MASK                  (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
91 #define APB_SEQ_BUS_WIDTH_8                     (0<<APB_SEQ_BUS_WIDTH_SHIFT)
92 #define APB_SEQ_BUS_WIDTH_16                    (1<<APB_SEQ_BUS_WIDTH_SHIFT)
93 #define APB_SEQ_BUS_WIDTH_32                    (2<<APB_SEQ_BUS_WIDTH_SHIFT)
94 #define APB_SEQ_BUS_WIDTH_64                    (3<<APB_SEQ_BUS_WIDTH_SHIFT)
95 #define APB_SEQ_BUS_WIDTH_128                   (4<<APB_SEQ_BUS_WIDTH_SHIFT)
96 #define APB_SEQ_DATA_SWAP                       (1<<27)
97 #define APB_SEQ_WRAP_SHIFT                      16
98 #define APB_SEQ_WRAP_MASK                       (0x7<<APB_SEQ_WRAP_SHIFT)
99
100 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
101 #define TEGRA_SYSTEM_DMA_CH_NR                  16
102 #else
103 #define TEGRA_SYSTEM_DMA_CH_NR                  32
104 #endif
105 #define TEGRA_SYSTEM_DMA_AVP_CH_NUM             4
106 #define TEGRA_SYSTEM_DMA_CH_MIN                 0
107 #define TEGRA_SYSTEM_DMA_CH_MAX \
108         (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
109
110 static struct clk *dma_clk;
111 const unsigned int ahb_addr_wrap_table[8] = {
112         0, 32, 64, 128, 256, 512, 1024, 2048
113 };
114
115 const unsigned int apb_addr_wrap_table[8] = {0, 1, 2, 4, 8, 16, 32, 64};
116
117 const unsigned int bus_width_table[5] = {8, 16, 32, 64, 128};
118
119 #define TEGRA_DMA_NAME_SIZE 16
120 struct tegra_dma_channel {
121         struct list_head        list;
122         int                     id;
123         spinlock_t              lock;
124         char                    name[TEGRA_DMA_NAME_SIZE];
125         char                    client_name[TEGRA_DMA_NAME_SIZE];
126         void  __iomem           *addr;
127         int                     mode;
128         int                     irq;
129         int                     req_transfer_count;
130 };
131
132 #define  NV_DMA_MAX_CHANNELS  32
133
134 static bool tegra_dma_initialized;
135 static DEFINE_MUTEX(tegra_dma_lock);
136 static DEFINE_SPINLOCK(enable_lock);
137
138 static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
139 static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
140
141 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
142         struct tegra_dma_req *req);
143 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
144         struct tegra_dma_req *req);
145 static void tegra_dma_stop(struct tegra_dma_channel *ch);
146
147 void tegra_dma_flush(struct tegra_dma_channel *ch)
148 {
149 }
150 EXPORT_SYMBOL(tegra_dma_flush);
151
152 void tegra_dma_dequeue(struct tegra_dma_channel *ch)
153 {
154         struct tegra_dma_req *req;
155
156         if (tegra_dma_is_empty(ch))
157                 return;
158
159         req = list_entry(ch->list.next, typeof(*req), node);
160
161         tegra_dma_dequeue_req(ch, req);
162         return;
163 }
164
165 void tegra_dma_stop(struct tegra_dma_channel *ch)
166 {
167         u32 csr;
168         u32 status;
169
170         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
171         csr &= ~CSR_IE_EOC;
172         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
173
174         csr &= ~CSR_ENB;
175         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
176
177         status = readl(ch->addr + APB_DMA_CHAN_STA);
178         if (status & STA_ISE_EOC)
179                 writel(status, ch->addr + APB_DMA_CHAN_STA);
180 }
181
182 bool tegra_dma_is_stopped(struct tegra_dma_channel *ch)
183 {
184         return !!(readl(ch->addr + APB_DMA_CHAN_STA) & CSR_ENB);
185 }
186
187 int tegra_dma_cancel(struct tegra_dma_channel *ch)
188 {
189         unsigned long irq_flags;
190
191         spin_lock_irqsave(&ch->lock, irq_flags);
192         while (!list_empty(&ch->list))
193                 list_del(ch->list.next);
194
195         tegra_dma_stop(ch);
196
197         spin_unlock_irqrestore(&ch->lock, irq_flags);
198         return 0;
199 }
200 EXPORT_SYMBOL(tegra_dma_cancel);
201
202 static unsigned int get_channel_status(struct tegra_dma_channel *ch,
203                         struct tegra_dma_req *req, bool is_stop_dma)
204 {
205         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
206         unsigned int status;
207
208         if (is_stop_dma) {
209                 /* STOP the DMA and get the transfer count.
210                  * Getting the transfer count is tricky.
211                  *  - Globally disable DMA on all channels
212                  *  - Read the channel's status register to know the number
213                  *    of pending bytes to be transfered.
214                  *  - Stop the dma channel
215                  *  - Globally re-enable DMA to resume other transfers
216                  */
217                 spin_lock(&enable_lock);
218                 writel(0, addr + APB_DMA_GEN);
219                 udelay(20);
220                 status = readl(ch->addr + APB_DMA_CHAN_STA);
221                 tegra_dma_stop(ch);
222                 writel(GEN_ENABLE, addr + APB_DMA_GEN);
223                 spin_unlock(&enable_lock);
224                 if (status & STA_ISE_EOC) {
225                         pr_err("Got Dma Int here clearing");
226                         writel(status, ch->addr + APB_DMA_CHAN_STA);
227                 }
228                 req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
229         } else {
230                 status = readl(ch->addr + APB_DMA_CHAN_STA);
231         }
232         return status;
233 }
234
235 /* should be called with the channel lock held */
236 static unsigned int dma_active_count(struct tegra_dma_channel *ch,
237         struct tegra_dma_req *req, unsigned int status)
238 {
239         unsigned int to_transfer;
240         unsigned int req_transfer_count;
241
242         unsigned int bytes_transferred;
243
244         to_transfer = (status & STA_COUNT_MASK) >> STA_COUNT_SHIFT;
245         req_transfer_count = ch->req_transfer_count;
246         req_transfer_count += 1;
247         to_transfer += 1;
248
249         bytes_transferred = req_transfer_count;
250
251         if (status & STA_BUSY)
252                 bytes_transferred -= to_transfer;
253
254         /* In continuous transfer mode, DMA only tracks the count of the
255          * half DMA buffer. So, if the DMA already finished half the DMA
256          * then add the half buffer to the completed count.
257          */
258         if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
259                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
260                         bytes_transferred += req_transfer_count;
261
262         if (status & STA_ISE_EOC)
263                 bytes_transferred += req_transfer_count;
264
265         bytes_transferred *= 4;
266
267         return bytes_transferred;
268 }
269
270 int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
271         struct tegra_dma_req *_req)
272 {
273         struct tegra_dma_req *req = NULL;
274         int found = 0;
275         unsigned int status;
276         unsigned long irq_flags;
277         int stop = 0;
278
279         spin_lock_irqsave(&ch->lock, irq_flags);
280
281         if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
282                 stop = 1;
283
284         list_for_each_entry(req, &ch->list, node) {
285                 if (req == _req) {
286                         list_del(&req->node);
287                         found = 1;
288                         break;
289                 }
290         }
291         if (!found) {
292                 spin_unlock_irqrestore(&ch->lock, irq_flags);
293                 return -ENOENT;
294         }
295
296         if (!stop)
297                 goto skip_status;
298
299         status = get_channel_status(ch, req, true);
300         req->bytes_transferred = dma_active_count(ch, req, status);
301
302         if (!list_empty(&ch->list)) {
303                 /* if the list is not empty, queue the next request */
304                 struct tegra_dma_req *next_req;
305                 next_req = list_entry(ch->list.next,
306                         typeof(*next_req), node);
307                 tegra_dma_update_hw(ch, next_req);
308         }
309 skip_status:
310         req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
311
312         spin_unlock_irqrestore(&ch->lock, irq_flags);
313
314         /* Callback should be called without any lock */
315         req->complete(req);
316         return 0;
317 }
318 EXPORT_SYMBOL(tegra_dma_dequeue_req);
319
320 bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
321 {
322         unsigned long irq_flags;
323         bool is_empty;
324
325         spin_lock_irqsave(&ch->lock, irq_flags);
326         if (list_empty(&ch->list))
327                 is_empty = true;
328         else
329                 is_empty = false;
330         spin_unlock_irqrestore(&ch->lock, irq_flags);
331         return is_empty;
332 }
333 EXPORT_SYMBOL(tegra_dma_is_empty);
334
335 bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
336         struct tegra_dma_req *_req)
337 {
338         unsigned long irq_flags;
339         struct tegra_dma_req *req;
340
341         spin_lock_irqsave(&ch->lock, irq_flags);
342         list_for_each_entry(req, &ch->list, node) {
343                 if (req == _req) {
344                         spin_unlock_irqrestore(&ch->lock, irq_flags);
345                         return true;
346                 }
347         }
348         spin_unlock_irqrestore(&ch->lock, irq_flags);
349         return false;
350 }
351 EXPORT_SYMBOL(tegra_dma_is_req_inflight);
352 int tegra_dma_get_transfer_count(struct tegra_dma_channel *ch,
353                         struct tegra_dma_req *req, bool is_stop_dma)
354 {
355         unsigned int status;
356         unsigned long irq_flags;
357         int bytes_transferred = 0;
358
359         if (IS_ERR_OR_NULL(ch))
360                 BUG();
361
362         spin_lock_irqsave(&ch->lock, irq_flags);
363
364         if (list_entry(ch->list.next, struct tegra_dma_req, node) != req) {
365                 spin_unlock_irqrestore(&ch->lock, irq_flags);
366                 pr_debug("The dma request is not the head req\n");
367                 return req->bytes_transferred;
368         }
369
370         if (req->status != TEGRA_DMA_REQ_INFLIGHT) {
371                 spin_unlock_irqrestore(&ch->lock, irq_flags);
372                 pr_debug("The dma request is not running\n");
373                 return req->bytes_transferred;
374         }
375
376         status = get_channel_status(ch, req, is_stop_dma);
377         bytes_transferred = dma_active_count(ch, req, status);
378         spin_unlock_irqrestore(&ch->lock, irq_flags);
379         return bytes_transferred;
380 }
381 EXPORT_SYMBOL(tegra_dma_get_transfer_count);
382
383 int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
384         struct tegra_dma_req *req)
385 {
386         unsigned long irq_flags;
387         struct tegra_dma_req *_req;
388         int start_dma = 0;
389
390         if (req->size > TEGRA_DMA_MAX_TRANSFER_SIZE ||
391                 req->source_addr & 0x3 || req->dest_addr & 0x3) {
392                 pr_err("Invalid DMA request for channel %d\n", ch->id);
393                 return -EINVAL;
394         }
395
396         spin_lock_irqsave(&ch->lock, irq_flags);
397
398         list_for_each_entry(_req, &ch->list, node) {
399                 if (req == _req) {
400                         spin_unlock_irqrestore(&ch->lock, irq_flags);
401                         return -EEXIST;
402                 }
403         }
404
405         req->bytes_transferred = 0;
406         req->status = 0;
407         /* STATUS_EMPTY just means the DMA hasn't processed the buf yet. */
408         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_EMPTY;
409         if (list_empty(&ch->list))
410                 start_dma = 1;
411
412         list_add_tail(&req->node, &ch->list);
413
414         if (start_dma)
415                 tegra_dma_update_hw(ch, req);
416         /* Check to see if this request needs to be pushed immediately.
417          * For continuous single-buffer DMA:
418          * The first buffer is always in-flight.  The 2nd buffer should
419          * also be in-flight.  The 3rd buffer becomes in-flight when the
420          * first is completed in the interrupt.
421          */
422         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_SINGLE) {
423                 struct tegra_dma_req *first_req, *second_req;
424                 first_req = list_entry(ch->list.next,
425                                         typeof(*first_req), node);
426                 second_req = list_entry(first_req->node.next,
427                                         typeof(*second_req), node);
428                 if (second_req == req) {
429                         unsigned long status =
430                                 readl(ch->addr + APB_DMA_CHAN_STA);
431                         if (!(status & STA_ISE_EOC))
432                                 tegra_dma_update_hw_partial(ch, req);
433                         /* Handle the case where the IRQ fired while we're
434                          * writing the interrupts.
435                          */
436                         if (status & STA_ISE_EOC) {
437                                 /* Interrupt fired, let the IRQ stop/restart
438                                  * the DMA with this buffer in a clean way.
439                                  */
440                                 req->status = TEGRA_DMA_REQ_SUCCESS;
441                         }
442                 }
443         }
444
445         spin_unlock_irqrestore(&ch->lock, irq_flags);
446
447         return 0;
448 }
449 EXPORT_SYMBOL(tegra_dma_enqueue_req);
450
451 static void tegra_dma_dump_channel_usage(void)
452 {
453         int i;
454         pr_info("DMA channel allocation dump:\n");
455         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
456                 struct tegra_dma_channel *ch = &dma_channels[i];
457                 pr_warn("dma %d used by %s\n", i, ch->client_name);
458         }
459         return;
460 }
461
462 struct tegra_dma_channel *tegra_dma_allocate_channel(int mode,
463                 const char namefmt[], ...)
464 {
465         int channel;
466         struct tegra_dma_channel *ch = NULL;
467         va_list args;
468
469         if (WARN_ON(!tegra_dma_initialized))
470                 return NULL;
471
472         mutex_lock(&tegra_dma_lock);
473
474         /* first channel is the shared channel */
475         if (mode & TEGRA_DMA_SHARED) {
476                 channel = TEGRA_SYSTEM_DMA_CH_MIN;
477         } else {
478                 channel = find_first_zero_bit(channel_usage,
479                         ARRAY_SIZE(dma_channels));
480                 if (channel >= ARRAY_SIZE(dma_channels)) {
481                         tegra_dma_dump_channel_usage();
482                         goto out;
483                 }
484         }
485         __set_bit(channel, channel_usage);
486         ch = &dma_channels[channel];
487         ch->mode = mode;
488         va_start(args, namefmt);
489         vsnprintf(ch->client_name, sizeof(ch->client_name),
490                 namefmt, args);
491         va_end(args);
492
493 out:
494         mutex_unlock(&tegra_dma_lock);
495         return ch;
496 }
497 EXPORT_SYMBOL(tegra_dma_allocate_channel);
498
499 void tegra_dma_free_channel(struct tegra_dma_channel *ch)
500 {
501         if (ch->mode & TEGRA_DMA_SHARED)
502                 return;
503         tegra_dma_cancel(ch);
504         mutex_lock(&tegra_dma_lock);
505         __clear_bit(ch->id, channel_usage);
506         memset(ch->client_name, 0, sizeof(ch->client_name));
507         mutex_unlock(&tegra_dma_lock);
508 }
509 EXPORT_SYMBOL(tegra_dma_free_channel);
510
511 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
512         struct tegra_dma_req *req)
513 {
514         u32 apb_ptr;
515         u32 ahb_ptr;
516         u32 csr;
517
518         if (req->to_memory) {
519                 apb_ptr = req->source_addr;
520                 ahb_ptr = req->dest_addr;
521         } else {
522                 apb_ptr = req->dest_addr;
523                 ahb_ptr = req->source_addr;
524         }
525         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
526         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
527
528         if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
529                 ch->req_transfer_count = (req->size >> 3) - 1;
530         else
531                 ch->req_transfer_count = (req->size >> 2) - 1;
532         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
533         csr &= ~CSR_WCOUNT_MASK;
534         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
535         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
536
537         req->status = TEGRA_DMA_REQ_INFLIGHT;
538         return;
539 }
540
541 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
542         struct tegra_dma_req *req)
543 {
544         int ahb_addr_wrap;
545         int apb_addr_wrap;
546         int ahb_bus_width;
547         int apb_bus_width;
548         int index;
549
550         u32 ahb_seq;
551         u32 apb_seq;
552         u32 ahb_ptr;
553         u32 apb_ptr;
554         u32 csr;
555
556         csr = CSR_IE_EOC | CSR_FLOW;
557         ahb_seq = AHB_SEQ_INTR_ENB;
558
559         switch (req->req_sel) {
560         case TEGRA_DMA_REQ_SEL_SL2B1:
561         case TEGRA_DMA_REQ_SEL_SL2B2:
562         case TEGRA_DMA_REQ_SEL_SL2B3:
563         case TEGRA_DMA_REQ_SEL_SL2B4:
564 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
565         case TEGRA_DMA_REQ_SEL_SL2B5:
566         case TEGRA_DMA_REQ_SEL_SL2B6:
567         case TEGRA_DMA_REQ_SEL_APBIF_CH0:
568         case TEGRA_DMA_REQ_SEL_APBIF_CH1:
569         case TEGRA_DMA_REQ_SEL_APBIF_CH2:
570         case TEGRA_DMA_REQ_SEL_APBIF_CH3:
571 #endif
572         case TEGRA_DMA_REQ_SEL_SPI:
573                 /* dtv interface has fixed burst size of 4 */
574                 if (req->fixed_burst_size) {
575                         ahb_seq |= AHB_SEQ_BURST_4;
576                         break;
577                 }
578                 /* For spi/slink the burst size based on transfer size
579                  * i.e. if multiple of 32 bytes then busrt is 8
580                  * word(8x32bits) else if multiple of 16 bytes then
581                  * burst is 4 word(4x32bits) else burst size is 1
582                  * word(1x32bits) */
583                 if (req->size & 0xF)
584                         ahb_seq |= AHB_SEQ_BURST_1;
585                 else if ((req->size >> 4) & 0x1)
586                         ahb_seq |= AHB_SEQ_BURST_4;
587                 else
588                         ahb_seq |= AHB_SEQ_BURST_8;
589                 break;
590 #if defined(CONFIG_ARCH_TEGRA_2x_SOC)
591         case TEGRA_DMA_REQ_SEL_I2S_2:
592         case TEGRA_DMA_REQ_SEL_I2S_1:
593         case TEGRA_DMA_REQ_SEL_SPD_I:
594         case TEGRA_DMA_REQ_SEL_UI_I:
595         case TEGRA_DMA_REQ_SEL_I2S2_2:
596         case TEGRA_DMA_REQ_SEL_I2S2_1:
597                 /* For ARCH_2x i2s/spdif burst size is 4 word */
598                 ahb_seq |= AHB_SEQ_BURST_4;
599                 break;
600 #endif
601
602         default:
603                 ahb_seq |= AHB_SEQ_BURST_1;
604                 break;
605         }
606
607         apb_seq = 0;
608
609         csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
610
611         ch->req_transfer_count = (req->size >> 2) - 1;
612
613         /* One shot mode is always single buffered.  Continuous mode could
614          * support either.
615          */
616         if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
617                 csr |= CSR_ONCE;
618         } else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE) {
619                 ahb_seq |= AHB_SEQ_DBL_BUF;
620                 /* We want an interrupt halfway through, then on the
621                  * completion.  The double buffer means 2 interrupts
622                  * pass before the DMA HW latches a new AHB_PTR etc.
623                  */
624                 ch->req_transfer_count = (req->size >> 3) - 1;
625         }
626         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
627
628         if (req->to_memory) {
629                 apb_ptr = req->source_addr;
630                 ahb_ptr = req->dest_addr;
631
632                 apb_addr_wrap = req->source_wrap;
633                 ahb_addr_wrap = req->dest_wrap;
634                 apb_bus_width = req->source_bus_width;
635                 ahb_bus_width = req->dest_bus_width;
636
637         } else {
638                 csr |= CSR_DIR;
639                 apb_ptr = req->dest_addr;
640                 ahb_ptr = req->source_addr;
641
642                 apb_addr_wrap = req->dest_wrap;
643                 ahb_addr_wrap = req->source_wrap;
644                 apb_bus_width = req->dest_bus_width;
645                 ahb_bus_width = req->source_bus_width;
646         }
647
648         apb_addr_wrap >>= 2;
649         ahb_addr_wrap >>= 2;
650
651         /* set address wrap for APB size */
652         index = 0;
653         do  {
654                 if (apb_addr_wrap_table[index] == apb_addr_wrap)
655                         break;
656                 index++;
657         } while (index < ARRAY_SIZE(apb_addr_wrap_table));
658         BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
659         apb_seq |= index << APB_SEQ_WRAP_SHIFT;
660
661         /* set address wrap for AHB size */
662         index = 0;
663         do  {
664                 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
665                         break;
666                 index++;
667         } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
668         BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
669         ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
670
671         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
672                 if (bus_width_table[index] == ahb_bus_width)
673                         break;
674         }
675         BUG_ON(index == ARRAY_SIZE(bus_width_table));
676         ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
677
678         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
679                 if (bus_width_table[index] == apb_bus_width)
680                         break;
681         }
682         BUG_ON(index == ARRAY_SIZE(bus_width_table));
683         apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
684
685         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
686         writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
687         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
688         writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
689         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
690
691         csr |= CSR_ENB;
692         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
693
694         req->status = TEGRA_DMA_REQ_INFLIGHT;
695 }
696
697 static void handle_oneshot_dma(struct tegra_dma_channel *ch)
698 {
699         struct tegra_dma_req *req;
700         unsigned long irq_flags;
701
702         spin_lock_irqsave(&ch->lock, irq_flags);
703         if (list_empty(&ch->list)) {
704                 spin_unlock_irqrestore(&ch->lock, irq_flags);
705                 return;
706         }
707
708         req = list_entry(ch->list.next, typeof(*req), node);
709         if (req) {
710                 list_del(&req->node);
711                 req->bytes_transferred = req->size;
712                 req->status = TEGRA_DMA_REQ_SUCCESS;
713
714                 spin_unlock_irqrestore(&ch->lock, irq_flags);
715                 /* Callback should be called without any lock */
716                 pr_debug("%s: transferred %d bytes\n", __func__,
717                         req->bytes_transferred);
718                 req->complete(req);
719                 spin_lock_irqsave(&ch->lock, irq_flags);
720         }
721
722         if (!list_empty(&ch->list)) {
723                 req = list_entry(ch->list.next, typeof(*req), node);
724                 /* the complete function we just called may have enqueued
725                    another req, in which case dma has already started */
726                 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
727                         tegra_dma_update_hw(ch, req);
728         }
729         spin_unlock_irqrestore(&ch->lock, irq_flags);
730 }
731
732 static void handle_continuous_dbl_dma(struct tegra_dma_channel *ch)
733 {
734         struct tegra_dma_req *req;
735         struct tegra_dma_req *next_req;
736         unsigned long irq_flags;
737
738         spin_lock_irqsave(&ch->lock, irq_flags);
739         if (list_empty(&ch->list)) {
740                 spin_unlock_irqrestore(&ch->lock, irq_flags);
741                 return;
742         }
743
744         req = list_entry(ch->list.next, typeof(*req), node);
745         if (req) {
746                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
747                         bool is_dma_ping_complete;
748                         is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
749                                                 & STA_PING_PONG) ? true : false;
750                         if (req->to_memory)
751                                 is_dma_ping_complete = !is_dma_ping_complete;
752                         /* Out of sync - Release current buffer */
753                         if (!is_dma_ping_complete) {
754                                 int bytes_transferred;
755
756                                 bytes_transferred = ch->req_transfer_count;
757                                 bytes_transferred += 1;
758                                 bytes_transferred <<= 3;
759                                 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
760                                 req->bytes_transferred = bytes_transferred;
761                                 req->status = TEGRA_DMA_REQ_SUCCESS;
762                                 tegra_dma_stop(ch);
763
764                                 if (!list_is_last(&req->node, &ch->list)) {
765                                         next_req = list_entry(req->node.next,
766                                                 typeof(*next_req), node);
767                                         tegra_dma_update_hw(ch, next_req);
768                                 }
769
770                                 list_del(&req->node);
771
772                                 /* DMA lock is NOT held when callbak is called */
773                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
774                                 req->complete(req);
775                                 return;
776                         }
777                         /* Load the next request into the hardware, if available
778                          * */
779                         if (!list_is_last(&req->node, &ch->list)) {
780                                 next_req = list_entry(req->node.next,
781                                         typeof(*next_req), node);
782                                 tegra_dma_update_hw_partial(ch, next_req);
783                         }
784                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
785                         req->bytes_transferred = req->size >> 1;
786                         /* DMA lock is NOT held when callback is called */
787                         spin_unlock_irqrestore(&ch->lock, irq_flags);
788                         if (likely(req->threshold))
789                                 req->threshold(req);
790                         return;
791
792                 } else if (req->buffer_status ==
793                         TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
794                         /* Callback when the buffer is completely full (i.e on
795                          * the second  interrupt */
796
797                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
798                         req->bytes_transferred = req->size;
799                         req->status = TEGRA_DMA_REQ_SUCCESS;
800                         if (list_is_last(&req->node, &ch->list))
801                                 tegra_dma_stop(ch);
802                         else {
803                                 /* It may be possible that req came after
804                                  * half dma complete so it need to start
805                                  * immediately */
806                                 next_req = list_entry(req->node.next, typeof(*next_req), node);
807                                 if (next_req->status != TEGRA_DMA_REQ_INFLIGHT) {
808                                         tegra_dma_stop(ch);
809                                         tegra_dma_update_hw(ch, next_req);
810                                 }
811                         }
812
813                         list_del(&req->node);
814
815                         /* DMA lock is NOT held when callbak is called */
816                         spin_unlock_irqrestore(&ch->lock, irq_flags);
817                         req->complete(req);
818                         return;
819
820                 } else {
821                         tegra_dma_stop(ch);
822                         /* Dma should be stop much earlier */
823                         BUG();
824                 }
825         }
826         spin_unlock_irqrestore(&ch->lock, irq_flags);
827 }
828
829 static void handle_continuous_sngl_dma(struct tegra_dma_channel *ch)
830 {
831         struct tegra_dma_req *req;
832         struct tegra_dma_req *next_req;
833         struct tegra_dma_req *next_next_req;
834         unsigned long irq_flags;
835
836         spin_lock_irqsave(&ch->lock, irq_flags);
837         if (list_empty(&ch->list)) {
838                 tegra_dma_stop(ch);
839                 spin_unlock_irqrestore(&ch->lock, irq_flags);
840                 pr_err("%s: No requests in the list.\n", __func__);
841                 return;
842         }
843         req = list_entry(ch->list.next, typeof(*req), node);
844         if (!req || (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_FULL)) {
845                 tegra_dma_stop(ch);
846                 spin_unlock_irqrestore(&ch->lock, irq_flags);
847                 pr_err("%s: DMA complete irq without corresponding req\n",
848                                 __func__);
849                 return;
850         }
851
852         /* Handle the case when buffer is completely full */
853         req->bytes_transferred = req->size;
854         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
855         req->status = TEGRA_DMA_REQ_SUCCESS;
856         if (list_is_last(&req->node, &ch->list)) {
857                 pr_debug("%s: stop\n", __func__);
858                 tegra_dma_stop(ch);
859         } else {
860                 /* The next entry should have already been queued and is now
861                  * in the middle of xfer.  We can then write the next->next one
862                  * if it exists.
863                  */
864                 next_req = list_entry(req->node.next, typeof(*next_req), node);
865                 if (next_req->status != TEGRA_DMA_REQ_INFLIGHT) {
866                         pr_debug("%s: interrupt during enqueue\n", __func__);
867                         tegra_dma_stop(ch);
868                         tegra_dma_update_hw(ch, next_req);
869                 } else if (!list_is_last(&next_req->node, &ch->list)) {
870                         next_next_req = list_entry(next_req->node.next,
871                                                 typeof(*next_next_req), node);
872                         tegra_dma_update_hw_partial(ch, next_next_req);
873                 }
874         }
875         list_del(&req->node);
876         spin_unlock_irqrestore(&ch->lock, irq_flags);
877         req->complete(req);
878 }
879
880 static irqreturn_t dma_isr(int irq, void *data)
881 {
882         struct tegra_dma_channel *ch = data;
883         unsigned long status;
884
885         status = readl(ch->addr + APB_DMA_CHAN_STA);
886         if (status & STA_ISE_EOC)
887                 writel(status, ch->addr + APB_DMA_CHAN_STA);
888         else {
889                 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
890                 return IRQ_HANDLED;
891         }
892
893         if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
894                 handle_oneshot_dma(ch);
895         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
896                 handle_continuous_dbl_dma(ch);
897         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_SINGLE)
898                 handle_continuous_sngl_dma(ch);
899         else
900                 pr_err("Bad channel mode for DMA ISR to handle\n");
901         return IRQ_HANDLED;
902 }
903
904 int __init tegra_dma_init(void)
905 {
906         int ret = 0;
907         int i;
908         unsigned int irq;
909         void __iomem *addr;
910         struct clk *c;
911
912         bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
913
914         c = clk_get_sys("tegra-dma", NULL);
915         if (IS_ERR(c)) {
916                 pr_err("Unable to get clock for APB DMA\n");
917                 ret = PTR_ERR(c);
918                 goto fail;
919         }
920         ret = clk_enable(c);
921         if (ret != 0) {
922                 pr_err("Unable to enable clock for APB DMA\n");
923                 goto fail;
924         }
925
926         dma_clk = clk_get_sys("apbdma", "apbdma");
927         if (!IS_ERR_OR_NULL(dma_clk)) {
928                 clk_enable(dma_clk);
929                 tegra_periph_reset_assert(dma_clk);
930                 udelay(10);
931                 tegra_periph_reset_deassert(dma_clk);
932                 udelay(10);
933         }
934
935         addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
936         writel(GEN_ENABLE, addr + APB_DMA_GEN);
937         writel(0, addr + APB_DMA_CNTRL);
938         writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
939                         addr + APB_DMA_IRQ_MASK_SET);
940
941         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
942                 struct tegra_dma_channel *ch = &dma_channels[i];
943
944                 ch->id = i;
945                 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
946
947                 memset(ch->client_name, 0, sizeof(ch->client_name));
948
949                 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
950                         TEGRA_APB_DMA_CH0_SIZE * i);
951
952                 spin_lock_init(&ch->lock);
953                 INIT_LIST_HEAD(&ch->list);
954
955 #ifndef CONFIG_ARCH_TEGRA_2x_SOC
956                 if (i >= 16)
957                         irq = INT_APB_DMA_CH16 + i - 16;
958                 else
959 #endif
960                         irq = INT_APB_DMA_CH0 + i;
961                 ret = request_irq(irq, dma_isr, 0, dma_channels[i].name, ch);
962                 if (ret) {
963                         pr_err("Failed to register IRQ %d for DMA %d\n",
964                                 irq, i);
965                         goto fail;
966                 }
967                 ch->irq = irq;
968
969                 __clear_bit(i, channel_usage);
970         }
971         /* mark the shared channel allocated */
972         __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
973
974         tegra_dma_initialized = true;
975
976         return 0;
977 fail:
978         writel(0, addr + APB_DMA_GEN);
979         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
980                 struct tegra_dma_channel *ch = &dma_channels[i];
981                 if (ch->irq)
982                         free_irq(ch->irq, ch);
983         }
984         return ret;
985 }
986 postcore_initcall(tegra_dma_init);
987
988 #ifdef CONFIG_PM_SLEEP
989
990 static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
991
992 static int tegra_dma_suspend(void)
993 {
994         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
995         u32 *ctx = apb_dma;
996         int i;
997
998         *ctx++ = readl(addr + APB_DMA_GEN);
999         *ctx++ = readl(addr + APB_DMA_CNTRL);
1000         *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
1001
1002         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
1003                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
1004                                   TEGRA_APB_DMA_CH0_SIZE * i);
1005
1006                 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
1007                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
1008                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
1009                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
1010                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
1011         }
1012
1013         return 0;
1014 }
1015
1016 static void tegra_dma_resume(void)
1017 {
1018         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
1019         u32 *ctx = apb_dma;
1020         int i;
1021
1022         writel(*ctx++, addr + APB_DMA_GEN);
1023         writel(*ctx++, addr + APB_DMA_CNTRL);
1024         writel(*ctx++, addr + APB_DMA_IRQ_MASK);
1025
1026         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
1027                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
1028                                   TEGRA_APB_DMA_CH0_SIZE * i);
1029
1030                 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
1031                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
1032                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
1033                 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
1034                 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
1035         }
1036 }
1037
1038 static struct syscore_ops tegra_dma_syscore_ops = {
1039         .suspend = tegra_dma_suspend,
1040         .resume = tegra_dma_resume,
1041 };
1042
1043 static int tegra_dma_syscore_init(void)
1044 {
1045         register_syscore_ops(&tegra_dma_syscore_ops);
1046
1047         return 0;
1048 }
1049 subsys_initcall(tegra_dma_syscore_init);
1050 #endif
1051
1052 #ifdef CONFIG_DEBUG_FS
1053
1054 #include <linux/debugfs.h>
1055 #include <linux/seq_file.h>
1056
1057 static int dbg_dma_show(struct seq_file *s, void *unused)
1058 {
1059         int i;
1060         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
1061
1062         seq_printf(s, "    APBDMA global register\n");
1063         seq_printf(s, "DMA_GEN:   0x%08x\n", __raw_readl(addr + APB_DMA_GEN));
1064         seq_printf(s, "DMA_CNTRL: 0x%08x\n", __raw_readl(addr + APB_DMA_CNTRL));
1065         seq_printf(s, "IRQ_MASK:  0x%08x\n",
1066                                         __raw_readl(addr + APB_DMA_IRQ_MASK));
1067
1068         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
1069                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
1070                                   TEGRA_APB_DMA_CH0_SIZE * i);
1071
1072                 seq_printf(s, "    APBDMA channel %02d register\n", i);
1073                 seq_printf(s, "0x00: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1074                                         __raw_readl(addr + 0x0),
1075                                         __raw_readl(addr + 0x4),
1076                                         __raw_readl(addr + 0x8),
1077                                         __raw_readl(addr + 0xC));
1078                 seq_printf(s, "0x10: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1079                                         __raw_readl(addr + 0x10),
1080                                         __raw_readl(addr + 0x14),
1081                                         __raw_readl(addr + 0x18),
1082                                         __raw_readl(addr + 0x1C));
1083         }
1084         seq_printf(s, "\nAPB DMA users\n");
1085         seq_printf(s, "-------------\n");
1086         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
1087                 struct tegra_dma_channel *ch = &dma_channels[i];
1088                 if (strlen(ch->client_name) > 0)
1089                         seq_printf(s, "dma %d -> %s\n", i, ch->client_name);
1090         }
1091         return 0;
1092 }
1093
1094 static int dbg_dma_open(struct inode *inode, struct file *file)
1095 {
1096         return single_open(file, dbg_dma_show, &inode->i_private);
1097 }
1098
1099 static const struct file_operations debug_fops = {
1100         .open      = dbg_dma_open,
1101         .read      = seq_read,
1102         .llseek  = seq_lseek,
1103         .release        = single_release,
1104 };
1105
1106 static int __init tegra_dma_debuginit(void)
1107 {
1108         (void) debugfs_create_file("tegra_dma", S_IRUGO,
1109                                         NULL, NULL, &debug_fops);
1110         return 0;
1111 }
1112 late_initcall(tegra_dma_debuginit);
1113 #endif