794b84d0d62d275fc69fe57559071447152f88ca
[linux-2.6.git] / arch / arm / mach-tegra / dma.c
1 /*
2  * arch/arm/mach-tegra/dma.c
3  *
4  * System DMA driver for NVIDIA Tegra SoCs
5  *
6  * Copyright (c) 2008-2009, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/io.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/err.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <linux/clk.h>
31 #include <linux/syscore_ops.h>
32 #include <mach/dma.h>
33 #include <mach/irqs.h>
34 #include <mach/iomap.h>
35
36 #define APB_DMA_GEN                             0x000
37 #define GEN_ENABLE                              (1<<31)
38
39 #define APB_DMA_CNTRL                           0x010
40
41 #define APB_DMA_IRQ_MASK                        0x01c
42
43 #define APB_DMA_IRQ_MASK_SET                    0x020
44
45 #define APB_DMA_CHAN_CSR                        0x000
46 #define CSR_ENB                                 (1<<31)
47 #define CSR_IE_EOC                              (1<<30)
48 #define CSR_HOLD                                (1<<29)
49 #define CSR_DIR                                 (1<<28)
50 #define CSR_ONCE                                (1<<27)
51 #define CSR_FLOW                                (1<<21)
52 #define CSR_REQ_SEL_SHIFT                       16
53 #define CSR_REQ_SEL_MASK                        (0x1F<<CSR_REQ_SEL_SHIFT)
54 #define CSR_WCOUNT_SHIFT                        2
55 #define CSR_WCOUNT_MASK                         0xFFFC
56
57 #define APB_DMA_CHAN_STA                                0x004
58 #define STA_BUSY                                (1<<31)
59 #define STA_ISE_EOC                             (1<<30)
60 #define STA_HALT                                (1<<29)
61 #define STA_PING_PONG                           (1<<28)
62 #define STA_COUNT_SHIFT                         2
63 #define STA_COUNT_MASK                          0xFFFC
64
65 #define APB_DMA_CHAN_AHB_PTR                            0x010
66
67 #define APB_DMA_CHAN_AHB_SEQ                            0x014
68 #define AHB_SEQ_INTR_ENB                        (1<<31)
69 #define AHB_SEQ_BUS_WIDTH_SHIFT                 28
70 #define AHB_SEQ_BUS_WIDTH_MASK                  (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
71 #define AHB_SEQ_BUS_WIDTH_8                     (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
72 #define AHB_SEQ_BUS_WIDTH_16                    (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
73 #define AHB_SEQ_BUS_WIDTH_32                    (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
74 #define AHB_SEQ_BUS_WIDTH_64                    (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
75 #define AHB_SEQ_BUS_WIDTH_128                   (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
76 #define AHB_SEQ_DATA_SWAP                       (1<<27)
77 #define AHB_SEQ_BURST_MASK                      (0x7<<24)
78 #define AHB_SEQ_BURST_1                         (4<<24)
79 #define AHB_SEQ_BURST_4                         (5<<24)
80 #define AHB_SEQ_BURST_8                         (6<<24)
81 #define AHB_SEQ_DBL_BUF                         (1<<19)
82 #define AHB_SEQ_WRAP_SHIFT                      16
83 #define AHB_SEQ_WRAP_MASK                       (0x7<<AHB_SEQ_WRAP_SHIFT)
84
85 #define APB_DMA_CHAN_APB_PTR                            0x018
86
87 #define APB_DMA_CHAN_APB_SEQ                            0x01c
88 #define APB_SEQ_BUS_WIDTH_SHIFT                 28
89 #define APB_SEQ_BUS_WIDTH_MASK                  (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
90 #define APB_SEQ_BUS_WIDTH_8                     (0<<APB_SEQ_BUS_WIDTH_SHIFT)
91 #define APB_SEQ_BUS_WIDTH_16                    (1<<APB_SEQ_BUS_WIDTH_SHIFT)
92 #define APB_SEQ_BUS_WIDTH_32                    (2<<APB_SEQ_BUS_WIDTH_SHIFT)
93 #define APB_SEQ_BUS_WIDTH_64                    (3<<APB_SEQ_BUS_WIDTH_SHIFT)
94 #define APB_SEQ_BUS_WIDTH_128                   (4<<APB_SEQ_BUS_WIDTH_SHIFT)
95 #define APB_SEQ_DATA_SWAP                       (1<<27)
96 #define APB_SEQ_WRAP_SHIFT                      16
97 #define APB_SEQ_WRAP_MASK                       (0x7<<APB_SEQ_WRAP_SHIFT)
98
99 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
100 #define TEGRA_SYSTEM_DMA_CH_NR                  16
101 #else
102 #define TEGRA_SYSTEM_DMA_CH_NR                  32
103 #endif
104 #define TEGRA_SYSTEM_DMA_AVP_CH_NUM             4
105 #define TEGRA_SYSTEM_DMA_CH_MIN                 0
106 #define TEGRA_SYSTEM_DMA_CH_MAX \
107         (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
108
109 const unsigned int ahb_addr_wrap_table[8] = {
110         0, 32, 64, 128, 256, 512, 1024, 2048
111 };
112
113 const unsigned int apb_addr_wrap_table[8] = {0, 1, 2, 4, 8, 16, 32, 64};
114
115 const unsigned int bus_width_table[5] = {8, 16, 32, 64, 128};
116
117 #define TEGRA_DMA_NAME_SIZE 16
118 struct tegra_dma_channel {
119         struct list_head        list;
120         int                     id;
121         spinlock_t              lock;
122         char                    name[TEGRA_DMA_NAME_SIZE];
123         void  __iomem           *addr;
124         int                     mode;
125         int                     irq;
126         int                     req_transfer_count;
127 };
128
129 #define  NV_DMA_MAX_CHANNELS  32
130
131 static bool tegra_dma_initialized;
132 static DEFINE_MUTEX(tegra_dma_lock);
133 static DEFINE_SPINLOCK(enable_lock);
134
135 static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
136 static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
137
138 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
139         struct tegra_dma_req *req);
140 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
141         struct tegra_dma_req *req);
142 static void tegra_dma_stop(struct tegra_dma_channel *ch);
143
144 void tegra_dma_flush(struct tegra_dma_channel *ch)
145 {
146 }
147 EXPORT_SYMBOL(tegra_dma_flush);
148
149 void tegra_dma_dequeue(struct tegra_dma_channel *ch)
150 {
151         struct tegra_dma_req *req;
152
153         if (tegra_dma_is_empty(ch))
154                 return;
155
156         req = list_entry(ch->list.next, typeof(*req), node);
157
158         tegra_dma_dequeue_req(ch, req);
159         return;
160 }
161
162 void tegra_dma_stop(struct tegra_dma_channel *ch)
163 {
164         u32 csr;
165         u32 status;
166
167         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
168         csr &= ~CSR_IE_EOC;
169         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
170
171         csr &= ~CSR_ENB;
172         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
173
174         status = readl(ch->addr + APB_DMA_CHAN_STA);
175         if (status & STA_ISE_EOC)
176                 writel(status, ch->addr + APB_DMA_CHAN_STA);
177 }
178
179 bool tegra_dma_is_stopped(struct tegra_dma_channel *ch)
180 {
181         return !!(readl(ch->addr + APB_DMA_CHAN_STA) & CSR_ENB);
182 }
183
184 int tegra_dma_cancel(struct tegra_dma_channel *ch)
185 {
186         unsigned long irq_flags;
187
188         spin_lock_irqsave(&ch->lock, irq_flags);
189         while (!list_empty(&ch->list))
190                 list_del(ch->list.next);
191
192         tegra_dma_stop(ch);
193
194         spin_unlock_irqrestore(&ch->lock, irq_flags);
195         return 0;
196 }
197
198 static unsigned int get_channel_status(struct tegra_dma_channel *ch,
199                         struct tegra_dma_req *req, bool is_stop_dma)
200 {
201         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
202         unsigned int status;
203
204         if (is_stop_dma) {
205                 /* STOP the DMA and get the transfer count.
206                  * Getting the transfer count is tricky.
207                  *  - Globally disable DMA on all channels
208                  *  - Read the channel's status register to know the number
209                  *    of pending bytes to be transfered.
210                  *  - Stop the dma channel
211                  *  - Globally re-enable DMA to resume other transfers
212                  */
213                 spin_lock(&enable_lock);
214                 writel(0, addr + APB_DMA_GEN);
215                 udelay(20);
216                 status = readl(ch->addr + APB_DMA_CHAN_STA);
217                 tegra_dma_stop(ch);
218                 writel(GEN_ENABLE, addr + APB_DMA_GEN);
219                 spin_unlock(&enable_lock);
220                 if (status & STA_ISE_EOC) {
221                         pr_err("Got Dma Int here clearing");
222                         writel(status, ch->addr + APB_DMA_CHAN_STA);
223                 }
224                 req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
225         } else {
226                 status = readl(ch->addr + APB_DMA_CHAN_STA);
227         }
228         return status;
229 }
230
231 /* should be called with the channel lock held */
232 static unsigned int dma_active_count(struct tegra_dma_channel *ch,
233         struct tegra_dma_req *req, unsigned int status)
234 {
235         unsigned int to_transfer;
236         unsigned int req_transfer_count;
237
238         unsigned int bytes_transferred;
239
240         to_transfer = (status & STA_COUNT_MASK) >> STA_COUNT_SHIFT;
241         req_transfer_count = ch->req_transfer_count;
242         req_transfer_count += 1;
243         to_transfer += 1;
244
245         bytes_transferred = req_transfer_count;
246
247         if (status & STA_BUSY)
248                 bytes_transferred -= to_transfer;
249
250         /* In continuous transfer mode, DMA only tracks the count of the
251          * half DMA buffer. So, if the DMA already finished half the DMA
252          * then add the half buffer to the completed count.
253          */
254         if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
255                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
256                         bytes_transferred += req_transfer_count;
257
258         if (status & STA_ISE_EOC)
259                 bytes_transferred += req_transfer_count;
260
261         bytes_transferred *= 4;
262
263         return bytes_transferred;
264 }
265
266 int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
267         struct tegra_dma_req *_req)
268 {
269         struct tegra_dma_req *req = NULL;
270         int found = 0;
271         unsigned int status;
272         unsigned long irq_flags;
273         int stop = 0;
274
275         spin_lock_irqsave(&ch->lock, irq_flags);
276
277         if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
278                 stop = 1;
279
280         list_for_each_entry(req, &ch->list, node) {
281                 if (req == _req) {
282                         list_del(&req->node);
283                         found = 1;
284                         break;
285                 }
286         }
287         if (!found) {
288                 spin_unlock_irqrestore(&ch->lock, irq_flags);
289                 return 0;
290         }
291
292         if (!stop)
293                 goto skip_status;
294
295         status = get_channel_status(ch, req, true);
296         req->bytes_transferred = dma_active_count(ch, req, status);
297
298         if (!list_empty(&ch->list)) {
299                 /* if the list is not empty, queue the next request */
300                 struct tegra_dma_req *next_req;
301                 next_req = list_entry(ch->list.next,
302                         typeof(*next_req), node);
303                 tegra_dma_update_hw(ch, next_req);
304         }
305 skip_status:
306         req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
307
308         spin_unlock_irqrestore(&ch->lock, irq_flags);
309
310         /* Callback should be called without any lock */
311         req->complete(req);
312         return 0;
313 }
314 EXPORT_SYMBOL(tegra_dma_dequeue_req);
315
316 bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
317 {
318         unsigned long irq_flags;
319         bool is_empty;
320
321         spin_lock_irqsave(&ch->lock, irq_flags);
322         if (list_empty(&ch->list))
323                 is_empty = true;
324         else
325                 is_empty = false;
326         spin_unlock_irqrestore(&ch->lock, irq_flags);
327         return is_empty;
328 }
329 EXPORT_SYMBOL(tegra_dma_is_empty);
330
331 bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
332         struct tegra_dma_req *_req)
333 {
334         unsigned long irq_flags;
335         struct tegra_dma_req *req;
336
337         spin_lock_irqsave(&ch->lock, irq_flags);
338         list_for_each_entry(req, &ch->list, node) {
339                 if (req == _req) {
340                         spin_unlock_irqrestore(&ch->lock, irq_flags);
341                         return true;
342                 }
343         }
344         spin_unlock_irqrestore(&ch->lock, irq_flags);
345         return false;
346 }
347 EXPORT_SYMBOL(tegra_dma_is_req_inflight);
348 int tegra_dma_get_transfer_count(struct tegra_dma_channel *ch,
349                         struct tegra_dma_req *req, bool is_stop_dma)
350 {
351         unsigned int status;
352         unsigned long irq_flags;
353         int bytes_transferred = 0;
354
355         if (IS_ERR_OR_NULL(ch))
356                 BUG();
357
358         spin_lock_irqsave(&ch->lock, irq_flags);
359
360         if (list_entry(ch->list.next, struct tegra_dma_req, node) != req) {
361                 spin_unlock_irqrestore(&ch->lock, irq_flags);
362                 pr_debug("The dma request is not the head req\n");
363                 return req->bytes_transferred;
364         }
365
366         if (req->status != TEGRA_DMA_REQ_INFLIGHT) {
367                 spin_unlock_irqrestore(&ch->lock, irq_flags);
368                 pr_debug("The dma request is not running\n");
369                 return req->bytes_transferred;
370         }
371
372         status = get_channel_status(ch, req, is_stop_dma);
373         bytes_transferred = dma_active_count(ch, req, status);
374         spin_unlock_irqrestore(&ch->lock, irq_flags);
375         return bytes_transferred;
376 }
377 EXPORT_SYMBOL(tegra_dma_get_transfer_count);
378
379 int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
380         struct tegra_dma_req *req)
381 {
382         unsigned long irq_flags;
383         struct tegra_dma_req *_req;
384         int start_dma = 0;
385
386         if (req->size > TEGRA_DMA_MAX_TRANSFER_SIZE ||
387                 req->source_addr & 0x3 || req->dest_addr & 0x3) {
388                 pr_err("Invalid DMA request for channel %d\n", ch->id);
389                 return -EINVAL;
390         }
391
392         spin_lock_irqsave(&ch->lock, irq_flags);
393
394         list_for_each_entry(_req, &ch->list, node) {
395                 if (req == _req) {
396                     spin_unlock_irqrestore(&ch->lock, irq_flags);
397                     return -EEXIST;
398                 }
399         }
400
401         req->bytes_transferred = 0;
402         req->status = 0;
403         /* STATUS_EMPTY just means the DMA hasn't processed the buf yet. */
404         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_EMPTY;
405         if (list_empty(&ch->list))
406                 start_dma = 1;
407
408         list_add_tail(&req->node, &ch->list);
409
410         if (start_dma)
411                 tegra_dma_update_hw(ch, req);
412         /* Check to see if this request needs to be pushed immediately.
413          * For continuous single-buffer DMA:
414          * The first buffer is always in-flight.  The 2nd buffer should
415          * also be in-flight.  The 3rd buffer becomes in-flight when the
416          * first is completed in the interrupt.
417          */
418         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_SINGLE) {
419                 struct tegra_dma_req *first_req, *second_req;
420                 first_req = list_entry(ch->list.next,
421                                         typeof(*first_req), node);
422                 second_req = list_entry(first_req->node.next,
423                                         typeof(*second_req), node);
424                 if (second_req == req) {
425                         unsigned long status =
426                                 readl(ch->addr + APB_DMA_CHAN_STA);
427                         if (!(status & STA_ISE_EOC))
428                                 tegra_dma_update_hw_partial(ch, req);
429                         /* Handle the case where the IRQ fired while we're
430                          * writing the interrupts.
431                          */
432                         if (status & STA_ISE_EOC) {
433                                 /* Interrupt fired, let the IRQ stop/restart
434                                  * the DMA with this buffer in a clean way.
435                                  */
436                                 req->status = TEGRA_DMA_REQ_SUCCESS;
437                         }
438                 }
439         }
440
441         spin_unlock_irqrestore(&ch->lock, irq_flags);
442
443         return 0;
444 }
445 EXPORT_SYMBOL(tegra_dma_enqueue_req);
446
447 struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
448 {
449         int channel;
450         struct tegra_dma_channel *ch = NULL;
451
452         if (WARN_ON(!tegra_dma_initialized))
453                 return NULL;
454
455         mutex_lock(&tegra_dma_lock);
456
457         /* first channel is the shared channel */
458         if (mode & TEGRA_DMA_SHARED) {
459                 channel = TEGRA_SYSTEM_DMA_CH_MIN;
460         } else {
461                 channel = find_first_zero_bit(channel_usage,
462                         ARRAY_SIZE(dma_channels));
463                 if (channel >= ARRAY_SIZE(dma_channels))
464                         goto out;
465         }
466         __set_bit(channel, channel_usage);
467         ch = &dma_channels[channel];
468         ch->mode = mode;
469
470 out:
471         mutex_unlock(&tegra_dma_lock);
472         return ch;
473 }
474 EXPORT_SYMBOL(tegra_dma_allocate_channel);
475
476 void tegra_dma_free_channel(struct tegra_dma_channel *ch)
477 {
478         if (ch->mode & TEGRA_DMA_SHARED)
479                 return;
480         tegra_dma_cancel(ch);
481         mutex_lock(&tegra_dma_lock);
482         __clear_bit(ch->id, channel_usage);
483         mutex_unlock(&tegra_dma_lock);
484 }
485 EXPORT_SYMBOL(tegra_dma_free_channel);
486
487 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
488         struct tegra_dma_req *req)
489 {
490         u32 apb_ptr;
491         u32 ahb_ptr;
492         u32 csr;
493
494         if (req->to_memory) {
495                 apb_ptr = req->source_addr;
496                 ahb_ptr = req->dest_addr;
497         } else {
498                 apb_ptr = req->dest_addr;
499                 ahb_ptr = req->source_addr;
500         }
501         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
502         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
503
504         if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
505                 ch->req_transfer_count = (req->size >> 3) - 1;
506         else
507                 ch->req_transfer_count = (req->size >> 2) - 1;
508         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
509         csr &= ~CSR_WCOUNT_MASK;
510         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
511         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
512
513         req->status = TEGRA_DMA_REQ_INFLIGHT;
514         return;
515 }
516
517 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
518         struct tegra_dma_req *req)
519 {
520         int ahb_addr_wrap;
521         int apb_addr_wrap;
522         int ahb_bus_width;
523         int apb_bus_width;
524         int index;
525
526         u32 ahb_seq;
527         u32 apb_seq;
528         u32 ahb_ptr;
529         u32 apb_ptr;
530         u32 csr;
531
532         csr = CSR_IE_EOC | CSR_FLOW;
533         ahb_seq = AHB_SEQ_INTR_ENB;
534
535         switch(req->req_sel) {
536         case TEGRA_DMA_REQ_SEL_SL2B1:
537         case TEGRA_DMA_REQ_SEL_SL2B2:
538         case TEGRA_DMA_REQ_SEL_SL2B3:
539         case TEGRA_DMA_REQ_SEL_SL2B4:
540 #if !defined(CONFIG_ARCH_TEGRA_2x_SOC)
541         case TEGRA_DMA_REQ_SEL_SL2B5:
542         case TEGRA_DMA_REQ_SEL_SL2B6:
543         case TEGRA_DMA_REQ_SEL_APBIF_CH0:
544         case TEGRA_DMA_REQ_SEL_APBIF_CH1:
545         case TEGRA_DMA_REQ_SEL_APBIF_CH2:
546         case TEGRA_DMA_REQ_SEL_APBIF_CH3:
547 #endif
548         case TEGRA_DMA_REQ_SEL_SPI:
549                 /* For spi/slink the burst size based on transfer size
550                  * i.e. if multiple of 32 bytes then busrt is
551                  * 8 word else if multiple of 16 bytes then burst is
552                  * 4 word else burst size is 1 word */
553                 if (req->size & 0xF)
554                         ahb_seq |= AHB_SEQ_BURST_1;
555                 else if ((req->size >> 4) & 0x1)
556                         ahb_seq |= AHB_SEQ_BURST_4;
557                 else
558                         ahb_seq |= AHB_SEQ_BURST_8;
559                 break;
560
561 #if defined(CONFIG_ARCH_TEGRA_2x_SOC)
562         case TEGRA_DMA_REQ_SEL_I2S_2:
563         case TEGRA_DMA_REQ_SEL_I2S_1:
564         case TEGRA_DMA_REQ_SEL_SPD_I:
565         case TEGRA_DMA_REQ_SEL_UI_I:
566         case TEGRA_DMA_REQ_SEL_I2S2_2:
567         case TEGRA_DMA_REQ_SEL_I2S2_1:
568                 /* For ARCH_2x i2s/spdif burst size is 4 word */
569                 ahb_seq |= AHB_SEQ_BURST_4;
570                 break;
571 #endif
572
573         default:
574                 ahb_seq |= AHB_SEQ_BURST_1;
575                 break;
576         }
577
578         apb_seq = 0;
579
580         csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
581
582         ch->req_transfer_count = (req->size >> 2) - 1;
583
584         /* One shot mode is always single buffered.  Continuous mode could
585          * support either.
586          */
587         if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
588                 csr |= CSR_ONCE;
589         } else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE) {
590                 ahb_seq |= AHB_SEQ_DBL_BUF;
591                 /* We want an interrupt halfway through, then on the
592                  * completion.  The double buffer means 2 interrupts
593                  * pass before the DMA HW latches a new AHB_PTR etc.
594                  */
595                 ch->req_transfer_count = (req->size >> 3) - 1;
596         }
597         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
598
599         if (req->to_memory) {
600                 apb_ptr = req->source_addr;
601                 ahb_ptr = req->dest_addr;
602
603                 apb_addr_wrap = req->source_wrap;
604                 ahb_addr_wrap = req->dest_wrap;
605                 apb_bus_width = req->source_bus_width;
606                 ahb_bus_width = req->dest_bus_width;
607
608         } else {
609                 csr |= CSR_DIR;
610                 apb_ptr = req->dest_addr;
611                 ahb_ptr = req->source_addr;
612
613                 apb_addr_wrap = req->dest_wrap;
614                 ahb_addr_wrap = req->source_wrap;
615                 apb_bus_width = req->dest_bus_width;
616                 ahb_bus_width = req->source_bus_width;
617         }
618
619         apb_addr_wrap >>= 2;
620         ahb_addr_wrap >>= 2;
621
622         /* set address wrap for APB size */
623         index = 0;
624         do  {
625                 if (apb_addr_wrap_table[index] == apb_addr_wrap)
626                         break;
627                 index++;
628         } while (index < ARRAY_SIZE(apb_addr_wrap_table));
629         BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
630         apb_seq |= index << APB_SEQ_WRAP_SHIFT;
631
632         /* set address wrap for AHB size */
633         index = 0;
634         do  {
635                 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
636                         break;
637                 index++;
638         } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
639         BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
640         ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
641
642         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
643                 if (bus_width_table[index] == ahb_bus_width)
644                         break;
645         }
646         BUG_ON(index == ARRAY_SIZE(bus_width_table));
647         ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
648
649         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
650                 if (bus_width_table[index] == apb_bus_width)
651                         break;
652         }
653         BUG_ON(index == ARRAY_SIZE(bus_width_table));
654         apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
655
656         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
657         writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
658         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
659         writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
660         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
661
662         csr |= CSR_ENB;
663         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
664
665         req->status = TEGRA_DMA_REQ_INFLIGHT;
666 }
667
668 static void handle_oneshot_dma(struct tegra_dma_channel *ch)
669 {
670         struct tegra_dma_req *req;
671         unsigned long irq_flags;
672
673         spin_lock_irqsave(&ch->lock, irq_flags);
674         if (list_empty(&ch->list)) {
675                 spin_unlock_irqrestore(&ch->lock, irq_flags);
676                 return;
677         }
678
679         req = list_entry(ch->list.next, typeof(*req), node);
680         if (req) {
681                 list_del(&req->node);
682                 req->bytes_transferred = req->size;
683                 req->status = TEGRA_DMA_REQ_SUCCESS;
684
685                 spin_unlock_irqrestore(&ch->lock, irq_flags);
686                 /* Callback should be called without any lock */
687                 pr_debug("%s: transferred %d bytes\n", __func__,
688                         req->bytes_transferred);
689                 req->complete(req);
690                 spin_lock_irqsave(&ch->lock, irq_flags);
691         }
692
693         if (!list_empty(&ch->list)) {
694                 req = list_entry(ch->list.next, typeof(*req), node);
695                 /* the complete function we just called may have enqueued
696                    another req, in which case dma has already started */
697                 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
698                         tegra_dma_update_hw(ch, req);
699         }
700         spin_unlock_irqrestore(&ch->lock, irq_flags);
701 }
702
703 static void handle_continuous_dbl_dma(struct tegra_dma_channel *ch)
704 {
705         struct tegra_dma_req *req;
706         struct tegra_dma_req *next_req;
707         unsigned long irq_flags;
708
709         spin_lock_irqsave(&ch->lock, irq_flags);
710         if (list_empty(&ch->list)) {
711                 spin_unlock_irqrestore(&ch->lock, irq_flags);
712                 return;
713         }
714
715         req = list_entry(ch->list.next, typeof(*req), node);
716         if (req) {
717                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
718                         bool is_dma_ping_complete;
719                         is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
720                                                 & STA_PING_PONG) ? true : false;
721                         if (req->to_memory)
722                                 is_dma_ping_complete = !is_dma_ping_complete;
723                         /* Out of sync - Release current buffer */
724                         if (!is_dma_ping_complete) {
725                                 int bytes_transferred;
726
727                                 bytes_transferred = ch->req_transfer_count;
728                                 bytes_transferred += 1;
729                                 bytes_transferred <<= 3;
730                                 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
731                                 req->bytes_transferred = bytes_transferred;
732                                 req->status = TEGRA_DMA_REQ_SUCCESS;
733                                 tegra_dma_stop(ch);
734
735                                 if (!list_is_last(&req->node, &ch->list)) {
736                                         next_req = list_entry(req->node.next,
737                                                 typeof(*next_req), node);
738                                         tegra_dma_update_hw(ch, next_req);
739                                 }
740
741                                 list_del(&req->node);
742
743                                 /* DMA lock is NOT held when callbak is called */
744                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
745                                 req->complete(req);
746                                 return;
747                         }
748                         /* Load the next request into the hardware, if available
749                          * */
750                         if (!list_is_last(&req->node, &ch->list)) {
751                                 next_req = list_entry(req->node.next,
752                                         typeof(*next_req), node);
753                                 tegra_dma_update_hw_partial(ch, next_req);
754                         }
755                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
756                         req->bytes_transferred = req->size >> 1;
757                         /* DMA lock is NOT held when callback is called */
758                         spin_unlock_irqrestore(&ch->lock, irq_flags);
759                         if (likely(req->threshold))
760                                 req->threshold(req);
761                         return;
762
763                 } else if (req->buffer_status ==
764                         TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
765                         /* Callback when the buffer is completely full (i.e on
766                          * the second  interrupt */
767
768                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
769                         req->bytes_transferred = req->size;
770                         req->status = TEGRA_DMA_REQ_SUCCESS;
771                         if (list_is_last(&req->node, &ch->list))
772                                 tegra_dma_stop(ch);
773                         else {
774                                 /* It may be possible that req came after
775                                  * half dma complete so it need to start
776                                  * immediately */
777                                 next_req = list_entry(req->node.next, typeof(*next_req), node);
778                                 if (next_req->status != TEGRA_DMA_REQ_INFLIGHT) {
779                                         tegra_dma_stop(ch);
780                                         tegra_dma_update_hw(ch, next_req);
781                                 }
782                         }
783
784                         list_del(&req->node);
785
786                         /* DMA lock is NOT held when callbak is called */
787                         spin_unlock_irqrestore(&ch->lock, irq_flags);
788                         req->complete(req);
789                         return;
790
791                 } else {
792                         tegra_dma_stop(ch);
793                         /* Dma should be stop much earlier */
794                         BUG();
795                 }
796         }
797         spin_unlock_irqrestore(&ch->lock, irq_flags);
798 }
799
800 static void handle_continuous_sngl_dma(struct tegra_dma_channel *ch)
801 {
802         struct tegra_dma_req *req;
803         struct tegra_dma_req *next_req;
804         struct tegra_dma_req *next_next_req;
805         unsigned long irq_flags;
806
807         spin_lock_irqsave(&ch->lock, irq_flags);
808         if (list_empty(&ch->list)) {
809                 tegra_dma_stop(ch);
810                 spin_unlock_irqrestore(&ch->lock, irq_flags);
811                 pr_err("%s: No requests in the list.\n", __func__);
812                 return;
813         }
814         req = list_entry(ch->list.next, typeof(*req), node);
815         if (!req || (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_FULL)) {
816                 tegra_dma_stop(ch);
817                 spin_unlock_irqrestore(&ch->lock, irq_flags);
818                 pr_err("%s: DMA complete irq without corresponding req\n",
819                                 __func__);
820                 return;
821         }
822
823         /* Handle the case when buffer is completely full */
824         req->bytes_transferred = req->size;
825         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
826         req->status = TEGRA_DMA_REQ_SUCCESS;
827         if (list_is_last(&req->node, &ch->list)) {
828                 pr_debug("%s: stop\n", __func__);
829                 tegra_dma_stop(ch);
830         } else {
831                 /* The next entry should have already been queued and is now
832                  * in the middle of xfer.  We can then write the next->next one
833                  * if it exists.
834                  */
835                 next_req = list_entry(req->node.next, typeof(*next_req), node);
836                 if (next_req->status != TEGRA_DMA_REQ_INFLIGHT) {
837                         pr_debug("%s: interrupt during enqueue\n", __func__);
838                         tegra_dma_stop(ch);
839                         tegra_dma_update_hw(ch, next_req);
840                 } else if (!list_is_last(&next_req->node, &ch->list)) {
841                         next_next_req = list_entry(next_req->node.next,
842                                                 typeof(*next_next_req), node);
843                         tegra_dma_update_hw_partial(ch, next_next_req);
844                 }
845         }
846         list_del(&req->node);
847         spin_unlock_irqrestore(&ch->lock, irq_flags);
848         req->complete(req);
849 }
850
851 static irqreturn_t dma_isr(int irq, void *data)
852 {
853         struct tegra_dma_channel *ch = data;
854         unsigned long status;
855
856         status = readl(ch->addr + APB_DMA_CHAN_STA);
857         if (status & STA_ISE_EOC)
858                 writel(status, ch->addr + APB_DMA_CHAN_STA);
859         else {
860                 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
861                 return IRQ_HANDLED;
862         }
863
864         if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
865                 handle_oneshot_dma(ch);
866         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
867                 handle_continuous_dbl_dma(ch);
868         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_SINGLE)
869                 handle_continuous_sngl_dma(ch);
870         else
871                 pr_err("Bad channel mode for DMA ISR to handle\n");
872         return IRQ_HANDLED;
873 }
874
875 int __init tegra_dma_init(void)
876 {
877         int ret = 0;
878         int i;
879         unsigned int irq;
880         void __iomem *addr;
881         struct clk *c;
882
883         bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
884
885         c = clk_get_sys("tegra-dma", NULL);
886         if (IS_ERR(c)) {
887                 pr_err("Unable to get clock for APB DMA\n");
888                 ret = PTR_ERR(c);
889                 goto fail;
890         }
891         ret = clk_enable(c);
892         if (ret != 0) {
893                 pr_err("Unable to enable clock for APB DMA\n");
894                 goto fail;
895         }
896
897         addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
898         writel(GEN_ENABLE, addr + APB_DMA_GEN);
899         writel(0, addr + APB_DMA_CNTRL);
900         writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
901                addr + APB_DMA_IRQ_MASK_SET);
902
903         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
904                 struct tegra_dma_channel *ch = &dma_channels[i];
905
906                 ch->id = i;
907                 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
908
909                 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
910                         TEGRA_APB_DMA_CH0_SIZE * i);
911
912                 spin_lock_init(&ch->lock);
913                 INIT_LIST_HEAD(&ch->list);
914
915 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
916                 if (i >= 16)
917                         irq = INT_APB_DMA_CH16 + i - 16;
918                 else
919 #endif
920                         irq = INT_APB_DMA_CH0 + i;
921                 ret = request_irq(irq, dma_isr, 0, dma_channels[i].name, ch);
922                 if (ret) {
923                         pr_err("Failed to register IRQ %d for DMA %d\n",
924                                 irq, i);
925                         goto fail;
926                 }
927                 ch->irq = irq;
928
929                 __clear_bit(i, channel_usage);
930         }
931         /* mark the shared channel allocated */
932         __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
933
934         tegra_dma_initialized = true;
935
936         return 0;
937 fail:
938         writel(0, addr + APB_DMA_GEN);
939         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
940                 struct tegra_dma_channel *ch = &dma_channels[i];
941                 if (ch->irq)
942                         free_irq(ch->irq, ch);
943         }
944         return ret;
945 }
946 postcore_initcall(tegra_dma_init);
947
948 #ifdef CONFIG_PM
949
950 static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
951
952 static int tegra_dma_suspend(void)
953 {
954         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
955         u32 *ctx = apb_dma;
956         int i;
957
958         *ctx++ = readl(addr + APB_DMA_GEN);
959         *ctx++ = readl(addr + APB_DMA_CNTRL);
960         *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
961
962         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
963                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
964                                   TEGRA_APB_DMA_CH0_SIZE * i);
965
966                 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
967                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
968                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
969                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
970                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
971         }
972
973         return 0;
974 }
975
976 static void tegra_dma_resume(void)
977 {
978         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
979         u32 *ctx = apb_dma;
980         int i;
981
982         writel(*ctx++, addr + APB_DMA_GEN);
983         writel(*ctx++, addr + APB_DMA_CNTRL);
984         writel(*ctx++, addr + APB_DMA_IRQ_MASK);
985
986         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
987                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
988                                   TEGRA_APB_DMA_CH0_SIZE * i);
989
990                 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
991                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
992                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
993                 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
994                 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
995         }
996 }
997
998 static struct syscore_ops tegra_dma_syscore_ops = {
999         .suspend = tegra_dma_suspend,
1000         .resume = tegra_dma_resume,
1001 };
1002
1003 static int tegra_dma_syscore_init(void)
1004 {
1005         register_syscore_ops(&tegra_dma_syscore_ops);
1006
1007         return 0;
1008 }
1009 subsys_initcall(tegra_dma_syscore_init);
1010 #endif