eb1c7cab01dbfa12dc9e7d3cd70c352e337eb1c9
[linux-2.6.git] / arch / arm / mach-tegra / dma.c
1 /*
2  * arch/arm/mach-tegra/dma.c
3  *
4  * System DMA driver for NVIDIA Tegra SoCs
5  *
6  * Copyright (c) 2008-2009, NVIDIA Corporation.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful, but WITHOUT
14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
16  * more details.
17  *
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, write to the Free Software Foundation, Inc.,
20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
21  */
22
23 #include <linux/io.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/err.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <linux/clk.h>
31 #include <linux/syscore_ops.h>
32 #include <mach/dma.h>
33 #include <mach/irqs.h>
34 #include <mach/iomap.h>
35
36 #define APB_DMA_GEN                             0x000
37 #define GEN_ENABLE                              (1<<31)
38
39 #define APB_DMA_CNTRL                           0x010
40
41 #define APB_DMA_IRQ_MASK                        0x01c
42
43 #define APB_DMA_IRQ_MASK_SET                    0x020
44
45 #define APB_DMA_CHAN_CSR                        0x000
46 #define CSR_ENB                                 (1<<31)
47 #define CSR_IE_EOC                              (1<<30)
48 #define CSR_HOLD                                (1<<29)
49 #define CSR_DIR                                 (1<<28)
50 #define CSR_ONCE                                (1<<27)
51 #define CSR_FLOW                                (1<<21)
52 #define CSR_REQ_SEL_SHIFT                       16
53 #define CSR_REQ_SEL_MASK                        (0x1F<<CSR_REQ_SEL_SHIFT)
54 #define CSR_WCOUNT_SHIFT                        2
55 #define CSR_WCOUNT_MASK                         0xFFFC
56
57 #define APB_DMA_CHAN_STA                                0x004
58 #define STA_BUSY                                (1<<31)
59 #define STA_ISE_EOC                             (1<<30)
60 #define STA_HALT                                (1<<29)
61 #define STA_PING_PONG                           (1<<28)
62 #define STA_COUNT_SHIFT                         2
63 #define STA_COUNT_MASK                          0xFFFC
64
65 #define APB_DMA_CHAN_AHB_PTR                            0x010
66
67 #define APB_DMA_CHAN_AHB_SEQ                            0x014
68 #define AHB_SEQ_INTR_ENB                        (1<<31)
69 #define AHB_SEQ_BUS_WIDTH_SHIFT                 28
70 #define AHB_SEQ_BUS_WIDTH_MASK                  (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
71 #define AHB_SEQ_BUS_WIDTH_8                     (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
72 #define AHB_SEQ_BUS_WIDTH_16                    (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
73 #define AHB_SEQ_BUS_WIDTH_32                    (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
74 #define AHB_SEQ_BUS_WIDTH_64                    (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
75 #define AHB_SEQ_BUS_WIDTH_128                   (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
76 #define AHB_SEQ_DATA_SWAP                       (1<<27)
77 #define AHB_SEQ_BURST_MASK                      (0x7<<24)
78 #define AHB_SEQ_BURST_1                         (4<<24)
79 #define AHB_SEQ_BURST_4                         (5<<24)
80 #define AHB_SEQ_BURST_8                         (6<<24)
81 #define AHB_SEQ_DBL_BUF                         (1<<19)
82 #define AHB_SEQ_WRAP_SHIFT                      16
83 #define AHB_SEQ_WRAP_MASK                       (0x7<<AHB_SEQ_WRAP_SHIFT)
84
85 #define APB_DMA_CHAN_APB_PTR                            0x018
86
87 #define APB_DMA_CHAN_APB_SEQ                            0x01c
88 #define APB_SEQ_BUS_WIDTH_SHIFT                 28
89 #define APB_SEQ_BUS_WIDTH_MASK                  (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
90 #define APB_SEQ_BUS_WIDTH_8                     (0<<APB_SEQ_BUS_WIDTH_SHIFT)
91 #define APB_SEQ_BUS_WIDTH_16                    (1<<APB_SEQ_BUS_WIDTH_SHIFT)
92 #define APB_SEQ_BUS_WIDTH_32                    (2<<APB_SEQ_BUS_WIDTH_SHIFT)
93 #define APB_SEQ_BUS_WIDTH_64                    (3<<APB_SEQ_BUS_WIDTH_SHIFT)
94 #define APB_SEQ_BUS_WIDTH_128                   (4<<APB_SEQ_BUS_WIDTH_SHIFT)
95 #define APB_SEQ_DATA_SWAP                       (1<<27)
96 #define APB_SEQ_WRAP_SHIFT                      16
97 #define APB_SEQ_WRAP_MASK                       (0x7<<APB_SEQ_WRAP_SHIFT)
98
99 #define TEGRA_SYSTEM_DMA_CH_NR                  16
100 #define TEGRA_SYSTEM_DMA_AVP_CH_NUM             4
101 #define TEGRA_SYSTEM_DMA_CH_MIN                 0
102 #define TEGRA_SYSTEM_DMA_CH_MAX \
103         (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
104
105 const unsigned int ahb_addr_wrap_table[8] = {
106         0, 32, 64, 128, 256, 512, 1024, 2048
107 };
108
109 const unsigned int apb_addr_wrap_table[8] = {0, 1, 2, 4, 8, 16, 32, 64};
110
111 const unsigned int bus_width_table[5] = {8, 16, 32, 64, 128};
112
113 #define TEGRA_DMA_NAME_SIZE 16
114 struct tegra_dma_channel {
115         struct list_head        list;
116         int                     id;
117         spinlock_t              lock;
118         char                    name[TEGRA_DMA_NAME_SIZE];
119         void  __iomem           *addr;
120         int                     mode;
121         int                     irq;
122         int                     req_transfer_count;
123 };
124
125 #define  NV_DMA_MAX_CHANNELS  32
126
127 static bool tegra_dma_initialized;
128 static DEFINE_MUTEX(tegra_dma_lock);
129 static DEFINE_SPINLOCK(enable_lock);
130
131 static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
132 static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
133
134 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
135         struct tegra_dma_req *req);
136 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
137         struct tegra_dma_req *req);
138 static void tegra_dma_stop(struct tegra_dma_channel *ch);
139
140 void tegra_dma_flush(struct tegra_dma_channel *ch)
141 {
142 }
143 EXPORT_SYMBOL(tegra_dma_flush);
144
145 void tegra_dma_dequeue(struct tegra_dma_channel *ch)
146 {
147         struct tegra_dma_req *req;
148
149         if (tegra_dma_is_empty(ch))
150                 return;
151
152         req = list_entry(ch->list.next, typeof(*req), node);
153
154         tegra_dma_dequeue_req(ch, req);
155         return;
156 }
157
158 void tegra_dma_stop(struct tegra_dma_channel *ch)
159 {
160         u32 csr;
161         u32 status;
162
163         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
164         csr &= ~CSR_IE_EOC;
165         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
166
167         csr &= ~CSR_ENB;
168         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
169
170         status = readl(ch->addr + APB_DMA_CHAN_STA);
171         if (status & STA_ISE_EOC)
172                 writel(status, ch->addr + APB_DMA_CHAN_STA);
173 }
174
175 bool tegra_dma_is_stopped(struct tegra_dma_channel *ch)
176 {
177         return !!(readl(ch->addr + APB_DMA_CHAN_STA) & CSR_ENB);
178 }
179
180 int tegra_dma_cancel(struct tegra_dma_channel *ch)
181 {
182         unsigned long irq_flags;
183
184         spin_lock_irqsave(&ch->lock, irq_flags);
185         while (!list_empty(&ch->list))
186                 list_del(ch->list.next);
187
188         tegra_dma_stop(ch);
189
190         spin_unlock_irqrestore(&ch->lock, irq_flags);
191         return 0;
192 }
193
194 static unsigned int get_channel_status(struct tegra_dma_channel *ch,
195                         struct tegra_dma_req *req, bool is_stop_dma)
196 {
197         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
198         unsigned int status;
199
200         if (is_stop_dma) {
201                 /* STOP the DMA and get the transfer count.
202                  * Getting the transfer count is tricky.
203                  *  - Globally disable DMA on all channels
204                  *  - Read the channel's status register to know the number
205                  *    of pending bytes to be transfered.
206                  *  - Stop the dma channel
207                  *  - Globally re-enable DMA to resume other transfers
208                  */
209                 spin_lock(&enable_lock);
210                 writel(0, addr + APB_DMA_GEN);
211                 udelay(20);
212                 status = readl(ch->addr + APB_DMA_CHAN_STA);
213                 tegra_dma_stop(ch);
214                 writel(GEN_ENABLE, addr + APB_DMA_GEN);
215                 spin_unlock(&enable_lock);
216                 if (status & STA_ISE_EOC) {
217                         pr_err("Got Dma Int here clearing");
218                         writel(status, ch->addr + APB_DMA_CHAN_STA);
219                 }
220                 req->status = TEGRA_DMA_REQ_ERROR_ABORTED;
221         } else {
222                 status = readl(ch->addr + APB_DMA_CHAN_STA);
223         }
224         return status;
225 }
226
227 /* should be called with the channel lock held */
228 static unsigned int dma_active_count(struct tegra_dma_channel *ch,
229         struct tegra_dma_req *req, unsigned int status)
230 {
231         unsigned int to_transfer;
232         unsigned int req_transfer_count;
233
234         unsigned int bytes_transferred;
235
236         to_transfer = (status & STA_COUNT_MASK) >> STA_COUNT_SHIFT;
237         req_transfer_count = ch->req_transfer_count;
238         req_transfer_count += 1;
239         to_transfer += 1;
240
241         bytes_transferred = req_transfer_count;
242
243         if (status & STA_BUSY)
244                 bytes_transferred -= to_transfer;
245
246         /* In continuous transfer mode, DMA only tracks the count of the
247          * half DMA buffer. So, if the DMA already finished half the DMA
248          * then add the half buffer to the completed count.
249          */
250         if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
251                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
252                         bytes_transferred += req_transfer_count;
253
254         if (status & STA_ISE_EOC)
255                 bytes_transferred += req_transfer_count;
256
257         bytes_transferred *= 4;
258
259         return bytes_transferred;
260 }
261
262 int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
263         struct tegra_dma_req *_req)
264 {
265         struct tegra_dma_req *req = NULL;
266         int found = 0;
267         unsigned int status;
268         unsigned long irq_flags;
269         int stop = 0;
270
271         spin_lock_irqsave(&ch->lock, irq_flags);
272
273         if (list_entry(ch->list.next, struct tegra_dma_req, node) == _req)
274                 stop = 1;
275
276         list_for_each_entry(req, &ch->list, node) {
277                 if (req == _req) {
278                         list_del(&req->node);
279                         found = 1;
280                         break;
281                 }
282         }
283         if (!found) {
284                 spin_unlock_irqrestore(&ch->lock, irq_flags);
285                 return 0;
286         }
287
288         if (!stop)
289                 goto skip_status;
290
291         status = get_channel_status(ch, req, true);
292         req->bytes_transferred = dma_active_count(ch, req, status);
293
294         if (!list_empty(&ch->list)) {
295                 /* if the list is not empty, queue the next request */
296                 struct tegra_dma_req *next_req;
297                 next_req = list_entry(ch->list.next,
298                         typeof(*next_req), node);
299                 tegra_dma_update_hw(ch, next_req);
300         }
301 skip_status:
302         req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
303
304         spin_unlock_irqrestore(&ch->lock, irq_flags);
305
306         /* Callback should be called without any lock */
307         req->complete(req);
308         return 0;
309 }
310 EXPORT_SYMBOL(tegra_dma_dequeue_req);
311
312 bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
313 {
314         unsigned long irq_flags;
315         bool is_empty;
316
317         spin_lock_irqsave(&ch->lock, irq_flags);
318         if (list_empty(&ch->list))
319                 is_empty = true;
320         else
321                 is_empty = false;
322         spin_unlock_irqrestore(&ch->lock, irq_flags);
323         return is_empty;
324 }
325 EXPORT_SYMBOL(tegra_dma_is_empty);
326
327 bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
328         struct tegra_dma_req *_req)
329 {
330         unsigned long irq_flags;
331         struct tegra_dma_req *req;
332
333         spin_lock_irqsave(&ch->lock, irq_flags);
334         list_for_each_entry(req, &ch->list, node) {
335                 if (req == _req) {
336                         spin_unlock_irqrestore(&ch->lock, irq_flags);
337                         return true;
338                 }
339         }
340         spin_unlock_irqrestore(&ch->lock, irq_flags);
341         return false;
342 }
343 EXPORT_SYMBOL(tegra_dma_is_req_inflight);
344 int tegra_dma_get_transfer_count(struct tegra_dma_channel *ch,
345                         struct tegra_dma_req *req, bool is_stop_dma)
346 {
347         unsigned int status;
348         unsigned long irq_flags;
349         int bytes_transferred = 0;
350
351         if (IS_ERR_OR_NULL(ch))
352                 BUG();
353
354         spin_lock_irqsave(&ch->lock, irq_flags);
355
356         if (list_entry(ch->list.next, struct tegra_dma_req, node) != req) {
357                 spin_unlock_irqrestore(&ch->lock, irq_flags);
358                 pr_debug("The dma request is not the head req\n");
359                 return req->bytes_transferred;
360         }
361
362         if (req->status != TEGRA_DMA_REQ_INFLIGHT) {
363                 spin_unlock_irqrestore(&ch->lock, irq_flags);
364                 pr_debug("The dma request is not running\n");
365                 return req->bytes_transferred;
366         }
367
368         status = get_channel_status(ch, req, is_stop_dma);
369         bytes_transferred = dma_active_count(ch, req, status);
370         spin_unlock_irqrestore(&ch->lock, irq_flags);
371         return bytes_transferred;
372 }
373 EXPORT_SYMBOL(tegra_dma_get_transfer_count);
374
375 int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
376         struct tegra_dma_req *req)
377 {
378         unsigned long irq_flags;
379         struct tegra_dma_req *_req;
380         int start_dma = 0;
381
382         if (req->size > TEGRA_DMA_MAX_TRANSFER_SIZE ||
383                 req->source_addr & 0x3 || req->dest_addr & 0x3) {
384                 pr_err("Invalid DMA request for channel %d\n", ch->id);
385                 return -EINVAL;
386         }
387
388         spin_lock_irqsave(&ch->lock, irq_flags);
389
390         list_for_each_entry(_req, &ch->list, node) {
391                 if (req == _req) {
392                     spin_unlock_irqrestore(&ch->lock, irq_flags);
393                     return -EEXIST;
394                 }
395         }
396
397         req->bytes_transferred = 0;
398         req->status = 0;
399         /* STATUS_EMPTY just means the DMA hasn't processed the buf yet. */
400         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_EMPTY;
401         if (list_empty(&ch->list))
402                 start_dma = 1;
403
404         list_add_tail(&req->node, &ch->list);
405
406         if (start_dma)
407                 tegra_dma_update_hw(ch, req);
408         /* Check to see if this request needs to be pushed immediately.
409          * For continuous single-buffer DMA:
410          * The first buffer is always in-flight.  The 2nd buffer should
411          * also be in-flight.  The 3rd buffer becomes in-flight when the
412          * first is completed in the interrupt.
413          */
414         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_SINGLE) {
415                 struct tegra_dma_req *first_req, *second_req;
416                 first_req = list_entry(ch->list.next,
417                                         typeof(*first_req), node);
418                 second_req = list_entry(first_req->node.next,
419                                         typeof(*second_req), node);
420                 if (second_req == req) {
421                         unsigned long status =
422                                 readl(ch->addr + APB_DMA_CHAN_STA);
423                         if (!(status & STA_ISE_EOC))
424                                 tegra_dma_update_hw_partial(ch, req);
425                         /* Handle the case where the IRQ fired while we're
426                          * writing the interrupts.
427                          */
428                         if (status & STA_ISE_EOC) {
429                                 /* Interrupt fired, let the IRQ stop/restart
430                                  * the DMA with this buffer in a clean way.
431                                  */
432                                 req->status = TEGRA_DMA_REQ_SUCCESS;
433                         }
434                 }
435         }
436
437         spin_unlock_irqrestore(&ch->lock, irq_flags);
438
439         return 0;
440 }
441 EXPORT_SYMBOL(tegra_dma_enqueue_req);
442
443 struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
444 {
445         int channel;
446         struct tegra_dma_channel *ch = NULL;
447
448         if (WARN_ON(!tegra_dma_initialized))
449                 return NULL;
450
451         mutex_lock(&tegra_dma_lock);
452
453         /* first channel is the shared channel */
454         if (mode & TEGRA_DMA_SHARED) {
455                 channel = TEGRA_SYSTEM_DMA_CH_MIN;
456         } else {
457                 channel = find_first_zero_bit(channel_usage,
458                         ARRAY_SIZE(dma_channels));
459                 if (channel >= ARRAY_SIZE(dma_channels))
460                         goto out;
461         }
462         __set_bit(channel, channel_usage);
463         ch = &dma_channels[channel];
464         ch->mode = mode;
465
466 out:
467         mutex_unlock(&tegra_dma_lock);
468         return ch;
469 }
470 EXPORT_SYMBOL(tegra_dma_allocate_channel);
471
472 void tegra_dma_free_channel(struct tegra_dma_channel *ch)
473 {
474         if (ch->mode & TEGRA_DMA_SHARED)
475                 return;
476         tegra_dma_cancel(ch);
477         mutex_lock(&tegra_dma_lock);
478         __clear_bit(ch->id, channel_usage);
479         mutex_unlock(&tegra_dma_lock);
480 }
481 EXPORT_SYMBOL(tegra_dma_free_channel);
482
483 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
484         struct tegra_dma_req *req)
485 {
486         u32 apb_ptr;
487         u32 ahb_ptr;
488         u32 csr;
489
490         if (req->to_memory) {
491                 apb_ptr = req->source_addr;
492                 ahb_ptr = req->dest_addr;
493         } else {
494                 apb_ptr = req->dest_addr;
495                 ahb_ptr = req->source_addr;
496         }
497         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
498         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
499
500         if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
501                 ch->req_transfer_count = (req->size >> 3) - 1;
502         else
503                 ch->req_transfer_count = (req->size >> 2) - 1;
504         csr = readl(ch->addr + APB_DMA_CHAN_CSR);
505         csr &= ~CSR_WCOUNT_MASK;
506         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
507         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
508
509         req->status = TEGRA_DMA_REQ_INFLIGHT;
510         return;
511 }
512
513 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
514         struct tegra_dma_req *req)
515 {
516         int ahb_addr_wrap;
517         int apb_addr_wrap;
518         int ahb_bus_width;
519         int apb_bus_width;
520         int index;
521
522         u32 ahb_seq;
523         u32 apb_seq;
524         u32 ahb_ptr;
525         u32 apb_ptr;
526         u32 csr;
527
528         csr = CSR_IE_EOC | CSR_FLOW;
529         ahb_seq = AHB_SEQ_INTR_ENB;
530
531         switch(req->req_sel) {
532         case TEGRA_DMA_REQ_SEL_SL2B1:
533         case TEGRA_DMA_REQ_SEL_SL2B2:
534         case TEGRA_DMA_REQ_SEL_SL2B3:
535         case TEGRA_DMA_REQ_SEL_SL2B4:
536         case TEGRA_DMA_REQ_SEL_SPI:
537                 /* For spi/slink the burst size based on transfer size
538                  * i.e. if multiple of 32 bytes then busrt is
539                  * 8 word else if multiple of 16 bytes then burst is
540                  * 4 word else burst size is 1 word */
541                 if (req->size & 0xF)
542                         ahb_seq |= AHB_SEQ_BURST_1;
543                 else if ((req->size >> 4) & 0x1)
544                         ahb_seq |= AHB_SEQ_BURST_4;
545                 else
546                         ahb_seq |= AHB_SEQ_BURST_8;
547                 break;
548         default:
549                 ahb_seq |= AHB_SEQ_BURST_1;
550                 break;
551         }
552
553         apb_seq = 0;
554
555         csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
556
557         ch->req_transfer_count = (req->size >> 2) - 1;
558
559         /* One shot mode is always single buffered.  Continuous mode could
560          * support either.
561          */
562         if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
563                 csr |= CSR_ONCE;
564         } else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE) {
565                 ahb_seq |= AHB_SEQ_DBL_BUF;
566                 /* We want an interrupt halfway through, then on the
567                  * completion.  The double buffer means 2 interrupts
568                  * pass before the DMA HW latches a new AHB_PTR etc.
569                  */
570                 ch->req_transfer_count = (req->size >> 3) - 1;
571         }
572         csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
573
574         if (req->to_memory) {
575                 apb_ptr = req->source_addr;
576                 ahb_ptr = req->dest_addr;
577
578                 apb_addr_wrap = req->source_wrap;
579                 ahb_addr_wrap = req->dest_wrap;
580                 apb_bus_width = req->source_bus_width;
581                 ahb_bus_width = req->dest_bus_width;
582
583         } else {
584                 csr |= CSR_DIR;
585                 apb_ptr = req->dest_addr;
586                 ahb_ptr = req->source_addr;
587
588                 apb_addr_wrap = req->dest_wrap;
589                 ahb_addr_wrap = req->source_wrap;
590                 apb_bus_width = req->dest_bus_width;
591                 ahb_bus_width = req->source_bus_width;
592         }
593
594         apb_addr_wrap >>= 2;
595         ahb_addr_wrap >>= 2;
596
597         /* set address wrap for APB size */
598         index = 0;
599         do  {
600                 if (apb_addr_wrap_table[index] == apb_addr_wrap)
601                         break;
602                 index++;
603         } while (index < ARRAY_SIZE(apb_addr_wrap_table));
604         BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
605         apb_seq |= index << APB_SEQ_WRAP_SHIFT;
606
607         /* set address wrap for AHB size */
608         index = 0;
609         do  {
610                 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
611                         break;
612                 index++;
613         } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
614         BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
615         ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
616
617         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
618                 if (bus_width_table[index] == ahb_bus_width)
619                         break;
620         }
621         BUG_ON(index == ARRAY_SIZE(bus_width_table));
622         ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
623
624         for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
625                 if (bus_width_table[index] == apb_bus_width)
626                         break;
627         }
628         BUG_ON(index == ARRAY_SIZE(bus_width_table));
629         apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
630
631         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
632         writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
633         writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
634         writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
635         writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
636
637         csr |= CSR_ENB;
638         writel(csr, ch->addr + APB_DMA_CHAN_CSR);
639
640         req->status = TEGRA_DMA_REQ_INFLIGHT;
641 }
642
643 static void handle_oneshot_dma(struct tegra_dma_channel *ch)
644 {
645         struct tegra_dma_req *req;
646         unsigned long irq_flags;
647
648         spin_lock_irqsave(&ch->lock, irq_flags);
649         if (list_empty(&ch->list)) {
650                 spin_unlock_irqrestore(&ch->lock, irq_flags);
651                 return;
652         }
653
654         req = list_entry(ch->list.next, typeof(*req), node);
655         if (req) {
656                 list_del(&req->node);
657                 req->bytes_transferred = req->size;
658                 req->status = TEGRA_DMA_REQ_SUCCESS;
659
660                 spin_unlock_irqrestore(&ch->lock, irq_flags);
661                 /* Callback should be called without any lock */
662                 pr_debug("%s: transferred %d bytes\n", __func__,
663                         req->bytes_transferred);
664                 req->complete(req);
665                 spin_lock_irqsave(&ch->lock, irq_flags);
666         }
667
668         if (!list_empty(&ch->list)) {
669                 req = list_entry(ch->list.next, typeof(*req), node);
670                 /* the complete function we just called may have enqueued
671                    another req, in which case dma has already started */
672                 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
673                         tegra_dma_update_hw(ch, req);
674         }
675         spin_unlock_irqrestore(&ch->lock, irq_flags);
676 }
677
678 static void handle_continuous_dbl_dma(struct tegra_dma_channel *ch)
679 {
680         struct tegra_dma_req *req;
681         struct tegra_dma_req *next_req;
682         unsigned long irq_flags;
683
684         spin_lock_irqsave(&ch->lock, irq_flags);
685         if (list_empty(&ch->list)) {
686                 spin_unlock_irqrestore(&ch->lock, irq_flags);
687                 return;
688         }
689
690         req = list_entry(ch->list.next, typeof(*req), node);
691         if (req) {
692                 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
693                         bool is_dma_ping_complete;
694                         is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
695                                                 & STA_PING_PONG) ? true : false;
696                         if (req->to_memory)
697                                 is_dma_ping_complete = !is_dma_ping_complete;
698                         /* Out of sync - Release current buffer */
699                         if (!is_dma_ping_complete) {
700                                 int bytes_transferred;
701
702                                 bytes_transferred = ch->req_transfer_count;
703                                 bytes_transferred += 1;
704                                 bytes_transferred <<= 3;
705                                 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
706                                 req->bytes_transferred = bytes_transferred;
707                                 req->status = TEGRA_DMA_REQ_SUCCESS;
708                                 tegra_dma_stop(ch);
709
710                                 if (!list_is_last(&req->node, &ch->list)) {
711                                         next_req = list_entry(req->node.next,
712                                                 typeof(*next_req), node);
713                                         tegra_dma_update_hw(ch, next_req);
714                                 }
715
716                                 list_del(&req->node);
717
718                                 /* DMA lock is NOT held when callbak is called */
719                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
720                                 req->complete(req);
721                                 return;
722                         }
723                         /* Load the next request into the hardware, if available
724                          * */
725                         if (!list_is_last(&req->node, &ch->list)) {
726                                 next_req = list_entry(req->node.next,
727                                         typeof(*next_req), node);
728                                 tegra_dma_update_hw_partial(ch, next_req);
729                         }
730                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
731                         req->bytes_transferred = req->size >> 1;
732                         /* DMA lock is NOT held when callback is called */
733                         spin_unlock_irqrestore(&ch->lock, irq_flags);
734                         if (likely(req->threshold))
735                                 req->threshold(req);
736                         return;
737
738                 } else if (req->buffer_status ==
739                         TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
740                         /* Callback when the buffer is completely full (i.e on
741                          * the second  interrupt */
742
743                         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
744                         req->bytes_transferred = req->size;
745                         req->status = TEGRA_DMA_REQ_SUCCESS;
746                         if (list_is_last(&req->node, &ch->list))
747                                 tegra_dma_stop(ch);
748                         else {
749                                 /* It may be possible that req came after
750                                  * half dma complete so it need to start
751                                  * immediately */
752                                 next_req = list_entry(req->node.next, typeof(*next_req), node);
753                                 if (next_req->status != TEGRA_DMA_REQ_INFLIGHT) {
754                                         tegra_dma_stop(ch);
755                                         tegra_dma_update_hw(ch, next_req);
756                                 }
757                         }
758
759                         list_del(&req->node);
760
761                         /* DMA lock is NOT held when callbak is called */
762                         spin_unlock_irqrestore(&ch->lock, irq_flags);
763                         req->complete(req);
764                         return;
765
766                 } else {
767                         tegra_dma_stop(ch);
768                         /* Dma should be stop much earlier */
769                         BUG();
770                 }
771         }
772         spin_unlock_irqrestore(&ch->lock, irq_flags);
773 }
774
775 static void handle_continuous_sngl_dma(struct tegra_dma_channel *ch)
776 {
777         struct tegra_dma_req *req;
778         struct tegra_dma_req *next_req;
779         struct tegra_dma_req *next_next_req;
780         unsigned long irq_flags;
781
782         spin_lock_irqsave(&ch->lock, irq_flags);
783         if (list_empty(&ch->list)) {
784                 tegra_dma_stop(ch);
785                 spin_unlock_irqrestore(&ch->lock, irq_flags);
786                 pr_err("%s: No requests in the list.\n", __func__);
787                 return;
788         }
789         req = list_entry(ch->list.next, typeof(*req), node);
790         if (!req || (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_FULL)) {
791                 tegra_dma_stop(ch);
792                 spin_unlock_irqrestore(&ch->lock, irq_flags);
793                 pr_err("%s: DMA complete irq without corresponding req\n",
794                                 __func__);
795                 return;
796         }
797
798         /* Handle the case when buffer is completely full */
799         req->bytes_transferred = req->size;
800         req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
801         req->status = TEGRA_DMA_REQ_SUCCESS;
802         if (list_is_last(&req->node, &ch->list)) {
803                 pr_debug("%s: stop\n", __func__);
804                 tegra_dma_stop(ch);
805         } else {
806                 /* The next entry should have already been queued and is now
807                  * in the middle of xfer.  We can then write the next->next one
808                  * if it exists.
809                  */
810                 next_req = list_entry(req->node.next, typeof(*next_req), node);
811                 if (next_req->status != TEGRA_DMA_REQ_INFLIGHT) {
812                         pr_debug("%s: interrupt during enqueue\n", __func__);
813                         tegra_dma_stop(ch);
814                         tegra_dma_update_hw(ch, next_req);
815                 } else if (!list_is_last(&next_req->node, &ch->list)) {
816                         next_next_req = list_entry(next_req->node.next,
817                                                 typeof(*next_next_req), node);
818                         tegra_dma_update_hw_partial(ch, next_next_req);
819                 }
820         }
821         list_del(&req->node);
822         spin_unlock_irqrestore(&ch->lock, irq_flags);
823         req->complete(req);
824 }
825
826 static irqreturn_t dma_isr(int irq, void *data)
827 {
828         struct tegra_dma_channel *ch = data;
829         unsigned long status;
830
831         status = readl(ch->addr + APB_DMA_CHAN_STA);
832         if (status & STA_ISE_EOC)
833                 writel(status, ch->addr + APB_DMA_CHAN_STA);
834         else {
835                 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
836                 return IRQ_HANDLED;
837         }
838
839         if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
840                 handle_oneshot_dma(ch);
841         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_DOUBLE)
842                 handle_continuous_dbl_dma(ch);
843         else if (ch->mode & TEGRA_DMA_MODE_CONTINUOUS_SINGLE)
844                 handle_continuous_sngl_dma(ch);
845         else
846                 pr_err("Bad channel mode for DMA ISR to handle\n");
847         return IRQ_HANDLED;
848 }
849
850 int __init tegra_dma_init(void)
851 {
852         int ret = 0;
853         int i;
854         unsigned int irq;
855         void __iomem *addr;
856         struct clk *c;
857
858         bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
859
860         c = clk_get_sys("tegra-dma", NULL);
861         if (IS_ERR(c)) {
862                 pr_err("Unable to get clock for APB DMA\n");
863                 ret = PTR_ERR(c);
864                 goto fail;
865         }
866         ret = clk_enable(c);
867         if (ret != 0) {
868                 pr_err("Unable to enable clock for APB DMA\n");
869                 goto fail;
870         }
871
872         addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
873         writel(GEN_ENABLE, addr + APB_DMA_GEN);
874         writel(0, addr + APB_DMA_CNTRL);
875         writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
876                addr + APB_DMA_IRQ_MASK_SET);
877
878         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
879                 struct tegra_dma_channel *ch = &dma_channels[i];
880
881                 ch->id = i;
882                 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
883
884                 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
885                         TEGRA_APB_DMA_CH0_SIZE * i);
886
887                 spin_lock_init(&ch->lock);
888                 INIT_LIST_HEAD(&ch->list);
889
890                 irq = INT_APB_DMA_CH0 + i;
891                 ret = request_irq(irq, dma_isr, 0, dma_channels[i].name, ch);
892                 if (ret) {
893                         pr_err("Failed to register IRQ %d for DMA %d\n",
894                                 irq, i);
895                         goto fail;
896                 }
897                 ch->irq = irq;
898
899                 __clear_bit(i, channel_usage);
900         }
901         /* mark the shared channel allocated */
902         __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
903
904         tegra_dma_initialized = true;
905
906         return 0;
907 fail:
908         writel(0, addr + APB_DMA_GEN);
909         for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
910                 struct tegra_dma_channel *ch = &dma_channels[i];
911                 if (ch->irq)
912                         free_irq(ch->irq, ch);
913         }
914         return ret;
915 }
916 postcore_initcall(tegra_dma_init);
917
918 #ifdef CONFIG_PM
919
920 static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
921
922 static int tegra_dma_suspend(void)
923 {
924         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
925         u32 *ctx = apb_dma;
926         int i;
927
928         *ctx++ = readl(addr + APB_DMA_GEN);
929         *ctx++ = readl(addr + APB_DMA_CNTRL);
930         *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
931
932         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
933                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
934                                   TEGRA_APB_DMA_CH0_SIZE * i);
935
936                 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
937                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
938                 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
939                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
940                 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
941         }
942
943         return 0;
944 }
945
946 static void tegra_dma_resume(void)
947 {
948         void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
949         u32 *ctx = apb_dma;
950         int i;
951
952         writel(*ctx++, addr + APB_DMA_GEN);
953         writel(*ctx++, addr + APB_DMA_CNTRL);
954         writel(*ctx++, addr + APB_DMA_IRQ_MASK);
955
956         for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
957                 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
958                                   TEGRA_APB_DMA_CH0_SIZE * i);
959
960                 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
961                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
962                 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
963                 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
964                 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
965         }
966 }
967
968 static struct syscore_ops tegra_dma_syscore_ops = {
969         .suspend = tegra_dma_suspend,
970         .resume = tegra_dma_resume,
971 };
972
973 static int tegra_dma_syscore_init(void)
974 {
975         register_syscore_ops(&tegra_dma_syscore_ops);
976
977         return 0;
978 }
979 subsys_initcall(tegra_dma_syscore_init);
980 #endif