dma: tegra: Fix usage of clk_prepare
[linux-3.10.git] / drivers / dma / tegra20-apb-dma.c
1 /*
2  * DMA driver for Nvidia's Tegra20 APB DMA controller.
3  *
4  * Copyright (c) 2012-13, NVIDIA CORPORATION.  All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18
19 #include <linux/bitops.h>
20 #include <linux/clk.h>
21 #include <linux/delay.h>
22 #include <linux/dmaengine.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/err.h>
25 #include <linux/init.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/mm.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/platform_device.h>
33 #include <linux/pm.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/slab.h>
36 #include <linux/clk/tegra.h>
37
38 #include <mach/pm_domains.h>
39 #include "dmaengine.h"
40
41 #define TEGRA_APBDMA_GENERAL                    0x0
42 #define TEGRA_APBDMA_GENERAL_ENABLE             BIT(31)
43
44 #define TEGRA_APBDMA_CONTROL                    0x010
45 #define TEGRA_APBDMA_IRQ_MASK                   0x01c
46 #define TEGRA_APBDMA_IRQ_MASK_SET               0x020
47
48 /* CSR register */
49 #define TEGRA_APBDMA_CHAN_CSR                   0x00
50 #define TEGRA_APBDMA_CSR_ENB                    BIT(31)
51 #define TEGRA_APBDMA_CSR_IE_EOC                 BIT(30)
52 #define TEGRA_APBDMA_CSR_HOLD                   BIT(29)
53 #define TEGRA_APBDMA_CSR_DIR                    BIT(28)
54 #define TEGRA_APBDMA_CSR_ONCE                   BIT(27)
55 #define TEGRA_APBDMA_CSR_FLOW                   BIT(21)
56 #define TEGRA_APBDMA_CSR_REQ_SEL_SHIFT          16
57 #define TEGRA_APBDMA_CSR_WCOUNT_MASK            0xFFFC
58
59 /* STATUS register */
60 #define TEGRA_APBDMA_CHAN_STATUS                0x004
61 #define TEGRA_APBDMA_STATUS_BUSY                BIT(31)
62 #define TEGRA_APBDMA_STATUS_ISE_EOC             BIT(30)
63 #define TEGRA_APBDMA_STATUS_HALT                BIT(29)
64 #define TEGRA_APBDMA_STATUS_PING_PONG           BIT(28)
65 #define TEGRA_APBDMA_STATUS_COUNT_SHIFT         2
66 #define TEGRA_APBDMA_STATUS_COUNT_MASK          0xFFFC
67
68 #define TEGRA_APBDMA_CHAN_CSRE                  0x00C
69 #define TEGRA_APBDMA_CHAN_CSRE_PAUSE            (1 << 31)
70
71 /* AHB memory address */
72 #define TEGRA_APBDMA_CHAN_AHBPTR                0x010
73
74 /* AHB sequence register */
75 #define TEGRA_APBDMA_CHAN_AHBSEQ                0x14
76 #define TEGRA_APBDMA_AHBSEQ_INTR_ENB            BIT(31)
77 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_8         (0 << 28)
78 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_16        (1 << 28)
79 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32        (2 << 28)
80 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_64        (3 << 28)
81 #define TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_128       (4 << 28)
82 #define TEGRA_APBDMA_AHBSEQ_DATA_SWAP           BIT(27)
83 #define TEGRA_APBDMA_AHBSEQ_BURST_1             (4 << 24)
84 #define TEGRA_APBDMA_AHBSEQ_BURST_4             (5 << 24)
85 #define TEGRA_APBDMA_AHBSEQ_BURST_8             (6 << 24)
86 #define TEGRA_APBDMA_AHBSEQ_DBL_BUF             BIT(19)
87 #define TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT          16
88 #define TEGRA_APBDMA_AHBSEQ_WRAP_NONE           0
89
90 /* APB address */
91 #define TEGRA_APBDMA_CHAN_APBPTR                0x018
92
93 /* APB sequence register */
94 #define TEGRA_APBDMA_CHAN_APBSEQ                0x01c
95 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8         (0 << 28)
96 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16        (1 << 28)
97 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32        (2 << 28)
98 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64        (3 << 28)
99 #define TEGRA_APBDMA_APBSEQ_BUS_WIDTH_128       (4 << 28)
100 #define TEGRA_APBDMA_APBSEQ_DATA_SWAP           BIT(27)
101 #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1         (1 << 16)
102
103
104 /* T148 specific change */
105 #define TEGRA_APBDMA_CHAN_WCOUNT                0x20
106 #define TEGRA_APBDMA_WCOUNT_WCOUNT_SHIFT        2
107 #define TEGRA_APBDMA_WCOUNT_WCOUNT_MASK         0xFFFFFFFC
108
109 #define TEGRA_APBDMA_CHAN_WORD_TRANSFER         0x24
110 #define TEGRA_APBDMA_WORD_TRANSFER_WXFER_SHIFT  2
111 #define TEGRA_APBDMA_WORD_TRANSFER_WXFER_MASK   0xFFFC
112
113 /*
114  * If any burst is in flight and DMA paused then this is the time to complete
115  * on-flight burst and update DMA status register.
116  */
117 #define TEGRA_APBDMA_BURST_COMPLETE_TIME        20
118
119 /* Channel base address offset from APBDMA base address */
120 #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET    0x1000
121
122 struct tegra_dma;
123
124 /*
125  * tegra_dma_chip_data Tegra chip specific DMA data
126  * @nr_channels: Number of channels available in the controller.
127  * @channel_reg_size: Channel register size.
128  * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
129  * @support_channel_pause: Support channel wise pause of dma.
130  * support_separate_wcount_reg: Support separate word count register.
131  */
132 struct tegra_dma_chip_data {
133         int nr_channels;
134         int channel_reg_size;
135         int max_dma_count;
136         bool support_channel_pause;
137         bool support_separate_wcount_reg;
138 };
139
140 /* DMA channel registers */
141 struct tegra_dma_channel_regs {
142         unsigned long   csr;
143         unsigned long   ahb_ptr;
144         unsigned long   apb_ptr;
145         unsigned long   ahb_seq;
146         unsigned long   apb_seq;
147         unsigned long   wcount;
148 };
149
150 /*
151  * tegra_dma_sg_req: Dma request details to configure hardware. This
152  * contains the details for one transfer to configure DMA hw.
153  * The client's request for data transfer can be broken into multiple
154  * sub-transfer as per requester details and hw support.
155  * This sub transfer get added in the list of transfer and point to Tegra
156  * DMA descriptor which manages the transfer details.
157  */
158 struct tegra_dma_sg_req {
159         struct tegra_dma_channel_regs   ch_regs;
160         int                             req_len;
161         bool                            configured;
162         bool                            last_sg;
163         bool                            half_done;
164         struct list_head                node;
165         struct tegra_dma_desc           *dma_desc;
166 };
167
168 /*
169  * tegra_dma_desc: Tegra DMA descriptors which manages the client requests.
170  * This descriptor keep track of transfer status, callbacks and request
171  * counts etc.
172  */
173 struct tegra_dma_desc {
174         struct dma_async_tx_descriptor  txd;
175         int                             bytes_requested;
176         int                             bytes_transferred;
177         enum dma_status                 dma_status;
178         struct list_head                node;
179         struct list_head                tx_list;
180         struct list_head                cb_node;
181         int                             cb_count;
182 };
183
184 struct tegra_dma_channel;
185
186 typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc,
187                                 bool to_terminate);
188
189 /* tegra_dma_channel: Channel specific information */
190 struct tegra_dma_channel {
191         struct dma_chan         dma_chan;
192         char                    name[30];
193         bool                    config_init;
194         int                     id;
195         int                     irq;
196         unsigned long           chan_base_offset;
197         spinlock_t              lock;
198         bool                    busy;
199         struct tegra_dma        *tdma;
200         bool                    cyclic;
201
202         /* Different lists for managing the requests */
203         struct list_head        free_sg_req;
204         struct list_head        pending_sg_req;
205         struct list_head        free_dma_desc;
206         struct list_head        cb_desc;
207
208         /* ISR handler and tasklet for bottom half of isr handling */
209         dma_isr_handler         isr_handler;
210         struct tasklet_struct   tasklet;
211         dma_async_tx_callback   callback;
212         void                    *callback_param;
213
214         /* Channel-slave specific configuration */
215         struct dma_slave_config dma_sconfig;
216         struct tegra_dma_channel_regs   channel_reg;
217 };
218
219 /* tegra_dma: Tegra DMA specific information */
220 struct tegra_dma {
221         struct dma_device               dma_dev;
222         struct device                   *dev;
223         struct clk                      *dma_clk;
224         spinlock_t                      global_lock;
225         void __iomem                    *base_addr;
226         const struct tegra_dma_chip_data *chip_data;
227
228         /* Some register need to be cache before suspend */
229         u32                             reg_gen;
230
231         /* Last member of the structure */
232         struct tegra_dma_channel channels[0];
233 };
234
235 static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val)
236 {
237         writel(val, tdma->base_addr + reg);
238 }
239
240 static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg)
241 {
242         return readl(tdma->base_addr + reg);
243 }
244
245 static inline void tdc_write(struct tegra_dma_channel *tdc,
246                 u32 reg, u32 val)
247 {
248         writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);
249 }
250
251 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
252 {
253         return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg);
254 }
255
256 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
257 {
258         return container_of(dc, struct tegra_dma_channel, dma_chan);
259 }
260
261 static inline struct tegra_dma_desc *txd_to_tegra_dma_desc(
262                 struct dma_async_tx_descriptor *td)
263 {
264         return container_of(td, struct tegra_dma_desc, txd);
265 }
266
267 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
268 {
269         return &tdc->dma_chan.dev->device;
270 }
271
272 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx);
273 static int tegra_dma_runtime_suspend(struct device *dev);
274 static int tegra_dma_runtime_resume(struct device *dev);
275
276 /* Get DMA desc from free list, if not there then allocate it.  */
277 static struct tegra_dma_desc *tegra_dma_desc_get(
278                 struct tegra_dma_channel *tdc)
279 {
280         struct tegra_dma_desc *dma_desc;
281         unsigned long flags;
282
283         spin_lock_irqsave(&tdc->lock, flags);
284
285         /* Do not allocate if desc are waiting for ack */
286         list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {
287                 if (async_tx_test_ack(&dma_desc->txd)) {
288                         list_del(&dma_desc->node);
289                         spin_unlock_irqrestore(&tdc->lock, flags);
290                         dma_desc->txd.flags = 0;
291                         return dma_desc;
292                 }
293         }
294
295         spin_unlock_irqrestore(&tdc->lock, flags);
296
297         /* Allocate DMA desc */
298         dma_desc = kzalloc(sizeof(*dma_desc), GFP_ATOMIC);
299         if (!dma_desc) {
300                 dev_err(tdc2dev(tdc), "dma_desc alloc failed\n");
301                 return NULL;
302         }
303
304         dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan);
305         dma_desc->txd.tx_submit = tegra_dma_tx_submit;
306         dma_desc->txd.flags = 0;
307         return dma_desc;
308 }
309
310 static void tegra_dma_desc_put(struct tegra_dma_channel *tdc,
311                 struct tegra_dma_desc *dma_desc)
312 {
313         unsigned long flags;
314
315         spin_lock_irqsave(&tdc->lock, flags);
316         if (!list_empty(&dma_desc->tx_list))
317                 list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req);
318         list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
319         spin_unlock_irqrestore(&tdc->lock, flags);
320 }
321
322 static struct tegra_dma_sg_req *tegra_dma_sg_req_get(
323                 struct tegra_dma_channel *tdc)
324 {
325         struct tegra_dma_sg_req *sg_req = NULL;
326         unsigned long flags;
327
328         spin_lock_irqsave(&tdc->lock, flags);
329         if (!list_empty(&tdc->free_sg_req)) {
330                 sg_req = list_first_entry(&tdc->free_sg_req,
331                                         typeof(*sg_req), node);
332                 list_del(&sg_req->node);
333                 spin_unlock_irqrestore(&tdc->lock, flags);
334                 return sg_req;
335         }
336         spin_unlock_irqrestore(&tdc->lock, flags);
337
338         sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_ATOMIC);
339         if (!sg_req)
340                 dev_err(tdc2dev(tdc), "sg_req alloc failed\n");
341         return sg_req;
342 }
343
344 static int tegra_dma_slave_config(struct dma_chan *dc,
345                 struct dma_slave_config *sconfig)
346 {
347         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
348
349         if (!list_empty(&tdc->pending_sg_req)) {
350                 dev_err(tdc2dev(tdc), "Configuration not allowed\n");
351                 return -EBUSY;
352         }
353
354         memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
355         tdc->config_init = true;
356         return 0;
357 }
358
359 static void tegra_dma_global_pause(struct tegra_dma_channel *tdc,
360         bool wait_for_burst_complete)
361 {
362         struct tegra_dma *tdma = tdc->tdma;
363
364         spin_lock(&tdma->global_lock);
365         tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0);
366         if (wait_for_burst_complete)
367                 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
368 }
369
370 static void tegra_dma_global_resume(struct tegra_dma_channel *tdc)
371 {
372         struct tegra_dma *tdma = tdc->tdma;
373
374         tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
375         spin_unlock(&tdma->global_lock);
376 }
377
378 static void tegra_dma_pause(struct tegra_dma_channel *tdc,
379         bool wait_for_burst_complete)
380 {
381         struct tegra_dma *tdma = tdc->tdma;
382
383         if (tdma->chip_data->support_channel_pause) {
384                 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE,
385                                 TEGRA_APBDMA_CHAN_CSRE_PAUSE);
386                 if (wait_for_burst_complete)
387                         udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
388         } else {
389                 tegra_dma_global_pause(tdc, wait_for_burst_complete);
390         }
391 }
392
393 static void tegra_dma_resume(struct tegra_dma_channel *tdc)
394 {
395         struct tegra_dma *tdma = tdc->tdma;
396
397         if (tdma->chip_data->support_channel_pause) {
398                 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSRE, 0);
399         } else {
400                 tegra_dma_global_resume(tdc);
401         }
402 }
403
404 static void tegra_dma_stop(struct tegra_dma_channel *tdc)
405 {
406         u32 csr;
407         u32 status;
408
409         /* Disable interrupts */
410         csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
411         csr &= ~TEGRA_APBDMA_CSR_IE_EOC;
412         tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);
413
414         /* Disable DMA */
415         csr &= ~TEGRA_APBDMA_CSR_ENB;
416         tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, csr);
417
418         /* Clear interrupt status if it is there */
419         status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
420         if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
421                 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
422                 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
423         }
424         tdc->busy = false;
425 }
426
427 static void tegra_dma_start(struct tegra_dma_channel *tdc,
428                 struct tegra_dma_sg_req *sg_req)
429 {
430         struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs;
431
432         if (tdc->tdma->chip_data->support_separate_wcount_reg) {
433                 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount);
434                 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, 0);
435         }
436         tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, ch_regs->csr);
437         tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_regs->apb_seq);
438         tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr);
439         tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq);
440         tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr);
441
442         /* Start DMA */
443         tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
444                                 ch_regs->csr | TEGRA_APBDMA_CSR_ENB);
445 }
446
447 static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc,
448                 struct tegra_dma_sg_req *nsg_req)
449 {
450         unsigned long status;
451
452         /*
453          * The DMA controller reloads the new configuration for next transfer
454          * after last burst of current transfer completes.
455          * If there is no IEC status then this makes sure that last burst
456          * has not be completed. There may be case that last burst is on
457          * flight and so it can complete but because DMA is paused, it
458          * will not generates interrupt as well as not reload the new
459          * configuration.
460          * If there is already IEC status then interrupt handler need to
461          * load new configuration.
462          */
463         tegra_dma_pause(tdc, false);
464         status  = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
465
466         /*
467          * If interrupt is pending then do nothing as the ISR will handle
468          * the programing for new request.
469          */
470         if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
471                 dev_err(tdc2dev(tdc),
472                         "Skipping new configuration as interrupt is pending\n");
473                 tegra_dma_resume(tdc);
474                 return;
475         }
476
477         /* Safe to program new configuration */
478         tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr);
479         tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr);
480         if (tdc->tdma->chip_data->support_separate_wcount_reg)
481                 tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,
482                                                 nsg_req->ch_regs.wcount);
483         tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
484                                 nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB);
485         nsg_req->configured = true;
486
487         tegra_dma_resume(tdc);
488 }
489
490 static void tdc_start_head_req(struct tegra_dma_channel *tdc)
491 {
492         struct tegra_dma_sg_req *sg_req;
493
494         if (list_empty(&tdc->pending_sg_req))
495                 return;
496
497         sg_req = list_first_entry(&tdc->pending_sg_req,
498                                         typeof(*sg_req), node);
499         tegra_dma_start(tdc, sg_req);
500         sg_req->configured = true;
501         tdc->busy = true;
502 }
503
504 static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc)
505 {
506         struct tegra_dma_sg_req *hsgreq;
507         struct tegra_dma_sg_req *hnsgreq;
508
509         if (list_empty(&tdc->pending_sg_req))
510                 return;
511
512         hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
513         if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) {
514                 hnsgreq = list_first_entry(&hsgreq->node,
515                                         typeof(*hnsgreq), node);
516                 tegra_dma_configure_for_next(tdc, hnsgreq);
517         }
518 }
519
520 static inline int get_current_xferred_count(struct tegra_dma_channel *tdc,
521         struct tegra_dma_sg_req *sg_req, unsigned long status)
522 {
523         return sg_req->req_len - (status & TEGRA_APBDMA_STATUS_COUNT_MASK) - 4;
524 }
525
526 static void tegra_dma_abort_all(struct tegra_dma_channel *tdc)
527 {
528         struct tegra_dma_sg_req *sgreq;
529         struct tegra_dma_desc *dma_desc;
530
531         while (!list_empty(&tdc->pending_sg_req)) {
532                 sgreq = list_first_entry(&tdc->pending_sg_req,
533                                                 typeof(*sgreq), node);
534                 list_move_tail(&sgreq->node, &tdc->free_sg_req);
535                 if (sgreq->last_sg) {
536                         dma_desc = sgreq->dma_desc;
537                         dma_desc->dma_status = DMA_ERROR;
538                         list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
539
540                         /* Add in cb list if it is not there. */
541                         if (!dma_desc->cb_count)
542                                 list_add_tail(&dma_desc->cb_node,
543                                                         &tdc->cb_desc);
544                         dma_desc->cb_count++;
545                 }
546         }
547         tdc->isr_handler = NULL;
548 }
549
550 static bool handle_continuous_head_request(struct tegra_dma_channel *tdc,
551                 struct tegra_dma_sg_req *last_sg_req, bool to_terminate)
552 {
553         struct tegra_dma_sg_req *hsgreq = NULL;
554
555         if (list_empty(&tdc->pending_sg_req)) {
556                 dev_err(tdc2dev(tdc), "Dma is running without req\n");
557                 tegra_dma_stop(tdc);
558                 return false;
559         }
560
561         /*
562          * Check that head req on list should be in flight.
563          * If it is not in flight then abort transfer as
564          * looping of transfer can not continue.
565          */
566         hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node);
567         if (!hsgreq->configured) {
568                 tegra_dma_stop(tdc);
569                 dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n");
570                 tegra_dma_abort_all(tdc);
571                 return false;
572         }
573
574         /* Configure next request */
575         if (!to_terminate)
576                 tdc_configure_next_head_desc(tdc);
577         return true;
578 }
579
580 static void handle_once_dma_done(struct tegra_dma_channel *tdc,
581         bool to_terminate)
582 {
583         struct tegra_dma_sg_req *sgreq;
584         struct tegra_dma_desc *dma_desc;
585
586         tdc->busy = false;
587         sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
588         dma_desc = sgreq->dma_desc;
589         dma_desc->bytes_transferred += sgreq->req_len;
590
591         list_del(&sgreq->node);
592         if (sgreq->last_sg) {
593                 dma_desc->dma_status = DMA_SUCCESS;
594                 dma_cookie_complete(&dma_desc->txd);
595                 if (!dma_desc->cb_count)
596                         list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
597                 dma_desc->cb_count++;
598                 list_add_tail(&dma_desc->node, &tdc->free_dma_desc);
599         }
600         list_add_tail(&sgreq->node, &tdc->free_sg_req);
601
602         /* Do not start DMA if it is going to be terminate */
603         if (list_empty(&tdc->pending_sg_req) && (!to_terminate)) {
604                 clk_disable(tdc->tdma->dma_clk);
605                 pm_runtime_put(tdc->tdma->dev);
606         }
607
608         if (to_terminate || list_empty(&tdc->pending_sg_req))
609                 return;
610
611         tdc_start_head_req(tdc);
612         return;
613 }
614
615 static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
616                 bool to_terminate)
617 {
618         struct tegra_dma_sg_req *sgreq;
619         struct tegra_dma_desc *dma_desc;
620         bool st;
621
622         sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node);
623         dma_desc = sgreq->dma_desc;
624         dma_desc->bytes_transferred += sgreq->req_len;
625
626         /* Callback need to be call */
627         if (!dma_desc->cb_count)
628                 list_add_tail(&dma_desc->cb_node, &tdc->cb_desc);
629         dma_desc->cb_count++;
630
631         /* If not last req then put at end of pending list */
632         if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) {
633                 list_move_tail(&sgreq->node, &tdc->pending_sg_req);
634                 sgreq->configured = false;
635                 st = handle_continuous_head_request(tdc, sgreq, to_terminate);
636                 if (!st)
637                         dma_desc->dma_status = DMA_ERROR;
638         }
639         return;
640 }
641
642 static void tegra_dma_tasklet(unsigned long data)
643 {
644         struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data;
645         dma_async_tx_callback callback = NULL;
646         void *callback_param = NULL;
647         struct tegra_dma_desc *dma_desc;
648         unsigned long flags;
649         int cb_count;
650
651         spin_lock_irqsave(&tdc->lock, flags);
652         while (!list_empty(&tdc->cb_desc)) {
653                 dma_desc  = list_first_entry(&tdc->cb_desc,
654                                         typeof(*dma_desc), cb_node);
655                 list_del(&dma_desc->cb_node);
656                 callback = dma_desc->txd.callback;
657                 callback_param = dma_desc->txd.callback_param;
658                 cb_count = dma_desc->cb_count;
659                 dma_desc->cb_count = 0;
660                 spin_unlock_irqrestore(&tdc->lock, flags);
661                 while (cb_count-- && callback)
662                         callback(callback_param);
663                 spin_lock_irqsave(&tdc->lock, flags);
664         }
665         spin_unlock_irqrestore(&tdc->lock, flags);
666 }
667
668 static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
669 {
670         struct tegra_dma_channel *tdc = dev_id;
671         unsigned long status;
672         unsigned long flags;
673
674         spin_lock_irqsave(&tdc->lock, flags);
675
676         status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
677         if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
678                 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, status);
679                 tdc_write(tdc, TEGRA_APBDMA_CHAN_STATUS, TEGRA_APBDMA_STATUS_ISE_EOC);
680                 tdc->isr_handler(tdc, false);
681                 tasklet_schedule(&tdc->tasklet);
682                 spin_unlock_irqrestore(&tdc->lock, flags);
683                 return IRQ_HANDLED;
684         }
685
686         spin_unlock_irqrestore(&tdc->lock, flags);
687         dev_info(tdc2dev(tdc),
688                 "Interrupt already served status 0x%08lx\n", status);
689         return IRQ_NONE;
690 }
691
692 static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd)
693 {
694         struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd);
695         struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan);
696         unsigned long flags;
697         dma_cookie_t cookie;
698
699         spin_lock_irqsave(&tdc->lock, flags);
700         dma_desc->dma_status = DMA_IN_PROGRESS;
701         cookie = dma_cookie_assign(&dma_desc->txd);
702         list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req);
703         spin_unlock_irqrestore(&tdc->lock, flags);
704         return cookie;
705 }
706
707 static void tegra_dma_issue_pending(struct dma_chan *dc)
708 {
709         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
710         unsigned long flags;
711         int ret;
712
713         spin_lock_irqsave(&tdc->lock, flags);
714         if (list_empty(&tdc->pending_sg_req)) {
715                 dev_err(tdc2dev(tdc), "No DMA request\n");
716                 goto end;
717         }
718
719         pm_runtime_get(tdc->tdma->dev);
720         ret = clk_enable(tdc->tdma->dma_clk);
721         if (ret < 0) {
722                 dev_err(tdc2dev(tdc), "clk_enable failed: %d\n", ret);
723                 return;
724         }
725
726         if (!tdc->busy) {
727                 tdc_start_head_req(tdc);
728
729                 /* Continuous single mode: Configure next req */
730                 if (tdc->cyclic) {
731                         /*
732                          * Wait for 1 burst time for configure DMA for
733                          * next transfer.
734                          */
735                         udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
736                         tdc_configure_next_head_desc(tdc);
737                 }
738         }
739 end:
740         spin_unlock_irqrestore(&tdc->lock, flags);
741         return;
742 }
743
744 static void tegra_dma_terminate_all(struct dma_chan *dc)
745 {
746         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
747         struct tegra_dma_sg_req *sgreq;
748         struct tegra_dma_desc *dma_desc;
749         unsigned long flags;
750         unsigned long status;
751         unsigned long wcount = 0;
752         bool was_busy;
753
754         spin_lock_irqsave(&tdc->lock, flags);
755         if (list_empty(&tdc->pending_sg_req)) {
756                 spin_unlock_irqrestore(&tdc->lock, flags);
757                 return;
758         }
759
760         if (!tdc->busy)
761                 goto skip_dma_stop;
762
763         /* Pause DMA before checking the queue status */
764         tegra_dma_pause(tdc, true);
765
766         status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
767         if (tdc->tdma->chip_data->support_separate_wcount_reg)
768                 wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER);
769         if (status & TEGRA_APBDMA_STATUS_ISE_EOC) {
770                 dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__);
771                 tdc->isr_handler(tdc, true);
772                 status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS);
773                 if (tdc->tdma->chip_data->support_separate_wcount_reg)
774                         wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER);
775         }
776
777         was_busy = tdc->busy;
778         tegra_dma_stop(tdc);
779
780         if (!list_empty(&tdc->pending_sg_req) && was_busy) {
781                 sgreq = list_first_entry(&tdc->pending_sg_req,
782                                         typeof(*sgreq), node);
783                 if (!tdc->tdma->chip_data->support_separate_wcount_reg)
784                         wcount = status;
785                 sgreq->dma_desc->bytes_transferred +=
786                                 get_current_xferred_count(tdc, sgreq, wcount);
787         }
788         tegra_dma_resume(tdc);
789         clk_disable(tdc->tdma->dma_clk);
790         pm_runtime_put(tdc->tdma->dev);
791
792 skip_dma_stop:
793         tegra_dma_abort_all(tdc);
794
795         while (!list_empty(&tdc->cb_desc)) {
796                 dma_desc  = list_first_entry(&tdc->cb_desc,
797                                         typeof(*dma_desc), cb_node);
798                 list_del(&dma_desc->cb_node);
799                 dma_desc->cb_count = 0;
800         }
801         spin_unlock_irqrestore(&tdc->lock, flags);
802 }
803
804 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
805         dma_cookie_t cookie, struct dma_tx_state *txstate)
806 {
807         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
808         struct tegra_dma_desc *dma_desc;
809         struct tegra_dma_sg_req *sg_req;
810         enum dma_status ret;
811         unsigned long flags;
812         unsigned int residual;
813
814         spin_lock_irqsave(&tdc->lock, flags);
815
816         ret = dma_cookie_status(dc, cookie, txstate);
817         if (ret == DMA_SUCCESS) {
818                 spin_unlock_irqrestore(&tdc->lock, flags);
819                 return ret;
820         }
821
822         /* Check on wait_ack desc status */
823         list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) {
824                 if (dma_desc->txd.cookie == cookie) {
825                         residual =  dma_desc->bytes_requested -
826                                         (dma_desc->bytes_transferred %
827                                                 dma_desc->bytes_requested);
828                         dma_set_residue(txstate, residual);
829                         ret = dma_desc->dma_status;
830                         spin_unlock_irqrestore(&tdc->lock, flags);
831                         return ret;
832                 }
833         }
834
835         /* Check in pending list */
836         list_for_each_entry(sg_req, &tdc->pending_sg_req, node) {
837                 dma_desc = sg_req->dma_desc;
838                 if (dma_desc->txd.cookie == cookie) {
839                         residual =  dma_desc->bytes_requested -
840                                         (dma_desc->bytes_transferred %
841                                                 dma_desc->bytes_requested);
842                         dma_set_residue(txstate, residual);
843                         ret = dma_desc->dma_status;
844                         spin_unlock_irqrestore(&tdc->lock, flags);
845                         return ret;
846                 }
847         }
848
849         dev_dbg(tdc2dev(tdc), "cookie %d does not found\n", cookie);
850         spin_unlock_irqrestore(&tdc->lock, flags);
851         return ret;
852 }
853
854 static int tegra_dma_device_control(struct dma_chan *dc, enum dma_ctrl_cmd cmd,
855                         unsigned long arg)
856 {
857         switch (cmd) {
858         case DMA_SLAVE_CONFIG:
859                 return tegra_dma_slave_config(dc,
860                                 (struct dma_slave_config *)arg);
861
862         case DMA_TERMINATE_ALL:
863                 tegra_dma_terminate_all(dc);
864                 return 0;
865
866         default:
867                 break;
868         }
869
870         return -ENXIO;
871 }
872
873 static inline int get_bus_width(struct tegra_dma_channel *tdc,
874                 enum dma_slave_buswidth slave_bw)
875 {
876         switch (slave_bw) {
877         case DMA_SLAVE_BUSWIDTH_1_BYTE:
878                 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_8;
879         case DMA_SLAVE_BUSWIDTH_2_BYTES:
880                 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_16;
881         case DMA_SLAVE_BUSWIDTH_4_BYTES:
882                 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
883         case DMA_SLAVE_BUSWIDTH_8_BYTES:
884                 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_64;
885         default:
886                 dev_warn(tdc2dev(tdc),
887                         "slave bw is not supported, using 32bits\n");
888                 return TEGRA_APBDMA_APBSEQ_BUS_WIDTH_32;
889         }
890 }
891
892 static inline int get_burst_size(struct tegra_dma_channel *tdc,
893         u32 burst_size, enum dma_slave_buswidth slave_bw, int len)
894 {
895         int burst_byte;
896         int burst_ahb_width;
897
898         /*
899          * burst_size from client is in terms of the bus_width.
900          * convert them into AHB memory width which is 4 byte.
901          */
902         burst_byte = burst_size * slave_bw;
903         burst_ahb_width = burst_byte / 4;
904
905         /* If burst size is 0 then calculate the burst size based on length */
906         if (!burst_ahb_width) {
907                 if (len & 0xF)
908                         return TEGRA_APBDMA_AHBSEQ_BURST_1;
909                 else if ((len >> 4) & 0x1)
910                         return TEGRA_APBDMA_AHBSEQ_BURST_4;
911                 else
912                         return TEGRA_APBDMA_AHBSEQ_BURST_8;
913         }
914         if (burst_ahb_width < 4)
915                 return TEGRA_APBDMA_AHBSEQ_BURST_1;
916         else if (burst_ahb_width < 8)
917                 return TEGRA_APBDMA_AHBSEQ_BURST_4;
918         else
919                 return TEGRA_APBDMA_AHBSEQ_BURST_8;
920 }
921
922 static int get_transfer_param(struct tegra_dma_channel *tdc,
923         enum dma_transfer_direction direction, unsigned long *apb_addr,
924         unsigned long *apb_seq, unsigned long *csr, unsigned int *burst_size,
925         enum dma_slave_buswidth *slave_bw)
926 {
927
928         switch (direction) {
929         case DMA_MEM_TO_DEV:
930                 *apb_addr = tdc->dma_sconfig.dst_addr;
931                 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
932                 *burst_size = tdc->dma_sconfig.dst_maxburst;
933                 *slave_bw = tdc->dma_sconfig.dst_addr_width;
934                 *csr = TEGRA_APBDMA_CSR_DIR;
935                 return 0;
936
937         case DMA_DEV_TO_MEM:
938                 *apb_addr = tdc->dma_sconfig.src_addr;
939                 *apb_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
940                 *burst_size = tdc->dma_sconfig.src_maxburst;
941                 *slave_bw = tdc->dma_sconfig.src_addr_width;
942                 *csr = 0;
943                 return 0;
944
945         default:
946                 dev_err(tdc2dev(tdc), "Dma direction is not supported\n");
947                 return -EINVAL;
948         }
949         return -EINVAL;
950 }
951
952 static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
953         struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len,
954         enum dma_transfer_direction direction, unsigned long flags,
955         void *context)
956 {
957         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
958         struct tegra_dma_desc *dma_desc;
959         unsigned int        i;
960         struct scatterlist      *sg;
961         unsigned long csr, ahb_seq, apb_ptr, apb_seq;
962         struct list_head req_list;
963         struct tegra_dma_sg_req  *sg_req = NULL;
964         u32 burst_size;
965         enum dma_slave_buswidth slave_bw;
966         int ret;
967
968         if (!tdc->config_init) {
969                 dev_err(tdc2dev(tdc), "dma channel is not configured\n");
970                 return NULL;
971         }
972         if (sg_len < 1) {
973                 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
974                 return NULL;
975         }
976
977         ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
978                                 &burst_size, &slave_bw);
979         if (ret < 0)
980                 return NULL;
981
982         INIT_LIST_HEAD(&req_list);
983
984         ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
985         ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
986                                         TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
987         ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
988
989         csr |= TEGRA_APBDMA_CSR_ONCE | TEGRA_APBDMA_CSR_FLOW;
990         csr |= tdc->dma_sconfig.slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
991         if (flags & DMA_PREP_INTERRUPT)
992                 csr |= TEGRA_APBDMA_CSR_IE_EOC;
993
994         apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
995
996         dma_desc = tegra_dma_desc_get(tdc);
997         if (!dma_desc) {
998                 dev_err(tdc2dev(tdc), "Dma descriptors not available\n");
999                 return NULL;
1000         }
1001         INIT_LIST_HEAD(&dma_desc->tx_list);
1002         INIT_LIST_HEAD(&dma_desc->cb_node);
1003         dma_desc->cb_count = 0;
1004         dma_desc->bytes_requested = 0;
1005         dma_desc->bytes_transferred = 0;
1006         dma_desc->dma_status = DMA_IN_PROGRESS;
1007
1008         /* Make transfer requests */
1009         for_each_sg(sgl, sg, sg_len, i) {
1010                 u32 len, mem;
1011
1012                 mem = sg_dma_address(sg);
1013                 len = sg_dma_len(sg);
1014
1015                 if ((len & 3) || (mem & 3) ||
1016                                 (len > tdc->tdma->chip_data->max_dma_count)) {
1017                         dev_err(tdc2dev(tdc),
1018                                 "Dma length/memory address is not supported\n");
1019                         tegra_dma_desc_put(tdc, dma_desc);
1020                         return NULL;
1021                 }
1022
1023                 sg_req = tegra_dma_sg_req_get(tdc);
1024                 if (!sg_req) {
1025                         dev_err(tdc2dev(tdc), "Dma sg-req not available\n");
1026                         tegra_dma_desc_put(tdc, dma_desc);
1027                         return NULL;
1028                 }
1029
1030                 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1031                 dma_desc->bytes_requested += len;
1032
1033                 sg_req->ch_regs.apb_ptr = apb_ptr;
1034                 sg_req->ch_regs.ahb_ptr = mem;
1035                 if (tdc->tdma->chip_data->support_separate_wcount_reg) {
1036                         sg_req->ch_regs.wcount = (len - 4) & 0xFFFC;
1037                         sg_req->ch_regs.csr = csr;
1038                 } else {
1039                         sg_req->ch_regs.csr = csr | ((len - 4) & 0xFFFC);
1040                 }
1041                 sg_req->ch_regs.apb_seq = apb_seq;
1042                 sg_req->ch_regs.ahb_seq = ahb_seq;
1043                 sg_req->configured = false;
1044                 sg_req->last_sg = false;
1045                 sg_req->dma_desc = dma_desc;
1046                 sg_req->req_len = len;
1047
1048                 list_add_tail(&sg_req->node, &dma_desc->tx_list);
1049         }
1050         sg_req->last_sg = true;
1051         if (flags & DMA_CTRL_ACK)
1052                 dma_desc->txd.flags = DMA_CTRL_ACK;
1053
1054         /*
1055          * Make sure that mode should not be conflicting with currently
1056          * configured mode.
1057          */
1058         if (!tdc->isr_handler) {
1059                 tdc->isr_handler = handle_once_dma_done;
1060                 tdc->cyclic = false;
1061         } else {
1062                 if (tdc->cyclic) {
1063                         dev_err(tdc2dev(tdc), "DMA configured in cyclic mode\n");
1064                         tegra_dma_desc_put(tdc, dma_desc);
1065                         return NULL;
1066                 }
1067         }
1068
1069         return &dma_desc->txd;
1070 }
1071
1072 struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
1073         struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1074         size_t period_len, enum dma_transfer_direction direction,
1075         unsigned long flags, void *context)
1076 {
1077         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1078         struct tegra_dma_desc *dma_desc = NULL;
1079         struct tegra_dma_sg_req  *sg_req = NULL;
1080         unsigned long csr, ahb_seq, apb_ptr, apb_seq;
1081         int len;
1082         size_t remain_len;
1083         dma_addr_t mem = buf_addr;
1084         u32 burst_size;
1085         enum dma_slave_buswidth slave_bw;
1086         int ret;
1087
1088         if (!buf_len || !period_len) {
1089                 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
1090                 return NULL;
1091         }
1092
1093         if (!tdc->config_init) {
1094                 dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
1095                 return NULL;
1096         }
1097
1098         /*
1099          * We allow to take more number of requests till DMA is
1100          * not started. The driver will loop over all requests.
1101          * Once DMA is started then new requests can be queued only after
1102          * terminating the DMA.
1103          */
1104         if (tdc->busy) {
1105                 dev_err(tdc2dev(tdc), "Request not allowed when dma running\n");
1106                 return NULL;
1107         }
1108
1109         /*
1110          * We only support cycle transfer when buf_len is multiple of
1111          * period_len.
1112          */
1113         if (buf_len % period_len) {
1114                 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
1115                 return NULL;
1116         }
1117
1118         len = period_len;
1119         if ((len & 3) || (buf_addr & 3) ||
1120                         (len > tdc->tdma->chip_data->max_dma_count)) {
1121                 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1122                 return NULL;
1123         }
1124
1125         ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
1126                                 &burst_size, &slave_bw);
1127         if (ret < 0)
1128                 return NULL;
1129
1130
1131         ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
1132         ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
1133                                         TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
1134         ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
1135
1136         csr |= TEGRA_APBDMA_CSR_FLOW;
1137         if (flags & DMA_PREP_INTERRUPT)
1138                 csr |= TEGRA_APBDMA_CSR_IE_EOC;
1139         csr |= tdc->dma_sconfig.slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
1140
1141         apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
1142
1143         dma_desc = tegra_dma_desc_get(tdc);
1144         if (!dma_desc) {
1145                 dev_err(tdc2dev(tdc), "not enough descriptors available\n");
1146                 return NULL;
1147         }
1148
1149         INIT_LIST_HEAD(&dma_desc->tx_list);
1150         INIT_LIST_HEAD(&dma_desc->cb_node);
1151         dma_desc->cb_count = 0;
1152
1153         dma_desc->bytes_transferred = 0;
1154         dma_desc->bytes_requested = buf_len;
1155         remain_len = buf_len;
1156
1157         /* Split transfer equal to period size */
1158         while (remain_len) {
1159                 sg_req = tegra_dma_sg_req_get(tdc);
1160                 if (!sg_req) {
1161                         dev_err(tdc2dev(tdc), "Dma sg-req not available\n");
1162                         tegra_dma_desc_put(tdc, dma_desc);
1163                         return NULL;
1164                 }
1165
1166                 ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1167                 sg_req->ch_regs.apb_ptr = apb_ptr;
1168                 sg_req->ch_regs.ahb_ptr = mem;
1169                 if (tdc->tdma->chip_data->support_separate_wcount_reg) {
1170                         sg_req->ch_regs.wcount = (len - 4) & 0xFFFC;
1171                         sg_req->ch_regs.csr = csr;
1172                 } else {
1173                         sg_req->ch_regs.csr = csr | ((len - 4) & 0xFFFC);
1174                 }
1175                 sg_req->ch_regs.apb_seq = apb_seq;
1176                 sg_req->ch_regs.ahb_seq = ahb_seq;
1177                 sg_req->configured = false;
1178                 sg_req->half_done = false;
1179                 sg_req->last_sg = false;
1180                 sg_req->dma_desc = dma_desc;
1181                 sg_req->req_len = len;
1182
1183                 list_add_tail(&sg_req->node, &dma_desc->tx_list);
1184                 remain_len -= len;
1185                 mem += len;
1186         }
1187         sg_req->last_sg = true;
1188         if (flags & DMA_CTRL_ACK)
1189                 dma_desc->txd.flags = DMA_CTRL_ACK;
1190
1191         /*
1192          * Make sure that mode should not be conflicting with currently
1193          * configured mode.
1194          */
1195         if (!tdc->isr_handler) {
1196                 tdc->isr_handler = handle_cont_sngl_cycle_dma_done;
1197                 tdc->cyclic = true;
1198         } else {
1199                 if (!tdc->cyclic) {
1200                         dev_err(tdc2dev(tdc), "DMA configuration conflict\n");
1201                         tegra_dma_desc_put(tdc, dma_desc);
1202                         return NULL;
1203                 }
1204         }
1205
1206         return &dma_desc->txd;
1207 }
1208
1209 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1210 {
1211         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1212
1213         clk_prepare(tdc->tdma->dma_clk);
1214         dma_cookie_init(&tdc->dma_chan);
1215         tdc->config_init = false;
1216         return 0;
1217 }
1218
1219 static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1220 {
1221         struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1222         struct tegra_dma_desc *dma_desc;
1223         struct tegra_dma_sg_req *sg_req;
1224         struct list_head dma_desc_list;
1225         struct list_head sg_req_list;
1226         unsigned long flags;
1227
1228         INIT_LIST_HEAD(&dma_desc_list);
1229         INIT_LIST_HEAD(&sg_req_list);
1230
1231         dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1232
1233         if (tdc->busy)
1234                 tegra_dma_terminate_all(dc);
1235         clk_unprepare(tdc->tdma->dma_clk);
1236         spin_lock_irqsave(&tdc->lock, flags);
1237         list_splice_init(&tdc->pending_sg_req, &sg_req_list);
1238         list_splice_init(&tdc->free_sg_req, &sg_req_list);
1239         list_splice_init(&tdc->free_dma_desc, &dma_desc_list);
1240         INIT_LIST_HEAD(&tdc->cb_desc);
1241         tdc->config_init = false;
1242         tdc->isr_handler = NULL;
1243         spin_unlock_irqrestore(&tdc->lock, flags);
1244
1245         while (!list_empty(&dma_desc_list)) {
1246                 dma_desc = list_first_entry(&dma_desc_list,
1247                                         typeof(*dma_desc), node);
1248                 list_del(&dma_desc->node);
1249                 kfree(dma_desc);
1250         }
1251
1252         while (!list_empty(&sg_req_list)) {
1253                 sg_req = list_first_entry(&sg_req_list, typeof(*sg_req), node);
1254                 list_del(&sg_req->node);
1255                 kfree(sg_req);
1256         }
1257 }
1258
1259 /* Tegra20 specific DMA controller information */
1260 static const struct tegra_dma_chip_data tegra20_dma_chip_data = {
1261         .nr_channels            = 16,
1262         .channel_reg_size       = 0x20,
1263         .max_dma_count          = 1024UL * 64,
1264         .support_channel_pause  = false,
1265         .support_separate_wcount_reg = false,
1266 };
1267
1268 #if defined(CONFIG_OF)
1269 /* Tegra30 specific DMA controller information */
1270 static const struct tegra_dma_chip_data tegra30_dma_chip_data = {
1271         .nr_channels            = 32,
1272         .channel_reg_size       = 0x20,
1273         .max_dma_count          = 1024UL * 64,
1274         .support_channel_pause  = false,
1275         .support_separate_wcount_reg = false,
1276 };
1277
1278 /* Tegra114 specific DMA controller information */
1279 static const struct tegra_dma_chip_data tegra114_dma_chip_data = {
1280         .nr_channels            = 32,
1281         .channel_reg_size       = 0x20,
1282         .max_dma_count          = 1024UL * 64,
1283         .support_channel_pause  = true,
1284         .support_separate_wcount_reg = false,
1285 };
1286
1287 /* Tegra148 specific DMA controller information */
1288 static const struct tegra_dma_chip_data tegra148_dma_chip_data = {
1289         .nr_channels            = 32,
1290         .channel_reg_size       = 0x40,
1291         .max_dma_count          = 1024UL * 64,
1292         .support_channel_pause  = true,
1293         .support_separate_wcount_reg = true,
1294 };
1295
1296 static const struct tegra_dma_chip_data tegra124_dma_chip_data = {
1297         .nr_channels            = 32,
1298         .channel_reg_size       = 0x40,
1299         .max_dma_count          = 1024UL * 64,
1300         .support_channel_pause  = true,
1301         .support_separate_wcount_reg = true,
1302 };
1303
1304 static const struct of_device_id tegra_dma_of_match[] = {
1305         {
1306                 .compatible = "nvidia,tegra124-apbdma",
1307                 .data = &tegra124_dma_chip_data,
1308         }, {
1309                 .compatible = "nvidia,tegra148-apbdma",
1310                 .data = &tegra148_dma_chip_data,
1311         }, {
1312                 .compatible = "nvidia,tegra114-apbdma",
1313                 .data = &tegra114_dma_chip_data,
1314         }, {
1315                 .compatible = "nvidia,tegra30-apbdma",
1316                 .data = &tegra30_dma_chip_data,
1317         }, {
1318                 .compatible = "nvidia,tegra20-apbdma",
1319                 .data = &tegra20_dma_chip_data,
1320         }, {
1321         },
1322 };
1323 MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
1324 #endif
1325
1326 static struct platform_device_id tegra_dma_devtype[] = {
1327         {
1328                 .name = "tegra20-apbdma",
1329                 .driver_data = (unsigned long)&tegra20_dma_chip_data,
1330         },
1331         {
1332                 .name = "tegra30-apbdma",
1333                 .driver_data = (unsigned long)&tegra30_dma_chip_data,
1334         },
1335         {
1336                 .name = "tegra114-apbdma",
1337                 .driver_data = (unsigned long)&tegra114_dma_chip_data,
1338         },
1339         {
1340                 .name = "tegra148-apbdma",
1341                 .driver_data = (unsigned long)&tegra148_dma_chip_data,
1342         },
1343         {
1344                 .name = "tegra124-apbdma",
1345                 .driver_data = (unsigned long)&tegra124_dma_chip_data,
1346         },
1347 };
1348
1349 static struct device *dma_device;
1350
1351 static int tegra_dma_probe(struct platform_device *pdev)
1352 {
1353         struct resource *res;
1354         struct tegra_dma *tdma;
1355         int ret;
1356         int i;
1357         const struct tegra_dma_chip_data *cdata = NULL;
1358
1359         if (pdev->dev.of_node) {
1360                 const struct of_device_id *match;
1361                 match = of_match_device(of_match_ptr(tegra_dma_of_match),
1362                                         &pdev->dev);
1363                 if (!match) {
1364                         dev_err(&pdev->dev, "Error: No device match found\n");
1365                         return -ENODEV;
1366                 }
1367                 cdata = match->data;
1368         } else {
1369                 /* If no device tree then fallback to tegra20 */
1370                 cdata = (struct tegra_dma_chip_data *)pdev->id_entry->driver_data;
1371         }
1372
1373         tdma = devm_kzalloc(&pdev->dev, sizeof(*tdma) + cdata->nr_channels *
1374                         sizeof(struct tegra_dma_channel), GFP_KERNEL);
1375         if (!tdma) {
1376                 dev_err(&pdev->dev, "Error: memory allocation failed\n");
1377                 return -ENOMEM;
1378         }
1379
1380         tdma->dev = &pdev->dev;
1381         tdma->chip_data = cdata;
1382         platform_set_drvdata(pdev, tdma);
1383
1384         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1385         if (!res) {
1386                 dev_err(&pdev->dev, "No mem resource for DMA\n");
1387                 return -EINVAL;
1388         }
1389
1390         tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);
1391         if (IS_ERR(tdma->base_addr))
1392                 return PTR_ERR(tdma->base_addr);
1393
1394         tdma->dma_clk = devm_clk_get(&pdev->dev, NULL);
1395         if (IS_ERR(tdma->dma_clk)) {
1396                 dev_err(&pdev->dev, "Error: Missing controller clock\n");
1397                 return PTR_ERR(tdma->dma_clk);
1398         }
1399
1400         spin_lock_init(&tdma->global_lock);
1401
1402         dma_device = &pdev->dev;
1403
1404         tegra_pd_add_device(&pdev->dev);
1405         pm_runtime_enable(&pdev->dev);
1406         if (!pm_runtime_enabled(&pdev->dev)) {
1407                 ret = tegra_dma_runtime_resume(&pdev->dev);
1408                 if (ret) {
1409                         dev_err(&pdev->dev, "dma_runtime_resume failed %d\n",
1410                                 ret);
1411                         goto err_pm_disable;
1412                 }
1413         }
1414
1415         /* Enable clock before accessing registers */
1416         ret = clk_prepare_enable(tdma->dma_clk);
1417         if (ret < 0) {
1418                 dev_err(&pdev->dev, "clk_prepare_enable failed: %d\n", ret);
1419                 goto err_pm_disable;
1420         }
1421
1422         /* Reset DMA controller */
1423         tegra_periph_reset_assert(tdma->dma_clk);
1424         udelay(2);
1425         tegra_periph_reset_deassert(tdma->dma_clk);
1426
1427         /* Enable global DMA registers */
1428         tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);
1429         tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
1430         tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
1431
1432         clk_disable_unprepare(tdma->dma_clk);
1433
1434         INIT_LIST_HEAD(&tdma->dma_dev.channels);
1435         for (i = 0; i < cdata->nr_channels; i++) {
1436                 struct tegra_dma_channel *tdc = &tdma->channels[i];
1437
1438                 tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET +
1439                                         i * cdata->channel_reg_size;
1440
1441                 res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
1442                 if (!res) {
1443                         ret = -EINVAL;
1444                         dev_err(&pdev->dev, "No irq resource for chan %d\n", i);
1445                         goto err_irq;
1446                 }
1447                 tdc->irq = res->start;
1448                 snprintf(tdc->name, sizeof(tdc->name), "apbdma.%d", i);
1449                 ret = devm_request_irq(&pdev->dev, tdc->irq,
1450                                 tegra_dma_isr, 0, tdc->name, tdc);
1451                 if (ret) {
1452                         dev_err(&pdev->dev,
1453                                 "request_irq failed with err %d channel %d\n",
1454                                 i, ret);
1455                         goto err_irq;
1456                 }
1457
1458                 tdc->dma_chan.device = &tdma->dma_dev;
1459                 dma_cookie_init(&tdc->dma_chan);
1460                 list_add_tail(&tdc->dma_chan.device_node,
1461                                 &tdma->dma_dev.channels);
1462                 tdc->tdma = tdma;
1463                 tdc->id = i;
1464
1465                 tasklet_init(&tdc->tasklet, tegra_dma_tasklet,
1466                                 (unsigned long)tdc);
1467                 spin_lock_init(&tdc->lock);
1468
1469                 INIT_LIST_HEAD(&tdc->pending_sg_req);
1470                 INIT_LIST_HEAD(&tdc->free_sg_req);
1471                 INIT_LIST_HEAD(&tdc->free_dma_desc);
1472                 INIT_LIST_HEAD(&tdc->cb_desc);
1473         }
1474
1475         dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
1476         dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1477         dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1478
1479         tdma->dma_dev.dev = &pdev->dev;
1480         tdma->dma_dev.device_alloc_chan_resources =
1481                                         tegra_dma_alloc_chan_resources;
1482         tdma->dma_dev.device_free_chan_resources =
1483                                         tegra_dma_free_chan_resources;
1484         tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
1485         tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
1486         tdma->dma_dev.device_control = tegra_dma_device_control;
1487         tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1488         tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1489
1490         ret = dma_async_device_register(&tdma->dma_dev);
1491         if (ret < 0) {
1492                 dev_err(&pdev->dev,
1493                         "Tegra20 APB DMA driver registration failed %d\n", ret);
1494                 goto err_irq;
1495         }
1496
1497         dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n",
1498                         cdata->nr_channels);
1499         return 0;
1500
1501 err_irq:
1502         while (--i >= 0) {
1503                 struct tegra_dma_channel *tdc = &tdma->channels[i];
1504                 tasklet_kill(&tdc->tasklet);
1505         }
1506
1507 err_pm_disable:
1508         pm_runtime_disable(&pdev->dev);
1509         if (!pm_runtime_status_suspended(&pdev->dev))
1510                 tegra_dma_runtime_suspend(&pdev->dev);
1511         tegra_pd_remove_device(&pdev->dev);
1512         return ret;
1513 }
1514
1515 static int tegra_dma_remove(struct platform_device *pdev)
1516 {
1517         struct tegra_dma *tdma = platform_get_drvdata(pdev);
1518         int i;
1519         struct tegra_dma_channel *tdc;
1520
1521         dma_async_device_unregister(&tdma->dma_dev);
1522
1523         for (i = 0; i < tdma->chip_data->nr_channels; ++i) {
1524                 tdc = &tdma->channels[i];
1525                 tasklet_kill(&tdc->tasklet);
1526         }
1527
1528         pm_runtime_disable(&pdev->dev);
1529         if (!pm_runtime_status_suspended(&pdev->dev))
1530                 tegra_dma_runtime_suspend(&pdev->dev);
1531
1532         tegra_pd_remove_device(&pdev->dev);
1533         return 0;
1534 }
1535
1536 static int tegra_dma_runtime_suspend(struct device *dev)
1537 {
1538         return 0;
1539 }
1540
1541 static int tegra_dma_runtime_resume(struct device *dev)
1542 {
1543         return 0;
1544 }
1545
1546 #ifdef CONFIG_PM_SLEEP
1547 static int tegra_dma_pm_suspend(struct device *dev)
1548 {
1549         struct tegra_dma *tdma = dev_get_drvdata(dev);
1550         int i;
1551         int ret;
1552
1553         ret = tegra_dma_runtime_resume(dev);
1554         if (ret < 0)
1555                 return ret;
1556
1557         tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL);
1558         for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1559                 struct tegra_dma_channel *tdc = &tdma->channels[i];
1560                 struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
1561
1562                 ch_reg->csr = tdc_read(tdc, TEGRA_APBDMA_CHAN_CSR);
1563                 ch_reg->ahb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBPTR);
1564                 ch_reg->apb_ptr = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBPTR);
1565                 ch_reg->ahb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_AHBSEQ);
1566                 ch_reg->apb_seq = tdc_read(tdc, TEGRA_APBDMA_CHAN_APBSEQ);
1567                 if (tdc->tdma->chip_data->support_separate_wcount_reg)
1568                         ch_reg->wcount =
1569                                 tdc_read(tdc, TEGRA_APBDMA_CHAN_WCOUNT);
1570         }
1571         tegra_dma_runtime_suspend(dev);
1572         return 0;
1573 }
1574
1575 static int tegra_dma_pm_resume(struct device *dev)
1576 {
1577         struct tegra_dma *tdma = dev_get_drvdata(dev);
1578         int i;
1579         int ret;
1580
1581         ret = tegra_dma_runtime_resume(dev);
1582         if (ret < 0)
1583                 return ret;
1584
1585         tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen);
1586         tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);
1587         tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);
1588
1589         for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1590                 struct tegra_dma_channel *tdc = &tdma->channels[i];
1591                 struct tegra_dma_channel_regs *ch_reg = &tdc->channel_reg;
1592
1593                 if (tdc->tdma->chip_data->support_separate_wcount_reg)
1594                         tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT,
1595                                         ch_reg->wcount);
1596                 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBSEQ, ch_reg->apb_seq);
1597                 tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_reg->apb_ptr);
1598                 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_reg->ahb_seq);
1599                 tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_reg->ahb_ptr);
1600                 tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR,
1601                         (ch_reg->csr & ~TEGRA_APBDMA_CSR_ENB));
1602         }
1603         tegra_dma_runtime_suspend(dev);
1604         return 0;
1605 }
1606 #endif
1607
1608 int tegra_dma_save(void)
1609 {
1610         return tegra_dma_pm_suspend(dma_device);
1611 }
1612
1613 int tegra_dma_restore(void)
1614 {
1615         return tegra_dma_pm_resume(dma_device);
1616 }
1617
1618 static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1619 #ifdef CONFIG_PM_RUNTIME
1620         .runtime_suspend = tegra_dma_runtime_suspend,
1621         .runtime_resume = tegra_dma_runtime_resume,
1622 #endif
1623         SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)
1624 };
1625
1626 static struct platform_driver tegra_dmac_driver = {
1627         .driver = {
1628                 .name   = "tegra-apbdma",
1629                 .owner = THIS_MODULE,
1630                 .pm     = &tegra_dma_dev_pm_ops,
1631                 .of_match_table = of_match_ptr(tegra_dma_of_match),
1632         },
1633         .probe          = tegra_dma_probe,
1634         .remove         = tegra_dma_remove,
1635         .id_table       = tegra_dma_devtype,
1636 };
1637
1638 module_platform_driver(tegra_dmac_driver);
1639
1640 MODULE_ALIAS("platform:tegra20-apbdma");
1641 MODULE_DESCRIPTION("NVIDIA Tegra APB DMA Controller driver");
1642 MODULE_AUTHOR("Laxman Dewangan <ldewangan@nvidia.com>");
1643 MODULE_LICENSE("GPL v2");