ste_dma40: move channel mode to a separate field
[linux-2.6.git] / drivers / dma / ste_dma40.c
1 /*
2  * Copyright (C) ST-Ericsson SA 2007-2010
3  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
4  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
5  * License terms: GNU General Public License (GPL) version 2
6  */
7
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/dmaengine.h>
11 #include <linux/platform_device.h>
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/err.h>
15
16 #include <plat/ste_dma40.h>
17
18 #include "ste_dma40_ll.h"
19
20 #define D40_NAME "dma40"
21
22 #define D40_PHY_CHAN -1
23
24 /* For masking out/in 2 bit channel positions */
25 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
26 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
27
28 /* Maximum iterations taken before giving up suspending a channel */
29 #define D40_SUSPEND_MAX_IT 500
30
31 /* Hardware requirement on LCLA alignment */
32 #define LCLA_ALIGNMENT 0x40000
33
34 /* Max number of links per event group */
35 #define D40_LCLA_LINK_PER_EVENT_GRP 128
36 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
37
38 /* Attempts before giving up to trying to get pages that are aligned */
39 #define MAX_LCLA_ALLOC_ATTEMPTS 256
40
41 /* Bit markings for allocation map */
42 #define D40_ALLOC_FREE          (1 << 31)
43 #define D40_ALLOC_PHY           (1 << 30)
44 #define D40_ALLOC_LOG_FREE      0
45
46 /* Hardware designer of the block */
47 #define D40_HW_DESIGNER 0x8
48
49 /**
50  * enum 40_command - The different commands and/or statuses.
51  *
52  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
53  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
54  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
55  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
56  */
57 enum d40_command {
58         D40_DMA_STOP            = 0,
59         D40_DMA_RUN             = 1,
60         D40_DMA_SUSPEND_REQ     = 2,
61         D40_DMA_SUSPENDED       = 3
62 };
63
64 /**
65  * struct d40_lli_pool - Structure for keeping LLIs in memory
66  *
67  * @base: Pointer to memory area when the pre_alloc_lli's are not large
68  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
69  * pre_alloc_lli is used.
70  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
71  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
72  * one buffer to one buffer.
73  */
74 struct d40_lli_pool {
75         void    *base;
76         int      size;
77         /* Space for dst and src, plus an extra for padding */
78         u8       pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
79 };
80
81 /**
82  * struct d40_desc - A descriptor is one DMA job.
83  *
84  * @lli_phy: LLI settings for physical channel. Both src and dst=
85  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
86  * lli_len equals one.
87  * @lli_log: Same as above but for logical channels.
88  * @lli_pool: The pool with two entries pre-allocated.
89  * @lli_len: Number of llis of current descriptor.
90  * @lli_current: Number of transfered llis.
91  * @lcla_alloc: Number of LCLA entries allocated.
92  * @txd: DMA engine struct. Used for among other things for communication
93  * during a transfer.
94  * @node: List entry.
95  * @is_in_client_list: true if the client owns this descriptor.
96  * @is_hw_linked: true if this job will automatically be continued for
97  * the previous one.
98  *
99  * This descriptor is used for both logical and physical transfers.
100  */
101 struct d40_desc {
102         /* LLI physical */
103         struct d40_phy_lli_bidir         lli_phy;
104         /* LLI logical */
105         struct d40_log_lli_bidir         lli_log;
106
107         struct d40_lli_pool              lli_pool;
108         int                              lli_len;
109         int                              lli_current;
110         int                              lcla_alloc;
111
112         struct dma_async_tx_descriptor   txd;
113         struct list_head                 node;
114
115         bool                             is_in_client_list;
116         bool                             is_hw_linked;
117 };
118
119 /**
120  * struct d40_lcla_pool - LCLA pool settings and data.
121  *
122  * @base: The virtual address of LCLA. 18 bit aligned.
123  * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
124  * This pointer is only there for clean-up on error.
125  * @pages: The number of pages needed for all physical channels.
126  * Only used later for clean-up on error
127  * @lock: Lock to protect the content in this struct.
128  * @alloc_map: big map over which LCLA entry is own by which job.
129  */
130 struct d40_lcla_pool {
131         void            *base;
132         void            *base_unaligned;
133         int              pages;
134         spinlock_t       lock;
135         struct d40_desc **alloc_map;
136 };
137
138 /**
139  * struct d40_phy_res - struct for handling eventlines mapped to physical
140  * channels.
141  *
142  * @lock: A lock protection this entity.
143  * @num: The physical channel number of this entity.
144  * @allocated_src: Bit mapped to show which src event line's are mapped to
145  * this physical channel. Can also be free or physically allocated.
146  * @allocated_dst: Same as for src but is dst.
147  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
148  * event line number.
149  */
150 struct d40_phy_res {
151         spinlock_t lock;
152         int        num;
153         u32        allocated_src;
154         u32        allocated_dst;
155 };
156
157 struct d40_base;
158
159 /**
160  * struct d40_chan - Struct that describes a channel.
161  *
162  * @lock: A spinlock to protect this struct.
163  * @log_num: The logical number, if any of this channel.
164  * @completed: Starts with 1, after first interrupt it is set to dma engine's
165  * current cookie.
166  * @pending_tx: The number of pending transfers. Used between interrupt handler
167  * and tasklet.
168  * @busy: Set to true when transfer is ongoing on this channel.
169  * @phy_chan: Pointer to physical channel which this instance runs on. If this
170  * point is NULL, then the channel is not allocated.
171  * @chan: DMA engine handle.
172  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
173  * transfer and call client callback.
174  * @client: Cliented owned descriptor list.
175  * @active: Active descriptor.
176  * @queue: Queued jobs.
177  * @dma_cfg: The client configuration of this dma channel.
178  * @configured: whether the dma_cfg configuration is valid
179  * @base: Pointer to the device instance struct.
180  * @src_def_cfg: Default cfg register setting for src.
181  * @dst_def_cfg: Default cfg register setting for dst.
182  * @log_def: Default logical channel settings.
183  * @lcla: Space for one dst src pair for logical channel transfers.
184  * @lcpa: Pointer to dst and src lcpa settings.
185  *
186  * This struct can either "be" a logical or a physical channel.
187  */
188 struct d40_chan {
189         spinlock_t                       lock;
190         int                              log_num;
191         /* ID of the most recent completed transfer */
192         int                              completed;
193         int                              pending_tx;
194         bool                             busy;
195         struct d40_phy_res              *phy_chan;
196         struct dma_chan                  chan;
197         struct tasklet_struct            tasklet;
198         struct list_head                 client;
199         struct list_head                 active;
200         struct list_head                 queue;
201         struct stedma40_chan_cfg         dma_cfg;
202         bool                             configured;
203         struct d40_base                 *base;
204         /* Default register configurations */
205         u32                              src_def_cfg;
206         u32                              dst_def_cfg;
207         struct d40_def_lcsp              log_def;
208         struct d40_log_lli_full         *lcpa;
209         /* Runtime reconfiguration */
210         dma_addr_t                      runtime_addr;
211         enum dma_data_direction         runtime_direction;
212 };
213
214 /**
215  * struct d40_base - The big global struct, one for each probe'd instance.
216  *
217  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
218  * @execmd_lock: Lock for execute command usage since several channels share
219  * the same physical register.
220  * @dev: The device structure.
221  * @virtbase: The virtual base address of the DMA's register.
222  * @rev: silicon revision detected.
223  * @clk: Pointer to the DMA clock structure.
224  * @phy_start: Physical memory start of the DMA registers.
225  * @phy_size: Size of the DMA register map.
226  * @irq: The IRQ number.
227  * @num_phy_chans: The number of physical channels. Read from HW. This
228  * is the number of available channels for this driver, not counting "Secure
229  * mode" allocated physical channels.
230  * @num_log_chans: The number of logical channels. Calculated from
231  * num_phy_chans.
232  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
233  * @dma_slave: dma_device channels that can do only do slave transfers.
234  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
235  * @log_chans: Room for all possible logical channels in system.
236  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
237  * to log_chans entries.
238  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
239  * to phy_chans entries.
240  * @plat_data: Pointer to provided platform_data which is the driver
241  * configuration.
242  * @phy_res: Vector containing all physical channels.
243  * @lcla_pool: lcla pool settings and data.
244  * @lcpa_base: The virtual mapped address of LCPA.
245  * @phy_lcpa: The physical address of the LCPA.
246  * @lcpa_size: The size of the LCPA area.
247  * @desc_slab: cache for descriptors.
248  */
249 struct d40_base {
250         spinlock_t                       interrupt_lock;
251         spinlock_t                       execmd_lock;
252         struct device                    *dev;
253         void __iomem                     *virtbase;
254         u8                                rev:4;
255         struct clk                       *clk;
256         phys_addr_t                       phy_start;
257         resource_size_t                   phy_size;
258         int                               irq;
259         int                               num_phy_chans;
260         int                               num_log_chans;
261         struct dma_device                 dma_both;
262         struct dma_device                 dma_slave;
263         struct dma_device                 dma_memcpy;
264         struct d40_chan                  *phy_chans;
265         struct d40_chan                  *log_chans;
266         struct d40_chan                 **lookup_log_chans;
267         struct d40_chan                 **lookup_phy_chans;
268         struct stedma40_platform_data    *plat_data;
269         /* Physical half channels */
270         struct d40_phy_res               *phy_res;
271         struct d40_lcla_pool              lcla_pool;
272         void                             *lcpa_base;
273         dma_addr_t                        phy_lcpa;
274         resource_size_t                   lcpa_size;
275         struct kmem_cache                *desc_slab;
276 };
277
278 /**
279  * struct d40_interrupt_lookup - lookup table for interrupt handler
280  *
281  * @src: Interrupt mask register.
282  * @clr: Interrupt clear register.
283  * @is_error: true if this is an error interrupt.
284  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
285  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
286  */
287 struct d40_interrupt_lookup {
288         u32 src;
289         u32 clr;
290         bool is_error;
291         int offset;
292 };
293
294 /**
295  * struct d40_reg_val - simple lookup struct
296  *
297  * @reg: The register.
298  * @val: The value that belongs to the register in reg.
299  */
300 struct d40_reg_val {
301         unsigned int reg;
302         unsigned int val;
303 };
304
305 static int d40_pool_lli_alloc(struct d40_desc *d40d,
306                               int lli_len, bool is_log)
307 {
308         u32 align;
309         void *base;
310
311         if (is_log)
312                 align = sizeof(struct d40_log_lli);
313         else
314                 align = sizeof(struct d40_phy_lli);
315
316         if (lli_len == 1) {
317                 base = d40d->lli_pool.pre_alloc_lli;
318                 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
319                 d40d->lli_pool.base = NULL;
320         } else {
321                 d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
322
323                 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
324                 d40d->lli_pool.base = base;
325
326                 if (d40d->lli_pool.base == NULL)
327                         return -ENOMEM;
328         }
329
330         if (is_log) {
331                 d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
332                                               align);
333                 d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
334                                               align);
335         } else {
336                 d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
337                                               align);
338                 d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
339                                               align);
340         }
341
342         return 0;
343 }
344
345 static void d40_pool_lli_free(struct d40_desc *d40d)
346 {
347         kfree(d40d->lli_pool.base);
348         d40d->lli_pool.base = NULL;
349         d40d->lli_pool.size = 0;
350         d40d->lli_log.src = NULL;
351         d40d->lli_log.dst = NULL;
352         d40d->lli_phy.src = NULL;
353         d40d->lli_phy.dst = NULL;
354 }
355
356 static int d40_lcla_alloc_one(struct d40_chan *d40c,
357                               struct d40_desc *d40d)
358 {
359         unsigned long flags;
360         int i;
361         int ret = -EINVAL;
362         int p;
363
364         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
365
366         p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
367
368         /*
369          * Allocate both src and dst at the same time, therefore the half
370          * start on 1 since 0 can't be used since zero is used as end marker.
371          */
372         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
373                 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
374                         d40c->base->lcla_pool.alloc_map[p + i] = d40d;
375                         d40d->lcla_alloc++;
376                         ret = i;
377                         break;
378                 }
379         }
380
381         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
382
383         return ret;
384 }
385
386 static int d40_lcla_free_all(struct d40_chan *d40c,
387                              struct d40_desc *d40d)
388 {
389         unsigned long flags;
390         int i;
391         int ret = -EINVAL;
392
393         if (d40c->log_num == D40_PHY_CHAN)
394                 return 0;
395
396         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
397
398         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
399                 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
400                                                     D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
401                         d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
402                                                         D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
403                         d40d->lcla_alloc--;
404                         if (d40d->lcla_alloc == 0) {
405                                 ret = 0;
406                                 break;
407                         }
408                 }
409         }
410
411         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
412
413         return ret;
414
415 }
416
417 static void d40_desc_remove(struct d40_desc *d40d)
418 {
419         list_del(&d40d->node);
420 }
421
422 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
423 {
424         struct d40_desc *desc = NULL;
425
426         if (!list_empty(&d40c->client)) {
427                 struct d40_desc *d;
428                 struct d40_desc *_d;
429
430                 list_for_each_entry_safe(d, _d, &d40c->client, node)
431                         if (async_tx_test_ack(&d->txd)) {
432                                 d40_pool_lli_free(d);
433                                 d40_desc_remove(d);
434                                 desc = d;
435                                 memset(desc, 0, sizeof(*desc));
436                                 break;
437                         }
438         }
439
440         if (!desc)
441                 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
442
443         if (desc)
444                 INIT_LIST_HEAD(&desc->node);
445
446         return desc;
447 }
448
449 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
450 {
451
452         d40_lcla_free_all(d40c, d40d);
453         kmem_cache_free(d40c->base->desc_slab, d40d);
454 }
455
456 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
457 {
458         list_add_tail(&desc->node, &d40c->active);
459 }
460
461 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
462 {
463         int curr_lcla = -EINVAL, next_lcla;
464
465         if (d40c->log_num == D40_PHY_CHAN) {
466                 d40_phy_lli_write(d40c->base->virtbase,
467                                   d40c->phy_chan->num,
468                                   d40d->lli_phy.dst,
469                                   d40d->lli_phy.src);
470                 d40d->lli_current = d40d->lli_len;
471         } else {
472
473                 if ((d40d->lli_len - d40d->lli_current) > 1)
474                         curr_lcla = d40_lcla_alloc_one(d40c, d40d);
475
476                 d40_log_lli_lcpa_write(d40c->lcpa,
477                                        &d40d->lli_log.dst[d40d->lli_current],
478                                        &d40d->lli_log.src[d40d->lli_current],
479                                        curr_lcla);
480
481                 d40d->lli_current++;
482                 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
483                         struct d40_log_lli *lcla;
484
485                         if (d40d->lli_current + 1 < d40d->lli_len)
486                                 next_lcla = d40_lcla_alloc_one(d40c, d40d);
487                         else
488                                 next_lcla = -EINVAL;
489
490                         lcla = d40c->base->lcla_pool.base +
491                                 d40c->phy_chan->num * 1024 +
492                                 8 * curr_lcla * 2;
493
494                         d40_log_lli_lcla_write(lcla,
495                                                &d40d->lli_log.dst[d40d->lli_current],
496                                                &d40d->lli_log.src[d40d->lli_current],
497                                                next_lcla);
498
499                         (void) dma_map_single(d40c->base->dev, lcla,
500                                               2 * sizeof(struct d40_log_lli),
501                                               DMA_TO_DEVICE);
502
503                         curr_lcla = next_lcla;
504
505                         if (curr_lcla == -EINVAL) {
506                                 d40d->lli_current++;
507                                 break;
508                         }
509
510                 }
511         }
512 }
513
514 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
515 {
516         struct d40_desc *d;
517
518         if (list_empty(&d40c->active))
519                 return NULL;
520
521         d = list_first_entry(&d40c->active,
522                              struct d40_desc,
523                              node);
524         return d;
525 }
526
527 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
528 {
529         list_add_tail(&desc->node, &d40c->queue);
530 }
531
532 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
533 {
534         struct d40_desc *d;
535
536         if (list_empty(&d40c->queue))
537                 return NULL;
538
539         d = list_first_entry(&d40c->queue,
540                              struct d40_desc,
541                              node);
542         return d;
543 }
544
545 static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
546 {
547         struct d40_desc *d;
548
549         if (list_empty(&d40c->queue))
550                 return NULL;
551         list_for_each_entry(d, &d40c->queue, node)
552                 if (list_is_last(&d->node, &d40c->queue))
553                         break;
554         return d;
555 }
556
557 /* Support functions for logical channels */
558
559
560 static int d40_channel_execute_command(struct d40_chan *d40c,
561                                        enum d40_command command)
562 {
563         u32 status;
564         int i;
565         void __iomem *active_reg;
566         int ret = 0;
567         unsigned long flags;
568         u32 wmask;
569
570         spin_lock_irqsave(&d40c->base->execmd_lock, flags);
571
572         if (d40c->phy_chan->num % 2 == 0)
573                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
574         else
575                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
576
577         if (command == D40_DMA_SUSPEND_REQ) {
578                 status = (readl(active_reg) &
579                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
580                         D40_CHAN_POS(d40c->phy_chan->num);
581
582                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
583                         goto done;
584         }
585
586         wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
587         writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
588                active_reg);
589
590         if (command == D40_DMA_SUSPEND_REQ) {
591
592                 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
593                         status = (readl(active_reg) &
594                                   D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
595                                 D40_CHAN_POS(d40c->phy_chan->num);
596
597                         cpu_relax();
598                         /*
599                          * Reduce the number of bus accesses while
600                          * waiting for the DMA to suspend.
601                          */
602                         udelay(3);
603
604                         if (status == D40_DMA_STOP ||
605                             status == D40_DMA_SUSPENDED)
606                                 break;
607                 }
608
609                 if (i == D40_SUSPEND_MAX_IT) {
610                         dev_err(&d40c->chan.dev->device,
611                                 "[%s]: unable to suspend the chl %d (log: %d) status %x\n",
612                                 __func__, d40c->phy_chan->num, d40c->log_num,
613                                 status);
614                         dump_stack();
615                         ret = -EBUSY;
616                 }
617
618         }
619 done:
620         spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
621         return ret;
622 }
623
624 static void d40_term_all(struct d40_chan *d40c)
625 {
626         struct d40_desc *d40d;
627
628         /* Release active descriptors */
629         while ((d40d = d40_first_active_get(d40c))) {
630                 d40_desc_remove(d40d);
631                 d40_desc_free(d40c, d40d);
632         }
633
634         /* Release queued descriptors waiting for transfer */
635         while ((d40d = d40_first_queued(d40c))) {
636                 d40_desc_remove(d40d);
637                 d40_desc_free(d40c, d40d);
638         }
639
640
641         d40c->pending_tx = 0;
642         d40c->busy = false;
643 }
644
645 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
646 {
647         u32 val;
648         unsigned long flags;
649
650         /* Notice, that disable requires the physical channel to be stopped */
651         if (do_enable)
652                 val = D40_ACTIVATE_EVENTLINE;
653         else
654                 val = D40_DEACTIVATE_EVENTLINE;
655
656         spin_lock_irqsave(&d40c->phy_chan->lock, flags);
657
658         /* Enable event line connected to device (or memcpy) */
659         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
660             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
661                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
662
663                 writel((val << D40_EVENTLINE_POS(event)) |
664                        ~D40_EVENTLINE_MASK(event),
665                        d40c->base->virtbase + D40_DREG_PCBASE +
666                        d40c->phy_chan->num * D40_DREG_PCDELTA +
667                        D40_CHAN_REG_SSLNK);
668         }
669         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
670                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
671
672                 writel((val << D40_EVENTLINE_POS(event)) |
673                        ~D40_EVENTLINE_MASK(event),
674                        d40c->base->virtbase + D40_DREG_PCBASE +
675                        d40c->phy_chan->num * D40_DREG_PCDELTA +
676                        D40_CHAN_REG_SDLNK);
677         }
678
679         spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
680 }
681
682 static u32 d40_chan_has_events(struct d40_chan *d40c)
683 {
684         u32 val;
685
686         val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
687                     d40c->phy_chan->num * D40_DREG_PCDELTA +
688                     D40_CHAN_REG_SSLNK);
689
690         val |= readl(d40c->base->virtbase + D40_DREG_PCBASE +
691                      d40c->phy_chan->num * D40_DREG_PCDELTA +
692                      D40_CHAN_REG_SDLNK);
693         return val;
694 }
695
696 static void d40_config_write(struct d40_chan *d40c)
697 {
698         u32 addr_base;
699         u32 var;
700
701         /* Odd addresses are even addresses + 4 */
702         addr_base = (d40c->phy_chan->num % 2) * 4;
703         /* Setup channel mode to logical or physical */
704         var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
705                 D40_CHAN_POS(d40c->phy_chan->num);
706         writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
707
708         /* Setup operational mode option register */
709         var = ((d40c->dma_cfg.channel_type >> STEDMA40_INFO_CH_MODE_OPT_POS) &
710                0x3) << D40_CHAN_POS(d40c->phy_chan->num);
711
712         writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
713
714         if (d40c->log_num != D40_PHY_CHAN) {
715                 /* Set default config for CFG reg */
716                 writel(d40c->src_def_cfg,
717                        d40c->base->virtbase + D40_DREG_PCBASE +
718                        d40c->phy_chan->num * D40_DREG_PCDELTA +
719                        D40_CHAN_REG_SSCFG);
720                 writel(d40c->dst_def_cfg,
721                        d40c->base->virtbase + D40_DREG_PCBASE +
722                        d40c->phy_chan->num * D40_DREG_PCDELTA +
723                        D40_CHAN_REG_SDCFG);
724
725                 /* Set LIDX for lcla */
726                 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
727                        D40_SREG_ELEM_LOG_LIDX_MASK,
728                        d40c->base->virtbase + D40_DREG_PCBASE +
729                        d40c->phy_chan->num * D40_DREG_PCDELTA +
730                        D40_CHAN_REG_SDELT);
731
732                 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
733                        D40_SREG_ELEM_LOG_LIDX_MASK,
734                        d40c->base->virtbase + D40_DREG_PCBASE +
735                        d40c->phy_chan->num * D40_DREG_PCDELTA +
736                        D40_CHAN_REG_SSELT);
737
738         }
739 }
740
741 static u32 d40_residue(struct d40_chan *d40c)
742 {
743         u32 num_elt;
744
745         if (d40c->log_num != D40_PHY_CHAN)
746                 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
747                         >> D40_MEM_LCSP2_ECNT_POS;
748         else
749                 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
750                                  d40c->phy_chan->num * D40_DREG_PCDELTA +
751                                  D40_CHAN_REG_SDELT) &
752                            D40_SREG_ELEM_PHY_ECNT_MASK) >>
753                         D40_SREG_ELEM_PHY_ECNT_POS;
754         return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
755 }
756
757 static bool d40_tx_is_linked(struct d40_chan *d40c)
758 {
759         bool is_link;
760
761         if (d40c->log_num != D40_PHY_CHAN)
762                 is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
763         else
764                 is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
765                                 d40c->phy_chan->num * D40_DREG_PCDELTA +
766                                 D40_CHAN_REG_SDLNK) &
767                         D40_SREG_LNK_PHYS_LNK_MASK;
768         return is_link;
769 }
770
771 static int d40_pause(struct dma_chan *chan)
772 {
773         struct d40_chan *d40c =
774                 container_of(chan, struct d40_chan, chan);
775         int res = 0;
776         unsigned long flags;
777
778         if (!d40c->busy)
779                 return 0;
780
781         spin_lock_irqsave(&d40c->lock, flags);
782
783         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
784         if (res == 0) {
785                 if (d40c->log_num != D40_PHY_CHAN) {
786                         d40_config_set_event(d40c, false);
787                         /* Resume the other logical channels if any */
788                         if (d40_chan_has_events(d40c))
789                                 res = d40_channel_execute_command(d40c,
790                                                                   D40_DMA_RUN);
791                 }
792         }
793
794         spin_unlock_irqrestore(&d40c->lock, flags);
795         return res;
796 }
797
798 static int d40_resume(struct dma_chan *chan)
799 {
800         struct d40_chan *d40c =
801                 container_of(chan, struct d40_chan, chan);
802         int res = 0;
803         unsigned long flags;
804
805         if (!d40c->busy)
806                 return 0;
807
808         spin_lock_irqsave(&d40c->lock, flags);
809
810         if (d40c->base->rev == 0)
811                 if (d40c->log_num != D40_PHY_CHAN) {
812                         res = d40_channel_execute_command(d40c,
813                                                           D40_DMA_SUSPEND_REQ);
814                         goto no_suspend;
815                 }
816
817         /* If bytes left to transfer or linked tx resume job */
818         if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
819
820                 if (d40c->log_num != D40_PHY_CHAN)
821                         d40_config_set_event(d40c, true);
822
823                 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
824         }
825
826 no_suspend:
827         spin_unlock_irqrestore(&d40c->lock, flags);
828         return res;
829 }
830
831 static void d40_tx_submit_log(struct d40_chan *d40c, struct d40_desc *d40d)
832 {
833         /* TODO: Write */
834 }
835
836 static void d40_tx_submit_phy(struct d40_chan *d40c, struct d40_desc *d40d)
837 {
838         struct d40_desc *d40d_prev = NULL;
839         int i;
840         u32 val;
841
842         if (!list_empty(&d40c->queue))
843                 d40d_prev = d40_last_queued(d40c);
844         else if (!list_empty(&d40c->active))
845                 d40d_prev = d40_first_active_get(d40c);
846
847         if (!d40d_prev)
848                 return;
849
850         /* Here we try to join this job with previous jobs */
851         val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
852                     d40c->phy_chan->num * D40_DREG_PCDELTA +
853                     D40_CHAN_REG_SSLNK);
854
855         /* Figure out which link we're currently transmitting */
856         for (i = 0; i < d40d_prev->lli_len; i++)
857                 if (val == d40d_prev->lli_phy.src[i].reg_lnk)
858                         break;
859
860         val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
861                     d40c->phy_chan->num * D40_DREG_PCDELTA +
862                     D40_CHAN_REG_SSELT) >> D40_SREG_ELEM_LOG_ECNT_POS;
863
864         if (i == (d40d_prev->lli_len - 1) && val > 0) {
865                 /* Change the current one */
866                 writel(virt_to_phys(d40d->lli_phy.src),
867                        d40c->base->virtbase + D40_DREG_PCBASE +
868                        d40c->phy_chan->num * D40_DREG_PCDELTA +
869                        D40_CHAN_REG_SSLNK);
870                 writel(virt_to_phys(d40d->lli_phy.dst),
871                        d40c->base->virtbase + D40_DREG_PCBASE +
872                        d40c->phy_chan->num * D40_DREG_PCDELTA +
873                        D40_CHAN_REG_SDLNK);
874
875                 d40d->is_hw_linked = true;
876
877         } else if (i < d40d_prev->lli_len) {
878                 (void) dma_unmap_single(d40c->base->dev,
879                                         virt_to_phys(d40d_prev->lli_phy.src),
880                                         d40d_prev->lli_pool.size,
881                                         DMA_TO_DEVICE);
882
883                 /* Keep the settings */
884                 val = d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk &
885                         ~D40_SREG_LNK_PHYS_LNK_MASK;
886                 d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk =
887                         val | virt_to_phys(d40d->lli_phy.src);
888
889                 val = d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk &
890                         ~D40_SREG_LNK_PHYS_LNK_MASK;
891                 d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk =
892                         val | virt_to_phys(d40d->lli_phy.dst);
893
894                 (void) dma_map_single(d40c->base->dev,
895                                       d40d_prev->lli_phy.src,
896                                       d40d_prev->lli_pool.size,
897                                       DMA_TO_DEVICE);
898                 d40d->is_hw_linked = true;
899         }
900 }
901
902 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
903 {
904         struct d40_chan *d40c = container_of(tx->chan,
905                                              struct d40_chan,
906                                              chan);
907         struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
908         unsigned long flags;
909
910         (void) d40_pause(&d40c->chan);
911
912         spin_lock_irqsave(&d40c->lock, flags);
913
914         d40c->chan.cookie++;
915
916         if (d40c->chan.cookie < 0)
917                 d40c->chan.cookie = 1;
918
919         d40d->txd.cookie = d40c->chan.cookie;
920
921         if (d40c->log_num == D40_PHY_CHAN)
922                 d40_tx_submit_phy(d40c, d40d);
923         else
924                 d40_tx_submit_log(d40c, d40d);
925
926         d40_desc_queue(d40c, d40d);
927
928         spin_unlock_irqrestore(&d40c->lock, flags);
929
930         (void) d40_resume(&d40c->chan);
931
932         return tx->cookie;
933 }
934
935 static int d40_start(struct d40_chan *d40c)
936 {
937         if (d40c->base->rev == 0) {
938                 int err;
939
940                 if (d40c->log_num != D40_PHY_CHAN) {
941                         err = d40_channel_execute_command(d40c,
942                                                           D40_DMA_SUSPEND_REQ);
943                         if (err)
944                                 return err;
945                 }
946         }
947
948         if (d40c->log_num != D40_PHY_CHAN)
949                 d40_config_set_event(d40c, true);
950
951         return d40_channel_execute_command(d40c, D40_DMA_RUN);
952 }
953
954 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
955 {
956         struct d40_desc *d40d;
957         int err;
958
959         /* Start queued jobs, if any */
960         d40d = d40_first_queued(d40c);
961
962         if (d40d != NULL) {
963                 d40c->busy = true;
964
965                 /* Remove from queue */
966                 d40_desc_remove(d40d);
967
968                 /* Add to active queue */
969                 d40_desc_submit(d40c, d40d);
970
971                 /*
972                  * If this job is already linked in hw,
973                  * do not submit it.
974                  */
975
976                 if (!d40d->is_hw_linked) {
977                         /* Initiate DMA job */
978                         d40_desc_load(d40c, d40d);
979
980                         /* Start dma job */
981                         err = d40_start(d40c);
982
983                         if (err)
984                                 return NULL;
985                 }
986         }
987
988         return d40d;
989 }
990
991 /* called from interrupt context */
992 static void dma_tc_handle(struct d40_chan *d40c)
993 {
994         struct d40_desc *d40d;
995
996         /* Get first active entry from list */
997         d40d = d40_first_active_get(d40c);
998
999         if (d40d == NULL)
1000                 return;
1001
1002         d40_lcla_free_all(d40c, d40d);
1003
1004         if (d40d->lli_current < d40d->lli_len) {
1005                 d40_desc_load(d40c, d40d);
1006                 /* Start dma job */
1007                 (void) d40_start(d40c);
1008                 return;
1009         }
1010
1011         if (d40_queue_start(d40c) == NULL)
1012                 d40c->busy = false;
1013
1014         d40c->pending_tx++;
1015         tasklet_schedule(&d40c->tasklet);
1016
1017 }
1018
1019 static void dma_tasklet(unsigned long data)
1020 {
1021         struct d40_chan *d40c = (struct d40_chan *) data;
1022         struct d40_desc *d40d;
1023         unsigned long flags;
1024         dma_async_tx_callback callback;
1025         void *callback_param;
1026
1027         spin_lock_irqsave(&d40c->lock, flags);
1028
1029         /* Get first active entry from list */
1030         d40d = d40_first_active_get(d40c);
1031
1032         if (d40d == NULL)
1033                 goto err;
1034
1035         d40c->completed = d40d->txd.cookie;
1036
1037         /*
1038          * If terminating a channel pending_tx is set to zero.
1039          * This prevents any finished active jobs to return to the client.
1040          */
1041         if (d40c->pending_tx == 0) {
1042                 spin_unlock_irqrestore(&d40c->lock, flags);
1043                 return;
1044         }
1045
1046         /* Callback to client */
1047         callback = d40d->txd.callback;
1048         callback_param = d40d->txd.callback_param;
1049
1050         if (async_tx_test_ack(&d40d->txd)) {
1051                 d40_pool_lli_free(d40d);
1052                 d40_desc_remove(d40d);
1053                 d40_desc_free(d40c, d40d);
1054         } else {
1055                 if (!d40d->is_in_client_list) {
1056                         d40_desc_remove(d40d);
1057                         d40_lcla_free_all(d40c, d40d);
1058                         list_add_tail(&d40d->node, &d40c->client);
1059                         d40d->is_in_client_list = true;
1060                 }
1061         }
1062
1063         d40c->pending_tx--;
1064
1065         if (d40c->pending_tx)
1066                 tasklet_schedule(&d40c->tasklet);
1067
1068         spin_unlock_irqrestore(&d40c->lock, flags);
1069
1070         if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1071                 callback(callback_param);
1072
1073         return;
1074
1075  err:
1076         /* Rescue manouver if receiving double interrupts */
1077         if (d40c->pending_tx > 0)
1078                 d40c->pending_tx--;
1079         spin_unlock_irqrestore(&d40c->lock, flags);
1080 }
1081
1082 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1083 {
1084         static const struct d40_interrupt_lookup il[] = {
1085                 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
1086                 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1087                 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1088                 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1089                 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
1090                 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
1091                 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
1092                 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
1093                 {D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
1094                 {D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
1095         };
1096
1097         int i;
1098         u32 regs[ARRAY_SIZE(il)];
1099         u32 idx;
1100         u32 row;
1101         long chan = -1;
1102         struct d40_chan *d40c;
1103         unsigned long flags;
1104         struct d40_base *base = data;
1105
1106         spin_lock_irqsave(&base->interrupt_lock, flags);
1107
1108         /* Read interrupt status of both logical and physical channels */
1109         for (i = 0; i < ARRAY_SIZE(il); i++)
1110                 regs[i] = readl(base->virtbase + il[i].src);
1111
1112         for (;;) {
1113
1114                 chan = find_next_bit((unsigned long *)regs,
1115                                      BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1116
1117                 /* No more set bits found? */
1118                 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1119                         break;
1120
1121                 row = chan / BITS_PER_LONG;
1122                 idx = chan & (BITS_PER_LONG - 1);
1123
1124                 /* ACK interrupt */
1125                 writel(1 << idx, base->virtbase + il[row].clr);
1126
1127                 if (il[row].offset == D40_PHY_CHAN)
1128                         d40c = base->lookup_phy_chans[idx];
1129                 else
1130                         d40c = base->lookup_log_chans[il[row].offset + idx];
1131                 spin_lock(&d40c->lock);
1132
1133                 if (!il[row].is_error)
1134                         dma_tc_handle(d40c);
1135                 else
1136                         dev_err(base->dev,
1137                                 "[%s] IRQ chan: %ld offset %d idx %d\n",
1138                                 __func__, chan, il[row].offset, idx);
1139
1140                 spin_unlock(&d40c->lock);
1141         }
1142
1143         spin_unlock_irqrestore(&base->interrupt_lock, flags);
1144
1145         return IRQ_HANDLED;
1146 }
1147
1148 static int d40_validate_conf(struct d40_chan *d40c,
1149                              struct stedma40_chan_cfg *conf)
1150 {
1151         int res = 0;
1152         u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1153         u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1154         bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1155
1156         if (!conf->dir) {
1157                 dev_err(&d40c->chan.dev->device, "[%s] Invalid direction.\n",
1158                         __func__);
1159                 res = -EINVAL;
1160         }
1161
1162         if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1163             d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1164             d40c->runtime_addr == 0) {
1165
1166                 dev_err(&d40c->chan.dev->device,
1167                         "[%s] Invalid TX channel address (%d)\n",
1168                         __func__, conf->dst_dev_type);
1169                 res = -EINVAL;
1170         }
1171
1172         if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1173             d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1174             d40c->runtime_addr == 0) {
1175                 dev_err(&d40c->chan.dev->device,
1176                         "[%s] Invalid RX channel address (%d)\n",
1177                         __func__, conf->src_dev_type);
1178                 res = -EINVAL;
1179         }
1180
1181         if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1182             dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1183                 dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
1184                         __func__);
1185                 res = -EINVAL;
1186         }
1187
1188         if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1189             src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1190                 dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
1191                         __func__);
1192                 res = -EINVAL;
1193         }
1194
1195         if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1196             dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1197                 dev_err(&d40c->chan.dev->device,
1198                         "[%s] No event line\n", __func__);
1199                 res = -EINVAL;
1200         }
1201
1202         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1203             (src_event_group != dst_event_group)) {
1204                 dev_err(&d40c->chan.dev->device,
1205                         "[%s] Invalid event group\n", __func__);
1206                 res = -EINVAL;
1207         }
1208
1209         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1210                 /*
1211                  * DMAC HW supports it. Will be added to this driver,
1212                  * in case any dma client requires it.
1213                  */
1214                 dev_err(&d40c->chan.dev->device,
1215                         "[%s] periph to periph not supported\n",
1216                         __func__);
1217                 res = -EINVAL;
1218         }
1219
1220         return res;
1221 }
1222
1223 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1224                                int log_event_line, bool is_log)
1225 {
1226         unsigned long flags;
1227         spin_lock_irqsave(&phy->lock, flags);
1228         if (!is_log) {
1229                 /* Physical interrupts are masked per physical full channel */
1230                 if (phy->allocated_src == D40_ALLOC_FREE &&
1231                     phy->allocated_dst == D40_ALLOC_FREE) {
1232                         phy->allocated_dst = D40_ALLOC_PHY;
1233                         phy->allocated_src = D40_ALLOC_PHY;
1234                         goto found;
1235                 } else
1236                         goto not_found;
1237         }
1238
1239         /* Logical channel */
1240         if (is_src) {
1241                 if (phy->allocated_src == D40_ALLOC_PHY)
1242                         goto not_found;
1243
1244                 if (phy->allocated_src == D40_ALLOC_FREE)
1245                         phy->allocated_src = D40_ALLOC_LOG_FREE;
1246
1247                 if (!(phy->allocated_src & (1 << log_event_line))) {
1248                         phy->allocated_src |= 1 << log_event_line;
1249                         goto found;
1250                 } else
1251                         goto not_found;
1252         } else {
1253                 if (phy->allocated_dst == D40_ALLOC_PHY)
1254                         goto not_found;
1255
1256                 if (phy->allocated_dst == D40_ALLOC_FREE)
1257                         phy->allocated_dst = D40_ALLOC_LOG_FREE;
1258
1259                 if (!(phy->allocated_dst & (1 << log_event_line))) {
1260                         phy->allocated_dst |= 1 << log_event_line;
1261                         goto found;
1262                 } else
1263                         goto not_found;
1264         }
1265
1266 not_found:
1267         spin_unlock_irqrestore(&phy->lock, flags);
1268         return false;
1269 found:
1270         spin_unlock_irqrestore(&phy->lock, flags);
1271         return true;
1272 }
1273
1274 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1275                                int log_event_line)
1276 {
1277         unsigned long flags;
1278         bool is_free = false;
1279
1280         spin_lock_irqsave(&phy->lock, flags);
1281         if (!log_event_line) {
1282                 phy->allocated_dst = D40_ALLOC_FREE;
1283                 phy->allocated_src = D40_ALLOC_FREE;
1284                 is_free = true;
1285                 goto out;
1286         }
1287
1288         /* Logical channel */
1289         if (is_src) {
1290                 phy->allocated_src &= ~(1 << log_event_line);
1291                 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1292                         phy->allocated_src = D40_ALLOC_FREE;
1293         } else {
1294                 phy->allocated_dst &= ~(1 << log_event_line);
1295                 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1296                         phy->allocated_dst = D40_ALLOC_FREE;
1297         }
1298
1299         is_free = ((phy->allocated_src | phy->allocated_dst) ==
1300                    D40_ALLOC_FREE);
1301
1302 out:
1303         spin_unlock_irqrestore(&phy->lock, flags);
1304
1305         return is_free;
1306 }
1307
1308 static int d40_allocate_channel(struct d40_chan *d40c)
1309 {
1310         int dev_type;
1311         int event_group;
1312         int event_line;
1313         struct d40_phy_res *phys;
1314         int i;
1315         int j;
1316         int log_num;
1317         bool is_src;
1318         bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1319
1320         phys = d40c->base->phy_res;
1321
1322         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1323                 dev_type = d40c->dma_cfg.src_dev_type;
1324                 log_num = 2 * dev_type;
1325                 is_src = true;
1326         } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1327                    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1328                 /* dst event lines are used for logical memcpy */
1329                 dev_type = d40c->dma_cfg.dst_dev_type;
1330                 log_num = 2 * dev_type + 1;
1331                 is_src = false;
1332         } else
1333                 return -EINVAL;
1334
1335         event_group = D40_TYPE_TO_GROUP(dev_type);
1336         event_line = D40_TYPE_TO_EVENT(dev_type);
1337
1338         if (!is_log) {
1339                 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1340                         /* Find physical half channel */
1341                         for (i = 0; i < d40c->base->num_phy_chans; i++) {
1342
1343                                 if (d40_alloc_mask_set(&phys[i], is_src,
1344                                                        0, is_log))
1345                                         goto found_phy;
1346                         }
1347                 } else
1348                         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1349                                 int phy_num = j  + event_group * 2;
1350                                 for (i = phy_num; i < phy_num + 2; i++) {
1351                                         if (d40_alloc_mask_set(&phys[i],
1352                                                                is_src,
1353                                                                0,
1354                                                                is_log))
1355                                                 goto found_phy;
1356                                 }
1357                         }
1358                 return -EINVAL;
1359 found_phy:
1360                 d40c->phy_chan = &phys[i];
1361                 d40c->log_num = D40_PHY_CHAN;
1362                 goto out;
1363         }
1364         if (dev_type == -1)
1365                 return -EINVAL;
1366
1367         /* Find logical channel */
1368         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1369                 int phy_num = j + event_group * 2;
1370                 /*
1371                  * Spread logical channels across all available physical rather
1372                  * than pack every logical channel at the first available phy
1373                  * channels.
1374                  */
1375                 if (is_src) {
1376                         for (i = phy_num; i < phy_num + 2; i++) {
1377                                 if (d40_alloc_mask_set(&phys[i], is_src,
1378                                                        event_line, is_log))
1379                                         goto found_log;
1380                         }
1381                 } else {
1382                         for (i = phy_num + 1; i >= phy_num; i--) {
1383                                 if (d40_alloc_mask_set(&phys[i], is_src,
1384                                                        event_line, is_log))
1385                                         goto found_log;
1386                         }
1387                 }
1388         }
1389         return -EINVAL;
1390
1391 found_log:
1392         d40c->phy_chan = &phys[i];
1393         d40c->log_num = log_num;
1394 out:
1395
1396         if (is_log)
1397                 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1398         else
1399                 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1400
1401         return 0;
1402
1403 }
1404
1405 static int d40_config_memcpy(struct d40_chan *d40c)
1406 {
1407         dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1408
1409         if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1410                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1411                 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1412                 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1413                         memcpy[d40c->chan.chan_id];
1414
1415         } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1416                    dma_has_cap(DMA_SLAVE, cap)) {
1417                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1418         } else {
1419                 dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
1420                         __func__);
1421                 return -EINVAL;
1422         }
1423
1424         return 0;
1425 }
1426
1427
1428 static int d40_free_dma(struct d40_chan *d40c)
1429 {
1430
1431         int res = 0;
1432         u32 event;
1433         struct d40_phy_res *phy = d40c->phy_chan;
1434         bool is_src;
1435         struct d40_desc *d;
1436         struct d40_desc *_d;
1437
1438
1439         /* Terminate all queued and active transfers */
1440         d40_term_all(d40c);
1441
1442         /* Release client owned descriptors */
1443         if (!list_empty(&d40c->client))
1444                 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1445                         d40_pool_lli_free(d);
1446                         d40_desc_remove(d);
1447                         d40_desc_free(d40c, d);
1448                 }
1449
1450         if (phy == NULL) {
1451                 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
1452                         __func__);
1453                 return -EINVAL;
1454         }
1455
1456         if (phy->allocated_src == D40_ALLOC_FREE &&
1457             phy->allocated_dst == D40_ALLOC_FREE) {
1458                 dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
1459                         __func__);
1460                 return -EINVAL;
1461         }
1462
1463         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1464             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1465                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1466                 is_src = false;
1467         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1468                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1469                 is_src = true;
1470         } else {
1471                 dev_err(&d40c->chan.dev->device,
1472                         "[%s] Unknown direction\n", __func__);
1473                 return -EINVAL;
1474         }
1475
1476         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1477         if (res) {
1478                 dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n",
1479                         __func__);
1480                 return res;
1481         }
1482
1483         if (d40c->log_num != D40_PHY_CHAN) {
1484                 /* Release logical channel, deactivate the event line */
1485
1486                 d40_config_set_event(d40c, false);
1487                 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1488
1489                 /*
1490                  * Check if there are more logical allocation
1491                  * on this phy channel.
1492                  */
1493                 if (!d40_alloc_mask_free(phy, is_src, event)) {
1494                         /* Resume the other logical channels if any */
1495                         if (d40_chan_has_events(d40c)) {
1496                                 res = d40_channel_execute_command(d40c,
1497                                                                   D40_DMA_RUN);
1498                                 if (res) {
1499                                         dev_err(&d40c->chan.dev->device,
1500                                                 "[%s] Executing RUN command\n",
1501                                                 __func__);
1502                                         return res;
1503                                 }
1504                         }
1505                         return 0;
1506                 }
1507         } else {
1508                 (void) d40_alloc_mask_free(phy, is_src, 0);
1509         }
1510
1511         /* Release physical channel */
1512         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1513         if (res) {
1514                 dev_err(&d40c->chan.dev->device,
1515                         "[%s] Failed to stop channel\n", __func__);
1516                 return res;
1517         }
1518         d40c->phy_chan = NULL;
1519         d40c->configured = false;
1520         d40c->base->lookup_phy_chans[phy->num] = NULL;
1521
1522         return 0;
1523 }
1524
1525 static bool d40_is_paused(struct d40_chan *d40c)
1526 {
1527         bool is_paused = false;
1528         unsigned long flags;
1529         void __iomem *active_reg;
1530         u32 status;
1531         u32 event;
1532
1533         spin_lock_irqsave(&d40c->lock, flags);
1534
1535         if (d40c->log_num == D40_PHY_CHAN) {
1536                 if (d40c->phy_chan->num % 2 == 0)
1537                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1538                 else
1539                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1540
1541                 status = (readl(active_reg) &
1542                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1543                         D40_CHAN_POS(d40c->phy_chan->num);
1544                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1545                         is_paused = true;
1546
1547                 goto _exit;
1548         }
1549
1550         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1551             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1552                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1553                 status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1554                                d40c->phy_chan->num * D40_DREG_PCDELTA +
1555                                D40_CHAN_REG_SDLNK);
1556         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1557                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1558                 status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1559                                d40c->phy_chan->num * D40_DREG_PCDELTA +
1560                                D40_CHAN_REG_SSLNK);
1561         } else {
1562                 dev_err(&d40c->chan.dev->device,
1563                         "[%s] Unknown direction\n", __func__);
1564                 goto _exit;
1565         }
1566
1567         status = (status & D40_EVENTLINE_MASK(event)) >>
1568                 D40_EVENTLINE_POS(event);
1569
1570         if (status != D40_DMA_RUN)
1571                 is_paused = true;
1572 _exit:
1573         spin_unlock_irqrestore(&d40c->lock, flags);
1574         return is_paused;
1575
1576 }
1577
1578
1579 static u32 stedma40_residue(struct dma_chan *chan)
1580 {
1581         struct d40_chan *d40c =
1582                 container_of(chan, struct d40_chan, chan);
1583         u32 bytes_left;
1584         unsigned long flags;
1585
1586         spin_lock_irqsave(&d40c->lock, flags);
1587         bytes_left = d40_residue(d40c);
1588         spin_unlock_irqrestore(&d40c->lock, flags);
1589
1590         return bytes_left;
1591 }
1592
1593 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1594                                                    struct scatterlist *sgl_dst,
1595                                                    struct scatterlist *sgl_src,
1596                                                    unsigned int sgl_len,
1597                                                    unsigned long dma_flags)
1598 {
1599         int res;
1600         struct d40_desc *d40d;
1601         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1602                                              chan);
1603         unsigned long flags;
1604
1605         if (d40c->phy_chan == NULL) {
1606                 dev_err(&d40c->chan.dev->device,
1607                         "[%s] Unallocated channel.\n", __func__);
1608                 return ERR_PTR(-EINVAL);
1609         }
1610
1611         spin_lock_irqsave(&d40c->lock, flags);
1612         d40d = d40_desc_get(d40c);
1613
1614         if (d40d == NULL)
1615                 goto err;
1616
1617         d40d->lli_len = sgl_len;
1618         d40d->lli_current = 0;
1619         d40d->txd.flags = dma_flags;
1620
1621         if (d40c->log_num != D40_PHY_CHAN) {
1622
1623                 if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) {
1624                         dev_err(&d40c->chan.dev->device,
1625                                 "[%s] Out of memory\n", __func__);
1626                         goto err;
1627                 }
1628
1629                 (void) d40_log_sg_to_lli(sgl_src,
1630                                          sgl_len,
1631                                          d40d->lli_log.src,
1632                                          d40c->log_def.lcsp1,
1633                                          d40c->dma_cfg.src_info.data_width);
1634
1635                 (void) d40_log_sg_to_lli(sgl_dst,
1636                                          sgl_len,
1637                                          d40d->lli_log.dst,
1638                                          d40c->log_def.lcsp3,
1639                                          d40c->dma_cfg.dst_info.data_width);
1640         } else {
1641                 if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
1642                         dev_err(&d40c->chan.dev->device,
1643                                 "[%s] Out of memory\n", __func__);
1644                         goto err;
1645                 }
1646
1647                 res = d40_phy_sg_to_lli(sgl_src,
1648                                         sgl_len,
1649                                         0,
1650                                         d40d->lli_phy.src,
1651                                         virt_to_phys(d40d->lli_phy.src),
1652                                         d40c->src_def_cfg,
1653                                         d40c->dma_cfg.src_info.data_width,
1654                                         d40c->dma_cfg.src_info.psize);
1655
1656                 if (res < 0)
1657                         goto err;
1658
1659                 res = d40_phy_sg_to_lli(sgl_dst,
1660                                         sgl_len,
1661                                         0,
1662                                         d40d->lli_phy.dst,
1663                                         virt_to_phys(d40d->lli_phy.dst),
1664                                         d40c->dst_def_cfg,
1665                                         d40c->dma_cfg.dst_info.data_width,
1666                                         d40c->dma_cfg.dst_info.psize);
1667
1668                 if (res < 0)
1669                         goto err;
1670
1671                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1672                                       d40d->lli_pool.size, DMA_TO_DEVICE);
1673         }
1674
1675         dma_async_tx_descriptor_init(&d40d->txd, chan);
1676
1677         d40d->txd.tx_submit = d40_tx_submit;
1678
1679         spin_unlock_irqrestore(&d40c->lock, flags);
1680
1681         return &d40d->txd;
1682 err:
1683         if (d40d)
1684                 d40_desc_free(d40c, d40d);
1685         spin_unlock_irqrestore(&d40c->lock, flags);
1686         return NULL;
1687 }
1688 EXPORT_SYMBOL(stedma40_memcpy_sg);
1689
1690 bool stedma40_filter(struct dma_chan *chan, void *data)
1691 {
1692         struct stedma40_chan_cfg *info = data;
1693         struct d40_chan *d40c =
1694                 container_of(chan, struct d40_chan, chan);
1695         int err;
1696
1697         if (data) {
1698                 err = d40_validate_conf(d40c, info);
1699                 if (!err)
1700                         d40c->dma_cfg = *info;
1701         } else
1702                 err = d40_config_memcpy(d40c);
1703
1704         if (!err)
1705                 d40c->configured = true;
1706
1707         return err == 0;
1708 }
1709 EXPORT_SYMBOL(stedma40_filter);
1710
1711 /* DMA ENGINE functions */
1712 static int d40_alloc_chan_resources(struct dma_chan *chan)
1713 {
1714         int err;
1715         unsigned long flags;
1716         struct d40_chan *d40c =
1717                 container_of(chan, struct d40_chan, chan);
1718         bool is_free_phy;
1719         spin_lock_irqsave(&d40c->lock, flags);
1720
1721         d40c->completed = chan->cookie = 1;
1722
1723         /* If no dma configuration is set use default configuration (memcpy) */
1724         if (!d40c->configured) {
1725                 err = d40_config_memcpy(d40c);
1726                 if (err) {
1727                         dev_err(&d40c->chan.dev->device,
1728                                 "[%s] Failed to configure memcpy channel\n",
1729                                 __func__);
1730                         goto fail;
1731                 }
1732         }
1733         is_free_phy = (d40c->phy_chan == NULL);
1734
1735         err = d40_allocate_channel(d40c);
1736         if (err) {
1737                 dev_err(&d40c->chan.dev->device,
1738                         "[%s] Failed to allocate channel\n", __func__);
1739                 goto fail;
1740         }
1741
1742         /* Fill in basic CFG register values */
1743         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1744                     &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1745
1746         if (d40c->log_num != D40_PHY_CHAN) {
1747                 d40_log_cfg(&d40c->dma_cfg,
1748                             &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1749
1750                 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1751                         d40c->lcpa = d40c->base->lcpa_base +
1752                           d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1753                 else
1754                         d40c->lcpa = d40c->base->lcpa_base +
1755                           d40c->dma_cfg.dst_dev_type *
1756                           D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1757         }
1758
1759         /*
1760          * Only write channel configuration to the DMA if the physical
1761          * resource is free. In case of multiple logical channels
1762          * on the same physical resource, only the first write is necessary.
1763          */
1764         if (is_free_phy)
1765                 d40_config_write(d40c);
1766 fail:
1767         spin_unlock_irqrestore(&d40c->lock, flags);
1768         return err;
1769 }
1770
1771 static void d40_free_chan_resources(struct dma_chan *chan)
1772 {
1773         struct d40_chan *d40c =
1774                 container_of(chan, struct d40_chan, chan);
1775         int err;
1776         unsigned long flags;
1777
1778         if (d40c->phy_chan == NULL) {
1779                 dev_err(&d40c->chan.dev->device,
1780                         "[%s] Cannot free unallocated channel\n", __func__);
1781                 return;
1782         }
1783
1784
1785         spin_lock_irqsave(&d40c->lock, flags);
1786
1787         err = d40_free_dma(d40c);
1788
1789         if (err)
1790                 dev_err(&d40c->chan.dev->device,
1791                         "[%s] Failed to free channel\n", __func__);
1792         spin_unlock_irqrestore(&d40c->lock, flags);
1793 }
1794
1795 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1796                                                        dma_addr_t dst,
1797                                                        dma_addr_t src,
1798                                                        size_t size,
1799                                                        unsigned long dma_flags)
1800 {
1801         struct d40_desc *d40d;
1802         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1803                                              chan);
1804         unsigned long flags;
1805         int err = 0;
1806
1807         if (d40c->phy_chan == NULL) {
1808                 dev_err(&d40c->chan.dev->device,
1809                         "[%s] Channel is not allocated.\n", __func__);
1810                 return ERR_PTR(-EINVAL);
1811         }
1812
1813         spin_lock_irqsave(&d40c->lock, flags);
1814         d40d = d40_desc_get(d40c);
1815
1816         if (d40d == NULL) {
1817                 dev_err(&d40c->chan.dev->device,
1818                         "[%s] Descriptor is NULL\n", __func__);
1819                 goto err;
1820         }
1821
1822         d40d->txd.flags = dma_flags;
1823
1824         dma_async_tx_descriptor_init(&d40d->txd, chan);
1825
1826         d40d->txd.tx_submit = d40_tx_submit;
1827
1828         if (d40c->log_num != D40_PHY_CHAN) {
1829
1830                 if (d40_pool_lli_alloc(d40d, 1, true) < 0) {
1831                         dev_err(&d40c->chan.dev->device,
1832                                 "[%s] Out of memory\n", __func__);
1833                         goto err;
1834                 }
1835                 d40d->lli_len = 1;
1836                 d40d->lli_current = 0;
1837
1838                 d40_log_fill_lli(d40d->lli_log.src,
1839                                  src,
1840                                  size,
1841                                  d40c->log_def.lcsp1,
1842                                  d40c->dma_cfg.src_info.data_width,
1843                                  true);
1844
1845                 d40_log_fill_lli(d40d->lli_log.dst,
1846                                  dst,
1847                                  size,
1848                                  d40c->log_def.lcsp3,
1849                                  d40c->dma_cfg.dst_info.data_width,
1850                                  true);
1851
1852         } else {
1853
1854                 if (d40_pool_lli_alloc(d40d, 1, false) < 0) {
1855                         dev_err(&d40c->chan.dev->device,
1856                                 "[%s] Out of memory\n", __func__);
1857                         goto err;
1858                 }
1859
1860                 err = d40_phy_fill_lli(d40d->lli_phy.src,
1861                                        src,
1862                                        size,
1863                                        d40c->dma_cfg.src_info.psize,
1864                                        0,
1865                                        d40c->src_def_cfg,
1866                                        true,
1867                                        d40c->dma_cfg.src_info.data_width,
1868                                        false);
1869                 if (err)
1870                         goto err_fill_lli;
1871
1872                 err = d40_phy_fill_lli(d40d->lli_phy.dst,
1873                                        dst,
1874                                        size,
1875                                        d40c->dma_cfg.dst_info.psize,
1876                                        0,
1877                                        d40c->dst_def_cfg,
1878                                        true,
1879                                        d40c->dma_cfg.dst_info.data_width,
1880                                        false);
1881
1882                 if (err)
1883                         goto err_fill_lli;
1884
1885                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1886                                       d40d->lli_pool.size, DMA_TO_DEVICE);
1887         }
1888
1889         spin_unlock_irqrestore(&d40c->lock, flags);
1890         return &d40d->txd;
1891
1892 err_fill_lli:
1893         dev_err(&d40c->chan.dev->device,
1894                 "[%s] Failed filling in PHY LLI\n", __func__);
1895 err:
1896         if (d40d)
1897                 d40_desc_free(d40c, d40d);
1898         spin_unlock_irqrestore(&d40c->lock, flags);
1899         return NULL;
1900 }
1901
1902 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1903                                  struct d40_chan *d40c,
1904                                  struct scatterlist *sgl,
1905                                  unsigned int sg_len,
1906                                  enum dma_data_direction direction,
1907                                  unsigned long dma_flags)
1908 {
1909         dma_addr_t dev_addr = 0;
1910         int total_size;
1911
1912         if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) {
1913                 dev_err(&d40c->chan.dev->device,
1914                         "[%s] Out of memory\n", __func__);
1915                 return -ENOMEM;
1916         }
1917
1918         d40d->lli_len = sg_len;
1919         d40d->lli_current = 0;
1920
1921         if (direction == DMA_FROM_DEVICE)
1922                 if (d40c->runtime_addr)
1923                         dev_addr = d40c->runtime_addr;
1924                 else
1925                         dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1926         else if (direction == DMA_TO_DEVICE)
1927                 if (d40c->runtime_addr)
1928                         dev_addr = d40c->runtime_addr;
1929                 else
1930                         dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1931
1932         else
1933                 return -EINVAL;
1934
1935         total_size = d40_log_sg_to_dev(sgl, sg_len,
1936                                        &d40d->lli_log,
1937                                        &d40c->log_def,
1938                                        d40c->dma_cfg.src_info.data_width,
1939                                        d40c->dma_cfg.dst_info.data_width,
1940                                        direction,
1941                                        dev_addr);
1942
1943         if (total_size < 0)
1944                 return -EINVAL;
1945
1946         return 0;
1947 }
1948
1949 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1950                                  struct d40_chan *d40c,
1951                                  struct scatterlist *sgl,
1952                                  unsigned int sgl_len,
1953                                  enum dma_data_direction direction,
1954                                  unsigned long dma_flags)
1955 {
1956         dma_addr_t src_dev_addr;
1957         dma_addr_t dst_dev_addr;
1958         int res;
1959
1960         if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
1961                 dev_err(&d40c->chan.dev->device,
1962                         "[%s] Out of memory\n", __func__);
1963                 return -ENOMEM;
1964         }
1965
1966         d40d->lli_len = sgl_len;
1967         d40d->lli_current = 0;
1968
1969         if (direction == DMA_FROM_DEVICE) {
1970                 dst_dev_addr = 0;
1971                 if (d40c->runtime_addr)
1972                         src_dev_addr = d40c->runtime_addr;
1973                 else
1974                         src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1975         } else if (direction == DMA_TO_DEVICE) {
1976                 if (d40c->runtime_addr)
1977                         dst_dev_addr = d40c->runtime_addr;
1978                 else
1979                         dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1980                 src_dev_addr = 0;
1981         } else
1982                 return -EINVAL;
1983
1984         res = d40_phy_sg_to_lli(sgl,
1985                                 sgl_len,
1986                                 src_dev_addr,
1987                                 d40d->lli_phy.src,
1988                                 virt_to_phys(d40d->lli_phy.src),
1989                                 d40c->src_def_cfg,
1990                                 d40c->dma_cfg.src_info.data_width,
1991                                 d40c->dma_cfg.src_info.psize);
1992         if (res < 0)
1993                 return res;
1994
1995         res = d40_phy_sg_to_lli(sgl,
1996                                 sgl_len,
1997                                 dst_dev_addr,
1998                                 d40d->lli_phy.dst,
1999                                 virt_to_phys(d40d->lli_phy.dst),
2000                                 d40c->dst_def_cfg,
2001                                 d40c->dma_cfg.dst_info.data_width,
2002                                 d40c->dma_cfg.dst_info.psize);
2003         if (res < 0)
2004                 return res;
2005
2006         (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2007                               d40d->lli_pool.size, DMA_TO_DEVICE);
2008         return 0;
2009 }
2010
2011 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2012                                                          struct scatterlist *sgl,
2013                                                          unsigned int sg_len,
2014                                                          enum dma_data_direction direction,
2015                                                          unsigned long dma_flags)
2016 {
2017         struct d40_desc *d40d;
2018         struct d40_chan *d40c = container_of(chan, struct d40_chan,
2019                                              chan);
2020         unsigned long flags;
2021         int err;
2022
2023         if (d40c->phy_chan == NULL) {
2024                 dev_err(&d40c->chan.dev->device,
2025                         "[%s] Cannot prepare unallocated channel\n", __func__);
2026                 return ERR_PTR(-EINVAL);
2027         }
2028
2029         spin_lock_irqsave(&d40c->lock, flags);
2030         d40d = d40_desc_get(d40c);
2031
2032         if (d40d == NULL)
2033                 goto err;
2034
2035         if (d40c->log_num != D40_PHY_CHAN)
2036                 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2037                                             direction, dma_flags);
2038         else
2039                 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2040                                             direction, dma_flags);
2041         if (err) {
2042                 dev_err(&d40c->chan.dev->device,
2043                         "[%s] Failed to prepare %s slave sg job: %d\n",
2044                         __func__,
2045                         d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
2046                 goto err;
2047         }
2048
2049         d40d->txd.flags = dma_flags;
2050
2051         dma_async_tx_descriptor_init(&d40d->txd, chan);
2052
2053         d40d->txd.tx_submit = d40_tx_submit;
2054
2055         spin_unlock_irqrestore(&d40c->lock, flags);
2056         return &d40d->txd;
2057
2058 err:
2059         if (d40d)
2060                 d40_desc_free(d40c, d40d);
2061         spin_unlock_irqrestore(&d40c->lock, flags);
2062         return NULL;
2063 }
2064
2065 static enum dma_status d40_tx_status(struct dma_chan *chan,
2066                                      dma_cookie_t cookie,
2067                                      struct dma_tx_state *txstate)
2068 {
2069         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2070         dma_cookie_t last_used;
2071         dma_cookie_t last_complete;
2072         int ret;
2073
2074         if (d40c->phy_chan == NULL) {
2075                 dev_err(&d40c->chan.dev->device,
2076                         "[%s] Cannot read status of unallocated channel\n",
2077                         __func__);
2078                 return -EINVAL;
2079         }
2080
2081         last_complete = d40c->completed;
2082         last_used = chan->cookie;
2083
2084         if (d40_is_paused(d40c))
2085                 ret = DMA_PAUSED;
2086         else
2087                 ret = dma_async_is_complete(cookie, last_complete, last_used);
2088
2089         dma_set_tx_state(txstate, last_complete, last_used,
2090                          stedma40_residue(chan));
2091
2092         return ret;
2093 }
2094
2095 static void d40_issue_pending(struct dma_chan *chan)
2096 {
2097         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2098         unsigned long flags;
2099
2100         if (d40c->phy_chan == NULL) {
2101                 dev_err(&d40c->chan.dev->device,
2102                         "[%s] Channel is not allocated!\n", __func__);
2103                 return;
2104         }
2105
2106         spin_lock_irqsave(&d40c->lock, flags);
2107
2108         /* Busy means that pending jobs are already being processed */
2109         if (!d40c->busy)
2110                 (void) d40_queue_start(d40c);
2111
2112         spin_unlock_irqrestore(&d40c->lock, flags);
2113 }
2114
2115 /* Runtime reconfiguration extension */
2116 static void d40_set_runtime_config(struct dma_chan *chan,
2117                                struct dma_slave_config *config)
2118 {
2119         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2120         struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2121         enum dma_slave_buswidth config_addr_width;
2122         dma_addr_t config_addr;
2123         u32 config_maxburst;
2124         enum stedma40_periph_data_width addr_width;
2125         int psize;
2126
2127         if (config->direction == DMA_FROM_DEVICE) {
2128                 dma_addr_t dev_addr_rx =
2129                         d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2130
2131                 config_addr = config->src_addr;
2132                 if (dev_addr_rx)
2133                         dev_dbg(d40c->base->dev,
2134                                 "channel has a pre-wired RX address %08x "
2135                                 "overriding with %08x\n",
2136                                 dev_addr_rx, config_addr);
2137                 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2138                         dev_dbg(d40c->base->dev,
2139                                 "channel was not configured for peripheral "
2140                                 "to memory transfer (%d) overriding\n",
2141                                 cfg->dir);
2142                 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2143
2144                 config_addr_width = config->src_addr_width;
2145                 config_maxburst = config->src_maxburst;
2146
2147         } else if (config->direction == DMA_TO_DEVICE) {
2148                 dma_addr_t dev_addr_tx =
2149                         d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2150
2151                 config_addr = config->dst_addr;
2152                 if (dev_addr_tx)
2153                         dev_dbg(d40c->base->dev,
2154                                 "channel has a pre-wired TX address %08x "
2155                                 "overriding with %08x\n",
2156                                 dev_addr_tx, config_addr);
2157                 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2158                         dev_dbg(d40c->base->dev,
2159                                 "channel was not configured for memory "
2160                                 "to peripheral transfer (%d) overriding\n",
2161                                 cfg->dir);
2162                 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2163
2164                 config_addr_width = config->dst_addr_width;
2165                 config_maxburst = config->dst_maxburst;
2166
2167         } else {
2168                 dev_err(d40c->base->dev,
2169                         "unrecognized channel direction %d\n",
2170                         config->direction);
2171                 return;
2172         }
2173
2174         switch (config_addr_width) {
2175         case DMA_SLAVE_BUSWIDTH_1_BYTE:
2176                 addr_width = STEDMA40_BYTE_WIDTH;
2177                 break;
2178         case DMA_SLAVE_BUSWIDTH_2_BYTES:
2179                 addr_width = STEDMA40_HALFWORD_WIDTH;
2180                 break;
2181         case DMA_SLAVE_BUSWIDTH_4_BYTES:
2182                 addr_width = STEDMA40_WORD_WIDTH;
2183                 break;
2184         case DMA_SLAVE_BUSWIDTH_8_BYTES:
2185                 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2186                 break;
2187         default:
2188                 dev_err(d40c->base->dev,
2189                         "illegal peripheral address width "
2190                         "requested (%d)\n",
2191                         config->src_addr_width);
2192                 return;
2193         }
2194
2195         if (d40c->log_num != D40_PHY_CHAN) {
2196                 if (config_maxburst >= 16)
2197                         psize = STEDMA40_PSIZE_LOG_16;
2198                 else if (config_maxburst >= 8)
2199                         psize = STEDMA40_PSIZE_LOG_8;
2200                 else if (config_maxburst >= 4)
2201                         psize = STEDMA40_PSIZE_LOG_4;
2202                 else
2203                         psize = STEDMA40_PSIZE_LOG_1;
2204         } else {
2205                 if (config_maxburst >= 16)
2206                         psize = STEDMA40_PSIZE_PHY_16;
2207                 else if (config_maxburst >= 8)
2208                         psize = STEDMA40_PSIZE_PHY_8;
2209                 else if (config_maxburst >= 4)
2210                         psize = STEDMA40_PSIZE_PHY_4;
2211                 else
2212                         psize = STEDMA40_PSIZE_PHY_1;
2213         }
2214
2215         /* Set up all the endpoint configs */
2216         cfg->src_info.data_width = addr_width;
2217         cfg->src_info.psize = psize;
2218         cfg->src_info.endianess = STEDMA40_LITTLE_ENDIAN;
2219         cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2220         cfg->dst_info.data_width = addr_width;
2221         cfg->dst_info.psize = psize;
2222         cfg->dst_info.endianess = STEDMA40_LITTLE_ENDIAN;
2223         cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2224
2225         /* Fill in register values */
2226         if (d40c->log_num != D40_PHY_CHAN)
2227                 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2228         else
2229                 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2230                             &d40c->dst_def_cfg, false);
2231
2232         /* These settings will take precedence later */
2233         d40c->runtime_addr = config_addr;
2234         d40c->runtime_direction = config->direction;
2235         dev_dbg(d40c->base->dev,
2236                 "configured channel %s for %s, data width %d, "
2237                 "maxburst %d bytes, LE, no flow control\n",
2238                 dma_chan_name(chan),
2239                 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2240                 config_addr_width,
2241                 config_maxburst);
2242 }
2243
2244 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2245                        unsigned long arg)
2246 {
2247         unsigned long flags;
2248         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2249
2250         if (d40c->phy_chan == NULL) {
2251                 dev_err(&d40c->chan.dev->device,
2252                         "[%s] Channel is not allocated!\n", __func__);
2253                 return -EINVAL;
2254         }
2255
2256         switch (cmd) {
2257         case DMA_TERMINATE_ALL:
2258                 spin_lock_irqsave(&d40c->lock, flags);
2259                 d40_term_all(d40c);
2260                 spin_unlock_irqrestore(&d40c->lock, flags);
2261                 return 0;
2262         case DMA_PAUSE:
2263                 return d40_pause(chan);
2264         case DMA_RESUME:
2265                 return d40_resume(chan);
2266         case DMA_SLAVE_CONFIG:
2267                 d40_set_runtime_config(chan,
2268                         (struct dma_slave_config *) arg);
2269                 return 0;
2270         default:
2271                 break;
2272         }
2273
2274         /* Other commands are unimplemented */
2275         return -ENXIO;
2276 }
2277
2278 /* Initialization functions */
2279
2280 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2281                                  struct d40_chan *chans, int offset,
2282                                  int num_chans)
2283 {
2284         int i = 0;
2285         struct d40_chan *d40c;
2286
2287         INIT_LIST_HEAD(&dma->channels);
2288
2289         for (i = offset; i < offset + num_chans; i++) {
2290                 d40c = &chans[i];
2291                 d40c->base = base;
2292                 d40c->chan.device = dma;
2293
2294                 spin_lock_init(&d40c->lock);
2295
2296                 d40c->log_num = D40_PHY_CHAN;
2297
2298                 INIT_LIST_HEAD(&d40c->active);
2299                 INIT_LIST_HEAD(&d40c->queue);
2300                 INIT_LIST_HEAD(&d40c->client);
2301
2302                 tasklet_init(&d40c->tasklet, dma_tasklet,
2303                              (unsigned long) d40c);
2304
2305                 list_add_tail(&d40c->chan.device_node,
2306                               &dma->channels);
2307         }
2308 }
2309
2310 static int __init d40_dmaengine_init(struct d40_base *base,
2311                                      int num_reserved_chans)
2312 {
2313         int err ;
2314
2315         d40_chan_init(base, &base->dma_slave, base->log_chans,
2316                       0, base->num_log_chans);
2317
2318         dma_cap_zero(base->dma_slave.cap_mask);
2319         dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2320
2321         base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2322         base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2323         base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2324         base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2325         base->dma_slave.device_tx_status = d40_tx_status;
2326         base->dma_slave.device_issue_pending = d40_issue_pending;
2327         base->dma_slave.device_control = d40_control;
2328         base->dma_slave.dev = base->dev;
2329
2330         err = dma_async_device_register(&base->dma_slave);
2331
2332         if (err) {
2333                 dev_err(base->dev,
2334                         "[%s] Failed to register slave channels\n",
2335                         __func__);
2336                 goto failure1;
2337         }
2338
2339         d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2340                       base->num_log_chans, base->plat_data->memcpy_len);
2341
2342         dma_cap_zero(base->dma_memcpy.cap_mask);
2343         dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2344
2345         base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2346         base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2347         base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2348         base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2349         base->dma_memcpy.device_tx_status = d40_tx_status;
2350         base->dma_memcpy.device_issue_pending = d40_issue_pending;
2351         base->dma_memcpy.device_control = d40_control;
2352         base->dma_memcpy.dev = base->dev;
2353         /*
2354          * This controller can only access address at even
2355          * 32bit boundaries, i.e. 2^2
2356          */
2357         base->dma_memcpy.copy_align = 2;
2358
2359         err = dma_async_device_register(&base->dma_memcpy);
2360
2361         if (err) {
2362                 dev_err(base->dev,
2363                         "[%s] Failed to regsiter memcpy only channels\n",
2364                         __func__);
2365                 goto failure2;
2366         }
2367
2368         d40_chan_init(base, &base->dma_both, base->phy_chans,
2369                       0, num_reserved_chans);
2370
2371         dma_cap_zero(base->dma_both.cap_mask);
2372         dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2373         dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2374
2375         base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2376         base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2377         base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2378         base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2379         base->dma_both.device_tx_status = d40_tx_status;
2380         base->dma_both.device_issue_pending = d40_issue_pending;
2381         base->dma_both.device_control = d40_control;
2382         base->dma_both.dev = base->dev;
2383         base->dma_both.copy_align = 2;
2384         err = dma_async_device_register(&base->dma_both);
2385
2386         if (err) {
2387                 dev_err(base->dev,
2388                         "[%s] Failed to register logical and physical capable channels\n",
2389                         __func__);
2390                 goto failure3;
2391         }
2392         return 0;
2393 failure3:
2394         dma_async_device_unregister(&base->dma_memcpy);
2395 failure2:
2396         dma_async_device_unregister(&base->dma_slave);
2397 failure1:
2398         return err;
2399 }
2400
2401 /* Initialization functions. */
2402
2403 static int __init d40_phy_res_init(struct d40_base *base)
2404 {
2405         int i;
2406         int num_phy_chans_avail = 0;
2407         u32 val[2];
2408         int odd_even_bit = -2;
2409
2410         val[0] = readl(base->virtbase + D40_DREG_PRSME);
2411         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2412
2413         for (i = 0; i < base->num_phy_chans; i++) {
2414                 base->phy_res[i].num = i;
2415                 odd_even_bit += 2 * ((i % 2) == 0);
2416                 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2417                         /* Mark security only channels as occupied */
2418                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2419                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2420                 } else {
2421                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2422                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2423                         num_phy_chans_avail++;
2424                 }
2425                 spin_lock_init(&base->phy_res[i].lock);
2426         }
2427
2428         /* Mark disabled channels as occupied */
2429         for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2430                 int chan = base->plat_data->disabled_channels[i];
2431
2432                 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2433                 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2434                 num_phy_chans_avail--;
2435         }
2436
2437         dev_info(base->dev, "%d of %d physical DMA channels available\n",
2438                  num_phy_chans_avail, base->num_phy_chans);
2439
2440         /* Verify settings extended vs standard */
2441         val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2442
2443         for (i = 0; i < base->num_phy_chans; i++) {
2444
2445                 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2446                     (val[0] & 0x3) != 1)
2447                         dev_info(base->dev,
2448                                  "[%s] INFO: channel %d is misconfigured (%d)\n",
2449                                  __func__, i, val[0] & 0x3);
2450
2451                 val[0] = val[0] >> 2;
2452         }
2453
2454         return num_phy_chans_avail;
2455 }
2456
2457 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2458 {
2459         static const struct d40_reg_val dma_id_regs[] = {
2460                 /* Peripheral Id */
2461                 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2462                 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2463                 /*
2464                  * D40_DREG_PERIPHID2 Depends on HW revision:
2465                  *  MOP500/HREF ED has 0x0008,
2466                  *  ? has 0x0018,
2467                  *  HREF V1 has 0x0028
2468                  */
2469                 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2470
2471                 /* PCell Id */
2472                 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2473                 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2474                 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2475                 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2476         };
2477         struct stedma40_platform_data *plat_data;
2478         struct clk *clk = NULL;
2479         void __iomem *virtbase = NULL;
2480         struct resource *res = NULL;
2481         struct d40_base *base = NULL;
2482         int num_log_chans = 0;
2483         int num_phy_chans;
2484         int i;
2485         u32 val;
2486         u32 rev;
2487
2488         clk = clk_get(&pdev->dev, NULL);
2489
2490         if (IS_ERR(clk)) {
2491                 dev_err(&pdev->dev, "[%s] No matching clock found\n",
2492                         __func__);
2493                 goto failure;
2494         }
2495
2496         clk_enable(clk);
2497
2498         /* Get IO for DMAC base address */
2499         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2500         if (!res)
2501                 goto failure;
2502
2503         if (request_mem_region(res->start, resource_size(res),
2504                                D40_NAME " I/O base") == NULL)
2505                 goto failure;
2506
2507         virtbase = ioremap(res->start, resource_size(res));
2508         if (!virtbase)
2509                 goto failure;
2510
2511         /* HW version check */
2512         for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2513                 if (dma_id_regs[i].val !=
2514                     readl(virtbase + dma_id_regs[i].reg)) {
2515                         dev_err(&pdev->dev,
2516                                 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2517                                 __func__,
2518                                 dma_id_regs[i].val,
2519                                 dma_id_regs[i].reg,
2520                                 readl(virtbase + dma_id_regs[i].reg));
2521                         goto failure;
2522                 }
2523         }
2524
2525         /* Get silicon revision and designer */
2526         val = readl(virtbase + D40_DREG_PERIPHID2);
2527
2528         if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2529             D40_HW_DESIGNER) {
2530                 dev_err(&pdev->dev,
2531                         "[%s] Unknown designer! Got %x wanted %x\n",
2532                         __func__, val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2533                         D40_HW_DESIGNER);
2534                 goto failure;
2535         }
2536
2537         rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2538                 D40_DREG_PERIPHID2_REV_POS;
2539
2540         /* The number of physical channels on this HW */
2541         num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2542
2543         dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2544                  rev, res->start);
2545
2546         plat_data = pdev->dev.platform_data;
2547
2548         /* Count the number of logical channels in use */
2549         for (i = 0; i < plat_data->dev_len; i++)
2550                 if (plat_data->dev_rx[i] != 0)
2551                         num_log_chans++;
2552
2553         for (i = 0; i < plat_data->dev_len; i++)
2554                 if (plat_data->dev_tx[i] != 0)
2555                         num_log_chans++;
2556
2557         base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2558                        (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2559                        sizeof(struct d40_chan), GFP_KERNEL);
2560
2561         if (base == NULL) {
2562                 dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
2563                 goto failure;
2564         }
2565
2566         base->rev = rev;
2567         base->clk = clk;
2568         base->num_phy_chans = num_phy_chans;
2569         base->num_log_chans = num_log_chans;
2570         base->phy_start = res->start;
2571         base->phy_size = resource_size(res);
2572         base->virtbase = virtbase;
2573         base->plat_data = plat_data;
2574         base->dev = &pdev->dev;
2575         base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2576         base->log_chans = &base->phy_chans[num_phy_chans];
2577
2578         base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2579                                 GFP_KERNEL);
2580         if (!base->phy_res)
2581                 goto failure;
2582
2583         base->lookup_phy_chans = kzalloc(num_phy_chans *
2584                                          sizeof(struct d40_chan *),
2585                                          GFP_KERNEL);
2586         if (!base->lookup_phy_chans)
2587                 goto failure;
2588
2589         if (num_log_chans + plat_data->memcpy_len) {
2590                 /*
2591                  * The max number of logical channels are event lines for all
2592                  * src devices and dst devices
2593                  */
2594                 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2595                                                  sizeof(struct d40_chan *),
2596                                                  GFP_KERNEL);
2597                 if (!base->lookup_log_chans)
2598                         goto failure;
2599         }
2600
2601         base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2602                                             sizeof(struct d40_desc *) *
2603                                             D40_LCLA_LINK_PER_EVENT_GRP,
2604                                             GFP_KERNEL);
2605         if (!base->lcla_pool.alloc_map)
2606                 goto failure;
2607
2608         base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2609                                             0, SLAB_HWCACHE_ALIGN,
2610                                             NULL);
2611         if (base->desc_slab == NULL)
2612                 goto failure;
2613
2614         return base;
2615
2616 failure:
2617         if (!IS_ERR(clk)) {
2618                 clk_disable(clk);
2619                 clk_put(clk);
2620         }
2621         if (virtbase)
2622                 iounmap(virtbase);
2623         if (res)
2624                 release_mem_region(res->start,
2625                                    resource_size(res));
2626         if (virtbase)
2627                 iounmap(virtbase);
2628
2629         if (base) {
2630                 kfree(base->lcla_pool.alloc_map);
2631                 kfree(base->lookup_log_chans);
2632                 kfree(base->lookup_phy_chans);
2633                 kfree(base->phy_res);
2634                 kfree(base);
2635         }
2636
2637         return NULL;
2638 }
2639
2640 static void __init d40_hw_init(struct d40_base *base)
2641 {
2642
2643         static const struct d40_reg_val dma_init_reg[] = {
2644                 /* Clock every part of the DMA block from start */
2645                 { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
2646
2647                 /* Interrupts on all logical channels */
2648                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2649                 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2650                 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2651                 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2652                 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2653                 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2654                 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2655                 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2656                 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2657                 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2658                 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2659                 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2660         };
2661         int i;
2662         u32 prmseo[2] = {0, 0};
2663         u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2664         u32 pcmis = 0;
2665         u32 pcicr = 0;
2666
2667         for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2668                 writel(dma_init_reg[i].val,
2669                        base->virtbase + dma_init_reg[i].reg);
2670
2671         /* Configure all our dma channels to default settings */
2672         for (i = 0; i < base->num_phy_chans; i++) {
2673
2674                 activeo[i % 2] = activeo[i % 2] << 2;
2675
2676                 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2677                     == D40_ALLOC_PHY) {
2678                         activeo[i % 2] |= 3;
2679                         continue;
2680                 }
2681
2682                 /* Enable interrupt # */
2683                 pcmis = (pcmis << 1) | 1;
2684
2685                 /* Clear interrupt # */
2686                 pcicr = (pcicr << 1) | 1;
2687
2688                 /* Set channel to physical mode */
2689                 prmseo[i % 2] = prmseo[i % 2] << 2;
2690                 prmseo[i % 2] |= 1;
2691
2692         }
2693
2694         writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2695         writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2696         writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2697         writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2698
2699         /* Write which interrupt to enable */
2700         writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2701
2702         /* Write which interrupt to clear */
2703         writel(pcicr, base->virtbase + D40_DREG_PCICR);
2704
2705 }
2706
2707 static int __init d40_lcla_allocate(struct d40_base *base)
2708 {
2709         unsigned long *page_list;
2710         int i, j;
2711         int ret = 0;
2712
2713         /*
2714          * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2715          * To full fill this hardware requirement without wasting 256 kb
2716          * we allocate pages until we get an aligned one.
2717          */
2718         page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2719                             GFP_KERNEL);
2720
2721         if (!page_list) {
2722                 ret = -ENOMEM;
2723                 goto failure;
2724         }
2725
2726         /* Calculating how many pages that are required */
2727         base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2728
2729         for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2730                 page_list[i] = __get_free_pages(GFP_KERNEL,
2731                                                 base->lcla_pool.pages);
2732                 if (!page_list[i]) {
2733
2734                         dev_err(base->dev,
2735                                 "[%s] Failed to allocate %d pages.\n",
2736                                 __func__, base->lcla_pool.pages);
2737
2738                         for (j = 0; j < i; j++)
2739                                 free_pages(page_list[j], base->lcla_pool.pages);
2740                         goto failure;
2741                 }
2742
2743                 if ((virt_to_phys((void *)page_list[i]) &
2744                      (LCLA_ALIGNMENT - 1)) == 0)
2745                         break;
2746         }
2747
2748         for (j = 0; j < i; j++)
2749                 free_pages(page_list[j], base->lcla_pool.pages);
2750
2751         if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2752                 base->lcla_pool.base = (void *)page_list[i];
2753         } else {
2754                 /*
2755                  * After many attempts and no succees with finding the correct
2756                  * alignment, try with allocating a big buffer.
2757                  */
2758                 dev_warn(base->dev,
2759                          "[%s] Failed to get %d pages @ 18 bit align.\n",
2760                          __func__, base->lcla_pool.pages);
2761                 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2762                                                          base->num_phy_chans +
2763                                                          LCLA_ALIGNMENT,
2764                                                          GFP_KERNEL);
2765                 if (!base->lcla_pool.base_unaligned) {
2766                         ret = -ENOMEM;
2767                         goto failure;
2768                 }
2769
2770                 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2771                                                  LCLA_ALIGNMENT);
2772         }
2773
2774         writel(virt_to_phys(base->lcla_pool.base),
2775                base->virtbase + D40_DREG_LCLA);
2776 failure:
2777         kfree(page_list);
2778         return ret;
2779 }
2780
2781 static int __init d40_probe(struct platform_device *pdev)
2782 {
2783         int err;
2784         int ret = -ENOENT;
2785         struct d40_base *base;
2786         struct resource *res = NULL;
2787         int num_reserved_chans;
2788         u32 val;
2789
2790         base = d40_hw_detect_init(pdev);
2791
2792         if (!base)
2793                 goto failure;
2794
2795         num_reserved_chans = d40_phy_res_init(base);
2796
2797         platform_set_drvdata(pdev, base);
2798
2799         spin_lock_init(&base->interrupt_lock);
2800         spin_lock_init(&base->execmd_lock);
2801
2802         /* Get IO for logical channel parameter address */
2803         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2804         if (!res) {
2805                 ret = -ENOENT;
2806                 dev_err(&pdev->dev,
2807                         "[%s] No \"lcpa\" memory resource\n",
2808                         __func__);
2809                 goto failure;
2810         }
2811         base->lcpa_size = resource_size(res);
2812         base->phy_lcpa = res->start;
2813
2814         if (request_mem_region(res->start, resource_size(res),
2815                                D40_NAME " I/O lcpa") == NULL) {
2816                 ret = -EBUSY;
2817                 dev_err(&pdev->dev,
2818                         "[%s] Failed to request LCPA region 0x%x-0x%x\n",
2819                         __func__, res->start, res->end);
2820                 goto failure;
2821         }
2822
2823         /* We make use of ESRAM memory for this. */
2824         val = readl(base->virtbase + D40_DREG_LCPA);
2825         if (res->start != val && val != 0) {
2826                 dev_warn(&pdev->dev,
2827                          "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2828                          __func__, val, res->start);
2829         } else
2830                 writel(res->start, base->virtbase + D40_DREG_LCPA);
2831
2832         base->lcpa_base = ioremap(res->start, resource_size(res));
2833         if (!base->lcpa_base) {
2834                 ret = -ENOMEM;
2835                 dev_err(&pdev->dev,
2836                         "[%s] Failed to ioremap LCPA region\n",
2837                         __func__);
2838                 goto failure;
2839         }
2840
2841         ret = d40_lcla_allocate(base);
2842         if (ret) {
2843                 dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n",
2844                         __func__);
2845                 goto failure;
2846         }
2847
2848         spin_lock_init(&base->lcla_pool.lock);
2849
2850         base->irq = platform_get_irq(pdev, 0);
2851
2852         ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2853
2854         if (ret) {
2855                 dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
2856                 goto failure;
2857         }
2858
2859         err = d40_dmaengine_init(base, num_reserved_chans);
2860         if (err)
2861                 goto failure;
2862
2863         d40_hw_init(base);
2864
2865         dev_info(base->dev, "initialized\n");
2866         return 0;
2867
2868 failure:
2869         if (base) {
2870                 if (base->desc_slab)
2871                         kmem_cache_destroy(base->desc_slab);
2872                 if (base->virtbase)
2873                         iounmap(base->virtbase);
2874                 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2875                         free_pages((unsigned long)base->lcla_pool.base,
2876                                    base->lcla_pool.pages);
2877
2878                 kfree(base->lcla_pool.base_unaligned);
2879
2880                 if (base->phy_lcpa)
2881                         release_mem_region(base->phy_lcpa,
2882                                            base->lcpa_size);
2883                 if (base->phy_start)
2884                         release_mem_region(base->phy_start,
2885                                            base->phy_size);
2886                 if (base->clk) {
2887                         clk_disable(base->clk);
2888                         clk_put(base->clk);
2889                 }
2890
2891                 kfree(base->lcla_pool.alloc_map);
2892                 kfree(base->lookup_log_chans);
2893                 kfree(base->lookup_phy_chans);
2894                 kfree(base->phy_res);
2895                 kfree(base);
2896         }
2897
2898         dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
2899         return ret;
2900 }
2901
2902 static struct platform_driver d40_driver = {
2903         .driver = {
2904                 .owner = THIS_MODULE,
2905                 .name  = D40_NAME,
2906         },
2907 };
2908
2909 int __init stedma40_init(void)
2910 {
2911         return platform_driver_probe(&d40_driver, d40_probe);
2912 }
2913 arch_initcall(stedma40_init);