Merge branch 'for-linus' of git://git.infradead.org/users/vkoul/slave-dma
[linux-2.6.git] / drivers / dma / ipu / ipu_idmac.c
1 /*
2  * Copyright (C) 2008
3  * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
4  *
5  * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11
12 #include <linux/dma-mapping.h>
13 #include <linux/init.h>
14 #include <linux/platform_device.h>
15 #include <linux/err.h>
16 #include <linux/spinlock.h>
17 #include <linux/delay.h>
18 #include <linux/list.h>
19 #include <linux/clk.h>
20 #include <linux/vmalloc.h>
21 #include <linux/string.h>
22 #include <linux/interrupt.h>
23 #include <linux/io.h>
24
25 #include <mach/ipu.h>
26
27 #include "ipu_intern.h"
28
29 #define FS_VF_IN_VALID  0x00000002
30 #define FS_ENC_IN_VALID 0x00000001
31
32 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
33                                bool wait_for_stop);
34
35 /*
36  * There can be only one, we could allocate it dynamically, but then we'd have
37  * to add an extra parameter to some functions, and use something as ugly as
38  *      struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
39  * in the ISR
40  */
41 static struct ipu ipu_data;
42
43 #define to_ipu(id) container_of(id, struct ipu, idmac)
44
45 static u32 __idmac_read_icreg(struct ipu *ipu, unsigned long reg)
46 {
47         return __raw_readl(ipu->reg_ic + reg);
48 }
49
50 #define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
51
52 static void __idmac_write_icreg(struct ipu *ipu, u32 value, unsigned long reg)
53 {
54         __raw_writel(value, ipu->reg_ic + reg);
55 }
56
57 #define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
58
59 static u32 idmac_read_ipureg(struct ipu *ipu, unsigned long reg)
60 {
61         return __raw_readl(ipu->reg_ipu + reg);
62 }
63
64 static void idmac_write_ipureg(struct ipu *ipu, u32 value, unsigned long reg)
65 {
66         __raw_writel(value, ipu->reg_ipu + reg);
67 }
68
69 /*****************************************************************************
70  * IPU / IC common functions
71  */
72 static void dump_idmac_reg(struct ipu *ipu)
73 {
74         dev_dbg(ipu->dev, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
75                 "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
76                 idmac_read_icreg(ipu, IDMAC_CONF),
77                 idmac_read_icreg(ipu, IC_CONF),
78                 idmac_read_icreg(ipu, IDMAC_CHA_EN),
79                 idmac_read_icreg(ipu, IDMAC_CHA_PRI),
80                 idmac_read_icreg(ipu, IDMAC_CHA_BUSY));
81         dev_dbg(ipu->dev, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
82                 "DB_MODE 0x%x, TASKS_STAT 0x%x\n",
83                 idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY),
84                 idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY),
85                 idmac_read_ipureg(ipu, IPU_CHA_CUR_BUF),
86                 idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL),
87                 idmac_read_ipureg(ipu, IPU_TASKS_STAT));
88 }
89
90 static uint32_t bytes_per_pixel(enum pixel_fmt fmt)
91 {
92         switch (fmt) {
93         case IPU_PIX_FMT_GENERIC:       /* generic data */
94         case IPU_PIX_FMT_RGB332:
95         case IPU_PIX_FMT_YUV420P:
96         case IPU_PIX_FMT_YUV422P:
97         default:
98                 return 1;
99         case IPU_PIX_FMT_RGB565:
100         case IPU_PIX_FMT_YUYV:
101         case IPU_PIX_FMT_UYVY:
102                 return 2;
103         case IPU_PIX_FMT_BGR24:
104         case IPU_PIX_FMT_RGB24:
105                 return 3;
106         case IPU_PIX_FMT_GENERIC_32:    /* generic data */
107         case IPU_PIX_FMT_BGR32:
108         case IPU_PIX_FMT_RGB32:
109         case IPU_PIX_FMT_ABGR32:
110                 return 4;
111         }
112 }
113
114 /* Enable direct write to memory by the Camera Sensor Interface */
115 static void ipu_ic_enable_task(struct ipu *ipu, enum ipu_channel channel)
116 {
117         uint32_t ic_conf, mask;
118
119         switch (channel) {
120         case IDMAC_IC_0:
121                 mask = IC_CONF_PRPENC_EN;
122                 break;
123         case IDMAC_IC_7:
124                 mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN;
125                 break;
126         default:
127                 return;
128         }
129         ic_conf = idmac_read_icreg(ipu, IC_CONF) | mask;
130         idmac_write_icreg(ipu, ic_conf, IC_CONF);
131 }
132
133 /* Called under spin_lock_irqsave(&ipu_data.lock) */
134 static void ipu_ic_disable_task(struct ipu *ipu, enum ipu_channel channel)
135 {
136         uint32_t ic_conf, mask;
137
138         switch (channel) {
139         case IDMAC_IC_0:
140                 mask = IC_CONF_PRPENC_EN;
141                 break;
142         case IDMAC_IC_7:
143                 mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN;
144                 break;
145         default:
146                 return;
147         }
148         ic_conf = idmac_read_icreg(ipu, IC_CONF) & ~mask;
149         idmac_write_icreg(ipu, ic_conf, IC_CONF);
150 }
151
152 static uint32_t ipu_channel_status(struct ipu *ipu, enum ipu_channel channel)
153 {
154         uint32_t stat = TASK_STAT_IDLE;
155         uint32_t task_stat_reg = idmac_read_ipureg(ipu, IPU_TASKS_STAT);
156
157         switch (channel) {
158         case IDMAC_IC_7:
159                 stat = (task_stat_reg & TSTAT_CSI2MEM_MASK) >>
160                         TSTAT_CSI2MEM_OFFSET;
161                 break;
162         case IDMAC_IC_0:
163         case IDMAC_SDC_0:
164         case IDMAC_SDC_1:
165         default:
166                 break;
167         }
168         return stat;
169 }
170
171 struct chan_param_mem_planar {
172         /* Word 0 */
173         u32     xv:10;
174         u32     yv:10;
175         u32     xb:12;
176
177         u32     yb:12;
178         u32     res1:2;
179         u32     nsb:1;
180         u32     lnpb:6;
181         u32     ubo_l:11;
182
183         u32     ubo_h:15;
184         u32     vbo_l:17;
185
186         u32     vbo_h:9;
187         u32     res2:3;
188         u32     fw:12;
189         u32     fh_l:8;
190
191         u32     fh_h:4;
192         u32     res3:28;
193
194         /* Word 1 */
195         u32     eba0;
196
197         u32     eba1;
198
199         u32     bpp:3;
200         u32     sl:14;
201         u32     pfs:3;
202         u32     bam:3;
203         u32     res4:2;
204         u32     npb:6;
205         u32     res5:1;
206
207         u32     sat:2;
208         u32     res6:30;
209 } __attribute__ ((packed));
210
211 struct chan_param_mem_interleaved {
212         /* Word 0 */
213         u32     xv:10;
214         u32     yv:10;
215         u32     xb:12;
216
217         u32     yb:12;
218         u32     sce:1;
219         u32     res1:1;
220         u32     nsb:1;
221         u32     lnpb:6;
222         u32     sx:10;
223         u32     sy_l:1;
224
225         u32     sy_h:9;
226         u32     ns:10;
227         u32     sm:10;
228         u32     sdx_l:3;
229
230         u32     sdx_h:2;
231         u32     sdy:5;
232         u32     sdrx:1;
233         u32     sdry:1;
234         u32     sdr1:1;
235         u32     res2:2;
236         u32     fw:12;
237         u32     fh_l:8;
238
239         u32     fh_h:4;
240         u32     res3:28;
241
242         /* Word 1 */
243         u32     eba0;
244
245         u32     eba1;
246
247         u32     bpp:3;
248         u32     sl:14;
249         u32     pfs:3;
250         u32     bam:3;
251         u32     res4:2;
252         u32     npb:6;
253         u32     res5:1;
254
255         u32     sat:2;
256         u32     scc:1;
257         u32     ofs0:5;
258         u32     ofs1:5;
259         u32     ofs2:5;
260         u32     ofs3:5;
261         u32     wid0:3;
262         u32     wid1:3;
263         u32     wid2:3;
264
265         u32     wid3:3;
266         u32     dec_sel:1;
267         u32     res6:28;
268 } __attribute__ ((packed));
269
270 union chan_param_mem {
271         struct chan_param_mem_planar            pp;
272         struct chan_param_mem_interleaved       ip;
273 };
274
275 static void ipu_ch_param_set_plane_offset(union chan_param_mem *params,
276                                           u32 u_offset, u32 v_offset)
277 {
278         params->pp.ubo_l = u_offset & 0x7ff;
279         params->pp.ubo_h = u_offset >> 11;
280         params->pp.vbo_l = v_offset & 0x1ffff;
281         params->pp.vbo_h = v_offset >> 17;
282 }
283
284 static void ipu_ch_param_set_size(union chan_param_mem *params,
285                                   uint32_t pixel_fmt, uint16_t width,
286                                   uint16_t height, uint16_t stride)
287 {
288         u32 u_offset;
289         u32 v_offset;
290
291         params->pp.fw           = width - 1;
292         params->pp.fh_l         = height - 1;
293         params->pp.fh_h         = (height - 1) >> 8;
294         params->pp.sl           = stride - 1;
295
296         switch (pixel_fmt) {
297         case IPU_PIX_FMT_GENERIC:
298                 /*Represents 8-bit Generic data */
299                 params->pp.bpp  = 3;
300                 params->pp.pfs  = 7;
301                 params->pp.npb  = 31;
302                 params->pp.sat  = 2;            /* SAT = use 32-bit access */
303                 break;
304         case IPU_PIX_FMT_GENERIC_32:
305                 /*Represents 32-bit Generic data */
306                 params->pp.bpp  = 0;
307                 params->pp.pfs  = 7;
308                 params->pp.npb  = 7;
309                 params->pp.sat  = 2;            /* SAT = use 32-bit access */
310                 break;
311         case IPU_PIX_FMT_RGB565:
312                 params->ip.bpp  = 2;
313                 params->ip.pfs  = 4;
314                 params->ip.npb  = 7;
315                 params->ip.sat  = 2;            /* SAT = 32-bit access */
316                 params->ip.ofs0 = 0;            /* Red bit offset */
317                 params->ip.ofs1 = 5;            /* Green bit offset */
318                 params->ip.ofs2 = 11;           /* Blue bit offset */
319                 params->ip.ofs3 = 16;           /* Alpha bit offset */
320                 params->ip.wid0 = 4;            /* Red bit width - 1 */
321                 params->ip.wid1 = 5;            /* Green bit width - 1 */
322                 params->ip.wid2 = 4;            /* Blue bit width - 1 */
323                 break;
324         case IPU_PIX_FMT_BGR24:
325                 params->ip.bpp  = 1;            /* 24 BPP & RGB PFS */
326                 params->ip.pfs  = 4;
327                 params->ip.npb  = 7;
328                 params->ip.sat  = 2;            /* SAT = 32-bit access */
329                 params->ip.ofs0 = 0;            /* Red bit offset */
330                 params->ip.ofs1 = 8;            /* Green bit offset */
331                 params->ip.ofs2 = 16;           /* Blue bit offset */
332                 params->ip.ofs3 = 24;           /* Alpha bit offset */
333                 params->ip.wid0 = 7;            /* Red bit width - 1 */
334                 params->ip.wid1 = 7;            /* Green bit width - 1 */
335                 params->ip.wid2 = 7;            /* Blue bit width - 1 */
336                 break;
337         case IPU_PIX_FMT_RGB24:
338                 params->ip.bpp  = 1;            /* 24 BPP & RGB PFS */
339                 params->ip.pfs  = 4;
340                 params->ip.npb  = 7;
341                 params->ip.sat  = 2;            /* SAT = 32-bit access */
342                 params->ip.ofs0 = 16;           /* Red bit offset */
343                 params->ip.ofs1 = 8;            /* Green bit offset */
344                 params->ip.ofs2 = 0;            /* Blue bit offset */
345                 params->ip.ofs3 = 24;           /* Alpha bit offset */
346                 params->ip.wid0 = 7;            /* Red bit width - 1 */
347                 params->ip.wid1 = 7;            /* Green bit width - 1 */
348                 params->ip.wid2 = 7;            /* Blue bit width - 1 */
349                 break;
350         case IPU_PIX_FMT_BGRA32:
351         case IPU_PIX_FMT_BGR32:
352         case IPU_PIX_FMT_ABGR32:
353                 params->ip.bpp  = 0;
354                 params->ip.pfs  = 4;
355                 params->ip.npb  = 7;
356                 params->ip.sat  = 2;            /* SAT = 32-bit access */
357                 params->ip.ofs0 = 8;            /* Red bit offset */
358                 params->ip.ofs1 = 16;           /* Green bit offset */
359                 params->ip.ofs2 = 24;           /* Blue bit offset */
360                 params->ip.ofs3 = 0;            /* Alpha bit offset */
361                 params->ip.wid0 = 7;            /* Red bit width - 1 */
362                 params->ip.wid1 = 7;            /* Green bit width - 1 */
363                 params->ip.wid2 = 7;            /* Blue bit width - 1 */
364                 params->ip.wid3 = 7;            /* Alpha bit width - 1 */
365                 break;
366         case IPU_PIX_FMT_RGBA32:
367         case IPU_PIX_FMT_RGB32:
368                 params->ip.bpp  = 0;
369                 params->ip.pfs  = 4;
370                 params->ip.npb  = 7;
371                 params->ip.sat  = 2;            /* SAT = 32-bit access */
372                 params->ip.ofs0 = 24;           /* Red bit offset */
373                 params->ip.ofs1 = 16;           /* Green bit offset */
374                 params->ip.ofs2 = 8;            /* Blue bit offset */
375                 params->ip.ofs3 = 0;            /* Alpha bit offset */
376                 params->ip.wid0 = 7;            /* Red bit width - 1 */
377                 params->ip.wid1 = 7;            /* Green bit width - 1 */
378                 params->ip.wid2 = 7;            /* Blue bit width - 1 */
379                 params->ip.wid3 = 7;            /* Alpha bit width - 1 */
380                 break;
381         case IPU_PIX_FMT_UYVY:
382                 params->ip.bpp  = 2;
383                 params->ip.pfs  = 6;
384                 params->ip.npb  = 7;
385                 params->ip.sat  = 2;            /* SAT = 32-bit access */
386                 break;
387         case IPU_PIX_FMT_YUV420P2:
388         case IPU_PIX_FMT_YUV420P:
389                 params->ip.bpp  = 3;
390                 params->ip.pfs  = 3;
391                 params->ip.npb  = 7;
392                 params->ip.sat  = 2;            /* SAT = 32-bit access */
393                 u_offset = stride * height;
394                 v_offset = u_offset + u_offset / 4;
395                 ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
396                 break;
397         case IPU_PIX_FMT_YVU422P:
398                 params->ip.bpp  = 3;
399                 params->ip.pfs  = 2;
400                 params->ip.npb  = 7;
401                 params->ip.sat  = 2;            /* SAT = 32-bit access */
402                 v_offset = stride * height;
403                 u_offset = v_offset + v_offset / 2;
404                 ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
405                 break;
406         case IPU_PIX_FMT_YUV422P:
407                 params->ip.bpp  = 3;
408                 params->ip.pfs  = 2;
409                 params->ip.npb  = 7;
410                 params->ip.sat  = 2;            /* SAT = 32-bit access */
411                 u_offset = stride * height;
412                 v_offset = u_offset + u_offset / 2;
413                 ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
414                 break;
415         default:
416                 dev_err(ipu_data.dev,
417                         "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt);
418                 break;
419         }
420
421         params->pp.nsb = 1;
422 }
423
424 static void ipu_ch_param_set_burst_size(union chan_param_mem *params,
425                                         uint16_t burst_pixels)
426 {
427         params->pp.npb = burst_pixels - 1;
428 }
429
430 static void ipu_ch_param_set_buffer(union chan_param_mem *params,
431                                     dma_addr_t buf0, dma_addr_t buf1)
432 {
433         params->pp.eba0 = buf0;
434         params->pp.eba1 = buf1;
435 }
436
437 static void ipu_ch_param_set_rotation(union chan_param_mem *params,
438                                       enum ipu_rotate_mode rotate)
439 {
440         params->pp.bam = rotate;
441 }
442
443 static void ipu_write_param_mem(uint32_t addr, uint32_t *data,
444                                 uint32_t num_words)
445 {
446         for (; num_words > 0; num_words--) {
447                 dev_dbg(ipu_data.dev,
448                         "write param mem - addr = 0x%08X, data = 0x%08X\n",
449                         addr, *data);
450                 idmac_write_ipureg(&ipu_data, addr, IPU_IMA_ADDR);
451                 idmac_write_ipureg(&ipu_data, *data++, IPU_IMA_DATA);
452                 addr++;
453                 if ((addr & 0x7) == 5) {
454                         addr &= ~0x7;   /* set to word 0 */
455                         addr += 8;      /* increment to next row */
456                 }
457         }
458 }
459
460 static int calc_resize_coeffs(uint32_t in_size, uint32_t out_size,
461                               uint32_t *resize_coeff,
462                               uint32_t *downsize_coeff)
463 {
464         uint32_t temp_size;
465         uint32_t temp_downsize;
466
467         *resize_coeff   = 1 << 13;
468         *downsize_coeff = 1 << 13;
469
470         /* Cannot downsize more than 8:1 */
471         if (out_size << 3 < in_size)
472                 return -EINVAL;
473
474         /* compute downsizing coefficient */
475         temp_downsize = 0;
476         temp_size = in_size;
477         while (temp_size >= out_size * 2 && temp_downsize < 2) {
478                 temp_size >>= 1;
479                 temp_downsize++;
480         }
481         *downsize_coeff = temp_downsize;
482
483         /*
484          * compute resizing coefficient using the following formula:
485          * resize_coeff = M*(SI -1)/(SO - 1)
486          * where M = 2^13, SI - input size, SO - output size
487          */
488         *resize_coeff = (8192L * (temp_size - 1)) / (out_size - 1);
489         if (*resize_coeff >= 16384L) {
490                 dev_err(ipu_data.dev, "Warning! Overflow on resize coeff.\n");
491                 *resize_coeff = 0x3FFF;
492         }
493
494         dev_dbg(ipu_data.dev, "resizing from %u -> %u pixels, "
495                 "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size, out_size,
496                 *downsize_coeff, *resize_coeff >= 8192L ? 1 : 0,
497                 ((*resize_coeff & 0x1FFF) * 10000L) / 8192L, *resize_coeff);
498
499         return 0;
500 }
501
502 static enum ipu_color_space format_to_colorspace(enum pixel_fmt fmt)
503 {
504         switch (fmt) {
505         case IPU_PIX_FMT_RGB565:
506         case IPU_PIX_FMT_BGR24:
507         case IPU_PIX_FMT_RGB24:
508         case IPU_PIX_FMT_BGR32:
509         case IPU_PIX_FMT_RGB32:
510                 return IPU_COLORSPACE_RGB;
511         default:
512                 return IPU_COLORSPACE_YCBCR;
513         }
514 }
515
516 static int ipu_ic_init_prpenc(struct ipu *ipu,
517                               union ipu_channel_param *params, bool src_is_csi)
518 {
519         uint32_t reg, ic_conf;
520         uint32_t downsize_coeff, resize_coeff;
521         enum ipu_color_space in_fmt, out_fmt;
522
523         /* Setup vertical resizing */
524         calc_resize_coeffs(params->video.in_height,
525                             params->video.out_height,
526                             &resize_coeff, &downsize_coeff);
527         reg = (downsize_coeff << 30) | (resize_coeff << 16);
528
529         /* Setup horizontal resizing */
530         calc_resize_coeffs(params->video.in_width,
531                             params->video.out_width,
532                             &resize_coeff, &downsize_coeff);
533         reg |= (downsize_coeff << 14) | resize_coeff;
534
535         /* Setup color space conversion */
536         in_fmt = format_to_colorspace(params->video.in_pixel_fmt);
537         out_fmt = format_to_colorspace(params->video.out_pixel_fmt);
538
539         /*
540          * Colourspace conversion unsupported yet - see _init_csc() in
541          * Freescale sources
542          */
543         if (in_fmt != out_fmt) {
544                 dev_err(ipu->dev, "Colourspace conversion unsupported!\n");
545                 return -EOPNOTSUPP;
546         }
547
548         idmac_write_icreg(ipu, reg, IC_PRP_ENC_RSC);
549
550         ic_conf = idmac_read_icreg(ipu, IC_CONF);
551
552         if (src_is_csi)
553                 ic_conf &= ~IC_CONF_RWS_EN;
554         else
555                 ic_conf |= IC_CONF_RWS_EN;
556
557         idmac_write_icreg(ipu, ic_conf, IC_CONF);
558
559         return 0;
560 }
561
562 static uint32_t dma_param_addr(uint32_t dma_ch)
563 {
564         /* Channel Parameter Memory */
565         return 0x10000 | (dma_ch << 4);
566 }
567
568 static void ipu_channel_set_priority(struct ipu *ipu, enum ipu_channel channel,
569                                      bool prio)
570 {
571         u32 reg = idmac_read_icreg(ipu, IDMAC_CHA_PRI);
572
573         if (prio)
574                 reg |= 1UL << channel;
575         else
576                 reg &= ~(1UL << channel);
577
578         idmac_write_icreg(ipu, reg, IDMAC_CHA_PRI);
579
580         dump_idmac_reg(ipu);
581 }
582
583 static uint32_t ipu_channel_conf_mask(enum ipu_channel channel)
584 {
585         uint32_t mask;
586
587         switch (channel) {
588         case IDMAC_IC_0:
589         case IDMAC_IC_7:
590                 mask = IPU_CONF_CSI_EN | IPU_CONF_IC_EN;
591                 break;
592         case IDMAC_SDC_0:
593         case IDMAC_SDC_1:
594                 mask = IPU_CONF_SDC_EN | IPU_CONF_DI_EN;
595                 break;
596         default:
597                 mask = 0;
598                 break;
599         }
600
601         return mask;
602 }
603
604 /**
605  * ipu_enable_channel() - enable an IPU channel.
606  * @idmac:      IPU DMAC context.
607  * @ichan:      IDMAC channel.
608  * @return:     0 on success or negative error code on failure.
609  */
610 static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan)
611 {
612         struct ipu *ipu = to_ipu(idmac);
613         enum ipu_channel channel = ichan->dma_chan.chan_id;
614         uint32_t reg;
615         unsigned long flags;
616
617         spin_lock_irqsave(&ipu->lock, flags);
618
619         /* Reset to buffer 0 */
620         idmac_write_ipureg(ipu, 1UL << channel, IPU_CHA_CUR_BUF);
621         ichan->active_buffer = 0;
622         ichan->status = IPU_CHANNEL_ENABLED;
623
624         switch (channel) {
625         case IDMAC_SDC_0:
626         case IDMAC_SDC_1:
627         case IDMAC_IC_7:
628                 ipu_channel_set_priority(ipu, channel, true);
629         default:
630                 break;
631         }
632
633         reg = idmac_read_icreg(ipu, IDMAC_CHA_EN);
634
635         idmac_write_icreg(ipu, reg | (1UL << channel), IDMAC_CHA_EN);
636
637         ipu_ic_enable_task(ipu, channel);
638
639         spin_unlock_irqrestore(&ipu->lock, flags);
640         return 0;
641 }
642
643 /**
644  * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
645  * @ichan:      IDMAC channel.
646  * @pixel_fmt:  pixel format of buffer. Pixel format is a FOURCC ASCII code.
647  * @width:      width of buffer in pixels.
648  * @height:     height of buffer in pixels.
649  * @stride:     stride length of buffer in pixels.
650  * @rot_mode:   rotation mode of buffer. A rotation setting other than
651  *              IPU_ROTATE_VERT_FLIP should only be used for input buffers of
652  *              rotation channels.
653  * @phyaddr_0:  buffer 0 physical address.
654  * @phyaddr_1:  buffer 1 physical address. Setting this to a value other than
655  *              NULL enables double buffering mode.
656  * @return:     0 on success or negative error code on failure.
657  */
658 static int ipu_init_channel_buffer(struct idmac_channel *ichan,
659                                    enum pixel_fmt pixel_fmt,
660                                    uint16_t width, uint16_t height,
661                                    uint32_t stride,
662                                    enum ipu_rotate_mode rot_mode,
663                                    dma_addr_t phyaddr_0, dma_addr_t phyaddr_1)
664 {
665         enum ipu_channel channel = ichan->dma_chan.chan_id;
666         struct idmac *idmac = to_idmac(ichan->dma_chan.device);
667         struct ipu *ipu = to_ipu(idmac);
668         union chan_param_mem params = {};
669         unsigned long flags;
670         uint32_t reg;
671         uint32_t stride_bytes;
672
673         stride_bytes = stride * bytes_per_pixel(pixel_fmt);
674
675         if (stride_bytes % 4) {
676                 dev_err(ipu->dev,
677                         "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
678                         stride, stride_bytes);
679                 return -EINVAL;
680         }
681
682         /* IC channel's stride must be a multiple of 8 pixels */
683         if ((channel <= IDMAC_IC_13) && (stride % 8)) {
684                 dev_err(ipu->dev, "Stride must be 8 pixel multiple\n");
685                 return -EINVAL;
686         }
687
688         /* Build parameter memory data for DMA channel */
689         ipu_ch_param_set_size(&params, pixel_fmt, width, height, stride_bytes);
690         ipu_ch_param_set_buffer(&params, phyaddr_0, phyaddr_1);
691         ipu_ch_param_set_rotation(&params, rot_mode);
692         /* Some channels (rotation) have restriction on burst length */
693         switch (channel) {
694         case IDMAC_IC_7:        /* Hangs with burst 8, 16, other values
695                                    invalid - Table 44-30 */
696 /*
697                 ipu_ch_param_set_burst_size(&params, 8);
698  */
699                 break;
700         case IDMAC_SDC_0:
701         case IDMAC_SDC_1:
702                 /* In original code only IPU_PIX_FMT_RGB565 was setting burst */
703                 ipu_ch_param_set_burst_size(&params, 16);
704                 break;
705         case IDMAC_IC_0:
706         default:
707                 break;
708         }
709
710         spin_lock_irqsave(&ipu->lock, flags);
711
712         ipu_write_param_mem(dma_param_addr(channel), (uint32_t *)&params, 10);
713
714         reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL);
715
716         if (phyaddr_1)
717                 reg |= 1UL << channel;
718         else
719                 reg &= ~(1UL << channel);
720
721         idmac_write_ipureg(ipu, reg, IPU_CHA_DB_MODE_SEL);
722
723         ichan->status = IPU_CHANNEL_READY;
724
725         spin_unlock_irqrestore(&ipu->lock, flags);
726
727         return 0;
728 }
729
730 /**
731  * ipu_select_buffer() - mark a channel's buffer as ready.
732  * @channel:    channel ID.
733  * @buffer_n:   buffer number to mark ready.
734  */
735 static void ipu_select_buffer(enum ipu_channel channel, int buffer_n)
736 {
737         /* No locking - this is a write-one-to-set register, cleared by IPU */
738         if (buffer_n == 0)
739                 /* Mark buffer 0 as ready. */
740                 idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF0_RDY);
741         else
742                 /* Mark buffer 1 as ready. */
743                 idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF1_RDY);
744 }
745
746 /**
747  * ipu_update_channel_buffer() - update physical address of a channel buffer.
748  * @ichan:      IDMAC channel.
749  * @buffer_n:   buffer number to update.
750  *              0 or 1 are the only valid values.
751  * @phyaddr:    buffer physical address.
752  */
753 /* Called under spin_lock(_irqsave)(&ichan->lock) */
754 static void ipu_update_channel_buffer(struct idmac_channel *ichan,
755                                       int buffer_n, dma_addr_t phyaddr)
756 {
757         enum ipu_channel channel = ichan->dma_chan.chan_id;
758         uint32_t reg;
759         unsigned long flags;
760
761         spin_lock_irqsave(&ipu_data.lock, flags);
762
763         if (buffer_n == 0) {
764                 reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
765                 if (reg & (1UL << channel)) {
766                         ipu_ic_disable_task(&ipu_data, channel);
767                         ichan->status = IPU_CHANNEL_READY;
768                 }
769
770                 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
771                 idmac_write_ipureg(&ipu_data, dma_param_addr(channel) +
772                                    0x0008UL, IPU_IMA_ADDR);
773                 idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA);
774         } else {
775                 reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
776                 if (reg & (1UL << channel)) {
777                         ipu_ic_disable_task(&ipu_data, channel);
778                         ichan->status = IPU_CHANNEL_READY;
779                 }
780
781                 /* Check if double-buffering is already enabled */
782                 reg = idmac_read_ipureg(&ipu_data, IPU_CHA_DB_MODE_SEL);
783
784                 if (!(reg & (1UL << channel)))
785                         idmac_write_ipureg(&ipu_data, reg | (1UL << channel),
786                                            IPU_CHA_DB_MODE_SEL);
787
788                 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
789                 idmac_write_ipureg(&ipu_data, dma_param_addr(channel) +
790                                    0x0009UL, IPU_IMA_ADDR);
791                 idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA);
792         }
793
794         spin_unlock_irqrestore(&ipu_data.lock, flags);
795 }
796
797 /* Called under spin_lock_irqsave(&ichan->lock) */
798 static int ipu_submit_buffer(struct idmac_channel *ichan,
799         struct idmac_tx_desc *desc, struct scatterlist *sg, int buf_idx)
800 {
801         unsigned int chan_id = ichan->dma_chan.chan_id;
802         struct device *dev = &ichan->dma_chan.dev->device;
803
804         if (async_tx_test_ack(&desc->txd))
805                 return -EINTR;
806
807         /*
808          * On first invocation this shouldn't be necessary, the call to
809          * ipu_init_channel_buffer() above will set addresses for us, so we
810          * could make it conditional on status >= IPU_CHANNEL_ENABLED, but
811          * doing it again shouldn't hurt either.
812          */
813         ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg));
814
815         ipu_select_buffer(chan_id, buf_idx);
816         dev_dbg(dev, "Updated sg %p on channel 0x%x buffer %d\n",
817                 sg, chan_id, buf_idx);
818
819         return 0;
820 }
821
822 /* Called under spin_lock_irqsave(&ichan->lock) */
823 static int ipu_submit_channel_buffers(struct idmac_channel *ichan,
824                                       struct idmac_tx_desc *desc)
825 {
826         struct scatterlist *sg;
827         int i, ret = 0;
828
829         for (i = 0, sg = desc->sg; i < 2 && sg; i++) {
830                 if (!ichan->sg[i]) {
831                         ichan->sg[i] = sg;
832
833                         ret = ipu_submit_buffer(ichan, desc, sg, i);
834                         if (ret < 0)
835                                 return ret;
836
837                         sg = sg_next(sg);
838                 }
839         }
840
841         return ret;
842 }
843
844 static dma_cookie_t idmac_tx_submit(struct dma_async_tx_descriptor *tx)
845 {
846         struct idmac_tx_desc *desc = to_tx_desc(tx);
847         struct idmac_channel *ichan = to_idmac_chan(tx->chan);
848         struct idmac *idmac = to_idmac(tx->chan->device);
849         struct ipu *ipu = to_ipu(idmac);
850         struct device *dev = &ichan->dma_chan.dev->device;
851         dma_cookie_t cookie;
852         unsigned long flags;
853         int ret;
854
855         /* Sanity check */
856         if (!list_empty(&desc->list)) {
857                 /* The descriptor doesn't belong to client */
858                 dev_err(dev, "Descriptor %p not prepared!\n", tx);
859                 return -EBUSY;
860         }
861
862         mutex_lock(&ichan->chan_mutex);
863
864         async_tx_clear_ack(tx);
865
866         if (ichan->status < IPU_CHANNEL_READY) {
867                 struct idmac_video_param *video = &ichan->params.video;
868                 /*
869                  * Initial buffer assignment - the first two sg-entries from
870                  * the descriptor will end up in the IDMAC buffers
871                  */
872                 dma_addr_t dma_1 = sg_is_last(desc->sg) ? 0 :
873                         sg_dma_address(&desc->sg[1]);
874
875                 WARN_ON(ichan->sg[0] || ichan->sg[1]);
876
877                 cookie = ipu_init_channel_buffer(ichan,
878                                                  video->out_pixel_fmt,
879                                                  video->out_width,
880                                                  video->out_height,
881                                                  video->out_stride,
882                                                  IPU_ROTATE_NONE,
883                                                  sg_dma_address(&desc->sg[0]),
884                                                  dma_1);
885                 if (cookie < 0)
886                         goto out;
887         }
888
889         dev_dbg(dev, "Submitting sg %p\n", &desc->sg[0]);
890
891         cookie = ichan->dma_chan.cookie;
892
893         if (++cookie < 0)
894                 cookie = 1;
895
896         /* from dmaengine.h: "last cookie value returned to client" */
897         ichan->dma_chan.cookie = cookie;
898         tx->cookie = cookie;
899
900         /* ipu->lock can be taken under ichan->lock, but not v.v. */
901         spin_lock_irqsave(&ichan->lock, flags);
902
903         list_add_tail(&desc->list, &ichan->queue);
904         /* submit_buffers() atomically verifies and fills empty sg slots */
905         ret = ipu_submit_channel_buffers(ichan, desc);
906
907         spin_unlock_irqrestore(&ichan->lock, flags);
908
909         if (ret < 0) {
910                 cookie = ret;
911                 goto dequeue;
912         }
913
914         if (ichan->status < IPU_CHANNEL_ENABLED) {
915                 ret = ipu_enable_channel(idmac, ichan);
916                 if (ret < 0) {
917                         cookie = ret;
918                         goto dequeue;
919                 }
920         }
921
922         dump_idmac_reg(ipu);
923
924 dequeue:
925         if (cookie < 0) {
926                 spin_lock_irqsave(&ichan->lock, flags);
927                 list_del_init(&desc->list);
928                 spin_unlock_irqrestore(&ichan->lock, flags);
929                 tx->cookie = cookie;
930                 ichan->dma_chan.cookie = cookie;
931         }
932
933 out:
934         mutex_unlock(&ichan->chan_mutex);
935
936         return cookie;
937 }
938
939 /* Called with ichan->chan_mutex held */
940 static int idmac_desc_alloc(struct idmac_channel *ichan, int n)
941 {
942         struct idmac_tx_desc *desc = vmalloc(n * sizeof(struct idmac_tx_desc));
943         struct idmac *idmac = to_idmac(ichan->dma_chan.device);
944
945         if (!desc)
946                 return -ENOMEM;
947
948         /* No interrupts, just disable the tasklet for a moment */
949         tasklet_disable(&to_ipu(idmac)->tasklet);
950
951         ichan->n_tx_desc = n;
952         ichan->desc = desc;
953         INIT_LIST_HEAD(&ichan->queue);
954         INIT_LIST_HEAD(&ichan->free_list);
955
956         while (n--) {
957                 struct dma_async_tx_descriptor *txd = &desc->txd;
958
959                 memset(txd, 0, sizeof(*txd));
960                 dma_async_tx_descriptor_init(txd, &ichan->dma_chan);
961                 txd->tx_submit          = idmac_tx_submit;
962
963                 list_add(&desc->list, &ichan->free_list);
964
965                 desc++;
966         }
967
968         tasklet_enable(&to_ipu(idmac)->tasklet);
969
970         return 0;
971 }
972
973 /**
974  * ipu_init_channel() - initialize an IPU channel.
975  * @idmac:      IPU DMAC context.
976  * @ichan:      pointer to the channel object.
977  * @return      0 on success or negative error code on failure.
978  */
979 static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan)
980 {
981         union ipu_channel_param *params = &ichan->params;
982         uint32_t ipu_conf;
983         enum ipu_channel channel = ichan->dma_chan.chan_id;
984         unsigned long flags;
985         uint32_t reg;
986         struct ipu *ipu = to_ipu(idmac);
987         int ret = 0, n_desc = 0;
988
989         dev_dbg(ipu->dev, "init channel = %d\n", channel);
990
991         if (channel != IDMAC_SDC_0 && channel != IDMAC_SDC_1 &&
992             channel != IDMAC_IC_7)
993                 return -EINVAL;
994
995         spin_lock_irqsave(&ipu->lock, flags);
996
997         switch (channel) {
998         case IDMAC_IC_7:
999                 n_desc = 16;
1000                 reg = idmac_read_icreg(ipu, IC_CONF);
1001                 idmac_write_icreg(ipu, reg & ~IC_CONF_CSI_MEM_WR_EN, IC_CONF);
1002                 break;
1003         case IDMAC_IC_0:
1004                 n_desc = 16;
1005                 reg = idmac_read_ipureg(ipu, IPU_FS_PROC_FLOW);
1006                 idmac_write_ipureg(ipu, reg & ~FS_ENC_IN_VALID, IPU_FS_PROC_FLOW);
1007                 ret = ipu_ic_init_prpenc(ipu, params, true);
1008                 break;
1009         case IDMAC_SDC_0:
1010         case IDMAC_SDC_1:
1011                 n_desc = 4;
1012         default:
1013                 break;
1014         }
1015
1016         ipu->channel_init_mask |= 1L << channel;
1017
1018         /* Enable IPU sub module */
1019         ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) |
1020                 ipu_channel_conf_mask(channel);
1021         idmac_write_ipureg(ipu, ipu_conf, IPU_CONF);
1022
1023         spin_unlock_irqrestore(&ipu->lock, flags);
1024
1025         if (n_desc && !ichan->desc)
1026                 ret = idmac_desc_alloc(ichan, n_desc);
1027
1028         dump_idmac_reg(ipu);
1029
1030         return ret;
1031 }
1032
1033 /**
1034  * ipu_uninit_channel() - uninitialize an IPU channel.
1035  * @idmac:      IPU DMAC context.
1036  * @ichan:      pointer to the channel object.
1037  */
1038 static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan)
1039 {
1040         enum ipu_channel channel = ichan->dma_chan.chan_id;
1041         unsigned long flags;
1042         uint32_t reg;
1043         unsigned long chan_mask = 1UL << channel;
1044         uint32_t ipu_conf;
1045         struct ipu *ipu = to_ipu(idmac);
1046
1047         spin_lock_irqsave(&ipu->lock, flags);
1048
1049         if (!(ipu->channel_init_mask & chan_mask)) {
1050                 dev_err(ipu->dev, "Channel already uninitialized %d\n",
1051                         channel);
1052                 spin_unlock_irqrestore(&ipu->lock, flags);
1053                 return;
1054         }
1055
1056         /* Reset the double buffer */
1057         reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL);
1058         idmac_write_ipureg(ipu, reg & ~chan_mask, IPU_CHA_DB_MODE_SEL);
1059
1060         ichan->sec_chan_en = false;
1061
1062         switch (channel) {
1063         case IDMAC_IC_7:
1064                 reg = idmac_read_icreg(ipu, IC_CONF);
1065                 idmac_write_icreg(ipu, reg & ~(IC_CONF_RWS_EN | IC_CONF_PRPENC_EN),
1066                              IC_CONF);
1067                 break;
1068         case IDMAC_IC_0:
1069                 reg = idmac_read_icreg(ipu, IC_CONF);
1070                 idmac_write_icreg(ipu, reg & ~(IC_CONF_PRPENC_EN | IC_CONF_PRPENC_CSC1),
1071                                   IC_CONF);
1072                 break;
1073         case IDMAC_SDC_0:
1074         case IDMAC_SDC_1:
1075         default:
1076                 break;
1077         }
1078
1079         ipu->channel_init_mask &= ~(1L << channel);
1080
1081         ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) &
1082                 ~ipu_channel_conf_mask(channel);
1083         idmac_write_ipureg(ipu, ipu_conf, IPU_CONF);
1084
1085         spin_unlock_irqrestore(&ipu->lock, flags);
1086
1087         ichan->n_tx_desc = 0;
1088         vfree(ichan->desc);
1089         ichan->desc = NULL;
1090 }
1091
1092 /**
1093  * ipu_disable_channel() - disable an IPU channel.
1094  * @idmac:              IPU DMAC context.
1095  * @ichan:              channel object pointer.
1096  * @wait_for_stop:      flag to set whether to wait for channel end of frame or
1097  *                      return immediately.
1098  * @return:             0 on success or negative error code on failure.
1099  */
1100 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
1101                                bool wait_for_stop)
1102 {
1103         enum ipu_channel channel = ichan->dma_chan.chan_id;
1104         struct ipu *ipu = to_ipu(idmac);
1105         uint32_t reg;
1106         unsigned long flags;
1107         unsigned long chan_mask = 1UL << channel;
1108         unsigned int timeout;
1109
1110         if (wait_for_stop && channel != IDMAC_SDC_1 && channel != IDMAC_SDC_0) {
1111                 timeout = 40;
1112                 /* This waiting always fails. Related to spurious irq problem */
1113                 while ((idmac_read_icreg(ipu, IDMAC_CHA_BUSY) & chan_mask) ||
1114                        (ipu_channel_status(ipu, channel) == TASK_STAT_ACTIVE)) {
1115                         timeout--;
1116                         msleep(10);
1117
1118                         if (!timeout) {
1119                                 dev_dbg(ipu->dev,
1120                                         "Warning: timeout waiting for channel %u to "
1121                                         "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
1122                                         "busy = 0x%08X, tstat = 0x%08X\n", channel,
1123                                         idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY),
1124                                         idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY),
1125                                         idmac_read_icreg(ipu, IDMAC_CHA_BUSY),
1126                                         idmac_read_ipureg(ipu, IPU_TASKS_STAT));
1127                                 break;
1128                         }
1129                 }
1130                 dev_dbg(ipu->dev, "timeout = %d * 10ms\n", 40 - timeout);
1131         }
1132         /* SDC BG and FG must be disabled before DMA is disabled */
1133         if (wait_for_stop && (channel == IDMAC_SDC_0 ||
1134                               channel == IDMAC_SDC_1)) {
1135                 for (timeout = 5;
1136                      timeout && !ipu_irq_status(ichan->eof_irq); timeout--)
1137                         msleep(5);
1138         }
1139
1140         spin_lock_irqsave(&ipu->lock, flags);
1141
1142         /* Disable IC task */
1143         ipu_ic_disable_task(ipu, channel);
1144
1145         /* Disable DMA channel(s) */
1146         reg = idmac_read_icreg(ipu, IDMAC_CHA_EN);
1147         idmac_write_icreg(ipu, reg & ~chan_mask, IDMAC_CHA_EN);
1148
1149         spin_unlock_irqrestore(&ipu->lock, flags);
1150
1151         return 0;
1152 }
1153
1154 static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan,
1155         struct idmac_tx_desc **desc, struct scatterlist *sg)
1156 {
1157         struct scatterlist *sgnew = sg ? sg_next(sg) : NULL;
1158
1159         if (sgnew)
1160                 /* next sg-element in this list */
1161                 return sgnew;
1162
1163         if ((*desc)->list.next == &ichan->queue)
1164                 /* No more descriptors on the queue */
1165                 return NULL;
1166
1167         /* Fetch next descriptor */
1168         *desc = list_entry((*desc)->list.next, struct idmac_tx_desc, list);
1169         return (*desc)->sg;
1170 }
1171
1172 /*
1173  * We have several possibilities here:
1174  * current BUF          next BUF
1175  *
1176  * not last sg          next not last sg
1177  * not last sg          next last sg
1178  * last sg              first sg from next descriptor
1179  * last sg              NULL
1180  *
1181  * Besides, the descriptor queue might be empty or not. We process all these
1182  * cases carefully.
1183  */
1184 static irqreturn_t idmac_interrupt(int irq, void *dev_id)
1185 {
1186         struct idmac_channel *ichan = dev_id;
1187         struct device *dev = &ichan->dma_chan.dev->device;
1188         unsigned int chan_id = ichan->dma_chan.chan_id;
1189         struct scatterlist **sg, *sgnext, *sgnew = NULL;
1190         /* Next transfer descriptor */
1191         struct idmac_tx_desc *desc, *descnew;
1192         dma_async_tx_callback callback;
1193         void *callback_param;
1194         bool done = false;
1195         u32 ready0, ready1, curbuf, err;
1196         unsigned long flags;
1197
1198         /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
1199
1200         dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);
1201
1202         spin_lock_irqsave(&ipu_data.lock, flags);
1203
1204         ready0  = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
1205         ready1  = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
1206         curbuf  = idmac_read_ipureg(&ipu_data, IPU_CHA_CUR_BUF);
1207         err     = idmac_read_ipureg(&ipu_data, IPU_INT_STAT_4);
1208
1209         if (err & (1 << chan_id)) {
1210                 idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4);
1211                 spin_unlock_irqrestore(&ipu_data.lock, flags);
1212                 /*
1213                  * Doing this
1214                  * ichan->sg[0] = ichan->sg[1] = NULL;
1215                  * you can force channel re-enable on the next tx_submit(), but
1216                  * this is dirty - think about descriptors with multiple
1217                  * sg elements.
1218                  */
1219                 dev_warn(dev, "NFB4EOF on channel %d, ready %x, %x, cur %x\n",
1220                          chan_id, ready0, ready1, curbuf);
1221                 return IRQ_HANDLED;
1222         }
1223         spin_unlock_irqrestore(&ipu_data.lock, flags);
1224
1225         /* Other interrupts do not interfere with this channel */
1226         spin_lock(&ichan->lock);
1227         if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) ||
1228                      (!ichan->active_buffer && (ready0 >> chan_id) & 1)
1229                      )) {
1230                 spin_unlock(&ichan->lock);
1231                 dev_dbg(dev,
1232                         "IRQ with active buffer still ready on channel %x, "
1233                         "active %d, ready %x, %x!\n", chan_id,
1234                         ichan->active_buffer, ready0, ready1);
1235                 return IRQ_NONE;
1236         }
1237
1238         if (unlikely(list_empty(&ichan->queue))) {
1239                 ichan->sg[ichan->active_buffer] = NULL;
1240                 spin_unlock(&ichan->lock);
1241                 dev_err(dev,
1242                         "IRQ without queued buffers on channel %x, active %d, "
1243                         "ready %x, %x!\n", chan_id,
1244                         ichan->active_buffer, ready0, ready1);
1245                 return IRQ_NONE;
1246         }
1247
1248         /*
1249          * active_buffer is a software flag, it shows which buffer we are
1250          * currently expecting back from the hardware, IDMAC should be
1251          * processing the other buffer already
1252          */
1253         sg = &ichan->sg[ichan->active_buffer];
1254         sgnext = ichan->sg[!ichan->active_buffer];
1255
1256         if (!*sg) {
1257                 spin_unlock(&ichan->lock);
1258                 return IRQ_HANDLED;
1259         }
1260
1261         desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list);
1262         descnew = desc;
1263
1264         dev_dbg(dev, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n",
1265                 irq, sg_dma_address(*sg), sgnext ? sg_dma_address(sgnext) : 0, ichan->active_buffer, curbuf);
1266
1267         /* Find the descriptor of sgnext */
1268         sgnew = idmac_sg_next(ichan, &descnew, *sg);
1269         if (sgnext != sgnew)
1270                 dev_err(dev, "Submitted buffer %p, next buffer %p\n", sgnext, sgnew);
1271
1272         /*
1273          * if sgnext == NULL sg must be the last element in a scatterlist and
1274          * queue must be empty
1275          */
1276         if (unlikely(!sgnext)) {
1277                 if (!WARN_ON(sg_next(*sg)))
1278                         dev_dbg(dev, "Underrun on channel %x\n", chan_id);
1279                 ichan->sg[!ichan->active_buffer] = sgnew;
1280
1281                 if (unlikely(sgnew)) {
1282                         ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);
1283                 } else {
1284                         spin_lock_irqsave(&ipu_data.lock, flags);
1285                         ipu_ic_disable_task(&ipu_data, chan_id);
1286                         spin_unlock_irqrestore(&ipu_data.lock, flags);
1287                         ichan->status = IPU_CHANNEL_READY;
1288                         /* Continue to check for complete descriptor */
1289                 }
1290         }
1291
1292         /* Calculate and submit the next sg element */
1293         sgnew = idmac_sg_next(ichan, &descnew, sgnew);
1294
1295         if (unlikely(!sg_next(*sg)) || !sgnext) {
1296                 /*
1297                  * Last element in scatterlist done, remove from the queue,
1298                  * _init for debugging
1299                  */
1300                 list_del_init(&desc->list);
1301                 done = true;
1302         }
1303
1304         *sg = sgnew;
1305
1306         if (likely(sgnew) &&
1307             ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {
1308                 callback = descnew->txd.callback;
1309                 callback_param = descnew->txd.callback_param;
1310                 spin_unlock(&ichan->lock);
1311                 if (callback)
1312                         callback(callback_param);
1313                 spin_lock(&ichan->lock);
1314         }
1315
1316         /* Flip the active buffer - even if update above failed */
1317         ichan->active_buffer = !ichan->active_buffer;
1318         if (done)
1319                 ichan->completed = desc->txd.cookie;
1320
1321         callback = desc->txd.callback;
1322         callback_param = desc->txd.callback_param;
1323
1324         spin_unlock(&ichan->lock);
1325
1326         if (done && (desc->txd.flags & DMA_PREP_INTERRUPT) && callback)
1327                 callback(callback_param);
1328
1329         return IRQ_HANDLED;
1330 }
1331
1332 static void ipu_gc_tasklet(unsigned long arg)
1333 {
1334         struct ipu *ipu = (struct ipu *)arg;
1335         int i;
1336
1337         for (i = 0; i < IPU_CHANNELS_NUM; i++) {
1338                 struct idmac_channel *ichan = ipu->channel + i;
1339                 struct idmac_tx_desc *desc;
1340                 unsigned long flags;
1341                 struct scatterlist *sg;
1342                 int j, k;
1343
1344                 for (j = 0; j < ichan->n_tx_desc; j++) {
1345                         desc = ichan->desc + j;
1346                         spin_lock_irqsave(&ichan->lock, flags);
1347                         if (async_tx_test_ack(&desc->txd)) {
1348                                 list_move(&desc->list, &ichan->free_list);
1349                                 for_each_sg(desc->sg, sg, desc->sg_len, k) {
1350                                         if (ichan->sg[0] == sg)
1351                                                 ichan->sg[0] = NULL;
1352                                         else if (ichan->sg[1] == sg)
1353                                                 ichan->sg[1] = NULL;
1354                                 }
1355                                 async_tx_clear_ack(&desc->txd);
1356                         }
1357                         spin_unlock_irqrestore(&ichan->lock, flags);
1358                 }
1359         }
1360 }
1361
1362 /* Allocate and initialise a transfer descriptor. */
1363 static struct dma_async_tx_descriptor *idmac_prep_slave_sg(struct dma_chan *chan,
1364                 struct scatterlist *sgl, unsigned int sg_len,
1365                 enum dma_data_direction direction, unsigned long tx_flags)
1366 {
1367         struct idmac_channel *ichan = to_idmac_chan(chan);
1368         struct idmac_tx_desc *desc = NULL;
1369         struct dma_async_tx_descriptor *txd = NULL;
1370         unsigned long flags;
1371
1372         /* We only can handle these three channels so far */
1373         if (chan->chan_id != IDMAC_SDC_0 && chan->chan_id != IDMAC_SDC_1 &&
1374             chan->chan_id != IDMAC_IC_7)
1375                 return NULL;
1376
1377         if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE) {
1378                 dev_err(chan->device->dev, "Invalid DMA direction %d!\n", direction);
1379                 return NULL;
1380         }
1381
1382         mutex_lock(&ichan->chan_mutex);
1383
1384         spin_lock_irqsave(&ichan->lock, flags);
1385         if (!list_empty(&ichan->free_list)) {
1386                 desc = list_entry(ichan->free_list.next,
1387                                   struct idmac_tx_desc, list);
1388
1389                 list_del_init(&desc->list);
1390
1391                 desc->sg_len    = sg_len;
1392                 desc->sg        = sgl;
1393                 txd             = &desc->txd;
1394                 txd->flags      = tx_flags;
1395         }
1396         spin_unlock_irqrestore(&ichan->lock, flags);
1397
1398         mutex_unlock(&ichan->chan_mutex);
1399
1400         tasklet_schedule(&to_ipu(to_idmac(chan->device))->tasklet);
1401
1402         return txd;
1403 }
1404
1405 /* Re-select the current buffer and re-activate the channel */
1406 static void idmac_issue_pending(struct dma_chan *chan)
1407 {
1408         struct idmac_channel *ichan = to_idmac_chan(chan);
1409         struct idmac *idmac = to_idmac(chan->device);
1410         struct ipu *ipu = to_ipu(idmac);
1411         unsigned long flags;
1412
1413         /* This is not always needed, but doesn't hurt either */
1414         spin_lock_irqsave(&ipu->lock, flags);
1415         ipu_select_buffer(chan->chan_id, ichan->active_buffer);
1416         spin_unlock_irqrestore(&ipu->lock, flags);
1417
1418         /*
1419          * Might need to perform some parts of initialisation from
1420          * ipu_enable_channel(), but not all, we do not want to reset to buffer
1421          * 0, don't need to set priority again either, but re-enabling the task
1422          * and the channel might be a good idea.
1423          */
1424 }
1425
1426 static int __idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1427                            unsigned long arg)
1428 {
1429         struct idmac_channel *ichan = to_idmac_chan(chan);
1430         struct idmac *idmac = to_idmac(chan->device);
1431         unsigned long flags;
1432         int i;
1433
1434         /* Only supports DMA_TERMINATE_ALL */
1435         if (cmd != DMA_TERMINATE_ALL)
1436                 return -ENXIO;
1437
1438         ipu_disable_channel(idmac, ichan,
1439                             ichan->status >= IPU_CHANNEL_ENABLED);
1440
1441         tasklet_disable(&to_ipu(idmac)->tasklet);
1442
1443         /* ichan->queue is modified in ISR, have to spinlock */
1444         spin_lock_irqsave(&ichan->lock, flags);
1445         list_splice_init(&ichan->queue, &ichan->free_list);
1446
1447         if (ichan->desc)
1448                 for (i = 0; i < ichan->n_tx_desc; i++) {
1449                         struct idmac_tx_desc *desc = ichan->desc + i;
1450                         if (list_empty(&desc->list))
1451                                 /* Descriptor was prepared, but not submitted */
1452                                 list_add(&desc->list, &ichan->free_list);
1453
1454                         async_tx_clear_ack(&desc->txd);
1455                 }
1456
1457         ichan->sg[0] = NULL;
1458         ichan->sg[1] = NULL;
1459         spin_unlock_irqrestore(&ichan->lock, flags);
1460
1461         tasklet_enable(&to_ipu(idmac)->tasklet);
1462
1463         ichan->status = IPU_CHANNEL_INITIALIZED;
1464
1465         return 0;
1466 }
1467
1468 static int idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1469                          unsigned long arg)
1470 {
1471         struct idmac_channel *ichan = to_idmac_chan(chan);
1472         int ret;
1473
1474         mutex_lock(&ichan->chan_mutex);
1475
1476         ret = __idmac_control(chan, cmd, arg);
1477
1478         mutex_unlock(&ichan->chan_mutex);
1479
1480         return ret;
1481 }
1482
1483 #ifdef DEBUG
1484 static irqreturn_t ic_sof_irq(int irq, void *dev_id)
1485 {
1486         struct idmac_channel *ichan = dev_id;
1487         printk(KERN_DEBUG "Got SOF IRQ %d on Channel %d\n",
1488                irq, ichan->dma_chan.chan_id);
1489         disable_irq_nosync(irq);
1490         return IRQ_HANDLED;
1491 }
1492
1493 static irqreturn_t ic_eof_irq(int irq, void *dev_id)
1494 {
1495         struct idmac_channel *ichan = dev_id;
1496         printk(KERN_DEBUG "Got EOF IRQ %d on Channel %d\n",
1497                irq, ichan->dma_chan.chan_id);
1498         disable_irq_nosync(irq);
1499         return IRQ_HANDLED;
1500 }
1501
1502 static int ic_sof = -EINVAL, ic_eof = -EINVAL;
1503 #endif
1504
1505 static int idmac_alloc_chan_resources(struct dma_chan *chan)
1506 {
1507         struct idmac_channel *ichan = to_idmac_chan(chan);
1508         struct idmac *idmac = to_idmac(chan->device);
1509         int ret;
1510
1511         /* dmaengine.c now guarantees to only offer free channels */
1512         BUG_ON(chan->client_count > 1);
1513         WARN_ON(ichan->status != IPU_CHANNEL_FREE);
1514
1515         chan->cookie            = 1;
1516         ichan->completed        = -ENXIO;
1517
1518         ret = ipu_irq_map(chan->chan_id);
1519         if (ret < 0)
1520                 goto eimap;
1521
1522         ichan->eof_irq = ret;
1523
1524         /*
1525          * Important to first disable the channel, because maybe someone
1526          * used it before us, e.g., the bootloader
1527          */
1528         ipu_disable_channel(idmac, ichan, true);
1529
1530         ret = ipu_init_channel(idmac, ichan);
1531         if (ret < 0)
1532                 goto eichan;
1533
1534         ret = request_irq(ichan->eof_irq, idmac_interrupt, 0,
1535                           ichan->eof_name, ichan);
1536         if (ret < 0)
1537                 goto erirq;
1538
1539 #ifdef DEBUG
1540         if (chan->chan_id == IDMAC_IC_7) {
1541                 ic_sof = ipu_irq_map(69);
1542                 if (ic_sof > 0)
1543                         request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan);
1544                 ic_eof = ipu_irq_map(70);
1545                 if (ic_eof > 0)
1546                         request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan);
1547         }
1548 #endif
1549
1550         ichan->status = IPU_CHANNEL_INITIALIZED;
1551
1552         dev_dbg(&chan->dev->device, "Found channel 0x%x, irq %d\n",
1553                 chan->chan_id, ichan->eof_irq);
1554
1555         return ret;
1556
1557 erirq:
1558         ipu_uninit_channel(idmac, ichan);
1559 eichan:
1560         ipu_irq_unmap(chan->chan_id);
1561 eimap:
1562         return ret;
1563 }
1564
1565 static void idmac_free_chan_resources(struct dma_chan *chan)
1566 {
1567         struct idmac_channel *ichan = to_idmac_chan(chan);
1568         struct idmac *idmac = to_idmac(chan->device);
1569
1570         mutex_lock(&ichan->chan_mutex);
1571
1572         __idmac_control(chan, DMA_TERMINATE_ALL, 0);
1573
1574         if (ichan->status > IPU_CHANNEL_FREE) {
1575 #ifdef DEBUG
1576                 if (chan->chan_id == IDMAC_IC_7) {
1577                         if (ic_sof > 0) {
1578                                 free_irq(ic_sof, ichan);
1579                                 ipu_irq_unmap(69);
1580                                 ic_sof = -EINVAL;
1581                         }
1582                         if (ic_eof > 0) {
1583                                 free_irq(ic_eof, ichan);
1584                                 ipu_irq_unmap(70);
1585                                 ic_eof = -EINVAL;
1586                         }
1587                 }
1588 #endif
1589                 free_irq(ichan->eof_irq, ichan);
1590                 ipu_irq_unmap(chan->chan_id);
1591         }
1592
1593         ichan->status = IPU_CHANNEL_FREE;
1594
1595         ipu_uninit_channel(idmac, ichan);
1596
1597         mutex_unlock(&ichan->chan_mutex);
1598
1599         tasklet_schedule(&to_ipu(idmac)->tasklet);
1600 }
1601
1602 static enum dma_status idmac_tx_status(struct dma_chan *chan,
1603                        dma_cookie_t cookie, struct dma_tx_state *txstate)
1604 {
1605         struct idmac_channel *ichan = to_idmac_chan(chan);
1606
1607         dma_set_tx_state(txstate, ichan->completed, chan->cookie, 0);
1608         if (cookie != chan->cookie)
1609                 return DMA_ERROR;
1610         return DMA_SUCCESS;
1611 }
1612
1613 static int __init ipu_idmac_init(struct ipu *ipu)
1614 {
1615         struct idmac *idmac = &ipu->idmac;
1616         struct dma_device *dma = &idmac->dma;
1617         int i;
1618
1619         dma_cap_set(DMA_SLAVE, dma->cap_mask);
1620         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
1621
1622         /* Compulsory common fields */
1623         dma->dev                                = ipu->dev;
1624         dma->device_alloc_chan_resources        = idmac_alloc_chan_resources;
1625         dma->device_free_chan_resources         = idmac_free_chan_resources;
1626         dma->device_tx_status                   = idmac_tx_status;
1627         dma->device_issue_pending               = idmac_issue_pending;
1628
1629         /* Compulsory for DMA_SLAVE fields */
1630         dma->device_prep_slave_sg               = idmac_prep_slave_sg;
1631         dma->device_control                     = idmac_control;
1632
1633         INIT_LIST_HEAD(&dma->channels);
1634         for (i = 0; i < IPU_CHANNELS_NUM; i++) {
1635                 struct idmac_channel *ichan = ipu->channel + i;
1636                 struct dma_chan *dma_chan = &ichan->dma_chan;
1637
1638                 spin_lock_init(&ichan->lock);
1639                 mutex_init(&ichan->chan_mutex);
1640
1641                 ichan->status           = IPU_CHANNEL_FREE;
1642                 ichan->sec_chan_en      = false;
1643                 ichan->completed        = -ENXIO;
1644                 snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i);
1645
1646                 dma_chan->device        = &idmac->dma;
1647                 dma_chan->cookie        = 1;
1648                 dma_chan->chan_id       = i;
1649                 list_add_tail(&dma_chan->device_node, &dma->channels);
1650         }
1651
1652         idmac_write_icreg(ipu, 0x00000070, IDMAC_CONF);
1653
1654         return dma_async_device_register(&idmac->dma);
1655 }
1656
1657 static void __exit ipu_idmac_exit(struct ipu *ipu)
1658 {
1659         int i;
1660         struct idmac *idmac = &ipu->idmac;
1661
1662         for (i = 0; i < IPU_CHANNELS_NUM; i++) {
1663                 struct idmac_channel *ichan = ipu->channel + i;
1664
1665                 idmac_control(&ichan->dma_chan, DMA_TERMINATE_ALL, 0);
1666                 idmac_prep_slave_sg(&ichan->dma_chan, NULL, 0, DMA_NONE, 0);
1667         }
1668
1669         dma_async_device_unregister(&idmac->dma);
1670 }
1671
1672 /*****************************************************************************
1673  * IPU common probe / remove
1674  */
1675
1676 static int __init ipu_probe(struct platform_device *pdev)
1677 {
1678         struct ipu_platform_data *pdata = pdev->dev.platform_data;
1679         struct resource *mem_ipu, *mem_ic;
1680         int ret;
1681
1682         spin_lock_init(&ipu_data.lock);
1683
1684         mem_ipu = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1685         mem_ic  = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1686         if (!pdata || !mem_ipu || !mem_ic)
1687                 return -EINVAL;
1688
1689         ipu_data.dev = &pdev->dev;
1690
1691         platform_set_drvdata(pdev, &ipu_data);
1692
1693         ret = platform_get_irq(pdev, 0);
1694         if (ret < 0)
1695                 goto err_noirq;
1696
1697         ipu_data.irq_fn = ret;
1698         ret = platform_get_irq(pdev, 1);
1699         if (ret < 0)
1700                 goto err_noirq;
1701
1702         ipu_data.irq_err = ret;
1703         ipu_data.irq_base = pdata->irq_base;
1704
1705         dev_dbg(&pdev->dev, "fn irq %u, err irq %u, irq-base %u\n",
1706                 ipu_data.irq_fn, ipu_data.irq_err, ipu_data.irq_base);
1707
1708         /* Remap IPU common registers */
1709         ipu_data.reg_ipu = ioremap(mem_ipu->start, resource_size(mem_ipu));
1710         if (!ipu_data.reg_ipu) {
1711                 ret = -ENOMEM;
1712                 goto err_ioremap_ipu;
1713         }
1714
1715         /* Remap Image Converter and Image DMA Controller registers */
1716         ipu_data.reg_ic = ioremap(mem_ic->start, resource_size(mem_ic));
1717         if (!ipu_data.reg_ic) {
1718                 ret = -ENOMEM;
1719                 goto err_ioremap_ic;
1720         }
1721
1722         /* Get IPU clock */
1723         ipu_data.ipu_clk = clk_get(&pdev->dev, NULL);
1724         if (IS_ERR(ipu_data.ipu_clk)) {
1725                 ret = PTR_ERR(ipu_data.ipu_clk);
1726                 goto err_clk_get;
1727         }
1728
1729         /* Make sure IPU HSP clock is running */
1730         clk_enable(ipu_data.ipu_clk);
1731
1732         /* Disable all interrupts */
1733         idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_1);
1734         idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_2);
1735         idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_3);
1736         idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_4);
1737         idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_5);
1738
1739         dev_dbg(&pdev->dev, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev->name,
1740                 (unsigned long)mem_ipu->start, ipu_data.irq_fn, ipu_data.irq_err);
1741
1742         ret = ipu_irq_attach_irq(&ipu_data, pdev);
1743         if (ret < 0)
1744                 goto err_attach_irq;
1745
1746         /* Initialize DMA engine */
1747         ret = ipu_idmac_init(&ipu_data);
1748         if (ret < 0)
1749                 goto err_idmac_init;
1750
1751         tasklet_init(&ipu_data.tasklet, ipu_gc_tasklet, (unsigned long)&ipu_data);
1752
1753         ipu_data.dev = &pdev->dev;
1754
1755         dev_dbg(ipu_data.dev, "IPU initialized\n");
1756
1757         return 0;
1758
1759 err_idmac_init:
1760 err_attach_irq:
1761         ipu_irq_detach_irq(&ipu_data, pdev);
1762         clk_disable(ipu_data.ipu_clk);
1763         clk_put(ipu_data.ipu_clk);
1764 err_clk_get:
1765         iounmap(ipu_data.reg_ic);
1766 err_ioremap_ic:
1767         iounmap(ipu_data.reg_ipu);
1768 err_ioremap_ipu:
1769 err_noirq:
1770         dev_err(&pdev->dev, "Failed to probe IPU: %d\n", ret);
1771         return ret;
1772 }
1773
1774 static int __exit ipu_remove(struct platform_device *pdev)
1775 {
1776         struct ipu *ipu = platform_get_drvdata(pdev);
1777
1778         ipu_idmac_exit(ipu);
1779         ipu_irq_detach_irq(ipu, pdev);
1780         clk_disable(ipu->ipu_clk);
1781         clk_put(ipu->ipu_clk);
1782         iounmap(ipu->reg_ic);
1783         iounmap(ipu->reg_ipu);
1784         tasklet_kill(&ipu->tasklet);
1785         platform_set_drvdata(pdev, NULL);
1786
1787         return 0;
1788 }
1789
1790 /*
1791  * We need two MEM resources - with IPU-common and Image Converter registers,
1792  * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
1793  */
1794 static struct platform_driver ipu_platform_driver = {
1795         .driver = {
1796                 .name   = "ipu-core",
1797                 .owner  = THIS_MODULE,
1798         },
1799         .remove         = __exit_p(ipu_remove),
1800 };
1801
1802 static int __init ipu_init(void)
1803 {
1804         return platform_driver_probe(&ipu_platform_driver, ipu_probe);
1805 }
1806 subsys_initcall(ipu_init);
1807
1808 MODULE_DESCRIPTION("IPU core driver");
1809 MODULE_LICENSE("GPL v2");
1810 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
1811 MODULE_ALIAS("platform:ipu-core");