Blackfin arch: Functional power management support
[linux-2.6.git] / arch / blackfin / kernel / bfin_dma_5xx.c
1 /*
2  * File:         arch/blackfin/kernel/bfin_dma_5xx.c
3  * Based on:
4  * Author:
5  *
6  * Created:
7  * Description:  This file contains the simple DMA Implementation for Blackfin
8  *
9  * Modified:
10  *               Copyright 2004-2006 Analog Devices Inc.
11  *
12  * Bugs:         Enter bugs at http://blackfin.uclinux.org/
13  *
14  * This program is free software; you can redistribute it and/or modify
15  * it under the terms of the GNU General Public License as published by
16  * the Free Software Foundation; either version 2 of the License, or
17  * (at your option) any later version.
18  *
19  * This program is distributed in the hope that it will be useful,
20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
22  * GNU General Public License for more details.
23  *
24  * You should have received a copy of the GNU General Public License
25  * along with this program; if not, see the file COPYING, or write
26  * to the Free Software Foundation, Inc.,
27  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
28  */
29
30 #include <linux/errno.h>
31 #include <linux/module.h>
32 #include <linux/sched.h>
33 #include <linux/interrupt.h>
34 #include <linux/kernel.h>
35 #include <linux/param.h>
36
37 #include <asm/blackfin.h>
38 #include <asm/dma.h>
39 #include <asm/cacheflush.h>
40
41 /* Remove unused code not exported by symbol or internally called */
42 #define REMOVE_DEAD_CODE
43
44 /**************************************************************************
45  * Global Variables
46 ***************************************************************************/
47
48 static struct dma_channel dma_ch[MAX_BLACKFIN_DMA_CHANNEL];
49
50 /*------------------------------------------------------------------------------
51  *       Set the Buffer Clear bit in the Configuration register of specific DMA
52  *       channel. This will stop the descriptor based DMA operation.
53  *-----------------------------------------------------------------------------*/
54 static void clear_dma_buffer(unsigned int channel)
55 {
56         dma_ch[channel].regs->cfg |= RESTART;
57         SSYNC();
58         dma_ch[channel].regs->cfg &= ~RESTART;
59         SSYNC();
60 }
61
62 static int __init blackfin_dma_init(void)
63 {
64         int i;
65
66         printk(KERN_INFO "Blackfin DMA Controller\n");
67
68         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
69                 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
70                 dma_ch[i].regs = dma_io_base_addr[i];
71                 mutex_init(&(dma_ch[i].dmalock));
72         }
73         /* Mark MEMDMA Channel 0 as requested since we're using it internally */
74         dma_ch[CH_MEM_STREAM0_DEST].chan_status = DMA_CHANNEL_REQUESTED;
75         dma_ch[CH_MEM_STREAM0_SRC].chan_status = DMA_CHANNEL_REQUESTED;
76
77 #if defined(CONFIG_DEB_DMA_URGENT)
78         bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
79                          | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
80 #endif
81         return 0;
82 }
83
84 arch_initcall(blackfin_dma_init);
85
86 /*------------------------------------------------------------------------------
87  *      Request the specific DMA channel from the system.
88  *-----------------------------------------------------------------------------*/
89 int request_dma(unsigned int channel, char *device_id)
90 {
91
92         pr_debug("request_dma() : BEGIN \n");
93
94 #if defined(CONFIG_BF561) && ANOMALY_05000182
95         if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
96                 if (get_cclk() > 500000000) {
97                         printk(KERN_WARNING
98                                "Request IMDMA failed due to ANOMALY 05000182\n");
99                         return -EFAULT;
100                 }
101         }
102 #endif
103
104         mutex_lock(&(dma_ch[channel].dmalock));
105
106         if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
107             || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
108                 mutex_unlock(&(dma_ch[channel].dmalock));
109                 pr_debug("DMA CHANNEL IN USE  \n");
110                 return -EBUSY;
111         } else {
112                 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
113                 pr_debug("DMA CHANNEL IS ALLOCATED  \n");
114         }
115
116         mutex_unlock(&(dma_ch[channel].dmalock));
117
118 #ifdef CONFIG_BF54x
119         if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
120                 if (strncmp(device_id, "BFIN_UART", 9) == 0) {
121                         dma_ch[channel].regs->peripheral_map &= 0x0FFF;
122                         dma_ch[channel].regs->peripheral_map |=
123                                 ((channel - CH_UART2_RX + 0xC)<<12);
124                 } else {
125                         dma_ch[channel].regs->peripheral_map &= 0x0FFF;
126                         dma_ch[channel].regs->peripheral_map |=
127                                 ((channel - CH_UART2_RX + 0x6)<<12);
128                 }
129         }
130 #endif
131
132         dma_ch[channel].device_id = device_id;
133         dma_ch[channel].irq_callback = NULL;
134
135         /* This is to be enabled by putting a restriction -
136          * you have to request DMA, before doing any operations on
137          * descriptor/channel
138          */
139         pr_debug("request_dma() : END  \n");
140         return channel;
141 }
142 EXPORT_SYMBOL(request_dma);
143
144 int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
145 {
146         int ret_irq = 0;
147
148         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
149                && channel < MAX_BLACKFIN_DMA_CHANNEL));
150
151         if (callback != NULL) {
152                 int ret_val;
153                 ret_irq = channel2irq(channel);
154
155                 dma_ch[channel].data = data;
156
157                 ret_val =
158                     request_irq(ret_irq, (void *)callback, IRQF_DISABLED,
159                                 dma_ch[channel].device_id, data);
160                 if (ret_val) {
161                         printk(KERN_NOTICE
162                                "Request irq in DMA engine failed.\n");
163                         return -EPERM;
164                 }
165                 dma_ch[channel].irq_callback = callback;
166         }
167         return 0;
168 }
169 EXPORT_SYMBOL(set_dma_callback);
170
171 void free_dma(unsigned int channel)
172 {
173         int ret_irq;
174
175         pr_debug("freedma() : BEGIN \n");
176         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
177                && channel < MAX_BLACKFIN_DMA_CHANNEL));
178
179         /* Halt the DMA */
180         disable_dma(channel);
181         clear_dma_buffer(channel);
182
183         if (dma_ch[channel].irq_callback != NULL) {
184                 ret_irq = channel2irq(channel);
185                 free_irq(ret_irq, dma_ch[channel].data);
186         }
187
188         /* Clear the DMA Variable in the Channel */
189         mutex_lock(&(dma_ch[channel].dmalock));
190         dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
191         mutex_unlock(&(dma_ch[channel].dmalock));
192
193         pr_debug("freedma() : END \n");
194 }
195 EXPORT_SYMBOL(free_dma);
196
197 void dma_enable_irq(unsigned int channel)
198 {
199         int ret_irq;
200
201         pr_debug("dma_enable_irq() : BEGIN \n");
202         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
203                && channel < MAX_BLACKFIN_DMA_CHANNEL));
204
205         ret_irq = channel2irq(channel);
206         enable_irq(ret_irq);
207 }
208 EXPORT_SYMBOL(dma_enable_irq);
209
210 void dma_disable_irq(unsigned int channel)
211 {
212         int ret_irq;
213
214         pr_debug("dma_disable_irq() : BEGIN \n");
215         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
216                && channel < MAX_BLACKFIN_DMA_CHANNEL));
217
218         ret_irq = channel2irq(channel);
219         disable_irq(ret_irq);
220 }
221 EXPORT_SYMBOL(dma_disable_irq);
222
223 int dma_channel_active(unsigned int channel)
224 {
225         if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
226                 return 0;
227         } else {
228                 return 1;
229         }
230 }
231 EXPORT_SYMBOL(dma_channel_active);
232
233 /*------------------------------------------------------------------------------
234 *       stop the specific DMA channel.
235 *-----------------------------------------------------------------------------*/
236 void disable_dma(unsigned int channel)
237 {
238         pr_debug("stop_dma() : BEGIN \n");
239
240         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
241                && channel < MAX_BLACKFIN_DMA_CHANNEL));
242
243         dma_ch[channel].regs->cfg &= ~DMAEN;    /* Clean the enable bit */
244         SSYNC();
245         dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
246         /* Needs to be enabled Later */
247         pr_debug("stop_dma() : END \n");
248         return;
249 }
250 EXPORT_SYMBOL(disable_dma);
251
252 void enable_dma(unsigned int channel)
253 {
254         pr_debug("enable_dma() : BEGIN \n");
255
256         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
257                && channel < MAX_BLACKFIN_DMA_CHANNEL));
258
259         dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
260         dma_ch[channel].regs->curr_x_count = 0;
261         dma_ch[channel].regs->curr_y_count = 0;
262
263         dma_ch[channel].regs->cfg |= DMAEN;     /* Set the enable bit */
264         SSYNC();
265         pr_debug("enable_dma() : END \n");
266         return;
267 }
268 EXPORT_SYMBOL(enable_dma);
269
270 /*------------------------------------------------------------------------------
271 *               Set the Start Address register for the specific DMA channel
272 *               This function can be used for register based DMA,
273 *               to setup the start address
274 *               addr:           Starting address of the DMA Data to be transferred.
275 *-----------------------------------------------------------------------------*/
276 void set_dma_start_addr(unsigned int channel, unsigned long addr)
277 {
278         pr_debug("set_dma_start_addr() : BEGIN \n");
279
280         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
281                && channel < MAX_BLACKFIN_DMA_CHANNEL));
282
283         dma_ch[channel].regs->start_addr = addr;
284         SSYNC();
285         pr_debug("set_dma_start_addr() : END\n");
286 }
287 EXPORT_SYMBOL(set_dma_start_addr);
288
289 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
290 {
291         pr_debug("set_dma_next_desc_addr() : BEGIN \n");
292
293         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
294                && channel < MAX_BLACKFIN_DMA_CHANNEL));
295
296         dma_ch[channel].regs->next_desc_ptr = addr;
297         SSYNC();
298         pr_debug("set_dma_next_desc_addr() : END\n");
299 }
300 EXPORT_SYMBOL(set_dma_next_desc_addr);
301
302 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
303 {
304         pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
305
306         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
307                && channel < MAX_BLACKFIN_DMA_CHANNEL));
308
309         dma_ch[channel].regs->curr_desc_ptr = addr;
310         SSYNC();
311         pr_debug("set_dma_curr_desc_addr() : END\n");
312 }
313 EXPORT_SYMBOL(set_dma_curr_desc_addr);
314
315 void set_dma_x_count(unsigned int channel, unsigned short x_count)
316 {
317         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
318                && channel < MAX_BLACKFIN_DMA_CHANNEL));
319
320         dma_ch[channel].regs->x_count = x_count;
321         SSYNC();
322 }
323 EXPORT_SYMBOL(set_dma_x_count);
324
325 void set_dma_y_count(unsigned int channel, unsigned short y_count)
326 {
327         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
328                && channel < MAX_BLACKFIN_DMA_CHANNEL));
329
330         dma_ch[channel].regs->y_count = y_count;
331         SSYNC();
332 }
333 EXPORT_SYMBOL(set_dma_y_count);
334
335 void set_dma_x_modify(unsigned int channel, short x_modify)
336 {
337         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
338                && channel < MAX_BLACKFIN_DMA_CHANNEL));
339
340         dma_ch[channel].regs->x_modify = x_modify;
341         SSYNC();
342 }
343 EXPORT_SYMBOL(set_dma_x_modify);
344
345 void set_dma_y_modify(unsigned int channel, short y_modify)
346 {
347         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
348                && channel < MAX_BLACKFIN_DMA_CHANNEL));
349
350         dma_ch[channel].regs->y_modify = y_modify;
351         SSYNC();
352 }
353 EXPORT_SYMBOL(set_dma_y_modify);
354
355 void set_dma_config(unsigned int channel, unsigned short config)
356 {
357         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
358                && channel < MAX_BLACKFIN_DMA_CHANNEL));
359
360         dma_ch[channel].regs->cfg = config;
361         SSYNC();
362 }
363 EXPORT_SYMBOL(set_dma_config);
364
365 unsigned short
366 set_bfin_dma_config(char direction, char flow_mode,
367                     char intr_mode, char dma_mode, char width, char syncmode)
368 {
369         unsigned short config;
370
371         config =
372             ((direction << 1) | (width << 2) | (dma_mode << 4) |
373              (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
374         return config;
375 }
376 EXPORT_SYMBOL(set_bfin_dma_config);
377
378 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
379 {
380         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
381                && channel < MAX_BLACKFIN_DMA_CHANNEL));
382
383         dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
384
385         dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
386
387         SSYNC();
388 }
389 EXPORT_SYMBOL(set_dma_sg);
390
391 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
392 {
393         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
394                && channel < MAX_BLACKFIN_DMA_CHANNEL));
395
396         dma_ch[channel].regs->curr_addr_ptr = addr;
397         SSYNC();
398 }
399 EXPORT_SYMBOL(set_dma_curr_addr);
400
401 /*------------------------------------------------------------------------------
402  *      Get the DMA status of a specific DMA channel from the system.
403  *-----------------------------------------------------------------------------*/
404 unsigned short get_dma_curr_irqstat(unsigned int channel)
405 {
406         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
407                && channel < MAX_BLACKFIN_DMA_CHANNEL));
408
409         return dma_ch[channel].regs->irq_status;
410 }
411 EXPORT_SYMBOL(get_dma_curr_irqstat);
412
413 /*------------------------------------------------------------------------------
414  *      Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
415  *-----------------------------------------------------------------------------*/
416 void clear_dma_irqstat(unsigned int channel)
417 {
418         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
419                && channel < MAX_BLACKFIN_DMA_CHANNEL));
420         dma_ch[channel].regs->irq_status |= 3;
421 }
422 EXPORT_SYMBOL(clear_dma_irqstat);
423
424 /*------------------------------------------------------------------------------
425  *      Get current DMA xcount of a specific DMA channel from the system.
426  *-----------------------------------------------------------------------------*/
427 unsigned short get_dma_curr_xcount(unsigned int channel)
428 {
429         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
430                && channel < MAX_BLACKFIN_DMA_CHANNEL));
431
432         return dma_ch[channel].regs->curr_x_count;
433 }
434 EXPORT_SYMBOL(get_dma_curr_xcount);
435
436 /*------------------------------------------------------------------------------
437  *      Get current DMA ycount of a specific DMA channel from the system.
438  *-----------------------------------------------------------------------------*/
439 unsigned short get_dma_curr_ycount(unsigned int channel)
440 {
441         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
442                && channel < MAX_BLACKFIN_DMA_CHANNEL));
443
444         return dma_ch[channel].regs->curr_y_count;
445 }
446 EXPORT_SYMBOL(get_dma_curr_ycount);
447
448 unsigned long get_dma_next_desc_ptr(unsigned int channel)
449 {
450         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
451               && channel < MAX_BLACKFIN_DMA_CHANNEL));
452
453         return dma_ch[channel].regs->next_desc_ptr;
454 }
455 EXPORT_SYMBOL(get_dma_next_desc_ptr);
456
457 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
458 {
459         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
460               && channel < MAX_BLACKFIN_DMA_CHANNEL));
461
462         return dma_ch[channel].regs->curr_desc_ptr;
463 }
464 EXPORT_SYMBOL(get_dma_curr_desc_ptr);
465
466 unsigned long get_dma_curr_addr(unsigned int channel)
467 {
468         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
469               && channel < MAX_BLACKFIN_DMA_CHANNEL));
470
471         return dma_ch[channel].regs->curr_addr_ptr;
472 }
473 EXPORT_SYMBOL(get_dma_curr_addr);
474
475 #ifdef CONFIG_PM
476 int blackfin_dma_suspend(void)
477 {
478         int i;
479
480         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
481                 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
482                         printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
483                         return -EBUSY;
484                 }
485
486                 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
487         }
488
489         return 0;
490 }
491
492 void blackfin_dma_resume(void)
493 {
494         int i;
495
496         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++)
497                 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
498 }
499 #endif
500
501 static void *__dma_memcpy(void *dest, const void *src, size_t size)
502 {
503         int direction;  /* 1 - address decrease, 0 - address increase */
504         int flag_align; /* 1 - address aligned,  0 - address unaligned */
505         int flag_2D;    /* 1 - 2D DMA needed,    0 - 1D DMA needed */
506         unsigned long flags;
507
508         if (size <= 0)
509                 return NULL;
510
511         local_irq_save(flags);
512
513         if ((unsigned long)src < memory_end)
514                 blackfin_dcache_flush_range((unsigned int)src,
515                                             (unsigned int)(src + size));
516
517         if ((unsigned long)dest < memory_end)
518                 blackfin_dcache_invalidate_range((unsigned int)dest,
519                                                  (unsigned int)(dest + size));
520
521         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
522
523         if ((unsigned long)src < (unsigned long)dest)
524                 direction = 1;
525         else
526                 direction = 0;
527
528         if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
529             && ((size % 2) == 0))
530                 flag_align = 1;
531         else
532                 flag_align = 0;
533
534         if (size > 0x10000)     /* size > 64K */
535                 flag_2D = 1;
536         else
537                 flag_2D = 0;
538
539         /* Setup destination and source start address */
540         if (direction) {
541                 if (flag_align) {
542                         bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
543                         bfin_write_MDMA_S0_START_ADDR(src + size - 2);
544                 } else {
545                         bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
546                         bfin_write_MDMA_S0_START_ADDR(src + size - 1);
547                 }
548         } else {
549                 bfin_write_MDMA_D0_START_ADDR(dest);
550                 bfin_write_MDMA_S0_START_ADDR(src);
551         }
552
553         /* Setup destination and source xcount */
554         if (flag_2D) {
555                 if (flag_align) {
556                         bfin_write_MDMA_D0_X_COUNT(1024 / 2);
557                         bfin_write_MDMA_S0_X_COUNT(1024 / 2);
558                 } else {
559                         bfin_write_MDMA_D0_X_COUNT(1024);
560                         bfin_write_MDMA_S0_X_COUNT(1024);
561                 }
562                 bfin_write_MDMA_D0_Y_COUNT(size >> 10);
563                 bfin_write_MDMA_S0_Y_COUNT(size >> 10);
564         } else {
565                 if (flag_align) {
566                         bfin_write_MDMA_D0_X_COUNT(size / 2);
567                         bfin_write_MDMA_S0_X_COUNT(size / 2);
568                 } else {
569                         bfin_write_MDMA_D0_X_COUNT(size);
570                         bfin_write_MDMA_S0_X_COUNT(size);
571                 }
572         }
573
574         /* Setup destination and source xmodify and ymodify */
575         if (direction) {
576                 if (flag_align) {
577                         bfin_write_MDMA_D0_X_MODIFY(-2);
578                         bfin_write_MDMA_S0_X_MODIFY(-2);
579                         if (flag_2D) {
580                                 bfin_write_MDMA_D0_Y_MODIFY(-2);
581                                 bfin_write_MDMA_S0_Y_MODIFY(-2);
582                         }
583                 } else {
584                         bfin_write_MDMA_D0_X_MODIFY(-1);
585                         bfin_write_MDMA_S0_X_MODIFY(-1);
586                         if (flag_2D) {
587                                 bfin_write_MDMA_D0_Y_MODIFY(-1);
588                                 bfin_write_MDMA_S0_Y_MODIFY(-1);
589                         }
590                 }
591         } else {
592                 if (flag_align) {
593                         bfin_write_MDMA_D0_X_MODIFY(2);
594                         bfin_write_MDMA_S0_X_MODIFY(2);
595                         if (flag_2D) {
596                                 bfin_write_MDMA_D0_Y_MODIFY(2);
597                                 bfin_write_MDMA_S0_Y_MODIFY(2);
598                         }
599                 } else {
600                         bfin_write_MDMA_D0_X_MODIFY(1);
601                         bfin_write_MDMA_S0_X_MODIFY(1);
602                         if (flag_2D) {
603                                 bfin_write_MDMA_D0_Y_MODIFY(1);
604                                 bfin_write_MDMA_S0_Y_MODIFY(1);
605                         }
606                 }
607         }
608
609         /* Enable source DMA */
610         if (flag_2D) {
611                 if (flag_align) {
612                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D | WDSIZE_16);
613                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D | WDSIZE_16);
614                 } else {
615                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D);
616                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D);
617                 }
618         } else {
619                 if (flag_align) {
620                         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
621                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
622                 } else {
623                         bfin_write_MDMA_S0_CONFIG(DMAEN);
624                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN);
625                 }
626         }
627
628         SSYNC();
629
630         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
631                 ;
632
633         bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() |
634                                       (DMA_DONE | DMA_ERR));
635
636         bfin_write_MDMA_S0_CONFIG(0);
637         bfin_write_MDMA_D0_CONFIG(0);
638
639         local_irq_restore(flags);
640
641         return dest;
642 }
643
644 void *dma_memcpy(void *dest, const void *src, size_t size)
645 {
646         size_t bulk;
647         size_t rest;
648         void * addr;
649
650         bulk = (size >> 16) << 16;
651         rest = size - bulk;
652         if (bulk)
653                 __dma_memcpy(dest, src, bulk);
654         addr = __dma_memcpy(dest+bulk, src+bulk, rest);
655         return addr;
656 }
657 EXPORT_SYMBOL(dma_memcpy);
658
659 void *safe_dma_memcpy(void *dest, const void *src, size_t size)
660 {
661         void *addr;
662         addr = dma_memcpy(dest, src, size);
663         return addr;
664 }
665 EXPORT_SYMBOL(safe_dma_memcpy);
666
667 void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
668 {
669         unsigned long flags;
670
671         local_irq_save(flags);
672
673         blackfin_dcache_flush_range((unsigned int)buf,
674                          (unsigned int)(buf) + len);
675
676         bfin_write_MDMA_D0_START_ADDR(addr);
677         bfin_write_MDMA_D0_X_COUNT(len);
678         bfin_write_MDMA_D0_X_MODIFY(0);
679         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
680
681         bfin_write_MDMA_S0_START_ADDR(buf);
682         bfin_write_MDMA_S0_X_COUNT(len);
683         bfin_write_MDMA_S0_X_MODIFY(1);
684         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
685
686         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
687         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
688
689         SSYNC();
690
691         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
692
693         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
694
695         bfin_write_MDMA_S0_CONFIG(0);
696         bfin_write_MDMA_D0_CONFIG(0);
697         local_irq_restore(flags);
698
699 }
700 EXPORT_SYMBOL(dma_outsb);
701
702
703 void dma_insb(unsigned long addr, void *buf, unsigned short len)
704 {
705         unsigned long flags;
706
707         blackfin_dcache_invalidate_range((unsigned int)buf,
708                          (unsigned int)(buf) + len);
709
710         local_irq_save(flags);
711         bfin_write_MDMA_D0_START_ADDR(buf);
712         bfin_write_MDMA_D0_X_COUNT(len);
713         bfin_write_MDMA_D0_X_MODIFY(1);
714         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
715
716         bfin_write_MDMA_S0_START_ADDR(addr);
717         bfin_write_MDMA_S0_X_COUNT(len);
718         bfin_write_MDMA_S0_X_MODIFY(0);
719         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
720
721         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
722         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
723
724         SSYNC();
725
726         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
727
728         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
729
730         bfin_write_MDMA_S0_CONFIG(0);
731         bfin_write_MDMA_D0_CONFIG(0);
732         local_irq_restore(flags);
733
734 }
735 EXPORT_SYMBOL(dma_insb);
736
737 void dma_outsw(unsigned long addr, const void  *buf, unsigned short len)
738 {
739         unsigned long flags;
740
741         local_irq_save(flags);
742
743         blackfin_dcache_flush_range((unsigned int)buf,
744                          (unsigned int)(buf) + len * sizeof(short));
745
746         bfin_write_MDMA_D0_START_ADDR(addr);
747         bfin_write_MDMA_D0_X_COUNT(len);
748         bfin_write_MDMA_D0_X_MODIFY(0);
749         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
750
751         bfin_write_MDMA_S0_START_ADDR(buf);
752         bfin_write_MDMA_S0_X_COUNT(len);
753         bfin_write_MDMA_S0_X_MODIFY(2);
754         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
755
756         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
757         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
758
759         SSYNC();
760
761         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
762
763         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
764
765         bfin_write_MDMA_S0_CONFIG(0);
766         bfin_write_MDMA_D0_CONFIG(0);
767         local_irq_restore(flags);
768
769 }
770 EXPORT_SYMBOL(dma_outsw);
771
772 void dma_insw(unsigned long addr, void *buf, unsigned short len)
773 {
774         unsigned long flags;
775
776         blackfin_dcache_invalidate_range((unsigned int)buf,
777                          (unsigned int)(buf) + len * sizeof(short));
778
779         local_irq_save(flags);
780
781         bfin_write_MDMA_D0_START_ADDR(buf);
782         bfin_write_MDMA_D0_X_COUNT(len);
783         bfin_write_MDMA_D0_X_MODIFY(2);
784         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
785
786         bfin_write_MDMA_S0_START_ADDR(addr);
787         bfin_write_MDMA_S0_X_COUNT(len);
788         bfin_write_MDMA_S0_X_MODIFY(0);
789         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
790
791         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
792         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
793
794         SSYNC();
795
796         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
797
798         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
799
800         bfin_write_MDMA_S0_CONFIG(0);
801         bfin_write_MDMA_D0_CONFIG(0);
802         local_irq_restore(flags);
803
804 }
805 EXPORT_SYMBOL(dma_insw);
806
807 void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
808 {
809         unsigned long flags;
810
811         local_irq_save(flags);
812
813         blackfin_dcache_flush_range((unsigned int)buf,
814                          (unsigned int)(buf) + len * sizeof(long));
815
816         bfin_write_MDMA_D0_START_ADDR(addr);
817         bfin_write_MDMA_D0_X_COUNT(len);
818         bfin_write_MDMA_D0_X_MODIFY(0);
819         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
820
821         bfin_write_MDMA_S0_START_ADDR(buf);
822         bfin_write_MDMA_S0_X_COUNT(len);
823         bfin_write_MDMA_S0_X_MODIFY(4);
824         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
825
826         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
827         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
828
829         SSYNC();
830
831         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
832
833         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
834
835         bfin_write_MDMA_S0_CONFIG(0);
836         bfin_write_MDMA_D0_CONFIG(0);
837         local_irq_restore(flags);
838
839 }
840 EXPORT_SYMBOL(dma_outsl);
841
842 void dma_insl(unsigned long addr, void *buf, unsigned short len)
843 {
844         unsigned long flags;
845
846         blackfin_dcache_invalidate_range((unsigned int)buf,
847                          (unsigned int)(buf) + len * sizeof(long));
848
849         local_irq_save(flags);
850
851         bfin_write_MDMA_D0_START_ADDR(buf);
852         bfin_write_MDMA_D0_X_COUNT(len);
853         bfin_write_MDMA_D0_X_MODIFY(4);
854         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
855
856         bfin_write_MDMA_S0_START_ADDR(addr);
857         bfin_write_MDMA_S0_X_COUNT(len);
858         bfin_write_MDMA_S0_X_MODIFY(0);
859         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
860
861         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
862         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
863
864         SSYNC();
865
866         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
867
868         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
869
870         bfin_write_MDMA_S0_CONFIG(0);
871         bfin_write_MDMA_D0_CONFIG(0);
872         local_irq_restore(flags);
873
874 }
875 EXPORT_SYMBOL(dma_insl);