2 * DMA controller driver for CSR SiRFprimaII
4 * Copyright (c) 2011 Cambridge Silicon Radio Limited, a CSR plc group company.
6 * Licensed under GPLv2 or later.
9 #include <linux/module.h>
10 #include <linux/dmaengine.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/interrupt.h>
14 #include <linux/slab.h>
15 #include <linux/of_irq.h>
16 #include <linux/of_address.h>
17 #include <linux/of_device.h>
18 #include <linux/of_platform.h>
19 #include <linux/sirfsoc_dma.h>
21 #define SIRFSOC_DMA_DESCRIPTORS 16
22 #define SIRFSOC_DMA_CHANNELS 16
24 #define SIRFSOC_DMA_CH_ADDR 0x00
25 #define SIRFSOC_DMA_CH_XLEN 0x04
26 #define SIRFSOC_DMA_CH_YLEN 0x08
27 #define SIRFSOC_DMA_CH_CTRL 0x0C
29 #define SIRFSOC_DMA_WIDTH_0 0x100
30 #define SIRFSOC_DMA_CH_VALID 0x140
31 #define SIRFSOC_DMA_CH_INT 0x144
32 #define SIRFSOC_DMA_INT_EN 0x148
33 #define SIRFSOC_DMA_CH_LOOP_CTRL 0x150
35 #define SIRFSOC_DMA_MODE_CTRL_BIT 4
36 #define SIRFSOC_DMA_DIR_CTRL_BIT 5
38 /* xlen and dma_width register is in 4 bytes boundary */
39 #define SIRFSOC_DMA_WORD_LEN 4
41 struct sirfsoc_dma_desc {
42 struct dma_async_tx_descriptor desc;
43 struct list_head node;
45 /* SiRFprimaII 2D-DMA parameters */
47 int xlen; /* DMA xlen */
48 int ylen; /* DMA ylen */
49 int width; /* DMA width */
51 bool cyclic; /* is loop DMA? */
52 u32 addr; /* DMA buffer address */
55 struct sirfsoc_dma_chan {
57 struct list_head free;
58 struct list_head prepared;
59 struct list_head queued;
60 struct list_head active;
61 struct list_head completed;
62 unsigned long happened_cyclic;
63 unsigned long completed_cyclic;
65 /* Lock for this structure */
72 struct dma_device dma;
73 struct tasklet_struct tasklet;
74 struct sirfsoc_dma_chan channels[SIRFSOC_DMA_CHANNELS];
79 #define DRV_NAME "sirfsoc_dma"
81 /* Convert struct dma_chan to struct sirfsoc_dma_chan */
83 struct sirfsoc_dma_chan *dma_chan_to_sirfsoc_dma_chan(struct dma_chan *c)
85 return container_of(c, struct sirfsoc_dma_chan, chan);
88 /* Convert struct dma_chan to struct sirfsoc_dma */
89 static inline struct sirfsoc_dma *dma_chan_to_sirfsoc_dma(struct dma_chan *c)
91 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(c);
92 return container_of(schan, struct sirfsoc_dma, channels[c->chan_id]);
95 /* Execute all queued DMA descriptors */
96 static void sirfsoc_dma_execute(struct sirfsoc_dma_chan *schan)
98 struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan);
99 int cid = schan->chan.chan_id;
100 struct sirfsoc_dma_desc *sdesc = NULL;
103 * lock has been held by functions calling this, so we don't hold
107 sdesc = list_first_entry(&schan->queued, struct sirfsoc_dma_desc,
109 /* Move the first queued descriptor to active list */
110 list_move_tail(&schan->queued, &schan->active);
112 /* Start the DMA transfer */
113 writel_relaxed(sdesc->width, sdma->base + SIRFSOC_DMA_WIDTH_0 +
115 writel_relaxed(cid | (schan->mode << SIRFSOC_DMA_MODE_CTRL_BIT) |
116 (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT),
117 sdma->base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL);
118 writel_relaxed(sdesc->xlen, sdma->base + cid * 0x10 +
119 SIRFSOC_DMA_CH_XLEN);
120 writel_relaxed(sdesc->ylen, sdma->base + cid * 0x10 +
121 SIRFSOC_DMA_CH_YLEN);
122 writel_relaxed(readl_relaxed(sdma->base + SIRFSOC_DMA_INT_EN) |
123 (1 << cid), sdma->base + SIRFSOC_DMA_INT_EN);
126 * writel has an implict memory write barrier to make sure data is
127 * flushed into memory before starting DMA
129 writel(sdesc->addr >> 2, sdma->base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR);
132 writel((1 << cid) | 1 << (cid + 16) |
133 readl_relaxed(sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL),
134 sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL);
135 schan->happened_cyclic = schan->completed_cyclic = 0;
139 /* Interrupt handler */
140 static irqreturn_t sirfsoc_dma_irq(int irq, void *data)
142 struct sirfsoc_dma *sdma = data;
143 struct sirfsoc_dma_chan *schan;
144 struct sirfsoc_dma_desc *sdesc = NULL;
148 is = readl(sdma->base + SIRFSOC_DMA_CH_INT);
149 while ((ch = fls(is) - 1) >= 0) {
151 writel_relaxed(1 << ch, sdma->base + SIRFSOC_DMA_CH_INT);
152 schan = &sdma->channels[ch];
154 spin_lock(&schan->lock);
156 sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc,
158 if (!sdesc->cyclic) {
159 /* Execute queued descriptors */
160 list_splice_tail_init(&schan->active, &schan->completed);
161 if (!list_empty(&schan->queued))
162 sirfsoc_dma_execute(schan);
164 schan->happened_cyclic++;
166 spin_unlock(&schan->lock);
169 /* Schedule tasklet */
170 tasklet_schedule(&sdma->tasklet);
175 /* process completed descriptors */
176 static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma)
178 dma_cookie_t last_cookie = 0;
179 struct sirfsoc_dma_chan *schan;
180 struct sirfsoc_dma_desc *sdesc;
181 struct dma_async_tx_descriptor *desc;
183 unsigned long happened_cyclic;
187 for (i = 0; i < sdma->dma.chancnt; i++) {
188 schan = &sdma->channels[i];
190 /* Get all completed descriptors */
191 spin_lock_irqsave(&schan->lock, flags);
192 if (!list_empty(&schan->completed)) {
193 list_splice_tail_init(&schan->completed, &list);
194 spin_unlock_irqrestore(&schan->lock, flags);
196 /* Execute callbacks and run dependencies */
197 list_for_each_entry(sdesc, &list, node) {
201 desc->callback(desc->callback_param);
203 last_cookie = desc->cookie;
204 dma_run_dependencies(desc);
207 /* Free descriptors */
208 spin_lock_irqsave(&schan->lock, flags);
209 list_splice_tail_init(&list, &schan->free);
210 schan->chan.completed_cookie = last_cookie;
211 spin_unlock_irqrestore(&schan->lock, flags);
213 /* for cyclic channel, desc is always in active list */
214 sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc,
217 if (!sdesc || (sdesc && !sdesc->cyclic)) {
218 /* without active cyclic DMA */
219 spin_unlock_irqrestore(&schan->lock, flags);
224 happened_cyclic = schan->happened_cyclic;
225 spin_unlock_irqrestore(&schan->lock, flags);
228 while (happened_cyclic != schan->completed_cyclic) {
230 desc->callback(desc->callback_param);
231 schan->completed_cyclic++;
238 static void sirfsoc_dma_tasklet(unsigned long data)
240 struct sirfsoc_dma *sdma = (void *)data;
242 sirfsoc_dma_process_completed(sdma);
245 /* Submit descriptor to hardware */
246 static dma_cookie_t sirfsoc_dma_tx_submit(struct dma_async_tx_descriptor *txd)
248 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(txd->chan);
249 struct sirfsoc_dma_desc *sdesc;
253 sdesc = container_of(txd, struct sirfsoc_dma_desc, desc);
255 spin_lock_irqsave(&schan->lock, flags);
257 /* Move descriptor to queue */
258 list_move_tail(&sdesc->node, &schan->queued);
261 cookie = dma_cookie_assign(txd);
263 spin_unlock_irqrestore(&schan->lock, flags);
268 static int sirfsoc_dma_slave_config(struct sirfsoc_dma_chan *schan,
269 struct dma_slave_config *config)
273 if ((config->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) ||
274 (config->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES))
277 spin_lock_irqsave(&schan->lock, flags);
278 schan->mode = (config->src_maxburst == 4 ? 1 : 0);
279 spin_unlock_irqrestore(&schan->lock, flags);
284 static int sirfsoc_dma_terminate_all(struct sirfsoc_dma_chan *schan)
286 struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan);
287 int cid = schan->chan.chan_id;
290 writel_relaxed(readl_relaxed(sdma->base + SIRFSOC_DMA_INT_EN) &
291 ~(1 << cid), sdma->base + SIRFSOC_DMA_INT_EN);
292 writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID);
294 writel_relaxed(readl_relaxed(sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL)
295 & ~((1 << cid) | 1 << (cid + 16)),
296 sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL);
298 spin_lock_irqsave(&schan->lock, flags);
299 list_splice_tail_init(&schan->active, &schan->free);
300 list_splice_tail_init(&schan->queued, &schan->free);
301 spin_unlock_irqrestore(&schan->lock, flags);
306 static int sirfsoc_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
309 struct dma_slave_config *config;
310 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
313 case DMA_TERMINATE_ALL:
314 return sirfsoc_dma_terminate_all(schan);
315 case DMA_SLAVE_CONFIG:
316 config = (struct dma_slave_config *)arg;
317 return sirfsoc_dma_slave_config(schan, config);
326 /* Alloc channel resources */
327 static int sirfsoc_dma_alloc_chan_resources(struct dma_chan *chan)
329 struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan);
330 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
331 struct sirfsoc_dma_desc *sdesc;
336 /* Alloc descriptors for this channel */
337 for (i = 0; i < SIRFSOC_DMA_DESCRIPTORS; i++) {
338 sdesc = kzalloc(sizeof(*sdesc), GFP_KERNEL);
340 dev_notice(sdma->dma.dev, "Memory allocation error. "
341 "Allocated only %u descriptors\n", i);
345 dma_async_tx_descriptor_init(&sdesc->desc, chan);
346 sdesc->desc.flags = DMA_CTRL_ACK;
347 sdesc->desc.tx_submit = sirfsoc_dma_tx_submit;
349 list_add_tail(&sdesc->node, &descs);
352 /* Return error only if no descriptors were allocated */
356 spin_lock_irqsave(&schan->lock, flags);
358 list_splice_tail_init(&descs, &schan->free);
359 spin_unlock_irqrestore(&schan->lock, flags);
364 /* Free channel resources */
365 static void sirfsoc_dma_free_chan_resources(struct dma_chan *chan)
367 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
368 struct sirfsoc_dma_desc *sdesc, *tmp;
372 spin_lock_irqsave(&schan->lock, flags);
374 /* Channel must be idle */
375 BUG_ON(!list_empty(&schan->prepared));
376 BUG_ON(!list_empty(&schan->queued));
377 BUG_ON(!list_empty(&schan->active));
378 BUG_ON(!list_empty(&schan->completed));
381 list_splice_tail_init(&schan->free, &descs);
383 spin_unlock_irqrestore(&schan->lock, flags);
385 /* Free descriptors */
386 list_for_each_entry_safe(sdesc, tmp, &descs, node)
390 /* Send pending descriptor to hardware */
391 static void sirfsoc_dma_issue_pending(struct dma_chan *chan)
393 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
396 spin_lock_irqsave(&schan->lock, flags);
398 if (list_empty(&schan->active) && !list_empty(&schan->queued))
399 sirfsoc_dma_execute(schan);
401 spin_unlock_irqrestore(&schan->lock, flags);
404 /* Check request completion status */
405 static enum dma_status
406 sirfsoc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
407 struct dma_tx_state *txstate)
409 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
413 spin_lock_irqsave(&schan->lock, flags);
414 ret = dma_cookie_status(chan, cookie, txstate);
415 spin_unlock_irqrestore(&schan->lock, flags);
420 static struct dma_async_tx_descriptor *sirfsoc_dma_prep_interleaved(
421 struct dma_chan *chan, struct dma_interleaved_template *xt,
424 struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan);
425 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
426 struct sirfsoc_dma_desc *sdesc = NULL;
427 unsigned long iflags;
430 if ((xt->dir != DMA_MEM_TO_DEV) || (xt->dir != DMA_DEV_TO_MEM)) {
435 /* Get free descriptor */
436 spin_lock_irqsave(&schan->lock, iflags);
437 if (!list_empty(&schan->free)) {
438 sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc,
440 list_del(&sdesc->node);
442 spin_unlock_irqrestore(&schan->lock, iflags);
445 /* try to free completed descriptors */
446 sirfsoc_dma_process_completed(sdma);
451 /* Place descriptor in prepared list */
452 spin_lock_irqsave(&schan->lock, iflags);
455 * Number of chunks in a frame can only be 1 for prima2
456 * and ylen (number of frame - 1) must be at least 0
458 if ((xt->frame_size == 1) && (xt->numf > 0)) {
460 sdesc->xlen = xt->sgl[0].size / SIRFSOC_DMA_WORD_LEN;
461 sdesc->width = (xt->sgl[0].size + xt->sgl[0].icg) /
462 SIRFSOC_DMA_WORD_LEN;
463 sdesc->ylen = xt->numf - 1;
464 if (xt->dir == DMA_MEM_TO_DEV) {
465 sdesc->addr = xt->src_start;
468 sdesc->addr = xt->dst_start;
472 list_add_tail(&sdesc->node, &schan->prepared);
474 pr_err("sirfsoc DMA Invalid xfer\n");
478 spin_unlock_irqrestore(&schan->lock, iflags);
482 spin_unlock_irqrestore(&schan->lock, iflags);
488 static struct dma_async_tx_descriptor *
489 sirfsoc_dma_prep_cyclic(struct dma_chan *chan, dma_addr_t addr,
490 size_t buf_len, size_t period_len,
491 enum dma_transfer_direction direction, void *context)
493 struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan);
494 struct sirfsoc_dma_desc *sdesc = NULL;
495 unsigned long iflags;
498 * we only support cycle transfer with 2 period
499 * If the X-length is set to 0, it would be the loop mode.
500 * The DMA address keeps increasing until reaching the end of a loop
501 * area whose size is defined by (DMA_WIDTH x (Y_LENGTH + 1)). Then
502 * the DMA address goes back to the beginning of this area.
503 * In loop mode, the DMA data region is divided into two parts, BUFA
504 * and BUFB. DMA controller generates interrupts twice in each loop:
505 * when the DMA address reaches the end of BUFA or the end of the
508 if (buf_len != 2 * period_len)
509 return ERR_PTR(-EINVAL);
511 /* Get free descriptor */
512 spin_lock_irqsave(&schan->lock, iflags);
513 if (!list_empty(&schan->free)) {
514 sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc,
516 list_del(&sdesc->node);
518 spin_unlock_irqrestore(&schan->lock, iflags);
523 /* Place descriptor in prepared list */
524 spin_lock_irqsave(&schan->lock, iflags);
528 sdesc->ylen = buf_len / SIRFSOC_DMA_WORD_LEN - 1;
530 list_add_tail(&sdesc->node, &schan->prepared);
531 spin_unlock_irqrestore(&schan->lock, iflags);
537 * The DMA controller consists of 16 independent DMA channels.
538 * Each channel is allocated to a different function
540 bool sirfsoc_dma_filter_id(struct dma_chan *chan, void *chan_id)
542 unsigned int ch_nr = (unsigned int) chan_id;
544 if (ch_nr == chan->chan_id +
545 chan->device->dev_id * SIRFSOC_DMA_CHANNELS)
550 EXPORT_SYMBOL(sirfsoc_dma_filter_id);
552 static int __devinit sirfsoc_dma_probe(struct platform_device *op)
554 struct device_node *dn = op->dev.of_node;
555 struct device *dev = &op->dev;
556 struct dma_device *dma;
557 struct sirfsoc_dma *sdma;
558 struct sirfsoc_dma_chan *schan;
560 ulong regs_start, regs_size;
564 sdma = devm_kzalloc(dev, sizeof(*sdma), GFP_KERNEL);
566 dev_err(dev, "Memory exhausted!\n");
570 if (of_property_read_u32(dn, "cell-index", &id)) {
571 dev_err(dev, "Fail to get DMAC index\n");
576 sdma->irq = irq_of_parse_and_map(dn, 0);
577 if (sdma->irq == NO_IRQ) {
578 dev_err(dev, "Error mapping IRQ!\n");
583 ret = of_address_to_resource(dn, 0, &res);
585 dev_err(dev, "Error parsing memory region!\n");
589 regs_start = res.start;
590 regs_size = resource_size(&res);
592 sdma->base = devm_ioremap(dev, regs_start, regs_size);
594 dev_err(dev, "Error mapping memory region!\n");
599 ret = devm_request_irq(dev, sdma->irq, &sirfsoc_dma_irq, 0, DRV_NAME,
602 dev_err(dev, "Error requesting IRQ!\n");
609 dma->chancnt = SIRFSOC_DMA_CHANNELS;
611 dma->device_alloc_chan_resources = sirfsoc_dma_alloc_chan_resources;
612 dma->device_free_chan_resources = sirfsoc_dma_free_chan_resources;
613 dma->device_issue_pending = sirfsoc_dma_issue_pending;
614 dma->device_control = sirfsoc_dma_control;
615 dma->device_tx_status = sirfsoc_dma_tx_status;
616 dma->device_prep_interleaved_dma = sirfsoc_dma_prep_interleaved;
617 dma->device_prep_dma_cyclic = sirfsoc_dma_prep_cyclic;
619 INIT_LIST_HEAD(&dma->channels);
620 dma_cap_set(DMA_SLAVE, dma->cap_mask);
621 dma_cap_set(DMA_CYCLIC, dma->cap_mask);
622 dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
623 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
625 for (i = 0; i < dma->chancnt; i++) {
626 schan = &sdma->channels[i];
628 schan->chan.device = dma;
629 dma_cookie_init(&schan->chan);
631 INIT_LIST_HEAD(&schan->free);
632 INIT_LIST_HEAD(&schan->prepared);
633 INIT_LIST_HEAD(&schan->queued);
634 INIT_LIST_HEAD(&schan->active);
635 INIT_LIST_HEAD(&schan->completed);
637 spin_lock_init(&schan->lock);
638 list_add_tail(&schan->chan.device_node, &dma->channels);
641 tasklet_init(&sdma->tasklet, sirfsoc_dma_tasklet, (unsigned long)sdma);
643 /* Register DMA engine */
644 dev_set_drvdata(dev, sdma);
645 ret = dma_async_device_register(dma);
649 dev_info(dev, "initialized SIRFSOC DMAC driver\n");
654 devm_free_irq(dev, sdma->irq, sdma);
656 irq_dispose_mapping(sdma->irq);
660 devm_kfree(dev, sdma);
664 static int __devexit sirfsoc_dma_remove(struct platform_device *op)
666 struct device *dev = &op->dev;
667 struct sirfsoc_dma *sdma = dev_get_drvdata(dev);
669 dma_async_device_unregister(&sdma->dma);
670 devm_free_irq(dev, sdma->irq, sdma);
671 irq_dispose_mapping(sdma->irq);
673 devm_kfree(dev, sdma);
677 static struct of_device_id sirfsoc_dma_match[] = {
678 { .compatible = "sirf,prima2-dmac", },
682 static struct platform_driver sirfsoc_dma_driver = {
683 .probe = sirfsoc_dma_probe,
684 .remove = __devexit_p(sirfsoc_dma_remove),
687 .owner = THIS_MODULE,
688 .of_match_table = sirfsoc_dma_match,
692 module_platform_driver(sirfsoc_dma_driver);
694 MODULE_AUTHOR("Rongjun Ying <rongjun.ying@csr.com>, "
695 "Barry Song <baohua.song@csr.com>");
696 MODULE_DESCRIPTION("SIRFSOC DMA control driver");
697 MODULE_LICENSE("GPL v2");