ioat: prepare the code for ioat[12]_dma_chan split
[linux-2.6.git] / drivers / dma / ioat / dma.c
1 /*
2  * Intel I/OAT DMA Linux driver
3  * Copyright(c) 2004 - 2009 Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17  *
18  * The full GNU General Public License is included in this distribution in
19  * the file called "COPYING".
20  *
21  */
22
23 /*
24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
25  * copy operations.
26  */
27
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/workqueue.h>
36 #include <linux/i7300_idle.h>
37 #include "dma.h"
38 #include "registers.h"
39 #include "hw.h"
40
41 static int ioat_pending_level = 4;
42 module_param(ioat_pending_level, int, 0644);
43 MODULE_PARM_DESC(ioat_pending_level,
44                  "high-water mark for pushing ioat descriptors (default: 4)");
45
46 static void ioat_dma_chan_reset_part2(struct work_struct *work);
47 static void ioat_dma_chan_watchdog(struct work_struct *work);
48
49 /* internal functions */
50 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat);
51 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat);
52
53 static struct ioat_desc_sw *
54 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
55 static struct ioat_desc_sw *
56 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat);
57
58 static inline struct ioat_chan_common *
59 ioat_chan_by_index(struct ioatdma_device *device, int index)
60 {
61         return device->idx[index];
62 }
63
64 /**
65  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
66  * @irq: interrupt id
67  * @data: interrupt data
68  */
69 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
70 {
71         struct ioatdma_device *instance = data;
72         struct ioat_chan_common *chan;
73         unsigned long attnstatus;
74         int bit;
75         u8 intrctrl;
76
77         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
78
79         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
80                 return IRQ_NONE;
81
82         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
83                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
84                 return IRQ_NONE;
85         }
86
87         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
88         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
89                 chan = ioat_chan_by_index(instance, bit);
90                 tasklet_schedule(&chan->cleanup_task);
91         }
92
93         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
94         return IRQ_HANDLED;
95 }
96
97 /**
98  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
99  * @irq: interrupt id
100  * @data: interrupt data
101  */
102 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
103 {
104         struct ioat_chan_common *chan = data;
105
106         tasklet_schedule(&chan->cleanup_task);
107
108         return IRQ_HANDLED;
109 }
110
111 static void ioat_dma_cleanup_tasklet(unsigned long data);
112
113 /**
114  * ioat_dma_enumerate_channels - find and initialize the device's channels
115  * @device: the device to be enumerated
116  */
117 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
118 {
119         u8 xfercap_scale;
120         u32 xfercap;
121         int i;
122         struct ioat_chan_common *chan;
123         struct ioat_dma_chan *ioat;
124         struct device *dev = &device->pdev->dev;
125         struct dma_device *dma = &device->common;
126
127         INIT_LIST_HEAD(&dma->channels);
128         dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
129         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
130         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
131
132 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
133         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
134                 dma->chancnt--;
135 #endif
136         for (i = 0; i < dma->chancnt; i++) {
137                 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
138                 if (!ioat) {
139                         dma->chancnt = i;
140                         break;
141                 }
142
143                 chan = &ioat->base;
144                 chan->device = device;
145                 chan->reg_base = device->reg_base + (0x80 * (i + 1));
146                 ioat->xfercap = xfercap;
147                 ioat->desccount = 0;
148                 INIT_DELAYED_WORK(&chan->work, ioat_dma_chan_reset_part2);
149                 spin_lock_init(&chan->cleanup_lock);
150                 spin_lock_init(&ioat->desc_lock);
151                 INIT_LIST_HEAD(&ioat->free_desc);
152                 INIT_LIST_HEAD(&ioat->used_desc);
153                 /* This should be made common somewhere in dmaengine.c */
154                 chan->common.device = &device->common;
155                 list_add_tail(&chan->common.device_node, &dma->channels);
156                 device->idx[i] = chan;
157                 tasklet_init(&chan->cleanup_task,
158                              ioat_dma_cleanup_tasklet,
159                              (unsigned long) ioat);
160                 tasklet_disable(&chan->cleanup_task);
161         }
162         return dma->chancnt;
163 }
164
165 /**
166  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
167  *                                 descriptors to hw
168  * @chan: DMA channel handle
169  */
170 static inline void
171 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
172 {
173         void __iomem *reg_base = ioat->base.reg_base;
174
175         ioat->pending = 0;
176         writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
177 }
178
179 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
180 {
181         struct ioat_dma_chan *ioat = to_ioat_chan(chan);
182
183         if (ioat->pending > 0) {
184                 spin_lock_bh(&ioat->desc_lock);
185                 __ioat1_dma_memcpy_issue_pending(ioat);
186                 spin_unlock_bh(&ioat->desc_lock);
187         }
188 }
189
190 static inline void
191 __ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
192 {
193         void __iomem *reg_base = ioat->base.reg_base;
194
195         ioat->pending = 0;
196         writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
197 }
198
199 static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
200 {
201         struct ioat_dma_chan *ioat = to_ioat_chan(chan);
202
203         if (ioat->pending > 0) {
204                 spin_lock_bh(&ioat->desc_lock);
205                 __ioat2_dma_memcpy_issue_pending(ioat);
206                 spin_unlock_bh(&ioat->desc_lock);
207         }
208 }
209
210
211 /**
212  * ioat_dma_chan_reset_part2 - reinit the channel after a reset
213  */
214 static void ioat_dma_chan_reset_part2(struct work_struct *work)
215 {
216         struct ioat_chan_common *chan;
217         struct ioat_dma_chan *ioat;
218         struct ioat_desc_sw *desc;
219
220         chan = container_of(work, struct ioat_chan_common, work.work);
221         ioat = container_of(chan, struct ioat_dma_chan, base);
222         spin_lock_bh(&chan->cleanup_lock);
223         spin_lock_bh(&ioat->desc_lock);
224
225         chan->completion_virt->low = 0;
226         chan->completion_virt->high = 0;
227         ioat->pending = 0;
228
229         /*
230          * count the descriptors waiting, and be sure to do it
231          * right for both the CB1 line and the CB2 ring
232          */
233         ioat->dmacount = 0;
234         if (ioat->used_desc.prev) {
235                 desc = to_ioat_desc(ioat->used_desc.prev);
236                 do {
237                         ioat->dmacount++;
238                         desc = to_ioat_desc(desc->node.next);
239                 } while (&desc->node != ioat->used_desc.next);
240         }
241
242         /*
243          * write the new starting descriptor address
244          * this puts channel engine into ARMED state
245          */
246         desc = to_ioat_desc(ioat->used_desc.prev);
247         switch (chan->device->version) {
248         case IOAT_VER_1_2:
249                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
250                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
251                 writel(((u64) desc->txd.phys) >> 32,
252                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
253
254                 writeb(IOAT_CHANCMD_START, chan->reg_base
255                         + IOAT_CHANCMD_OFFSET(chan->device->version));
256                 break;
257         case IOAT_VER_2_0:
258                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
259                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
260                 writel(((u64) desc->txd.phys) >> 32,
261                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
262
263                 /* tell the engine to go with what's left to be done */
264                 writew(ioat->dmacount,
265                        chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
266
267                 break;
268         }
269         dev_err(to_dev(chan),
270                 "chan%d reset - %d descs waiting, %d total desc\n",
271                 chan_num(chan), ioat->dmacount, ioat->desccount);
272
273         spin_unlock_bh(&ioat->desc_lock);
274         spin_unlock_bh(&chan->cleanup_lock);
275 }
276
277 /**
278  * ioat_dma_reset_channel - restart a channel
279  * @ioat: IOAT DMA channel handle
280  */
281 static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat)
282 {
283         struct ioat_chan_common *chan = &ioat->base;
284         void __iomem *reg_base = chan->reg_base;
285         u32 chansts, chanerr;
286
287         if (!ioat->used_desc.prev)
288                 return;
289
290         chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
291         chansts = (chan->completion_virt->low
292                                         & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
293         if (chanerr) {
294                 dev_err(to_dev(chan),
295                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
296                         chan_num(chan), chansts, chanerr);
297                 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
298         }
299
300         /*
301          * whack it upside the head with a reset
302          * and wait for things to settle out.
303          * force the pending count to a really big negative
304          * to make sure no one forces an issue_pending
305          * while we're waiting.
306          */
307
308         spin_lock_bh(&ioat->desc_lock);
309         ioat->pending = INT_MIN;
310         writeb(IOAT_CHANCMD_RESET,
311                reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
312         spin_unlock_bh(&ioat->desc_lock);
313
314         /* schedule the 2nd half instead of sleeping a long time */
315         schedule_delayed_work(&chan->work, RESET_DELAY);
316 }
317
318 /**
319  * ioat_dma_chan_watchdog - watch for stuck channels
320  */
321 static void ioat_dma_chan_watchdog(struct work_struct *work)
322 {
323         struct ioatdma_device *device =
324                 container_of(work, struct ioatdma_device, work.work);
325         struct ioat_dma_chan *ioat;
326         struct ioat_chan_common *chan;
327         int i;
328
329         union {
330                 u64 full;
331                 struct {
332                         u32 low;
333                         u32 high;
334                 };
335         } completion_hw;
336         unsigned long compl_desc_addr_hw;
337
338         for (i = 0; i < device->common.chancnt; i++) {
339                 chan = ioat_chan_by_index(device, i);
340                 ioat = container_of(chan, struct ioat_dma_chan, base);
341
342                 if (chan->device->version == IOAT_VER_1_2
343                         /* have we started processing anything yet */
344                     && chan->last_completion
345                         /* have we completed any since last watchdog cycle? */
346                     && (chan->last_completion == chan->watchdog_completion)
347                         /* has TCP stuck on one cookie since last watchdog? */
348                     && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
349                     && (chan->watchdog_tcp_cookie != chan->completed_cookie)
350                         /* is there something in the chain to be processed? */
351                         /* CB1 chain always has at least the last one processed */
352                     && (ioat->used_desc.prev != ioat->used_desc.next)
353                     && ioat->pending == 0) {
354
355                         /*
356                          * check CHANSTS register for completed
357                          * descriptor address.
358                          * if it is different than completion writeback,
359                          * it is not zero
360                          * and it has changed since the last watchdog
361                          *     we can assume that channel
362                          *     is still working correctly
363                          *     and the problem is in completion writeback.
364                          *     update completion writeback
365                          *     with actual CHANSTS value
366                          * else
367                          *     try resetting the channel
368                          */
369
370                         completion_hw.low = readl(chan->reg_base +
371                                 IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
372                         completion_hw.high = readl(chan->reg_base +
373                                 IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
374 #if (BITS_PER_LONG == 64)
375                         compl_desc_addr_hw =
376                                 completion_hw.full
377                                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
378 #else
379                         compl_desc_addr_hw =
380                                 completion_hw.low & IOAT_LOW_COMPLETION_MASK;
381 #endif
382
383                         if ((compl_desc_addr_hw != 0)
384                            && (compl_desc_addr_hw != chan->watchdog_completion)
385                            && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
386                                 chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
387                                 chan->completion_virt->low = completion_hw.low;
388                                 chan->completion_virt->high = completion_hw.high;
389                         } else {
390                                 ioat_dma_reset_channel(ioat);
391                                 chan->watchdog_completion = 0;
392                                 chan->last_compl_desc_addr_hw = 0;
393                         }
394
395                 /*
396                  * for version 2.0 if there are descriptors yet to be processed
397                  * and the last completed hasn't changed since the last watchdog
398                  *      if they haven't hit the pending level
399                  *          issue the pending to push them through
400                  *      else
401                  *          try resetting the channel
402                  */
403                 } else if (chan->device->version == IOAT_VER_2_0
404                     && ioat->used_desc.prev
405                     && chan->last_completion
406                     && chan->last_completion == chan->watchdog_completion) {
407
408                         if (ioat->pending < ioat_pending_level)
409                                 ioat2_dma_memcpy_issue_pending(&chan->common);
410                         else {
411                                 ioat_dma_reset_channel(ioat);
412                                 chan->watchdog_completion = 0;
413                         }
414                 } else {
415                         chan->last_compl_desc_addr_hw = 0;
416                         chan->watchdog_completion = chan->last_completion;
417                 }
418                 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
419         }
420
421         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
422 }
423
424 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
425 {
426         struct dma_chan *c = tx->chan;
427         struct ioat_dma_chan *ioat = to_ioat_chan(c);
428         struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
429         struct ioat_desc_sw *first;
430         struct ioat_desc_sw *chain_tail;
431         dma_cookie_t cookie;
432
433         spin_lock_bh(&ioat->desc_lock);
434         /* cookie incr and addition to used_list must be atomic */
435         cookie = c->cookie;
436         cookie++;
437         if (cookie < 0)
438                 cookie = 1;
439         c->cookie = cookie;
440         tx->cookie = cookie;
441
442         /* write address into NextDescriptor field of last desc in chain */
443         first = to_ioat_desc(tx->tx_list.next);
444         chain_tail = to_ioat_desc(ioat->used_desc.prev);
445         /* make descriptor updates globally visible before chaining */
446         wmb();
447         chain_tail->hw->next = first->txd.phys;
448         list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
449
450         ioat->dmacount += desc->tx_cnt;
451         ioat->pending += desc->tx_cnt;
452         if (ioat->pending >= ioat_pending_level)
453                 __ioat1_dma_memcpy_issue_pending(ioat);
454         spin_unlock_bh(&ioat->desc_lock);
455
456         return cookie;
457 }
458
459 static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
460 {
461         struct ioat_dma_chan *ioat = to_ioat_chan(tx->chan);
462         struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
463         struct ioat_desc_sw *new;
464         struct ioat_dma_descriptor *hw;
465         dma_cookie_t cookie;
466         u32 copy;
467         size_t len;
468         dma_addr_t src, dst;
469         unsigned long orig_flags;
470         unsigned int desc_count = 0;
471
472         /* src and dest and len are stored in the initial descriptor */
473         len = first->len;
474         src = first->src;
475         dst = first->dst;
476         orig_flags = first->txd.flags;
477         new = first;
478
479         /*
480          * ioat->desc_lock is still in force in version 2 path
481          * it gets unlocked at end of this function
482          */
483         do {
484                 copy = min_t(size_t, len, ioat->xfercap);
485
486                 async_tx_ack(&new->txd);
487
488                 hw = new->hw;
489                 hw->size = copy;
490                 hw->ctl = 0;
491                 hw->src_addr = src;
492                 hw->dst_addr = dst;
493
494                 len -= copy;
495                 dst += copy;
496                 src += copy;
497                 desc_count++;
498         } while (len && (new = ioat2_dma_get_next_descriptor(ioat)));
499
500         if (!new) {
501                 dev_err(to_dev(&ioat->base), "tx submit failed\n");
502                 spin_unlock_bh(&ioat->desc_lock);
503                 return -ENOMEM;
504         }
505
506         hw->ctl_f.compl_write = 1;
507         if (first->txd.callback) {
508                 hw->ctl_f.int_en = 1;
509                 if (first != new) {
510                         /* move callback into to last desc */
511                         new->txd.callback = first->txd.callback;
512                         new->txd.callback_param
513                                         = first->txd.callback_param;
514                         first->txd.callback = NULL;
515                         first->txd.callback_param = NULL;
516                 }
517         }
518
519         new->tx_cnt = desc_count;
520         new->txd.flags = orig_flags; /* client is in control of this ack */
521
522         /* store the original values for use in later cleanup */
523         if (new != first) {
524                 new->src = first->src;
525                 new->dst = first->dst;
526                 new->len = first->len;
527         }
528
529         /* cookie incr and addition to used_list must be atomic */
530         cookie = ioat->base.common.cookie;
531         cookie++;
532         if (cookie < 0)
533                 cookie = 1;
534         ioat->base.common.cookie = new->txd.cookie = cookie;
535
536         ioat->dmacount += desc_count;
537         ioat->pending += desc_count;
538         if (ioat->pending >= ioat_pending_level)
539                 __ioat2_dma_memcpy_issue_pending(ioat);
540         spin_unlock_bh(&ioat->desc_lock);
541
542         return cookie;
543 }
544
545 /**
546  * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
547  * @ioat: the channel supplying the memory pool for the descriptors
548  * @flags: allocation flags
549  */
550 static struct ioat_desc_sw *
551 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
552 {
553         struct ioat_dma_descriptor *desc;
554         struct ioat_desc_sw *desc_sw;
555         struct ioatdma_device *ioatdma_device;
556         dma_addr_t phys;
557
558         ioatdma_device = ioat->base.device;
559         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
560         if (unlikely(!desc))
561                 return NULL;
562
563         desc_sw = kzalloc(sizeof(*desc_sw), flags);
564         if (unlikely(!desc_sw)) {
565                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
566                 return NULL;
567         }
568
569         memset(desc, 0, sizeof(*desc));
570         dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
571         switch (ioatdma_device->version) {
572         case IOAT_VER_1_2:
573                 desc_sw->txd.tx_submit = ioat1_tx_submit;
574                 break;
575         case IOAT_VER_2_0:
576         case IOAT_VER_3_0:
577                 desc_sw->txd.tx_submit = ioat2_tx_submit;
578                 break;
579         }
580
581         desc_sw->hw = desc;
582         desc_sw->txd.phys = phys;
583
584         return desc_sw;
585 }
586
587 static int ioat_initial_desc_count = 256;
588 module_param(ioat_initial_desc_count, int, 0644);
589 MODULE_PARM_DESC(ioat_initial_desc_count,
590                  "initial descriptors per channel (default: 256)");
591
592 /**
593  * ioat2_dma_massage_chan_desc - link the descriptors into a circle
594  * @ioat: the channel to be massaged
595  */
596 static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat)
597 {
598         struct ioat_desc_sw *desc, *_desc;
599
600         /* setup used_desc */
601         ioat->used_desc.next = ioat->free_desc.next;
602         ioat->used_desc.prev = NULL;
603
604         /* pull free_desc out of the circle so that every node is a hw
605          * descriptor, but leave it pointing to the list
606          */
607         ioat->free_desc.prev->next = ioat->free_desc.next;
608         ioat->free_desc.next->prev = ioat->free_desc.prev;
609
610         /* circle link the hw descriptors */
611         desc = to_ioat_desc(ioat->free_desc.next);
612         desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
613         list_for_each_entry_safe(desc, _desc, ioat->free_desc.next, node) {
614                 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
615         }
616 }
617
618 /**
619  * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
620  * @chan: the channel to be filled out
621  */
622 static int ioat_dma_alloc_chan_resources(struct dma_chan *c)
623 {
624         struct ioat_dma_chan *ioat = to_ioat_chan(c);
625         struct ioat_chan_common *chan = &ioat->base;
626         struct ioat_desc_sw *desc;
627         u16 chanctrl;
628         u32 chanerr;
629         int i;
630         LIST_HEAD(tmp_list);
631
632         /* have we already been set up? */
633         if (!list_empty(&ioat->free_desc))
634                 return ioat->desccount;
635
636         /* Setup register to interrupt and write completion status on error */
637         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
638                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
639                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
640         writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
641
642         chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
643         if (chanerr) {
644                 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
645                 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
646         }
647
648         /* Allocate descriptors */
649         for (i = 0; i < ioat_initial_desc_count; i++) {
650                 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
651                 if (!desc) {
652                         dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
653                         break;
654                 }
655                 list_add_tail(&desc->node, &tmp_list);
656         }
657         spin_lock_bh(&ioat->desc_lock);
658         ioat->desccount = i;
659         list_splice(&tmp_list, &ioat->free_desc);
660         if (chan->device->version != IOAT_VER_1_2)
661                 ioat2_dma_massage_chan_desc(ioat);
662         spin_unlock_bh(&ioat->desc_lock);
663
664         /* allocate a completion writeback area */
665         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
666         chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
667                                                GFP_KERNEL,
668                                                &chan->completion_addr);
669         memset(chan->completion_virt, 0,
670                sizeof(*chan->completion_virt));
671         writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
672                chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
673         writel(((u64) chan->completion_addr) >> 32,
674                chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
675
676         tasklet_enable(&chan->cleanup_task);
677         ioat_dma_start_null_desc(ioat);  /* give chain to dma device */
678         return ioat->desccount;
679 }
680
681 /**
682  * ioat_dma_free_chan_resources - release all the descriptors
683  * @chan: the channel to be cleaned
684  */
685 static void ioat_dma_free_chan_resources(struct dma_chan *c)
686 {
687         struct ioat_dma_chan *ioat = to_ioat_chan(c);
688         struct ioat_chan_common *chan = &ioat->base;
689         struct ioatdma_device *ioatdma_device = chan->device;
690         struct ioat_desc_sw *desc, *_desc;
691         int in_use_descs = 0;
692
693         /* Before freeing channel resources first check
694          * if they have been previously allocated for this channel.
695          */
696         if (ioat->desccount == 0)
697                 return;
698
699         tasklet_disable(&chan->cleanup_task);
700         ioat_dma_memcpy_cleanup(ioat);
701
702         /* Delay 100ms after reset to allow internal DMA logic to quiesce
703          * before removing DMA descriptor resources.
704          */
705         writeb(IOAT_CHANCMD_RESET,
706                chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
707         mdelay(100);
708
709         spin_lock_bh(&ioat->desc_lock);
710         switch (chan->device->version) {
711         case IOAT_VER_1_2:
712                 list_for_each_entry_safe(desc, _desc,
713                                          &ioat->used_desc, node) {
714                         in_use_descs++;
715                         list_del(&desc->node);
716                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
717                                       desc->txd.phys);
718                         kfree(desc);
719                 }
720                 list_for_each_entry_safe(desc, _desc,
721                                          &ioat->free_desc, node) {
722                         list_del(&desc->node);
723                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
724                                       desc->txd.phys);
725                         kfree(desc);
726                 }
727                 break;
728         case IOAT_VER_2_0:
729         case IOAT_VER_3_0:
730                 list_for_each_entry_safe(desc, _desc,
731                                          ioat->free_desc.next, node) {
732                         list_del(&desc->node);
733                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
734                                       desc->txd.phys);
735                         kfree(desc);
736                 }
737                 desc = to_ioat_desc(ioat->free_desc.next);
738                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
739                               desc->txd.phys);
740                 kfree(desc);
741                 INIT_LIST_HEAD(&ioat->free_desc);
742                 INIT_LIST_HEAD(&ioat->used_desc);
743                 break;
744         }
745         spin_unlock_bh(&ioat->desc_lock);
746
747         pci_pool_free(ioatdma_device->completion_pool,
748                       chan->completion_virt,
749                       chan->completion_addr);
750
751         /* one is ok since we left it on there on purpose */
752         if (in_use_descs > 1)
753                 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
754                         in_use_descs - 1);
755
756         chan->last_completion = chan->completion_addr = 0;
757         chan->watchdog_completion = 0;
758         chan->last_compl_desc_addr_hw = 0;
759         chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
760         ioat->pending = 0;
761         ioat->dmacount = 0;
762         ioat->desccount = 0;
763 }
764
765 /**
766  * ioat1_dma_get_next_descriptor - return the next available descriptor
767  * @ioat: IOAT DMA channel handle
768  *
769  * Gets the next descriptor from the chain, and must be called with the
770  * channel's desc_lock held.  Allocates more descriptors if the channel
771  * has run out.
772  */
773 static struct ioat_desc_sw *
774 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
775 {
776         struct ioat_desc_sw *new;
777
778         if (!list_empty(&ioat->free_desc)) {
779                 new = to_ioat_desc(ioat->free_desc.next);
780                 list_del(&new->node);
781         } else {
782                 /* try to get another desc */
783                 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
784                 if (!new) {
785                         dev_err(to_dev(&ioat->base), "alloc failed\n");
786                         return NULL;
787                 }
788         }
789
790         prefetch(new->hw);
791         return new;
792 }
793
794 static struct ioat_desc_sw *
795 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
796 {
797         struct ioat_desc_sw *new;
798
799         /*
800          * used.prev points to where to start processing
801          * used.next points to next free descriptor
802          * if used.prev == NULL, there are none waiting to be processed
803          * if used.next == used.prev.prev, there is only one free descriptor,
804          *      and we need to use it to as a noop descriptor before
805          *      linking in a new set of descriptors, since the device
806          *      has probably already read the pointer to it
807          */
808         if (ioat->used_desc.prev &&
809             ioat->used_desc.next == ioat->used_desc.prev->prev) {
810
811                 struct ioat_desc_sw *desc;
812                 struct ioat_desc_sw *noop_desc;
813                 int i;
814
815                 /* set up the noop descriptor */
816                 noop_desc = to_ioat_desc(ioat->used_desc.next);
817                 /* set size to non-zero value (channel returns error when size is 0) */
818                 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
819                 noop_desc->hw->ctl = 0;
820                 noop_desc->hw->ctl_f.null = 1;
821                 noop_desc->hw->src_addr = 0;
822                 noop_desc->hw->dst_addr = 0;
823
824                 ioat->used_desc.next = ioat->used_desc.next->next;
825                 ioat->pending++;
826                 ioat->dmacount++;
827
828                 /* try to get a few more descriptors */
829                 for (i = 16; i; i--) {
830                         desc = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
831                         if (!desc) {
832                                 dev_err(to_dev(&ioat->base),
833                                         "alloc failed\n");
834                                 break;
835                         }
836                         list_add_tail(&desc->node, ioat->used_desc.next);
837
838                         desc->hw->next
839                                 = to_ioat_desc(desc->node.next)->txd.phys;
840                         to_ioat_desc(desc->node.prev)->hw->next
841                                 = desc->txd.phys;
842                         ioat->desccount++;
843                 }
844
845                 ioat->used_desc.next = noop_desc->node.next;
846         }
847         new = to_ioat_desc(ioat->used_desc.next);
848         prefetch(new);
849         ioat->used_desc.next = new->node.next;
850
851         if (ioat->used_desc.prev == NULL)
852                 ioat->used_desc.prev = &new->node;
853
854         prefetch(new->hw);
855         return new;
856 }
857
858 static struct ioat_desc_sw *
859 ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
860 {
861         if (!ioat)
862                 return NULL;
863
864         switch (ioat->base.device->version) {
865         case IOAT_VER_1_2:
866                 return ioat1_dma_get_next_descriptor(ioat);
867         case IOAT_VER_2_0:
868         case IOAT_VER_3_0:
869                 return ioat2_dma_get_next_descriptor(ioat);
870         }
871         return NULL;
872 }
873
874 static struct dma_async_tx_descriptor *
875 ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
876                       dma_addr_t dma_src, size_t len, unsigned long flags)
877 {
878         struct ioat_dma_chan *ioat = to_ioat_chan(c);
879         struct ioat_desc_sw *desc;
880         size_t copy;
881         LIST_HEAD(chain);
882         dma_addr_t src = dma_src;
883         dma_addr_t dest = dma_dest;
884         size_t total_len = len;
885         struct ioat_dma_descriptor *hw = NULL;
886         int tx_cnt = 0;
887
888         spin_lock_bh(&ioat->desc_lock);
889         desc = ioat_dma_get_next_descriptor(ioat);
890         do {
891                 if (!desc)
892                         break;
893
894                 tx_cnt++;
895                 copy = min_t(size_t, len, ioat->xfercap);
896
897                 hw = desc->hw;
898                 hw->size = copy;
899                 hw->ctl = 0;
900                 hw->src_addr = src;
901                 hw->dst_addr = dest;
902
903                 list_add_tail(&desc->node, &chain);
904
905                 len -= copy;
906                 dest += copy;
907                 src += copy;
908                 if (len) {
909                         struct ioat_desc_sw *next;
910
911                         async_tx_ack(&desc->txd);
912                         next = ioat_dma_get_next_descriptor(ioat);
913                         hw->next = next ? next->txd.phys : 0;
914                         desc = next;
915                 } else
916                         hw->next = 0;
917         } while (len);
918
919         if (!desc) {
920                 struct ioat_chan_common *chan = &ioat->base;
921
922                 dev_err(to_dev(chan),
923                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
924                         chan_num(chan), ioat->dmacount, ioat->desccount);
925                 list_splice(&chain, &ioat->free_desc);
926                 spin_unlock_bh(&ioat->desc_lock);
927                 return NULL;
928         }
929         spin_unlock_bh(&ioat->desc_lock);
930
931         desc->txd.flags = flags;
932         desc->tx_cnt = tx_cnt;
933         desc->src = dma_src;
934         desc->dst = dma_dest;
935         desc->len = total_len;
936         list_splice(&chain, &desc->txd.tx_list);
937         hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
938         hw->ctl_f.compl_write = 1;
939
940         return &desc->txd;
941 }
942
943 static struct dma_async_tx_descriptor *
944 ioat2_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
945                       dma_addr_t dma_src, size_t len, unsigned long flags)
946 {
947         struct ioat_dma_chan *ioat = to_ioat_chan(c);
948         struct ioat_desc_sw *new;
949
950         spin_lock_bh(&ioat->desc_lock);
951         new = ioat2_dma_get_next_descriptor(ioat);
952
953         /*
954          * leave ioat->desc_lock set in ioat 2 path
955          * it will get unlocked at end of tx_submit
956          */
957
958         if (new) {
959                 new->len = len;
960                 new->dst = dma_dest;
961                 new->src = dma_src;
962                 new->txd.flags = flags;
963                 return &new->txd;
964         } else {
965                 struct ioat_chan_common *chan = &ioat->base;
966
967                 spin_unlock_bh(&ioat->desc_lock);
968                 dev_err(to_dev(chan),
969                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
970                         chan_num(chan), ioat->dmacount, ioat->desccount);
971                 return NULL;
972         }
973 }
974
975 static void ioat_dma_cleanup_tasklet(unsigned long data)
976 {
977         struct ioat_dma_chan *chan = (void *)data;
978         ioat_dma_memcpy_cleanup(chan);
979         writew(IOAT_CHANCTRL_INT_DISABLE,
980                chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
981 }
982
983 static void
984 ioat_dma_unmap(struct ioat_chan_common *chan, struct ioat_desc_sw *desc)
985 {
986         if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
987                 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
988                         pci_unmap_single(chan->device->pdev,
989                                          pci_unmap_addr(desc, dst),
990                                          pci_unmap_len(desc, len),
991                                          PCI_DMA_FROMDEVICE);
992                 else
993                         pci_unmap_page(chan->device->pdev,
994                                        pci_unmap_addr(desc, dst),
995                                        pci_unmap_len(desc, len),
996                                        PCI_DMA_FROMDEVICE);
997         }
998
999         if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1000                 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1001                         pci_unmap_single(chan->device->pdev,
1002                                          pci_unmap_addr(desc, src),
1003                                          pci_unmap_len(desc, len),
1004                                          PCI_DMA_TODEVICE);
1005                 else
1006                         pci_unmap_page(chan->device->pdev,
1007                                        pci_unmap_addr(desc, src),
1008                                        pci_unmap_len(desc, len),
1009                                        PCI_DMA_TODEVICE);
1010         }
1011 }
1012
1013 /**
1014  * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
1015  * @chan: ioat channel to be cleaned up
1016  */
1017 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat)
1018 {
1019         struct ioat_chan_common *chan = &ioat->base;
1020         unsigned long phys_complete;
1021         struct ioat_desc_sw *desc, *_desc;
1022         dma_cookie_t cookie = 0;
1023         unsigned long desc_phys;
1024         struct ioat_desc_sw *latest_desc;
1025         struct dma_async_tx_descriptor *tx;
1026
1027         prefetch(chan->completion_virt);
1028
1029         if (!spin_trylock_bh(&chan->cleanup_lock))
1030                 return;
1031
1032         /* The completion writeback can happen at any time,
1033            so reads by the driver need to be atomic operations
1034            The descriptor physical addresses are limited to 32-bits
1035            when the CPU can only do a 32-bit mov */
1036
1037 #if (BITS_PER_LONG == 64)
1038         phys_complete =
1039                 chan->completion_virt->full
1040                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1041 #else
1042         phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
1043 #endif
1044
1045         if ((chan->completion_virt->full
1046                 & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
1047                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
1048                 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
1049                         readl(chan->reg_base + IOAT_CHANERR_OFFSET));
1050
1051                 /* TODO do something to salvage the situation */
1052         }
1053
1054         if (phys_complete == chan->last_completion) {
1055                 spin_unlock_bh(&chan->cleanup_lock);
1056                 /*
1057                  * perhaps we're stuck so hard that the watchdog can't go off?
1058                  * try to catch it after 2 seconds
1059                  */
1060                 if (chan->device->version != IOAT_VER_3_0) {
1061                         if (time_after(jiffies,
1062                                        chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1063                                 ioat_dma_chan_watchdog(&(chan->device->work.work));
1064                                 chan->last_completion_time = jiffies;
1065                         }
1066                 }
1067                 return;
1068         }
1069         chan->last_completion_time = jiffies;
1070
1071         cookie = 0;
1072         if (!spin_trylock_bh(&ioat->desc_lock)) {
1073                 spin_unlock_bh(&chan->cleanup_lock);
1074                 return;
1075         }
1076
1077         switch (chan->device->version) {
1078         case IOAT_VER_1_2:
1079                 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
1080                         tx = &desc->txd;
1081                         /*
1082                          * Incoming DMA requests may use multiple descriptors,
1083                          * due to exceeding xfercap, perhaps. If so, only the
1084                          * last one will have a cookie, and require unmapping.
1085                          */
1086                         if (tx->cookie) {
1087                                 cookie = tx->cookie;
1088                                 ioat_dma_unmap(chan, desc);
1089                                 if (tx->callback) {
1090                                         tx->callback(tx->callback_param);
1091                                         tx->callback = NULL;
1092                                 }
1093                         }
1094
1095                         if (tx->phys != phys_complete) {
1096                                 /*
1097                                  * a completed entry, but not the last, so clean
1098                                  * up if the client is done with the descriptor
1099                                  */
1100                                 if (async_tx_test_ack(tx)) {
1101                                         list_move_tail(&desc->node,
1102                                                        &ioat->free_desc);
1103                                 } else
1104                                         tx->cookie = 0;
1105                         } else {
1106                                 /*
1107                                  * last used desc. Do not remove, so we can
1108                                  * append from it, but don't look at it next
1109                                  * time, either
1110                                  */
1111                                 tx->cookie = 0;
1112
1113                                 /* TODO check status bits? */
1114                                 break;
1115                         }
1116                 }
1117                 break;
1118         case IOAT_VER_2_0:
1119         case IOAT_VER_3_0:
1120                 /* has some other thread has already cleaned up? */
1121                 if (ioat->used_desc.prev == NULL)
1122                         break;
1123
1124                 /* work backwards to find latest finished desc */
1125                 desc = to_ioat_desc(ioat->used_desc.next);
1126                 tx = &desc->txd;
1127                 latest_desc = NULL;
1128                 do {
1129                         desc = to_ioat_desc(desc->node.prev);
1130                         desc_phys = (unsigned long)tx->phys
1131                                        & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1132                         if (desc_phys == phys_complete) {
1133                                 latest_desc = desc;
1134                                 break;
1135                         }
1136                 } while (&desc->node != ioat->used_desc.prev);
1137
1138                 if (latest_desc != NULL) {
1139                         /* work forwards to clear finished descriptors */
1140                         for (desc = to_ioat_desc(ioat->used_desc.prev);
1141                              &desc->node != latest_desc->node.next &&
1142                              &desc->node != ioat->used_desc.next;
1143                              desc = to_ioat_desc(desc->node.next)) {
1144                                 if (tx->cookie) {
1145                                         cookie = tx->cookie;
1146                                         tx->cookie = 0;
1147                                         ioat_dma_unmap(chan, desc);
1148                                         if (tx->callback) {
1149                                                 tx->callback(tx->callback_param);
1150                                                 tx->callback = NULL;
1151                                         }
1152                                 }
1153                         }
1154
1155                         /* move used.prev up beyond those that are finished */
1156                         if (&desc->node == ioat->used_desc.next)
1157                                 ioat->used_desc.prev = NULL;
1158                         else
1159                                 ioat->used_desc.prev = &desc->node;
1160                 }
1161                 break;
1162         }
1163
1164         spin_unlock_bh(&ioat->desc_lock);
1165
1166         chan->last_completion = phys_complete;
1167         if (cookie != 0)
1168                 chan->completed_cookie = cookie;
1169
1170         spin_unlock_bh(&chan->cleanup_lock);
1171 }
1172
1173 /**
1174  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1175  * @chan: IOAT DMA channel handle
1176  * @cookie: DMA transaction identifier
1177  * @done: if not %NULL, updated with last completed transaction
1178  * @used: if not %NULL, updated with last used transaction
1179  */
1180 static enum dma_status
1181 ioat_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
1182                      dma_cookie_t *done, dma_cookie_t *used)
1183 {
1184         struct ioat_dma_chan *ioat = to_ioat_chan(c);
1185         struct ioat_chan_common *chan = &ioat->base;
1186         dma_cookie_t last_used;
1187         dma_cookie_t last_complete;
1188         enum dma_status ret;
1189
1190         last_used = c->cookie;
1191         last_complete = chan->completed_cookie;
1192         chan->watchdog_tcp_cookie = cookie;
1193
1194         if (done)
1195                 *done = last_complete;
1196         if (used)
1197                 *used = last_used;
1198
1199         ret = dma_async_is_complete(cookie, last_complete, last_used);
1200         if (ret == DMA_SUCCESS)
1201                 return ret;
1202
1203         ioat_dma_memcpy_cleanup(ioat);
1204
1205         last_used = c->cookie;
1206         last_complete = chan->completed_cookie;
1207
1208         if (done)
1209                 *done = last_complete;
1210         if (used)
1211                 *used = last_used;
1212
1213         return dma_async_is_complete(cookie, last_complete, last_used);
1214 }
1215
1216 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat)
1217 {
1218         struct ioat_chan_common *chan = &ioat->base;
1219         struct ioat_desc_sw *desc;
1220         struct ioat_dma_descriptor *hw;
1221
1222         spin_lock_bh(&ioat->desc_lock);
1223
1224         desc = ioat_dma_get_next_descriptor(ioat);
1225
1226         if (!desc) {
1227                 dev_err(to_dev(chan),
1228                         "Unable to start null desc - get next desc failed\n");
1229                 spin_unlock_bh(&ioat->desc_lock);
1230                 return;
1231         }
1232
1233         hw = desc->hw;
1234         hw->ctl = 0;
1235         hw->ctl_f.null = 1;
1236         hw->ctl_f.int_en = 1;
1237         hw->ctl_f.compl_write = 1;
1238         /* set size to non-zero value (channel returns error when size is 0) */
1239         hw->size = NULL_DESC_BUFFER_SIZE;
1240         hw->src_addr = 0;
1241         hw->dst_addr = 0;
1242         async_tx_ack(&desc->txd);
1243         switch (chan->device->version) {
1244         case IOAT_VER_1_2:
1245                 hw->next = 0;
1246                 list_add_tail(&desc->node, &ioat->used_desc);
1247
1248                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1249                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1250                 writel(((u64) desc->txd.phys) >> 32,
1251                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1252
1253                 writeb(IOAT_CHANCMD_START, chan->reg_base
1254                         + IOAT_CHANCMD_OFFSET(chan->device->version));
1255                 break;
1256         case IOAT_VER_2_0:
1257         case IOAT_VER_3_0:
1258                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1259                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1260                 writel(((u64) desc->txd.phys) >> 32,
1261                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1262
1263                 ioat->dmacount++;
1264                 __ioat2_dma_memcpy_issue_pending(ioat);
1265                 break;
1266         }
1267         spin_unlock_bh(&ioat->desc_lock);
1268 }
1269
1270 /*
1271  * Perform a IOAT transaction to verify the HW works.
1272  */
1273 #define IOAT_TEST_SIZE 2000
1274
1275 static void ioat_dma_test_callback(void *dma_async_param)
1276 {
1277         struct completion *cmp = dma_async_param;
1278
1279         complete(cmp);
1280 }
1281
1282 /**
1283  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
1284  * @device: device to be tested
1285  */
1286 static int ioat_dma_self_test(struct ioatdma_device *device)
1287 {
1288         int i;
1289         u8 *src;
1290         u8 *dest;
1291         struct dma_device *dma = &device->common;
1292         struct device *dev = &device->pdev->dev;
1293         struct dma_chan *dma_chan;
1294         struct dma_async_tx_descriptor *tx;
1295         dma_addr_t dma_dest, dma_src;
1296         dma_cookie_t cookie;
1297         int err = 0;
1298         struct completion cmp;
1299         unsigned long tmo;
1300         unsigned long flags;
1301
1302         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1303         if (!src)
1304                 return -ENOMEM;
1305         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1306         if (!dest) {
1307                 kfree(src);
1308                 return -ENOMEM;
1309         }
1310
1311         /* Fill in src buffer */
1312         for (i = 0; i < IOAT_TEST_SIZE; i++)
1313                 src[i] = (u8)i;
1314
1315         /* Start copy, using first DMA channel */
1316         dma_chan = container_of(dma->channels.next, struct dma_chan,
1317                                 device_node);
1318         if (dma->device_alloc_chan_resources(dma_chan) < 1) {
1319                 dev_err(dev, "selftest cannot allocate chan resource\n");
1320                 err = -ENODEV;
1321                 goto out;
1322         }
1323
1324         dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
1325         dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
1326         flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
1327                 DMA_PREP_INTERRUPT;
1328         tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
1329                                                    IOAT_TEST_SIZE, flags);
1330         if (!tx) {
1331                 dev_err(dev, "Self-test prep failed, disabling\n");
1332                 err = -ENODEV;
1333                 goto free_resources;
1334         }
1335
1336         async_tx_ack(tx);
1337         init_completion(&cmp);
1338         tx->callback = ioat_dma_test_callback;
1339         tx->callback_param = &cmp;
1340         cookie = tx->tx_submit(tx);
1341         if (cookie < 0) {
1342                 dev_err(dev, "Self-test setup failed, disabling\n");
1343                 err = -ENODEV;
1344                 goto free_resources;
1345         }
1346         dma->device_issue_pending(dma_chan);
1347
1348         tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1349
1350         if (tmo == 0 ||
1351             dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
1352                                         != DMA_SUCCESS) {
1353                 dev_err(dev, "Self-test copy timed out, disabling\n");
1354                 err = -ENODEV;
1355                 goto free_resources;
1356         }
1357         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
1358                 dev_err(dev, "Self-test copy failed compare, disabling\n");
1359                 err = -ENODEV;
1360                 goto free_resources;
1361         }
1362
1363 free_resources:
1364         dma->device_free_chan_resources(dma_chan);
1365 out:
1366         kfree(src);
1367         kfree(dest);
1368         return err;
1369 }
1370
1371 static char ioat_interrupt_style[32] = "msix";
1372 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
1373                     sizeof(ioat_interrupt_style), 0644);
1374 MODULE_PARM_DESC(ioat_interrupt_style,
1375                  "set ioat interrupt style: msix (default), "
1376                  "msix-single-vector, msi, intx)");
1377
1378 /**
1379  * ioat_dma_setup_interrupts - setup interrupt handler
1380  * @device: ioat device
1381  */
1382 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
1383 {
1384         struct ioat_chan_common *chan;
1385         struct pci_dev *pdev = device->pdev;
1386         struct device *dev = &pdev->dev;
1387         struct msix_entry *msix;
1388         int i, j, msixcnt;
1389         int err = -EINVAL;
1390         u8 intrctrl = 0;
1391
1392         if (!strcmp(ioat_interrupt_style, "msix"))
1393                 goto msix;
1394         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
1395                 goto msix_single_vector;
1396         if (!strcmp(ioat_interrupt_style, "msi"))
1397                 goto msi;
1398         if (!strcmp(ioat_interrupt_style, "intx"))
1399                 goto intx;
1400         dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1401         goto err_no_irq;
1402
1403 msix:
1404         /* The number of MSI-X vectors should equal the number of channels */
1405         msixcnt = device->common.chancnt;
1406         for (i = 0; i < msixcnt; i++)
1407                 device->msix_entries[i].entry = i;
1408
1409         err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1410         if (err < 0)
1411                 goto msi;
1412         if (err > 0)
1413                 goto msix_single_vector;
1414
1415         for (i = 0; i < msixcnt; i++) {
1416                 msix = &device->msix_entries[i];
1417                 chan = ioat_chan_by_index(device, i);
1418                 err = devm_request_irq(dev, msix->vector,
1419                                        ioat_dma_do_interrupt_msix, 0,
1420                                        "ioat-msix", chan);
1421                 if (err) {
1422                         for (j = 0; j < i; j++) {
1423                                 msix = &device->msix_entries[j];
1424                                 chan = ioat_chan_by_index(device, j);
1425                                 devm_free_irq(dev, msix->vector, chan);
1426                         }
1427                         goto msix_single_vector;
1428                 }
1429         }
1430         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1431         goto done;
1432
1433 msix_single_vector:
1434         msix = &device->msix_entries[0];
1435         msix->entry = 0;
1436         err = pci_enable_msix(pdev, device->msix_entries, 1);
1437         if (err)
1438                 goto msi;
1439
1440         err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1441                                "ioat-msix", device);
1442         if (err) {
1443                 pci_disable_msix(pdev);
1444                 goto msi;
1445         }
1446         goto done;
1447
1448 msi:
1449         err = pci_enable_msi(pdev);
1450         if (err)
1451                 goto intx;
1452
1453         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1454                                "ioat-msi", device);
1455         if (err) {
1456                 pci_disable_msi(pdev);
1457                 goto intx;
1458         }
1459         goto done;
1460
1461 intx:
1462         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1463                                IRQF_SHARED, "ioat-intx", device);
1464         if (err)
1465                 goto err_no_irq;
1466
1467 done:
1468         if (device->intr_quirk)
1469                 device->intr_quirk(device);
1470         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1471         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1472         return 0;
1473
1474 err_no_irq:
1475         /* Disable all interrupt generation */
1476         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1477         dev_err(dev, "no usable interrupts\n");
1478         return err;
1479 }
1480
1481 static void ioat_disable_interrupts(struct ioatdma_device *device)
1482 {
1483         /* Disable all interrupt generation */
1484         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1485 }
1486
1487 static int ioat_probe(struct ioatdma_device *device)
1488 {
1489         int err = -ENODEV;
1490         struct dma_device *dma = &device->common;
1491         struct pci_dev *pdev = device->pdev;
1492         struct device *dev = &pdev->dev;
1493
1494         /* DMA coherent memory pool for DMA descriptor allocations */
1495         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1496                                            sizeof(struct ioat_dma_descriptor),
1497                                            64, 0);
1498         if (!device->dma_pool) {
1499                 err = -ENOMEM;
1500                 goto err_dma_pool;
1501         }
1502
1503         device->completion_pool = pci_pool_create("completion_pool", pdev,
1504                                                   sizeof(u64), SMP_CACHE_BYTES,
1505                                                   SMP_CACHE_BYTES);
1506         if (!device->completion_pool) {
1507                 err = -ENOMEM;
1508                 goto err_completion_pool;
1509         }
1510
1511         ioat_dma_enumerate_channels(device);
1512
1513         dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1514         dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1515         dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1516         dma->device_is_tx_complete = ioat_dma_is_complete;
1517         dma->dev = &pdev->dev;
1518
1519         dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1520                 " %d channels, device version 0x%02x, driver version %s\n",
1521                 dma->chancnt, device->version, IOAT_DMA_VERSION);
1522
1523         if (!dma->chancnt) {
1524                 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1525                         "zero channels detected\n");
1526                 goto err_setup_interrupts;
1527         }
1528
1529         err = ioat_dma_setup_interrupts(device);
1530         if (err)
1531                 goto err_setup_interrupts;
1532
1533         err = ioat_dma_self_test(device);
1534         if (err)
1535                 goto err_self_test;
1536
1537         return 0;
1538
1539 err_self_test:
1540         ioat_disable_interrupts(device);
1541 err_setup_interrupts:
1542         pci_pool_destroy(device->completion_pool);
1543 err_completion_pool:
1544         pci_pool_destroy(device->dma_pool);
1545 err_dma_pool:
1546         return err;
1547 }
1548
1549 static int ioat_register(struct ioatdma_device *device)
1550 {
1551         int err = dma_async_device_register(&device->common);
1552
1553         if (err) {
1554                 ioat_disable_interrupts(device);
1555                 pci_pool_destroy(device->completion_pool);
1556                 pci_pool_destroy(device->dma_pool);
1557         }
1558
1559         return err;
1560 }
1561
1562 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1563 static void ioat1_intr_quirk(struct ioatdma_device *device)
1564 {
1565         struct pci_dev *pdev = device->pdev;
1566         u32 dmactrl;
1567
1568         pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1569         if (pdev->msi_enabled)
1570                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1571         else
1572                 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1573         pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1574 }
1575
1576 int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1577 {
1578         struct pci_dev *pdev = device->pdev;
1579         struct dma_device *dma;
1580         int err;
1581
1582         device->intr_quirk = ioat1_intr_quirk;
1583         dma = &device->common;
1584         dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1585         dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1586
1587         err = ioat_probe(device);
1588         if (err)
1589                 return err;
1590         ioat_set_tcp_copy_break(4096);
1591         err = ioat_register(device);
1592         if (err)
1593                 return err;
1594         if (dca)
1595                 device->dca = ioat_dca_init(pdev, device->reg_base);
1596
1597         INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1598         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1599
1600         return err;
1601 }
1602
1603 int ioat2_dma_probe(struct ioatdma_device *device, int dca)
1604 {
1605         struct pci_dev *pdev = device->pdev;
1606         struct dma_device *dma;
1607         struct dma_chan *c;
1608         struct ioat_chan_common *chan;
1609         int err;
1610
1611         dma = &device->common;
1612         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1613         dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1614
1615         err = ioat_probe(device);
1616         if (err)
1617                 return err;
1618         ioat_set_tcp_copy_break(2048);
1619
1620         list_for_each_entry(c, &dma->channels, device_node) {
1621                 chan = to_chan_common(c);
1622                 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
1623                        chan->reg_base + IOAT_DCACTRL_OFFSET);
1624         }
1625
1626         err = ioat_register(device);
1627         if (err)
1628                 return err;
1629         if (dca)
1630                 device->dca = ioat2_dca_init(pdev, device->reg_base);
1631
1632         INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1633         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1634
1635         return err;
1636 }
1637
1638 int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1639 {
1640         struct pci_dev *pdev = device->pdev;
1641         struct dma_device *dma;
1642         struct dma_chan *c;
1643         struct ioat_chan_common *chan;
1644         int err;
1645         u16 dev_id;
1646
1647         dma = &device->common;
1648         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1649         dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1650
1651         /* -= IOAT ver.3 workarounds =- */
1652         /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1653          * that can cause stability issues for IOAT ver.3
1654          */
1655         pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1656
1657         /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1658          * (workaround for spurious config parity error after restart)
1659          */
1660         pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1661         if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1662                 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1663
1664         err = ioat_probe(device);
1665         if (err)
1666                 return err;
1667         ioat_set_tcp_copy_break(262144);
1668
1669         list_for_each_entry(c, &dma->channels, device_node) {
1670                 chan = to_chan_common(c);
1671                 writel(IOAT_DMA_DCA_ANY_CPU,
1672                        chan->reg_base + IOAT_DCACTRL_OFFSET);
1673         }
1674
1675         err = ioat_register(device);
1676         if (err)
1677                 return err;
1678         if (dca)
1679                 device->dca = ioat3_dca_init(pdev, device->reg_base);
1680
1681         return err;
1682 }
1683
1684 void ioat_dma_remove(struct ioatdma_device *device)
1685 {
1686         struct dma_device *dma = &device->common;
1687
1688         if (device->version != IOAT_VER_3_0)
1689                 cancel_delayed_work(&device->work);
1690
1691         ioat_disable_interrupts(device);
1692
1693         dma_async_device_unregister(dma);
1694
1695         pci_pool_destroy(device->dma_pool);
1696         pci_pool_destroy(device->completion_pool);
1697
1698         INIT_LIST_HEAD(&dma->channels);
1699 }
1700