ioat2,3: convert to a true ring buffer
[linux-2.6.git] / drivers / dma / ioat / dma_v2.c
1 /*
2  * Intel I/OAT DMA Linux driver
3  * Copyright(c) 2004 - 2009 Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17  *
18  * The full GNU General Public License is included in this distribution in
19  * the file called "COPYING".
20  *
21  */
22
23 /*
24  * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
25  * does asynchronous data movement and checksumming operations.
26  */
27
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/workqueue.h>
36 #include <linux/i7300_idle.h>
37 #include "dma.h"
38 #include "dma_v2.h"
39 #include "registers.h"
40 #include "hw.h"
41
42 static int ioat_ring_alloc_order = 8;
43 module_param(ioat_ring_alloc_order, int, 0644);
44 MODULE_PARM_DESC(ioat_ring_alloc_order,
45                  "ioat2+: allocate 2^n descriptors per channel (default: n=8)");
46
47 static void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
48 {
49         void * __iomem reg_base = ioat->base.reg_base;
50
51         ioat->pending = 0;
52         ioat->dmacount += ioat2_ring_pending(ioat);
53         ioat->issued = ioat->head;
54         /* make descriptor updates globally visible before notifying channel */
55         wmb();
56         writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
57
58 }
59
60 static void ioat2_issue_pending(struct dma_chan *chan)
61 {
62         struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
63
64         spin_lock_bh(&ioat->ring_lock);
65         if (ioat->pending == 1)
66                 __ioat2_issue_pending(ioat);
67         spin_unlock_bh(&ioat->ring_lock);
68 }
69
70 /**
71  * ioat2_update_pending - log pending descriptors
72  * @ioat: ioat2+ channel
73  *
74  * set pending to '1' unless pending is already set to '2', pending == 2
75  * indicates that submission is temporarily blocked due to an in-flight
76  * reset.  If we are already above the ioat_pending_level threshold then
77  * just issue pending.
78  *
79  * called with ring_lock held
80  */
81 static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
82 {
83         if (unlikely(ioat->pending == 2))
84                 return;
85         else if (ioat2_ring_pending(ioat) > ioat_pending_level)
86                 __ioat2_issue_pending(ioat);
87         else
88                 ioat->pending = 1;
89 }
90
91 static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
92 {
93         void __iomem *reg_base = ioat->base.reg_base;
94         struct ioat_ring_ent *desc;
95         struct ioat_dma_descriptor *hw;
96         int idx;
97
98         if (ioat2_ring_space(ioat) < 1) {
99                 dev_err(to_dev(&ioat->base),
100                         "Unable to start null desc - ring full\n");
101                 return;
102         }
103
104         idx = ioat2_desc_alloc(ioat, 1);
105         desc = ioat2_get_ring_ent(ioat, idx);
106
107         hw = desc->hw;
108         hw->ctl = 0;
109         hw->ctl_f.null = 1;
110         hw->ctl_f.int_en = 1;
111         hw->ctl_f.compl_write = 1;
112         /* set size to non-zero value (channel returns error when size is 0) */
113         hw->size = NULL_DESC_BUFFER_SIZE;
114         hw->src_addr = 0;
115         hw->dst_addr = 0;
116         async_tx_ack(&desc->txd);
117         writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
118                reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
119         writel(((u64) desc->txd.phys) >> 32,
120                reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
121         __ioat2_issue_pending(ioat);
122 }
123
124 static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
125 {
126         spin_lock_bh(&ioat->ring_lock);
127         __ioat2_start_null_desc(ioat);
128         spin_unlock_bh(&ioat->ring_lock);
129 }
130
131 static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
132
133 /**
134  * ioat2_reset_part2 - reinit the channel after a reset
135  */
136 static void ioat2_reset_part2(struct work_struct *work)
137 {
138         struct ioat_chan_common *chan;
139         struct ioat2_dma_chan *ioat;
140
141         chan = container_of(work, struct ioat_chan_common, work.work);
142         ioat = container_of(chan, struct ioat2_dma_chan, base);
143
144         /* ensure that ->tail points to the stalled descriptor
145          * (ioat->pending is set to 2 at this point so no new
146          * descriptors will be issued while we perform this cleanup)
147          */
148         ioat2_cleanup(ioat);
149
150         spin_lock_bh(&chan->cleanup_lock);
151         spin_lock_bh(&ioat->ring_lock);
152
153         /* set the tail to be re-issued */
154         ioat->issued = ioat->tail;
155         ioat->dmacount = 0;
156
157         if (ioat2_ring_pending(ioat)) {
158                 struct ioat_ring_ent *desc;
159
160                 desc = ioat2_get_ring_ent(ioat, ioat->tail);
161                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
162                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
163                 writel(((u64) desc->txd.phys) >> 32,
164                        chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
165                 __ioat2_issue_pending(ioat);
166         } else
167                 __ioat2_start_null_desc(ioat);
168
169         spin_unlock_bh(&ioat->ring_lock);
170         spin_unlock_bh(&chan->cleanup_lock);
171
172         dev_info(to_dev(chan),
173                  "chan%d reset - %d descs waiting, %d total desc\n",
174                  chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
175 }
176
177 /**
178  * ioat2_reset_channel - restart a channel
179  * @ioat: IOAT DMA channel handle
180  */
181 static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
182 {
183         u32 chansts, chanerr;
184         struct ioat_chan_common *chan = &ioat->base;
185         u16 active;
186
187         spin_lock_bh(&ioat->ring_lock);
188         active = ioat2_ring_active(ioat);
189         spin_unlock_bh(&ioat->ring_lock);
190         if (!active)
191                 return;
192
193         chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
194         chansts = (chan->completion_virt->low
195                                         & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
196         if (chanerr) {
197                 dev_err(to_dev(chan),
198                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
199                         chan_num(chan), chansts, chanerr);
200                 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
201         }
202
203         spin_lock_bh(&ioat->ring_lock);
204         ioat->pending = 2;
205         writeb(IOAT_CHANCMD_RESET,
206                chan->reg_base
207                + IOAT_CHANCMD_OFFSET(chan->device->version));
208         spin_unlock_bh(&ioat->ring_lock);
209         schedule_delayed_work(&chan->work, RESET_DELAY);
210 }
211
212 /**
213  * ioat2_chan_watchdog - watch for stuck channels
214  */
215 static void ioat2_chan_watchdog(struct work_struct *work)
216 {
217         struct ioatdma_device *device =
218                 container_of(work, struct ioatdma_device, work.work);
219         struct ioat2_dma_chan *ioat;
220         struct ioat_chan_common *chan;
221         u16 active;
222         int i;
223
224         for (i = 0; i < device->common.chancnt; i++) {
225                 chan = ioat_chan_by_index(device, i);
226                 ioat = container_of(chan, struct ioat2_dma_chan, base);
227
228                 /*
229                  * for version 2.0 if there are descriptors yet to be processed
230                  * and the last completed hasn't changed since the last watchdog
231                  *      if they haven't hit the pending level
232                  *          issue the pending to push them through
233                  *      else
234                  *          try resetting the channel
235                  */
236                 spin_lock_bh(&ioat->ring_lock);
237                 active = ioat2_ring_active(ioat);
238                 spin_unlock_bh(&ioat->ring_lock);
239
240                 if (active &&
241                     chan->last_completion &&
242                     chan->last_completion == chan->watchdog_completion) {
243
244                         if (ioat->pending == 1)
245                                 ioat2_issue_pending(&chan->common);
246                         else {
247                                 ioat2_reset_channel(ioat);
248                                 chan->watchdog_completion = 0;
249                         }
250                 } else {
251                         chan->last_compl_desc_addr_hw = 0;
252                         chan->watchdog_completion = chan->last_completion;
253                 }
254                 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
255         }
256         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
257 }
258
259 /**
260  * ioat2_cleanup - clean finished descriptors (advance tail pointer)
261  * @chan: ioat channel to be cleaned up
262  */
263 static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
264 {
265         struct ioat_chan_common *chan = &ioat->base;
266         unsigned long phys_complete;
267         struct ioat_ring_ent *desc;
268         bool seen_current = false;
269         u16 active;
270         int i;
271         struct dma_async_tx_descriptor *tx;
272
273         prefetch(chan->completion_virt);
274
275         spin_lock_bh(&chan->cleanup_lock);
276         phys_complete = ioat_get_current_completion(chan);
277         if (phys_complete == chan->last_completion) {
278                 spin_unlock_bh(&chan->cleanup_lock);
279                 /*
280                  * perhaps we're stuck so hard that the watchdog can't go off?
281                  * try to catch it after WATCHDOG_DELAY seconds
282                  */
283                 if (chan->device->version < IOAT_VER_3_0) {
284                         unsigned long tmo;
285
286                         tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
287                         if (time_after(jiffies, tmo)) {
288                                 ioat2_chan_watchdog(&(chan->device->work.work));
289                                 chan->last_completion_time = jiffies;
290                         }
291                 }
292                 return;
293         }
294         chan->last_completion_time = jiffies;
295
296         spin_lock_bh(&ioat->ring_lock);
297
298         active = ioat2_ring_active(ioat);
299         for (i = 0; i < active && !seen_current; i++) {
300                 prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
301                 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
302                 tx = &desc->txd;
303                 if (tx->cookie) {
304                         ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
305                         chan->completed_cookie = tx->cookie;
306                         tx->cookie = 0;
307                         if (tx->callback) {
308                                 tx->callback(tx->callback_param);
309                                 tx->callback = NULL;
310                         }
311                 }
312
313                 if (tx->phys == phys_complete)
314                         seen_current = true;
315         }
316         ioat->tail += i;
317         BUG_ON(!seen_current); /* no active descs have written a completion? */
318         spin_unlock_bh(&ioat->ring_lock);
319
320         chan->last_completion = phys_complete;
321
322         spin_unlock_bh(&chan->cleanup_lock);
323 }
324
325 static void ioat2_cleanup_tasklet(unsigned long data)
326 {
327         struct ioat2_dma_chan *ioat = (void *) data;
328
329         ioat2_cleanup(ioat);
330         writew(IOAT_CHANCTRL_INT_DISABLE,
331                ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
332 }
333
334 /**
335  * ioat2_enumerate_channels - find and initialize the device's channels
336  * @device: the device to be enumerated
337  */
338 static int ioat2_enumerate_channels(struct ioatdma_device *device)
339 {
340         struct ioat2_dma_chan *ioat;
341         struct device *dev = &device->pdev->dev;
342         struct dma_device *dma = &device->common;
343         u8 xfercap_log;
344         int i;
345
346         INIT_LIST_HEAD(&dma->channels);
347         dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
348         xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
349         if (xfercap_log == 0)
350                 return 0;
351
352         /* FIXME which i/oat version is i7300? */
353 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
354         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
355                 dma->chancnt--;
356 #endif
357         for (i = 0; i < dma->chancnt; i++) {
358                 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
359                 if (!ioat)
360                         break;
361
362                 ioat_init_channel(device, &ioat->base, i,
363                                   ioat2_reset_part2,
364                                   ioat2_cleanup_tasklet,
365                                   (unsigned long) ioat);
366                 ioat->xfercap_log = xfercap_log;
367                 spin_lock_init(&ioat->ring_lock);
368         }
369         dma->chancnt = i;
370         return i;
371 }
372
373 static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
374 {
375         struct dma_chan *c = tx->chan;
376         struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
377         dma_cookie_t cookie = c->cookie;
378
379         cookie++;
380         if (cookie < 0)
381                 cookie = 1;
382         tx->cookie = cookie;
383         c->cookie = cookie;
384         ioat2_update_pending(ioat);
385         spin_unlock_bh(&ioat->ring_lock);
386
387         return cookie;
388 }
389
390 static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan)
391 {
392         struct ioat_dma_descriptor *hw;
393         struct ioat_ring_ent *desc;
394         struct ioatdma_device *dma;
395         dma_addr_t phys;
396
397         dma = to_ioatdma_device(chan->device);
398         hw = pci_pool_alloc(dma->dma_pool, GFP_KERNEL, &phys);
399         if (!hw)
400                 return NULL;
401         memset(hw, 0, sizeof(*hw));
402
403         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
404         if (!desc) {
405                 pci_pool_free(dma->dma_pool, hw, phys);
406                 return NULL;
407         }
408
409         dma_async_tx_descriptor_init(&desc->txd, chan);
410         desc->txd.tx_submit = ioat2_tx_submit_unlock;
411         desc->hw = hw;
412         desc->txd.phys = phys;
413         return desc;
414 }
415
416 static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
417 {
418         struct ioatdma_device *dma;
419
420         dma = to_ioatdma_device(chan->device);
421         pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
422         kfree(desc);
423 }
424
425 /* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
426  * @chan: channel to be initialized
427  */
428 static int ioat2_alloc_chan_resources(struct dma_chan *c)
429 {
430         struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
431         struct ioat_chan_common *chan = &ioat->base;
432         struct ioat_ring_ent **ring;
433         u16 chanctrl;
434         u32 chanerr;
435         int descs;
436         int i;
437
438         /* have we already been set up? */
439         if (ioat->ring)
440                 return 1 << ioat->alloc_order;
441
442         /* Setup register to interrupt and write completion status on error */
443         chanctrl = IOAT_CHANCTRL_ERR_INT_EN | IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
444                    IOAT_CHANCTRL_ERR_COMPLETION_EN;
445         writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
446
447         chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
448         if (chanerr) {
449                 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
450                 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
451         }
452
453         /* allocate a completion writeback area */
454         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
455         chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
456                                                GFP_KERNEL,
457                                                &chan->completion_addr);
458         if (!chan->completion_virt)
459                 return -ENOMEM;
460
461         memset(chan->completion_virt, 0,
462                sizeof(*chan->completion_virt));
463         writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
464                chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
465         writel(((u64) chan->completion_addr) >> 32,
466                chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
467
468         ioat->alloc_order = ioat_get_alloc_order();
469         descs = 1 << ioat->alloc_order;
470
471         /* allocate the array to hold the software ring */
472         ring = kcalloc(descs, sizeof(*ring), GFP_KERNEL);
473         if (!ring)
474                 return -ENOMEM;
475         for (i = 0; i < descs; i++) {
476                 ring[i] = ioat2_alloc_ring_ent(c);
477                 if (!ring[i]) {
478                         while (i--)
479                                 ioat2_free_ring_ent(ring[i], c);
480                         kfree(ring);
481                         return -ENOMEM;
482                 }
483         }
484
485         /* link descs */
486         for (i = 0; i < descs-1; i++) {
487                 struct ioat_ring_ent *next = ring[i+1];
488                 struct ioat_dma_descriptor *hw = ring[i]->hw;
489
490                 hw->next = next->txd.phys;
491         }
492         ring[i]->hw->next = ring[0]->txd.phys;
493
494         spin_lock_bh(&ioat->ring_lock);
495         ioat->ring = ring;
496         ioat->head = 0;
497         ioat->issued = 0;
498         ioat->tail = 0;
499         ioat->pending = 0;
500         spin_unlock_bh(&ioat->ring_lock);
501
502         tasklet_enable(&chan->cleanup_task);
503         ioat2_start_null_desc(ioat);
504
505         return descs;
506 }
507
508 /**
509  * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
510  * @idx: gets starting descriptor index on successful allocation
511  * @ioat: ioat2,3 channel (ring) to operate on
512  * @num_descs: allocation length
513  */
514 static int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
515 {
516         struct ioat_chan_common *chan = &ioat->base;
517
518         spin_lock_bh(&ioat->ring_lock);
519         if (unlikely(ioat2_ring_space(ioat) < num_descs)) {
520                 if (printk_ratelimit())
521                         dev_dbg(to_dev(chan),
522                                 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
523                                 __func__, num_descs, ioat->head, ioat->tail,
524                                 ioat->issued);
525                 spin_unlock_bh(&ioat->ring_lock);
526
527                 /* do direct reclaim in the allocation failure case */
528                 ioat2_cleanup(ioat);
529
530                 return -ENOMEM;
531         }
532
533         dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
534                 __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
535
536         *idx = ioat2_desc_alloc(ioat, num_descs);
537         return 0;  /* with ioat->ring_lock held */
538 }
539
540 static struct dma_async_tx_descriptor *
541 ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
542                            dma_addr_t dma_src, size_t len, unsigned long flags)
543 {
544         struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
545         struct ioat_dma_descriptor *hw;
546         struct ioat_ring_ent *desc;
547         dma_addr_t dst = dma_dest;
548         dma_addr_t src = dma_src;
549         size_t total_len = len;
550         int num_descs;
551         u16 idx;
552         int i;
553
554         num_descs = ioat2_xferlen_to_descs(ioat, len);
555         if (likely(num_descs) &&
556             ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
557                 /* pass */;
558         else
559                 return NULL;
560         for (i = 0; i < num_descs; i++) {
561                 size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
562
563                 desc = ioat2_get_ring_ent(ioat, idx + i);
564                 hw = desc->hw;
565
566                 hw->size = copy;
567                 hw->ctl = 0;
568                 hw->src_addr = src;
569                 hw->dst_addr = dst;
570
571                 len -= copy;
572                 dst += copy;
573                 src += copy;
574         }
575
576         desc->txd.flags = flags;
577         desc->len = total_len;
578         hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
579         hw->ctl_f.compl_write = 1;
580         /* we leave the channel locked to ensure in order submission */
581
582         return &desc->txd;
583 }
584
585 /**
586  * ioat2_free_chan_resources - release all the descriptors
587  * @chan: the channel to be cleaned
588  */
589 static void ioat2_free_chan_resources(struct dma_chan *c)
590 {
591         struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
592         struct ioat_chan_common *chan = &ioat->base;
593         struct ioatdma_device *ioatdma_device = chan->device;
594         struct ioat_ring_ent *desc;
595         const u16 total_descs = 1 << ioat->alloc_order;
596         int descs;
597         int i;
598
599         /* Before freeing channel resources first check
600          * if they have been previously allocated for this channel.
601          */
602         if (!ioat->ring)
603                 return;
604
605         tasklet_disable(&chan->cleanup_task);
606         ioat2_cleanup(ioat);
607
608         /* Delay 100ms after reset to allow internal DMA logic to quiesce
609          * before removing DMA descriptor resources.
610          */
611         writeb(IOAT_CHANCMD_RESET,
612                chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
613         mdelay(100);
614
615         spin_lock_bh(&ioat->ring_lock);
616         descs = ioat2_ring_space(ioat);
617         for (i = 0; i < descs; i++) {
618                 desc = ioat2_get_ring_ent(ioat, ioat->head + i);
619                 ioat2_free_ring_ent(desc, c);
620         }
621
622         if (descs < total_descs)
623                 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
624                         total_descs - descs);
625
626         for (i = 0; i < total_descs - descs; i++) {
627                 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
628                 ioat2_free_ring_ent(desc, c);
629         }
630
631         kfree(ioat->ring);
632         ioat->ring = NULL;
633         ioat->alloc_order = 0;
634         pci_pool_free(ioatdma_device->completion_pool,
635                       chan->completion_virt,
636                       chan->completion_addr);
637         spin_unlock_bh(&ioat->ring_lock);
638
639         chan->last_completion = 0;
640         chan->completion_addr = 0;
641         ioat->pending = 0;
642         ioat->dmacount = 0;
643         chan->watchdog_completion = 0;
644         chan->last_compl_desc_addr_hw = 0;
645         chan->watchdog_tcp_cookie = 0;
646         chan->watchdog_last_tcp_cookie = 0;
647 }
648
649 static enum dma_status
650 ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
651                      dma_cookie_t *done, dma_cookie_t *used)
652 {
653         struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
654
655         if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
656                 return DMA_SUCCESS;
657
658         ioat2_cleanup(ioat);
659
660         return ioat_is_complete(c, cookie, done, used);
661 }
662
663 int ioat2_dma_probe(struct ioatdma_device *device, int dca)
664 {
665         struct pci_dev *pdev = device->pdev;
666         struct dma_device *dma;
667         struct dma_chan *c;
668         struct ioat_chan_common *chan;
669         int err;
670
671         device->enumerate_channels = ioat2_enumerate_channels;
672         dma = &device->common;
673         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
674         dma->device_issue_pending = ioat2_issue_pending;
675         dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
676         dma->device_free_chan_resources = ioat2_free_chan_resources;
677         dma->device_is_tx_complete = ioat2_is_complete;
678
679         err = ioat_probe(device);
680         if (err)
681                 return err;
682         ioat_set_tcp_copy_break(2048);
683
684         list_for_each_entry(c, &dma->channels, device_node) {
685                 chan = to_chan_common(c);
686                 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
687                        chan->reg_base + IOAT_DCACTRL_OFFSET);
688         }
689
690         err = ioat_register(device);
691         if (err)
692                 return err;
693         if (dca)
694                 device->dca = ioat2_dca_init(pdev, device->reg_base);
695
696         INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
697         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
698
699         return err;
700 }
701
702 int ioat3_dma_probe(struct ioatdma_device *device, int dca)
703 {
704         struct pci_dev *pdev = device->pdev;
705         struct dma_device *dma;
706         struct dma_chan *c;
707         struct ioat_chan_common *chan;
708         int err;
709         u16 dev_id;
710
711         device->enumerate_channels = ioat2_enumerate_channels;
712         dma = &device->common;
713         dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
714         dma->device_issue_pending = ioat2_issue_pending;
715         dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
716         dma->device_free_chan_resources = ioat2_free_chan_resources;
717         dma->device_is_tx_complete = ioat2_is_complete;
718
719         /* -= IOAT ver.3 workarounds =- */
720         /* Write CHANERRMSK_INT with 3E07h to mask out the errors
721          * that can cause stability issues for IOAT ver.3
722          */
723         pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
724
725         /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
726          * (workaround for spurious config parity error after restart)
727          */
728         pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
729         if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
730                 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
731
732         err = ioat_probe(device);
733         if (err)
734                 return err;
735         ioat_set_tcp_copy_break(262144);
736
737         list_for_each_entry(c, &dma->channels, device_node) {
738                 chan = to_chan_common(c);
739                 writel(IOAT_DMA_DCA_ANY_CPU,
740                        chan->reg_base + IOAT_DCACTRL_OFFSET);
741         }
742
743         err = ioat_register(device);
744         if (err)
745                 return err;
746         if (dca)
747                 device->dca = ioat3_dca_init(pdev, device->reg_base);
748
749         return err;
750 }