ioat: ignore reserved bits for chancnt and xfercap
[linux-2.6.git] / drivers / dma / ioat / dma.c
1 /*
2  * Intel I/OAT DMA Linux driver
3  * Copyright(c) 2004 - 2009 Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope that it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  *
14  * You should have received a copy of the GNU General Public License along with
15  * this program; if not, write to the Free Software Foundation, Inc.,
16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17  *
18  * The full GNU General Public License is included in this distribution in
19  * the file called "COPYING".
20  *
21  */
22
23 /*
24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
25  * copy operations.
26  */
27
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/workqueue.h>
36 #include <linux/i7300_idle.h>
37 #include "dma.h"
38 #include "registers.h"
39 #include "hw.h"
40
41 int ioat_pending_level = 4;
42 module_param(ioat_pending_level, int, 0644);
43 MODULE_PARM_DESC(ioat_pending_level,
44                  "high-water mark for pushing ioat descriptors (default: 4)");
45
46 /* internal functions */
47 static void ioat1_cleanup(struct ioat_dma_chan *ioat);
48 static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
49
50 /**
51  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
52  * @irq: interrupt id
53  * @data: interrupt data
54  */
55 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
56 {
57         struct ioatdma_device *instance = data;
58         struct ioat_chan_common *chan;
59         unsigned long attnstatus;
60         int bit;
61         u8 intrctrl;
62
63         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
64
65         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
66                 return IRQ_NONE;
67
68         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
69                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
70                 return IRQ_NONE;
71         }
72
73         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
74         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
75                 chan = ioat_chan_by_index(instance, bit);
76                 tasklet_schedule(&chan->cleanup_task);
77         }
78
79         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
80         return IRQ_HANDLED;
81 }
82
83 /**
84  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
85  * @irq: interrupt id
86  * @data: interrupt data
87  */
88 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
89 {
90         struct ioat_chan_common *chan = data;
91
92         tasklet_schedule(&chan->cleanup_task);
93
94         return IRQ_HANDLED;
95 }
96
97 static void ioat1_cleanup_tasklet(unsigned long data);
98
99 /* common channel initialization */
100 void ioat_init_channel(struct ioatdma_device *device,
101                        struct ioat_chan_common *chan, int idx,
102                        work_func_t work_fn, void (*tasklet)(unsigned long),
103                        unsigned long tasklet_data)
104 {
105         struct dma_device *dma = &device->common;
106
107         chan->device = device;
108         chan->reg_base = device->reg_base + (0x80 * (idx + 1));
109         INIT_DELAYED_WORK(&chan->work, work_fn);
110         spin_lock_init(&chan->cleanup_lock);
111         chan->common.device = dma;
112         list_add_tail(&chan->common.device_node, &dma->channels);
113         device->idx[idx] = chan;
114         tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
115         tasklet_disable(&chan->cleanup_task);
116 }
117
118 static void ioat1_reset_part2(struct work_struct *work);
119
120 /**
121  * ioat1_dma_enumerate_channels - find and initialize the device's channels
122  * @device: the device to be enumerated
123  */
124 static int ioat1_enumerate_channels(struct ioatdma_device *device)
125 {
126         u8 xfercap_scale;
127         u32 xfercap;
128         int i;
129         struct ioat_dma_chan *ioat;
130         struct device *dev = &device->pdev->dev;
131         struct dma_device *dma = &device->common;
132
133         INIT_LIST_HEAD(&dma->channels);
134         dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
135         dma->chancnt &= 0x1f; /* bits [4:0] valid */
136         if (dma->chancnt > ARRAY_SIZE(device->idx)) {
137                 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
138                          dma->chancnt, ARRAY_SIZE(device->idx));
139                 dma->chancnt = ARRAY_SIZE(device->idx);
140         }
141         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
142         xfercap_scale &= 0x1f; /* bits [4:0] valid */
143         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
144         dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
145
146 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
147         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
148                 dma->chancnt--;
149 #endif
150         for (i = 0; i < dma->chancnt; i++) {
151                 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
152                 if (!ioat)
153                         break;
154
155                 ioat_init_channel(device, &ioat->base, i,
156                                   ioat1_reset_part2,
157                                   ioat1_cleanup_tasklet,
158                                   (unsigned long) ioat);
159                 ioat->xfercap = xfercap;
160                 spin_lock_init(&ioat->desc_lock);
161                 INIT_LIST_HEAD(&ioat->free_desc);
162                 INIT_LIST_HEAD(&ioat->used_desc);
163         }
164         dma->chancnt = i;
165         return i;
166 }
167
168 /**
169  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
170  *                                 descriptors to hw
171  * @chan: DMA channel handle
172  */
173 static inline void
174 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
175 {
176         void __iomem *reg_base = ioat->base.reg_base;
177
178         dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
179                 __func__, ioat->pending);
180         ioat->pending = 0;
181         writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
182 }
183
184 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
185 {
186         struct ioat_dma_chan *ioat = to_ioat_chan(chan);
187
188         if (ioat->pending > 0) {
189                 spin_lock_bh(&ioat->desc_lock);
190                 __ioat1_dma_memcpy_issue_pending(ioat);
191                 spin_unlock_bh(&ioat->desc_lock);
192         }
193 }
194
195 /**
196  * ioat1_reset_part2 - reinit the channel after a reset
197  */
198 static void ioat1_reset_part2(struct work_struct *work)
199 {
200         struct ioat_chan_common *chan;
201         struct ioat_dma_chan *ioat;
202         struct ioat_desc_sw *desc;
203         int dmacount;
204         bool start_null = false;
205
206         chan = container_of(work, struct ioat_chan_common, work.work);
207         ioat = container_of(chan, struct ioat_dma_chan, base);
208         spin_lock_bh(&chan->cleanup_lock);
209         spin_lock_bh(&ioat->desc_lock);
210
211         *chan->completion = 0;
212         ioat->pending = 0;
213
214         /* count the descriptors waiting */
215         dmacount = 0;
216         if (ioat->used_desc.prev) {
217                 desc = to_ioat_desc(ioat->used_desc.prev);
218                 do {
219                         dmacount++;
220                         desc = to_ioat_desc(desc->node.next);
221                 } while (&desc->node != ioat->used_desc.next);
222         }
223
224         if (dmacount) {
225                 /*
226                  * write the new starting descriptor address
227                  * this puts channel engine into ARMED state
228                  */
229                 desc = to_ioat_desc(ioat->used_desc.prev);
230                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
231                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
232                 writel(((u64) desc->txd.phys) >> 32,
233                        chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
234
235                 writeb(IOAT_CHANCMD_START, chan->reg_base
236                         + IOAT_CHANCMD_OFFSET(chan->device->version));
237         } else
238                 start_null = true;
239         spin_unlock_bh(&ioat->desc_lock);
240         spin_unlock_bh(&chan->cleanup_lock);
241
242         dev_err(to_dev(chan),
243                 "chan%d reset - %d descs waiting, %d total desc\n",
244                 chan_num(chan), dmacount, ioat->desccount);
245
246         if (start_null)
247                 ioat1_dma_start_null_desc(ioat);
248 }
249
250 /**
251  * ioat1_reset_channel - restart a channel
252  * @ioat: IOAT DMA channel handle
253  */
254 static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
255 {
256         struct ioat_chan_common *chan = &ioat->base;
257         void __iomem *reg_base = chan->reg_base;
258         u32 chansts, chanerr;
259
260         if (!ioat->used_desc.prev)
261                 return;
262
263         dev_dbg(to_dev(chan), "%s\n", __func__);
264         chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
265         chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
266         if (chanerr) {
267                 dev_err(to_dev(chan),
268                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
269                         chan_num(chan), chansts, chanerr);
270                 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
271         }
272
273         /*
274          * whack it upside the head with a reset
275          * and wait for things to settle out.
276          * force the pending count to a really big negative
277          * to make sure no one forces an issue_pending
278          * while we're waiting.
279          */
280
281         spin_lock_bh(&ioat->desc_lock);
282         ioat->pending = INT_MIN;
283         writeb(IOAT_CHANCMD_RESET,
284                reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
285         spin_unlock_bh(&ioat->desc_lock);
286
287         /* schedule the 2nd half instead of sleeping a long time */
288         schedule_delayed_work(&chan->work, RESET_DELAY);
289 }
290
291 /**
292  * ioat1_chan_watchdog - watch for stuck channels
293  */
294 static void ioat1_chan_watchdog(struct work_struct *work)
295 {
296         struct ioatdma_device *device =
297                 container_of(work, struct ioatdma_device, work.work);
298         struct ioat_dma_chan *ioat;
299         struct ioat_chan_common *chan;
300         int i;
301         u64 completion;
302         u32 completion_low;
303         unsigned long compl_desc_addr_hw;
304
305         for (i = 0; i < device->common.chancnt; i++) {
306                 chan = ioat_chan_by_index(device, i);
307                 ioat = container_of(chan, struct ioat_dma_chan, base);
308
309                 if (/* have we started processing anything yet */
310                     chan->last_completion
311                     /* have we completed any since last watchdog cycle? */
312                     && (chan->last_completion == chan->watchdog_completion)
313                     /* has TCP stuck on one cookie since last watchdog? */
314                     && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
315                     && (chan->watchdog_tcp_cookie != chan->completed_cookie)
316                     /* is there something in the chain to be processed? */
317                     /* CB1 chain always has at least the last one processed */
318                     && (ioat->used_desc.prev != ioat->used_desc.next)
319                     && ioat->pending == 0) {
320
321                         /*
322                          * check CHANSTS register for completed
323                          * descriptor address.
324                          * if it is different than completion writeback,
325                          * it is not zero
326                          * and it has changed since the last watchdog
327                          *     we can assume that channel
328                          *     is still working correctly
329                          *     and the problem is in completion writeback.
330                          *     update completion writeback
331                          *     with actual CHANSTS value
332                          * else
333                          *     try resetting the channel
334                          */
335
336                         /* we need to read the low address first as this
337                          * causes the chipset to latch the upper bits
338                          * for the subsequent read
339                          */
340                         completion_low = readl(chan->reg_base +
341                                 IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
342                         completion = readl(chan->reg_base +
343                                 IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
344                         completion <<= 32;
345                         completion |= completion_low;
346                         compl_desc_addr_hw = completion &
347                                         IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
348
349                         if ((compl_desc_addr_hw != 0)
350                            && (compl_desc_addr_hw != chan->watchdog_completion)
351                            && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
352                                 chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
353                                 *chan->completion = completion;
354                         } else {
355                                 ioat1_reset_channel(ioat);
356                                 chan->watchdog_completion = 0;
357                                 chan->last_compl_desc_addr_hw = 0;
358                         }
359                 } else {
360                         chan->last_compl_desc_addr_hw = 0;
361                         chan->watchdog_completion = chan->last_completion;
362                 }
363
364                 chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
365         }
366
367         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
368 }
369
370 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
371 {
372         struct dma_chan *c = tx->chan;
373         struct ioat_dma_chan *ioat = to_ioat_chan(c);
374         struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
375         struct ioat_desc_sw *first;
376         struct ioat_desc_sw *chain_tail;
377         dma_cookie_t cookie;
378
379         spin_lock_bh(&ioat->desc_lock);
380         /* cookie incr and addition to used_list must be atomic */
381         cookie = c->cookie;
382         cookie++;
383         if (cookie < 0)
384                 cookie = 1;
385         c->cookie = cookie;
386         tx->cookie = cookie;
387         dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
388
389         /* write address into NextDescriptor field of last desc in chain */
390         first = to_ioat_desc(tx->tx_list.next);
391         chain_tail = to_ioat_desc(ioat->used_desc.prev);
392         /* make descriptor updates globally visible before chaining */
393         wmb();
394         chain_tail->hw->next = first->txd.phys;
395         list_splice_tail_init(&tx->tx_list, &ioat->used_desc);
396         dump_desc_dbg(ioat, chain_tail);
397         dump_desc_dbg(ioat, first);
398
399         ioat->pending += desc->tx_cnt;
400         if (ioat->pending >= ioat_pending_level)
401                 __ioat1_dma_memcpy_issue_pending(ioat);
402         spin_unlock_bh(&ioat->desc_lock);
403
404         return cookie;
405 }
406
407 /**
408  * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
409  * @ioat: the channel supplying the memory pool for the descriptors
410  * @flags: allocation flags
411  */
412 static struct ioat_desc_sw *
413 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
414 {
415         struct ioat_dma_descriptor *desc;
416         struct ioat_desc_sw *desc_sw;
417         struct ioatdma_device *ioatdma_device;
418         dma_addr_t phys;
419
420         ioatdma_device = ioat->base.device;
421         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
422         if (unlikely(!desc))
423                 return NULL;
424
425         desc_sw = kzalloc(sizeof(*desc_sw), flags);
426         if (unlikely(!desc_sw)) {
427                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
428                 return NULL;
429         }
430
431         memset(desc, 0, sizeof(*desc));
432
433         dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
434         desc_sw->txd.tx_submit = ioat1_tx_submit;
435         desc_sw->hw = desc;
436         desc_sw->txd.phys = phys;
437         set_desc_id(desc_sw, -1);
438
439         return desc_sw;
440 }
441
442 static int ioat_initial_desc_count = 256;
443 module_param(ioat_initial_desc_count, int, 0644);
444 MODULE_PARM_DESC(ioat_initial_desc_count,
445                  "ioat1: initial descriptors per channel (default: 256)");
446 /**
447  * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
448  * @chan: the channel to be filled out
449  */
450 static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
451 {
452         struct ioat_dma_chan *ioat = to_ioat_chan(c);
453         struct ioat_chan_common *chan = &ioat->base;
454         struct ioat_desc_sw *desc;
455         u16 chanctrl;
456         u32 chanerr;
457         int i;
458         LIST_HEAD(tmp_list);
459
460         /* have we already been set up? */
461         if (!list_empty(&ioat->free_desc))
462                 return ioat->desccount;
463
464         /* Setup register to interrupt and write completion status on error */
465         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
466                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
467                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
468         writew(chanctrl, chan->reg_base + IOAT_CHANCTRL_OFFSET);
469
470         chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
471         if (chanerr) {
472                 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
473                 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
474         }
475
476         /* Allocate descriptors */
477         for (i = 0; i < ioat_initial_desc_count; i++) {
478                 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
479                 if (!desc) {
480                         dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
481                         break;
482                 }
483                 set_desc_id(desc, i);
484                 list_add_tail(&desc->node, &tmp_list);
485         }
486         spin_lock_bh(&ioat->desc_lock);
487         ioat->desccount = i;
488         list_splice(&tmp_list, &ioat->free_desc);
489         spin_unlock_bh(&ioat->desc_lock);
490
491         /* allocate a completion writeback area */
492         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
493         chan->completion = pci_pool_alloc(chan->device->completion_pool,
494                                           GFP_KERNEL, &chan->completion_dma);
495         memset(chan->completion, 0, sizeof(*chan->completion));
496         writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
497                chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
498         writel(((u64) chan->completion_dma) >> 32,
499                chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
500
501         tasklet_enable(&chan->cleanup_task);
502         ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */
503         dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
504                 __func__, ioat->desccount);
505         return ioat->desccount;
506 }
507
508 /**
509  * ioat1_dma_free_chan_resources - release all the descriptors
510  * @chan: the channel to be cleaned
511  */
512 static void ioat1_dma_free_chan_resources(struct dma_chan *c)
513 {
514         struct ioat_dma_chan *ioat = to_ioat_chan(c);
515         struct ioat_chan_common *chan = &ioat->base;
516         struct ioatdma_device *ioatdma_device = chan->device;
517         struct ioat_desc_sw *desc, *_desc;
518         int in_use_descs = 0;
519
520         /* Before freeing channel resources first check
521          * if they have been previously allocated for this channel.
522          */
523         if (ioat->desccount == 0)
524                 return;
525
526         tasklet_disable(&chan->cleanup_task);
527         ioat1_cleanup(ioat);
528
529         /* Delay 100ms after reset to allow internal DMA logic to quiesce
530          * before removing DMA descriptor resources.
531          */
532         writeb(IOAT_CHANCMD_RESET,
533                chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
534         mdelay(100);
535
536         spin_lock_bh(&ioat->desc_lock);
537         list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
538                 dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
539                         __func__, desc_id(desc));
540                 dump_desc_dbg(ioat, desc);
541                 in_use_descs++;
542                 list_del(&desc->node);
543                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
544                               desc->txd.phys);
545                 kfree(desc);
546         }
547         list_for_each_entry_safe(desc, _desc,
548                                  &ioat->free_desc, node) {
549                 list_del(&desc->node);
550                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
551                               desc->txd.phys);
552                 kfree(desc);
553         }
554         spin_unlock_bh(&ioat->desc_lock);
555
556         pci_pool_free(ioatdma_device->completion_pool,
557                       chan->completion,
558                       chan->completion_dma);
559
560         /* one is ok since we left it on there on purpose */
561         if (in_use_descs > 1)
562                 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
563                         in_use_descs - 1);
564
565         chan->last_completion = 0;
566         chan->completion_dma = 0;
567         chan->watchdog_completion = 0;
568         chan->last_compl_desc_addr_hw = 0;
569         chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
570         ioat->pending = 0;
571         ioat->desccount = 0;
572 }
573
574 /**
575  * ioat1_dma_get_next_descriptor - return the next available descriptor
576  * @ioat: IOAT DMA channel handle
577  *
578  * Gets the next descriptor from the chain, and must be called with the
579  * channel's desc_lock held.  Allocates more descriptors if the channel
580  * has run out.
581  */
582 static struct ioat_desc_sw *
583 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
584 {
585         struct ioat_desc_sw *new;
586
587         if (!list_empty(&ioat->free_desc)) {
588                 new = to_ioat_desc(ioat->free_desc.next);
589                 list_del(&new->node);
590         } else {
591                 /* try to get another desc */
592                 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
593                 if (!new) {
594                         dev_err(to_dev(&ioat->base), "alloc failed\n");
595                         return NULL;
596                 }
597         }
598         dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
599                 __func__, desc_id(new));
600         prefetch(new->hw);
601         return new;
602 }
603
604 static struct dma_async_tx_descriptor *
605 ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
606                       dma_addr_t dma_src, size_t len, unsigned long flags)
607 {
608         struct ioat_dma_chan *ioat = to_ioat_chan(c);
609         struct ioat_desc_sw *desc;
610         size_t copy;
611         LIST_HEAD(chain);
612         dma_addr_t src = dma_src;
613         dma_addr_t dest = dma_dest;
614         size_t total_len = len;
615         struct ioat_dma_descriptor *hw = NULL;
616         int tx_cnt = 0;
617
618         spin_lock_bh(&ioat->desc_lock);
619         desc = ioat1_dma_get_next_descriptor(ioat);
620         do {
621                 if (!desc)
622                         break;
623
624                 tx_cnt++;
625                 copy = min_t(size_t, len, ioat->xfercap);
626
627                 hw = desc->hw;
628                 hw->size = copy;
629                 hw->ctl = 0;
630                 hw->src_addr = src;
631                 hw->dst_addr = dest;
632
633                 list_add_tail(&desc->node, &chain);
634
635                 len -= copy;
636                 dest += copy;
637                 src += copy;
638                 if (len) {
639                         struct ioat_desc_sw *next;
640
641                         async_tx_ack(&desc->txd);
642                         next = ioat1_dma_get_next_descriptor(ioat);
643                         hw->next = next ? next->txd.phys : 0;
644                         dump_desc_dbg(ioat, desc);
645                         desc = next;
646                 } else
647                         hw->next = 0;
648         } while (len);
649
650         if (!desc) {
651                 struct ioat_chan_common *chan = &ioat->base;
652
653                 dev_err(to_dev(chan),
654                         "chan%d - get_next_desc failed\n", chan_num(chan));
655                 list_splice(&chain, &ioat->free_desc);
656                 spin_unlock_bh(&ioat->desc_lock);
657                 return NULL;
658         }
659         spin_unlock_bh(&ioat->desc_lock);
660
661         desc->txd.flags = flags;
662         desc->tx_cnt = tx_cnt;
663         desc->len = total_len;
664         list_splice(&chain, &desc->txd.tx_list);
665         hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
666         hw->ctl_f.compl_write = 1;
667         dump_desc_dbg(ioat, desc);
668
669         return &desc->txd;
670 }
671
672 static void ioat1_cleanup_tasklet(unsigned long data)
673 {
674         struct ioat_dma_chan *chan = (void *)data;
675         ioat1_cleanup(chan);
676         writew(IOAT_CHANCTRL_INT_DISABLE,
677                chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
678 }
679
680 static void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
681                        int direction, enum dma_ctrl_flags flags, bool dst)
682 {
683         if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
684             (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
685                 pci_unmap_single(pdev, addr, len, direction);
686         else
687                 pci_unmap_page(pdev, addr, len, direction);
688 }
689
690
691 void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
692                     size_t len, struct ioat_dma_descriptor *hw)
693 {
694         struct pci_dev *pdev = chan->device->pdev;
695         size_t offset = len - hw->size;
696
697         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
698                 ioat_unmap(pdev, hw->dst_addr - offset, len,
699                            PCI_DMA_FROMDEVICE, flags, 1);
700
701         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
702                 ioat_unmap(pdev, hw->src_addr - offset, len,
703                            PCI_DMA_TODEVICE, flags, 0);
704 }
705
706 unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
707 {
708         unsigned long phys_complete;
709         u64 completion;
710
711         completion = *chan->completion;
712         phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
713
714         dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
715                 (unsigned long long) phys_complete);
716
717         if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
718                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
719                 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
720                         readl(chan->reg_base + IOAT_CHANERR_OFFSET));
721
722                 /* TODO do something to salvage the situation */
723         }
724
725         return phys_complete;
726 }
727
728 /**
729  * ioat1_cleanup - cleanup up finished descriptors
730  * @chan: ioat channel to be cleaned up
731  */
732 static void ioat1_cleanup(struct ioat_dma_chan *ioat)
733 {
734         struct ioat_chan_common *chan = &ioat->base;
735         unsigned long phys_complete;
736         struct ioat_desc_sw *desc, *_desc;
737         dma_cookie_t cookie = 0;
738         struct dma_async_tx_descriptor *tx;
739
740         prefetch(chan->completion);
741
742         if (!spin_trylock_bh(&chan->cleanup_lock))
743                 return;
744
745         phys_complete = ioat_get_current_completion(chan);
746         if (phys_complete == chan->last_completion) {
747                 spin_unlock_bh(&chan->cleanup_lock);
748                 /*
749                  * perhaps we're stuck so hard that the watchdog can't go off?
750                  * try to catch it after 2 seconds
751                  */
752                 if (time_after(jiffies,
753                                chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
754                         ioat1_chan_watchdog(&(chan->device->work.work));
755                         chan->last_completion_time = jiffies;
756                 }
757                 return;
758         }
759         chan->last_completion_time = jiffies;
760
761         cookie = 0;
762         if (!spin_trylock_bh(&ioat->desc_lock)) {
763                 spin_unlock_bh(&chan->cleanup_lock);
764                 return;
765         }
766
767         dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
768                  __func__, phys_complete);
769         list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
770                 tx = &desc->txd;
771                 /*
772                  * Incoming DMA requests may use multiple descriptors,
773                  * due to exceeding xfercap, perhaps. If so, only the
774                  * last one will have a cookie, and require unmapping.
775                  */
776                 dump_desc_dbg(ioat, desc);
777                 if (tx->cookie) {
778                         cookie = tx->cookie;
779                         ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
780                         if (tx->callback) {
781                                 tx->callback(tx->callback_param);
782                                 tx->callback = NULL;
783                         }
784                 }
785
786                 if (tx->phys != phys_complete) {
787                         /*
788                          * a completed entry, but not the last, so clean
789                          * up if the client is done with the descriptor
790                          */
791                         if (async_tx_test_ack(tx))
792                                 list_move_tail(&desc->node, &ioat->free_desc);
793                         else
794                                 tx->cookie = 0;
795                 } else {
796                         /*
797                          * last used desc. Do not remove, so we can
798                          * append from it, but don't look at it next
799                          * time, either
800                          */
801                         tx->cookie = 0;
802
803                         /* TODO check status bits? */
804                         break;
805                 }
806         }
807
808         spin_unlock_bh(&ioat->desc_lock);
809
810         chan->last_completion = phys_complete;
811         if (cookie != 0)
812                 chan->completed_cookie = cookie;
813
814         spin_unlock_bh(&chan->cleanup_lock);
815 }
816
817 static enum dma_status
818 ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
819                       dma_cookie_t *done, dma_cookie_t *used)
820 {
821         struct ioat_dma_chan *ioat = to_ioat_chan(c);
822
823         if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
824                 return DMA_SUCCESS;
825
826         ioat1_cleanup(ioat);
827
828         return ioat_is_complete(c, cookie, done, used);
829 }
830
831 static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
832 {
833         struct ioat_chan_common *chan = &ioat->base;
834         struct ioat_desc_sw *desc;
835         struct ioat_dma_descriptor *hw;
836
837         spin_lock_bh(&ioat->desc_lock);
838
839         desc = ioat1_dma_get_next_descriptor(ioat);
840
841         if (!desc) {
842                 dev_err(to_dev(chan),
843                         "Unable to start null desc - get next desc failed\n");
844                 spin_unlock_bh(&ioat->desc_lock);
845                 return;
846         }
847
848         hw = desc->hw;
849         hw->ctl = 0;
850         hw->ctl_f.null = 1;
851         hw->ctl_f.int_en = 1;
852         hw->ctl_f.compl_write = 1;
853         /* set size to non-zero value (channel returns error when size is 0) */
854         hw->size = NULL_DESC_BUFFER_SIZE;
855         hw->src_addr = 0;
856         hw->dst_addr = 0;
857         async_tx_ack(&desc->txd);
858         hw->next = 0;
859         list_add_tail(&desc->node, &ioat->used_desc);
860         dump_desc_dbg(ioat, desc);
861
862         writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
863                chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
864         writel(((u64) desc->txd.phys) >> 32,
865                chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
866
867         writeb(IOAT_CHANCMD_START, chan->reg_base
868                 + IOAT_CHANCMD_OFFSET(chan->device->version));
869         spin_unlock_bh(&ioat->desc_lock);
870 }
871
872 /*
873  * Perform a IOAT transaction to verify the HW works.
874  */
875 #define IOAT_TEST_SIZE 2000
876
877 static void ioat_dma_test_callback(void *dma_async_param)
878 {
879         struct completion *cmp = dma_async_param;
880
881         complete(cmp);
882 }
883
884 /**
885  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
886  * @device: device to be tested
887  */
888 static int ioat_dma_self_test(struct ioatdma_device *device)
889 {
890         int i;
891         u8 *src;
892         u8 *dest;
893         struct dma_device *dma = &device->common;
894         struct device *dev = &device->pdev->dev;
895         struct dma_chan *dma_chan;
896         struct dma_async_tx_descriptor *tx;
897         dma_addr_t dma_dest, dma_src;
898         dma_cookie_t cookie;
899         int err = 0;
900         struct completion cmp;
901         unsigned long tmo;
902         unsigned long flags;
903
904         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
905         if (!src)
906                 return -ENOMEM;
907         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
908         if (!dest) {
909                 kfree(src);
910                 return -ENOMEM;
911         }
912
913         /* Fill in src buffer */
914         for (i = 0; i < IOAT_TEST_SIZE; i++)
915                 src[i] = (u8)i;
916
917         /* Start copy, using first DMA channel */
918         dma_chan = container_of(dma->channels.next, struct dma_chan,
919                                 device_node);
920         if (dma->device_alloc_chan_resources(dma_chan) < 1) {
921                 dev_err(dev, "selftest cannot allocate chan resource\n");
922                 err = -ENODEV;
923                 goto out;
924         }
925
926         dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
927         dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
928         flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
929                 DMA_PREP_INTERRUPT;
930         tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
931                                                    IOAT_TEST_SIZE, flags);
932         if (!tx) {
933                 dev_err(dev, "Self-test prep failed, disabling\n");
934                 err = -ENODEV;
935                 goto free_resources;
936         }
937
938         async_tx_ack(tx);
939         init_completion(&cmp);
940         tx->callback = ioat_dma_test_callback;
941         tx->callback_param = &cmp;
942         cookie = tx->tx_submit(tx);
943         if (cookie < 0) {
944                 dev_err(dev, "Self-test setup failed, disabling\n");
945                 err = -ENODEV;
946                 goto free_resources;
947         }
948         dma->device_issue_pending(dma_chan);
949
950         tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
951
952         if (tmo == 0 ||
953             dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
954                                         != DMA_SUCCESS) {
955                 dev_err(dev, "Self-test copy timed out, disabling\n");
956                 err = -ENODEV;
957                 goto free_resources;
958         }
959         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
960                 dev_err(dev, "Self-test copy failed compare, disabling\n");
961                 err = -ENODEV;
962                 goto free_resources;
963         }
964
965 free_resources:
966         dma->device_free_chan_resources(dma_chan);
967 out:
968         kfree(src);
969         kfree(dest);
970         return err;
971 }
972
973 static char ioat_interrupt_style[32] = "msix";
974 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
975                     sizeof(ioat_interrupt_style), 0644);
976 MODULE_PARM_DESC(ioat_interrupt_style,
977                  "set ioat interrupt style: msix (default), "
978                  "msix-single-vector, msi, intx)");
979
980 /**
981  * ioat_dma_setup_interrupts - setup interrupt handler
982  * @device: ioat device
983  */
984 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
985 {
986         struct ioat_chan_common *chan;
987         struct pci_dev *pdev = device->pdev;
988         struct device *dev = &pdev->dev;
989         struct msix_entry *msix;
990         int i, j, msixcnt;
991         int err = -EINVAL;
992         u8 intrctrl = 0;
993
994         if (!strcmp(ioat_interrupt_style, "msix"))
995                 goto msix;
996         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
997                 goto msix_single_vector;
998         if (!strcmp(ioat_interrupt_style, "msi"))
999                 goto msi;
1000         if (!strcmp(ioat_interrupt_style, "intx"))
1001                 goto intx;
1002         dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1003         goto err_no_irq;
1004
1005 msix:
1006         /* The number of MSI-X vectors should equal the number of channels */
1007         msixcnt = device->common.chancnt;
1008         for (i = 0; i < msixcnt; i++)
1009                 device->msix_entries[i].entry = i;
1010
1011         err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1012         if (err < 0)
1013                 goto msi;
1014         if (err > 0)
1015                 goto msix_single_vector;
1016
1017         for (i = 0; i < msixcnt; i++) {
1018                 msix = &device->msix_entries[i];
1019                 chan = ioat_chan_by_index(device, i);
1020                 err = devm_request_irq(dev, msix->vector,
1021                                        ioat_dma_do_interrupt_msix, 0,
1022                                        "ioat-msix", chan);
1023                 if (err) {
1024                         for (j = 0; j < i; j++) {
1025                                 msix = &device->msix_entries[j];
1026                                 chan = ioat_chan_by_index(device, j);
1027                                 devm_free_irq(dev, msix->vector, chan);
1028                         }
1029                         goto msix_single_vector;
1030                 }
1031         }
1032         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1033         goto done;
1034
1035 msix_single_vector:
1036         msix = &device->msix_entries[0];
1037         msix->entry = 0;
1038         err = pci_enable_msix(pdev, device->msix_entries, 1);
1039         if (err)
1040                 goto msi;
1041
1042         err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1043                                "ioat-msix", device);
1044         if (err) {
1045                 pci_disable_msix(pdev);
1046                 goto msi;
1047         }
1048         goto done;
1049
1050 msi:
1051         err = pci_enable_msi(pdev);
1052         if (err)
1053                 goto intx;
1054
1055         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1056                                "ioat-msi", device);
1057         if (err) {
1058                 pci_disable_msi(pdev);
1059                 goto intx;
1060         }
1061         goto done;
1062
1063 intx:
1064         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1065                                IRQF_SHARED, "ioat-intx", device);
1066         if (err)
1067                 goto err_no_irq;
1068
1069 done:
1070         if (device->intr_quirk)
1071                 device->intr_quirk(device);
1072         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1073         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1074         return 0;
1075
1076 err_no_irq:
1077         /* Disable all interrupt generation */
1078         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1079         dev_err(dev, "no usable interrupts\n");
1080         return err;
1081 }
1082
1083 static void ioat_disable_interrupts(struct ioatdma_device *device)
1084 {
1085         /* Disable all interrupt generation */
1086         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1087 }
1088
1089 int ioat_probe(struct ioatdma_device *device)
1090 {
1091         int err = -ENODEV;
1092         struct dma_device *dma = &device->common;
1093         struct pci_dev *pdev = device->pdev;
1094         struct device *dev = &pdev->dev;
1095
1096         /* DMA coherent memory pool for DMA descriptor allocations */
1097         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1098                                            sizeof(struct ioat_dma_descriptor),
1099                                            64, 0);
1100         if (!device->dma_pool) {
1101                 err = -ENOMEM;
1102                 goto err_dma_pool;
1103         }
1104
1105         device->completion_pool = pci_pool_create("completion_pool", pdev,
1106                                                   sizeof(u64), SMP_CACHE_BYTES,
1107                                                   SMP_CACHE_BYTES);
1108
1109         if (!device->completion_pool) {
1110                 err = -ENOMEM;
1111                 goto err_completion_pool;
1112         }
1113
1114         device->enumerate_channels(device);
1115
1116         dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1117         dma->dev = &pdev->dev;
1118
1119         dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1120                 " %d channels, device version 0x%02x, driver version %s\n",
1121                 dma->chancnt, device->version, IOAT_DMA_VERSION);
1122
1123         if (!dma->chancnt) {
1124                 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1125                         "zero channels detected\n");
1126                 goto err_setup_interrupts;
1127         }
1128
1129         err = ioat_dma_setup_interrupts(device);
1130         if (err)
1131                 goto err_setup_interrupts;
1132
1133         err = ioat_dma_self_test(device);
1134         if (err)
1135                 goto err_self_test;
1136
1137         return 0;
1138
1139 err_self_test:
1140         ioat_disable_interrupts(device);
1141 err_setup_interrupts:
1142         pci_pool_destroy(device->completion_pool);
1143 err_completion_pool:
1144         pci_pool_destroy(device->dma_pool);
1145 err_dma_pool:
1146         return err;
1147 }
1148
1149 int ioat_register(struct ioatdma_device *device)
1150 {
1151         int err = dma_async_device_register(&device->common);
1152
1153         if (err) {
1154                 ioat_disable_interrupts(device);
1155                 pci_pool_destroy(device->completion_pool);
1156                 pci_pool_destroy(device->dma_pool);
1157         }
1158
1159         return err;
1160 }
1161
1162 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1163 static void ioat1_intr_quirk(struct ioatdma_device *device)
1164 {
1165         struct pci_dev *pdev = device->pdev;
1166         u32 dmactrl;
1167
1168         pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1169         if (pdev->msi_enabled)
1170                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1171         else
1172                 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1173         pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1174 }
1175
1176 int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1177 {
1178         struct pci_dev *pdev = device->pdev;
1179         struct dma_device *dma;
1180         int err;
1181
1182         device->intr_quirk = ioat1_intr_quirk;
1183         device->enumerate_channels = ioat1_enumerate_channels;
1184         dma = &device->common;
1185         dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1186         dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1187         dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1188         dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1189         dma->device_is_tx_complete = ioat1_dma_is_complete;
1190
1191         err = ioat_probe(device);
1192         if (err)
1193                 return err;
1194         ioat_set_tcp_copy_break(4096);
1195         err = ioat_register(device);
1196         if (err)
1197                 return err;
1198         if (dca)
1199                 device->dca = ioat_dca_init(pdev, device->reg_base);
1200
1201         INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
1202         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1203
1204         return err;
1205 }
1206
1207 void ioat_dma_remove(struct ioatdma_device *device)
1208 {
1209         struct dma_device *dma = &device->common;
1210
1211         if (device->version != IOAT_VER_3_0)
1212                 cancel_delayed_work(&device->work);
1213
1214         ioat_disable_interrupts(device);
1215
1216         dma_async_device_unregister(dma);
1217
1218         pci_pool_destroy(device->dma_pool);
1219         pci_pool_destroy(device->completion_pool);
1220
1221         INIT_LIST_HEAD(&dma->channels);
1222 }