IB/ipath: remove stale references to userspace SMA
[linux-2.6.git] / drivers / infiniband / hw / ipath / ipath_intr.c
1 /*
2  * Copyright (c) 2006 QLogic, Inc. All rights reserved.
3  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33
34 #include <linux/pci.h>
35
36 #include "ipath_kernel.h"
37 #include "ipath_verbs.h"
38 #include "ipath_common.h"
39
40 /* These are all rcv-related errors which we want to count for stats */
41 #define E_SUM_PKTERRS \
42         (INFINIPATH_E_RHDRLEN | INFINIPATH_E_RBADTID | \
43          INFINIPATH_E_RBADVERSION | INFINIPATH_E_RHDR | \
44          INFINIPATH_E_RLONGPKTLEN | INFINIPATH_E_RSHORTPKTLEN | \
45          INFINIPATH_E_RMAXPKTLEN | INFINIPATH_E_RMINPKTLEN | \
46          INFINIPATH_E_RFORMATERR | INFINIPATH_E_RUNSUPVL | \
47          INFINIPATH_E_RUNEXPCHAR | INFINIPATH_E_REBP)
48
49 /* These are all send-related errors which we want to count for stats */
50 #define E_SUM_ERRS \
51         (INFINIPATH_E_SPIOARMLAUNCH | INFINIPATH_E_SUNEXPERRPKTNUM | \
52          INFINIPATH_E_SDROPPEDDATAPKT | INFINIPATH_E_SDROPPEDSMPPKT | \
53          INFINIPATH_E_SMAXPKTLEN | INFINIPATH_E_SUNSUPVL | \
54          INFINIPATH_E_SMINPKTLEN | INFINIPATH_E_SPKTLEN | \
55          INFINIPATH_E_INVALIDADDR)
56
57 /*
58  * these are errors that can occur when the link changes state while
59  * a packet is being sent or received.  This doesn't cover things
60  * like EBP or VCRC that can be the result of a sending having the
61  * link change state, so we receive a "known bad" packet.
62  */
63 #define E_SUM_LINK_PKTERRS \
64         (INFINIPATH_E_SDROPPEDDATAPKT | INFINIPATH_E_SDROPPEDSMPPKT | \
65          INFINIPATH_E_SMINPKTLEN | INFINIPATH_E_SPKTLEN | \
66          INFINIPATH_E_RSHORTPKTLEN | INFINIPATH_E_RMINPKTLEN | \
67          INFINIPATH_E_RUNEXPCHAR)
68
69 static u64 handle_e_sum_errs(struct ipath_devdata *dd, ipath_err_t errs)
70 {
71         unsigned long sbuf[4];
72         u64 ignore_this_time = 0;
73         u32 piobcnt;
74
75         /* if possible that sendbuffererror could be valid */
76         piobcnt = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
77         /* read these before writing errorclear */
78         sbuf[0] = ipath_read_kreg64(
79                 dd, dd->ipath_kregs->kr_sendbuffererror);
80         sbuf[1] = ipath_read_kreg64(
81                 dd, dd->ipath_kregs->kr_sendbuffererror + 1);
82         if (piobcnt > 128) {
83                 sbuf[2] = ipath_read_kreg64(
84                         dd, dd->ipath_kregs->kr_sendbuffererror + 2);
85                 sbuf[3] = ipath_read_kreg64(
86                         dd, dd->ipath_kregs->kr_sendbuffererror + 3);
87         }
88
89         if (sbuf[0] || sbuf[1] || (piobcnt > 128 && (sbuf[2] || sbuf[3]))) {
90                 int i;
91
92                 ipath_cdbg(PKT, "SendbufErrs %lx %lx ", sbuf[0], sbuf[1]);
93                 if (ipath_debug & __IPATH_PKTDBG && piobcnt > 128)
94                         printk("%lx %lx ", sbuf[2], sbuf[3]);
95                 for (i = 0; i < piobcnt; i++) {
96                         if (test_bit(i, sbuf)) {
97                                 u32 __iomem *piobuf;
98                                 if (i < dd->ipath_piobcnt2k)
99                                         piobuf = (u32 __iomem *)
100                                                 (dd->ipath_pio2kbase +
101                                                  i * dd->ipath_palign);
102                                 else
103                                         piobuf = (u32 __iomem *)
104                                                 (dd->ipath_pio4kbase +
105                                                  (i - dd->ipath_piobcnt2k) *
106                                                  dd->ipath_4kalign);
107
108                                 ipath_cdbg(PKT,
109                                            "PIObuf[%u] @%p pbc is %x; ",
110                                            i, piobuf, readl(piobuf));
111
112                                 ipath_disarm_piobufs(dd, i, 1);
113                         }
114                 }
115                 if (ipath_debug & __IPATH_PKTDBG)
116                         printk("\n");
117         }
118         if ((errs & E_SUM_LINK_PKTERRS) &&
119             !(dd->ipath_flags & IPATH_LINKACTIVE)) {
120                 /*
121                  * This can happen when SMA is trying to bring the link
122                  * up, but the IB link changes state at the "wrong" time.
123                  * The IB logic then complains that the packet isn't
124                  * valid.  We don't want to confuse people, so we just
125                  * don't print them, except at debug
126                  */
127                 ipath_dbg("Ignoring packet errors %llx, because link not "
128                           "ACTIVE\n", (unsigned long long) errs);
129                 ignore_this_time = errs & E_SUM_LINK_PKTERRS;
130         }
131
132         return ignore_this_time;
133 }
134
135 /* return the strings for the most common link states */
136 static char *ib_linkstate(u32 linkstate)
137 {
138         char *ret;
139
140         switch (linkstate) {
141         case IPATH_IBSTATE_INIT:
142                 ret = "Init";
143                 break;
144         case IPATH_IBSTATE_ARM:
145                 ret = "Arm";
146                 break;
147         case IPATH_IBSTATE_ACTIVE:
148                 ret = "Active";
149                 break;
150         default:
151                 ret = "Down";
152         }
153
154         return ret;
155 }
156
157 static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
158                                      ipath_err_t errs, int noprint)
159 {
160         u64 val;
161         u32 ltstate, lstate;
162
163         /*
164          * even if diags are enabled, we want to notice LINKINIT, etc.
165          * We just don't want to change the LED state, or
166          * dd->ipath_kregs->kr_ibcctrl
167          */
168         val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus);
169         lstate = val & IPATH_IBSTATE_MASK;
170
171         /*
172          * this is confusing enough when it happens that I want to always put it
173          * on the console and in the logs.  If it was a requested state change,
174          * we'll have already cleared the flags, so we won't print this warning
175          */
176         if ((lstate != IPATH_IBSTATE_ARM && lstate != IPATH_IBSTATE_ACTIVE)
177                 && (dd->ipath_flags & (IPATH_LINKARMED | IPATH_LINKACTIVE))) {
178                 dev_info(&dd->pcidev->dev, "Link state changed from %s to %s\n",
179                                  (dd->ipath_flags & IPATH_LINKARMED) ? "ARM" : "ACTIVE",
180                                  ib_linkstate(lstate));
181                 /*
182                  * Flush all queued sends when link went to DOWN or INIT,
183                  * to be sure that they don't block SMA and other MAD packets
184                  */
185                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
186                                  INFINIPATH_S_ABORT);
187                 ipath_disarm_piobufs(dd, dd->ipath_lastport_piobuf,
188                                                         (unsigned)(dd->ipath_piobcnt2k +
189                                         dd->ipath_piobcnt4k) -
190                                         dd->ipath_lastport_piobuf);
191         }
192         else if (lstate == IPATH_IBSTATE_INIT || lstate == IPATH_IBSTATE_ARM ||
193             lstate == IPATH_IBSTATE_ACTIVE) {
194                 /*
195                  * only print at SMA if there is a change, debug if not
196                  * (sometimes we want to know that, usually not).
197                  */
198                 if (lstate == ((unsigned) dd->ipath_lastibcstat
199                                & IPATH_IBSTATE_MASK)) {
200                         ipath_dbg("Status change intr but no change (%s)\n",
201                                   ib_linkstate(lstate));
202                 }
203                 else
204                         ipath_cdbg(VERBOSE, "Unit %u link state %s, last "
205                                    "was %s\n", dd->ipath_unit,
206                                    ib_linkstate(lstate),
207                                    ib_linkstate((unsigned)
208                                                 dd->ipath_lastibcstat
209                                                 & IPATH_IBSTATE_MASK));
210         }
211         else {
212                 lstate = dd->ipath_lastibcstat & IPATH_IBSTATE_MASK;
213                 if (lstate == IPATH_IBSTATE_INIT ||
214                     lstate == IPATH_IBSTATE_ARM ||
215                     lstate == IPATH_IBSTATE_ACTIVE)
216                         ipath_cdbg(VERBOSE, "Unit %u link state down"
217                                    " (state 0x%x), from %s\n",
218                                    dd->ipath_unit,
219                                    (u32)val & IPATH_IBSTATE_MASK,
220                                    ib_linkstate(lstate));
221                 else
222                         ipath_cdbg(VERBOSE, "Unit %u link state changed "
223                                    "to 0x%x from down (%x)\n",
224                                    dd->ipath_unit, (u32) val, lstate);
225         }
226         ltstate = (val >> INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
227                 INFINIPATH_IBCS_LINKTRAININGSTATE_MASK;
228         lstate = (val >> INFINIPATH_IBCS_LINKSTATE_SHIFT) &
229                 INFINIPATH_IBCS_LINKSTATE_MASK;
230
231         if (ltstate == INFINIPATH_IBCS_LT_STATE_POLLACTIVE ||
232             ltstate == INFINIPATH_IBCS_LT_STATE_POLLQUIET) {
233                 u32 last_ltstate;
234
235                 /*
236                  * Ignore cycling back and forth from Polling.Active
237                  * to Polling.Quiet while waiting for the other end of
238                  * the link to come up. We will cycle back and forth
239                  * between them if no cable is plugged in,
240                  * the other device is powered off or disabled, etc.
241                  */
242                 last_ltstate = (dd->ipath_lastibcstat >>
243                                 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT)
244                         & INFINIPATH_IBCS_LINKTRAININGSTATE_MASK;
245                 if (last_ltstate == INFINIPATH_IBCS_LT_STATE_POLLACTIVE
246                     || last_ltstate ==
247                     INFINIPATH_IBCS_LT_STATE_POLLQUIET) {
248                         if (dd->ipath_ibpollcnt > 40) {
249                                 dd->ipath_flags |= IPATH_NOCABLE;
250                                 *dd->ipath_statusp |=
251                                         IPATH_STATUS_IB_NOCABLE;
252                         } else
253                                 dd->ipath_ibpollcnt++;
254                         goto skip_ibchange;
255                 }
256         }
257         dd->ipath_ibpollcnt = 0;        /* some state other than 2 or 3 */
258         ipath_stats.sps_iblink++;
259         if (ltstate != INFINIPATH_IBCS_LT_STATE_LINKUP) {
260                 dd->ipath_flags |= IPATH_LINKDOWN;
261                 dd->ipath_flags &= ~(IPATH_LINKUNK | IPATH_LINKINIT
262                                      | IPATH_LINKACTIVE |
263                                      IPATH_LINKARMED);
264                 *dd->ipath_statusp &= ~IPATH_STATUS_IB_READY;
265                 dd->ipath_lli_counter = 0;
266                 if (!noprint) {
267                         if (((dd->ipath_lastibcstat >>
268                               INFINIPATH_IBCS_LINKSTATE_SHIFT) &
269                              INFINIPATH_IBCS_LINKSTATE_MASK)
270                             == INFINIPATH_IBCS_L_STATE_ACTIVE)
271                                 /* if from up to down be more vocal */
272                                 ipath_cdbg(VERBOSE,
273                                            "Unit %u link now down (%s)\n",
274                                            dd->ipath_unit,
275                                            ipath_ibcstatus_str[ltstate]);
276                         else
277                                 ipath_cdbg(VERBOSE, "Unit %u link is "
278                                            "down (%s)\n", dd->ipath_unit,
279                                            ipath_ibcstatus_str[ltstate]);
280                 }
281
282                 dd->ipath_f_setextled(dd, lstate, ltstate);
283         } else if ((val & IPATH_IBSTATE_MASK) == IPATH_IBSTATE_ACTIVE) {
284                 dd->ipath_flags |= IPATH_LINKACTIVE;
285                 dd->ipath_flags &=
286                         ~(IPATH_LINKUNK | IPATH_LINKINIT | IPATH_LINKDOWN |
287                           IPATH_LINKARMED | IPATH_NOCABLE);
288                 *dd->ipath_statusp &= ~IPATH_STATUS_IB_NOCABLE;
289                 *dd->ipath_statusp |=
290                         IPATH_STATUS_IB_READY | IPATH_STATUS_IB_CONF;
291                 dd->ipath_f_setextled(dd, lstate, ltstate);
292         } else if ((val & IPATH_IBSTATE_MASK) == IPATH_IBSTATE_INIT) {
293                 /*
294                  * set INIT and DOWN.  Down is checked by most of the other
295                  * code, but INIT is useful to know in a few places.
296                  */
297                 dd->ipath_flags |= IPATH_LINKINIT | IPATH_LINKDOWN;
298                 dd->ipath_flags &=
299                         ~(IPATH_LINKUNK | IPATH_LINKACTIVE | IPATH_LINKARMED
300                           | IPATH_NOCABLE);
301                 *dd->ipath_statusp &= ~(IPATH_STATUS_IB_NOCABLE
302                                         | IPATH_STATUS_IB_READY);
303                 dd->ipath_f_setextled(dd, lstate, ltstate);
304         } else if ((val & IPATH_IBSTATE_MASK) == IPATH_IBSTATE_ARM) {
305                 dd->ipath_flags |= IPATH_LINKARMED;
306                 dd->ipath_flags &=
307                         ~(IPATH_LINKUNK | IPATH_LINKDOWN | IPATH_LINKINIT |
308                           IPATH_LINKACTIVE | IPATH_NOCABLE);
309                 *dd->ipath_statusp &= ~(IPATH_STATUS_IB_NOCABLE
310                                         | IPATH_STATUS_IB_READY);
311                 dd->ipath_f_setextled(dd, lstate, ltstate);
312         } else {
313                 if (!noprint)
314                         ipath_dbg("IBstatuschange unit %u: %s (%x)\n",
315                                   dd->ipath_unit,
316                                   ipath_ibcstatus_str[ltstate], ltstate);
317         }
318 skip_ibchange:
319         dd->ipath_lastibcstat = val;
320 }
321
322 static void handle_supp_msgs(struct ipath_devdata *dd,
323                              unsigned supp_msgs, char msg[512])
324 {
325         /*
326          * Print the message unless it's ibc status change only, which
327          * happens so often we never want to count it.
328          */
329         if (dd->ipath_lasterror & ~INFINIPATH_E_IBSTATUSCHANGED) {
330                 ipath_decode_err(msg, sizeof msg, dd->ipath_lasterror &
331                                  ~INFINIPATH_E_IBSTATUSCHANGED);
332                 if (dd->ipath_lasterror &
333                     ~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL))
334                         ipath_dev_err(dd, "Suppressed %u messages for "
335                                       "fast-repeating errors (%s) (%llx)\n",
336                                       supp_msgs, msg,
337                                       (unsigned long long)
338                                       dd->ipath_lasterror);
339                 else {
340                         /*
341                          * rcvegrfull and rcvhdrqfull are "normal", for some
342                          * types of processes (mostly benchmarks) that send
343                          * huge numbers of messages, while not processing
344                          * them. So only complain about these at debug
345                          * level.
346                          */
347                         ipath_dbg("Suppressed %u messages for %s\n",
348                                   supp_msgs, msg);
349                 }
350         }
351 }
352
353 static unsigned handle_frequent_errors(struct ipath_devdata *dd,
354                                        ipath_err_t errs, char msg[512],
355                                        int *noprint)
356 {
357         unsigned long nc;
358         static unsigned long nextmsg_time;
359         static unsigned nmsgs, supp_msgs;
360
361         /*
362          * Throttle back "fast" messages to no more than 10 per 5 seconds.
363          * This isn't perfect, but it's a reasonable heuristic. If we get
364          * more than 10, give a 6x longer delay.
365          */
366         nc = jiffies;
367         if (nmsgs > 10) {
368                 if (time_before(nc, nextmsg_time)) {
369                         *noprint = 1;
370                         if (!supp_msgs++)
371                                 nextmsg_time = nc + HZ * 3;
372                 }
373                 else if (supp_msgs) {
374                         handle_supp_msgs(dd, supp_msgs, msg);
375                         supp_msgs = 0;
376                         nmsgs = 0;
377                 }
378         }
379         else if (!nmsgs++ || time_after(nc, nextmsg_time))
380                 nextmsg_time = nc + HZ / 2;
381
382         return supp_msgs;
383 }
384
385 static int handle_errors(struct ipath_devdata *dd, ipath_err_t errs)
386 {
387         char msg[512];
388         u64 ignore_this_time = 0;
389         int i;
390         int chkerrpkts = 0, noprint = 0;
391         unsigned supp_msgs;
392
393         supp_msgs = handle_frequent_errors(dd, errs, msg, &noprint);
394
395         /*
396          * don't report errors that are masked (includes those always
397          * ignored)
398          */
399         errs &= ~dd->ipath_maskederrs;
400
401         /* do these first, they are most important */
402         if (errs & INFINIPATH_E_HARDWARE) {
403                 /* reuse same msg buf */
404                 dd->ipath_f_handle_hwerrors(dd, msg, sizeof msg);
405         }
406
407         if (!noprint && (errs & ~infinipath_e_bitsextant))
408                 ipath_dev_err(dd, "error interrupt with unknown errors "
409                               "%llx set\n", (unsigned long long)
410                               (errs & ~infinipath_e_bitsextant));
411
412         if (errs & E_SUM_ERRS)
413                 ignore_this_time = handle_e_sum_errs(dd, errs);
414         else if ((errs & E_SUM_LINK_PKTERRS) &&
415             !(dd->ipath_flags & IPATH_LINKACTIVE)) {
416                 /*
417                  * This can happen when SMA is trying to bring the link
418                  * up, but the IB link changes state at the "wrong" time.
419                  * The IB logic then complains that the packet isn't
420                  * valid.  We don't want to confuse people, so we just
421                  * don't print them, except at debug
422                  */
423                 ipath_dbg("Ignoring packet errors %llx, because link not "
424                           "ACTIVE\n", (unsigned long long) errs);
425                 ignore_this_time = errs & E_SUM_LINK_PKTERRS;
426         }
427
428         if (supp_msgs == 250000) {
429                 /*
430                  * It's not entirely reasonable assuming that the errors set
431                  * in the last clear period are all responsible for the
432                  * problem, but the alternative is to assume it's the only
433                  * ones on this particular interrupt, which also isn't great
434                  */
435                 dd->ipath_maskederrs |= dd->ipath_lasterror | errs;
436                 ipath_write_kreg(dd, dd->ipath_kregs->kr_errormask,
437                                  ~dd->ipath_maskederrs);
438                 ipath_decode_err(msg, sizeof msg,
439                                  (dd->ipath_maskederrs & ~dd->
440                                   ipath_ignorederrs));
441
442                 if ((dd->ipath_maskederrs & ~dd->ipath_ignorederrs) &
443                     ~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL))
444                         ipath_dev_err(dd, "Disabling error(s) %llx because "
445                                       "occurring too frequently (%s)\n",
446                                       (unsigned long long)
447                                       (dd->ipath_maskederrs &
448                                        ~dd->ipath_ignorederrs), msg);
449                 else {
450                         /*
451                          * rcvegrfull and rcvhdrqfull are "normal",
452                          * for some types of processes (mostly benchmarks)
453                          * that send huge numbers of messages, while not
454                          * processing them.  So only complain about
455                          * these at debug level.
456                          */
457                         ipath_dbg("Disabling frequent queue full errors "
458                                   "(%s)\n", msg);
459                 }
460
461                 /*
462                  * Re-enable the masked errors after around 3 minutes.  in
463                  * ipath_get_faststats().  If we have a series of fast
464                  * repeating but different errors, the interval will keep
465                  * stretching out, but that's OK, as that's pretty
466                  * catastrophic.
467                  */
468                 dd->ipath_unmasktime = jiffies + HZ * 180;
469         }
470
471         ipath_write_kreg(dd, dd->ipath_kregs->kr_errorclear, errs);
472         if (ignore_this_time)
473                 errs &= ~ignore_this_time;
474         if (errs & ~dd->ipath_lasterror) {
475                 errs &= ~dd->ipath_lasterror;
476                 /* never suppress duplicate hwerrors or ibstatuschange */
477                 dd->ipath_lasterror |= errs &
478                         ~(INFINIPATH_E_HARDWARE |
479                           INFINIPATH_E_IBSTATUSCHANGED);
480         }
481         if (!errs)
482                 return 0;
483
484         if (!noprint)
485                 /*
486                  * the ones we mask off are handled specially below or above
487                  */
488                 ipath_decode_err(msg, sizeof msg,
489                                  errs & ~(INFINIPATH_E_IBSTATUSCHANGED |
490                                           INFINIPATH_E_RRCVEGRFULL |
491                                           INFINIPATH_E_RRCVHDRFULL |
492                                           INFINIPATH_E_HARDWARE));
493         else
494                 /* so we don't need if (!noprint) at strlcat's below */
495                 *msg = 0;
496
497         if (errs & E_SUM_PKTERRS) {
498                 ipath_stats.sps_pkterrs++;
499                 chkerrpkts = 1;
500         }
501         if (errs & E_SUM_ERRS)
502                 ipath_stats.sps_errs++;
503
504         if (errs & (INFINIPATH_E_RICRC | INFINIPATH_E_RVCRC)) {
505                 ipath_stats.sps_crcerrs++;
506                 chkerrpkts = 1;
507         }
508
509         /*
510          * We don't want to print these two as they happen, or we can make
511          * the situation even worse, because it takes so long to print
512          * messages to serial consoles.  Kernel ports get printed from
513          * fast_stats, no more than every 5 seconds, user ports get printed
514          * on close
515          */
516         if (errs & INFINIPATH_E_RRCVHDRFULL) {
517                 int any;
518                 u32 hd, tl;
519                 ipath_stats.sps_hdrqfull++;
520                 for (any = i = 0; i < dd->ipath_cfgports; i++) {
521                         struct ipath_portdata *pd = dd->ipath_pd[i];
522                         if (i == 0) {
523                                 hd = dd->ipath_port0head;
524                                 tl = (u32) le64_to_cpu(
525                                         *dd->ipath_hdrqtailptr);
526                         } else if (pd && pd->port_cnt &&
527                                    pd->port_rcvhdrtail_kvaddr) {
528                                 /*
529                                  * don't report same point multiple times,
530                                  * except kernel
531                                  */
532                                 tl = (u32) * pd->port_rcvhdrtail_kvaddr;
533                                 if (tl == dd->ipath_lastrcvhdrqtails[i])
534                                         continue;
535                                 hd = ipath_read_ureg32(dd, ur_rcvhdrhead,
536                                                        i);
537                         } else
538                                 continue;
539                         if (hd == (tl + 1) ||
540                             (!hd && tl == dd->ipath_hdrqlast)) {
541                                 if (i == 0)
542                                         chkerrpkts = 1;
543                                 dd->ipath_lastrcvhdrqtails[i] = tl;
544                                 pd->port_hdrqfull++;
545                         }
546                 }
547         }
548         if (errs & INFINIPATH_E_RRCVEGRFULL) {
549                 /*
550                  * since this is of less importance and not likely to
551                  * happen without also getting hdrfull, only count
552                  * occurrences; don't check each port (or even the kernel
553                  * vs user)
554                  */
555                 ipath_stats.sps_etidfull++;
556                 if (dd->ipath_port0head !=
557                     (u32) le64_to_cpu(*dd->ipath_hdrqtailptr))
558                         chkerrpkts = 1;
559         }
560
561         /*
562          * do this before IBSTATUSCHANGED, in case both bits set in a single
563          * interrupt; we want the STATUSCHANGE to "win", so we do our
564          * internal copy of state machine correctly
565          */
566         if (errs & INFINIPATH_E_RIBLOSTLINK) {
567                 /*
568                  * force through block below
569                  */
570                 errs |= INFINIPATH_E_IBSTATUSCHANGED;
571                 ipath_stats.sps_iblink++;
572                 dd->ipath_flags |= IPATH_LINKDOWN;
573                 dd->ipath_flags &= ~(IPATH_LINKUNK | IPATH_LINKINIT
574                                      | IPATH_LINKARMED | IPATH_LINKACTIVE);
575                 *dd->ipath_statusp &= ~IPATH_STATUS_IB_READY;
576                 if (!noprint) {
577                         u64 st = ipath_read_kreg64(
578                                 dd, dd->ipath_kregs->kr_ibcstatus);
579
580                         ipath_dbg("Lost link, link now down (%s)\n",
581                                   ipath_ibcstatus_str[st & 0xf]);
582                 }
583         }
584         if (errs & INFINIPATH_E_IBSTATUSCHANGED)
585                 handle_e_ibstatuschanged(dd, errs, noprint);
586
587         if (errs & INFINIPATH_E_RESET) {
588                 if (!noprint)
589                         ipath_dev_err(dd, "Got reset, requires re-init "
590                                       "(unload and reload driver)\n");
591                 dd->ipath_flags &= ~IPATH_INITTED;      /* needs re-init */
592                 /* mark as having had error */
593                 *dd->ipath_statusp |= IPATH_STATUS_HWERROR;
594                 *dd->ipath_statusp &= ~IPATH_STATUS_IB_CONF;
595         }
596
597         if (!noprint && *msg)
598                 ipath_dev_err(dd, "%s error\n", msg);
599         if (dd->ipath_state_wanted & dd->ipath_flags) {
600                 ipath_cdbg(VERBOSE, "driver wanted state %x, iflags now %x, "
601                            "waking\n", dd->ipath_state_wanted,
602                            dd->ipath_flags);
603                 wake_up_interruptible(&ipath_state_wait);
604         }
605
606         return chkerrpkts;
607 }
608
609 /* this is separate to allow for better optimization of ipath_intr() */
610
611 static void ipath_bad_intr(struct ipath_devdata *dd, u32 * unexpectp)
612 {
613         /*
614          * sometimes happen during driver init and unload, don't want
615          * to process any interrupts at that point
616          */
617
618         /* this is just a bandaid, not a fix, if something goes badly
619          * wrong */
620         if (++*unexpectp > 100) {
621                 if (++*unexpectp > 105) {
622                         /*
623                          * ok, we must be taking somebody else's interrupts,
624                          * due to a messed up mptable and/or PIRQ table, so
625                          * unregister the interrupt.  We've seen this during
626                          * linuxbios development work, and it may happen in
627                          * the future again.
628                          */
629                         if (dd->pcidev && dd->pcidev->irq) {
630                                 ipath_dev_err(dd, "Now %u unexpected "
631                                               "interrupts, unregistering "
632                                               "interrupt handler\n",
633                                               *unexpectp);
634                                 ipath_dbg("free_irq of irq %x\n",
635                                           dd->pcidev->irq);
636                                 free_irq(dd->pcidev->irq, dd);
637                         }
638                 }
639                 if (ipath_read_kreg32(dd, dd->ipath_kregs->kr_intmask)) {
640                         ipath_dev_err(dd, "%u unexpected interrupts, "
641                                       "disabling interrupts completely\n",
642                                       *unexpectp);
643                         /*
644                          * disable all interrupts, something is very wrong
645                          */
646                         ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask,
647                                          0ULL);
648                 }
649         } else if (*unexpectp > 1)
650                 ipath_dbg("Interrupt when not ready, should not happen, "
651                           "ignoring\n");
652 }
653
654 static void ipath_bad_regread(struct ipath_devdata *dd)
655 {
656         static int allbits;
657
658         /* separate routine, for better optimization of ipath_intr() */
659
660         /*
661          * We print the message and disable interrupts, in hope of
662          * having a better chance of debugging the problem.
663          */
664         ipath_dev_err(dd,
665                       "Read of interrupt status failed (all bits set)\n");
666         if (allbits++) {
667                 /* disable all interrupts, something is very wrong */
668                 ipath_write_kreg(dd, dd->ipath_kregs->kr_intmask, 0ULL);
669                 if (allbits == 2) {
670                         ipath_dev_err(dd, "Still bad interrupt status, "
671                                       "unregistering interrupt\n");
672                         free_irq(dd->pcidev->irq, dd);
673                 } else if (allbits > 2) {
674                         if ((allbits % 10000) == 0)
675                                 printk(".");
676                 } else
677                         ipath_dev_err(dd, "Disabling interrupts, "
678                                       "multiple errors\n");
679         }
680 }
681
682 static void handle_port_pioavail(struct ipath_devdata *dd)
683 {
684         u32 i;
685         /*
686          * start from port 1, since for now port 0  is never using
687          * wait_event for PIO
688          */
689         for (i = 1; dd->ipath_portpiowait && i < dd->ipath_cfgports; i++) {
690                 struct ipath_portdata *pd = dd->ipath_pd[i];
691
692                 if (pd && pd->port_cnt &&
693                     dd->ipath_portpiowait & (1U << i)) {
694                         clear_bit(i, &dd->ipath_portpiowait);
695                         if (test_bit(IPATH_PORT_WAITING_PIO,
696                                      &pd->port_flag)) {
697                                 clear_bit(IPATH_PORT_WAITING_PIO,
698                                           &pd->port_flag);
699                                 wake_up_interruptible(&pd->port_wait);
700                         }
701                 }
702         }
703 }
704
705 static void handle_layer_pioavail(struct ipath_devdata *dd)
706 {
707         int ret;
708
709         ret = ipath_ib_piobufavail(dd->verbs_dev);
710         if (ret > 0)
711                 goto set;
712
713         return;
714 set:
715         set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
716         ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
717                          dd->ipath_sendctrl);
718 }
719
720 /*
721  * Handle receive interrupts for user ports; this means a user
722  * process was waiting for a packet to arrive, and didn't want
723  * to poll
724  */
725 static void handle_urcv(struct ipath_devdata *dd, u32 istat)
726 {
727         u64 portr;
728         int i;
729         int rcvdint = 0;
730
731         portr = ((istat >> INFINIPATH_I_RCVAVAIL_SHIFT) &
732                  infinipath_i_rcvavail_mask)
733                 | ((istat >> INFINIPATH_I_RCVURG_SHIFT) &
734                    infinipath_i_rcvurg_mask);
735         for (i = 1; i < dd->ipath_cfgports; i++) {
736                 struct ipath_portdata *pd = dd->ipath_pd[i];
737                 if (portr & (1 << i) && pd && pd->port_cnt &&
738                         test_bit(IPATH_PORT_WAITING_RCV, &pd->port_flag)) {
739                         int rcbit;
740                         clear_bit(IPATH_PORT_WAITING_RCV,
741                                   &pd->port_flag);
742                         rcbit = i + INFINIPATH_R_INTRAVAIL_SHIFT;
743                         clear_bit(1UL << rcbit, &dd->ipath_rcvctrl);
744                         wake_up_interruptible(&pd->port_wait);
745                         rcvdint = 1;
746                 }
747         }
748         if (rcvdint) {
749                 /* only want to take one interrupt, so turn off the rcv
750                  * interrupt for all the ports that we did the wakeup on
751                  * (but never for kernel port)
752                  */
753                 ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
754                                  dd->ipath_rcvctrl);
755         }
756 }
757
758 irqreturn_t ipath_intr(int irq, void *data, struct pt_regs *regs)
759 {
760         struct ipath_devdata *dd = data;
761         u32 istat, chk0rcv = 0;
762         ipath_err_t estat = 0;
763         irqreturn_t ret;
764         u32 oldhead, curtail;
765         static unsigned unexpected = 0;
766         static const u32 port0rbits = (1U<<INFINIPATH_I_RCVAVAIL_SHIFT) |
767                  (1U<<INFINIPATH_I_RCVURG_SHIFT);
768
769         ipath_stats.sps_ints++;
770
771         if (!(dd->ipath_flags & IPATH_PRESENT)) {
772                 /*
773                  * This return value is not great, but we do not want the
774                  * interrupt core code to remove our interrupt handler
775                  * because we don't appear to be handling an interrupt
776                  * during a chip reset.
777                  */
778                 return IRQ_HANDLED;
779         }
780
781         /*
782          * this needs to be flags&initted, not statusp, so we keep
783          * taking interrupts even after link goes down, etc.
784          * Also, we *must* clear the interrupt at some point, or we won't
785          * take it again, which can be real bad for errors, etc...
786          */
787
788         if (!(dd->ipath_flags & IPATH_INITTED)) {
789                 ipath_bad_intr(dd, &unexpected);
790                 ret = IRQ_NONE;
791                 goto bail;
792         }
793
794         /*
795          * We try to avoid reading the interrupt status register, since
796          * that's a PIO read, and stalls the processor for up to about
797          * ~0.25 usec. The idea is that if we processed a port0 packet,
798          * we blindly clear the  port 0 receive interrupt bits, and nothing
799          * else, then return.  If other interrupts are pending, the chip
800          * will re-interrupt us as soon as we write the intclear register.
801          * We then won't process any more kernel packets (if not the 2nd
802          * time, then the 3rd or 4th) and we'll then handle the other
803          * interrupts.   We clear the interrupts first so that we don't
804          * lose intr for later packets that arrive while we are processing.
805          */
806         oldhead = dd->ipath_port0head;
807         curtail = (u32)le64_to_cpu(*dd->ipath_hdrqtailptr);
808         if (oldhead != curtail) {
809                 if (dd->ipath_flags & IPATH_GPIO_INTR) {
810                         ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_clear,
811                                          (u64) (1 << 2));
812                         istat = port0rbits | INFINIPATH_I_GPIO;
813                 }
814                 else
815                         istat = port0rbits;
816                 ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, istat);
817                 ipath_kreceive(dd);
818                 if (oldhead != dd->ipath_port0head) {
819                         ipath_stats.sps_fastrcvint++;
820                         goto done;
821                 }
822         }
823
824         istat = ipath_read_kreg32(dd, dd->ipath_kregs->kr_intstatus);
825
826         if (unlikely(!istat)) {
827                 ipath_stats.sps_nullintr++;
828                 ret = IRQ_NONE; /* not our interrupt, or already handled */
829                 goto bail;
830         }
831         if (unlikely(istat == -1)) {
832                 ipath_bad_regread(dd);
833                 /* don't know if it was our interrupt or not */
834                 ret = IRQ_NONE;
835                 goto bail;
836         }
837
838         if (unexpected)
839                 unexpected = 0;
840
841         if (unlikely(istat & ~infinipath_i_bitsextant))
842                 ipath_dev_err(dd,
843                               "interrupt with unknown interrupts %x set\n",
844                               istat & (u32) ~ infinipath_i_bitsextant);
845         else
846                 ipath_cdbg(VERBOSE, "intr stat=0x%x\n", istat);
847
848         if (unlikely(istat & INFINIPATH_I_ERROR)) {
849                 ipath_stats.sps_errints++;
850                 estat = ipath_read_kreg64(dd,
851                                           dd->ipath_kregs->kr_errorstatus);
852                 if (!estat)
853                         dev_info(&dd->pcidev->dev, "error interrupt (%x), "
854                                  "but no error bits set!\n", istat);
855                 else if (estat == -1LL)
856                         /*
857                          * should we try clearing all, or hope next read
858                          * works?
859                          */
860                         ipath_dev_err(dd, "Read of error status failed "
861                                       "(all bits set); ignoring\n");
862                 else
863                         if (handle_errors(dd, estat))
864                                 /* force calling ipath_kreceive() */
865                                 chk0rcv = 1;
866         }
867
868         if (istat & INFINIPATH_I_GPIO) {
869                 /*
870                  * Packets are available in the port 0 rcv queue.
871                  * Eventually this needs to be generalized to check
872                  * IPATH_GPIO_INTR, and the specific GPIO bit, if
873                  * GPIO interrupts are used for anything else.
874                  */
875                 if (unlikely(!(dd->ipath_flags & IPATH_GPIO_INTR))) {
876                         u32 gpiostatus;
877                         gpiostatus = ipath_read_kreg32(
878                                 dd, dd->ipath_kregs->kr_gpio_status);
879                         ipath_dbg("Unexpected GPIO interrupt bits %x\n",
880                                   gpiostatus);
881                         ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_clear,
882                                          gpiostatus);
883                 }
884                 else {
885                         /* Clear GPIO status bit 2 */
886                         ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_clear,
887                                         (u64) (1 << 2));
888                         chk0rcv = 1;
889                 }
890         }
891         chk0rcv |= istat & port0rbits;
892
893         /*
894          * Clear the interrupt bits we found set, unless they are receive
895          * related, in which case we already cleared them above, and don't
896          * want to clear them again, because we might lose an interrupt.
897          * Clear it early, so we "know" know the chip will have seen this by
898          * the time we process the queue, and will re-interrupt if necessary.
899          * The processor itself won't take the interrupt again until we return.
900          */
901         ipath_write_kreg(dd, dd->ipath_kregs->kr_intclear, istat);
902
903         /*
904          * handle port0 receive  before checking for pio buffers available,
905          * since receives can overflow; piobuf waiters can afford a few
906          * extra cycles, since they were waiting anyway, and user's waiting
907          * for receive are at the bottom.
908          */
909         if (chk0rcv) {
910                 ipath_kreceive(dd);
911                 istat &= ~port0rbits;
912         }
913
914         if (istat & ((infinipath_i_rcvavail_mask <<
915                       INFINIPATH_I_RCVAVAIL_SHIFT)
916                      | (infinipath_i_rcvurg_mask <<
917                         INFINIPATH_I_RCVURG_SHIFT)))
918                 handle_urcv(dd, istat);
919
920         if (istat & INFINIPATH_I_SPIOBUFAVAIL) {
921                 clear_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
922                 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
923                                  dd->ipath_sendctrl);
924
925                 if (dd->ipath_portpiowait)
926                         handle_port_pioavail(dd);
927
928                 handle_layer_pioavail(dd);
929         }
930
931 done:
932         ret = IRQ_HANDLED;
933
934 bail:
935         return ret;
936 }