66cafa20c246cea2a024fdaf1bbdaf5b6b26279d
[linux-2.6.git] / drivers / infiniband / ulp / ipoib / ipoib_ib.c
1 /*
2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35
36 #include <linux/delay.h>
37 #include <linux/dma-mapping.h>
38
39 #include <rdma/ib_cache.h>
40 #include <linux/ip.h>
41 #include <linux/tcp.h>
42
43 #include "ipoib.h"
44
45 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG_DATA
46 static int data_debug_level;
47
48 module_param(data_debug_level, int, 0644);
49 MODULE_PARM_DESC(data_debug_level,
50                  "Enable data path debug tracing if > 0");
51 #endif
52
53 static DEFINE_MUTEX(pkey_mutex);
54
55 struct ipoib_ah *ipoib_create_ah(struct net_device *dev,
56                                  struct ib_pd *pd, struct ib_ah_attr *attr)
57 {
58         struct ipoib_ah *ah;
59
60         ah = kmalloc(sizeof *ah, GFP_KERNEL);
61         if (!ah)
62                 return NULL;
63
64         ah->dev       = dev;
65         ah->last_send = 0;
66         kref_init(&ah->ref);
67
68         ah->ah = ib_create_ah(pd, attr);
69         if (IS_ERR(ah->ah)) {
70                 kfree(ah);
71                 ah = NULL;
72         } else
73                 ipoib_dbg(netdev_priv(dev), "Created ah %p\n", ah->ah);
74
75         return ah;
76 }
77
78 void ipoib_free_ah(struct kref *kref)
79 {
80         struct ipoib_ah *ah = container_of(kref, struct ipoib_ah, ref);
81         struct ipoib_dev_priv *priv = netdev_priv(ah->dev);
82
83         unsigned long flags;
84
85         spin_lock_irqsave(&priv->lock, flags);
86         list_add_tail(&ah->list, &priv->dead_ahs);
87         spin_unlock_irqrestore(&priv->lock, flags);
88 }
89
90 static void ipoib_ud_dma_unmap_rx(struct ipoib_dev_priv *priv,
91                                   u64 mapping[IPOIB_UD_RX_SG])
92 {
93         if (ipoib_ud_need_sg(priv->max_ib_mtu)) {
94                 ib_dma_unmap_single(priv->ca, mapping[0], IPOIB_UD_HEAD_SIZE,
95                                     DMA_FROM_DEVICE);
96                 ib_dma_unmap_page(priv->ca, mapping[1], PAGE_SIZE,
97                                   DMA_FROM_DEVICE);
98         } else
99                 ib_dma_unmap_single(priv->ca, mapping[0],
100                                     IPOIB_UD_BUF_SIZE(priv->max_ib_mtu),
101                                     DMA_FROM_DEVICE);
102 }
103
104 static void ipoib_ud_skb_put_frags(struct ipoib_dev_priv *priv,
105                                    struct sk_buff *skb,
106                                    unsigned int length)
107 {
108         if (ipoib_ud_need_sg(priv->max_ib_mtu)) {
109                 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
110                 unsigned int size;
111                 /*
112                  * There is only two buffers needed for max_payload = 4K,
113                  * first buf size is IPOIB_UD_HEAD_SIZE
114                  */
115                 skb->tail += IPOIB_UD_HEAD_SIZE;
116                 skb->len  += length;
117
118                 size = length - IPOIB_UD_HEAD_SIZE;
119
120                 frag->size     = size;
121                 skb->data_len += size;
122                 skb->truesize += size;
123         } else
124                 skb_put(skb, length);
125
126 }
127
128 static int ipoib_ib_post_receive(struct net_device *dev, int id)
129 {
130         struct ipoib_dev_priv *priv = netdev_priv(dev);
131         struct ib_recv_wr *bad_wr;
132         int ret;
133
134         priv->rx_wr.wr_id   = id | IPOIB_OP_RECV;
135         priv->rx_sge[0].addr = priv->rx_ring[id].mapping[0];
136         priv->rx_sge[1].addr = priv->rx_ring[id].mapping[1];
137
138
139         ret = ib_post_recv(priv->qp, &priv->rx_wr, &bad_wr);
140         if (unlikely(ret)) {
141                 ipoib_warn(priv, "receive failed for buf %d (%d)\n", id, ret);
142                 ipoib_ud_dma_unmap_rx(priv, priv->rx_ring[id].mapping);
143                 dev_kfree_skb_any(priv->rx_ring[id].skb);
144                 priv->rx_ring[id].skb = NULL;
145         }
146
147         return ret;
148 }
149
150 static struct sk_buff *ipoib_alloc_rx_skb(struct net_device *dev, int id)
151 {
152         struct ipoib_dev_priv *priv = netdev_priv(dev);
153         struct sk_buff *skb;
154         int buf_size;
155         u64 *mapping;
156
157         if (ipoib_ud_need_sg(priv->max_ib_mtu))
158                 buf_size = IPOIB_UD_HEAD_SIZE;
159         else
160                 buf_size = IPOIB_UD_BUF_SIZE(priv->max_ib_mtu);
161
162         skb = dev_alloc_skb(buf_size + 4);
163         if (unlikely(!skb))
164                 return NULL;
165
166         /*
167          * IB will leave a 40 byte gap for a GRH and IPoIB adds a 4 byte
168          * header.  So we need 4 more bytes to get to 48 and align the
169          * IP header to a multiple of 16.
170          */
171         skb_reserve(skb, 4);
172
173         mapping = priv->rx_ring[id].mapping;
174         mapping[0] = ib_dma_map_single(priv->ca, skb->data, buf_size,
175                                        DMA_FROM_DEVICE);
176         if (unlikely(ib_dma_mapping_error(priv->ca, mapping[0])))
177                 goto error;
178
179         if (ipoib_ud_need_sg(priv->max_ib_mtu)) {
180                 struct page *page = alloc_page(GFP_ATOMIC);
181                 if (!page)
182                         goto partial_error;
183                 skb_fill_page_desc(skb, 0, page, 0, PAGE_SIZE);
184                 mapping[1] =
185                         ib_dma_map_page(priv->ca, skb_shinfo(skb)->frags[0].page,
186                                         0, PAGE_SIZE, DMA_FROM_DEVICE);
187                 if (unlikely(ib_dma_mapping_error(priv->ca, mapping[1])))
188                         goto partial_error;
189         }
190
191         priv->rx_ring[id].skb = skb;
192         return skb;
193
194 partial_error:
195         ib_dma_unmap_single(priv->ca, mapping[0], buf_size, DMA_FROM_DEVICE);
196 error:
197         dev_kfree_skb_any(skb);
198         return NULL;
199 }
200
201 static int ipoib_ib_post_receives(struct net_device *dev)
202 {
203         struct ipoib_dev_priv *priv = netdev_priv(dev);
204         int i;
205
206         for (i = 0; i < ipoib_recvq_size; ++i) {
207                 if (!ipoib_alloc_rx_skb(dev, i)) {
208                         ipoib_warn(priv, "failed to allocate receive buffer %d\n", i);
209                         return -ENOMEM;
210                 }
211                 if (ipoib_ib_post_receive(dev, i)) {
212                         ipoib_warn(priv, "ipoib_ib_post_receive failed for buf %d\n", i);
213                         return -EIO;
214                 }
215         }
216
217         return 0;
218 }
219
220 static void ipoib_ib_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
221 {
222         struct ipoib_dev_priv *priv = netdev_priv(dev);
223         unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV;
224         struct sk_buff *skb;
225         u64 mapping[IPOIB_UD_RX_SG];
226
227         ipoib_dbg_data(priv, "recv completion: id %d, status: %d\n",
228                        wr_id, wc->status);
229
230         if (unlikely(wr_id >= ipoib_recvq_size)) {
231                 ipoib_warn(priv, "recv completion event with wrid %d (> %d)\n",
232                            wr_id, ipoib_recvq_size);
233                 return;
234         }
235
236         skb  = priv->rx_ring[wr_id].skb;
237
238         if (unlikely(wc->status != IB_WC_SUCCESS)) {
239                 if (wc->status != IB_WC_WR_FLUSH_ERR)
240                         ipoib_warn(priv, "failed recv event "
241                                    "(status=%d, wrid=%d vend_err %x)\n",
242                                    wc->status, wr_id, wc->vendor_err);
243                 ipoib_ud_dma_unmap_rx(priv, priv->rx_ring[wr_id].mapping);
244                 dev_kfree_skb_any(skb);
245                 priv->rx_ring[wr_id].skb = NULL;
246                 return;
247         }
248
249         /*
250          * Drop packets that this interface sent, ie multicast packets
251          * that the HCA has replicated.
252          */
253         if (wc->slid == priv->local_lid && wc->src_qp == priv->qp->qp_num)
254                 goto repost;
255
256         memcpy(mapping, priv->rx_ring[wr_id].mapping,
257                IPOIB_UD_RX_SG * sizeof *mapping);
258
259         /*
260          * If we can't allocate a new RX buffer, dump
261          * this packet and reuse the old buffer.
262          */
263         if (unlikely(!ipoib_alloc_rx_skb(dev, wr_id))) {
264                 ++dev->stats.rx_dropped;
265                 goto repost;
266         }
267
268         ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
269                        wc->byte_len, wc->slid);
270
271         ipoib_ud_dma_unmap_rx(priv, mapping);
272         ipoib_ud_skb_put_frags(priv, skb, wc->byte_len);
273
274         skb_pull(skb, IB_GRH_BYTES);
275
276         skb->protocol = ((struct ipoib_header *) skb->data)->proto;
277         skb_reset_mac_header(skb);
278         skb_pull(skb, IPOIB_ENCAP_LEN);
279
280         dev->last_rx = jiffies;
281         ++dev->stats.rx_packets;
282         dev->stats.rx_bytes += skb->len;
283
284         skb->dev = dev;
285         /* XXX get correct PACKET_ type here */
286         skb->pkt_type = PACKET_HOST;
287
288         if (test_bit(IPOIB_FLAG_CSUM, &priv->flags) && likely(wc->csum_ok))
289                 skb->ip_summed = CHECKSUM_UNNECESSARY;
290
291         if (dev->features & NETIF_F_LRO)
292                 lro_receive_skb(&priv->lro.lro_mgr, skb, NULL);
293         else
294                 netif_receive_skb(skb);
295
296 repost:
297         if (unlikely(ipoib_ib_post_receive(dev, wr_id)))
298                 ipoib_warn(priv, "ipoib_ib_post_receive failed "
299                            "for buf %d\n", wr_id);
300 }
301
302 static int ipoib_dma_map_tx(struct ib_device *ca,
303                             struct ipoib_tx_buf *tx_req)
304 {
305         struct sk_buff *skb = tx_req->skb;
306         u64 *mapping = tx_req->mapping;
307         int i;
308         int off;
309
310         if (skb_headlen(skb)) {
311                 mapping[0] = ib_dma_map_single(ca, skb->data, skb_headlen(skb),
312                                                DMA_TO_DEVICE);
313                 if (unlikely(ib_dma_mapping_error(ca, mapping[0])))
314                         return -EIO;
315
316                 off = 1;
317         } else
318                 off = 0;
319
320         for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
321                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
322                 mapping[i + off] = ib_dma_map_page(ca, frag->page,
323                                                  frag->page_offset, frag->size,
324                                                  DMA_TO_DEVICE);
325                 if (unlikely(ib_dma_mapping_error(ca, mapping[i + off])))
326                         goto partial_error;
327         }
328         return 0;
329
330 partial_error:
331         for (; i > 0; --i) {
332                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
333                 ib_dma_unmap_page(ca, mapping[i - !off], frag->size, DMA_TO_DEVICE);
334         }
335
336         if (off)
337                 ib_dma_unmap_single(ca, mapping[0], skb_headlen(skb), DMA_TO_DEVICE);
338
339         return -EIO;
340 }
341
342 static void ipoib_dma_unmap_tx(struct ib_device *ca,
343                                struct ipoib_tx_buf *tx_req)
344 {
345         struct sk_buff *skb = tx_req->skb;
346         u64 *mapping = tx_req->mapping;
347         int i;
348         int off;
349
350         if (skb_headlen(skb)) {
351                 ib_dma_unmap_single(ca, mapping[0], skb_headlen(skb), DMA_TO_DEVICE);
352                 off = 1;
353         } else
354                 off = 0;
355
356         for (i = 0; i < skb_shinfo(skb)->nr_frags; ++i) {
357                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
358                 ib_dma_unmap_page(ca, mapping[i + off], frag->size,
359                                   DMA_TO_DEVICE);
360         }
361 }
362
363 static void ipoib_ib_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
364 {
365         struct ipoib_dev_priv *priv = netdev_priv(dev);
366         unsigned int wr_id = wc->wr_id;
367         struct ipoib_tx_buf *tx_req;
368
369         ipoib_dbg_data(priv, "send completion: id %d, status: %d\n",
370                        wr_id, wc->status);
371
372         if (unlikely(wr_id >= ipoib_sendq_size)) {
373                 ipoib_warn(priv, "send completion event with wrid %d (> %d)\n",
374                            wr_id, ipoib_sendq_size);
375                 return;
376         }
377
378         tx_req = &priv->tx_ring[wr_id];
379
380         ipoib_dma_unmap_tx(priv->ca, tx_req);
381
382         ++dev->stats.tx_packets;
383         dev->stats.tx_bytes += tx_req->skb->len;
384
385         dev_kfree_skb_any(tx_req->skb);
386
387         ++priv->tx_tail;
388         if (unlikely(--priv->tx_outstanding == ipoib_sendq_size >> 1) &&
389             netif_queue_stopped(dev) &&
390             test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
391                 netif_wake_queue(dev);
392
393         if (wc->status != IB_WC_SUCCESS &&
394             wc->status != IB_WC_WR_FLUSH_ERR)
395                 ipoib_warn(priv, "failed send event "
396                            "(status=%d, wrid=%d vend_err %x)\n",
397                            wc->status, wr_id, wc->vendor_err);
398 }
399
400 static int poll_tx(struct ipoib_dev_priv *priv)
401 {
402         int n, i;
403
404         n = ib_poll_cq(priv->send_cq, MAX_SEND_CQE, priv->send_wc);
405         for (i = 0; i < n; ++i)
406                 ipoib_ib_handle_tx_wc(priv->dev, priv->send_wc + i);
407
408         return n == MAX_SEND_CQE;
409 }
410
411 int ipoib_poll(struct napi_struct *napi, int budget)
412 {
413         struct ipoib_dev_priv *priv = container_of(napi, struct ipoib_dev_priv, napi);
414         struct net_device *dev = priv->dev;
415         int done;
416         int t;
417         int n, i;
418
419         done  = 0;
420
421 poll_more:
422         while (done < budget) {
423                 int max = (budget - done);
424
425                 t = min(IPOIB_NUM_WC, max);
426                 n = ib_poll_cq(priv->recv_cq, t, priv->ibwc);
427
428                 for (i = 0; i < n; i++) {
429                         struct ib_wc *wc = priv->ibwc + i;
430
431                         if (wc->wr_id & IPOIB_OP_RECV) {
432                                 ++done;
433                                 if (wc->wr_id & IPOIB_OP_CM)
434                                         ipoib_cm_handle_rx_wc(dev, wc);
435                                 else
436                                         ipoib_ib_handle_rx_wc(dev, wc);
437                         } else
438                                 ipoib_cm_handle_tx_wc(priv->dev, wc);
439                 }
440
441                 if (n != t)
442                         break;
443         }
444
445         if (done < budget) {
446                 if (dev->features & NETIF_F_LRO)
447                         lro_flush_all(&priv->lro.lro_mgr);
448
449                 netif_rx_complete(dev, napi);
450                 if (unlikely(ib_req_notify_cq(priv->recv_cq,
451                                               IB_CQ_NEXT_COMP |
452                                               IB_CQ_REPORT_MISSED_EVENTS)) &&
453                     netif_rx_reschedule(dev, napi))
454                         goto poll_more;
455         }
456
457         return done;
458 }
459
460 void ipoib_ib_completion(struct ib_cq *cq, void *dev_ptr)
461 {
462         struct net_device *dev = dev_ptr;
463         struct ipoib_dev_priv *priv = netdev_priv(dev);
464
465         netif_rx_schedule(dev, &priv->napi);
466 }
467
468 static void drain_tx_cq(struct net_device *dev)
469 {
470         struct ipoib_dev_priv *priv = netdev_priv(dev);
471         unsigned long flags;
472
473         spin_lock_irqsave(&priv->tx_lock, flags);
474         while (poll_tx(priv))
475                 ; /* nothing */
476
477         if (netif_queue_stopped(dev))
478                 mod_timer(&priv->poll_timer, jiffies + 1);
479
480         spin_unlock_irqrestore(&priv->tx_lock, flags);
481 }
482
483 void ipoib_send_comp_handler(struct ib_cq *cq, void *dev_ptr)
484 {
485         drain_tx_cq((struct net_device *)dev_ptr);
486 }
487
488 static inline int post_send(struct ipoib_dev_priv *priv,
489                             unsigned int wr_id,
490                             struct ib_ah *address, u32 qpn,
491                             struct ipoib_tx_buf *tx_req,
492                             void *head, int hlen)
493 {
494         struct ib_send_wr *bad_wr;
495         int i, off;
496         struct sk_buff *skb = tx_req->skb;
497         skb_frag_t *frags = skb_shinfo(skb)->frags;
498         int nr_frags = skb_shinfo(skb)->nr_frags;
499         u64 *mapping = tx_req->mapping;
500
501         if (skb_headlen(skb)) {
502                 priv->tx_sge[0].addr         = mapping[0];
503                 priv->tx_sge[0].length       = skb_headlen(skb);
504                 off = 1;
505         } else
506                 off = 0;
507
508         for (i = 0; i < nr_frags; ++i) {
509                 priv->tx_sge[i + off].addr = mapping[i + off];
510                 priv->tx_sge[i + off].length = frags[i].size;
511         }
512         priv->tx_wr.num_sge          = nr_frags + off;
513         priv->tx_wr.wr_id            = wr_id;
514         priv->tx_wr.wr.ud.remote_qpn = qpn;
515         priv->tx_wr.wr.ud.ah         = address;
516
517         if (head) {
518                 priv->tx_wr.wr.ud.mss    = skb_shinfo(skb)->gso_size;
519                 priv->tx_wr.wr.ud.header = head;
520                 priv->tx_wr.wr.ud.hlen   = hlen;
521                 priv->tx_wr.opcode       = IB_WR_LSO;
522         } else
523                 priv->tx_wr.opcode       = IB_WR_SEND;
524
525         return ib_post_send(priv->qp, &priv->tx_wr, &bad_wr);
526 }
527
528 void ipoib_send(struct net_device *dev, struct sk_buff *skb,
529                 struct ipoib_ah *address, u32 qpn)
530 {
531         struct ipoib_dev_priv *priv = netdev_priv(dev);
532         struct ipoib_tx_buf *tx_req;
533         int hlen;
534         void *phead;
535
536         if (skb_is_gso(skb)) {
537                 hlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
538                 phead = skb->data;
539                 if (unlikely(!skb_pull(skb, hlen))) {
540                         ipoib_warn(priv, "linear data too small\n");
541                         ++dev->stats.tx_dropped;
542                         ++dev->stats.tx_errors;
543                         dev_kfree_skb_any(skb);
544                         return;
545                 }
546         } else {
547                 if (unlikely(skb->len > priv->mcast_mtu + IPOIB_ENCAP_LEN)) {
548                         ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n",
549                                    skb->len, priv->mcast_mtu + IPOIB_ENCAP_LEN);
550                         ++dev->stats.tx_dropped;
551                         ++dev->stats.tx_errors;
552                         ipoib_cm_skb_too_long(dev, skb, priv->mcast_mtu);
553                         return;
554                 }
555                 phead = NULL;
556                 hlen  = 0;
557         }
558
559         ipoib_dbg_data(priv, "sending packet, length=%d address=%p qpn=0x%06x\n",
560                        skb->len, address, qpn);
561
562         /*
563          * We put the skb into the tx_ring _before_ we call post_send()
564          * because it's entirely possible that the completion handler will
565          * run before we execute anything after the post_send().  That
566          * means we have to make sure everything is properly recorded and
567          * our state is consistent before we call post_send().
568          */
569         tx_req = &priv->tx_ring[priv->tx_head & (ipoib_sendq_size - 1)];
570         tx_req->skb = skb;
571         if (unlikely(ipoib_dma_map_tx(priv->ca, tx_req))) {
572                 ++dev->stats.tx_errors;
573                 dev_kfree_skb_any(skb);
574                 return;
575         }
576
577         if (skb->ip_summed == CHECKSUM_PARTIAL)
578                 priv->tx_wr.send_flags |= IB_SEND_IP_CSUM;
579         else
580                 priv->tx_wr.send_flags &= ~IB_SEND_IP_CSUM;
581
582         if (++priv->tx_outstanding == ipoib_sendq_size) {
583                 ipoib_dbg(priv, "TX ring full, stopping kernel net queue\n");
584                 if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP))
585                         ipoib_warn(priv, "request notify on send CQ failed\n");
586                 netif_stop_queue(dev);
587         }
588
589         if (unlikely(post_send(priv, priv->tx_head & (ipoib_sendq_size - 1),
590                                address->ah, qpn, tx_req, phead, hlen))) {
591                 ipoib_warn(priv, "post_send failed\n");
592                 ++dev->stats.tx_errors;
593                 --priv->tx_outstanding;
594                 ipoib_dma_unmap_tx(priv->ca, tx_req);
595                 dev_kfree_skb_any(skb);
596                 if (netif_queue_stopped(dev))
597                         netif_wake_queue(dev);
598         } else {
599                 dev->trans_start = jiffies;
600
601                 address->last_send = priv->tx_head;
602                 ++priv->tx_head;
603                 skb_orphan(skb);
604
605         }
606
607         if (unlikely(priv->tx_outstanding > MAX_SEND_CQE))
608                 while (poll_tx(priv))
609                         ; /* nothing */
610 }
611
612 static void __ipoib_reap_ah(struct net_device *dev)
613 {
614         struct ipoib_dev_priv *priv = netdev_priv(dev);
615         struct ipoib_ah *ah, *tah;
616         LIST_HEAD(remove_list);
617
618         spin_lock_irq(&priv->tx_lock);
619         spin_lock(&priv->lock);
620         list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
621                 if ((int) priv->tx_tail - (int) ah->last_send >= 0) {
622                         list_del(&ah->list);
623                         ib_destroy_ah(ah->ah);
624                         kfree(ah);
625                 }
626         spin_unlock(&priv->lock);
627         spin_unlock_irq(&priv->tx_lock);
628 }
629
630 void ipoib_reap_ah(struct work_struct *work)
631 {
632         struct ipoib_dev_priv *priv =
633                 container_of(work, struct ipoib_dev_priv, ah_reap_task.work);
634         struct net_device *dev = priv->dev;
635
636         __ipoib_reap_ah(dev);
637
638         if (!test_bit(IPOIB_STOP_REAPER, &priv->flags))
639                 queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task,
640                                    round_jiffies_relative(HZ));
641 }
642
643 static void ipoib_ib_tx_timer_func(unsigned long ctx)
644 {
645         drain_tx_cq((struct net_device *)ctx);
646 }
647
648 int ipoib_ib_dev_open(struct net_device *dev)
649 {
650         struct ipoib_dev_priv *priv = netdev_priv(dev);
651         int ret;
652
653         if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &priv->pkey_index)) {
654                 ipoib_warn(priv, "P_Key 0x%04x not found\n", priv->pkey);
655                 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
656                 return -1;
657         }
658         set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
659
660         ret = ipoib_init_qp(dev);
661         if (ret) {
662                 ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
663                 return -1;
664         }
665
666         ret = ipoib_ib_post_receives(dev);
667         if (ret) {
668                 ipoib_warn(priv, "ipoib_ib_post_receives returned %d\n", ret);
669                 ipoib_ib_dev_stop(dev, 1);
670                 return -1;
671         }
672
673         ret = ipoib_cm_dev_open(dev);
674         if (ret) {
675                 ipoib_warn(priv, "ipoib_cm_dev_open returned %d\n", ret);
676                 ipoib_ib_dev_stop(dev, 1);
677                 return -1;
678         }
679
680         clear_bit(IPOIB_STOP_REAPER, &priv->flags);
681         queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task,
682                            round_jiffies_relative(HZ));
683
684         init_timer(&priv->poll_timer);
685         priv->poll_timer.function = ipoib_ib_tx_timer_func;
686         priv->poll_timer.data = (unsigned long)dev;
687
688         set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
689
690         return 0;
691 }
692
693 static void ipoib_pkey_dev_check_presence(struct net_device *dev)
694 {
695         struct ipoib_dev_priv *priv = netdev_priv(dev);
696         u16 pkey_index = 0;
697
698         if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
699                 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
700         else
701                 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
702 }
703
704 int ipoib_ib_dev_up(struct net_device *dev)
705 {
706         struct ipoib_dev_priv *priv = netdev_priv(dev);
707
708         ipoib_pkey_dev_check_presence(dev);
709
710         if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
711                 ipoib_dbg(priv, "PKEY is not assigned.\n");
712                 return 0;
713         }
714
715         set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
716
717         return ipoib_mcast_start_thread(dev);
718 }
719
720 int ipoib_ib_dev_down(struct net_device *dev, int flush)
721 {
722         struct ipoib_dev_priv *priv = netdev_priv(dev);
723
724         ipoib_dbg(priv, "downing ib_dev\n");
725
726         clear_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
727         netif_carrier_off(dev);
728
729         /* Shutdown the P_Key thread if still active */
730         if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
731                 mutex_lock(&pkey_mutex);
732                 set_bit(IPOIB_PKEY_STOP, &priv->flags);
733                 cancel_delayed_work(&priv->pkey_poll_task);
734                 mutex_unlock(&pkey_mutex);
735                 if (flush)
736                         flush_workqueue(ipoib_workqueue);
737         }
738
739         ipoib_mcast_stop_thread(dev, flush);
740         ipoib_mcast_dev_flush(dev);
741
742         ipoib_flush_paths(dev);
743
744         return 0;
745 }
746
747 static int recvs_pending(struct net_device *dev)
748 {
749         struct ipoib_dev_priv *priv = netdev_priv(dev);
750         int pending = 0;
751         int i;
752
753         for (i = 0; i < ipoib_recvq_size; ++i)
754                 if (priv->rx_ring[i].skb)
755                         ++pending;
756
757         return pending;
758 }
759
760 void ipoib_drain_cq(struct net_device *dev)
761 {
762         struct ipoib_dev_priv *priv = netdev_priv(dev);
763         int i, n;
764         do {
765                 n = ib_poll_cq(priv->recv_cq, IPOIB_NUM_WC, priv->ibwc);
766                 for (i = 0; i < n; ++i) {
767                         /*
768                          * Convert any successful completions to flush
769                          * errors to avoid passing packets up the
770                          * stack after bringing the device down.
771                          */
772                         if (priv->ibwc[i].status == IB_WC_SUCCESS)
773                                 priv->ibwc[i].status = IB_WC_WR_FLUSH_ERR;
774
775                         if (priv->ibwc[i].wr_id & IPOIB_OP_RECV) {
776                                 if (priv->ibwc[i].wr_id & IPOIB_OP_CM)
777                                         ipoib_cm_handle_rx_wc(dev, priv->ibwc + i);
778                                 else
779                                         ipoib_ib_handle_rx_wc(dev, priv->ibwc + i);
780                         } else
781                                 ipoib_cm_handle_tx_wc(dev, priv->ibwc + i);
782                 }
783         } while (n == IPOIB_NUM_WC);
784
785         while (poll_tx(priv))
786                 ; /* nothing */
787 }
788
789 int ipoib_ib_dev_stop(struct net_device *dev, int flush)
790 {
791         struct ipoib_dev_priv *priv = netdev_priv(dev);
792         struct ib_qp_attr qp_attr;
793         unsigned long begin;
794         struct ipoib_tx_buf *tx_req;
795         int i;
796
797         clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
798
799         ipoib_cm_dev_stop(dev);
800
801         /*
802          * Move our QP to the error state and then reinitialize in
803          * when all work requests have completed or have been flushed.
804          */
805         qp_attr.qp_state = IB_QPS_ERR;
806         if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
807                 ipoib_warn(priv, "Failed to modify QP to ERROR state\n");
808
809         /* Wait for all sends and receives to complete */
810         begin = jiffies;
811
812         while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
813                 if (time_after(jiffies, begin + 5 * HZ)) {
814                         ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",
815                                    priv->tx_head - priv->tx_tail, recvs_pending(dev));
816
817                         /*
818                          * assume the HW is wedged and just free up
819                          * all our pending work requests.
820                          */
821                         while ((int) priv->tx_tail - (int) priv->tx_head < 0) {
822                                 tx_req = &priv->tx_ring[priv->tx_tail &
823                                                         (ipoib_sendq_size - 1)];
824                                 ipoib_dma_unmap_tx(priv->ca, tx_req);
825                                 dev_kfree_skb_any(tx_req->skb);
826                                 ++priv->tx_tail;
827                                 --priv->tx_outstanding;
828                         }
829
830                         for (i = 0; i < ipoib_recvq_size; ++i) {
831                                 struct ipoib_rx_buf *rx_req;
832
833                                 rx_req = &priv->rx_ring[i];
834                                 if (!rx_req->skb)
835                                         continue;
836                                 ipoib_ud_dma_unmap_rx(priv,
837                                                       priv->rx_ring[i].mapping);
838                                 dev_kfree_skb_any(rx_req->skb);
839                                 rx_req->skb = NULL;
840                         }
841
842                         goto timeout;
843                 }
844
845                 ipoib_drain_cq(dev);
846
847                 msleep(1);
848         }
849
850         ipoib_dbg(priv, "All sends and receives done.\n");
851
852 timeout:
853         del_timer_sync(&priv->poll_timer);
854         qp_attr.qp_state = IB_QPS_RESET;
855         if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
856                 ipoib_warn(priv, "Failed to modify QP to RESET state\n");
857
858         /* Wait for all AHs to be reaped */
859         set_bit(IPOIB_STOP_REAPER, &priv->flags);
860         cancel_delayed_work(&priv->ah_reap_task);
861         if (flush)
862                 flush_workqueue(ipoib_workqueue);
863
864         begin = jiffies;
865
866         while (!list_empty(&priv->dead_ahs)) {
867                 __ipoib_reap_ah(dev);
868
869                 if (time_after(jiffies, begin + HZ)) {
870                         ipoib_warn(priv, "timing out; will leak address handles\n");
871                         break;
872                 }
873
874                 msleep(1);
875         }
876
877         ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
878
879         return 0;
880 }
881
882 int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
883 {
884         struct ipoib_dev_priv *priv = netdev_priv(dev);
885
886         priv->ca = ca;
887         priv->port = port;
888         priv->qp = NULL;
889
890         if (ipoib_transport_dev_init(dev, ca)) {
891                 printk(KERN_WARNING "%s: ipoib_transport_dev_init failed\n", ca->name);
892                 return -ENODEV;
893         }
894
895         if (dev->flags & IFF_UP) {
896                 if (ipoib_ib_dev_open(dev)) {
897                         ipoib_transport_dev_cleanup(dev);
898                         return -ENODEV;
899                 }
900         }
901
902         return 0;
903 }
904
905 static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
906                                 enum ipoib_flush_level level)
907 {
908         struct ipoib_dev_priv *cpriv;
909         struct net_device *dev = priv->dev;
910         u16 new_index;
911
912         mutex_lock(&priv->vlan_mutex);
913
914         /*
915          * Flush any child interfaces too -- they might be up even if
916          * the parent is down.
917          */
918         list_for_each_entry(cpriv, &priv->child_intfs, list)
919                 __ipoib_ib_dev_flush(cpriv, level);
920
921         mutex_unlock(&priv->vlan_mutex);
922
923         if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags)) {
924                 ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
925                 return;
926         }
927
928         if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
929                 ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_ADMIN_UP not set.\n");
930                 return;
931         }
932
933         if (level == IPOIB_FLUSH_HEAVY) {
934                 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &new_index)) {
935                         clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
936                         ipoib_ib_dev_down(dev, 0);
937                         ipoib_ib_dev_stop(dev, 0);
938                         if (ipoib_pkey_dev_delay_open(dev))
939                                 return;
940                 }
941
942                 /* restart QP only if P_Key index is changed */
943                 if (test_and_set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags) &&
944                     new_index == priv->pkey_index) {
945                         ipoib_dbg(priv, "Not flushing - P_Key index not changed.\n");
946                         return;
947                 }
948                 priv->pkey_index = new_index;
949         }
950
951         if (level == IPOIB_FLUSH_LIGHT) {
952                 ipoib_mark_paths_invalid(dev);
953                 ipoib_mcast_dev_flush(dev);
954         }
955
956         if (level >= IPOIB_FLUSH_NORMAL)
957                 ipoib_ib_dev_down(dev, 0);
958
959         if (level == IPOIB_FLUSH_HEAVY) {
960                 ipoib_ib_dev_stop(dev, 0);
961                 ipoib_ib_dev_open(dev);
962         }
963
964         /*
965          * The device could have been brought down between the start and when
966          * we get here, don't bring it back up if it's not configured up
967          */
968         if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
969                 if (level >= IPOIB_FLUSH_NORMAL)
970                         ipoib_ib_dev_up(dev);
971                 ipoib_mcast_restart_task(&priv->restart_task);
972         }
973 }
974
975 void ipoib_ib_dev_flush_light(struct work_struct *work)
976 {
977         struct ipoib_dev_priv *priv =
978                 container_of(work, struct ipoib_dev_priv, flush_light);
979
980         __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_LIGHT);
981 }
982
983 void ipoib_ib_dev_flush_normal(struct work_struct *work)
984 {
985         struct ipoib_dev_priv *priv =
986                 container_of(work, struct ipoib_dev_priv, flush_normal);
987
988         __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_NORMAL);
989 }
990
991 void ipoib_ib_dev_flush_heavy(struct work_struct *work)
992 {
993         struct ipoib_dev_priv *priv =
994                 container_of(work, struct ipoib_dev_priv, flush_heavy);
995
996         __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_HEAVY);
997 }
998
999 void ipoib_ib_dev_cleanup(struct net_device *dev)
1000 {
1001         struct ipoib_dev_priv *priv = netdev_priv(dev);
1002
1003         ipoib_dbg(priv, "cleaning up ib_dev\n");
1004
1005         ipoib_mcast_stop_thread(dev, 1);
1006         ipoib_mcast_dev_flush(dev);
1007
1008         ipoib_transport_dev_cleanup(dev);
1009 }
1010
1011 /*
1012  * Delayed P_Key Assigment Interim Support
1013  *
1014  * The following is initial implementation of delayed P_Key assigment
1015  * mechanism. It is using the same approach implemented for the multicast
1016  * group join. The single goal of this implementation is to quickly address
1017  * Bug #2507. This implementation will probably be removed when the P_Key
1018  * change async notification is available.
1019  */
1020
1021 void ipoib_pkey_poll(struct work_struct *work)
1022 {
1023         struct ipoib_dev_priv *priv =
1024                 container_of(work, struct ipoib_dev_priv, pkey_poll_task.work);
1025         struct net_device *dev = priv->dev;
1026
1027         ipoib_pkey_dev_check_presence(dev);
1028
1029         if (test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
1030                 ipoib_open(dev);
1031         else {
1032                 mutex_lock(&pkey_mutex);
1033                 if (!test_bit(IPOIB_PKEY_STOP, &priv->flags))
1034                         queue_delayed_work(ipoib_workqueue,
1035                                            &priv->pkey_poll_task,
1036                                            HZ);
1037                 mutex_unlock(&pkey_mutex);
1038         }
1039 }
1040
1041 int ipoib_pkey_dev_delay_open(struct net_device *dev)
1042 {
1043         struct ipoib_dev_priv *priv = netdev_priv(dev);
1044
1045         /* Look for the interface pkey value in the IB Port P_Key table and */
1046         /* set the interface pkey assigment flag                            */
1047         ipoib_pkey_dev_check_presence(dev);
1048
1049         /* P_Key value not assigned yet - start polling */
1050         if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
1051                 mutex_lock(&pkey_mutex);
1052                 clear_bit(IPOIB_PKEY_STOP, &priv->flags);
1053                 queue_delayed_work(ipoib_workqueue,
1054                                    &priv->pkey_poll_task,
1055                                    HZ);
1056                 mutex_unlock(&pkey_mutex);
1057                 return 1;
1058         }
1059
1060         return 0;
1061 }