tg3: Remove 5750 PCI code
[linux-3.10.git] / drivers / net / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2  *
3  * Copyright (c) 2007-2010 Broadcom Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  *
9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
10  * Written by: Eliezer Tamir
11  * Based on code from Michael Chan's bnx2 driver
12  * UDP CSUM errata workaround by Arik Gendelman
13  * Slowpath and fastpath rework by Vladislav Zolotarov
14  * Statistics and Link management by Yitchak Gertner
15  *
16  */
17
18 #include <linux/etherdevice.h>
19 #include <linux/if_vlan.h>
20 #include <linux/ip.h>
21 #include <net/ipv6.h>
22 #include <net/ip6_checksum.h>
23 #include <linux/firmware.h>
24 #include "bnx2x_cmn.h"
25
26 #include "bnx2x_init.h"
27
28 static int bnx2x_setup_irqs(struct bnx2x *bp);
29
30 /* free skb in the packet ring at pos idx
31  * return idx of last bd freed
32  */
33 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
34                              u16 idx)
35 {
36         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
37         struct eth_tx_start_bd *tx_start_bd;
38         struct eth_tx_bd *tx_data_bd;
39         struct sk_buff *skb = tx_buf->skb;
40         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
41         int nbd;
42
43         /* prefetch skb end pointer to speedup dev_kfree_skb() */
44         prefetch(&skb->end);
45
46         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
47            idx, tx_buf, skb);
48
49         /* unmap first bd */
50         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
51         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
52         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
53                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
54
55         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
56 #ifdef BNX2X_STOP_ON_ERROR
57         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
58                 BNX2X_ERR("BAD nbd!\n");
59                 bnx2x_panic();
60         }
61 #endif
62         new_cons = nbd + tx_buf->first_bd;
63
64         /* Get the next bd */
65         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
66
67         /* Skip a parse bd... */
68         --nbd;
69         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
70
71         /* ...and the TSO split header bd since they have no mapping */
72         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
73                 --nbd;
74                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
75         }
76
77         /* now free frags */
78         while (nbd > 0) {
79
80                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
81                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
82                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
83                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
84                 if (--nbd)
85                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
86         }
87
88         /* release skb */
89         WARN_ON(!skb);
90         dev_kfree_skb(skb);
91         tx_buf->first_bd = 0;
92         tx_buf->skb = NULL;
93
94         return new_cons;
95 }
96
97 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
98 {
99         struct bnx2x *bp = fp->bp;
100         struct netdev_queue *txq;
101         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
102
103 #ifdef BNX2X_STOP_ON_ERROR
104         if (unlikely(bp->panic))
105                 return -1;
106 #endif
107
108         txq = netdev_get_tx_queue(bp->dev, fp->index);
109         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
110         sw_cons = fp->tx_pkt_cons;
111
112         while (sw_cons != hw_cons) {
113                 u16 pkt_cons;
114
115                 pkt_cons = TX_BD(sw_cons);
116
117                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
118                                       " pkt_cons %u\n",
119                    fp->index, hw_cons, sw_cons, pkt_cons);
120
121                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
122                 sw_cons++;
123         }
124
125         fp->tx_pkt_cons = sw_cons;
126         fp->tx_bd_cons = bd_cons;
127
128         /* Need to make the tx_bd_cons update visible to start_xmit()
129          * before checking for netif_tx_queue_stopped().  Without the
130          * memory barrier, there is a small possibility that
131          * start_xmit() will miss it and cause the queue to be stopped
132          * forever.
133          */
134         smp_mb();
135
136         if (unlikely(netif_tx_queue_stopped(txq))) {
137                 /* Taking tx_lock() is needed to prevent reenabling the queue
138                  * while it's empty. This could have happen if rx_action() gets
139                  * suspended in bnx2x_tx_int() after the condition before
140                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
141                  *
142                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
143                  * sends some packets consuming the whole queue again->
144                  * stops the queue
145                  */
146
147                 __netif_tx_lock(txq, smp_processor_id());
148
149                 if ((netif_tx_queue_stopped(txq)) &&
150                     (bp->state == BNX2X_STATE_OPEN) &&
151                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
152                         netif_tx_wake_queue(txq);
153
154                 __netif_tx_unlock(txq);
155         }
156         return 0;
157 }
158
159 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
160                                              u16 idx)
161 {
162         u16 last_max = fp->last_max_sge;
163
164         if (SUB_S16(idx, last_max) > 0)
165                 fp->last_max_sge = idx;
166 }
167
168 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
169                                   struct eth_fast_path_rx_cqe *fp_cqe)
170 {
171         struct bnx2x *bp = fp->bp;
172         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
173                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
174                       SGE_PAGE_SHIFT;
175         u16 last_max, last_elem, first_elem;
176         u16 delta = 0;
177         u16 i;
178
179         if (!sge_len)
180                 return;
181
182         /* First mark all used pages */
183         for (i = 0; i < sge_len; i++)
184                 SGE_MASK_CLEAR_BIT(fp,
185                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
186
187         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
188            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
189
190         /* Here we assume that the last SGE index is the biggest */
191         prefetch((void *)(fp->sge_mask));
192         bnx2x_update_last_max_sge(fp,
193                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
194
195         last_max = RX_SGE(fp->last_max_sge);
196         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
197         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
198
199         /* If ring is not full */
200         if (last_elem + 1 != first_elem)
201                 last_elem++;
202
203         /* Now update the prod */
204         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
205                 if (likely(fp->sge_mask[i]))
206                         break;
207
208                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
209                 delta += RX_SGE_MASK_ELEM_SZ;
210         }
211
212         if (delta > 0) {
213                 fp->rx_sge_prod += delta;
214                 /* clear page-end entries */
215                 bnx2x_clear_sge_mask_next_elems(fp);
216         }
217
218         DP(NETIF_MSG_RX_STATUS,
219            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
220            fp->last_max_sge, fp->rx_sge_prod);
221 }
222
223 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
224                             struct sk_buff *skb, u16 cons, u16 prod)
225 {
226         struct bnx2x *bp = fp->bp;
227         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
228         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
229         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
230         dma_addr_t mapping;
231
232         /* move empty skb from pool to prod and map it */
233         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
234         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
235                                  fp->rx_buf_size, DMA_FROM_DEVICE);
236         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
237
238         /* move partial skb from cons to pool (don't unmap yet) */
239         fp->tpa_pool[queue] = *cons_rx_buf;
240
241         /* mark bin state as start - print error if current state != stop */
242         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
243                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
244
245         fp->tpa_state[queue] = BNX2X_TPA_START;
246
247         /* point prod_bd to new skb */
248         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
249         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
250
251 #ifdef BNX2X_STOP_ON_ERROR
252         fp->tpa_queue_used |= (1 << queue);
253 #ifdef _ASM_GENERIC_INT_L64_H
254         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
255 #else
256         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
257 #endif
258            fp->tpa_queue_used);
259 #endif
260 }
261
262 /* Timestamp option length allowed for TPA aggregation:
263  *
264  *              nop nop kind length echo val
265  */
266 #define TPA_TSTAMP_OPT_LEN      12
267 /**
268  * Calculate the approximate value of the MSS for this
269  * aggregation using the first packet of it.
270  *
271  * @param bp
272  * @param parsing_flags Parsing flags from the START CQE
273  * @param len_on_bd Total length of the first packet for the
274  *                   aggregation.
275  */
276 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
277                                     u16 len_on_bd)
278 {
279         /* TPA arrgregation won't have an IP options and TCP options
280          * other than timestamp.
281          */
282         u16 hdrs_len = ETH_HLEN + sizeof(struct iphdr) + sizeof(struct tcphdr);
283
284
285         /* Check if there was a TCP timestamp, if there is it's will
286          * always be 12 bytes length: nop nop kind length echo val.
287          *
288          * Otherwise FW would close the aggregation.
289          */
290         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
291                 hdrs_len += TPA_TSTAMP_OPT_LEN;
292
293         return len_on_bd - hdrs_len;
294 }
295
296 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
297                                struct sk_buff *skb,
298                                struct eth_fast_path_rx_cqe *fp_cqe,
299                                u16 cqe_idx, u16 parsing_flags)
300 {
301         struct sw_rx_page *rx_pg, old_rx_pg;
302         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
303         u32 i, frag_len, frag_size, pages;
304         int err;
305         int j;
306
307         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
308         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
309
310         /* This is needed in order to enable forwarding support */
311         if (frag_size)
312                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp, parsing_flags,
313                                                               len_on_bd);
314
315 #ifdef BNX2X_STOP_ON_ERROR
316         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
317                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
318                           pages, cqe_idx);
319                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
320                           fp_cqe->pkt_len, len_on_bd);
321                 bnx2x_panic();
322                 return -EINVAL;
323         }
324 #endif
325
326         /* Run through the SGL and compose the fragmented skb */
327         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
328                 u16 sge_idx =
329                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[j]));
330
331                 /* FW gives the indices of the SGE as if the ring is an array
332                    (meaning that "next" element will consume 2 indices) */
333                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
334                 rx_pg = &fp->rx_page_ring[sge_idx];
335                 old_rx_pg = *rx_pg;
336
337                 /* If we fail to allocate a substitute page, we simply stop
338                    where we are and drop the whole packet */
339                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
340                 if (unlikely(err)) {
341                         fp->eth_q_stats.rx_skb_alloc_failed++;
342                         return err;
343                 }
344
345                 /* Unmap the page as we r going to pass it to the stack */
346                 dma_unmap_page(&bp->pdev->dev,
347                                dma_unmap_addr(&old_rx_pg, mapping),
348                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
349
350                 /* Add one frag and update the appropriate fields in the skb */
351                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
352
353                 skb->data_len += frag_len;
354                 skb->truesize += frag_len;
355                 skb->len += frag_len;
356
357                 frag_size -= frag_len;
358         }
359
360         return 0;
361 }
362
363 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
364                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
365                            u16 cqe_idx)
366 {
367         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
368         struct sk_buff *skb = rx_buf->skb;
369         /* alloc new skb */
370         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
371
372         /* Unmap skb in the pool anyway, as we are going to change
373            pool entry status to BNX2X_TPA_STOP even if new skb allocation
374            fails. */
375         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
376                          fp->rx_buf_size, DMA_FROM_DEVICE);
377
378         if (likely(new_skb)) {
379                 /* fix ip xsum and give it to the stack */
380                 /* (no need to map the new skb) */
381                 u16 parsing_flags =
382                         le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags);
383
384                 prefetch(skb);
385                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
386
387 #ifdef BNX2X_STOP_ON_ERROR
388                 if (pad + len > fp->rx_buf_size) {
389                         BNX2X_ERR("skb_put is about to fail...  "
390                                   "pad %d  len %d  rx_buf_size %d\n",
391                                   pad, len, fp->rx_buf_size);
392                         bnx2x_panic();
393                         return;
394                 }
395 #endif
396
397                 skb_reserve(skb, pad);
398                 skb_put(skb, len);
399
400                 skb->protocol = eth_type_trans(skb, bp->dev);
401                 skb->ip_summed = CHECKSUM_UNNECESSARY;
402
403                 {
404                         struct iphdr *iph;
405
406                         iph = (struct iphdr *)skb->data;
407                         iph->check = 0;
408                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
409                 }
410
411                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
412                                          &cqe->fast_path_cqe, cqe_idx,
413                                          parsing_flags)) {
414                         if (parsing_flags & PARSING_FLAGS_VLAN)
415                                 __vlan_hwaccel_put_tag(skb,
416                                                  le16_to_cpu(cqe->fast_path_cqe.
417                                                              vlan_tag));
418                         napi_gro_receive(&fp->napi, skb);
419                 } else {
420                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
421                            " - dropping packet!\n");
422                         dev_kfree_skb(skb);
423                 }
424
425
426                 /* put new skb in bin */
427                 fp->tpa_pool[queue].skb = new_skb;
428
429         } else {
430                 /* else drop the packet and keep the buffer in the bin */
431                 DP(NETIF_MSG_RX_STATUS,
432                    "Failed to allocate new skb - dropping packet!\n");
433                 fp->eth_q_stats.rx_skb_alloc_failed++;
434         }
435
436         fp->tpa_state[queue] = BNX2X_TPA_STOP;
437 }
438
439 /* Set Toeplitz hash value in the skb using the value from the
440  * CQE (calculated by HW).
441  */
442 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
443                                         struct sk_buff *skb)
444 {
445         /* Set Toeplitz hash from CQE */
446         if ((bp->dev->features & NETIF_F_RXHASH) &&
447             (cqe->fast_path_cqe.status_flags &
448              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
449                 skb->rxhash =
450                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
451 }
452
453 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
454 {
455         struct bnx2x *bp = fp->bp;
456         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
457         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
458         int rx_pkt = 0;
459
460 #ifdef BNX2X_STOP_ON_ERROR
461         if (unlikely(bp->panic))
462                 return 0;
463 #endif
464
465         /* CQ "next element" is of the size of the regular element,
466            that's why it's ok here */
467         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
468         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
469                 hw_comp_cons++;
470
471         bd_cons = fp->rx_bd_cons;
472         bd_prod = fp->rx_bd_prod;
473         bd_prod_fw = bd_prod;
474         sw_comp_cons = fp->rx_comp_cons;
475         sw_comp_prod = fp->rx_comp_prod;
476
477         /* Memory barrier necessary as speculative reads of the rx
478          * buffer can be ahead of the index in the status block
479          */
480         rmb();
481
482         DP(NETIF_MSG_RX_STATUS,
483            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
484            fp->index, hw_comp_cons, sw_comp_cons);
485
486         while (sw_comp_cons != hw_comp_cons) {
487                 struct sw_rx_bd *rx_buf = NULL;
488                 struct sk_buff *skb;
489                 union eth_rx_cqe *cqe;
490                 u8 cqe_fp_flags;
491                 u16 len, pad;
492
493                 comp_ring_cons = RCQ_BD(sw_comp_cons);
494                 bd_prod = RX_BD(bd_prod);
495                 bd_cons = RX_BD(bd_cons);
496
497                 /* Prefetch the page containing the BD descriptor
498                    at producer's index. It will be needed when new skb is
499                    allocated */
500                 prefetch((void *)(PAGE_ALIGN((unsigned long)
501                                              (&fp->rx_desc_ring[bd_prod])) -
502                                   PAGE_SIZE + 1));
503
504                 cqe = &fp->rx_comp_ring[comp_ring_cons];
505                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
506
507                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
508                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
509                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
510                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
511                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
512                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
513
514                 /* is this a slowpath msg? */
515                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
516                         bnx2x_sp_event(fp, cqe);
517                         goto next_cqe;
518
519                 /* this is an rx packet */
520                 } else {
521                         rx_buf = &fp->rx_buf_ring[bd_cons];
522                         skb = rx_buf->skb;
523                         prefetch(skb);
524                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
525                         pad = cqe->fast_path_cqe.placement_offset;
526
527                         /* - If CQE is marked both TPA_START and TPA_END it is
528                          *   a non-TPA CQE.
529                          * - FP CQE will always have either TPA_START or/and
530                          *   TPA_STOP flags set.
531                          */
532                         if ((!fp->disable_tpa) &&
533                             (TPA_TYPE(cqe_fp_flags) !=
534                                         (TPA_TYPE_START | TPA_TYPE_END))) {
535                                 u16 queue = cqe->fast_path_cqe.queue_index;
536
537                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
538                                         DP(NETIF_MSG_RX_STATUS,
539                                            "calling tpa_start on queue %d\n",
540                                            queue);
541
542                                         bnx2x_tpa_start(fp, queue, skb,
543                                                         bd_cons, bd_prod);
544
545                                         /* Set Toeplitz hash for an LRO skb */
546                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
547
548                                         goto next_rx;
549                                 } else { /* TPA_STOP */
550                                         DP(NETIF_MSG_RX_STATUS,
551                                            "calling tpa_stop on queue %d\n",
552                                            queue);
553
554                                         if (!BNX2X_RX_SUM_FIX(cqe))
555                                                 BNX2X_ERR("STOP on none TCP "
556                                                           "data\n");
557
558                                         /* This is a size of the linear data
559                                            on this skb */
560                                         len = le16_to_cpu(cqe->fast_path_cqe.
561                                                                 len_on_bd);
562                                         bnx2x_tpa_stop(bp, fp, queue, pad,
563                                                     len, cqe, comp_ring_cons);
564 #ifdef BNX2X_STOP_ON_ERROR
565                                         if (bp->panic)
566                                                 return 0;
567 #endif
568
569                                         bnx2x_update_sge_prod(fp,
570                                                         &cqe->fast_path_cqe);
571                                         goto next_cqe;
572                                 }
573                         }
574
575                         dma_sync_single_for_device(&bp->pdev->dev,
576                                         dma_unmap_addr(rx_buf, mapping),
577                                                    pad + RX_COPY_THRESH,
578                                                    DMA_FROM_DEVICE);
579                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
580
581                         /* is this an error packet? */
582                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
583                                 DP(NETIF_MSG_RX_ERR,
584                                    "ERROR  flags %x  rx packet %u\n",
585                                    cqe_fp_flags, sw_comp_cons);
586                                 fp->eth_q_stats.rx_err_discard_pkt++;
587                                 goto reuse_rx;
588                         }
589
590                         /* Since we don't have a jumbo ring
591                          * copy small packets if mtu > 1500
592                          */
593                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
594                             (len <= RX_COPY_THRESH)) {
595                                 struct sk_buff *new_skb;
596
597                                 new_skb = netdev_alloc_skb(bp->dev,
598                                                            len + pad);
599                                 if (new_skb == NULL) {
600                                         DP(NETIF_MSG_RX_ERR,
601                                            "ERROR  packet dropped "
602                                            "because of alloc failure\n");
603                                         fp->eth_q_stats.rx_skb_alloc_failed++;
604                                         goto reuse_rx;
605                                 }
606
607                                 /* aligned copy */
608                                 skb_copy_from_linear_data_offset(skb, pad,
609                                                     new_skb->data + pad, len);
610                                 skb_reserve(new_skb, pad);
611                                 skb_put(new_skb, len);
612
613                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
614
615                                 skb = new_skb;
616
617                         } else
618                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
619                                 dma_unmap_single(&bp->pdev->dev,
620                                         dma_unmap_addr(rx_buf, mapping),
621                                                  fp->rx_buf_size,
622                                                  DMA_FROM_DEVICE);
623                                 skb_reserve(skb, pad);
624                                 skb_put(skb, len);
625
626                         } else {
627                                 DP(NETIF_MSG_RX_ERR,
628                                    "ERROR  packet dropped because "
629                                    "of alloc failure\n");
630                                 fp->eth_q_stats.rx_skb_alloc_failed++;
631 reuse_rx:
632                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
633                                 goto next_rx;
634                         }
635
636                         skb->protocol = eth_type_trans(skb, bp->dev);
637
638                         /* Set Toeplitz hash for a none-LRO skb */
639                         bnx2x_set_skb_rxhash(bp, cqe, skb);
640
641                         skb_checksum_none_assert(skb);
642
643                         if (bp->rx_csum) {
644                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
645                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
646                                 else
647                                         fp->eth_q_stats.hw_csum_err++;
648                         }
649                 }
650
651                 skb_record_rx_queue(skb, fp->index);
652
653                 if (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
654                      PARSING_FLAGS_VLAN)
655                         __vlan_hwaccel_put_tag(skb,
656                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag));
657                 napi_gro_receive(&fp->napi, skb);
658
659
660 next_rx:
661                 rx_buf->skb = NULL;
662
663                 bd_cons = NEXT_RX_IDX(bd_cons);
664                 bd_prod = NEXT_RX_IDX(bd_prod);
665                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
666                 rx_pkt++;
667 next_cqe:
668                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
669                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
670
671                 if (rx_pkt == budget)
672                         break;
673         } /* while */
674
675         fp->rx_bd_cons = bd_cons;
676         fp->rx_bd_prod = bd_prod_fw;
677         fp->rx_comp_cons = sw_comp_cons;
678         fp->rx_comp_prod = sw_comp_prod;
679
680         /* Update producers */
681         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
682                              fp->rx_sge_prod);
683
684         fp->rx_pkt += rx_pkt;
685         fp->rx_calls++;
686
687         return rx_pkt;
688 }
689
690 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
691 {
692         struct bnx2x_fastpath *fp = fp_cookie;
693         struct bnx2x *bp = fp->bp;
694
695         /* Return here if interrupt is disabled */
696         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
697                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
698                 return IRQ_HANDLED;
699         }
700
701         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
702                          "[fp %d fw_sd %d igusb %d]\n",
703            fp->index, fp->fw_sb_id, fp->igu_sb_id);
704         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
705
706 #ifdef BNX2X_STOP_ON_ERROR
707         if (unlikely(bp->panic))
708                 return IRQ_HANDLED;
709 #endif
710
711         /* Handle Rx and Tx according to MSI-X vector */
712         prefetch(fp->rx_cons_sb);
713         prefetch(fp->tx_cons_sb);
714         prefetch(&fp->sb_running_index[SM_RX_ID]);
715         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
716
717         return IRQ_HANDLED;
718 }
719
720 /* HW Lock for shared dual port PHYs */
721 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
722 {
723         mutex_lock(&bp->port.phy_mutex);
724
725         if (bp->port.need_hw_lock)
726                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
727 }
728
729 void bnx2x_release_phy_lock(struct bnx2x *bp)
730 {
731         if (bp->port.need_hw_lock)
732                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
733
734         mutex_unlock(&bp->port.phy_mutex);
735 }
736
737 /* calculates MF speed according to current linespeed and MF configuration */
738 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
739 {
740         u16 line_speed = bp->link_vars.line_speed;
741         if (IS_MF(bp)) {
742                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
743                                                    bp->mf_config[BP_VN(bp)]);
744
745                 /* Calculate the current MAX line speed limit for the MF
746                  * devices
747                  */
748                 if (IS_MF_SI(bp))
749                         line_speed = (line_speed * maxCfg) / 100;
750                 else { /* SD mode */
751                         u16 vn_max_rate = maxCfg * 100;
752
753                         if (vn_max_rate < line_speed)
754                                 line_speed = vn_max_rate;
755                 }
756         }
757
758         return line_speed;
759 }
760
761 void bnx2x_link_report(struct bnx2x *bp)
762 {
763         if (bp->flags & MF_FUNC_DIS) {
764                 netif_carrier_off(bp->dev);
765                 netdev_err(bp->dev, "NIC Link is Down\n");
766                 return;
767         }
768
769         if (bp->link_vars.link_up) {
770                 u16 line_speed;
771
772                 if (bp->state == BNX2X_STATE_OPEN)
773                         netif_carrier_on(bp->dev);
774                 netdev_info(bp->dev, "NIC Link is Up, ");
775
776                 line_speed = bnx2x_get_mf_speed(bp);
777
778                 pr_cont("%d Mbps ", line_speed);
779
780                 if (bp->link_vars.duplex == DUPLEX_FULL)
781                         pr_cont("full duplex");
782                 else
783                         pr_cont("half duplex");
784
785                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
786                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
787                                 pr_cont(", receive ");
788                                 if (bp->link_vars.flow_ctrl &
789                                     BNX2X_FLOW_CTRL_TX)
790                                         pr_cont("& transmit ");
791                         } else {
792                                 pr_cont(", transmit ");
793                         }
794                         pr_cont("flow control ON");
795                 }
796                 pr_cont("\n");
797
798         } else { /* link_down */
799                 netif_carrier_off(bp->dev);
800                 netdev_err(bp->dev, "NIC Link is Down\n");
801         }
802 }
803
804 /* Returns the number of actually allocated BDs */
805 static inline int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
806                                       int rx_ring_size)
807 {
808         struct bnx2x *bp = fp->bp;
809         u16 ring_prod, cqe_ring_prod;
810         int i;
811
812         fp->rx_comp_cons = 0;
813         cqe_ring_prod = ring_prod = 0;
814         for (i = 0; i < rx_ring_size; i++) {
815                 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
816                         BNX2X_ERR("was only able to allocate "
817                                   "%d rx skbs on queue[%d]\n", i, fp->index);
818                         fp->eth_q_stats.rx_skb_alloc_failed++;
819                         break;
820                 }
821                 ring_prod = NEXT_RX_IDX(ring_prod);
822                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
823                 WARN_ON(ring_prod <= i);
824         }
825
826         fp->rx_bd_prod = ring_prod;
827         /* Limit the CQE producer by the CQE ring size */
828         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
829                                cqe_ring_prod);
830         fp->rx_pkt = fp->rx_calls = 0;
831
832         return i;
833 }
834
835 static inline void bnx2x_alloc_rx_bd_ring(struct bnx2x_fastpath *fp)
836 {
837         struct bnx2x *bp = fp->bp;
838         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
839                                               MAX_RX_AVAIL/bp->num_queues;
840
841         rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
842
843         bnx2x_alloc_rx_bds(fp, rx_ring_size);
844
845         /* Warning!
846          * this will generate an interrupt (to the TSTORM)
847          * must only be done after chip is initialized
848          */
849         bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
850                              fp->rx_sge_prod);
851 }
852
853 void bnx2x_init_rx_rings(struct bnx2x *bp)
854 {
855         int func = BP_FUNC(bp);
856         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
857                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
858         u16 ring_prod;
859         int i, j;
860
861         for_each_rx_queue(bp, j) {
862                 struct bnx2x_fastpath *fp = &bp->fp[j];
863
864                 DP(NETIF_MSG_IFUP,
865                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
866
867                 if (!fp->disable_tpa) {
868                         for (i = 0; i < max_agg_queues; i++) {
869                                 fp->tpa_pool[i].skb =
870                                    netdev_alloc_skb(bp->dev, fp->rx_buf_size);
871                                 if (!fp->tpa_pool[i].skb) {
872                                         BNX2X_ERR("Failed to allocate TPA "
873                                                   "skb pool for queue[%d] - "
874                                                   "disabling TPA on this "
875                                                   "queue!\n", j);
876                                         bnx2x_free_tpa_pool(bp, fp, i);
877                                         fp->disable_tpa = 1;
878                                         break;
879                                 }
880                                 dma_unmap_addr_set((struct sw_rx_bd *)
881                                                         &bp->fp->tpa_pool[i],
882                                                    mapping, 0);
883                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
884                         }
885
886                         /* "next page" elements initialization */
887                         bnx2x_set_next_page_sgl(fp);
888
889                         /* set SGEs bit mask */
890                         bnx2x_init_sge_ring_bit_mask(fp);
891
892                         /* Allocate SGEs and initialize the ring elements */
893                         for (i = 0, ring_prod = 0;
894                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
895
896                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
897                                         BNX2X_ERR("was only able to allocate "
898                                                   "%d rx sges\n", i);
899                                         BNX2X_ERR("disabling TPA for"
900                                                   " queue[%d]\n", j);
901                                         /* Cleanup already allocated elements */
902                                         bnx2x_free_rx_sge_range(bp,
903                                                                 fp, ring_prod);
904                                         bnx2x_free_tpa_pool(bp,
905                                                             fp, max_agg_queues);
906                                         fp->disable_tpa = 1;
907                                         ring_prod = 0;
908                                         break;
909                                 }
910                                 ring_prod = NEXT_SGE_IDX(ring_prod);
911                         }
912
913                         fp->rx_sge_prod = ring_prod;
914                 }
915         }
916
917         for_each_rx_queue(bp, j) {
918                 struct bnx2x_fastpath *fp = &bp->fp[j];
919
920                 fp->rx_bd_cons = 0;
921
922                 bnx2x_set_next_page_rx_bd(fp);
923
924                 /* CQ ring */
925                 bnx2x_set_next_page_rx_cq(fp);
926
927                 /* Allocate BDs and initialize BD ring */
928                 bnx2x_alloc_rx_bd_ring(fp);
929
930                 if (j != 0)
931                         continue;
932
933                 if (!CHIP_IS_E2(bp)) {
934                         REG_WR(bp, BAR_USTRORM_INTMEM +
935                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
936                                U64_LO(fp->rx_comp_mapping));
937                         REG_WR(bp, BAR_USTRORM_INTMEM +
938                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
939                                U64_HI(fp->rx_comp_mapping));
940                 }
941         }
942 }
943
944 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
945 {
946         int i;
947
948         for_each_tx_queue(bp, i) {
949                 struct bnx2x_fastpath *fp = &bp->fp[i];
950
951                 u16 bd_cons = fp->tx_bd_cons;
952                 u16 sw_prod = fp->tx_pkt_prod;
953                 u16 sw_cons = fp->tx_pkt_cons;
954
955                 while (sw_cons != sw_prod) {
956                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
957                         sw_cons++;
958                 }
959         }
960 }
961
962 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
963 {
964         int i, j;
965
966         for_each_rx_queue(bp, j) {
967                 struct bnx2x_fastpath *fp = &bp->fp[j];
968
969                 for (i = 0; i < NUM_RX_BD; i++) {
970                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
971                         struct sk_buff *skb = rx_buf->skb;
972
973                         if (skb == NULL)
974                                 continue;
975
976                         dma_unmap_single(&bp->pdev->dev,
977                                          dma_unmap_addr(rx_buf, mapping),
978                                          fp->rx_buf_size, DMA_FROM_DEVICE);
979
980                         rx_buf->skb = NULL;
981                         dev_kfree_skb(skb);
982                 }
983                 if (!fp->disable_tpa)
984                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
985                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
986                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
987         }
988 }
989
990 void bnx2x_free_skbs(struct bnx2x *bp)
991 {
992         bnx2x_free_tx_skbs(bp);
993         bnx2x_free_rx_skbs(bp);
994 }
995
996 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
997 {
998         int i, offset = 1;
999
1000         free_irq(bp->msix_table[0].vector, bp->dev);
1001         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1002            bp->msix_table[0].vector);
1003
1004 #ifdef BCM_CNIC
1005         offset++;
1006 #endif
1007         for_each_eth_queue(bp, i) {
1008                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
1009                    "state %x\n", i, bp->msix_table[i + offset].vector,
1010                    bnx2x_fp(bp, i, state));
1011
1012                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
1013         }
1014 }
1015
1016 void bnx2x_free_irq(struct bnx2x *bp)
1017 {
1018         if (bp->flags & USING_MSIX_FLAG)
1019                 bnx2x_free_msix_irqs(bp);
1020         else if (bp->flags & USING_MSI_FLAG)
1021                 free_irq(bp->pdev->irq, bp->dev);
1022         else
1023                 free_irq(bp->pdev->irq, bp->dev);
1024 }
1025
1026 int bnx2x_enable_msix(struct bnx2x *bp)
1027 {
1028         int msix_vec = 0, i, rc, req_cnt;
1029
1030         bp->msix_table[msix_vec].entry = msix_vec;
1031         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1032            bp->msix_table[0].entry);
1033         msix_vec++;
1034
1035 #ifdef BCM_CNIC
1036         bp->msix_table[msix_vec].entry = msix_vec;
1037         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1038            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1039         msix_vec++;
1040 #endif
1041         for_each_eth_queue(bp, i) {
1042                 bp->msix_table[msix_vec].entry = msix_vec;
1043                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1044                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
1045                 msix_vec++;
1046         }
1047
1048         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_CONTEXT_USE + 1;
1049
1050         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1051
1052         /*
1053          * reconfigure number of tx/rx queues according to available
1054          * MSI-X vectors
1055          */
1056         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1057                 /* how less vectors we will have? */
1058                 int diff = req_cnt - rc;
1059
1060                 DP(NETIF_MSG_IFUP,
1061                    "Trying to use less MSI-X vectors: %d\n", rc);
1062
1063                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1064
1065                 if (rc) {
1066                         DP(NETIF_MSG_IFUP,
1067                            "MSI-X is not attainable  rc %d\n", rc);
1068                         return rc;
1069                 }
1070                 /*
1071                  * decrease number of queues by number of unallocated entries
1072                  */
1073                 bp->num_queues -= diff;
1074
1075                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1076                                   bp->num_queues);
1077         } else if (rc) {
1078                 /* fall to INTx if not enough memory */
1079                 if (rc == -ENOMEM)
1080                         bp->flags |= DISABLE_MSI_FLAG;
1081                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1082                 return rc;
1083         }
1084
1085         bp->flags |= USING_MSIX_FLAG;
1086
1087         return 0;
1088 }
1089
1090 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1091 {
1092         int i, rc, offset = 1;
1093
1094         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1095                          bp->dev->name, bp->dev);
1096         if (rc) {
1097                 BNX2X_ERR("request sp irq failed\n");
1098                 return -EBUSY;
1099         }
1100
1101 #ifdef BCM_CNIC
1102         offset++;
1103 #endif
1104         for_each_eth_queue(bp, i) {
1105                 struct bnx2x_fastpath *fp = &bp->fp[i];
1106                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1107                          bp->dev->name, i);
1108
1109                 rc = request_irq(bp->msix_table[offset].vector,
1110                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1111                 if (rc) {
1112                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1113                         bnx2x_free_msix_irqs(bp);
1114                         return -EBUSY;
1115                 }
1116
1117                 offset++;
1118                 fp->state = BNX2X_FP_STATE_IRQ;
1119         }
1120
1121         i = BNX2X_NUM_ETH_QUEUES(bp);
1122         offset = 1 + CNIC_CONTEXT_USE;
1123         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1124                " ... fp[%d] %d\n",
1125                bp->msix_table[0].vector,
1126                0, bp->msix_table[offset].vector,
1127                i - 1, bp->msix_table[offset + i - 1].vector);
1128
1129         return 0;
1130 }
1131
1132 int bnx2x_enable_msi(struct bnx2x *bp)
1133 {
1134         int rc;
1135
1136         rc = pci_enable_msi(bp->pdev);
1137         if (rc) {
1138                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1139                 return -1;
1140         }
1141         bp->flags |= USING_MSI_FLAG;
1142
1143         return 0;
1144 }
1145
1146 static int bnx2x_req_irq(struct bnx2x *bp)
1147 {
1148         unsigned long flags;
1149         int rc;
1150
1151         if (bp->flags & USING_MSI_FLAG)
1152                 flags = 0;
1153         else
1154                 flags = IRQF_SHARED;
1155
1156         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1157                          bp->dev->name, bp->dev);
1158         if (!rc)
1159                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1160
1161         return rc;
1162 }
1163
1164 static void bnx2x_napi_enable(struct bnx2x *bp)
1165 {
1166         int i;
1167
1168         for_each_napi_queue(bp, i)
1169                 napi_enable(&bnx2x_fp(bp, i, napi));
1170 }
1171
1172 static void bnx2x_napi_disable(struct bnx2x *bp)
1173 {
1174         int i;
1175
1176         for_each_napi_queue(bp, i)
1177                 napi_disable(&bnx2x_fp(bp, i, napi));
1178 }
1179
1180 void bnx2x_netif_start(struct bnx2x *bp)
1181 {
1182         int intr_sem;
1183
1184         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1185         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1186
1187         if (intr_sem) {
1188                 if (netif_running(bp->dev)) {
1189                         bnx2x_napi_enable(bp);
1190                         bnx2x_int_enable(bp);
1191                         if (bp->state == BNX2X_STATE_OPEN)
1192                                 netif_tx_wake_all_queues(bp->dev);
1193                 }
1194         }
1195 }
1196
1197 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1198 {
1199         bnx2x_int_disable_sync(bp, disable_hw);
1200         bnx2x_napi_disable(bp);
1201         netif_tx_disable(bp->dev);
1202 }
1203
1204 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1205 {
1206 #ifdef BCM_CNIC
1207         struct bnx2x *bp = netdev_priv(dev);
1208         if (NO_FCOE(bp))
1209                 return skb_tx_hash(dev, skb);
1210         else {
1211                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1212                 u16 ether_type = ntohs(hdr->h_proto);
1213
1214                 /* Skip VLAN tag if present */
1215                 if (ether_type == ETH_P_8021Q) {
1216                         struct vlan_ethhdr *vhdr =
1217                                 (struct vlan_ethhdr *)skb->data;
1218
1219                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1220                 }
1221
1222                 /* If ethertype is FCoE or FIP - use FCoE ring */
1223                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1224                         return bnx2x_fcoe(bp, index);
1225         }
1226 #endif
1227         /* Select a none-FCoE queue:  if FCoE is enabled, exclude FCoE L2 ring
1228          */
1229         return __skb_tx_hash(dev, skb,
1230                         dev->real_num_tx_queues - FCOE_CONTEXT_USE);
1231 }
1232
1233 void bnx2x_set_num_queues(struct bnx2x *bp)
1234 {
1235         switch (bp->multi_mode) {
1236         case ETH_RSS_MODE_DISABLED:
1237                 bp->num_queues = 1;
1238                 break;
1239         case ETH_RSS_MODE_REGULAR:
1240                 bp->num_queues = bnx2x_calc_num_queues(bp);
1241                 break;
1242
1243         default:
1244                 bp->num_queues = 1;
1245                 break;
1246         }
1247
1248         /* Add special queues */
1249         bp->num_queues += NONE_ETH_CONTEXT_USE;
1250 }
1251
1252 #ifdef BCM_CNIC
1253 static inline void bnx2x_set_fcoe_eth_macs(struct bnx2x *bp)
1254 {
1255         if (!NO_FCOE(bp)) {
1256                 if (!IS_MF_SD(bp))
1257                         bnx2x_set_fip_eth_mac_addr(bp, 1);
1258                 bnx2x_set_all_enode_macs(bp, 1);
1259                 bp->flags |= FCOE_MACS_SET;
1260         }
1261 }
1262 #endif
1263
1264 static void bnx2x_release_firmware(struct bnx2x *bp)
1265 {
1266         kfree(bp->init_ops_offsets);
1267         kfree(bp->init_ops);
1268         kfree(bp->init_data);
1269         release_firmware(bp->firmware);
1270 }
1271
1272 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1273 {
1274         int rc, num = bp->num_queues;
1275
1276 #ifdef BCM_CNIC
1277         if (NO_FCOE(bp))
1278                 num -= FCOE_CONTEXT_USE;
1279
1280 #endif
1281         netif_set_real_num_tx_queues(bp->dev, num);
1282         rc = netif_set_real_num_rx_queues(bp->dev, num);
1283         return rc;
1284 }
1285
1286 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1287 {
1288         int i;
1289
1290         for_each_queue(bp, i) {
1291                 struct bnx2x_fastpath *fp = &bp->fp[i];
1292
1293                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1294                 if (IS_FCOE_IDX(i))
1295                         /*
1296                          * Although there are no IP frames expected to arrive to
1297                          * this ring we still want to add an
1298                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1299                          * overrun attack.
1300                          */
1301                         fp->rx_buf_size =
1302                                 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
1303                                 BNX2X_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1304                 else
1305                         fp->rx_buf_size =
1306                                 bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN +
1307                                 IP_HEADER_ALIGNMENT_PADDING;
1308         }
1309 }
1310
1311 /* must be called with rtnl_lock */
1312 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1313 {
1314         u32 load_code;
1315         int i, rc;
1316
1317         /* Set init arrays */
1318         rc = bnx2x_init_firmware(bp);
1319         if (rc) {
1320                 BNX2X_ERR("Error loading firmware\n");
1321                 return rc;
1322         }
1323
1324 #ifdef BNX2X_STOP_ON_ERROR
1325         if (unlikely(bp->panic))
1326                 return -EPERM;
1327 #endif
1328
1329         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1330
1331         /* must be called before memory allocation and HW init */
1332         bnx2x_ilt_set_info(bp);
1333
1334         /* Set the receive queues buffer size */
1335         bnx2x_set_rx_buf_size(bp);
1336
1337         if (bnx2x_alloc_mem(bp))
1338                 return -ENOMEM;
1339
1340         rc = bnx2x_set_real_num_queues(bp);
1341         if (rc) {
1342                 BNX2X_ERR("Unable to set real_num_queues\n");
1343                 goto load_error0;
1344         }
1345
1346         for_each_queue(bp, i)
1347                 bnx2x_fp(bp, i, disable_tpa) =
1348                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1349
1350 #ifdef BCM_CNIC
1351         /* We don't want TPA on FCoE L2 ring */
1352         bnx2x_fcoe(bp, disable_tpa) = 1;
1353 #endif
1354         bnx2x_napi_enable(bp);
1355
1356         /* Send LOAD_REQUEST command to MCP
1357            Returns the type of LOAD command:
1358            if it is the first port to be initialized
1359            common blocks should be initialized, otherwise - not
1360         */
1361         if (!BP_NOMCP(bp)) {
1362                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1363                 if (!load_code) {
1364                         BNX2X_ERR("MCP response failure, aborting\n");
1365                         rc = -EBUSY;
1366                         goto load_error1;
1367                 }
1368                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1369                         rc = -EBUSY; /* other port in diagnostic mode */
1370                         goto load_error1;
1371                 }
1372
1373         } else {
1374                 int path = BP_PATH(bp);
1375                 int port = BP_PORT(bp);
1376
1377                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1378                    path, load_count[path][0], load_count[path][1],
1379                    load_count[path][2]);
1380                 load_count[path][0]++;
1381                 load_count[path][1 + port]++;
1382                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1383                    path, load_count[path][0], load_count[path][1],
1384                    load_count[path][2]);
1385                 if (load_count[path][0] == 1)
1386                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1387                 else if (load_count[path][1 + port] == 1)
1388                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1389                 else
1390                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1391         }
1392
1393         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1394             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1395             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1396                 bp->port.pmf = 1;
1397         else
1398                 bp->port.pmf = 0;
1399         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1400
1401         /* Initialize HW */
1402         rc = bnx2x_init_hw(bp, load_code);
1403         if (rc) {
1404                 BNX2X_ERR("HW init failed, aborting\n");
1405                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1406                 goto load_error2;
1407         }
1408
1409         /* Connect to IRQs */
1410         rc = bnx2x_setup_irqs(bp);
1411         if (rc) {
1412                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1413                 goto load_error2;
1414         }
1415
1416         /* Setup NIC internals and enable interrupts */
1417         bnx2x_nic_init(bp, load_code);
1418
1419         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1420             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1421             (bp->common.shmem2_base))
1422                 SHMEM2_WR(bp, dcc_support,
1423                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1424                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1425
1426         /* Send LOAD_DONE command to MCP */
1427         if (!BP_NOMCP(bp)) {
1428                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1429                 if (!load_code) {
1430                         BNX2X_ERR("MCP response failure, aborting\n");
1431                         rc = -EBUSY;
1432                         goto load_error3;
1433                 }
1434         }
1435
1436         bnx2x_dcbx_init(bp);
1437
1438         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1439
1440         rc = bnx2x_func_start(bp);
1441         if (rc) {
1442                 BNX2X_ERR("Function start failed!\n");
1443 #ifndef BNX2X_STOP_ON_ERROR
1444                 goto load_error3;
1445 #else
1446                 bp->panic = 1;
1447                 return -EBUSY;
1448 #endif
1449         }
1450
1451         rc = bnx2x_setup_client(bp, &bp->fp[0], 1 /* Leading */);
1452         if (rc) {
1453                 BNX2X_ERR("Setup leading failed!\n");
1454 #ifndef BNX2X_STOP_ON_ERROR
1455                 goto load_error3;
1456 #else
1457                 bp->panic = 1;
1458                 return -EBUSY;
1459 #endif
1460         }
1461
1462         if (!CHIP_IS_E1(bp) &&
1463             (bp->mf_config[BP_VN(bp)] & FUNC_MF_CFG_FUNC_DISABLED)) {
1464                 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1465                 bp->flags |= MF_FUNC_DIS;
1466         }
1467
1468 #ifdef BCM_CNIC
1469         /* Enable Timer scan */
1470         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1471 #endif
1472
1473         for_each_nondefault_queue(bp, i) {
1474                 rc = bnx2x_setup_client(bp, &bp->fp[i], 0);
1475                 if (rc)
1476 #ifdef BCM_CNIC
1477                         goto load_error4;
1478 #else
1479                         goto load_error3;
1480 #endif
1481         }
1482
1483         /* Now when Clients are configured we are ready to work */
1484         bp->state = BNX2X_STATE_OPEN;
1485
1486 #ifdef BCM_CNIC
1487         bnx2x_set_fcoe_eth_macs(bp);
1488 #endif
1489
1490         bnx2x_set_eth_mac(bp, 1);
1491
1492         /* Clear MC configuration */
1493         if (CHIP_IS_E1(bp))
1494                 bnx2x_invalidate_e1_mc_list(bp);
1495         else
1496                 bnx2x_invalidate_e1h_mc_list(bp);
1497
1498         /* Clear UC lists configuration */
1499         bnx2x_invalidate_uc_list(bp);
1500
1501         if (bp->port.pmf)
1502                 bnx2x_initial_phy_init(bp, load_mode);
1503
1504         /* Initialize Rx filtering */
1505         bnx2x_set_rx_mode(bp->dev);
1506
1507         /* Start fast path */
1508         switch (load_mode) {
1509         case LOAD_NORMAL:
1510                 /* Tx queue should be only reenabled */
1511                 netif_tx_wake_all_queues(bp->dev);
1512                 /* Initialize the receive filter. */
1513                 break;
1514
1515         case LOAD_OPEN:
1516                 netif_tx_start_all_queues(bp->dev);
1517                 smp_mb__after_clear_bit();
1518                 break;
1519
1520         case LOAD_DIAG:
1521                 bp->state = BNX2X_STATE_DIAG;
1522                 break;
1523
1524         default:
1525                 break;
1526         }
1527
1528         if (!bp->port.pmf)
1529                 bnx2x__link_status_update(bp);
1530
1531         /* start the timer */
1532         mod_timer(&bp->timer, jiffies + bp->current_interval);
1533
1534 #ifdef BCM_CNIC
1535         bnx2x_setup_cnic_irq_info(bp);
1536         if (bp->state == BNX2X_STATE_OPEN)
1537                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1538 #endif
1539         bnx2x_inc_load_cnt(bp);
1540
1541         bnx2x_release_firmware(bp);
1542
1543         return 0;
1544
1545 #ifdef BCM_CNIC
1546 load_error4:
1547         /* Disable Timer scan */
1548         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1549 #endif
1550 load_error3:
1551         bnx2x_int_disable_sync(bp, 1);
1552
1553         /* Free SKBs, SGEs, TPA pool and driver internals */
1554         bnx2x_free_skbs(bp);
1555         for_each_rx_queue(bp, i)
1556                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1557
1558         /* Release IRQs */
1559         bnx2x_free_irq(bp);
1560 load_error2:
1561         if (!BP_NOMCP(bp)) {
1562                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1563                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1564         }
1565
1566         bp->port.pmf = 0;
1567 load_error1:
1568         bnx2x_napi_disable(bp);
1569 load_error0:
1570         bnx2x_free_mem(bp);
1571
1572         bnx2x_release_firmware(bp);
1573
1574         return rc;
1575 }
1576
1577 /* must be called with rtnl_lock */
1578 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1579 {
1580         int i;
1581
1582         if (bp->state == BNX2X_STATE_CLOSED) {
1583                 /* Interface has been removed - nothing to recover */
1584                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1585                 bp->is_leader = 0;
1586                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1587                 smp_wmb();
1588
1589                 return -EINVAL;
1590         }
1591
1592 #ifdef BCM_CNIC
1593         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1594 #endif
1595         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1596
1597         /* Set "drop all" */
1598         bp->rx_mode = BNX2X_RX_MODE_NONE;
1599         bnx2x_set_storm_rx_mode(bp);
1600
1601         /* Stop Tx */
1602         bnx2x_tx_disable(bp);
1603
1604         del_timer_sync(&bp->timer);
1605
1606         SHMEM_WR(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb,
1607                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1608
1609         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1610
1611         /* Cleanup the chip if needed */
1612         if (unload_mode != UNLOAD_RECOVERY)
1613                 bnx2x_chip_cleanup(bp, unload_mode);
1614         else {
1615                 /* Disable HW interrupts, NAPI and Tx */
1616                 bnx2x_netif_stop(bp, 1);
1617
1618                 /* Release IRQs */
1619                 bnx2x_free_irq(bp);
1620         }
1621
1622         bp->port.pmf = 0;
1623
1624         /* Free SKBs, SGEs, TPA pool and driver internals */
1625         bnx2x_free_skbs(bp);
1626         for_each_rx_queue(bp, i)
1627                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1628
1629         bnx2x_free_mem(bp);
1630
1631         bp->state = BNX2X_STATE_CLOSED;
1632
1633         /* The last driver must disable a "close the gate" if there is no
1634          * parity attention or "process kill" pending.
1635          */
1636         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1637             bnx2x_reset_is_done(bp))
1638                 bnx2x_disable_close_the_gate(bp);
1639
1640         /* Reset MCP mail box sequence if there is on going recovery */
1641         if (unload_mode == UNLOAD_RECOVERY)
1642                 bp->fw_seq = 0;
1643
1644         return 0;
1645 }
1646
1647 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1648 {
1649         u16 pmcsr;
1650
1651         /* If there is no power capability, silently succeed */
1652         if (!bp->pm_cap) {
1653                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
1654                 return 0;
1655         }
1656
1657         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1658
1659         switch (state) {
1660         case PCI_D0:
1661                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1662                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1663                                        PCI_PM_CTRL_PME_STATUS));
1664
1665                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1666                         /* delay required during transition out of D3hot */
1667                         msleep(20);
1668                 break;
1669
1670         case PCI_D3hot:
1671                 /* If there are other clients above don't
1672                    shut down the power */
1673                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1674                         return 0;
1675                 /* Don't shut down the power for emulation and FPGA */
1676                 if (CHIP_REV_IS_SLOW(bp))
1677                         return 0;
1678
1679                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1680                 pmcsr |= 3;
1681
1682                 if (bp->wol)
1683                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1684
1685                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1686                                       pmcsr);
1687
1688                 /* No more memory access after this point until
1689                 * device is brought back to D0.
1690                 */
1691                 break;
1692
1693         default:
1694                 return -EINVAL;
1695         }
1696         return 0;
1697 }
1698
1699 /*
1700  * net_device service functions
1701  */
1702 int bnx2x_poll(struct napi_struct *napi, int budget)
1703 {
1704         int work_done = 0;
1705         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1706                                                  napi);
1707         struct bnx2x *bp = fp->bp;
1708
1709         while (1) {
1710 #ifdef BNX2X_STOP_ON_ERROR
1711                 if (unlikely(bp->panic)) {
1712                         napi_complete(napi);
1713                         return 0;
1714                 }
1715 #endif
1716
1717                 if (bnx2x_has_tx_work(fp))
1718                         bnx2x_tx_int(fp);
1719
1720                 if (bnx2x_has_rx_work(fp)) {
1721                         work_done += bnx2x_rx_int(fp, budget - work_done);
1722
1723                         /* must not complete if we consumed full budget */
1724                         if (work_done >= budget)
1725                                 break;
1726                 }
1727
1728                 /* Fall out from the NAPI loop if needed */
1729                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1730 #ifdef BCM_CNIC
1731                         /* No need to update SB for FCoE L2 ring as long as
1732                          * it's connected to the default SB and the SB
1733                          * has been updated when NAPI was scheduled.
1734                          */
1735                         if (IS_FCOE_FP(fp)) {
1736                                 napi_complete(napi);
1737                                 break;
1738                         }
1739 #endif
1740
1741                         bnx2x_update_fpsb_idx(fp);
1742                         /* bnx2x_has_rx_work() reads the status block,
1743                          * thus we need to ensure that status block indices
1744                          * have been actually read (bnx2x_update_fpsb_idx)
1745                          * prior to this check (bnx2x_has_rx_work) so that
1746                          * we won't write the "newer" value of the status block
1747                          * to IGU (if there was a DMA right after
1748                          * bnx2x_has_rx_work and if there is no rmb, the memory
1749                          * reading (bnx2x_update_fpsb_idx) may be postponed
1750                          * to right before bnx2x_ack_sb). In this case there
1751                          * will never be another interrupt until there is
1752                          * another update of the status block, while there
1753                          * is still unhandled work.
1754                          */
1755                         rmb();
1756
1757                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1758                                 napi_complete(napi);
1759                                 /* Re-enable interrupts */
1760                                 DP(NETIF_MSG_HW,
1761                                    "Update index to %d\n", fp->fp_hc_idx);
1762                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
1763                                              le16_to_cpu(fp->fp_hc_idx),
1764                                              IGU_INT_ENABLE, 1);
1765                                 break;
1766                         }
1767                 }
1768         }
1769
1770         return work_done;
1771 }
1772
1773 /* we split the first BD into headers and data BDs
1774  * to ease the pain of our fellow microcode engineers
1775  * we use one mapping for both BDs
1776  * So far this has only been observed to happen
1777  * in Other Operating Systems(TM)
1778  */
1779 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1780                                    struct bnx2x_fastpath *fp,
1781                                    struct sw_tx_bd *tx_buf,
1782                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1783                                    u16 bd_prod, int nbd)
1784 {
1785         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1786         struct eth_tx_bd *d_tx_bd;
1787         dma_addr_t mapping;
1788         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1789
1790         /* first fix first BD */
1791         h_tx_bd->nbd = cpu_to_le16(nbd);
1792         h_tx_bd->nbytes = cpu_to_le16(hlen);
1793
1794         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1795            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1796            h_tx_bd->addr_lo, h_tx_bd->nbd);
1797
1798         /* now get a new data BD
1799          * (after the pbd) and fill it */
1800         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1801         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1802
1803         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1804                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1805
1806         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1807         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1808         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1809
1810         /* this marks the BD as one that has no individual mapping */
1811         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1812
1813         DP(NETIF_MSG_TX_QUEUED,
1814            "TSO split data size is %d (%x:%x)\n",
1815            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1816
1817         /* update tx_bd */
1818         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1819
1820         return bd_prod;
1821 }
1822
1823 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1824 {
1825         if (fix > 0)
1826                 csum = (u16) ~csum_fold(csum_sub(csum,
1827                                 csum_partial(t_header - fix, fix, 0)));
1828
1829         else if (fix < 0)
1830                 csum = (u16) ~csum_fold(csum_add(csum,
1831                                 csum_partial(t_header, -fix, 0)));
1832
1833         return swab16(csum);
1834 }
1835
1836 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1837 {
1838         u32 rc;
1839
1840         if (skb->ip_summed != CHECKSUM_PARTIAL)
1841                 rc = XMIT_PLAIN;
1842
1843         else {
1844                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
1845                         rc = XMIT_CSUM_V6;
1846                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1847                                 rc |= XMIT_CSUM_TCP;
1848
1849                 } else {
1850                         rc = XMIT_CSUM_V4;
1851                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1852                                 rc |= XMIT_CSUM_TCP;
1853                 }
1854         }
1855
1856         if (skb_is_gso_v6(skb))
1857                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
1858         else if (skb_is_gso(skb))
1859                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
1860
1861         return rc;
1862 }
1863
1864 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1865 /* check if packet requires linearization (packet is too fragmented)
1866    no need to check fragmentation if page size > 8K (there will be no
1867    violation to FW restrictions) */
1868 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1869                              u32 xmit_type)
1870 {
1871         int to_copy = 0;
1872         int hlen = 0;
1873         int first_bd_sz = 0;
1874
1875         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1876         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1877
1878                 if (xmit_type & XMIT_GSO) {
1879                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1880                         /* Check if LSO packet needs to be copied:
1881                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1882                         int wnd_size = MAX_FETCH_BD - 3;
1883                         /* Number of windows to check */
1884                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1885                         int wnd_idx = 0;
1886                         int frag_idx = 0;
1887                         u32 wnd_sum = 0;
1888
1889                         /* Headers length */
1890                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1891                                 tcp_hdrlen(skb);
1892
1893                         /* Amount of data (w/o headers) on linear part of SKB*/
1894                         first_bd_sz = skb_headlen(skb) - hlen;
1895
1896                         wnd_sum  = first_bd_sz;
1897
1898                         /* Calculate the first sum - it's special */
1899                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1900                                 wnd_sum +=
1901                                         skb_shinfo(skb)->frags[frag_idx].size;
1902
1903                         /* If there was data on linear skb data - check it */
1904                         if (first_bd_sz > 0) {
1905                                 if (unlikely(wnd_sum < lso_mss)) {
1906                                         to_copy = 1;
1907                                         goto exit_lbl;
1908                                 }
1909
1910                                 wnd_sum -= first_bd_sz;
1911                         }
1912
1913                         /* Others are easier: run through the frag list and
1914                            check all windows */
1915                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1916                                 wnd_sum +=
1917                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1918
1919                                 if (unlikely(wnd_sum < lso_mss)) {
1920                                         to_copy = 1;
1921                                         break;
1922                                 }
1923                                 wnd_sum -=
1924                                         skb_shinfo(skb)->frags[wnd_idx].size;
1925                         }
1926                 } else {
1927                         /* in non-LSO too fragmented packet should always
1928                            be linearized */
1929                         to_copy = 1;
1930                 }
1931         }
1932
1933 exit_lbl:
1934         if (unlikely(to_copy))
1935                 DP(NETIF_MSG_TX_QUEUED,
1936                    "Linearization IS REQUIRED for %s packet. "
1937                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1938                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1939                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1940
1941         return to_copy;
1942 }
1943 #endif
1944
1945 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
1946                                         u32 xmit_type)
1947 {
1948         *parsing_data |= (skb_shinfo(skb)->gso_size <<
1949                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
1950                               ETH_TX_PARSE_BD_E2_LSO_MSS;
1951         if ((xmit_type & XMIT_GSO_V6) &&
1952             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
1953                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
1954 }
1955
1956 /**
1957  * Update PBD in GSO case.
1958  *
1959  * @param skb
1960  * @param tx_start_bd
1961  * @param pbd
1962  * @param xmit_type
1963  */
1964 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
1965                                      struct eth_tx_parse_bd_e1x *pbd,
1966                                      u32 xmit_type)
1967 {
1968         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
1969         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
1970         pbd->tcp_flags = pbd_tcp_flags(skb);
1971
1972         if (xmit_type & XMIT_GSO_V4) {
1973                 pbd->ip_id = swab16(ip_hdr(skb)->id);
1974                 pbd->tcp_pseudo_csum =
1975                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1976                                                   ip_hdr(skb)->daddr,
1977                                                   0, IPPROTO_TCP, 0));
1978
1979         } else
1980                 pbd->tcp_pseudo_csum =
1981                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1982                                                 &ipv6_hdr(skb)->daddr,
1983                                                 0, IPPROTO_TCP, 0));
1984
1985         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
1986 }
1987
1988 /**
1989  *
1990  * @param skb
1991  * @param tx_start_bd
1992  * @param pbd_e2
1993  * @param xmit_type
1994  *
1995  * @return header len
1996  */
1997 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
1998         u32 *parsing_data, u32 xmit_type)
1999 {
2000         *parsing_data |= ((tcp_hdrlen(skb)/4) <<
2001                 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2002                 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2003
2004         *parsing_data |= ((((u8 *)tcp_hdr(skb) - skb->data) / 2) <<
2005                 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2006                 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2007
2008         return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2009 }
2010
2011 /**
2012  *
2013  * @param skb
2014  * @param tx_start_bd
2015  * @param pbd
2016  * @param xmit_type
2017  *
2018  * @return Header length
2019  */
2020 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2021         struct eth_tx_parse_bd_e1x *pbd,
2022         u32 xmit_type)
2023 {
2024         u8 hlen = (skb_network_header(skb) - skb->data) / 2;
2025
2026         /* for now NS flag is not used in Linux */
2027         pbd->global_data =
2028                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2029                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2030
2031         pbd->ip_hlen_w = (skb_transport_header(skb) -
2032                         skb_network_header(skb)) / 2;
2033
2034         hlen += pbd->ip_hlen_w + tcp_hdrlen(skb) / 2;
2035
2036         pbd->total_hlen_w = cpu_to_le16(hlen);
2037         hlen = hlen*2;
2038
2039         if (xmit_type & XMIT_CSUM_TCP) {
2040                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2041
2042         } else {
2043                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2044
2045                 DP(NETIF_MSG_TX_QUEUED,
2046                    "hlen %d  fix %d  csum before fix %x\n",
2047                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2048
2049                 /* HW bug: fixup the CSUM */
2050                 pbd->tcp_pseudo_csum =
2051                         bnx2x_csum_fix(skb_transport_header(skb),
2052                                        SKB_CS(skb), fix);
2053
2054                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2055                    pbd->tcp_pseudo_csum);
2056         }
2057
2058         return hlen;
2059 }
2060
2061 /* called with netif_tx_lock
2062  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2063  * netif_wake_queue()
2064  */
2065 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2066 {
2067         struct bnx2x *bp = netdev_priv(dev);
2068         struct bnx2x_fastpath *fp;
2069         struct netdev_queue *txq;
2070         struct sw_tx_bd *tx_buf;
2071         struct eth_tx_start_bd *tx_start_bd;
2072         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2073         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2074         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2075         u32 pbd_e2_parsing_data = 0;
2076         u16 pkt_prod, bd_prod;
2077         int nbd, fp_index;
2078         dma_addr_t mapping;
2079         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2080         int i;
2081         u8 hlen = 0;
2082         __le16 pkt_size = 0;
2083         struct ethhdr *eth;
2084         u8 mac_type = UNICAST_ADDRESS;
2085
2086 #ifdef BNX2X_STOP_ON_ERROR
2087         if (unlikely(bp->panic))
2088                 return NETDEV_TX_BUSY;
2089 #endif
2090
2091         fp_index = skb_get_queue_mapping(skb);
2092         txq = netdev_get_tx_queue(dev, fp_index);
2093
2094         fp = &bp->fp[fp_index];
2095
2096         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
2097                 fp->eth_q_stats.driver_xoff++;
2098                 netif_tx_stop_queue(txq);
2099                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2100                 return NETDEV_TX_BUSY;
2101         }
2102
2103         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
2104                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
2105            fp_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2106            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2107
2108         eth = (struct ethhdr *)skb->data;
2109
2110         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2111         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2112                 if (is_broadcast_ether_addr(eth->h_dest))
2113                         mac_type = BROADCAST_ADDRESS;
2114                 else
2115                         mac_type = MULTICAST_ADDRESS;
2116         }
2117
2118 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2119         /* First, check if we need to linearize the skb (due to FW
2120            restrictions). No need to check fragmentation if page size > 8K
2121            (there will be no violation to FW restrictions) */
2122         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2123                 /* Statistics of linearization */
2124                 bp->lin_cnt++;
2125                 if (skb_linearize(skb) != 0) {
2126                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2127                            "silently dropping this SKB\n");
2128                         dev_kfree_skb_any(skb);
2129                         return NETDEV_TX_OK;
2130                 }
2131         }
2132 #endif
2133
2134         /*
2135         Please read carefully. First we use one BD which we mark as start,
2136         then we have a parsing info BD (used for TSO or xsum),
2137         and only then we have the rest of the TSO BDs.
2138         (don't forget to mark the last one as last,
2139         and to unmap only AFTER you write to the BD ...)
2140         And above all, all pdb sizes are in words - NOT DWORDS!
2141         */
2142
2143         pkt_prod = fp->tx_pkt_prod++;
2144         bd_prod = TX_BD(fp->tx_bd_prod);
2145
2146         /* get a tx_buf and first BD */
2147         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
2148         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
2149
2150         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2151         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2152                  mac_type);
2153
2154         /* header nbd */
2155         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2156
2157         /* remember the first BD of the packet */
2158         tx_buf->first_bd = fp->tx_bd_prod;
2159         tx_buf->skb = skb;
2160         tx_buf->flags = 0;
2161
2162         DP(NETIF_MSG_TX_QUEUED,
2163            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2164            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
2165
2166         if (vlan_tx_tag_present(skb)) {
2167                 tx_start_bd->vlan_or_ethertype =
2168                     cpu_to_le16(vlan_tx_tag_get(skb));
2169                 tx_start_bd->bd_flags.as_bitfield |=
2170                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2171         } else
2172                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2173
2174         /* turn on parsing and get a BD */
2175         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2176
2177         if (xmit_type & XMIT_CSUM) {
2178                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2179
2180                 if (xmit_type & XMIT_CSUM_V4)
2181                         tx_start_bd->bd_flags.as_bitfield |=
2182                                                 ETH_TX_BD_FLAGS_IP_CSUM;
2183                 else
2184                         tx_start_bd->bd_flags.as_bitfield |=
2185                                                 ETH_TX_BD_FLAGS_IPV6;
2186
2187                 if (!(xmit_type & XMIT_CSUM_TCP))
2188                         tx_start_bd->bd_flags.as_bitfield |=
2189                                                 ETH_TX_BD_FLAGS_IS_UDP;
2190         }
2191
2192         if (CHIP_IS_E2(bp)) {
2193                 pbd_e2 = &fp->tx_desc_ring[bd_prod].parse_bd_e2;
2194                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2195                 /* Set PBD in checksum offload case */
2196                 if (xmit_type & XMIT_CSUM)
2197                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2198                                                      &pbd_e2_parsing_data,
2199                                                      xmit_type);
2200         } else {
2201                 pbd_e1x = &fp->tx_desc_ring[bd_prod].parse_bd_e1x;
2202                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2203                 /* Set PBD in checksum offload case */
2204                 if (xmit_type & XMIT_CSUM)
2205                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2206
2207         }
2208
2209         /* Map skb linear data for DMA */
2210         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2211                                  skb_headlen(skb), DMA_TO_DEVICE);
2212
2213         /* Setup the data pointer of the first BD of the packet */
2214         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2215         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2216         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2217         tx_start_bd->nbd = cpu_to_le16(nbd);
2218         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2219         pkt_size = tx_start_bd->nbytes;
2220
2221         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2222            "  nbytes %d  flags %x  vlan %x\n",
2223            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2224            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2225            tx_start_bd->bd_flags.as_bitfield,
2226            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2227
2228         if (xmit_type & XMIT_GSO) {
2229
2230                 DP(NETIF_MSG_TX_QUEUED,
2231                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2232                    skb->len, hlen, skb_headlen(skb),
2233                    skb_shinfo(skb)->gso_size);
2234
2235                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2236
2237                 if (unlikely(skb_headlen(skb) > hlen))
2238                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2239                                                  hlen, bd_prod, ++nbd);
2240                 if (CHIP_IS_E2(bp))
2241                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2242                                              xmit_type);
2243                 else
2244                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2245         }
2246
2247         /* Set the PBD's parsing_data field if not zero
2248          * (for the chips newer than 57711).
2249          */
2250         if (pbd_e2_parsing_data)
2251                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2252
2253         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2254
2255         /* Handle fragmented skb */
2256         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2257                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2258
2259                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2260                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2261                 if (total_pkt_bd == NULL)
2262                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2263
2264                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2265                                        frag->page_offset,
2266                                        frag->size, DMA_TO_DEVICE);
2267
2268                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2269                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2270                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2271                 le16_add_cpu(&pkt_size, frag->size);
2272
2273                 DP(NETIF_MSG_TX_QUEUED,
2274                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2275                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2276                    le16_to_cpu(tx_data_bd->nbytes));
2277         }
2278
2279         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2280
2281         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2282
2283         /* now send a tx doorbell, counting the next BD
2284          * if the packet contains or ends with it
2285          */
2286         if (TX_BD_POFF(bd_prod) < nbd)
2287                 nbd++;
2288
2289         if (total_pkt_bd != NULL)
2290                 total_pkt_bd->total_pkt_bytes = pkt_size;
2291
2292         if (pbd_e1x)
2293                 DP(NETIF_MSG_TX_QUEUED,
2294                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2295                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2296                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2297                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2298                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2299                     le16_to_cpu(pbd_e1x->total_hlen_w));
2300         if (pbd_e2)
2301                 DP(NETIF_MSG_TX_QUEUED,
2302                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2303                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2304                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2305                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2306                    pbd_e2->parsing_data);
2307         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2308
2309         /*
2310          * Make sure that the BD data is updated before updating the producer
2311          * since FW might read the BD right after the producer is updated.
2312          * This is only applicable for weak-ordered memory model archs such
2313          * as IA-64. The following barrier is also mandatory since FW will
2314          * assumes packets must have BDs.
2315          */
2316         wmb();
2317
2318         fp->tx_db.data.prod += nbd;
2319         barrier();
2320
2321         DOORBELL(bp, fp->cid, fp->tx_db.raw);
2322
2323         mmiowb();
2324
2325         fp->tx_bd_prod += nbd;
2326
2327         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2328                 netif_tx_stop_queue(txq);
2329
2330                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2331                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2332                  * fp->bd_tx_cons */
2333                 smp_mb();
2334
2335                 fp->eth_q_stats.driver_xoff++;
2336                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2337                         netif_tx_wake_queue(txq);
2338         }
2339         fp->tx_pkt++;
2340
2341         return NETDEV_TX_OK;
2342 }
2343
2344 /* called with rtnl_lock */
2345 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2346 {
2347         struct sockaddr *addr = p;
2348         struct bnx2x *bp = netdev_priv(dev);
2349
2350         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2351                 return -EINVAL;
2352
2353         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2354         if (netif_running(dev))
2355                 bnx2x_set_eth_mac(bp, 1);
2356
2357         return 0;
2358 }
2359
2360
2361 static int bnx2x_setup_irqs(struct bnx2x *bp)
2362 {
2363         int rc = 0;
2364         if (bp->flags & USING_MSIX_FLAG) {
2365                 rc = bnx2x_req_msix_irqs(bp);
2366                 if (rc)
2367                         return rc;
2368         } else {
2369                 bnx2x_ack_int(bp);
2370                 rc = bnx2x_req_irq(bp);
2371                 if (rc) {
2372                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
2373                         return rc;
2374                 }
2375                 if (bp->flags & USING_MSI_FLAG) {
2376                         bp->dev->irq = bp->pdev->irq;
2377                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
2378                                bp->pdev->irq);
2379                 }
2380         }
2381
2382         return 0;
2383 }
2384
2385 void bnx2x_free_mem_bp(struct bnx2x *bp)
2386 {
2387         kfree(bp->fp);
2388         kfree(bp->msix_table);
2389         kfree(bp->ilt);
2390 }
2391
2392 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
2393 {
2394         struct bnx2x_fastpath *fp;
2395         struct msix_entry *tbl;
2396         struct bnx2x_ilt *ilt;
2397
2398         /* fp array */
2399         fp = kzalloc(L2_FP_COUNT(bp->l2_cid_count)*sizeof(*fp), GFP_KERNEL);
2400         if (!fp)
2401                 goto alloc_err;
2402         bp->fp = fp;
2403
2404         /* msix table */
2405         tbl = kzalloc((FP_SB_COUNT(bp->l2_cid_count) + 1) * sizeof(*tbl),
2406                                   GFP_KERNEL);
2407         if (!tbl)
2408                 goto alloc_err;
2409         bp->msix_table = tbl;
2410
2411         /* ilt */
2412         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
2413         if (!ilt)
2414                 goto alloc_err;
2415         bp->ilt = ilt;
2416
2417         return 0;
2418 alloc_err:
2419         bnx2x_free_mem_bp(bp);
2420         return -ENOMEM;
2421
2422 }
2423
2424 /* called with rtnl_lock */
2425 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2426 {
2427         struct bnx2x *bp = netdev_priv(dev);
2428         int rc = 0;
2429
2430         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2431                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2432                 return -EAGAIN;
2433         }
2434
2435         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2436             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2437                 return -EINVAL;
2438
2439         /* This does not race with packet allocation
2440          * because the actual alloc size is
2441          * only updated as part of load
2442          */
2443         dev->mtu = new_mtu;
2444
2445         if (netif_running(dev)) {
2446                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2447                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2448         }
2449
2450         return rc;
2451 }
2452
2453 void bnx2x_tx_timeout(struct net_device *dev)
2454 {
2455         struct bnx2x *bp = netdev_priv(dev);
2456
2457 #ifdef BNX2X_STOP_ON_ERROR
2458         if (!bp->panic)
2459                 bnx2x_panic();
2460 #endif
2461         /* This allows the netif to be shutdown gracefully before resetting */
2462         schedule_delayed_work(&bp->reset_task, 0);
2463 }
2464
2465 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2466 {
2467         struct net_device *dev = pci_get_drvdata(pdev);
2468         struct bnx2x *bp;
2469
2470         if (!dev) {
2471                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2472                 return -ENODEV;
2473         }
2474         bp = netdev_priv(dev);
2475
2476         rtnl_lock();
2477
2478         pci_save_state(pdev);
2479
2480         if (!netif_running(dev)) {
2481                 rtnl_unlock();
2482                 return 0;
2483         }
2484
2485         netif_device_detach(dev);
2486
2487         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2488
2489         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2490
2491         rtnl_unlock();
2492
2493         return 0;
2494 }
2495
2496 int bnx2x_resume(struct pci_dev *pdev)
2497 {
2498         struct net_device *dev = pci_get_drvdata(pdev);
2499         struct bnx2x *bp;
2500         int rc;
2501
2502         if (!dev) {
2503                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2504                 return -ENODEV;
2505         }
2506         bp = netdev_priv(dev);
2507
2508         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2509                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2510                 return -EAGAIN;
2511         }
2512
2513         rtnl_lock();
2514
2515         pci_restore_state(pdev);
2516
2517         if (!netif_running(dev)) {
2518                 rtnl_unlock();
2519                 return 0;
2520         }
2521
2522         bnx2x_set_power_state(bp, PCI_D0);
2523         netif_device_attach(dev);
2524
2525         /* Since the chip was reset, clear the FW sequence number */
2526         bp->fw_seq = 0;
2527         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2528
2529         rtnl_unlock();
2530
2531         return rc;
2532 }