6a16f76f277e76cc669430baa222f998d39649bf
[linux-3.10.git] / drivers / net / vmxnet3 / vmxnet3_drv.c
1 /*
2  * Linux driver for VMware's vmxnet3 ethernet NIC.
3  *
4  * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License as published by the
8  * Free Software Foundation; version 2 of the License and no later version.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13  * NON INFRINGEMENT. See the GNU General Public License for more
14  * details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19  *
20  * The full GNU General Public License is included in this distribution in
21  * the file called "COPYING".
22  *
23  * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
24  *
25  */
26
27 #include "vmxnet3_int.h"
28
29 char vmxnet3_driver_name[] = "vmxnet3";
30 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
31
32
33 /*
34  * PCI Device ID Table
35  * Last entry must be all 0s
36  */
37 static const struct pci_device_id vmxnet3_pciid_table[] = {
38         {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
39         {0}
40 };
41
42 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
43
44 static atomic_t devices_found;
45
46
47 /*
48  *    Enable/Disable the given intr
49  */
50 static void
51 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
52 {
53         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
54 }
55
56
57 static void
58 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
59 {
60         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
61 }
62
63
64 /*
65  *    Enable/Disable all intrs used by the device
66  */
67 static void
68 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
69 {
70         int i;
71
72         for (i = 0; i < adapter->intr.num_intrs; i++)
73                 vmxnet3_enable_intr(adapter, i);
74 }
75
76
77 static void
78 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
79 {
80         int i;
81
82         for (i = 0; i < adapter->intr.num_intrs; i++)
83                 vmxnet3_disable_intr(adapter, i);
84 }
85
86
87 static void
88 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
89 {
90         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
91 }
92
93
94 static bool
95 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
96 {
97         return netif_queue_stopped(adapter->netdev);
98 }
99
100
101 static void
102 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
103 {
104         tq->stopped = false;
105         netif_start_queue(adapter->netdev);
106 }
107
108
109 static void
110 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111 {
112         tq->stopped = false;
113         netif_wake_queue(adapter->netdev);
114 }
115
116
117 static void
118 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119 {
120         tq->stopped = true;
121         tq->num_stop++;
122         netif_stop_queue(adapter->netdev);
123 }
124
125
126 /*
127  * Check the link state. This may start or stop the tx queue.
128  */
129 static void
130 vmxnet3_check_link(struct vmxnet3_adapter *adapter)
131 {
132         u32 ret;
133
134         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
135         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
136         adapter->link_speed = ret >> 16;
137         if (ret & 1) { /* Link is up. */
138                 printk(KERN_INFO "%s: NIC Link is Up %d Mbps\n",
139                        adapter->netdev->name, adapter->link_speed);
140                 if (!netif_carrier_ok(adapter->netdev))
141                         netif_carrier_on(adapter->netdev);
142
143                 vmxnet3_tq_start(&adapter->tx_queue, adapter);
144         } else {
145                 printk(KERN_INFO "%s: NIC Link is Down\n",
146                        adapter->netdev->name);
147                 if (netif_carrier_ok(adapter->netdev))
148                         netif_carrier_off(adapter->netdev);
149
150                 vmxnet3_tq_stop(&adapter->tx_queue, adapter);
151         }
152 }
153
154
155 static void
156 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
157 {
158         u32 events = adapter->shared->ecr;
159         if (!events)
160                 return;
161
162         vmxnet3_ack_events(adapter, events);
163
164         /* Check if link state has changed */
165         if (events & VMXNET3_ECR_LINK)
166                 vmxnet3_check_link(adapter);
167
168         /* Check if there is an error on xmit/recv queues */
169         if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
170                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
171                                        VMXNET3_CMD_GET_QUEUE_STATUS);
172
173                 if (adapter->tqd_start->status.stopped) {
174                         printk(KERN_ERR "%s: tq error 0x%x\n",
175                                adapter->netdev->name,
176                                adapter->tqd_start->status.error);
177                 }
178                 if (adapter->rqd_start->status.stopped) {
179                         printk(KERN_ERR "%s: rq error 0x%x\n",
180                                adapter->netdev->name,
181                                adapter->rqd_start->status.error);
182                 }
183
184                 schedule_work(&adapter->work);
185         }
186 }
187
188
189 static void
190 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
191                      struct pci_dev *pdev)
192 {
193         if (tbi->map_type == VMXNET3_MAP_SINGLE)
194                 pci_unmap_single(pdev, tbi->dma_addr, tbi->len,
195                                  PCI_DMA_TODEVICE);
196         else if (tbi->map_type == VMXNET3_MAP_PAGE)
197                 pci_unmap_page(pdev, tbi->dma_addr, tbi->len,
198                                PCI_DMA_TODEVICE);
199         else
200                 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
201
202         tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
203 }
204
205
206 static int
207 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
208                   struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
209 {
210         struct sk_buff *skb;
211         int entries = 0;
212
213         /* no out of order completion */
214         BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
215         BUG_ON(tq->tx_ring.base[eop_idx].txd.eop != 1);
216
217         skb = tq->buf_info[eop_idx].skb;
218         BUG_ON(skb == NULL);
219         tq->buf_info[eop_idx].skb = NULL;
220
221         VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
222
223         while (tq->tx_ring.next2comp != eop_idx) {
224                 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
225                                      pdev);
226
227                 /* update next2comp w/o tx_lock. Since we are marking more,
228                  * instead of less, tx ring entries avail, the worst case is
229                  * that the tx routine incorrectly re-queues a pkt due to
230                  * insufficient tx ring entries.
231                  */
232                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
233                 entries++;
234         }
235
236         dev_kfree_skb_any(skb);
237         return entries;
238 }
239
240
241 static int
242 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
243                         struct vmxnet3_adapter *adapter)
244 {
245         int completed = 0;
246         union Vmxnet3_GenericDesc *gdesc;
247
248         gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
249         while (gdesc->tcd.gen == tq->comp_ring.gen) {
250                 completed += vmxnet3_unmap_pkt(gdesc->tcd.txdIdx, tq,
251                                                adapter->pdev, adapter);
252
253                 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
254                 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
255         }
256
257         if (completed) {
258                 spin_lock(&tq->tx_lock);
259                 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
260                              vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
261                              VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
262                              netif_carrier_ok(adapter->netdev))) {
263                         vmxnet3_tq_wake(tq, adapter);
264                 }
265                 spin_unlock(&tq->tx_lock);
266         }
267         return completed;
268 }
269
270
271 static void
272 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
273                    struct vmxnet3_adapter *adapter)
274 {
275         int i;
276
277         while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
278                 struct vmxnet3_tx_buf_info *tbi;
279                 union Vmxnet3_GenericDesc *gdesc;
280
281                 tbi = tq->buf_info + tq->tx_ring.next2comp;
282                 gdesc = tq->tx_ring.base + tq->tx_ring.next2comp;
283
284                 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
285                 if (tbi->skb) {
286                         dev_kfree_skb_any(tbi->skb);
287                         tbi->skb = NULL;
288                 }
289                 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
290         }
291
292         /* sanity check, verify all buffers are indeed unmapped and freed */
293         for (i = 0; i < tq->tx_ring.size; i++) {
294                 BUG_ON(tq->buf_info[i].skb != NULL ||
295                        tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
296         }
297
298         tq->tx_ring.gen = VMXNET3_INIT_GEN;
299         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
300
301         tq->comp_ring.gen = VMXNET3_INIT_GEN;
302         tq->comp_ring.next2proc = 0;
303 }
304
305
306 void
307 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
308                    struct vmxnet3_adapter *adapter)
309 {
310         if (tq->tx_ring.base) {
311                 pci_free_consistent(adapter->pdev, tq->tx_ring.size *
312                                     sizeof(struct Vmxnet3_TxDesc),
313                                     tq->tx_ring.base, tq->tx_ring.basePA);
314                 tq->tx_ring.base = NULL;
315         }
316         if (tq->data_ring.base) {
317                 pci_free_consistent(adapter->pdev, tq->data_ring.size *
318                                     sizeof(struct Vmxnet3_TxDataDesc),
319                                     tq->data_ring.base, tq->data_ring.basePA);
320                 tq->data_ring.base = NULL;
321         }
322         if (tq->comp_ring.base) {
323                 pci_free_consistent(adapter->pdev, tq->comp_ring.size *
324                                     sizeof(struct Vmxnet3_TxCompDesc),
325                                     tq->comp_ring.base, tq->comp_ring.basePA);
326                 tq->comp_ring.base = NULL;
327         }
328         kfree(tq->buf_info);
329         tq->buf_info = NULL;
330 }
331
332
333 static void
334 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
335                 struct vmxnet3_adapter *adapter)
336 {
337         int i;
338
339         /* reset the tx ring contents to 0 and reset the tx ring states */
340         memset(tq->tx_ring.base, 0, tq->tx_ring.size *
341                sizeof(struct Vmxnet3_TxDesc));
342         tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
343         tq->tx_ring.gen = VMXNET3_INIT_GEN;
344
345         memset(tq->data_ring.base, 0, tq->data_ring.size *
346                sizeof(struct Vmxnet3_TxDataDesc));
347
348         /* reset the tx comp ring contents to 0 and reset comp ring states */
349         memset(tq->comp_ring.base, 0, tq->comp_ring.size *
350                sizeof(struct Vmxnet3_TxCompDesc));
351         tq->comp_ring.next2proc = 0;
352         tq->comp_ring.gen = VMXNET3_INIT_GEN;
353
354         /* reset the bookkeeping data */
355         memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
356         for (i = 0; i < tq->tx_ring.size; i++)
357                 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
358
359         /* stats are not reset */
360 }
361
362
363 static int
364 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
365                   struct vmxnet3_adapter *adapter)
366 {
367         BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
368                tq->comp_ring.base || tq->buf_info);
369
370         tq->tx_ring.base = pci_alloc_consistent(adapter->pdev, tq->tx_ring.size
371                            * sizeof(struct Vmxnet3_TxDesc),
372                            &tq->tx_ring.basePA);
373         if (!tq->tx_ring.base) {
374                 printk(KERN_ERR "%s: failed to allocate tx ring\n",
375                        adapter->netdev->name);
376                 goto err;
377         }
378
379         tq->data_ring.base = pci_alloc_consistent(adapter->pdev,
380                              tq->data_ring.size *
381                              sizeof(struct Vmxnet3_TxDataDesc),
382                              &tq->data_ring.basePA);
383         if (!tq->data_ring.base) {
384                 printk(KERN_ERR "%s: failed to allocate data ring\n",
385                        adapter->netdev->name);
386                 goto err;
387         }
388
389         tq->comp_ring.base = pci_alloc_consistent(adapter->pdev,
390                              tq->comp_ring.size *
391                              sizeof(struct Vmxnet3_TxCompDesc),
392                              &tq->comp_ring.basePA);
393         if (!tq->comp_ring.base) {
394                 printk(KERN_ERR "%s: failed to allocate tx comp ring\n",
395                        adapter->netdev->name);
396                 goto err;
397         }
398
399         tq->buf_info = kcalloc(tq->tx_ring.size, sizeof(tq->buf_info[0]),
400                                GFP_KERNEL);
401         if (!tq->buf_info) {
402                 printk(KERN_ERR "%s: failed to allocate tx bufinfo\n",
403                        adapter->netdev->name);
404                 goto err;
405         }
406
407         return 0;
408
409 err:
410         vmxnet3_tq_destroy(tq, adapter);
411         return -ENOMEM;
412 }
413
414
415 /*
416  *    starting from ring->next2fill, allocate rx buffers for the given ring
417  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
418  *    are allocated or allocation fails
419  */
420
421 static int
422 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
423                         int num_to_alloc, struct vmxnet3_adapter *adapter)
424 {
425         int num_allocated = 0;
426         struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
427         struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
428         u32 val;
429
430         while (num_allocated < num_to_alloc) {
431                 struct vmxnet3_rx_buf_info *rbi;
432                 union Vmxnet3_GenericDesc *gd;
433
434                 rbi = rbi_base + ring->next2fill;
435                 gd = ring->base + ring->next2fill;
436
437                 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
438                         if (rbi->skb == NULL) {
439                                 rbi->skb = dev_alloc_skb(rbi->len +
440                                                          NET_IP_ALIGN);
441                                 if (unlikely(rbi->skb == NULL)) {
442                                         rq->stats.rx_buf_alloc_failure++;
443                                         break;
444                                 }
445                                 rbi->skb->dev = adapter->netdev;
446
447                                 skb_reserve(rbi->skb, NET_IP_ALIGN);
448                                 rbi->dma_addr = pci_map_single(adapter->pdev,
449                                                 rbi->skb->data, rbi->len,
450                                                 PCI_DMA_FROMDEVICE);
451                         } else {
452                                 /* rx buffer skipped by the device */
453                         }
454                         val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
455                 } else {
456                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
457                                rbi->len  != PAGE_SIZE);
458
459                         if (rbi->page == NULL) {
460                                 rbi->page = alloc_page(GFP_ATOMIC);
461                                 if (unlikely(rbi->page == NULL)) {
462                                         rq->stats.rx_buf_alloc_failure++;
463                                         break;
464                                 }
465                                 rbi->dma_addr = pci_map_page(adapter->pdev,
466                                                 rbi->page, 0, PAGE_SIZE,
467                                                 PCI_DMA_FROMDEVICE);
468                         } else {
469                                 /* rx buffers skipped by the device */
470                         }
471                         val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
472                 }
473
474                 BUG_ON(rbi->dma_addr == 0);
475                 gd->rxd.addr = rbi->dma_addr;
476                 gd->dword[2] = (ring->gen << VMXNET3_RXD_GEN_SHIFT) | val |
477                                 rbi->len;
478
479                 num_allocated++;
480                 vmxnet3_cmd_ring_adv_next2fill(ring);
481         }
482         rq->uncommitted[ring_idx] += num_allocated;
483
484         dprintk(KERN_ERR "alloc_rx_buf: %d allocated, next2fill %u, next2comp "
485                 "%u, uncommited %u\n", num_allocated, ring->next2fill,
486                 ring->next2comp, rq->uncommitted[ring_idx]);
487
488         /* so that the device can distinguish a full ring and an empty ring */
489         BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
490
491         return num_allocated;
492 }
493
494
495 static void
496 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
497                     struct vmxnet3_rx_buf_info *rbi)
498 {
499         struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
500                 skb_shinfo(skb)->nr_frags;
501
502         BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
503
504         frag->page = rbi->page;
505         frag->page_offset = 0;
506         frag->size = rcd->len;
507         skb->data_len += frag->size;
508         skb_shinfo(skb)->nr_frags++;
509 }
510
511
512 static void
513 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
514                 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
515                 struct vmxnet3_adapter *adapter)
516 {
517         u32 dw2, len;
518         unsigned long buf_offset;
519         int i;
520         union Vmxnet3_GenericDesc *gdesc;
521         struct vmxnet3_tx_buf_info *tbi = NULL;
522
523         BUG_ON(ctx->copy_size > skb_headlen(skb));
524
525         /* use the previous gen bit for the SOP desc */
526         dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
527
528         ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
529         gdesc = ctx->sop_txd; /* both loops below can be skipped */
530
531         /* no need to map the buffer if headers are copied */
532         if (ctx->copy_size) {
533                 ctx->sop_txd->txd.addr = tq->data_ring.basePA +
534                                         tq->tx_ring.next2fill *
535                                         sizeof(struct Vmxnet3_TxDataDesc);
536                 ctx->sop_txd->dword[2] = dw2 | ctx->copy_size;
537                 ctx->sop_txd->dword[3] = 0;
538
539                 tbi = tq->buf_info + tq->tx_ring.next2fill;
540                 tbi->map_type = VMXNET3_MAP_NONE;
541
542                 dprintk(KERN_ERR "txd[%u]: 0x%Lx 0x%x 0x%x\n",
543                         tq->tx_ring.next2fill, ctx->sop_txd->txd.addr,
544                         ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
545                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
546
547                 /* use the right gen for non-SOP desc */
548                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
549         }
550
551         /* linear part can use multiple tx desc if it's big */
552         len = skb_headlen(skb) - ctx->copy_size;
553         buf_offset = ctx->copy_size;
554         while (len) {
555                 u32 buf_size;
556
557                 buf_size = len > VMXNET3_MAX_TX_BUF_SIZE ?
558                            VMXNET3_MAX_TX_BUF_SIZE : len;
559
560                 tbi = tq->buf_info + tq->tx_ring.next2fill;
561                 tbi->map_type = VMXNET3_MAP_SINGLE;
562                 tbi->dma_addr = pci_map_single(adapter->pdev,
563                                 skb->data + buf_offset, buf_size,
564                                 PCI_DMA_TODEVICE);
565
566                 tbi->len = buf_size; /* this automatically convert 2^14 to 0 */
567
568                 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
569                 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
570
571                 gdesc->txd.addr = tbi->dma_addr;
572                 gdesc->dword[2] = dw2 | buf_size;
573                 gdesc->dword[3] = 0;
574
575                 dprintk(KERN_ERR "txd[%u]: 0x%Lx 0x%x 0x%x\n",
576                         tq->tx_ring.next2fill, gdesc->txd.addr,
577                         gdesc->dword[2], gdesc->dword[3]);
578                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
579                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
580
581                 len -= buf_size;
582                 buf_offset += buf_size;
583         }
584
585         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
586                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
587
588                 tbi = tq->buf_info + tq->tx_ring.next2fill;
589                 tbi->map_type = VMXNET3_MAP_PAGE;
590                 tbi->dma_addr = pci_map_page(adapter->pdev, frag->page,
591                                              frag->page_offset, frag->size,
592                                              PCI_DMA_TODEVICE);
593
594                 tbi->len = frag->size;
595
596                 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
597                 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
598
599                 gdesc->txd.addr = tbi->dma_addr;
600                 gdesc->dword[2] = dw2 | frag->size;
601                 gdesc->dword[3] = 0;
602
603                 dprintk(KERN_ERR "txd[%u]: 0x%llu %u %u\n",
604                         tq->tx_ring.next2fill, gdesc->txd.addr,
605                         gdesc->dword[2], gdesc->dword[3]);
606                 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
607                 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
608         }
609
610         ctx->eop_txd = gdesc;
611
612         /* set the last buf_info for the pkt */
613         tbi->skb = skb;
614         tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
615 }
616
617
618 /*
619  *    parse and copy relevant protocol headers:
620  *      For a tso pkt, relevant headers are L2/3/4 including options
621  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
622  *      if it's a TCP/UDP pkt
623  *
624  * Returns:
625  *    -1:  error happens during parsing
626  *     0:  protocol headers parsed, but too big to be copied
627  *     1:  protocol headers parsed and copied
628  *
629  * Other effects:
630  *    1. related *ctx fields are updated.
631  *    2. ctx->copy_size is # of bytes copied
632  *    3. the portion copied is guaranteed to be in the linear part
633  *
634  */
635 static int
636 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
637                            struct vmxnet3_tx_ctx *ctx,
638                            struct vmxnet3_adapter *adapter)
639 {
640         struct Vmxnet3_TxDataDesc *tdd;
641
642         if (ctx->mss) {
643                 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
644                 ctx->l4_hdr_size = ((struct tcphdr *)
645                                    skb_transport_header(skb))->doff * 4;
646                 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
647         } else {
648                 unsigned int pull_size;
649
650                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
651                         ctx->eth_ip_hdr_size = skb_transport_offset(skb);
652
653                         if (ctx->ipv4) {
654                                 struct iphdr *iph = (struct iphdr *)
655                                                     skb_network_header(skb);
656                                 if (iph->protocol == IPPROTO_TCP) {
657                                         pull_size = ctx->eth_ip_hdr_size +
658                                                     sizeof(struct tcphdr);
659
660                                         if (unlikely(!pskb_may_pull(skb,
661                                                                 pull_size))) {
662                                                 goto err;
663                                         }
664                                         ctx->l4_hdr_size = ((struct tcphdr *)
665                                            skb_transport_header(skb))->doff * 4;
666                                 } else if (iph->protocol == IPPROTO_UDP) {
667                                         ctx->l4_hdr_size =
668                                                         sizeof(struct udphdr);
669                                 } else {
670                                         ctx->l4_hdr_size = 0;
671                                 }
672                         } else {
673                                 /* for simplicity, don't copy L4 headers */
674                                 ctx->l4_hdr_size = 0;
675                         }
676                         ctx->copy_size = ctx->eth_ip_hdr_size +
677                                          ctx->l4_hdr_size;
678                 } else {
679                         ctx->eth_ip_hdr_size = 0;
680                         ctx->l4_hdr_size = 0;
681                         /* copy as much as allowed */
682                         ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
683                                              , skb_headlen(skb));
684                 }
685
686                 /* make sure headers are accessible directly */
687                 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
688                         goto err;
689         }
690
691         if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
692                 tq->stats.oversized_hdr++;
693                 ctx->copy_size = 0;
694                 return 0;
695         }
696
697         tdd = tq->data_ring.base + tq->tx_ring.next2fill;
698
699         memcpy(tdd->data, skb->data, ctx->copy_size);
700         dprintk(KERN_ERR "copy %u bytes to dataRing[%u]\n",
701                 ctx->copy_size, tq->tx_ring.next2fill);
702         return 1;
703
704 err:
705         return -1;
706 }
707
708
709 static void
710 vmxnet3_prepare_tso(struct sk_buff *skb,
711                     struct vmxnet3_tx_ctx *ctx)
712 {
713         struct tcphdr *tcph = (struct tcphdr *)skb_transport_header(skb);
714         if (ctx->ipv4) {
715                 struct iphdr *iph = (struct iphdr *)skb_network_header(skb);
716                 iph->check = 0;
717                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
718                                                  IPPROTO_TCP, 0);
719         } else {
720                 struct ipv6hdr *iph = (struct ipv6hdr *)skb_network_header(skb);
721                 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
722                                                IPPROTO_TCP, 0);
723         }
724 }
725
726
727 /*
728  * Transmits a pkt thru a given tq
729  * Returns:
730  *    NETDEV_TX_OK:      descriptors are setup successfully
731  *    NETDEV_TX_OK:      error occured, the pkt is dropped
732  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
733  *
734  * Side-effects:
735  *    1. tx ring may be changed
736  *    2. tq stats may be updated accordingly
737  *    3. shared->txNumDeferred may be updated
738  */
739
740 static int
741 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
742                 struct vmxnet3_adapter *adapter, struct net_device *netdev)
743 {
744         int ret;
745         u32 count;
746         unsigned long flags;
747         struct vmxnet3_tx_ctx ctx;
748         union Vmxnet3_GenericDesc *gdesc;
749
750         /* conservatively estimate # of descriptors to use */
751         count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) +
752                 skb_shinfo(skb)->nr_frags + 1;
753
754         ctx.ipv4 = (skb->protocol == __constant_ntohs(ETH_P_IP));
755
756         ctx.mss = skb_shinfo(skb)->gso_size;
757         if (ctx.mss) {
758                 if (skb_header_cloned(skb)) {
759                         if (unlikely(pskb_expand_head(skb, 0, 0,
760                                                       GFP_ATOMIC) != 0)) {
761                                 tq->stats.drop_tso++;
762                                 goto drop_pkt;
763                         }
764                         tq->stats.copy_skb_header++;
765                 }
766                 vmxnet3_prepare_tso(skb, &ctx);
767         } else {
768                 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
769
770                         /* non-tso pkts must not use more than
771                          * VMXNET3_MAX_TXD_PER_PKT entries
772                          */
773                         if (skb_linearize(skb) != 0) {
774                                 tq->stats.drop_too_many_frags++;
775                                 goto drop_pkt;
776                         }
777                         tq->stats.linearized++;
778
779                         /* recalculate the # of descriptors to use */
780                         count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
781                 }
782         }
783
784         ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
785         if (ret >= 0) {
786                 BUG_ON(ret <= 0 && ctx.copy_size != 0);
787                 /* hdrs parsed, check against other limits */
788                 if (ctx.mss) {
789                         if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
790                                      VMXNET3_MAX_TX_BUF_SIZE)) {
791                                 goto hdr_too_big;
792                         }
793                 } else {
794                         if (skb->ip_summed == CHECKSUM_PARTIAL) {
795                                 if (unlikely(ctx.eth_ip_hdr_size +
796                                              skb->csum_offset >
797                                              VMXNET3_MAX_CSUM_OFFSET)) {
798                                         goto hdr_too_big;
799                                 }
800                         }
801                 }
802         } else {
803                 tq->stats.drop_hdr_inspect_err++;
804                 goto drop_pkt;
805         }
806
807         spin_lock_irqsave(&tq->tx_lock, flags);
808
809         if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
810                 tq->stats.tx_ring_full++;
811                 dprintk(KERN_ERR "tx queue stopped on %s, next2comp %u"
812                         " next2fill %u\n", adapter->netdev->name,
813                         tq->tx_ring.next2comp, tq->tx_ring.next2fill);
814
815                 vmxnet3_tq_stop(tq, adapter);
816                 spin_unlock_irqrestore(&tq->tx_lock, flags);
817                 return NETDEV_TX_BUSY;
818         }
819
820         /* fill tx descs related to addr & len */
821         vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
822
823         /* setup the EOP desc */
824         ctx.eop_txd->dword[3] = VMXNET3_TXD_CQ | VMXNET3_TXD_EOP;
825
826         /* setup the SOP desc */
827         gdesc = ctx.sop_txd;
828         if (ctx.mss) {
829                 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
830                 gdesc->txd.om = VMXNET3_OM_TSO;
831                 gdesc->txd.msscof = ctx.mss;
832                 tq->shared->txNumDeferred += (skb->len - gdesc->txd.hlen +
833                                              ctx.mss - 1) / ctx.mss;
834         } else {
835                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
836                         gdesc->txd.hlen = ctx.eth_ip_hdr_size;
837                         gdesc->txd.om = VMXNET3_OM_CSUM;
838                         gdesc->txd.msscof = ctx.eth_ip_hdr_size +
839                                             skb->csum_offset;
840                 } else {
841                         gdesc->txd.om = 0;
842                         gdesc->txd.msscof = 0;
843                 }
844                 tq->shared->txNumDeferred++;
845         }
846
847         if (vlan_tx_tag_present(skb)) {
848                 gdesc->txd.ti = 1;
849                 gdesc->txd.tci = vlan_tx_tag_get(skb);
850         }
851
852         wmb();
853
854         /* finally flips the GEN bit of the SOP desc */
855         gdesc->dword[2] ^= VMXNET3_TXD_GEN;
856         dprintk(KERN_ERR "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
857                 (u32)((union Vmxnet3_GenericDesc *)ctx.sop_txd -
858                 tq->tx_ring.base), gdesc->txd.addr, gdesc->dword[2],
859                 gdesc->dword[3]);
860
861         spin_unlock_irqrestore(&tq->tx_lock, flags);
862
863         if (tq->shared->txNumDeferred >= tq->shared->txThreshold) {
864                 tq->shared->txNumDeferred = 0;
865                 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_TXPROD,
866                                        tq->tx_ring.next2fill);
867         }
868         netdev->trans_start = jiffies;
869
870         return NETDEV_TX_OK;
871
872 hdr_too_big:
873         tq->stats.drop_oversized_hdr++;
874 drop_pkt:
875         tq->stats.drop_total++;
876         dev_kfree_skb(skb);
877         return NETDEV_TX_OK;
878 }
879
880
881 static netdev_tx_t
882 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
883 {
884         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
885         struct vmxnet3_tx_queue *tq = &adapter->tx_queue;
886
887         return vmxnet3_tq_xmit(skb, tq, adapter, netdev);
888 }
889
890
891 static void
892 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
893                 struct sk_buff *skb,
894                 union Vmxnet3_GenericDesc *gdesc)
895 {
896         if (!gdesc->rcd.cnc && adapter->rxcsum) {
897                 /* typical case: TCP/UDP over IP and both csums are correct */
898                 if ((gdesc->dword[3] & VMXNET3_RCD_CSUM_OK) ==
899                                                         VMXNET3_RCD_CSUM_OK) {
900                         skb->ip_summed = CHECKSUM_UNNECESSARY;
901                         BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
902                         BUG_ON(!(gdesc->rcd.v4  || gdesc->rcd.v6));
903                         BUG_ON(gdesc->rcd.frg);
904                 } else {
905                         if (gdesc->rcd.csum) {
906                                 skb->csum = htons(gdesc->rcd.csum);
907                                 skb->ip_summed = CHECKSUM_PARTIAL;
908                         } else {
909                                 skb->ip_summed = CHECKSUM_NONE;
910                         }
911                 }
912         } else {
913                 skb->ip_summed = CHECKSUM_NONE;
914         }
915 }
916
917
918 static void
919 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
920                  struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
921 {
922         rq->stats.drop_err++;
923         if (!rcd->fcs)
924                 rq->stats.drop_fcs++;
925
926         rq->stats.drop_total++;
927
928         /*
929          * We do not unmap and chain the rx buffer to the skb.
930          * We basically pretend this buffer is not used and will be recycled
931          * by vmxnet3_rq_alloc_rx_buf()
932          */
933
934         /*
935          * ctx->skb may be NULL if this is the first and the only one
936          * desc for the pkt
937          */
938         if (ctx->skb)
939                 dev_kfree_skb_irq(ctx->skb);
940
941         ctx->skb = NULL;
942 }
943
944
945 static int
946 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
947                        struct vmxnet3_adapter *adapter, int quota)
948 {
949         static u32 rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
950         u32 num_rxd = 0;
951         struct Vmxnet3_RxCompDesc *rcd;
952         struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
953
954         rcd = &rq->comp_ring.base[rq->comp_ring.next2proc].rcd;
955         while (rcd->gen == rq->comp_ring.gen) {
956                 struct vmxnet3_rx_buf_info *rbi;
957                 struct sk_buff *skb;
958                 int num_to_alloc;
959                 struct Vmxnet3_RxDesc *rxd;
960                 u32 idx, ring_idx;
961
962                 if (num_rxd >= quota) {
963                         /* we may stop even before we see the EOP desc of
964                          * the current pkt
965                          */
966                         break;
967                 }
968                 num_rxd++;
969
970                 idx = rcd->rxdIdx;
971                 ring_idx = rcd->rqID == rq->qid ? 0 : 1;
972
973                 rxd = &rq->rx_ring[ring_idx].base[idx].rxd;
974                 rbi = rq->buf_info[ring_idx] + idx;
975
976                 BUG_ON(rxd->addr != rbi->dma_addr || rxd->len != rbi->len);
977
978                 if (unlikely(rcd->eop && rcd->err)) {
979                         vmxnet3_rx_error(rq, rcd, ctx, adapter);
980                         goto rcd_done;
981                 }
982
983                 if (rcd->sop) { /* first buf of the pkt */
984                         BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
985                                rcd->rqID != rq->qid);
986
987                         BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
988                         BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
989
990                         if (unlikely(rcd->len == 0)) {
991                                 /* Pretend the rx buffer is skipped. */
992                                 BUG_ON(!(rcd->sop && rcd->eop));
993                                 dprintk(KERN_ERR "rxRing[%u][%u] 0 length\n",
994                                         ring_idx, idx);
995                                 goto rcd_done;
996                         }
997
998                         ctx->skb = rbi->skb;
999                         rbi->skb = NULL;
1000
1001                         pci_unmap_single(adapter->pdev, rbi->dma_addr, rbi->len,
1002                                          PCI_DMA_FROMDEVICE);
1003
1004                         skb_put(ctx->skb, rcd->len);
1005                 } else {
1006                         BUG_ON(ctx->skb == NULL);
1007                         /* non SOP buffer must be type 1 in most cases */
1008                         if (rbi->buf_type == VMXNET3_RX_BUF_PAGE) {
1009                                 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1010
1011                                 if (rcd->len) {
1012                                         pci_unmap_page(adapter->pdev,
1013                                                        rbi->dma_addr, rbi->len,
1014                                                        PCI_DMA_FROMDEVICE);
1015
1016                                         vmxnet3_append_frag(ctx->skb, rcd, rbi);
1017                                         rbi->page = NULL;
1018                                 }
1019                         } else {
1020                                 /*
1021                                  * The only time a non-SOP buffer is type 0 is
1022                                  * when it's EOP and error flag is raised, which
1023                                  * has already been handled.
1024                                  */
1025                                 BUG_ON(true);
1026                         }
1027                 }
1028
1029                 skb = ctx->skb;
1030                 if (rcd->eop) {
1031                         skb->len += skb->data_len;
1032                         skb->truesize += skb->data_len;
1033
1034                         vmxnet3_rx_csum(adapter, skb,
1035                                         (union Vmxnet3_GenericDesc *)rcd);
1036                         skb->protocol = eth_type_trans(skb, adapter->netdev);
1037
1038                         if (unlikely(adapter->vlan_grp && rcd->ts)) {
1039                                 vlan_hwaccel_receive_skb(skb,
1040                                                 adapter->vlan_grp, rcd->tci);
1041                         } else {
1042                                 netif_receive_skb(skb);
1043                         }
1044
1045                         adapter->netdev->last_rx = jiffies;
1046                         ctx->skb = NULL;
1047                 }
1048
1049 rcd_done:
1050                 /* device may skip some rx descs */
1051                 rq->rx_ring[ring_idx].next2comp = idx;
1052                 VMXNET3_INC_RING_IDX_ONLY(rq->rx_ring[ring_idx].next2comp,
1053                                           rq->rx_ring[ring_idx].size);
1054
1055                 /* refill rx buffers frequently to avoid starving the h/w */
1056                 num_to_alloc = vmxnet3_cmd_ring_desc_avail(rq->rx_ring +
1057                                                            ring_idx);
1058                 if (unlikely(num_to_alloc > VMXNET3_RX_ALLOC_THRESHOLD(rq,
1059                                                         ring_idx, adapter))) {
1060                         vmxnet3_rq_alloc_rx_buf(rq, ring_idx, num_to_alloc,
1061                                                 adapter);
1062
1063                         /* if needed, update the register */
1064                         if (unlikely(rq->shared->updateRxProd)) {
1065                                 VMXNET3_WRITE_BAR0_REG(adapter,
1066                                         rxprod_reg[ring_idx] + rq->qid * 8,
1067                                         rq->rx_ring[ring_idx].next2fill);
1068                                 rq->uncommitted[ring_idx] = 0;
1069                         }
1070                 }
1071
1072                 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1073                 rcd = &rq->comp_ring.base[rq->comp_ring.next2proc].rcd;
1074         }
1075
1076         return num_rxd;
1077 }
1078
1079
1080 static void
1081 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1082                    struct vmxnet3_adapter *adapter)
1083 {
1084         u32 i, ring_idx;
1085         struct Vmxnet3_RxDesc *rxd;
1086
1087         for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1088                 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1089                         rxd = &rq->rx_ring[ring_idx].base[i].rxd;
1090
1091                         if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1092                                         rq->buf_info[ring_idx][i].skb) {
1093                                 pci_unmap_single(adapter->pdev, rxd->addr,
1094                                                  rxd->len, PCI_DMA_FROMDEVICE);
1095                                 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1096                                 rq->buf_info[ring_idx][i].skb = NULL;
1097                         } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1098                                         rq->buf_info[ring_idx][i].page) {
1099                                 pci_unmap_page(adapter->pdev, rxd->addr,
1100                                                rxd->len, PCI_DMA_FROMDEVICE);
1101                                 put_page(rq->buf_info[ring_idx][i].page);
1102                                 rq->buf_info[ring_idx][i].page = NULL;
1103                         }
1104                 }
1105
1106                 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1107                 rq->rx_ring[ring_idx].next2fill =
1108                                         rq->rx_ring[ring_idx].next2comp = 0;
1109                 rq->uncommitted[ring_idx] = 0;
1110         }
1111
1112         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1113         rq->comp_ring.next2proc = 0;
1114 }
1115
1116
1117 void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1118                         struct vmxnet3_adapter *adapter)
1119 {
1120         int i;
1121         int j;
1122
1123         /* all rx buffers must have already been freed */
1124         for (i = 0; i < 2; i++) {
1125                 if (rq->buf_info[i]) {
1126                         for (j = 0; j < rq->rx_ring[i].size; j++)
1127                                 BUG_ON(rq->buf_info[i][j].page != NULL);
1128                 }
1129         }
1130
1131
1132         kfree(rq->buf_info[0]);
1133
1134         for (i = 0; i < 2; i++) {
1135                 if (rq->rx_ring[i].base) {
1136                         pci_free_consistent(adapter->pdev, rq->rx_ring[i].size
1137                                             * sizeof(struct Vmxnet3_RxDesc),
1138                                             rq->rx_ring[i].base,
1139                                             rq->rx_ring[i].basePA);
1140                         rq->rx_ring[i].base = NULL;
1141                 }
1142                 rq->buf_info[i] = NULL;
1143         }
1144
1145         if (rq->comp_ring.base) {
1146                 pci_free_consistent(adapter->pdev, rq->comp_ring.size *
1147                                     sizeof(struct Vmxnet3_RxCompDesc),
1148                                     rq->comp_ring.base, rq->comp_ring.basePA);
1149                 rq->comp_ring.base = NULL;
1150         }
1151 }
1152
1153
1154 static int
1155 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1156                 struct vmxnet3_adapter  *adapter)
1157 {
1158         int i;
1159
1160         /* initialize buf_info */
1161         for (i = 0; i < rq->rx_ring[0].size; i++) {
1162
1163                 /* 1st buf for a pkt is skbuff */
1164                 if (i % adapter->rx_buf_per_pkt == 0) {
1165                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1166                         rq->buf_info[0][i].len = adapter->skb_buf_size;
1167                 } else { /* subsequent bufs for a pkt is frag */
1168                         rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1169                         rq->buf_info[0][i].len = PAGE_SIZE;
1170                 }
1171         }
1172         for (i = 0; i < rq->rx_ring[1].size; i++) {
1173                 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1174                 rq->buf_info[1][i].len = PAGE_SIZE;
1175         }
1176
1177         /* reset internal state and allocate buffers for both rings */
1178         for (i = 0; i < 2; i++) {
1179                 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1180                 rq->uncommitted[i] = 0;
1181
1182                 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1183                        sizeof(struct Vmxnet3_RxDesc));
1184                 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1185         }
1186         if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1187                                     adapter) == 0) {
1188                 /* at least has 1 rx buffer for the 1st ring */
1189                 return -ENOMEM;
1190         }
1191         vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1192
1193         /* reset the comp ring */
1194         rq->comp_ring.next2proc = 0;
1195         memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1196                sizeof(struct Vmxnet3_RxCompDesc));
1197         rq->comp_ring.gen = VMXNET3_INIT_GEN;
1198
1199         /* reset rxctx */
1200         rq->rx_ctx.skb = NULL;
1201
1202         /* stats are not reset */
1203         return 0;
1204 }
1205
1206
1207 static int
1208 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1209 {
1210         int i;
1211         size_t sz;
1212         struct vmxnet3_rx_buf_info *bi;
1213
1214         for (i = 0; i < 2; i++) {
1215
1216                 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1217                 rq->rx_ring[i].base = pci_alloc_consistent(adapter->pdev, sz,
1218                                                         &rq->rx_ring[i].basePA);
1219                 if (!rq->rx_ring[i].base) {
1220                         printk(KERN_ERR "%s: failed to allocate rx ring %d\n",
1221                                adapter->netdev->name, i);
1222                         goto err;
1223                 }
1224         }
1225
1226         sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1227         rq->comp_ring.base = pci_alloc_consistent(adapter->pdev, sz,
1228                                                   &rq->comp_ring.basePA);
1229         if (!rq->comp_ring.base) {
1230                 printk(KERN_ERR "%s: failed to allocate rx comp ring\n",
1231                        adapter->netdev->name);
1232                 goto err;
1233         }
1234
1235         sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1236                                                    rq->rx_ring[1].size);
1237         bi = kmalloc(sz, GFP_KERNEL);
1238         if (!bi) {
1239                 printk(KERN_ERR "%s: failed to allocate rx bufinfo\n",
1240                        adapter->netdev->name);
1241                 goto err;
1242         }
1243         memset(bi, 0, sz);
1244         rq->buf_info[0] = bi;
1245         rq->buf_info[1] = bi + rq->rx_ring[0].size;
1246
1247         return 0;
1248
1249 err:
1250         vmxnet3_rq_destroy(rq, adapter);
1251         return -ENOMEM;
1252 }
1253
1254
1255 static int
1256 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1257 {
1258         if (unlikely(adapter->shared->ecr))
1259                 vmxnet3_process_events(adapter);
1260
1261         vmxnet3_tq_tx_complete(&adapter->tx_queue, adapter);
1262         return vmxnet3_rq_rx_complete(&adapter->rx_queue, adapter, budget);
1263 }
1264
1265
1266 static int
1267 vmxnet3_poll(struct napi_struct *napi, int budget)
1268 {
1269         struct vmxnet3_adapter *adapter = container_of(napi,
1270                                           struct vmxnet3_adapter, napi);
1271         int rxd_done;
1272
1273         rxd_done = vmxnet3_do_poll(adapter, budget);
1274
1275         if (rxd_done < budget) {
1276                 napi_complete(napi);
1277                 vmxnet3_enable_intr(adapter, 0);
1278         }
1279         return rxd_done;
1280 }
1281
1282
1283 /* Interrupt handler for vmxnet3  */
1284 static irqreturn_t
1285 vmxnet3_intr(int irq, void *dev_id)
1286 {
1287         struct net_device *dev = dev_id;
1288         struct vmxnet3_adapter *adapter = netdev_priv(dev);
1289
1290         if (unlikely(adapter->intr.type == VMXNET3_IT_INTX)) {
1291                 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1292                 if (unlikely(icr == 0))
1293                         /* not ours */
1294                         return IRQ_NONE;
1295         }
1296
1297
1298         /* disable intr if needed */
1299         if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1300                 vmxnet3_disable_intr(adapter, 0);
1301
1302         napi_schedule(&adapter->napi);
1303
1304         return IRQ_HANDLED;
1305 }
1306
1307 #ifdef CONFIG_NET_POLL_CONTROLLER
1308
1309
1310 /* netpoll callback. */
1311 static void
1312 vmxnet3_netpoll(struct net_device *netdev)
1313 {
1314         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1315         int irq;
1316
1317 #ifdef CONFIG_PCI_MSI
1318         if (adapter->intr.type == VMXNET3_IT_MSIX)
1319                 irq = adapter->intr.msix_entries[0].vector;
1320         else
1321 #endif
1322                 irq = adapter->pdev->irq;
1323
1324         disable_irq(irq);
1325         vmxnet3_intr(irq, netdev);
1326         enable_irq(irq);
1327 }
1328 #endif
1329
1330 static int
1331 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1332 {
1333         int err;
1334
1335 #ifdef CONFIG_PCI_MSI
1336         if (adapter->intr.type == VMXNET3_IT_MSIX) {
1337                 /* we only use 1 MSI-X vector */
1338                 err = request_irq(adapter->intr.msix_entries[0].vector,
1339                                   vmxnet3_intr, 0, adapter->netdev->name,
1340                                   adapter->netdev);
1341         } else
1342 #endif
1343         if (adapter->intr.type == VMXNET3_IT_MSI) {
1344                 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1345                                   adapter->netdev->name, adapter->netdev);
1346         } else {
1347                 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1348                                   IRQF_SHARED, adapter->netdev->name,
1349                                   adapter->netdev);
1350         }
1351
1352         if (err)
1353                 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error"
1354                        ":%d\n", adapter->netdev->name, adapter->intr.type, err);
1355
1356
1357         if (!err) {
1358                 int i;
1359                 /* init our intr settings */
1360                 for (i = 0; i < adapter->intr.num_intrs; i++)
1361                         adapter->intr.mod_levels[i] = UPT1_IML_ADAPTIVE;
1362
1363                 /* next setup intr index for all intr sources */
1364                 adapter->tx_queue.comp_ring.intr_idx = 0;
1365                 adapter->rx_queue.comp_ring.intr_idx = 0;
1366                 adapter->intr.event_intr_idx = 0;
1367
1368                 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors "
1369                        "allocated\n", adapter->netdev->name, adapter->intr.type,
1370                        adapter->intr.mask_mode, adapter->intr.num_intrs);
1371         }
1372
1373         return err;
1374 }
1375
1376
1377 static void
1378 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1379 {
1380         BUG_ON(adapter->intr.type == VMXNET3_IT_AUTO ||
1381                adapter->intr.num_intrs <= 0);
1382
1383         switch (adapter->intr.type) {
1384 #ifdef CONFIG_PCI_MSI
1385         case VMXNET3_IT_MSIX:
1386         {
1387                 int i;
1388
1389                 for (i = 0; i < adapter->intr.num_intrs; i++)
1390                         free_irq(adapter->intr.msix_entries[i].vector,
1391                                  adapter->netdev);
1392                 break;
1393         }
1394 #endif
1395         case VMXNET3_IT_MSI:
1396                 free_irq(adapter->pdev->irq, adapter->netdev);
1397                 break;
1398         case VMXNET3_IT_INTX:
1399                 free_irq(adapter->pdev->irq, adapter->netdev);
1400                 break;
1401         default:
1402                 BUG_ON(true);
1403         }
1404 }
1405
1406
1407 static void
1408 vmxnet3_vlan_rx_register(struct net_device *netdev, struct vlan_group *grp)
1409 {
1410         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1411         struct Vmxnet3_DriverShared *shared = adapter->shared;
1412         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1413
1414         if (grp) {
1415                 /* add vlan rx stripping. */
1416                 if (adapter->netdev->features & NETIF_F_HW_VLAN_RX) {
1417                         int i;
1418                         struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
1419                         adapter->vlan_grp = grp;
1420
1421                         /* update FEATURES to device */
1422                         devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
1423                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1424                                                VMXNET3_CMD_UPDATE_FEATURE);
1425                         /*
1426                          *  Clear entire vfTable; then enable untagged pkts.
1427                          *  Note: setting one entry in vfTable to non-zero turns
1428                          *  on VLAN rx filtering.
1429                          */
1430                         for (i = 0; i < VMXNET3_VFT_SIZE; i++)
1431                                 vfTable[i] = 0;
1432
1433                         VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1434                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1435                                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1436                 } else {
1437                         printk(KERN_ERR "%s: vlan_rx_register when device has "
1438                                "no NETIF_F_HW_VLAN_RX\n", netdev->name);
1439                 }
1440         } else {
1441                 /* remove vlan rx stripping. */
1442                 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
1443                 adapter->vlan_grp = NULL;
1444
1445                 if (devRead->misc.uptFeatures & UPT1_F_RXVLAN) {
1446                         int i;
1447
1448                         for (i = 0; i < VMXNET3_VFT_SIZE; i++) {
1449                                 /* clear entire vfTable; this also disables
1450                                  * VLAN rx filtering
1451                                  */
1452                                 vfTable[i] = 0;
1453                         }
1454                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1455                                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1456
1457                         /* update FEATURES to device */
1458                         devRead->misc.uptFeatures &= ~UPT1_F_RXVLAN;
1459                         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1460                                                VMXNET3_CMD_UPDATE_FEATURE);
1461                 }
1462         }
1463 }
1464
1465
1466 static void
1467 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1468 {
1469         if (adapter->vlan_grp) {
1470                 u16 vid;
1471                 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1472                 bool activeVlan = false;
1473
1474                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
1475                         if (vlan_group_get_device(adapter->vlan_grp, vid)) {
1476                                 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1477                                 activeVlan = true;
1478                         }
1479                 }
1480                 if (activeVlan) {
1481                         /* continue to allow untagged pkts */
1482                         VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1483                 }
1484         }
1485 }
1486
1487
1488 static void
1489 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1490 {
1491         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1492         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1493
1494         VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1495         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1496                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1497 }
1498
1499
1500 static void
1501 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1502 {
1503         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1504         u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1505
1506         VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1507         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1508                                VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1509 }
1510
1511
1512 static u8 *
1513 vmxnet3_copy_mc(struct net_device *netdev)
1514 {
1515         u8 *buf = NULL;
1516         u32 sz = netdev->mc_count * ETH_ALEN;
1517
1518         /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
1519         if (sz <= 0xffff) {
1520                 /* We may be called with BH disabled */
1521                 buf = kmalloc(sz, GFP_ATOMIC);
1522                 if (buf) {
1523                         int i;
1524                         struct dev_mc_list *mc = netdev->mc_list;
1525
1526                         for (i = 0; i < netdev->mc_count; i++) {
1527                                 BUG_ON(!mc);
1528                                 memcpy(buf + i * ETH_ALEN, mc->dmi_addr,
1529                                        ETH_ALEN);
1530                                 mc = mc->next;
1531                         }
1532                 }
1533         }
1534         return buf;
1535 }
1536
1537
1538 static void
1539 vmxnet3_set_mc(struct net_device *netdev)
1540 {
1541         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1542         struct Vmxnet3_RxFilterConf *rxConf =
1543                                         &adapter->shared->devRead.rxFilterConf;
1544         u8 *new_table = NULL;
1545         u32 new_mode = VMXNET3_RXM_UCAST;
1546
1547         if (netdev->flags & IFF_PROMISC)
1548                 new_mode |= VMXNET3_RXM_PROMISC;
1549
1550         if (netdev->flags & IFF_BROADCAST)
1551                 new_mode |= VMXNET3_RXM_BCAST;
1552
1553         if (netdev->flags & IFF_ALLMULTI)
1554                 new_mode |= VMXNET3_RXM_ALL_MULTI;
1555         else
1556                 if (netdev->mc_count > 0) {
1557                         new_table = vmxnet3_copy_mc(netdev);
1558                         if (new_table) {
1559                                 new_mode |= VMXNET3_RXM_MCAST;
1560                                 rxConf->mfTableLen = netdev->mc_count *
1561                                                      ETH_ALEN;
1562                                 rxConf->mfTablePA = virt_to_phys(new_table);
1563                         } else {
1564                                 printk(KERN_INFO "%s: failed to copy mcast list"
1565                                        ", setting ALL_MULTI\n", netdev->name);
1566                                 new_mode |= VMXNET3_RXM_ALL_MULTI;
1567                         }
1568                 }
1569
1570
1571         if (!(new_mode & VMXNET3_RXM_MCAST)) {
1572                 rxConf->mfTableLen = 0;
1573                 rxConf->mfTablePA = 0;
1574         }
1575
1576         if (new_mode != rxConf->rxMode) {
1577                 rxConf->rxMode = new_mode;
1578                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1579                                        VMXNET3_CMD_UPDATE_RX_MODE);
1580         }
1581
1582         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1583                                VMXNET3_CMD_UPDATE_MAC_FILTERS);
1584
1585         kfree(new_table);
1586 }
1587
1588
1589 /*
1590  *   Set up driver_shared based on settings in adapter.
1591  */
1592
1593 static void
1594 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
1595 {
1596         struct Vmxnet3_DriverShared *shared = adapter->shared;
1597         struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
1598         struct Vmxnet3_TxQueueConf *tqc;
1599         struct Vmxnet3_RxQueueConf *rqc;
1600         int i;
1601
1602         memset(shared, 0, sizeof(*shared));
1603
1604         /* driver settings */
1605         shared->magic = VMXNET3_REV1_MAGIC;
1606         devRead->misc.driverInfo.version = VMXNET3_DRIVER_VERSION_NUM;
1607         devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
1608                                 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
1609         devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
1610         devRead->misc.driverInfo.vmxnet3RevSpt = 1;
1611         devRead->misc.driverInfo.uptVerSpt = 1;
1612
1613         devRead->misc.ddPA = virt_to_phys(adapter);
1614         devRead->misc.ddLen = sizeof(struct vmxnet3_adapter);
1615
1616         /* set up feature flags */
1617         if (adapter->rxcsum)
1618                 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
1619
1620         if (adapter->lro) {
1621                 devRead->misc.uptFeatures |= UPT1_F_LRO;
1622                 devRead->misc.maxNumRxSG = 1 + MAX_SKB_FRAGS;
1623         }
1624         if ((adapter->netdev->features & NETIF_F_HW_VLAN_RX)
1625                         && adapter->vlan_grp) {
1626                 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
1627         }
1628
1629         devRead->misc.mtu = adapter->netdev->mtu;
1630         devRead->misc.queueDescPA = adapter->queue_desc_pa;
1631         devRead->misc.queueDescLen = sizeof(struct Vmxnet3_TxQueueDesc) +
1632                                      sizeof(struct Vmxnet3_RxQueueDesc);
1633
1634         /* tx queue settings */
1635         BUG_ON(adapter->tx_queue.tx_ring.base == NULL);
1636
1637         devRead->misc.numTxQueues = 1;
1638         tqc = &adapter->tqd_start->conf;
1639         tqc->txRingBasePA   = adapter->tx_queue.tx_ring.basePA;
1640         tqc->dataRingBasePA = adapter->tx_queue.data_ring.basePA;
1641         tqc->compRingBasePA = adapter->tx_queue.comp_ring.basePA;
1642         tqc->ddPA           = virt_to_phys(adapter->tx_queue.buf_info);
1643         tqc->txRingSize     = adapter->tx_queue.tx_ring.size;
1644         tqc->dataRingSize   = adapter->tx_queue.data_ring.size;
1645         tqc->compRingSize   = adapter->tx_queue.comp_ring.size;
1646         tqc->ddLen          = sizeof(struct vmxnet3_tx_buf_info) *
1647                               tqc->txRingSize;
1648         tqc->intrIdx        = adapter->tx_queue.comp_ring.intr_idx;
1649
1650         /* rx queue settings */
1651         devRead->misc.numRxQueues = 1;
1652         rqc = &adapter->rqd_start->conf;
1653         rqc->rxRingBasePA[0] = adapter->rx_queue.rx_ring[0].basePA;
1654         rqc->rxRingBasePA[1] = adapter->rx_queue.rx_ring[1].basePA;
1655         rqc->compRingBasePA  = adapter->rx_queue.comp_ring.basePA;
1656         rqc->ddPA            = virt_to_phys(adapter->rx_queue.buf_info);
1657         rqc->rxRingSize[0]   = adapter->rx_queue.rx_ring[0].size;
1658         rqc->rxRingSize[1]   = adapter->rx_queue.rx_ring[1].size;
1659         rqc->compRingSize    = adapter->rx_queue.comp_ring.size;
1660         rqc->ddLen           = sizeof(struct vmxnet3_rx_buf_info) *
1661                                (rqc->rxRingSize[0] + rqc->rxRingSize[1]);
1662         rqc->intrIdx         = adapter->rx_queue.comp_ring.intr_idx;
1663
1664         /* intr settings */
1665         devRead->intrConf.autoMask = adapter->intr.mask_mode ==
1666                                      VMXNET3_IMM_AUTO;
1667         devRead->intrConf.numIntrs = adapter->intr.num_intrs;
1668         for (i = 0; i < adapter->intr.num_intrs; i++)
1669                 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
1670
1671         devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
1672
1673         /* rx filter settings */
1674         devRead->rxFilterConf.rxMode = 0;
1675         vmxnet3_restore_vlan(adapter);
1676         /* the rest are already zeroed */
1677 }
1678
1679
1680 int
1681 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
1682 {
1683         int err;
1684         u32 ret;
1685
1686         dprintk(KERN_ERR "%s: skb_buf_size %d, rx_buf_per_pkt %d, ring sizes"
1687                 " %u %u %u\n", adapter->netdev->name, adapter->skb_buf_size,
1688                 adapter->rx_buf_per_pkt, adapter->tx_queue.tx_ring.size,
1689                 adapter->rx_queue.rx_ring[0].size,
1690                 adapter->rx_queue.rx_ring[1].size);
1691
1692         vmxnet3_tq_init(&adapter->tx_queue, adapter);
1693         err = vmxnet3_rq_init(&adapter->rx_queue, adapter);
1694         if (err) {
1695                 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n",
1696                        adapter->netdev->name, err);
1697                 goto rq_err;
1698         }
1699
1700         err = vmxnet3_request_irqs(adapter);
1701         if (err) {
1702                 printk(KERN_ERR "Failed to setup irq for %s: error %d\n",
1703                        adapter->netdev->name, err);
1704                 goto irq_err;
1705         }
1706
1707         vmxnet3_setup_driver_shared(adapter);
1708
1709         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL,
1710                                VMXNET3_GET_ADDR_LO(adapter->shared_pa));
1711         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH,
1712                                VMXNET3_GET_ADDR_HI(adapter->shared_pa));
1713
1714         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1715                                VMXNET3_CMD_ACTIVATE_DEV);
1716         ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
1717
1718         if (ret != 0) {
1719                 printk(KERN_ERR "Failed to activate dev %s: error %u\n",
1720                        adapter->netdev->name, ret);
1721                 err = -EINVAL;
1722                 goto activate_err;
1723         }
1724         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD,
1725                                adapter->rx_queue.rx_ring[0].next2fill);
1726         VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_RXPROD2,
1727                                adapter->rx_queue.rx_ring[1].next2fill);
1728
1729         /* Apply the rx filter settins last. */
1730         vmxnet3_set_mc(adapter->netdev);
1731
1732         /*
1733          * Check link state when first activating device. It will start the
1734          * tx queue if the link is up.
1735          */
1736         vmxnet3_check_link(adapter);
1737
1738         napi_enable(&adapter->napi);
1739         vmxnet3_enable_all_intrs(adapter);
1740         clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
1741         return 0;
1742
1743 activate_err:
1744         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
1745         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
1746         vmxnet3_free_irqs(adapter);
1747 irq_err:
1748 rq_err:
1749         /* free up buffers we allocated */
1750         vmxnet3_rq_cleanup(&adapter->rx_queue, adapter);
1751         return err;
1752 }
1753
1754
1755 void
1756 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
1757 {
1758         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
1759 }
1760
1761
1762 int
1763 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
1764 {
1765         if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
1766                 return 0;
1767
1768
1769         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1770                                VMXNET3_CMD_QUIESCE_DEV);
1771         vmxnet3_disable_all_intrs(adapter);
1772
1773         napi_disable(&adapter->napi);
1774         netif_tx_disable(adapter->netdev);
1775         adapter->link_speed = 0;
1776         netif_carrier_off(adapter->netdev);
1777
1778         vmxnet3_tq_cleanup(&adapter->tx_queue, adapter);
1779         vmxnet3_rq_cleanup(&adapter->rx_queue, adapter);
1780         vmxnet3_free_irqs(adapter);
1781         return 0;
1782 }
1783
1784
1785 static void
1786 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
1787 {
1788         u32 tmp;
1789
1790         tmp = *(u32 *)mac;
1791         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
1792
1793         tmp = (mac[5] << 8) | mac[4];
1794         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
1795 }
1796
1797
1798 static int
1799 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
1800 {
1801         struct sockaddr *addr = p;
1802         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1803
1804         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1805         vmxnet3_write_mac_addr(adapter, addr->sa_data);
1806
1807         return 0;
1808 }
1809
1810
1811 /* ==================== initialization and cleanup routines ============ */
1812
1813 static int
1814 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
1815 {
1816         int err;
1817         unsigned long mmio_start, mmio_len;
1818         struct pci_dev *pdev = adapter->pdev;
1819
1820         err = pci_enable_device(pdev);
1821         if (err) {
1822                 printk(KERN_ERR "Failed to enable adapter %s: error %d\n",
1823                        pci_name(pdev), err);
1824                 return err;
1825         }
1826
1827         if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
1828                 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
1829                         printk(KERN_ERR "pci_set_consistent_dma_mask failed "
1830                                "for adapter %s\n", pci_name(pdev));
1831                         err = -EIO;
1832                         goto err_set_mask;
1833                 }
1834                 *dma64 = true;
1835         } else {
1836                 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
1837                         printk(KERN_ERR "pci_set_dma_mask failed for adapter "
1838                                "%s\n",  pci_name(pdev));
1839                         err = -EIO;
1840                         goto err_set_mask;
1841                 }
1842                 *dma64 = false;
1843         }
1844
1845         err = pci_request_selected_regions(pdev, (1 << 2) - 1,
1846                                            vmxnet3_driver_name);
1847         if (err) {
1848                 printk(KERN_ERR "Failed to request region for adapter %s: "
1849                        "error %d\n", pci_name(pdev), err);
1850                 goto err_set_mask;
1851         }
1852
1853         pci_set_master(pdev);
1854
1855         mmio_start = pci_resource_start(pdev, 0);
1856         mmio_len = pci_resource_len(pdev, 0);
1857         adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
1858         if (!adapter->hw_addr0) {
1859                 printk(KERN_ERR "Failed to map bar0 for adapter %s\n",
1860                        pci_name(pdev));
1861                 err = -EIO;
1862                 goto err_ioremap;
1863         }
1864
1865         mmio_start = pci_resource_start(pdev, 1);
1866         mmio_len = pci_resource_len(pdev, 1);
1867         adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
1868         if (!adapter->hw_addr1) {
1869                 printk(KERN_ERR "Failed to map bar1 for adapter %s\n",
1870                        pci_name(pdev));
1871                 err = -EIO;
1872                 goto err_bar1;
1873         }
1874         return 0;
1875
1876 err_bar1:
1877         iounmap(adapter->hw_addr0);
1878 err_ioremap:
1879         pci_release_selected_regions(pdev, (1 << 2) - 1);
1880 err_set_mask:
1881         pci_disable_device(pdev);
1882         return err;
1883 }
1884
1885
1886 static void
1887 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
1888 {
1889         BUG_ON(!adapter->pdev);
1890
1891         iounmap(adapter->hw_addr0);
1892         iounmap(adapter->hw_addr1);
1893         pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
1894         pci_disable_device(adapter->pdev);
1895 }
1896
1897
1898 static void
1899 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
1900 {
1901         size_t sz;
1902
1903         if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
1904                                     VMXNET3_MAX_ETH_HDR_SIZE) {
1905                 adapter->skb_buf_size = adapter->netdev->mtu +
1906                                         VMXNET3_MAX_ETH_HDR_SIZE;
1907                 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
1908                         adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
1909
1910                 adapter->rx_buf_per_pkt = 1;
1911         } else {
1912                 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
1913                 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
1914                                             VMXNET3_MAX_ETH_HDR_SIZE;
1915                 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
1916         }
1917
1918         /*
1919          * for simplicity, force the ring0 size to be a multiple of
1920          * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
1921          */
1922         sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
1923         adapter->rx_queue.rx_ring[0].size = (adapter->rx_queue.rx_ring[0].size +
1924                                              sz - 1) / sz * sz;
1925         adapter->rx_queue.rx_ring[0].size = min_t(u32,
1926                                             adapter->rx_queue.rx_ring[0].size,
1927                                             VMXNET3_RX_RING_MAX_SIZE / sz * sz);
1928 }
1929
1930
1931 int
1932 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
1933                       u32 rx_ring_size, u32 rx_ring2_size)
1934 {
1935         int err;
1936
1937         adapter->tx_queue.tx_ring.size   = tx_ring_size;
1938         adapter->tx_queue.data_ring.size = tx_ring_size;
1939         adapter->tx_queue.comp_ring.size = tx_ring_size;
1940         adapter->tx_queue.shared = &adapter->tqd_start->ctrl;
1941         adapter->tx_queue.stopped = true;
1942         err = vmxnet3_tq_create(&adapter->tx_queue, adapter);
1943         if (err)
1944                 return err;
1945
1946         adapter->rx_queue.rx_ring[0].size = rx_ring_size;
1947         adapter->rx_queue.rx_ring[1].size = rx_ring2_size;
1948         vmxnet3_adjust_rx_ring_size(adapter);
1949         adapter->rx_queue.comp_ring.size  = adapter->rx_queue.rx_ring[0].size +
1950                                             adapter->rx_queue.rx_ring[1].size;
1951         adapter->rx_queue.qid  = 0;
1952         adapter->rx_queue.qid2 = 1;
1953         adapter->rx_queue.shared = &adapter->rqd_start->ctrl;
1954         err = vmxnet3_rq_create(&adapter->rx_queue, adapter);
1955         if (err)
1956                 vmxnet3_tq_destroy(&adapter->tx_queue, adapter);
1957
1958         return err;
1959 }
1960
1961 static int
1962 vmxnet3_open(struct net_device *netdev)
1963 {
1964         struct vmxnet3_adapter *adapter;
1965         int err;
1966
1967         adapter = netdev_priv(netdev);
1968
1969         spin_lock_init(&adapter->tx_queue.tx_lock);
1970
1971         err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
1972                                     VMXNET3_DEF_RX_RING_SIZE,
1973                                     VMXNET3_DEF_RX_RING_SIZE);
1974         if (err)
1975                 goto queue_err;
1976
1977         err = vmxnet3_activate_dev(adapter);
1978         if (err)
1979                 goto activate_err;
1980
1981         return 0;
1982
1983 activate_err:
1984         vmxnet3_rq_destroy(&adapter->rx_queue, adapter);
1985         vmxnet3_tq_destroy(&adapter->tx_queue, adapter);
1986 queue_err:
1987         return err;
1988 }
1989
1990
1991 static int
1992 vmxnet3_close(struct net_device *netdev)
1993 {
1994         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1995
1996         /*
1997          * Reset_work may be in the middle of resetting the device, wait for its
1998          * completion.
1999          */
2000         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2001                 msleep(1);
2002
2003         vmxnet3_quiesce_dev(adapter);
2004
2005         vmxnet3_rq_destroy(&adapter->rx_queue, adapter);
2006         vmxnet3_tq_destroy(&adapter->tx_queue, adapter);
2007
2008         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2009
2010
2011         return 0;
2012 }
2013
2014
2015 void
2016 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2017 {
2018         /*
2019          * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2020          * vmxnet3_close() will deadlock.
2021          */
2022         BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2023
2024         /* we need to enable NAPI, otherwise dev_close will deadlock */
2025         napi_enable(&adapter->napi);
2026         dev_close(adapter->netdev);
2027 }
2028
2029
2030 static int
2031 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2032 {
2033         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2034         int err = 0;
2035
2036         if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2037                 return -EINVAL;
2038
2039         if (new_mtu > 1500 && !adapter->jumbo_frame)
2040                 return -EINVAL;
2041
2042         netdev->mtu = new_mtu;
2043
2044         /*
2045          * Reset_work may be in the middle of resetting the device, wait for its
2046          * completion.
2047          */
2048         while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2049                 msleep(1);
2050
2051         if (netif_running(netdev)) {
2052                 vmxnet3_quiesce_dev(adapter);
2053                 vmxnet3_reset_dev(adapter);
2054
2055                 /* we need to re-create the rx queue based on the new mtu */
2056                 vmxnet3_rq_destroy(&adapter->rx_queue, adapter);
2057                 vmxnet3_adjust_rx_ring_size(adapter);
2058                 adapter->rx_queue.comp_ring.size  =
2059                                         adapter->rx_queue.rx_ring[0].size +
2060                                         adapter->rx_queue.rx_ring[1].size;
2061                 err = vmxnet3_rq_create(&adapter->rx_queue, adapter);
2062                 if (err) {
2063                         printk(KERN_ERR "%s: failed to re-create rx queue,"
2064                                 " error %d. Closing it.\n", netdev->name, err);
2065                         goto out;
2066                 }
2067
2068                 err = vmxnet3_activate_dev(adapter);
2069                 if (err) {
2070                         printk(KERN_ERR "%s: failed to re-activate, error %d. "
2071                                 "Closing it\n", netdev->name, err);
2072                         goto out;
2073                 }
2074         }
2075
2076 out:
2077         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2078         if (err)
2079                 vmxnet3_force_close(adapter);
2080
2081         return err;
2082 }
2083
2084
2085 static void
2086 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2087 {
2088         struct net_device *netdev = adapter->netdev;
2089
2090         netdev->features = NETIF_F_SG |
2091                 NETIF_F_HW_CSUM |
2092                 NETIF_F_HW_VLAN_TX |
2093                 NETIF_F_HW_VLAN_RX |
2094                 NETIF_F_HW_VLAN_FILTER |
2095                 NETIF_F_TSO |
2096                 NETIF_F_TSO6 |
2097                 NETIF_F_LRO;
2098
2099         printk(KERN_INFO "features: sg csum vlan jf tso tsoIPv6 lro");
2100
2101         adapter->rxcsum = true;
2102         adapter->jumbo_frame = true;
2103         adapter->lro = true;
2104
2105         if (dma64) {
2106                 netdev->features |= NETIF_F_HIGHDMA;
2107                 printk(" highDMA");
2108         }
2109
2110         netdev->vlan_features = netdev->features;
2111         printk("\n");
2112 }
2113
2114
2115 static void
2116 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2117 {
2118         u32 tmp;
2119
2120         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2121         *(u32 *)mac = tmp;
2122
2123         tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2124         mac[4] = tmp & 0xff;
2125         mac[5] = (tmp >> 8) & 0xff;
2126 }
2127
2128
2129 static void
2130 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2131 {
2132         u32 cfg;
2133
2134         /* intr settings */
2135         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2136                                VMXNET3_CMD_GET_CONF_INTR);
2137         cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2138         adapter->intr.type = cfg & 0x3;
2139         adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2140
2141         if (adapter->intr.type == VMXNET3_IT_AUTO) {
2142                 int err;
2143
2144 #ifdef CONFIG_PCI_MSI
2145                 adapter->intr.msix_entries[0].entry = 0;
2146                 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2147                                       VMXNET3_LINUX_MAX_MSIX_VECT);
2148                 if (!err) {
2149                         adapter->intr.num_intrs = 1;
2150                         adapter->intr.type = VMXNET3_IT_MSIX;
2151                         return;
2152                 }
2153 #endif
2154
2155                 err = pci_enable_msi(adapter->pdev);
2156                 if (!err) {
2157                         adapter->intr.num_intrs = 1;
2158                         adapter->intr.type = VMXNET3_IT_MSI;
2159                         return;
2160                 }
2161         }
2162
2163         adapter->intr.type = VMXNET3_IT_INTX;
2164
2165         /* INT-X related setting */
2166         adapter->intr.num_intrs = 1;
2167 }
2168
2169
2170 static void
2171 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2172 {
2173         if (adapter->intr.type == VMXNET3_IT_MSIX)
2174                 pci_disable_msix(adapter->pdev);
2175         else if (adapter->intr.type == VMXNET3_IT_MSI)
2176                 pci_disable_msi(adapter->pdev);
2177         else
2178                 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2179 }
2180
2181
2182 static void
2183 vmxnet3_tx_timeout(struct net_device *netdev)
2184 {
2185         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2186         adapter->tx_timeout_count++;
2187
2188         printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name);
2189         schedule_work(&adapter->work);
2190 }
2191
2192
2193 static void
2194 vmxnet3_reset_work(struct work_struct *data)
2195 {
2196         struct vmxnet3_adapter *adapter;
2197
2198         adapter = container_of(data, struct vmxnet3_adapter, work);
2199
2200         /* if another thread is resetting the device, no need to proceed */
2201         if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2202                 return;
2203
2204         /* if the device is closed, we must leave it alone */
2205         if (netif_running(adapter->netdev)) {
2206                 printk(KERN_INFO "%s: resetting\n", adapter->netdev->name);
2207                 vmxnet3_quiesce_dev(adapter);
2208                 vmxnet3_reset_dev(adapter);
2209                 vmxnet3_activate_dev(adapter);
2210         } else {
2211                 printk(KERN_INFO "%s: already closed\n", adapter->netdev->name);
2212         }
2213
2214         clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2215 }
2216
2217
2218 static int __devinit
2219 vmxnet3_probe_device(struct pci_dev *pdev,
2220                      const struct pci_device_id *id)
2221 {
2222         static const struct net_device_ops vmxnet3_netdev_ops = {
2223                 .ndo_open = vmxnet3_open,
2224                 .ndo_stop = vmxnet3_close,
2225                 .ndo_start_xmit = vmxnet3_xmit_frame,
2226                 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2227                 .ndo_change_mtu = vmxnet3_change_mtu,
2228                 .ndo_get_stats = vmxnet3_get_stats,
2229                 .ndo_tx_timeout = vmxnet3_tx_timeout,
2230                 .ndo_set_multicast_list = vmxnet3_set_mc,
2231                 .ndo_vlan_rx_register = vmxnet3_vlan_rx_register,
2232                 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2233                 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2234 #ifdef CONFIG_NET_POLL_CONTROLLER
2235                 .ndo_poll_controller = vmxnet3_netpoll,
2236 #endif
2237         };
2238         int err;
2239         bool dma64 = false; /* stupid gcc */
2240         u32 ver;
2241         struct net_device *netdev;
2242         struct vmxnet3_adapter *adapter;
2243         u8 mac[ETH_ALEN];
2244
2245         netdev = alloc_etherdev(sizeof(struct vmxnet3_adapter));
2246         if (!netdev) {
2247                 printk(KERN_ERR "Failed to alloc ethernet device for adapter "
2248                         "%s\n", pci_name(pdev));
2249                 return -ENOMEM;
2250         }
2251
2252         pci_set_drvdata(pdev, netdev);
2253         adapter = netdev_priv(netdev);
2254         adapter->netdev = netdev;
2255         adapter->pdev = pdev;
2256
2257         adapter->shared = pci_alloc_consistent(adapter->pdev,
2258                           sizeof(struct Vmxnet3_DriverShared),
2259                           &adapter->shared_pa);
2260         if (!adapter->shared) {
2261                 printk(KERN_ERR "Failed to allocate memory for %s\n",
2262                         pci_name(pdev));
2263                 err = -ENOMEM;
2264                 goto err_alloc_shared;
2265         }
2266
2267         adapter->tqd_start = pci_alloc_consistent(adapter->pdev,
2268                              sizeof(struct Vmxnet3_TxQueueDesc) +
2269                              sizeof(struct Vmxnet3_RxQueueDesc),
2270                              &adapter->queue_desc_pa);
2271
2272         if (!adapter->tqd_start) {
2273                 printk(KERN_ERR "Failed to allocate memory for %s\n",
2274                         pci_name(pdev));
2275                 err = -ENOMEM;
2276                 goto err_alloc_queue_desc;
2277         }
2278         adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start
2279                                                             + 1);
2280
2281         adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL);
2282         if (adapter->pm_conf == NULL) {
2283                 printk(KERN_ERR "Failed to allocate memory for %s\n",
2284                         pci_name(pdev));
2285                 err = -ENOMEM;
2286                 goto err_alloc_pm;
2287         }
2288
2289         err = vmxnet3_alloc_pci_resources(adapter, &dma64);
2290         if (err < 0)
2291                 goto err_alloc_pci;
2292
2293         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
2294         if (ver & 1) {
2295                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
2296         } else {
2297                 printk(KERN_ERR "Incompatible h/w version (0x%x) for adapter"
2298                        " %s\n", ver, pci_name(pdev));
2299                 err = -EBUSY;
2300                 goto err_ver;
2301         }
2302
2303         ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
2304         if (ver & 1) {
2305                 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
2306         } else {
2307                 printk(KERN_ERR "Incompatible upt version (0x%x) for "
2308                        "adapter %s\n", ver, pci_name(pdev));
2309                 err = -EBUSY;
2310                 goto err_ver;
2311         }
2312
2313         vmxnet3_declare_features(adapter, dma64);
2314
2315         adapter->dev_number = atomic_read(&devices_found);
2316         vmxnet3_alloc_intr_resources(adapter);
2317
2318         vmxnet3_read_mac_addr(adapter, mac);
2319         memcpy(netdev->dev_addr,  mac, netdev->addr_len);
2320
2321         netdev->netdev_ops = &vmxnet3_netdev_ops;
2322         netdev->watchdog_timeo = 5 * HZ;
2323         vmxnet3_set_ethtool_ops(netdev);
2324
2325         INIT_WORK(&adapter->work, vmxnet3_reset_work);
2326
2327         netif_napi_add(netdev, &adapter->napi, vmxnet3_poll, 64);
2328         SET_NETDEV_DEV(netdev, &pdev->dev);
2329         err = register_netdev(netdev);
2330
2331         if (err) {
2332                 printk(KERN_ERR "Failed to register adapter %s\n",
2333                         pci_name(pdev));
2334                 goto err_register;
2335         }
2336
2337         set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2338         atomic_inc(&devices_found);
2339         return 0;
2340
2341 err_register:
2342         vmxnet3_free_intr_resources(adapter);
2343 err_ver:
2344         vmxnet3_free_pci_resources(adapter);
2345 err_alloc_pci:
2346         kfree(adapter->pm_conf);
2347 err_alloc_pm:
2348         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) +
2349                             sizeof(struct Vmxnet3_RxQueueDesc),
2350                             adapter->tqd_start, adapter->queue_desc_pa);
2351 err_alloc_queue_desc:
2352         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2353                             adapter->shared, adapter->shared_pa);
2354 err_alloc_shared:
2355         pci_set_drvdata(pdev, NULL);
2356         free_netdev(netdev);
2357         return err;
2358 }
2359
2360
2361 static void __devexit
2362 vmxnet3_remove_device(struct pci_dev *pdev)
2363 {
2364         struct net_device *netdev = pci_get_drvdata(pdev);
2365         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2366
2367         flush_scheduled_work();
2368
2369         unregister_netdev(netdev);
2370
2371         vmxnet3_free_intr_resources(adapter);
2372         vmxnet3_free_pci_resources(adapter);
2373         kfree(adapter->pm_conf);
2374         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_TxQueueDesc) +
2375                             sizeof(struct Vmxnet3_RxQueueDesc),
2376                             adapter->tqd_start, adapter->queue_desc_pa);
2377         pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
2378                             adapter->shared, adapter->shared_pa);
2379         free_netdev(netdev);
2380 }
2381
2382
2383 #ifdef CONFIG_PM
2384
2385 static int
2386 vmxnet3_suspend(struct device *device)
2387 {
2388         struct pci_dev *pdev = to_pci_dev(device);
2389         struct net_device *netdev = pci_get_drvdata(pdev);
2390         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2391         struct Vmxnet3_PMConf *pmConf;
2392         struct ethhdr *ehdr;
2393         struct arphdr *ahdr;
2394         u8 *arpreq;
2395         struct in_device *in_dev;
2396         struct in_ifaddr *ifa;
2397         int i = 0;
2398
2399         if (!netif_running(netdev))
2400                 return 0;
2401
2402         vmxnet3_disable_all_intrs(adapter);
2403         vmxnet3_free_irqs(adapter);
2404         vmxnet3_free_intr_resources(adapter);
2405
2406         netif_device_detach(netdev);
2407         netif_stop_queue(netdev);
2408
2409         /* Create wake-up filters. */
2410         pmConf = adapter->pm_conf;
2411         memset(pmConf, 0, sizeof(*pmConf));
2412
2413         if (adapter->wol & WAKE_UCAST) {
2414                 pmConf->filters[i].patternSize = ETH_ALEN;
2415                 pmConf->filters[i].maskSize = 1;
2416                 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
2417                 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
2418
2419                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
2420                 i++;
2421         }
2422
2423         if (adapter->wol & WAKE_ARP) {
2424                 in_dev = in_dev_get(netdev);
2425                 if (!in_dev)
2426                         goto skip_arp;
2427
2428                 ifa = (struct in_ifaddr *)in_dev->ifa_list;
2429                 if (!ifa)
2430                         goto skip_arp;
2431
2432                 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
2433                         sizeof(struct arphdr) +         /* ARP header */
2434                         2 * ETH_ALEN +          /* 2 Ethernet addresses*/
2435                         2 * sizeof(u32);        /*2 IPv4 addresses */
2436                 pmConf->filters[i].maskSize =
2437                         (pmConf->filters[i].patternSize - 1) / 8 + 1;
2438
2439                 /* ETH_P_ARP in Ethernet header. */
2440                 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
2441                 ehdr->h_proto = htons(ETH_P_ARP);
2442
2443                 /* ARPOP_REQUEST in ARP header. */
2444                 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
2445                 ahdr->ar_op = htons(ARPOP_REQUEST);
2446                 arpreq = (u8 *)(ahdr + 1);
2447
2448                 /* The Unicast IPv4 address in 'tip' field. */
2449                 arpreq += 2 * ETH_ALEN + sizeof(u32);
2450                 *(u32 *)arpreq = ifa->ifa_address;
2451
2452                 /* The mask for the relevant bits. */
2453                 pmConf->filters[i].mask[0] = 0x00;
2454                 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
2455                 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
2456                 pmConf->filters[i].mask[3] = 0x00;
2457                 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
2458                 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
2459                 in_dev_put(in_dev);
2460
2461                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
2462                 i++;
2463         }
2464
2465 skip_arp:
2466         if (adapter->wol & WAKE_MAGIC)
2467                 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
2468
2469         pmConf->numFilters = i;
2470
2471         adapter->shared->devRead.pmConfDesc.confVer = 1;
2472         adapter->shared->devRead.pmConfDesc.confLen = sizeof(*pmConf);
2473         adapter->shared->devRead.pmConfDesc.confPA = virt_to_phys(pmConf);
2474
2475         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2476                                VMXNET3_CMD_UPDATE_PMCFG);
2477
2478         pci_save_state(pdev);
2479         pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
2480                         adapter->wol);
2481         pci_disable_device(pdev);
2482         pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
2483
2484         return 0;
2485 }
2486
2487
2488 static int
2489 vmxnet3_resume(struct device *device)
2490 {
2491         int err;
2492         struct pci_dev *pdev = to_pci_dev(device);
2493         struct net_device *netdev = pci_get_drvdata(pdev);
2494         struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2495         struct Vmxnet3_PMConf *pmConf;
2496
2497         if (!netif_running(netdev))
2498                 return 0;
2499
2500         /* Destroy wake-up filters. */
2501         pmConf = adapter->pm_conf;
2502         memset(pmConf, 0, sizeof(*pmConf));
2503
2504         adapter->shared->devRead.pmConfDesc.confVer = 1;
2505         adapter->shared->devRead.pmConfDesc.confLen = sizeof(*pmConf);
2506         adapter->shared->devRead.pmConfDesc.confPA = virt_to_phys(pmConf);
2507
2508         netif_device_attach(netdev);
2509         pci_set_power_state(pdev, PCI_D0);
2510         pci_restore_state(pdev);
2511         err = pci_enable_device_mem(pdev);
2512         if (err != 0)
2513                 return err;
2514
2515         pci_enable_wake(pdev, PCI_D0, 0);
2516
2517         VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2518                                VMXNET3_CMD_UPDATE_PMCFG);
2519         vmxnet3_alloc_intr_resources(adapter);
2520         vmxnet3_request_irqs(adapter);
2521         vmxnet3_enable_all_intrs(adapter);
2522
2523         return 0;
2524 }
2525
2526 static struct dev_pm_ops vmxnet3_pm_ops = {
2527         .suspend = vmxnet3_suspend,
2528         .resume = vmxnet3_resume,
2529 };
2530 #endif
2531
2532 static struct pci_driver vmxnet3_driver = {
2533         .name           = vmxnet3_driver_name,
2534         .id_table       = vmxnet3_pciid_table,
2535         .probe          = vmxnet3_probe_device,
2536         .remove         = __devexit_p(vmxnet3_remove_device),
2537 #ifdef CONFIG_PM
2538         .driver.pm      = &vmxnet3_pm_ops,
2539 #endif
2540 };
2541
2542
2543 static int __init
2544 vmxnet3_init_module(void)
2545 {
2546         printk(KERN_INFO "%s - version %s\n", VMXNET3_DRIVER_DESC,
2547                 VMXNET3_DRIVER_VERSION_REPORT);
2548         return pci_register_driver(&vmxnet3_driver);
2549 }
2550
2551 module_init(vmxnet3_init_module);
2552
2553
2554 static void
2555 vmxnet3_exit_module(void)
2556 {
2557         pci_unregister_driver(&vmxnet3_driver);
2558 }
2559
2560 module_exit(vmxnet3_exit_module);
2561
2562 MODULE_AUTHOR("VMware, Inc.");
2563 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
2564 MODULE_LICENSE("GPL v2");
2565 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);