Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net
[linux-3.10.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
1 /*******************************************************************************
2
3   Intel 82599 Virtual Function driver
4   Copyright(c) 1999 - 2012 Intel Corporation.
5
6   This program is free software; you can redistribute it and/or modify it
7   under the terms and conditions of the GNU General Public License,
8   version 2, as published by the Free Software Foundation.
9
10   This program is distributed in the hope it will be useful, but WITHOUT
11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13   more details.
14
15   You should have received a copy of the GNU General Public License along with
16   this program; if not, write to the Free Software Foundation, Inc.,
17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19   The full GNU General Public License is included in this distribution in
20   the file called "COPYING".
21
22   Contact Information:
23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28
29 /******************************************************************************
30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
31 ******************************************************************************/
32
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34
35 #include <linux/types.h>
36 #include <linux/bitops.h>
37 #include <linux/module.h>
38 #include <linux/pci.h>
39 #include <linux/netdevice.h>
40 #include <linux/vmalloc.h>
41 #include <linux/string.h>
42 #include <linux/in.h>
43 #include <linux/ip.h>
44 #include <linux/tcp.h>
45 #include <linux/sctp.h>
46 #include <linux/ipv6.h>
47 #include <linux/slab.h>
48 #include <net/checksum.h>
49 #include <net/ip6_checksum.h>
50 #include <linux/ethtool.h>
51 #include <linux/if.h>
52 #include <linux/if_vlan.h>
53 #include <linux/prefetch.h>
54
55 #include "ixgbevf.h"
56
57 const char ixgbevf_driver_name[] = "ixgbevf";
58 static const char ixgbevf_driver_string[] =
59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
60
61 #define DRV_VERSION "2.6.0-k"
62 const char ixgbevf_driver_version[] = DRV_VERSION;
63 static char ixgbevf_copyright[] =
64         "Copyright (c) 2009 - 2012 Intel Corporation.";
65
66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
67         [board_82599_vf] = &ixgbevf_82599_vf_info,
68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
69 };
70
71 /* ixgbevf_pci_tbl - PCI Device ID Table
72  *
73  * Wildcard entries (PCI_ANY_ID) should come last
74  * Last entry must be all 0s
75  *
76  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
77  *   Class, Class Mask, private data (not used) }
78  */
79 static struct pci_device_id ixgbevf_pci_tbl[] = {
80         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
81         board_82599_vf},
82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
83         board_X540_vf},
84
85         /* required last entry */
86         {0, }
87 };
88 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
89
90 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
91 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
92 MODULE_LICENSE("GPL");
93 MODULE_VERSION(DRV_VERSION);
94
95 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
96 static int debug = -1;
97 module_param(debug, int, 0);
98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
99
100 /* forward decls */
101 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
102
103 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
104                                            struct ixgbevf_ring *rx_ring,
105                                            u32 val)
106 {
107         /*
108          * Force memory writes to complete before letting h/w
109          * know there are new descriptors to fetch.  (Only
110          * applicable for weak-ordered memory model archs,
111          * such as IA-64).
112          */
113         wmb();
114         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
115 }
116
117 /**
118  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
119  * @adapter: pointer to adapter struct
120  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
121  * @queue: queue to map the corresponding interrupt to
122  * @msix_vector: the vector to map to the corresponding queue
123  *
124  */
125 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
126                              u8 queue, u8 msix_vector)
127 {
128         u32 ivar, index;
129         struct ixgbe_hw *hw = &adapter->hw;
130         if (direction == -1) {
131                 /* other causes */
132                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
133                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
134                 ivar &= ~0xFF;
135                 ivar |= msix_vector;
136                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
137         } else {
138                 /* tx or rx causes */
139                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
140                 index = ((16 * (queue & 1)) + (8 * direction));
141                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
142                 ivar &= ~(0xFF << index);
143                 ivar |= (msix_vector << index);
144                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
145         }
146 }
147
148 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
149                                                struct ixgbevf_tx_buffer
150                                                *tx_buffer_info)
151 {
152         if (tx_buffer_info->dma) {
153                 if (tx_buffer_info->mapped_as_page)
154                         dma_unmap_page(tx_ring->dev,
155                                        tx_buffer_info->dma,
156                                        tx_buffer_info->length,
157                                        DMA_TO_DEVICE);
158                 else
159                         dma_unmap_single(tx_ring->dev,
160                                          tx_buffer_info->dma,
161                                          tx_buffer_info->length,
162                                          DMA_TO_DEVICE);
163                 tx_buffer_info->dma = 0;
164         }
165         if (tx_buffer_info->skb) {
166                 dev_kfree_skb_any(tx_buffer_info->skb);
167                 tx_buffer_info->skb = NULL;
168         }
169         tx_buffer_info->time_stamp = 0;
170         /* tx_buffer_info must be completely set up in the transmit path */
171 }
172
173 #define IXGBE_MAX_TXD_PWR       14
174 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
175
176 /* Tx Descriptors needed, worst case */
177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
179
180 static void ixgbevf_tx_timeout(struct net_device *netdev);
181
182 /**
183  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
184  * @q_vector: board private structure
185  * @tx_ring: tx ring to clean
186  **/
187 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
188                                  struct ixgbevf_ring *tx_ring)
189 {
190         struct ixgbevf_adapter *adapter = q_vector->adapter;
191         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
192         struct ixgbevf_tx_buffer *tx_buffer_info;
193         unsigned int i, eop, count = 0;
194         unsigned int total_bytes = 0, total_packets = 0;
195
196         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
197                 return true;
198
199         i = tx_ring->next_to_clean;
200         eop = tx_ring->tx_buffer_info[i].next_to_watch;
201         eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
202
203         while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
204                (count < tx_ring->count)) {
205                 bool cleaned = false;
206                 rmb(); /* read buffer_info after eop_desc */
207                 /* eop could change between read and DD-check */
208                 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch))
209                         goto cont_loop;
210                 for ( ; !cleaned; count++) {
211                         struct sk_buff *skb;
212                         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
213                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
214                         cleaned = (i == eop);
215                         skb = tx_buffer_info->skb;
216
217                         if (cleaned && skb) {
218                                 unsigned int segs, bytecount;
219
220                                 /* gso_segs is currently only valid for tcp */
221                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
222                                 /* multiply data chunks by size of headers */
223                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
224                                             skb->len;
225                                 total_packets += segs;
226                                 total_bytes += bytecount;
227                         }
228
229                         ixgbevf_unmap_and_free_tx_resource(tx_ring,
230                                                            tx_buffer_info);
231
232                         tx_desc->wb.status = 0;
233
234                         i++;
235                         if (i == tx_ring->count)
236                                 i = 0;
237                 }
238
239 cont_loop:
240                 eop = tx_ring->tx_buffer_info[i].next_to_watch;
241                 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
242         }
243
244         tx_ring->next_to_clean = i;
245
246 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
247         if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
248                      (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
249                 /* Make sure that anybody stopping the queue after this
250                  * sees the new next_to_clean.
251                  */
252                 smp_mb();
253                 if (__netif_subqueue_stopped(tx_ring->netdev,
254                                              tx_ring->queue_index) &&
255                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
256                         netif_wake_subqueue(tx_ring->netdev,
257                                             tx_ring->queue_index);
258                         ++adapter->restart_queue;
259                 }
260         }
261
262         u64_stats_update_begin(&tx_ring->syncp);
263         tx_ring->total_bytes += total_bytes;
264         tx_ring->total_packets += total_packets;
265         u64_stats_update_end(&tx_ring->syncp);
266
267         return count < tx_ring->count;
268 }
269
270 /**
271  * ixgbevf_receive_skb - Send a completed packet up the stack
272  * @q_vector: structure containing interrupt and ring information
273  * @skb: packet to send up
274  * @status: hardware indication of status of receive
275  * @rx_ring: rx descriptor ring (for a specific queue) to setup
276  * @rx_desc: rx descriptor
277  **/
278 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
279                                 struct sk_buff *skb, u8 status,
280                                 struct ixgbevf_ring *ring,
281                                 union ixgbe_adv_rx_desc *rx_desc)
282 {
283         struct ixgbevf_adapter *adapter = q_vector->adapter;
284         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
285         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
286
287         if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
288                 __vlan_hwaccel_put_tag(skb, tag);
289
290         napi_gro_receive(&q_vector->napi, skb);
291 }
292
293 /**
294  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
295  * @adapter: address of board private structure
296  * @status_err: hardware indication of status of receive
297  * @skb: skb currently being received and modified
298  **/
299 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter,
300                                        struct ixgbevf_ring *ring,
301                                        u32 status_err, struct sk_buff *skb)
302 {
303         skb_checksum_none_assert(skb);
304
305         /* Rx csum disabled */
306         if (!(ring->netdev->features & NETIF_F_RXCSUM))
307                 return;
308
309         /* if IP and error */
310         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
311             (status_err & IXGBE_RXDADV_ERR_IPE)) {
312                 adapter->hw_csum_rx_error++;
313                 return;
314         }
315
316         if (!(status_err & IXGBE_RXD_STAT_L4CS))
317                 return;
318
319         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
320                 adapter->hw_csum_rx_error++;
321                 return;
322         }
323
324         /* It must be a TCP or UDP packet with a valid checksum */
325         skb->ip_summed = CHECKSUM_UNNECESSARY;
326         adapter->hw_csum_rx_good++;
327 }
328
329 /**
330  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
331  * @adapter: address of board private structure
332  **/
333 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
334                                      struct ixgbevf_ring *rx_ring,
335                                      int cleaned_count)
336 {
337         struct pci_dev *pdev = adapter->pdev;
338         union ixgbe_adv_rx_desc *rx_desc;
339         struct ixgbevf_rx_buffer *bi;
340         struct sk_buff *skb;
341         unsigned int i = rx_ring->next_to_use;
342
343         bi = &rx_ring->rx_buffer_info[i];
344
345         while (cleaned_count--) {
346                 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
347                 skb = bi->skb;
348                 if (!skb) {
349                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
350                                                         rx_ring->rx_buf_len);
351                         if (!skb) {
352                                 adapter->alloc_rx_buff_failed++;
353                                 goto no_buffers;
354                         }
355
356                         bi->skb = skb;
357                 }
358                 if (!bi->dma) {
359                         bi->dma = dma_map_single(&pdev->dev, skb->data,
360                                                  rx_ring->rx_buf_len,
361                                                  DMA_FROM_DEVICE);
362                 }
363                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
364
365                 i++;
366                 if (i == rx_ring->count)
367                         i = 0;
368                 bi = &rx_ring->rx_buffer_info[i];
369         }
370
371 no_buffers:
372         if (rx_ring->next_to_use != i) {
373                 rx_ring->next_to_use = i;
374
375                 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
376         }
377 }
378
379 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
380                                              u32 qmask)
381 {
382         struct ixgbe_hw *hw = &adapter->hw;
383
384         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
385 }
386
387 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
388                                  struct ixgbevf_ring *rx_ring,
389                                  int budget)
390 {
391         struct ixgbevf_adapter *adapter = q_vector->adapter;
392         struct pci_dev *pdev = adapter->pdev;
393         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
394         struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
395         struct sk_buff *skb;
396         unsigned int i;
397         u32 len, staterr;
398         int cleaned_count = 0;
399         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
400
401         i = rx_ring->next_to_clean;
402         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
403         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
404         rx_buffer_info = &rx_ring->rx_buffer_info[i];
405
406         while (staterr & IXGBE_RXD_STAT_DD) {
407                 if (!budget)
408                         break;
409                 budget--;
410
411                 rmb(); /* read descriptor and rx_buffer_info after status DD */
412                 len = le16_to_cpu(rx_desc->wb.upper.length);
413                 skb = rx_buffer_info->skb;
414                 prefetch(skb->data - NET_IP_ALIGN);
415                 rx_buffer_info->skb = NULL;
416
417                 if (rx_buffer_info->dma) {
418                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
419                                          rx_ring->rx_buf_len,
420                                          DMA_FROM_DEVICE);
421                         rx_buffer_info->dma = 0;
422                         skb_put(skb, len);
423                 }
424
425                 i++;
426                 if (i == rx_ring->count)
427                         i = 0;
428
429                 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
430                 prefetch(next_rxd);
431                 cleaned_count++;
432
433                 next_buffer = &rx_ring->rx_buffer_info[i];
434
435                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
436                         skb->next = next_buffer->skb;
437                         skb->next->prev = skb;
438                         adapter->non_eop_descs++;
439                         goto next_desc;
440                 }
441
442                 /* ERR_MASK will only have valid bits if EOP set */
443                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
444                         dev_kfree_skb_irq(skb);
445                         goto next_desc;
446                 }
447
448                 ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb);
449
450                 /* probably a little skewed due to removing CRC */
451                 total_rx_bytes += skb->len;
452                 total_rx_packets++;
453
454                 /*
455                  * Work around issue of some types of VM to VM loop back
456                  * packets not getting split correctly
457                  */
458                 if (staterr & IXGBE_RXD_STAT_LB) {
459                         u32 header_fixup_len = skb_headlen(skb);
460                         if (header_fixup_len < 14)
461                                 skb_push(skb, header_fixup_len);
462                 }
463                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
464
465                 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
466
467 next_desc:
468                 rx_desc->wb.upper.status_error = 0;
469
470                 /* return some buffers to hardware, one at a time is too slow */
471                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
472                         ixgbevf_alloc_rx_buffers(adapter, rx_ring,
473                                                  cleaned_count);
474                         cleaned_count = 0;
475                 }
476
477                 /* use prefetched values */
478                 rx_desc = next_rxd;
479                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
480
481                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
482         }
483
484         rx_ring->next_to_clean = i;
485         cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
486
487         if (cleaned_count)
488                 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
489
490         u64_stats_update_begin(&rx_ring->syncp);
491         rx_ring->total_packets += total_rx_packets;
492         rx_ring->total_bytes += total_rx_bytes;
493         u64_stats_update_end(&rx_ring->syncp);
494
495         return !!budget;
496 }
497
498 /**
499  * ixgbevf_poll - NAPI polling calback
500  * @napi: napi struct with our devices info in it
501  * @budget: amount of work driver is allowed to do this pass, in packets
502  *
503  * This function will clean more than one or more rings associated with a
504  * q_vector.
505  **/
506 static int ixgbevf_poll(struct napi_struct *napi, int budget)
507 {
508         struct ixgbevf_q_vector *q_vector =
509                 container_of(napi, struct ixgbevf_q_vector, napi);
510         struct ixgbevf_adapter *adapter = q_vector->adapter;
511         struct ixgbevf_ring *ring;
512         int per_ring_budget;
513         bool clean_complete = true;
514
515         ixgbevf_for_each_ring(ring, q_vector->tx)
516                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
517
518         /* attempt to distribute budget to each queue fairly, but don't allow
519          * the budget to go below 1 because we'll exit polling */
520         if (q_vector->rx.count > 1)
521                 per_ring_budget = max(budget/q_vector->rx.count, 1);
522         else
523                 per_ring_budget = budget;
524
525         ixgbevf_for_each_ring(ring, q_vector->rx)
526                 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring,
527                                                        per_ring_budget);
528
529         /* If all work not completed, return budget and keep polling */
530         if (!clean_complete)
531                 return budget;
532         /* all work done, exit the polling mode */
533         napi_complete(napi);
534         if (adapter->rx_itr_setting & 1)
535                 ixgbevf_set_itr(q_vector);
536         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
537                 ixgbevf_irq_enable_queues(adapter,
538                                           1 << q_vector->v_idx);
539
540         return 0;
541 }
542
543
544 /**
545  * ixgbevf_configure_msix - Configure MSI-X hardware
546  * @adapter: board private structure
547  *
548  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
549  * interrupts.
550  **/
551 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
552 {
553         struct ixgbevf_q_vector *q_vector;
554         int q_vectors, v_idx;
555
556         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
557         adapter->eims_enable_mask = 0;
558
559         /*
560          * Populate the IVAR table and set the ITR values to the
561          * corresponding register.
562          */
563         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
564                 struct ixgbevf_ring *ring;
565                 q_vector = adapter->q_vector[v_idx];
566
567                 ixgbevf_for_each_ring(ring, q_vector->rx)
568                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
569
570                 ixgbevf_for_each_ring(ring, q_vector->tx)
571                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
572
573                 if (q_vector->tx.ring && !q_vector->rx.ring) {
574                         /* tx only vector */
575                         if (adapter->tx_itr_setting == 1)
576                                 q_vector->itr = IXGBE_10K_ITR;
577                         else
578                                 q_vector->itr = adapter->tx_itr_setting;
579                 } else {
580                         /* rx or rx/tx vector */
581                         if (adapter->rx_itr_setting == 1)
582                                 q_vector->itr = IXGBE_20K_ITR;
583                         else
584                                 q_vector->itr = adapter->rx_itr_setting;
585                 }
586
587                 /* add q_vector eims value to global eims_enable_mask */
588                 adapter->eims_enable_mask |= 1 << v_idx;
589
590                 ixgbevf_write_eitr(q_vector);
591         }
592
593         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
594         /* setup eims_other and add value to global eims_enable_mask */
595         adapter->eims_other = 1 << v_idx;
596         adapter->eims_enable_mask |= adapter->eims_other;
597 }
598
599 enum latency_range {
600         lowest_latency = 0,
601         low_latency = 1,
602         bulk_latency = 2,
603         latency_invalid = 255
604 };
605
606 /**
607  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
608  * @q_vector: structure containing interrupt and ring information
609  * @ring_container: structure containing ring performance data
610  *
611  *      Stores a new ITR value based on packets and byte
612  *      counts during the last interrupt.  The advantage of per interrupt
613  *      computation is faster updates and more accurate ITR for the current
614  *      traffic pattern.  Constants in this function were computed
615  *      based on theoretical maximum wire speed and thresholds were set based
616  *      on testing data as well as attempting to minimize response time
617  *      while increasing bulk throughput.
618  **/
619 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
620                                struct ixgbevf_ring_container *ring_container)
621 {
622         int bytes = ring_container->total_bytes;
623         int packets = ring_container->total_packets;
624         u32 timepassed_us;
625         u64 bytes_perint;
626         u8 itr_setting = ring_container->itr;
627
628         if (packets == 0)
629                 return;
630
631         /* simple throttlerate management
632          *    0-20MB/s lowest (100000 ints/s)
633          *   20-100MB/s low   (20000 ints/s)
634          *  100-1249MB/s bulk (8000 ints/s)
635          */
636         /* what was last interrupt timeslice? */
637         timepassed_us = q_vector->itr >> 2;
638         bytes_perint = bytes / timepassed_us; /* bytes/usec */
639
640         switch (itr_setting) {
641         case lowest_latency:
642                 if (bytes_perint > 10)
643                         itr_setting = low_latency;
644                 break;
645         case low_latency:
646                 if (bytes_perint > 20)
647                         itr_setting = bulk_latency;
648                 else if (bytes_perint <= 10)
649                         itr_setting = lowest_latency;
650                 break;
651         case bulk_latency:
652                 if (bytes_perint <= 20)
653                         itr_setting = low_latency;
654                 break;
655         }
656
657         /* clear work counters since we have the values we need */
658         ring_container->total_bytes = 0;
659         ring_container->total_packets = 0;
660
661         /* write updated itr to ring container */
662         ring_container->itr = itr_setting;
663 }
664
665 /**
666  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
667  * @q_vector: structure containing interrupt and ring information
668  *
669  * This function is made to be called by ethtool and by the driver
670  * when it needs to update VTEITR registers at runtime.  Hardware
671  * specific quirks/differences are taken care of here.
672  */
673 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
674 {
675         struct ixgbevf_adapter *adapter = q_vector->adapter;
676         struct ixgbe_hw *hw = &adapter->hw;
677         int v_idx = q_vector->v_idx;
678         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
679
680         /*
681          * set the WDIS bit to not clear the timer bits and cause an
682          * immediate assertion of the interrupt
683          */
684         itr_reg |= IXGBE_EITR_CNT_WDIS;
685
686         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
687 }
688
689 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
690 {
691         u32 new_itr = q_vector->itr;
692         u8 current_itr;
693
694         ixgbevf_update_itr(q_vector, &q_vector->tx);
695         ixgbevf_update_itr(q_vector, &q_vector->rx);
696
697         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
698
699         switch (current_itr) {
700         /* counts and packets in update_itr are dependent on these numbers */
701         case lowest_latency:
702                 new_itr = IXGBE_100K_ITR;
703                 break;
704         case low_latency:
705                 new_itr = IXGBE_20K_ITR;
706                 break;
707         case bulk_latency:
708         default:
709                 new_itr = IXGBE_8K_ITR;
710                 break;
711         }
712
713         if (new_itr != q_vector->itr) {
714                 /* do an exponential smoothing */
715                 new_itr = (10 * new_itr * q_vector->itr) /
716                           ((9 * new_itr) + q_vector->itr);
717
718                 /* save the algorithm value here */
719                 q_vector->itr = new_itr;
720
721                 ixgbevf_write_eitr(q_vector);
722         }
723 }
724
725 static irqreturn_t ixgbevf_msix_mbx(int irq, void *data)
726 {
727         struct ixgbevf_adapter *adapter = data;
728         struct ixgbe_hw *hw = &adapter->hw;
729         u32 msg;
730         bool got_ack = false;
731
732         if (!hw->mbx.ops.check_for_ack(hw))
733                 got_ack = true;
734
735         if (!hw->mbx.ops.check_for_msg(hw)) {
736                 hw->mbx.ops.read(hw, &msg, 1);
737
738                 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG)
739                         mod_timer(&adapter->watchdog_timer,
740                                   round_jiffies(jiffies + 1));
741
742                 if (msg & IXGBE_VT_MSGTYPE_NACK)
743                         pr_warn("Last Request of type %2.2x to PF Nacked\n",
744                                 msg & 0xFF);
745                 /*
746                  * Restore the PFSTS bit in case someone is polling for a
747                  * return message from the PF
748                  */
749                 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS;
750         }
751
752         /*
753          * checking for the ack clears the PFACK bit.  Place
754          * it back in the v2p_mailbox cache so that anyone
755          * polling for an ack will not miss it
756          */
757         if (got_ack)
758                 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK;
759
760         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
761
762         return IRQ_HANDLED;
763 }
764
765
766 /**
767  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
768  * @irq: unused
769  * @data: pointer to our q_vector struct for this interrupt vector
770  **/
771 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
772 {
773         struct ixgbevf_q_vector *q_vector = data;
774
775         /* EIAM disabled interrupts (on this vector) for us */
776         if (q_vector->rx.ring || q_vector->tx.ring)
777                 napi_schedule(&q_vector->napi);
778
779         return IRQ_HANDLED;
780 }
781
782 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
783                                      int r_idx)
784 {
785         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
786
787         a->rx_ring[r_idx].next = q_vector->rx.ring;
788         q_vector->rx.ring = &a->rx_ring[r_idx];
789         q_vector->rx.count++;
790 }
791
792 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
793                                      int t_idx)
794 {
795         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
796
797         a->tx_ring[t_idx].next = q_vector->tx.ring;
798         q_vector->tx.ring = &a->tx_ring[t_idx];
799         q_vector->tx.count++;
800 }
801
802 /**
803  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
804  * @adapter: board private structure to initialize
805  *
806  * This function maps descriptor rings to the queue-specific vectors
807  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
808  * one vector per ring/queue, but on a constrained vector budget, we
809  * group the rings as "efficiently" as possible.  You would add new
810  * mapping configurations in here.
811  **/
812 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
813 {
814         int q_vectors;
815         int v_start = 0;
816         int rxr_idx = 0, txr_idx = 0;
817         int rxr_remaining = adapter->num_rx_queues;
818         int txr_remaining = adapter->num_tx_queues;
819         int i, j;
820         int rqpv, tqpv;
821         int err = 0;
822
823         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
824
825         /*
826          * The ideal configuration...
827          * We have enough vectors to map one per queue.
828          */
829         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
830                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
831                         map_vector_to_rxq(adapter, v_start, rxr_idx);
832
833                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
834                         map_vector_to_txq(adapter, v_start, txr_idx);
835                 goto out;
836         }
837
838         /*
839          * If we don't have enough vectors for a 1-to-1
840          * mapping, we'll have to group them so there are
841          * multiple queues per vector.
842          */
843         /* Re-adjusting *qpv takes care of the remainder. */
844         for (i = v_start; i < q_vectors; i++) {
845                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
846                 for (j = 0; j < rqpv; j++) {
847                         map_vector_to_rxq(adapter, i, rxr_idx);
848                         rxr_idx++;
849                         rxr_remaining--;
850                 }
851         }
852         for (i = v_start; i < q_vectors; i++) {
853                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
854                 for (j = 0; j < tqpv; j++) {
855                         map_vector_to_txq(adapter, i, txr_idx);
856                         txr_idx++;
857                         txr_remaining--;
858                 }
859         }
860
861 out:
862         return err;
863 }
864
865 /**
866  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
867  * @adapter: board private structure
868  *
869  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
870  * interrupts from the kernel.
871  **/
872 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
873 {
874         struct net_device *netdev = adapter->netdev;
875         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
876         int vector, err;
877         int ri = 0, ti = 0;
878
879         for (vector = 0; vector < q_vectors; vector++) {
880                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
881                 struct msix_entry *entry = &adapter->msix_entries[vector];
882
883                 if (q_vector->tx.ring && q_vector->rx.ring) {
884                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
885                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
886                         ti++;
887                 } else if (q_vector->rx.ring) {
888                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
889                                  "%s-%s-%d", netdev->name, "rx", ri++);
890                 } else if (q_vector->tx.ring) {
891                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
892                                  "%s-%s-%d", netdev->name, "tx", ti++);
893                 } else {
894                         /* skip this unused q_vector */
895                         continue;
896                 }
897                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
898                                   q_vector->name, q_vector);
899                 if (err) {
900                         hw_dbg(&adapter->hw,
901                                "request_irq failed for MSIX interrupt "
902                                "Error: %d\n", err);
903                         goto free_queue_irqs;
904                 }
905         }
906
907         err = request_irq(adapter->msix_entries[vector].vector,
908                           &ixgbevf_msix_mbx, 0, netdev->name, adapter);
909         if (err) {
910                 hw_dbg(&adapter->hw,
911                        "request_irq for msix_mbx failed: %d\n", err);
912                 goto free_queue_irqs;
913         }
914
915         return 0;
916
917 free_queue_irqs:
918         while (vector) {
919                 vector--;
920                 free_irq(adapter->msix_entries[vector].vector,
921                          adapter->q_vector[vector]);
922         }
923         pci_disable_msix(adapter->pdev);
924         kfree(adapter->msix_entries);
925         adapter->msix_entries = NULL;
926         return err;
927 }
928
929 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
930 {
931         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
932
933         for (i = 0; i < q_vectors; i++) {
934                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
935                 q_vector->rx.ring = NULL;
936                 q_vector->tx.ring = NULL;
937                 q_vector->rx.count = 0;
938                 q_vector->tx.count = 0;
939         }
940 }
941
942 /**
943  * ixgbevf_request_irq - initialize interrupts
944  * @adapter: board private structure
945  *
946  * Attempts to configure interrupts using the best available
947  * capabilities of the hardware and kernel.
948  **/
949 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
950 {
951         int err = 0;
952
953         err = ixgbevf_request_msix_irqs(adapter);
954
955         if (err)
956                 hw_dbg(&adapter->hw,
957                        "request_irq failed, Error %d\n", err);
958
959         return err;
960 }
961
962 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
963 {
964         int i, q_vectors;
965
966         q_vectors = adapter->num_msix_vectors;
967         i = q_vectors - 1;
968
969         free_irq(adapter->msix_entries[i].vector, adapter);
970         i--;
971
972         for (; i >= 0; i--) {
973                 /* free only the irqs that were actually requested */
974                 if (!adapter->q_vector[i]->rx.ring &&
975                     !adapter->q_vector[i]->tx.ring)
976                         continue;
977
978                 free_irq(adapter->msix_entries[i].vector,
979                          adapter->q_vector[i]);
980         }
981
982         ixgbevf_reset_q_vectors(adapter);
983 }
984
985 /**
986  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
987  * @adapter: board private structure
988  **/
989 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
990 {
991         struct ixgbe_hw *hw = &adapter->hw;
992         int i;
993
994         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
995         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
996         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
997
998         IXGBE_WRITE_FLUSH(hw);
999
1000         for (i = 0; i < adapter->num_msix_vectors; i++)
1001                 synchronize_irq(adapter->msix_entries[i].vector);
1002 }
1003
1004 /**
1005  * ixgbevf_irq_enable - Enable default interrupt generation settings
1006  * @adapter: board private structure
1007  **/
1008 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1009 {
1010         struct ixgbe_hw *hw = &adapter->hw;
1011
1012         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1013         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1014         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1015 }
1016
1017 /**
1018  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1019  * @adapter: board private structure
1020  *
1021  * Configure the Tx unit of the MAC after a reset.
1022  **/
1023 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1024 {
1025         u64 tdba;
1026         struct ixgbe_hw *hw = &adapter->hw;
1027         u32 i, j, tdlen, txctrl;
1028
1029         /* Setup the HW Tx Head and Tail descriptor pointers */
1030         for (i = 0; i < adapter->num_tx_queues; i++) {
1031                 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1032                 j = ring->reg_idx;
1033                 tdba = ring->dma;
1034                 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1035                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1036                                 (tdba & DMA_BIT_MASK(32)));
1037                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1038                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1039                 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1040                 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1041                 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1042                 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1043                 /* Disable Tx Head Writeback RO bit, since this hoses
1044                  * bookkeeping if things aren't delivered in order.
1045                  */
1046                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1047                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1048                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1049         }
1050 }
1051
1052 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1053
1054 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1055 {
1056         struct ixgbevf_ring *rx_ring;
1057         struct ixgbe_hw *hw = &adapter->hw;
1058         u32 srrctl;
1059
1060         rx_ring = &adapter->rx_ring[index];
1061
1062         srrctl = IXGBE_SRRCTL_DROP_EN;
1063
1064         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1065
1066         if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE)
1067                 srrctl |= IXGBEVF_RXBUFFER_2048 >>
1068                         IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1069         else
1070                 srrctl |= rx_ring->rx_buf_len >>
1071                         IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1072         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1073 }
1074
1075 /**
1076  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1077  * @adapter: board private structure
1078  *
1079  * Configure the Rx unit of the MAC after a reset.
1080  **/
1081 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1082 {
1083         u64 rdba;
1084         struct ixgbe_hw *hw = &adapter->hw;
1085         struct net_device *netdev = adapter->netdev;
1086         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1087         int i, j;
1088         u32 rdlen;
1089         int rx_buf_len;
1090
1091         /* PSRTYPE must be initialized in 82599 */
1092         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
1093         if (netdev->mtu <= ETH_DATA_LEN)
1094                 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1095         else
1096                 rx_buf_len = ALIGN(max_frame, 1024);
1097
1098         rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1099         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1100          * the Base and Length of the Rx Descriptor Ring */
1101         for (i = 0; i < adapter->num_rx_queues; i++) {
1102                 rdba = adapter->rx_ring[i].dma;
1103                 j = adapter->rx_ring[i].reg_idx;
1104                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1105                                 (rdba & DMA_BIT_MASK(32)));
1106                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1107                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1108                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1109                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1110                 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1111                 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1112                 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1113
1114                 ixgbevf_configure_srrctl(adapter, j);
1115         }
1116 }
1117
1118 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1119 {
1120         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1121         struct ixgbe_hw *hw = &adapter->hw;
1122
1123         /* add VID to filter table */
1124         if (hw->mac.ops.set_vfta)
1125                 hw->mac.ops.set_vfta(hw, vid, 0, true);
1126         set_bit(vid, adapter->active_vlans);
1127
1128         return 0;
1129 }
1130
1131 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1132 {
1133         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1134         struct ixgbe_hw *hw = &adapter->hw;
1135
1136         /* remove VID from filter table */
1137         if (hw->mac.ops.set_vfta)
1138                 hw->mac.ops.set_vfta(hw, vid, 0, false);
1139         clear_bit(vid, adapter->active_vlans);
1140
1141         return 0;
1142 }
1143
1144 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1145 {
1146         u16 vid;
1147
1148         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1149                 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
1150 }
1151
1152 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1153 {
1154         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1155         struct ixgbe_hw *hw = &adapter->hw;
1156         int count = 0;
1157
1158         if ((netdev_uc_count(netdev)) > 10) {
1159                 pr_err("Too many unicast filters - No Space\n");
1160                 return -ENOSPC;
1161         }
1162
1163         if (!netdev_uc_empty(netdev)) {
1164                 struct netdev_hw_addr *ha;
1165                 netdev_for_each_uc_addr(ha, netdev) {
1166                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1167                         udelay(200);
1168                 }
1169         } else {
1170                 /*
1171                  * If the list is empty then send message to PF driver to
1172                  * clear all macvlans on this VF.
1173                  */
1174                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1175         }
1176
1177         return count;
1178 }
1179
1180 /**
1181  * ixgbevf_set_rx_mode - Multicast set
1182  * @netdev: network interface device structure
1183  *
1184  * The set_rx_method entry point is called whenever the multicast address
1185  * list or the network interface flags are updated.  This routine is
1186  * responsible for configuring the hardware for proper multicast mode.
1187  **/
1188 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1189 {
1190         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1191         struct ixgbe_hw *hw = &adapter->hw;
1192
1193         /* reprogram multicast list */
1194         if (hw->mac.ops.update_mc_addr_list)
1195                 hw->mac.ops.update_mc_addr_list(hw, netdev);
1196
1197         ixgbevf_write_uc_addr_list(netdev);
1198 }
1199
1200 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1201 {
1202         int q_idx;
1203         struct ixgbevf_q_vector *q_vector;
1204         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1205
1206         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1207                 q_vector = adapter->q_vector[q_idx];
1208                 napi_enable(&q_vector->napi);
1209         }
1210 }
1211
1212 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1213 {
1214         int q_idx;
1215         struct ixgbevf_q_vector *q_vector;
1216         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1217
1218         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1219                 q_vector = adapter->q_vector[q_idx];
1220                 napi_disable(&q_vector->napi);
1221         }
1222 }
1223
1224 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1225 {
1226         struct net_device *netdev = adapter->netdev;
1227         int i;
1228
1229         ixgbevf_set_rx_mode(netdev);
1230
1231         ixgbevf_restore_vlan(adapter);
1232
1233         ixgbevf_configure_tx(adapter);
1234         ixgbevf_configure_rx(adapter);
1235         for (i = 0; i < adapter->num_rx_queues; i++) {
1236                 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1237                 ixgbevf_alloc_rx_buffers(adapter, ring,
1238                                          IXGBE_DESC_UNUSED(ring));
1239         }
1240 }
1241
1242 #define IXGBE_MAX_RX_DESC_POLL 10
1243 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1244                                                 int rxr)
1245 {
1246         struct ixgbe_hw *hw = &adapter->hw;
1247         int j = adapter->rx_ring[rxr].reg_idx;
1248         int k;
1249
1250         for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1251                 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1252                         break;
1253                 else
1254                         msleep(1);
1255         }
1256         if (k >= IXGBE_MAX_RX_DESC_POLL) {
1257                 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1258                        "not set within the polling period\n", rxr);
1259         }
1260
1261         ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1262                                 (adapter->rx_ring[rxr].count - 1));
1263 }
1264
1265 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1266 {
1267         /* Only save pre-reset stats if there are some */
1268         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1269                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1270                         adapter->stats.base_vfgprc;
1271                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1272                         adapter->stats.base_vfgptc;
1273                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1274                         adapter->stats.base_vfgorc;
1275                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1276                         adapter->stats.base_vfgotc;
1277                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1278                         adapter->stats.base_vfmprc;
1279         }
1280 }
1281
1282 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1283 {
1284         struct ixgbe_hw *hw = &adapter->hw;
1285
1286         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1287         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1288         adapter->stats.last_vfgorc |=
1289                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1290         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1291         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1292         adapter->stats.last_vfgotc |=
1293                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1294         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1295
1296         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1297         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1298         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1299         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1300         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1301 }
1302
1303 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1304 {
1305         struct net_device *netdev = adapter->netdev;
1306         struct ixgbe_hw *hw = &adapter->hw;
1307         int i, j = 0;
1308         int num_rx_rings = adapter->num_rx_queues;
1309         u32 txdctl, rxdctl;
1310         u32 msg[2];
1311
1312         for (i = 0; i < adapter->num_tx_queues; i++) {
1313                 j = adapter->tx_ring[i].reg_idx;
1314                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1315                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1316                 txdctl |= (8 << 16);
1317                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1318         }
1319
1320         for (i = 0; i < adapter->num_tx_queues; i++) {
1321                 j = adapter->tx_ring[i].reg_idx;
1322                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1323                 txdctl |= IXGBE_TXDCTL_ENABLE;
1324                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1325         }
1326
1327         for (i = 0; i < num_rx_rings; i++) {
1328                 j = adapter->rx_ring[i].reg_idx;
1329                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1330                 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1331                 if (hw->mac.type == ixgbe_mac_X540_vf) {
1332                         rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1333                         rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1334                                    IXGBE_RXDCTL_RLPML_EN);
1335                 }
1336                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1337                 ixgbevf_rx_desc_queue_enable(adapter, i);
1338         }
1339
1340         ixgbevf_configure_msix(adapter);
1341
1342         if (hw->mac.ops.set_rar) {
1343                 if (is_valid_ether_addr(hw->mac.addr))
1344                         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1345                 else
1346                         hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1347         }
1348
1349         msg[0] = IXGBE_VF_SET_LPE;
1350         msg[1] = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1351         hw->mbx.ops.write_posted(hw, msg, 2);
1352
1353         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1354         ixgbevf_napi_enable_all(adapter);
1355
1356         /* enable transmits */
1357         netif_tx_start_all_queues(netdev);
1358
1359         ixgbevf_save_reset_stats(adapter);
1360         ixgbevf_init_last_counter_stats(adapter);
1361
1362         mod_timer(&adapter->watchdog_timer, jiffies);
1363 }
1364
1365 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1366 {
1367         struct ixgbe_hw *hw = &adapter->hw;
1368
1369         ixgbevf_configure(adapter);
1370
1371         ixgbevf_up_complete(adapter);
1372
1373         /* clear any pending interrupts, may auto mask */
1374         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1375
1376         ixgbevf_irq_enable(adapter);
1377 }
1378
1379 /**
1380  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1381  * @adapter: board private structure
1382  * @rx_ring: ring to free buffers from
1383  **/
1384 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1385                                   struct ixgbevf_ring *rx_ring)
1386 {
1387         struct pci_dev *pdev = adapter->pdev;
1388         unsigned long size;
1389         unsigned int i;
1390
1391         if (!rx_ring->rx_buffer_info)
1392                 return;
1393
1394         /* Free all the Rx ring sk_buffs */
1395         for (i = 0; i < rx_ring->count; i++) {
1396                 struct ixgbevf_rx_buffer *rx_buffer_info;
1397
1398                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1399                 if (rx_buffer_info->dma) {
1400                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
1401                                          rx_ring->rx_buf_len,
1402                                          DMA_FROM_DEVICE);
1403                         rx_buffer_info->dma = 0;
1404                 }
1405                 if (rx_buffer_info->skb) {
1406                         struct sk_buff *skb = rx_buffer_info->skb;
1407                         rx_buffer_info->skb = NULL;
1408                         do {
1409                                 struct sk_buff *this = skb;
1410                                 skb = skb->prev;
1411                                 dev_kfree_skb(this);
1412                         } while (skb);
1413                 }
1414         }
1415
1416         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1417         memset(rx_ring->rx_buffer_info, 0, size);
1418
1419         /* Zero out the descriptor ring */
1420         memset(rx_ring->desc, 0, rx_ring->size);
1421
1422         rx_ring->next_to_clean = 0;
1423         rx_ring->next_to_use = 0;
1424
1425         if (rx_ring->head)
1426                 writel(0, adapter->hw.hw_addr + rx_ring->head);
1427         if (rx_ring->tail)
1428                 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1429 }
1430
1431 /**
1432  * ixgbevf_clean_tx_ring - Free Tx Buffers
1433  * @adapter: board private structure
1434  * @tx_ring: ring to be cleaned
1435  **/
1436 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1437                                   struct ixgbevf_ring *tx_ring)
1438 {
1439         struct ixgbevf_tx_buffer *tx_buffer_info;
1440         unsigned long size;
1441         unsigned int i;
1442
1443         if (!tx_ring->tx_buffer_info)
1444                 return;
1445
1446         /* Free all the Tx ring sk_buffs */
1447
1448         for (i = 0; i < tx_ring->count; i++) {
1449                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1450                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
1451         }
1452
1453         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1454         memset(tx_ring->tx_buffer_info, 0, size);
1455
1456         memset(tx_ring->desc, 0, tx_ring->size);
1457
1458         tx_ring->next_to_use = 0;
1459         tx_ring->next_to_clean = 0;
1460
1461         if (tx_ring->head)
1462                 writel(0, adapter->hw.hw_addr + tx_ring->head);
1463         if (tx_ring->tail)
1464                 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1465 }
1466
1467 /**
1468  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1469  * @adapter: board private structure
1470  **/
1471 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1472 {
1473         int i;
1474
1475         for (i = 0; i < adapter->num_rx_queues; i++)
1476                 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1477 }
1478
1479 /**
1480  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1481  * @adapter: board private structure
1482  **/
1483 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1484 {
1485         int i;
1486
1487         for (i = 0; i < adapter->num_tx_queues; i++)
1488                 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1489 }
1490
1491 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1492 {
1493         struct net_device *netdev = adapter->netdev;
1494         struct ixgbe_hw *hw = &adapter->hw;
1495         u32 txdctl;
1496         int i, j;
1497
1498         /* signal that we are down to the interrupt handler */
1499         set_bit(__IXGBEVF_DOWN, &adapter->state);
1500         /* disable receives */
1501
1502         netif_tx_disable(netdev);
1503
1504         msleep(10);
1505
1506         netif_tx_stop_all_queues(netdev);
1507
1508         ixgbevf_irq_disable(adapter);
1509
1510         ixgbevf_napi_disable_all(adapter);
1511
1512         del_timer_sync(&adapter->watchdog_timer);
1513         /* can't call flush scheduled work here because it can deadlock
1514          * if linkwatch_event tries to acquire the rtnl_lock which we are
1515          * holding */
1516         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1517                 msleep(1);
1518
1519         /* disable transmits in the hardware now that interrupts are off */
1520         for (i = 0; i < adapter->num_tx_queues; i++) {
1521                 j = adapter->tx_ring[i].reg_idx;
1522                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1523                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1524                                 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1525         }
1526
1527         netif_carrier_off(netdev);
1528
1529         if (!pci_channel_offline(adapter->pdev))
1530                 ixgbevf_reset(adapter);
1531
1532         ixgbevf_clean_all_tx_rings(adapter);
1533         ixgbevf_clean_all_rx_rings(adapter);
1534 }
1535
1536 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1537 {
1538         struct ixgbe_hw *hw = &adapter->hw;
1539
1540         WARN_ON(in_interrupt());
1541
1542         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1543                 msleep(1);
1544
1545         /*
1546          * Check if PF is up before re-init.  If not then skip until
1547          * later when the PF is up and ready to service requests from
1548          * the VF via mailbox.  If the VF is up and running then the
1549          * watchdog task will continue to schedule reset tasks until
1550          * the PF is up and running.
1551          */
1552         if (!hw->mac.ops.reset_hw(hw)) {
1553                 ixgbevf_down(adapter);
1554                 ixgbevf_up(adapter);
1555         }
1556
1557         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1558 }
1559
1560 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1561 {
1562         struct ixgbe_hw *hw = &adapter->hw;
1563         struct net_device *netdev = adapter->netdev;
1564
1565         if (hw->mac.ops.reset_hw(hw))
1566                 hw_dbg(hw, "PF still resetting\n");
1567         else
1568                 hw->mac.ops.init_hw(hw);
1569
1570         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1571                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1572                        netdev->addr_len);
1573                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1574                        netdev->addr_len);
1575         }
1576 }
1577
1578 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1579                                          int vectors)
1580 {
1581         int err, vector_threshold;
1582
1583         /* We'll want at least 2 (vector_threshold):
1584          * 1) TxQ[0] + RxQ[0] handler
1585          * 2) Other (Link Status Change, etc.)
1586          */
1587         vector_threshold = MIN_MSIX_COUNT;
1588
1589         /* The more we get, the more we will assign to Tx/Rx Cleanup
1590          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1591          * Right now, we simply care about how many we'll get; we'll
1592          * set them up later while requesting irq's.
1593          */
1594         while (vectors >= vector_threshold) {
1595                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1596                                       vectors);
1597                 if (!err) /* Success in acquiring all requested vectors. */
1598                         break;
1599                 else if (err < 0)
1600                         vectors = 0; /* Nasty failure, quit now */
1601                 else /* err == number of vectors we should try again with */
1602                         vectors = err;
1603         }
1604
1605         if (vectors < vector_threshold) {
1606                 /* Can't allocate enough MSI-X interrupts?  Oh well.
1607                  * This just means we'll go with either a single MSI
1608                  * vector or fall back to legacy interrupts.
1609                  */
1610                 hw_dbg(&adapter->hw,
1611                        "Unable to allocate MSI-X interrupts\n");
1612                 kfree(adapter->msix_entries);
1613                 adapter->msix_entries = NULL;
1614         } else {
1615                 /*
1616                  * Adjust for only the vectors we'll use, which is minimum
1617                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1618                  * vectors we were allocated.
1619                  */
1620                 adapter->num_msix_vectors = vectors;
1621         }
1622 }
1623
1624 /**
1625  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
1626  * @adapter: board private structure to initialize
1627  *
1628  * This is the top level queue allocation routine.  The order here is very
1629  * important, starting with the "most" number of features turned on at once,
1630  * and ending with the smallest set of features.  This way large combinations
1631  * can be allocated if they're turned on, and smaller combinations are the
1632  * fallthrough conditions.
1633  *
1634  **/
1635 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1636 {
1637         /* Start with base case */
1638         adapter->num_rx_queues = 1;
1639         adapter->num_tx_queues = 1;
1640 }
1641
1642 /**
1643  * ixgbevf_alloc_queues - Allocate memory for all rings
1644  * @adapter: board private structure to initialize
1645  *
1646  * We allocate one ring per queue at run-time since we don't know the
1647  * number of queues at compile-time.  The polling_netdev array is
1648  * intended for Multiqueue, but should work fine with a single queue.
1649  **/
1650 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1651 {
1652         int i;
1653
1654         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1655                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1656         if (!adapter->tx_ring)
1657                 goto err_tx_ring_allocation;
1658
1659         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1660                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1661         if (!adapter->rx_ring)
1662                 goto err_rx_ring_allocation;
1663
1664         for (i = 0; i < adapter->num_tx_queues; i++) {
1665                 adapter->tx_ring[i].count = adapter->tx_ring_count;
1666                 adapter->tx_ring[i].queue_index = i;
1667                 adapter->tx_ring[i].reg_idx = i;
1668                 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1669                 adapter->tx_ring[i].netdev = adapter->netdev;
1670         }
1671
1672         for (i = 0; i < adapter->num_rx_queues; i++) {
1673                 adapter->rx_ring[i].count = adapter->rx_ring_count;
1674                 adapter->rx_ring[i].queue_index = i;
1675                 adapter->rx_ring[i].reg_idx = i;
1676                 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1677                 adapter->rx_ring[i].netdev = adapter->netdev;
1678         }
1679
1680         return 0;
1681
1682 err_rx_ring_allocation:
1683         kfree(adapter->tx_ring);
1684 err_tx_ring_allocation:
1685         return -ENOMEM;
1686 }
1687
1688 /**
1689  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1690  * @adapter: board private structure to initialize
1691  *
1692  * Attempt to configure the interrupts using the best available
1693  * capabilities of the hardware and the kernel.
1694  **/
1695 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1696 {
1697         int err = 0;
1698         int vector, v_budget;
1699
1700         /*
1701          * It's easy to be greedy for MSI-X vectors, but it really
1702          * doesn't do us much good if we have a lot more vectors
1703          * than CPU's.  So let's be conservative and only ask for
1704          * (roughly) the same number of vectors as there are CPU's.
1705          * The default is to use pairs of vectors.
1706          */
1707         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1708         v_budget = min_t(int, v_budget, num_online_cpus());
1709         v_budget += NON_Q_VECTORS;
1710
1711         /* A failure in MSI-X entry allocation isn't fatal, but it does
1712          * mean we disable MSI-X capabilities of the adapter. */
1713         adapter->msix_entries = kcalloc(v_budget,
1714                                         sizeof(struct msix_entry), GFP_KERNEL);
1715         if (!adapter->msix_entries) {
1716                 err = -ENOMEM;
1717                 goto out;
1718         }
1719
1720         for (vector = 0; vector < v_budget; vector++)
1721                 adapter->msix_entries[vector].entry = vector;
1722
1723         ixgbevf_acquire_msix_vectors(adapter, v_budget);
1724
1725 out:
1726         return err;
1727 }
1728
1729 /**
1730  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1731  * @adapter: board private structure to initialize
1732  *
1733  * We allocate one q_vector per queue interrupt.  If allocation fails we
1734  * return -ENOMEM.
1735  **/
1736 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1737 {
1738         int q_idx, num_q_vectors;
1739         struct ixgbevf_q_vector *q_vector;
1740
1741         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1742
1743         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1744                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
1745                 if (!q_vector)
1746                         goto err_out;
1747                 q_vector->adapter = adapter;
1748                 q_vector->v_idx = q_idx;
1749                 netif_napi_add(adapter->netdev, &q_vector->napi,
1750                                ixgbevf_poll, 64);
1751                 adapter->q_vector[q_idx] = q_vector;
1752         }
1753
1754         return 0;
1755
1756 err_out:
1757         while (q_idx) {
1758                 q_idx--;
1759                 q_vector = adapter->q_vector[q_idx];
1760                 netif_napi_del(&q_vector->napi);
1761                 kfree(q_vector);
1762                 adapter->q_vector[q_idx] = NULL;
1763         }
1764         return -ENOMEM;
1765 }
1766
1767 /**
1768  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
1769  * @adapter: board private structure to initialize
1770  *
1771  * This function frees the memory allocated to the q_vectors.  In addition if
1772  * NAPI is enabled it will delete any references to the NAPI struct prior
1773  * to freeing the q_vector.
1774  **/
1775 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
1776 {
1777         int q_idx, num_q_vectors;
1778         int napi_vectors;
1779
1780         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1781         napi_vectors = adapter->num_rx_queues;
1782
1783         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1784                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
1785
1786                 adapter->q_vector[q_idx] = NULL;
1787                 if (q_idx < napi_vectors)
1788                         netif_napi_del(&q_vector->napi);
1789                 kfree(q_vector);
1790         }
1791 }
1792
1793 /**
1794  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
1795  * @adapter: board private structure
1796  *
1797  **/
1798 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
1799 {
1800         pci_disable_msix(adapter->pdev);
1801         kfree(adapter->msix_entries);
1802         adapter->msix_entries = NULL;
1803 }
1804
1805 /**
1806  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
1807  * @adapter: board private structure to initialize
1808  *
1809  **/
1810 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
1811 {
1812         int err;
1813
1814         /* Number of supported queues */
1815         ixgbevf_set_num_queues(adapter);
1816
1817         err = ixgbevf_set_interrupt_capability(adapter);
1818         if (err) {
1819                 hw_dbg(&adapter->hw,
1820                        "Unable to setup interrupt capabilities\n");
1821                 goto err_set_interrupt;
1822         }
1823
1824         err = ixgbevf_alloc_q_vectors(adapter);
1825         if (err) {
1826                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
1827                        "vectors\n");
1828                 goto err_alloc_q_vectors;
1829         }
1830
1831         err = ixgbevf_alloc_queues(adapter);
1832         if (err) {
1833                 pr_err("Unable to allocate memory for queues\n");
1834                 goto err_alloc_queues;
1835         }
1836
1837         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
1838                "Tx Queue count = %u\n",
1839                (adapter->num_rx_queues > 1) ? "Enabled" :
1840                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
1841
1842         set_bit(__IXGBEVF_DOWN, &adapter->state);
1843
1844         return 0;
1845 err_alloc_queues:
1846         ixgbevf_free_q_vectors(adapter);
1847 err_alloc_q_vectors:
1848         ixgbevf_reset_interrupt_capability(adapter);
1849 err_set_interrupt:
1850         return err;
1851 }
1852
1853 /**
1854  * ixgbevf_sw_init - Initialize general software structures
1855  * (struct ixgbevf_adapter)
1856  * @adapter: board private structure to initialize
1857  *
1858  * ixgbevf_sw_init initializes the Adapter private data structure.
1859  * Fields are initialized based on PCI device information and
1860  * OS network device settings (MTU size).
1861  **/
1862 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
1863 {
1864         struct ixgbe_hw *hw = &adapter->hw;
1865         struct pci_dev *pdev = adapter->pdev;
1866         int err;
1867
1868         /* PCI config space info */
1869
1870         hw->vendor_id = pdev->vendor;
1871         hw->device_id = pdev->device;
1872         hw->revision_id = pdev->revision;
1873         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1874         hw->subsystem_device_id = pdev->subsystem_device;
1875
1876         hw->mbx.ops.init_params(hw);
1877         hw->mac.max_tx_queues = MAX_TX_QUEUES;
1878         hw->mac.max_rx_queues = MAX_RX_QUEUES;
1879         err = hw->mac.ops.reset_hw(hw);
1880         if (err) {
1881                 dev_info(&pdev->dev,
1882                          "PF still in reset state, assigning new address\n");
1883                 eth_hw_addr_random(adapter->netdev);
1884                 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
1885                         adapter->netdev->addr_len);
1886         } else {
1887                 err = hw->mac.ops.init_hw(hw);
1888                 if (err) {
1889                         pr_err("init_shared_code failed: %d\n", err);
1890                         goto out;
1891                 }
1892                 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
1893                         adapter->netdev->addr_len);
1894         }
1895
1896         /* Enable dynamic interrupt throttling rates */
1897         adapter->rx_itr_setting = 1;
1898         adapter->tx_itr_setting = 1;
1899
1900         /* set default ring sizes */
1901         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
1902         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
1903
1904         set_bit(__IXGBEVF_DOWN, &adapter->state);
1905         return 0;
1906
1907 out:
1908         return err;
1909 }
1910
1911 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
1912         {                                                       \
1913                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
1914                 if (current_counter < last_counter)             \
1915                         counter += 0x100000000LL;               \
1916                 last_counter = current_counter;                 \
1917                 counter &= 0xFFFFFFFF00000000LL;                \
1918                 counter |= current_counter;                     \
1919         }
1920
1921 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
1922         {                                                                \
1923                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
1924                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
1925                 u64 current_counter = (current_counter_msb << 32) |      \
1926                         current_counter_lsb;                             \
1927                 if (current_counter < last_counter)                      \
1928                         counter += 0x1000000000LL;                       \
1929                 last_counter = current_counter;                          \
1930                 counter &= 0xFFFFFFF000000000LL;                         \
1931                 counter |= current_counter;                              \
1932         }
1933 /**
1934  * ixgbevf_update_stats - Update the board statistics counters.
1935  * @adapter: board private structure
1936  **/
1937 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
1938 {
1939         struct ixgbe_hw *hw = &adapter->hw;
1940
1941         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
1942                                 adapter->stats.vfgprc);
1943         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
1944                                 adapter->stats.vfgptc);
1945         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1946                                 adapter->stats.last_vfgorc,
1947                                 adapter->stats.vfgorc);
1948         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1949                                 adapter->stats.last_vfgotc,
1950                                 adapter->stats.vfgotc);
1951         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
1952                                 adapter->stats.vfmprc);
1953 }
1954
1955 /**
1956  * ixgbevf_watchdog - Timer Call-back
1957  * @data: pointer to adapter cast into an unsigned long
1958  **/
1959 static void ixgbevf_watchdog(unsigned long data)
1960 {
1961         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
1962         struct ixgbe_hw *hw = &adapter->hw;
1963         u32 eics = 0;
1964         int i;
1965
1966         /*
1967          * Do the watchdog outside of interrupt context due to the lovely
1968          * delays that some of the newer hardware requires
1969          */
1970
1971         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
1972                 goto watchdog_short_circuit;
1973
1974         /* get one bit for every active tx/rx interrupt vector */
1975         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
1976                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
1977                 if (qv->rx.ring || qv->tx.ring)
1978                         eics |= 1 << i;
1979         }
1980
1981         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
1982
1983 watchdog_short_circuit:
1984         schedule_work(&adapter->watchdog_task);
1985 }
1986
1987 /**
1988  * ixgbevf_tx_timeout - Respond to a Tx Hang
1989  * @netdev: network interface device structure
1990  **/
1991 static void ixgbevf_tx_timeout(struct net_device *netdev)
1992 {
1993         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1994
1995         /* Do the reset outside of interrupt context */
1996         schedule_work(&adapter->reset_task);
1997 }
1998
1999 static void ixgbevf_reset_task(struct work_struct *work)
2000 {
2001         struct ixgbevf_adapter *adapter;
2002         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2003
2004         /* If we're already down or resetting, just bail */
2005         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2006             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2007                 return;
2008
2009         adapter->tx_timeout_count++;
2010
2011         ixgbevf_reinit_locked(adapter);
2012 }
2013
2014 /**
2015  * ixgbevf_watchdog_task - worker thread to bring link up
2016  * @work: pointer to work_struct containing our data
2017  **/
2018 static void ixgbevf_watchdog_task(struct work_struct *work)
2019 {
2020         struct ixgbevf_adapter *adapter = container_of(work,
2021                                                        struct ixgbevf_adapter,
2022                                                        watchdog_task);
2023         struct net_device *netdev = adapter->netdev;
2024         struct ixgbe_hw *hw = &adapter->hw;
2025         u32 link_speed = adapter->link_speed;
2026         bool link_up = adapter->link_up;
2027
2028         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2029
2030         /*
2031          * Always check the link on the watchdog because we have
2032          * no LSC interrupt
2033          */
2034         if (hw->mac.ops.check_link) {
2035                 if ((hw->mac.ops.check_link(hw, &link_speed,
2036                                             &link_up, false)) != 0) {
2037                         adapter->link_up = link_up;
2038                         adapter->link_speed = link_speed;
2039                         netif_carrier_off(netdev);
2040                         netif_tx_stop_all_queues(netdev);
2041                         schedule_work(&adapter->reset_task);
2042                         goto pf_has_reset;
2043                 }
2044         } else {
2045                 /* always assume link is up, if no check link
2046                  * function */
2047                 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
2048                 link_up = true;
2049         }
2050         adapter->link_up = link_up;
2051         adapter->link_speed = link_speed;
2052
2053         if (link_up) {
2054                 if (!netif_carrier_ok(netdev)) {
2055                         hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n",
2056                                (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
2057                                10 : 1);
2058                         netif_carrier_on(netdev);
2059                         netif_tx_wake_all_queues(netdev);
2060                 }
2061         } else {
2062                 adapter->link_up = false;
2063                 adapter->link_speed = 0;
2064                 if (netif_carrier_ok(netdev)) {
2065                         hw_dbg(&adapter->hw, "NIC Link is Down\n");
2066                         netif_carrier_off(netdev);
2067                         netif_tx_stop_all_queues(netdev);
2068                 }
2069         }
2070
2071         ixgbevf_update_stats(adapter);
2072
2073 pf_has_reset:
2074         /* Reset the timer */
2075         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2076                 mod_timer(&adapter->watchdog_timer,
2077                           round_jiffies(jiffies + (2 * HZ)));
2078
2079         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2080 }
2081
2082 /**
2083  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2084  * @adapter: board private structure
2085  * @tx_ring: Tx descriptor ring for a specific queue
2086  *
2087  * Free all transmit software resources
2088  **/
2089 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2090                                struct ixgbevf_ring *tx_ring)
2091 {
2092         struct pci_dev *pdev = adapter->pdev;
2093
2094         ixgbevf_clean_tx_ring(adapter, tx_ring);
2095
2096         vfree(tx_ring->tx_buffer_info);
2097         tx_ring->tx_buffer_info = NULL;
2098
2099         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2100                           tx_ring->dma);
2101
2102         tx_ring->desc = NULL;
2103 }
2104
2105 /**
2106  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2107  * @adapter: board private structure
2108  *
2109  * Free all transmit software resources
2110  **/
2111 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2112 {
2113         int i;
2114
2115         for (i = 0; i < adapter->num_tx_queues; i++)
2116                 if (adapter->tx_ring[i].desc)
2117                         ixgbevf_free_tx_resources(adapter,
2118                                                   &adapter->tx_ring[i]);
2119
2120 }
2121
2122 /**
2123  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2124  * @adapter: board private structure
2125  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2126  *
2127  * Return 0 on success, negative on failure
2128  **/
2129 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2130                                struct ixgbevf_ring *tx_ring)
2131 {
2132         struct pci_dev *pdev = adapter->pdev;
2133         int size;
2134
2135         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2136         tx_ring->tx_buffer_info = vzalloc(size);
2137         if (!tx_ring->tx_buffer_info)
2138                 goto err;
2139
2140         /* round up to nearest 4K */
2141         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2142         tx_ring->size = ALIGN(tx_ring->size, 4096);
2143
2144         tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2145                                            &tx_ring->dma, GFP_KERNEL);
2146         if (!tx_ring->desc)
2147                 goto err;
2148
2149         tx_ring->next_to_use = 0;
2150         tx_ring->next_to_clean = 0;
2151         return 0;
2152
2153 err:
2154         vfree(tx_ring->tx_buffer_info);
2155         tx_ring->tx_buffer_info = NULL;
2156         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2157                "descriptor ring\n");
2158         return -ENOMEM;
2159 }
2160
2161 /**
2162  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2163  * @adapter: board private structure
2164  *
2165  * If this function returns with an error, then it's possible one or
2166  * more of the rings is populated (while the rest are not).  It is the
2167  * callers duty to clean those orphaned rings.
2168  *
2169  * Return 0 on success, negative on failure
2170  **/
2171 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2172 {
2173         int i, err = 0;
2174
2175         for (i = 0; i < adapter->num_tx_queues; i++) {
2176                 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2177                 if (!err)
2178                         continue;
2179                 hw_dbg(&adapter->hw,
2180                        "Allocation for Tx Queue %u failed\n", i);
2181                 break;
2182         }
2183
2184         return err;
2185 }
2186
2187 /**
2188  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2189  * @adapter: board private structure
2190  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2191  *
2192  * Returns 0 on success, negative on failure
2193  **/
2194 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2195                                struct ixgbevf_ring *rx_ring)
2196 {
2197         struct pci_dev *pdev = adapter->pdev;
2198         int size;
2199
2200         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2201         rx_ring->rx_buffer_info = vzalloc(size);
2202         if (!rx_ring->rx_buffer_info)
2203                 goto alloc_failed;
2204
2205         /* Round up to nearest 4K */
2206         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2207         rx_ring->size = ALIGN(rx_ring->size, 4096);
2208
2209         rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2210                                            &rx_ring->dma, GFP_KERNEL);
2211
2212         if (!rx_ring->desc) {
2213                 hw_dbg(&adapter->hw,
2214                        "Unable to allocate memory for "
2215                        "the receive descriptor ring\n");
2216                 vfree(rx_ring->rx_buffer_info);
2217                 rx_ring->rx_buffer_info = NULL;
2218                 goto alloc_failed;
2219         }
2220
2221         rx_ring->next_to_clean = 0;
2222         rx_ring->next_to_use = 0;
2223
2224         return 0;
2225 alloc_failed:
2226         return -ENOMEM;
2227 }
2228
2229 /**
2230  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2231  * @adapter: board private structure
2232  *
2233  * If this function returns with an error, then it's possible one or
2234  * more of the rings is populated (while the rest are not).  It is the
2235  * callers duty to clean those orphaned rings.
2236  *
2237  * Return 0 on success, negative on failure
2238  **/
2239 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2240 {
2241         int i, err = 0;
2242
2243         for (i = 0; i < adapter->num_rx_queues; i++) {
2244                 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2245                 if (!err)
2246                         continue;
2247                 hw_dbg(&adapter->hw,
2248                        "Allocation for Rx Queue %u failed\n", i);
2249                 break;
2250         }
2251         return err;
2252 }
2253
2254 /**
2255  * ixgbevf_free_rx_resources - Free Rx Resources
2256  * @adapter: board private structure
2257  * @rx_ring: ring to clean the resources from
2258  *
2259  * Free all receive software resources
2260  **/
2261 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2262                                struct ixgbevf_ring *rx_ring)
2263 {
2264         struct pci_dev *pdev = adapter->pdev;
2265
2266         ixgbevf_clean_rx_ring(adapter, rx_ring);
2267
2268         vfree(rx_ring->rx_buffer_info);
2269         rx_ring->rx_buffer_info = NULL;
2270
2271         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2272                           rx_ring->dma);
2273
2274         rx_ring->desc = NULL;
2275 }
2276
2277 /**
2278  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2279  * @adapter: board private structure
2280  *
2281  * Free all receive software resources
2282  **/
2283 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2284 {
2285         int i;
2286
2287         for (i = 0; i < adapter->num_rx_queues; i++)
2288                 if (adapter->rx_ring[i].desc)
2289                         ixgbevf_free_rx_resources(adapter,
2290                                                   &adapter->rx_ring[i]);
2291 }
2292
2293 /**
2294  * ixgbevf_open - Called when a network interface is made active
2295  * @netdev: network interface device structure
2296  *
2297  * Returns 0 on success, negative value on failure
2298  *
2299  * The open entry point is called when a network interface is made
2300  * active by the system (IFF_UP).  At this point all resources needed
2301  * for transmit and receive operations are allocated, the interrupt
2302  * handler is registered with the OS, the watchdog timer is started,
2303  * and the stack is notified that the interface is ready.
2304  **/
2305 static int ixgbevf_open(struct net_device *netdev)
2306 {
2307         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2308         struct ixgbe_hw *hw = &adapter->hw;
2309         int err;
2310
2311         /* disallow open during test */
2312         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2313                 return -EBUSY;
2314
2315         if (hw->adapter_stopped) {
2316                 ixgbevf_reset(adapter);
2317                 /* if adapter is still stopped then PF isn't up and
2318                  * the vf can't start. */
2319                 if (hw->adapter_stopped) {
2320                         err = IXGBE_ERR_MBX;
2321                         pr_err("Unable to start - perhaps the PF Driver isn't "
2322                                "up yet\n");
2323                         goto err_setup_reset;
2324                 }
2325         }
2326
2327         /* allocate transmit descriptors */
2328         err = ixgbevf_setup_all_tx_resources(adapter);
2329         if (err)
2330                 goto err_setup_tx;
2331
2332         /* allocate receive descriptors */
2333         err = ixgbevf_setup_all_rx_resources(adapter);
2334         if (err)
2335                 goto err_setup_rx;
2336
2337         ixgbevf_configure(adapter);
2338
2339         /*
2340          * Map the Tx/Rx rings to the vectors we were allotted.
2341          * if request_irq will be called in this function map_rings
2342          * must be called *before* up_complete
2343          */
2344         ixgbevf_map_rings_to_vectors(adapter);
2345
2346         ixgbevf_up_complete(adapter);
2347
2348         /* clear any pending interrupts, may auto mask */
2349         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2350         err = ixgbevf_request_irq(adapter);
2351         if (err)
2352                 goto err_req_irq;
2353
2354         ixgbevf_irq_enable(adapter);
2355
2356         return 0;
2357
2358 err_req_irq:
2359         ixgbevf_down(adapter);
2360         ixgbevf_free_irq(adapter);
2361 err_setup_rx:
2362         ixgbevf_free_all_rx_resources(adapter);
2363 err_setup_tx:
2364         ixgbevf_free_all_tx_resources(adapter);
2365         ixgbevf_reset(adapter);
2366
2367 err_setup_reset:
2368
2369         return err;
2370 }
2371
2372 /**
2373  * ixgbevf_close - Disables a network interface
2374  * @netdev: network interface device structure
2375  *
2376  * Returns 0, this is not allowed to fail
2377  *
2378  * The close entry point is called when an interface is de-activated
2379  * by the OS.  The hardware is still under the drivers control, but
2380  * needs to be disabled.  A global MAC reset is issued to stop the
2381  * hardware, and all transmit and receive resources are freed.
2382  **/
2383 static int ixgbevf_close(struct net_device *netdev)
2384 {
2385         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2386
2387         ixgbevf_down(adapter);
2388         ixgbevf_free_irq(adapter);
2389
2390         ixgbevf_free_all_tx_resources(adapter);
2391         ixgbevf_free_all_rx_resources(adapter);
2392
2393         return 0;
2394 }
2395
2396 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2397                                 u32 vlan_macip_lens, u32 type_tucmd,
2398                                 u32 mss_l4len_idx)
2399 {
2400         struct ixgbe_adv_tx_context_desc *context_desc;
2401         u16 i = tx_ring->next_to_use;
2402
2403         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
2404
2405         i++;
2406         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2407
2408         /* set bits to identify this as an advanced context descriptor */
2409         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2410
2411         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
2412         context_desc->seqnum_seed       = 0;
2413         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
2414         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
2415 }
2416
2417 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2418                        struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2419 {
2420         u32 vlan_macip_lens, type_tucmd;
2421         u32 mss_l4len_idx, l4len;
2422
2423         if (!skb_is_gso(skb))
2424                 return 0;
2425
2426         if (skb_header_cloned(skb)) {
2427                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2428                 if (err)
2429                         return err;
2430         }
2431
2432         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2433         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2434
2435         if (skb->protocol == htons(ETH_P_IP)) {
2436                 struct iphdr *iph = ip_hdr(skb);
2437                 iph->tot_len = 0;
2438                 iph->check = 0;
2439                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2440                                                          iph->daddr, 0,
2441                                                          IPPROTO_TCP,
2442                                                          0);
2443                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2444         } else if (skb_is_gso_v6(skb)) {
2445                 ipv6_hdr(skb)->payload_len = 0;
2446                 tcp_hdr(skb)->check =
2447                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2448                                      &ipv6_hdr(skb)->daddr,
2449                                      0, IPPROTO_TCP, 0);
2450         }
2451
2452         /* compute header lengths */
2453         l4len = tcp_hdrlen(skb);
2454         *hdr_len += l4len;
2455         *hdr_len = skb_transport_offset(skb) + l4len;
2456
2457         /* mss_l4len_id: use 1 as index for TSO */
2458         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2459         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2460         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2461
2462         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2463         vlan_macip_lens = skb_network_header_len(skb);
2464         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2465         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2466
2467         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2468                             type_tucmd, mss_l4len_idx);
2469
2470         return 1;
2471 }
2472
2473 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2474                             struct sk_buff *skb, u32 tx_flags)
2475 {
2476
2477
2478
2479         u32 vlan_macip_lens = 0;
2480         u32 mss_l4len_idx = 0;
2481         u32 type_tucmd = 0;
2482
2483         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2484                 u8 l4_hdr = 0;
2485                 switch (skb->protocol) {
2486                 case __constant_htons(ETH_P_IP):
2487                         vlan_macip_lens |= skb_network_header_len(skb);
2488                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2489                         l4_hdr = ip_hdr(skb)->protocol;
2490                         break;
2491                 case __constant_htons(ETH_P_IPV6):
2492                         vlan_macip_lens |= skb_network_header_len(skb);
2493                         l4_hdr = ipv6_hdr(skb)->nexthdr;
2494                         break;
2495                 default:
2496                         if (unlikely(net_ratelimit())) {
2497                                 dev_warn(tx_ring->dev,
2498                                  "partial checksum but proto=%x!\n",
2499                                  skb->protocol);
2500                         }
2501                         break;
2502                 }
2503
2504                 switch (l4_hdr) {
2505                 case IPPROTO_TCP:
2506                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2507                         mss_l4len_idx = tcp_hdrlen(skb) <<
2508                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2509                         break;
2510                 case IPPROTO_SCTP:
2511                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2512                         mss_l4len_idx = sizeof(struct sctphdr) <<
2513                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2514                         break;
2515                 case IPPROTO_UDP:
2516                         mss_l4len_idx = sizeof(struct udphdr) <<
2517                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2518                         break;
2519                 default:
2520                         if (unlikely(net_ratelimit())) {
2521                                 dev_warn(tx_ring->dev,
2522                                  "partial checksum but l4 proto=%x!\n",
2523                                  l4_hdr);
2524                         }
2525                         break;
2526                 }
2527         }
2528
2529         /* vlan_macip_lens: MACLEN, VLAN tag */
2530         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2531         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2532
2533         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2534                             type_tucmd, mss_l4len_idx);
2535
2536         return (skb->ip_summed == CHECKSUM_PARTIAL);
2537 }
2538
2539 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2540                           struct sk_buff *skb, u32 tx_flags,
2541                           unsigned int first)
2542 {
2543         struct ixgbevf_tx_buffer *tx_buffer_info;
2544         unsigned int len;
2545         unsigned int total = skb->len;
2546         unsigned int offset = 0, size;
2547         int count = 0;
2548         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2549         unsigned int f;
2550         int i;
2551
2552         i = tx_ring->next_to_use;
2553
2554         len = min(skb_headlen(skb), total);
2555         while (len) {
2556                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2557                 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2558
2559                 tx_buffer_info->length = size;
2560                 tx_buffer_info->mapped_as_page = false;
2561                 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
2562                                                      skb->data + offset,
2563                                                      size, DMA_TO_DEVICE);
2564                 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
2565                         goto dma_error;
2566                 tx_buffer_info->next_to_watch = i;
2567
2568                 len -= size;
2569                 total -= size;
2570                 offset += size;
2571                 count++;
2572                 i++;
2573                 if (i == tx_ring->count)
2574                         i = 0;
2575         }
2576
2577         for (f = 0; f < nr_frags; f++) {
2578                 const struct skb_frag_struct *frag;
2579
2580                 frag = &skb_shinfo(skb)->frags[f];
2581                 len = min((unsigned int)skb_frag_size(frag), total);
2582                 offset = 0;
2583
2584                 while (len) {
2585                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
2586                         size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2587
2588                         tx_buffer_info->length = size;
2589                         tx_buffer_info->dma =
2590                                 skb_frag_dma_map(tx_ring->dev, frag,
2591                                                  offset, size, DMA_TO_DEVICE);
2592                         tx_buffer_info->mapped_as_page = true;
2593                         if (dma_mapping_error(tx_ring->dev,
2594                                               tx_buffer_info->dma))
2595                                 goto dma_error;
2596                         tx_buffer_info->next_to_watch = i;
2597
2598                         len -= size;
2599                         total -= size;
2600                         offset += size;
2601                         count++;
2602                         i++;
2603                         if (i == tx_ring->count)
2604                                 i = 0;
2605                 }
2606                 if (total == 0)
2607                         break;
2608         }
2609
2610         if (i == 0)
2611                 i = tx_ring->count - 1;
2612         else
2613                 i = i - 1;
2614         tx_ring->tx_buffer_info[i].skb = skb;
2615         tx_ring->tx_buffer_info[first].next_to_watch = i;
2616         tx_ring->tx_buffer_info[first].time_stamp = jiffies;
2617
2618         return count;
2619
2620 dma_error:
2621         dev_err(tx_ring->dev, "TX DMA map failed\n");
2622
2623         /* clear timestamp and dma mappings for failed tx_buffer_info map */
2624         tx_buffer_info->dma = 0;
2625         tx_buffer_info->next_to_watch = 0;
2626         count--;
2627
2628         /* clear timestamp and dma mappings for remaining portion of packet */
2629         while (count >= 0) {
2630                 count--;
2631                 i--;
2632                 if (i < 0)
2633                         i += tx_ring->count;
2634                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2635                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2636         }
2637
2638         return count;
2639 }
2640
2641 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
2642                              int count, u32 paylen, u8 hdr_len)
2643 {
2644         union ixgbe_adv_tx_desc *tx_desc = NULL;
2645         struct ixgbevf_tx_buffer *tx_buffer_info;
2646         u32 olinfo_status = 0, cmd_type_len = 0;
2647         unsigned int i;
2648
2649         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2650
2651         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2652
2653         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2654
2655         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2656                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2657
2658         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2659                 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2660
2661         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2662                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2663
2664                 /* use index 1 context for tso */
2665                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2666                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2667                         olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
2668
2669         }
2670
2671         /*
2672          * Check Context must be set if Tx switch is enabled, which it
2673          * always is for case where virtual functions are running
2674          */
2675         olinfo_status |= IXGBE_ADVTXD_CC;
2676
2677         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
2678
2679         i = tx_ring->next_to_use;
2680         while (count--) {
2681                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2682                 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2683                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
2684                 tx_desc->read.cmd_type_len =
2685                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
2686                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2687                 i++;
2688                 if (i == tx_ring->count)
2689                         i = 0;
2690         }
2691
2692         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
2693
2694         tx_ring->next_to_use = i;
2695 }
2696
2697 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2698 {
2699         struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
2700
2701         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2702         /* Herbert's original patch had:
2703          *  smp_mb__after_netif_stop_queue();
2704          * but since that doesn't exist yet, just open code it. */
2705         smp_mb();
2706
2707         /* We need to check again in a case another CPU has just
2708          * made room available. */
2709         if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
2710                 return -EBUSY;
2711
2712         /* A reprieve! - use start_queue because it doesn't call schedule */
2713         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2714         ++adapter->restart_queue;
2715         return 0;
2716 }
2717
2718 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2719 {
2720         if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
2721                 return 0;
2722         return __ixgbevf_maybe_stop_tx(tx_ring, size);
2723 }
2724
2725 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2726 {
2727         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2728         struct ixgbevf_ring *tx_ring;
2729         unsigned int first;
2730         unsigned int tx_flags = 0;
2731         u8 hdr_len = 0;
2732         int r_idx = 0, tso;
2733         u16 count = TXD_USE_COUNT(skb_headlen(skb));
2734 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2735         unsigned short f;
2736 #endif
2737
2738         tx_ring = &adapter->tx_ring[r_idx];
2739
2740         /*
2741          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
2742          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
2743          *       + 2 desc gap to keep tail from touching head,
2744          *       + 1 desc for context descriptor,
2745          * otherwise try next time
2746          */
2747 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2748         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2749                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2750 #else
2751         count += skb_shinfo(skb)->nr_frags;
2752 #endif
2753         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
2754                 adapter->tx_busy++;
2755                 return NETDEV_TX_BUSY;
2756         }
2757
2758         if (vlan_tx_tag_present(skb)) {
2759                 tx_flags |= vlan_tx_tag_get(skb);
2760                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
2761                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
2762         }
2763
2764         first = tx_ring->next_to_use;
2765
2766         if (skb->protocol == htons(ETH_P_IP))
2767                 tx_flags |= IXGBE_TX_FLAGS_IPV4;
2768         tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
2769         if (tso < 0) {
2770                 dev_kfree_skb_any(skb);
2771                 return NETDEV_TX_OK;
2772         }
2773
2774         if (tso)
2775                 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
2776         else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
2777                 tx_flags |= IXGBE_TX_FLAGS_CSUM;
2778
2779         ixgbevf_tx_queue(tx_ring, tx_flags,
2780                          ixgbevf_tx_map(tx_ring, skb, tx_flags, first),
2781                          skb->len, hdr_len);
2782         /*
2783          * Force memory writes to complete before letting h/w
2784          * know there are new descriptors to fetch.  (Only
2785          * applicable for weak-ordered memory model archs,
2786          * such as IA-64).
2787          */
2788         wmb();
2789
2790         writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
2791
2792         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
2793
2794         return NETDEV_TX_OK;
2795 }
2796
2797 /**
2798  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
2799  * @netdev: network interface device structure
2800  * @p: pointer to an address structure
2801  *
2802  * Returns 0 on success, negative on failure
2803  **/
2804 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
2805 {
2806         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2807         struct ixgbe_hw *hw = &adapter->hw;
2808         struct sockaddr *addr = p;
2809
2810         if (!is_valid_ether_addr(addr->sa_data))
2811                 return -EADDRNOTAVAIL;
2812
2813         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2814         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2815
2816         if (hw->mac.ops.set_rar)
2817                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2818
2819         return 0;
2820 }
2821
2822 /**
2823  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
2824  * @netdev: network interface device structure
2825  * @new_mtu: new value for maximum frame size
2826  *
2827  * Returns 0 on success, negative on failure
2828  **/
2829 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
2830 {
2831         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2832         struct ixgbe_hw *hw = &adapter->hw;
2833         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2834         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
2835         u32 msg[2];
2836
2837         if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
2838                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
2839
2840         /* MTU < 68 is an error and causes problems on some kernels */
2841         if ((new_mtu < 68) || (max_frame > max_possible_frame))
2842                 return -EINVAL;
2843
2844         hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
2845                netdev->mtu, new_mtu);
2846         /* must set new MTU before calling down or up */
2847         netdev->mtu = new_mtu;
2848
2849         if (!netif_running(netdev)) {
2850                 msg[0] = IXGBE_VF_SET_LPE;
2851                 msg[1] = max_frame;
2852                 hw->mbx.ops.write_posted(hw, msg, 2);
2853         }
2854
2855         if (netif_running(netdev))
2856                 ixgbevf_reinit_locked(adapter);
2857
2858         return 0;
2859 }
2860
2861 static void ixgbevf_shutdown(struct pci_dev *pdev)
2862 {
2863         struct net_device *netdev = pci_get_drvdata(pdev);
2864         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2865
2866         netif_device_detach(netdev);
2867
2868         if (netif_running(netdev)) {
2869                 ixgbevf_down(adapter);
2870                 ixgbevf_free_irq(adapter);
2871                 ixgbevf_free_all_tx_resources(adapter);
2872                 ixgbevf_free_all_rx_resources(adapter);
2873         }
2874
2875         pci_save_state(pdev);
2876
2877         pci_disable_device(pdev);
2878 }
2879
2880 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
2881                                                 struct rtnl_link_stats64 *stats)
2882 {
2883         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2884         unsigned int start;
2885         u64 bytes, packets;
2886         const struct ixgbevf_ring *ring;
2887         int i;
2888
2889         ixgbevf_update_stats(adapter);
2890
2891         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
2892
2893         for (i = 0; i < adapter->num_rx_queues; i++) {
2894                 ring = &adapter->rx_ring[i];
2895                 do {
2896                         start = u64_stats_fetch_begin_bh(&ring->syncp);
2897                         bytes = ring->total_bytes;
2898                         packets = ring->total_packets;
2899                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
2900                 stats->rx_bytes += bytes;
2901                 stats->rx_packets += packets;
2902         }
2903
2904         for (i = 0; i < adapter->num_tx_queues; i++) {
2905                 ring = &adapter->tx_ring[i];
2906                 do {
2907                         start = u64_stats_fetch_begin_bh(&ring->syncp);
2908                         bytes = ring->total_bytes;
2909                         packets = ring->total_packets;
2910                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
2911                 stats->tx_bytes += bytes;
2912                 stats->tx_packets += packets;
2913         }
2914
2915         return stats;
2916 }
2917
2918 static const struct net_device_ops ixgbe_netdev_ops = {
2919         .ndo_open               = ixgbevf_open,
2920         .ndo_stop               = ixgbevf_close,
2921         .ndo_start_xmit         = ixgbevf_xmit_frame,
2922         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
2923         .ndo_get_stats64        = ixgbevf_get_stats,
2924         .ndo_validate_addr      = eth_validate_addr,
2925         .ndo_set_mac_address    = ixgbevf_set_mac,
2926         .ndo_change_mtu         = ixgbevf_change_mtu,
2927         .ndo_tx_timeout         = ixgbevf_tx_timeout,
2928         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
2929         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
2930 };
2931
2932 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
2933 {
2934         dev->netdev_ops = &ixgbe_netdev_ops;
2935         ixgbevf_set_ethtool_ops(dev);
2936         dev->watchdog_timeo = 5 * HZ;
2937 }
2938
2939 /**
2940  * ixgbevf_probe - Device Initialization Routine
2941  * @pdev: PCI device information struct
2942  * @ent: entry in ixgbevf_pci_tbl
2943  *
2944  * Returns 0 on success, negative on failure
2945  *
2946  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
2947  * The OS initialization, configuring of the adapter private structure,
2948  * and a hardware reset occur.
2949  **/
2950 static int __devinit ixgbevf_probe(struct pci_dev *pdev,
2951                                    const struct pci_device_id *ent)
2952 {
2953         struct net_device *netdev;
2954         struct ixgbevf_adapter *adapter = NULL;
2955         struct ixgbe_hw *hw = NULL;
2956         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
2957         static int cards_found;
2958         int err, pci_using_dac;
2959
2960         err = pci_enable_device(pdev);
2961         if (err)
2962                 return err;
2963
2964         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
2965             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
2966                 pci_using_dac = 1;
2967         } else {
2968                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2969                 if (err) {
2970                         err = dma_set_coherent_mask(&pdev->dev,
2971                                                     DMA_BIT_MASK(32));
2972                         if (err) {
2973                                 dev_err(&pdev->dev, "No usable DMA "
2974                                         "configuration, aborting\n");
2975                                 goto err_dma;
2976                         }
2977                 }
2978                 pci_using_dac = 0;
2979         }
2980
2981         err = pci_request_regions(pdev, ixgbevf_driver_name);
2982         if (err) {
2983                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
2984                 goto err_pci_reg;
2985         }
2986
2987         pci_set_master(pdev);
2988
2989         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
2990                                    MAX_TX_QUEUES);
2991         if (!netdev) {
2992                 err = -ENOMEM;
2993                 goto err_alloc_etherdev;
2994         }
2995
2996         SET_NETDEV_DEV(netdev, &pdev->dev);
2997
2998         pci_set_drvdata(pdev, netdev);
2999         adapter = netdev_priv(netdev);
3000
3001         adapter->netdev = netdev;
3002         adapter->pdev = pdev;
3003         hw = &adapter->hw;
3004         hw->back = adapter;
3005         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3006
3007         /*
3008          * call save state here in standalone driver because it relies on
3009          * adapter struct to exist, and needs to call netdev_priv
3010          */
3011         pci_save_state(pdev);
3012
3013         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3014                               pci_resource_len(pdev, 0));
3015         if (!hw->hw_addr) {
3016                 err = -EIO;
3017                 goto err_ioremap;
3018         }
3019
3020         ixgbevf_assign_netdev_ops(netdev);
3021
3022         adapter->bd_number = cards_found;
3023
3024         /* Setup hw api */
3025         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3026         hw->mac.type  = ii->mac;
3027
3028         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3029                sizeof(struct ixgbe_mbx_operations));
3030
3031         /* setup the private structure */
3032         err = ixgbevf_sw_init(adapter);
3033         if (err)
3034                 goto err_sw_init;
3035
3036         /* The HW MAC address was set and/or determined in sw_init */
3037         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
3038
3039         if (!is_valid_ether_addr(netdev->dev_addr)) {
3040                 pr_err("invalid MAC address\n");
3041                 err = -EIO;
3042                 goto err_sw_init;
3043         }
3044
3045         netdev->hw_features = NETIF_F_SG |
3046                            NETIF_F_IP_CSUM |
3047                            NETIF_F_IPV6_CSUM |
3048                            NETIF_F_TSO |
3049                            NETIF_F_TSO6 |
3050                            NETIF_F_RXCSUM;
3051
3052         netdev->features = netdev->hw_features |
3053                            NETIF_F_HW_VLAN_TX |
3054                            NETIF_F_HW_VLAN_RX |
3055                            NETIF_F_HW_VLAN_FILTER;
3056
3057         netdev->vlan_features |= NETIF_F_TSO;
3058         netdev->vlan_features |= NETIF_F_TSO6;
3059         netdev->vlan_features |= NETIF_F_IP_CSUM;
3060         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3061         netdev->vlan_features |= NETIF_F_SG;
3062
3063         if (pci_using_dac)
3064                 netdev->features |= NETIF_F_HIGHDMA;
3065
3066         netdev->priv_flags |= IFF_UNICAST_FLT;
3067
3068         init_timer(&adapter->watchdog_timer);
3069         adapter->watchdog_timer.function = ixgbevf_watchdog;
3070         adapter->watchdog_timer.data = (unsigned long)adapter;
3071
3072         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3073         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3074
3075         err = ixgbevf_init_interrupt_scheme(adapter);
3076         if (err)
3077                 goto err_sw_init;
3078
3079         /* pick up the PCI bus settings for reporting later */
3080         if (hw->mac.ops.get_bus_info)
3081                 hw->mac.ops.get_bus_info(hw);
3082
3083         strcpy(netdev->name, "eth%d");
3084
3085         err = register_netdev(netdev);
3086         if (err)
3087                 goto err_register;
3088
3089         netif_carrier_off(netdev);
3090
3091         ixgbevf_init_last_counter_stats(adapter);
3092
3093         /* print the MAC address */
3094         hw_dbg(hw, "%pM\n", netdev->dev_addr);
3095
3096         hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3097
3098         hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3099         cards_found++;
3100         return 0;
3101
3102 err_register:
3103 err_sw_init:
3104         ixgbevf_reset_interrupt_capability(adapter);
3105         iounmap(hw->hw_addr);
3106 err_ioremap:
3107         free_netdev(netdev);
3108 err_alloc_etherdev:
3109         pci_release_regions(pdev);
3110 err_pci_reg:
3111 err_dma:
3112         pci_disable_device(pdev);
3113         return err;
3114 }
3115
3116 /**
3117  * ixgbevf_remove - Device Removal Routine
3118  * @pdev: PCI device information struct
3119  *
3120  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3121  * that it should release a PCI device.  The could be caused by a
3122  * Hot-Plug event, or because the driver is going to be removed from
3123  * memory.
3124  **/
3125 static void __devexit ixgbevf_remove(struct pci_dev *pdev)
3126 {
3127         struct net_device *netdev = pci_get_drvdata(pdev);
3128         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3129
3130         set_bit(__IXGBEVF_DOWN, &adapter->state);
3131
3132         del_timer_sync(&adapter->watchdog_timer);
3133
3134         cancel_work_sync(&adapter->reset_task);
3135         cancel_work_sync(&adapter->watchdog_task);
3136
3137         if (netdev->reg_state == NETREG_REGISTERED)
3138                 unregister_netdev(netdev);
3139
3140         ixgbevf_reset_interrupt_capability(adapter);
3141
3142         iounmap(adapter->hw.hw_addr);
3143         pci_release_regions(pdev);
3144
3145         hw_dbg(&adapter->hw, "Remove complete\n");
3146
3147         kfree(adapter->tx_ring);
3148         kfree(adapter->rx_ring);
3149
3150         free_netdev(netdev);
3151
3152         pci_disable_device(pdev);
3153 }
3154
3155 static struct pci_driver ixgbevf_driver = {
3156         .name     = ixgbevf_driver_name,
3157         .id_table = ixgbevf_pci_tbl,
3158         .probe    = ixgbevf_probe,
3159         .remove   = __devexit_p(ixgbevf_remove),
3160         .shutdown = ixgbevf_shutdown,
3161 };
3162
3163 /**
3164  * ixgbevf_init_module - Driver Registration Routine
3165  *
3166  * ixgbevf_init_module is the first routine called when the driver is
3167  * loaded. All it does is register with the PCI subsystem.
3168  **/
3169 static int __init ixgbevf_init_module(void)
3170 {
3171         int ret;
3172         pr_info("%s - version %s\n", ixgbevf_driver_string,
3173                 ixgbevf_driver_version);
3174
3175         pr_info("%s\n", ixgbevf_copyright);
3176
3177         ret = pci_register_driver(&ixgbevf_driver);
3178         return ret;
3179 }
3180
3181 module_init(ixgbevf_init_module);
3182
3183 /**
3184  * ixgbevf_exit_module - Driver Exit Cleanup Routine
3185  *
3186  * ixgbevf_exit_module is called just before the driver is removed
3187  * from memory.
3188  **/
3189 static void __exit ixgbevf_exit_module(void)
3190 {
3191         pci_unregister_driver(&ixgbevf_driver);
3192 }
3193
3194 #ifdef DEBUG
3195 /**
3196  * ixgbevf_get_hw_dev_name - return device name string
3197  * used by hardware layer to print debugging information
3198  **/
3199 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3200 {
3201         struct ixgbevf_adapter *adapter = hw->back;
3202         return adapter->netdev->name;
3203 }
3204
3205 #endif
3206 module_exit(ixgbevf_exit_module);
3207
3208 /* ixgbevf_main.c */