virtio: rename virtqueue_add_buf_gfp to virtqueue_add_buf
[linux-2.6.git] / drivers / net / virtio_net.c
1 /* A network driver using virtio.
2  *
3  * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  */
19 //#define DEBUG
20 #include <linux/netdevice.h>
21 #include <linux/etherdevice.h>
22 #include <linux/ethtool.h>
23 #include <linux/module.h>
24 #include <linux/virtio.h>
25 #include <linux/virtio_net.h>
26 #include <linux/scatterlist.h>
27 #include <linux/if_vlan.h>
28 #include <linux/slab.h>
29
30 static int napi_weight = 128;
31 module_param(napi_weight, int, 0444);
32
33 static bool csum = true, gso = true;
34 module_param(csum, bool, 0444);
35 module_param(gso, bool, 0444);
36
37 /* FIXME: MTU in config. */
38 #define MAX_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
39 #define GOOD_COPY_LEN   128
40
41 #define VIRTNET_SEND_COMMAND_SG_MAX    2
42 #define VIRTNET_DRIVER_VERSION "1.0.0"
43
44 struct virtnet_stats {
45         struct u64_stats_sync syncp;
46         u64 tx_bytes;
47         u64 tx_packets;
48
49         u64 rx_bytes;
50         u64 rx_packets;
51 };
52
53 struct virtnet_info {
54         struct virtio_device *vdev;
55         struct virtqueue *rvq, *svq, *cvq;
56         struct net_device *dev;
57         struct napi_struct napi;
58         unsigned int status;
59
60         /* Number of input buffers, and max we've ever had. */
61         unsigned int num, max;
62
63         /* I like... big packets and I cannot lie! */
64         bool big_packets;
65
66         /* Host will merge rx buffers for big packets (shake it! shake it!) */
67         bool mergeable_rx_bufs;
68
69         /* Active statistics */
70         struct virtnet_stats __percpu *stats;
71
72         /* Work struct for refilling if we run low on memory. */
73         struct delayed_work refill;
74
75         /* Chain pages by the private ptr. */
76         struct page *pages;
77
78         /* fragments + linear part + virtio header */
79         struct scatterlist rx_sg[MAX_SKB_FRAGS + 2];
80         struct scatterlist tx_sg[MAX_SKB_FRAGS + 2];
81 };
82
83 struct skb_vnet_hdr {
84         union {
85                 struct virtio_net_hdr hdr;
86                 struct virtio_net_hdr_mrg_rxbuf mhdr;
87         };
88         unsigned int num_sg;
89 };
90
91 struct padded_vnet_hdr {
92         struct virtio_net_hdr hdr;
93         /*
94          * virtio_net_hdr should be in a separated sg buffer because of a
95          * QEMU bug, and data sg buffer shares same page with this header sg.
96          * This padding makes next sg 16 byte aligned after virtio_net_hdr.
97          */
98         char padding[6];
99 };
100
101 static inline struct skb_vnet_hdr *skb_vnet_hdr(struct sk_buff *skb)
102 {
103         return (struct skb_vnet_hdr *)skb->cb;
104 }
105
106 /*
107  * private is used to chain pages for big packets, put the whole
108  * most recent used list in the beginning for reuse
109  */
110 static void give_pages(struct virtnet_info *vi, struct page *page)
111 {
112         struct page *end;
113
114         /* Find end of list, sew whole thing into vi->pages. */
115         for (end = page; end->private; end = (struct page *)end->private);
116         end->private = (unsigned long)vi->pages;
117         vi->pages = page;
118 }
119
120 static struct page *get_a_page(struct virtnet_info *vi, gfp_t gfp_mask)
121 {
122         struct page *p = vi->pages;
123
124         if (p) {
125                 vi->pages = (struct page *)p->private;
126                 /* clear private here, it is used to chain pages */
127                 p->private = 0;
128         } else
129                 p = alloc_page(gfp_mask);
130         return p;
131 }
132
133 static void skb_xmit_done(struct virtqueue *svq)
134 {
135         struct virtnet_info *vi = svq->vdev->priv;
136
137         /* Suppress further interrupts. */
138         virtqueue_disable_cb(svq);
139
140         /* We were probably waiting for more output buffers. */
141         netif_wake_queue(vi->dev);
142 }
143
144 static void set_skb_frag(struct sk_buff *skb, struct page *page,
145                          unsigned int offset, unsigned int *len)
146 {
147         int size = min((unsigned)PAGE_SIZE - offset, *len);
148         int i = skb_shinfo(skb)->nr_frags;
149
150         __skb_fill_page_desc(skb, i, page, offset, size);
151
152         skb->data_len += size;
153         skb->len += size;
154         skb->truesize += PAGE_SIZE;
155         skb_shinfo(skb)->nr_frags++;
156         *len -= size;
157 }
158
159 /* Called from bottom half context */
160 static struct sk_buff *page_to_skb(struct virtnet_info *vi,
161                                    struct page *page, unsigned int len)
162 {
163         struct sk_buff *skb;
164         struct skb_vnet_hdr *hdr;
165         unsigned int copy, hdr_len, offset;
166         char *p;
167
168         p = page_address(page);
169
170         /* copy small packet so we can reuse these pages for small data */
171         skb = netdev_alloc_skb_ip_align(vi->dev, GOOD_COPY_LEN);
172         if (unlikely(!skb))
173                 return NULL;
174
175         hdr = skb_vnet_hdr(skb);
176
177         if (vi->mergeable_rx_bufs) {
178                 hdr_len = sizeof hdr->mhdr;
179                 offset = hdr_len;
180         } else {
181                 hdr_len = sizeof hdr->hdr;
182                 offset = sizeof(struct padded_vnet_hdr);
183         }
184
185         memcpy(hdr, p, hdr_len);
186
187         len -= hdr_len;
188         p += offset;
189
190         copy = len;
191         if (copy > skb_tailroom(skb))
192                 copy = skb_tailroom(skb);
193         memcpy(skb_put(skb, copy), p, copy);
194
195         len -= copy;
196         offset += copy;
197
198         /*
199          * Verify that we can indeed put this data into a skb.
200          * This is here to handle cases when the device erroneously
201          * tries to receive more than is possible. This is usually
202          * the case of a broken device.
203          */
204         if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) {
205                 if (net_ratelimit())
206                         pr_debug("%s: too much data\n", skb->dev->name);
207                 dev_kfree_skb(skb);
208                 return NULL;
209         }
210
211         while (len) {
212                 set_skb_frag(skb, page, offset, &len);
213                 page = (struct page *)page->private;
214                 offset = 0;
215         }
216
217         if (page)
218                 give_pages(vi, page);
219
220         return skb;
221 }
222
223 static int receive_mergeable(struct virtnet_info *vi, struct sk_buff *skb)
224 {
225         struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
226         struct page *page;
227         int num_buf, i, len;
228
229         num_buf = hdr->mhdr.num_buffers;
230         while (--num_buf) {
231                 i = skb_shinfo(skb)->nr_frags;
232                 if (i >= MAX_SKB_FRAGS) {
233                         pr_debug("%s: packet too long\n", skb->dev->name);
234                         skb->dev->stats.rx_length_errors++;
235                         return -EINVAL;
236                 }
237                 page = virtqueue_get_buf(vi->rvq, &len);
238                 if (!page) {
239                         pr_debug("%s: rx error: %d buffers missing\n",
240                                  skb->dev->name, hdr->mhdr.num_buffers);
241                         skb->dev->stats.rx_length_errors++;
242                         return -EINVAL;
243                 }
244
245                 if (len > PAGE_SIZE)
246                         len = PAGE_SIZE;
247
248                 set_skb_frag(skb, page, 0, &len);
249
250                 --vi->num;
251         }
252         return 0;
253 }
254
255 static void receive_buf(struct net_device *dev, void *buf, unsigned int len)
256 {
257         struct virtnet_info *vi = netdev_priv(dev);
258         struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
259         struct sk_buff *skb;
260         struct page *page;
261         struct skb_vnet_hdr *hdr;
262
263         if (unlikely(len < sizeof(struct virtio_net_hdr) + ETH_HLEN)) {
264                 pr_debug("%s: short packet %i\n", dev->name, len);
265                 dev->stats.rx_length_errors++;
266                 if (vi->mergeable_rx_bufs || vi->big_packets)
267                         give_pages(vi, buf);
268                 else
269                         dev_kfree_skb(buf);
270                 return;
271         }
272
273         if (!vi->mergeable_rx_bufs && !vi->big_packets) {
274                 skb = buf;
275                 len -= sizeof(struct virtio_net_hdr);
276                 skb_trim(skb, len);
277         } else {
278                 page = buf;
279                 skb = page_to_skb(vi, page, len);
280                 if (unlikely(!skb)) {
281                         dev->stats.rx_dropped++;
282                         give_pages(vi, page);
283                         return;
284                 }
285                 if (vi->mergeable_rx_bufs)
286                         if (receive_mergeable(vi, skb)) {
287                                 dev_kfree_skb(skb);
288                                 return;
289                         }
290         }
291
292         hdr = skb_vnet_hdr(skb);
293
294         u64_stats_update_begin(&stats->syncp);
295         stats->rx_bytes += skb->len;
296         stats->rx_packets++;
297         u64_stats_update_end(&stats->syncp);
298
299         if (hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
300                 pr_debug("Needs csum!\n");
301                 if (!skb_partial_csum_set(skb,
302                                           hdr->hdr.csum_start,
303                                           hdr->hdr.csum_offset))
304                         goto frame_err;
305         } else if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID) {
306                 skb->ip_summed = CHECKSUM_UNNECESSARY;
307         }
308
309         skb->protocol = eth_type_trans(skb, dev);
310         pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
311                  ntohs(skb->protocol), skb->len, skb->pkt_type);
312
313         if (hdr->hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
314                 pr_debug("GSO!\n");
315                 switch (hdr->hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
316                 case VIRTIO_NET_HDR_GSO_TCPV4:
317                         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
318                         break;
319                 case VIRTIO_NET_HDR_GSO_UDP:
320                         skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
321                         break;
322                 case VIRTIO_NET_HDR_GSO_TCPV6:
323                         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
324                         break;
325                 default:
326                         if (net_ratelimit())
327                                 printk(KERN_WARNING "%s: bad gso type %u.\n",
328                                        dev->name, hdr->hdr.gso_type);
329                         goto frame_err;
330                 }
331
332                 if (hdr->hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
333                         skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
334
335                 skb_shinfo(skb)->gso_size = hdr->hdr.gso_size;
336                 if (skb_shinfo(skb)->gso_size == 0) {
337                         if (net_ratelimit())
338                                 printk(KERN_WARNING "%s: zero gso size.\n",
339                                        dev->name);
340                         goto frame_err;
341                 }
342
343                 /* Header must be checked, and gso_segs computed. */
344                 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
345                 skb_shinfo(skb)->gso_segs = 0;
346         }
347
348         netif_receive_skb(skb);
349         return;
350
351 frame_err:
352         dev->stats.rx_frame_errors++;
353         dev_kfree_skb(skb);
354 }
355
356 static int add_recvbuf_small(struct virtnet_info *vi, gfp_t gfp)
357 {
358         struct sk_buff *skb;
359         struct skb_vnet_hdr *hdr;
360         int err;
361
362         skb = __netdev_alloc_skb_ip_align(vi->dev, MAX_PACKET_LEN, gfp);
363         if (unlikely(!skb))
364                 return -ENOMEM;
365
366         skb_put(skb, MAX_PACKET_LEN);
367
368         hdr = skb_vnet_hdr(skb);
369         sg_set_buf(vi->rx_sg, &hdr->hdr, sizeof hdr->hdr);
370
371         skb_to_sgvec(skb, vi->rx_sg + 1, 0, skb->len);
372
373         err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, 2, skb, gfp);
374         if (err < 0)
375                 dev_kfree_skb(skb);
376
377         return err;
378 }
379
380 static int add_recvbuf_big(struct virtnet_info *vi, gfp_t gfp)
381 {
382         struct page *first, *list = NULL;
383         char *p;
384         int i, err, offset;
385
386         /* page in vi->rx_sg[MAX_SKB_FRAGS + 1] is list tail */
387         for (i = MAX_SKB_FRAGS + 1; i > 1; --i) {
388                 first = get_a_page(vi, gfp);
389                 if (!first) {
390                         if (list)
391                                 give_pages(vi, list);
392                         return -ENOMEM;
393                 }
394                 sg_set_buf(&vi->rx_sg[i], page_address(first), PAGE_SIZE);
395
396                 /* chain new page in list head to match sg */
397                 first->private = (unsigned long)list;
398                 list = first;
399         }
400
401         first = get_a_page(vi, gfp);
402         if (!first) {
403                 give_pages(vi, list);
404                 return -ENOMEM;
405         }
406         p = page_address(first);
407
408         /* vi->rx_sg[0], vi->rx_sg[1] share the same page */
409         /* a separated vi->rx_sg[0] for virtio_net_hdr only due to QEMU bug */
410         sg_set_buf(&vi->rx_sg[0], p, sizeof(struct virtio_net_hdr));
411
412         /* vi->rx_sg[1] for data packet, from offset */
413         offset = sizeof(struct padded_vnet_hdr);
414         sg_set_buf(&vi->rx_sg[1], p + offset, PAGE_SIZE - offset);
415
416         /* chain first in list head */
417         first->private = (unsigned long)list;
418         err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, MAX_SKB_FRAGS + 2,
419                                 first, gfp);
420         if (err < 0)
421                 give_pages(vi, first);
422
423         return err;
424 }
425
426 static int add_recvbuf_mergeable(struct virtnet_info *vi, gfp_t gfp)
427 {
428         struct page *page;
429         int err;
430
431         page = get_a_page(vi, gfp);
432         if (!page)
433                 return -ENOMEM;
434
435         sg_init_one(vi->rx_sg, page_address(page), PAGE_SIZE);
436
437         err = virtqueue_add_buf(vi->rvq, vi->rx_sg, 0, 1, page, gfp);
438         if (err < 0)
439                 give_pages(vi, page);
440
441         return err;
442 }
443
444 /*
445  * Returns false if we couldn't fill entirely (OOM).
446  *
447  * Normally run in the receive path, but can also be run from ndo_open
448  * before we're receiving packets, or from refill_work which is
449  * careful to disable receiving (using napi_disable).
450  */
451 static bool try_fill_recv(struct virtnet_info *vi, gfp_t gfp)
452 {
453         int err;
454         bool oom;
455
456         do {
457                 if (vi->mergeable_rx_bufs)
458                         err = add_recvbuf_mergeable(vi, gfp);
459                 else if (vi->big_packets)
460                         err = add_recvbuf_big(vi, gfp);
461                 else
462                         err = add_recvbuf_small(vi, gfp);
463
464                 oom = err == -ENOMEM;
465                 if (err < 0)
466                         break;
467                 ++vi->num;
468         } while (err > 0);
469         if (unlikely(vi->num > vi->max))
470                 vi->max = vi->num;
471         virtqueue_kick(vi->rvq);
472         return !oom;
473 }
474
475 static void skb_recv_done(struct virtqueue *rvq)
476 {
477         struct virtnet_info *vi = rvq->vdev->priv;
478         /* Schedule NAPI, Suppress further interrupts if successful. */
479         if (napi_schedule_prep(&vi->napi)) {
480                 virtqueue_disable_cb(rvq);
481                 __napi_schedule(&vi->napi);
482         }
483 }
484
485 static void virtnet_napi_enable(struct virtnet_info *vi)
486 {
487         napi_enable(&vi->napi);
488
489         /* If all buffers were filled by other side before we napi_enabled, we
490          * won't get another interrupt, so process any outstanding packets
491          * now.  virtnet_poll wants re-enable the queue, so we disable here.
492          * We synchronize against interrupts via NAPI_STATE_SCHED */
493         if (napi_schedule_prep(&vi->napi)) {
494                 virtqueue_disable_cb(vi->rvq);
495                 __napi_schedule(&vi->napi);
496         }
497 }
498
499 static void refill_work(struct work_struct *work)
500 {
501         struct virtnet_info *vi;
502         bool still_empty;
503
504         vi = container_of(work, struct virtnet_info, refill.work);
505         napi_disable(&vi->napi);
506         still_empty = !try_fill_recv(vi, GFP_KERNEL);
507         virtnet_napi_enable(vi);
508
509         /* In theory, this can happen: if we don't get any buffers in
510          * we will *never* try to fill again. */
511         if (still_empty)
512                 queue_delayed_work(system_nrt_wq, &vi->refill, HZ/2);
513 }
514
515 static int virtnet_poll(struct napi_struct *napi, int budget)
516 {
517         struct virtnet_info *vi = container_of(napi, struct virtnet_info, napi);
518         void *buf;
519         unsigned int len, received = 0;
520
521 again:
522         while (received < budget &&
523                (buf = virtqueue_get_buf(vi->rvq, &len)) != NULL) {
524                 receive_buf(vi->dev, buf, len);
525                 --vi->num;
526                 received++;
527         }
528
529         if (vi->num < vi->max / 2) {
530                 if (!try_fill_recv(vi, GFP_ATOMIC))
531                         queue_delayed_work(system_nrt_wq, &vi->refill, 0);
532         }
533
534         /* Out of packets? */
535         if (received < budget) {
536                 napi_complete(napi);
537                 if (unlikely(!virtqueue_enable_cb(vi->rvq)) &&
538                     napi_schedule_prep(napi)) {
539                         virtqueue_disable_cb(vi->rvq);
540                         __napi_schedule(napi);
541                         goto again;
542                 }
543         }
544
545         return received;
546 }
547
548 static unsigned int free_old_xmit_skbs(struct virtnet_info *vi)
549 {
550         struct sk_buff *skb;
551         unsigned int len, tot_sgs = 0;
552         struct virtnet_stats __percpu *stats = this_cpu_ptr(vi->stats);
553
554         while ((skb = virtqueue_get_buf(vi->svq, &len)) != NULL) {
555                 pr_debug("Sent skb %p\n", skb);
556
557                 u64_stats_update_begin(&stats->syncp);
558                 stats->tx_bytes += skb->len;
559                 stats->tx_packets++;
560                 u64_stats_update_end(&stats->syncp);
561
562                 tot_sgs += skb_vnet_hdr(skb)->num_sg;
563                 dev_kfree_skb_any(skb);
564         }
565         return tot_sgs;
566 }
567
568 static int xmit_skb(struct virtnet_info *vi, struct sk_buff *skb)
569 {
570         struct skb_vnet_hdr *hdr = skb_vnet_hdr(skb);
571         const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
572
573         pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest);
574
575         if (skb->ip_summed == CHECKSUM_PARTIAL) {
576                 hdr->hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
577                 hdr->hdr.csum_start = skb_checksum_start_offset(skb);
578                 hdr->hdr.csum_offset = skb->csum_offset;
579         } else {
580                 hdr->hdr.flags = 0;
581                 hdr->hdr.csum_offset = hdr->hdr.csum_start = 0;
582         }
583
584         if (skb_is_gso(skb)) {
585                 hdr->hdr.hdr_len = skb_headlen(skb);
586                 hdr->hdr.gso_size = skb_shinfo(skb)->gso_size;
587                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
588                         hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
589                 else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
590                         hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
591                 else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP)
592                         hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
593                 else
594                         BUG();
595                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCP_ECN)
596                         hdr->hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
597         } else {
598                 hdr->hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
599                 hdr->hdr.gso_size = hdr->hdr.hdr_len = 0;
600         }
601
602         hdr->mhdr.num_buffers = 0;
603
604         /* Encode metadata header at front. */
605         if (vi->mergeable_rx_bufs)
606                 sg_set_buf(vi->tx_sg, &hdr->mhdr, sizeof hdr->mhdr);
607         else
608                 sg_set_buf(vi->tx_sg, &hdr->hdr, sizeof hdr->hdr);
609
610         hdr->num_sg = skb_to_sgvec(skb, vi->tx_sg + 1, 0, skb->len) + 1;
611         return virtqueue_add_buf(vi->svq, vi->tx_sg, hdr->num_sg,
612                                  0, skb, GFP_ATOMIC);
613 }
614
615 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
616 {
617         struct virtnet_info *vi = netdev_priv(dev);
618         int capacity;
619
620         /* Free up any pending old buffers before queueing new ones. */
621         free_old_xmit_skbs(vi);
622
623         /* Try to transmit */
624         capacity = xmit_skb(vi, skb);
625
626         /* This can happen with OOM and indirect buffers. */
627         if (unlikely(capacity < 0)) {
628                 if (net_ratelimit()) {
629                         if (likely(capacity == -ENOMEM)) {
630                                 dev_warn(&dev->dev,
631                                          "TX queue failure: out of memory\n");
632                         } else {
633                                 dev->stats.tx_fifo_errors++;
634                                 dev_warn(&dev->dev,
635                                          "Unexpected TX queue failure: %d\n",
636                                          capacity);
637                         }
638                 }
639                 dev->stats.tx_dropped++;
640                 kfree_skb(skb);
641                 return NETDEV_TX_OK;
642         }
643         virtqueue_kick(vi->svq);
644
645         /* Don't wait up for transmitted skbs to be freed. */
646         skb_orphan(skb);
647         nf_reset(skb);
648
649         /* Apparently nice girls don't return TX_BUSY; stop the queue
650          * before it gets out of hand.  Naturally, this wastes entries. */
651         if (capacity < 2+MAX_SKB_FRAGS) {
652                 netif_stop_queue(dev);
653                 if (unlikely(!virtqueue_enable_cb_delayed(vi->svq))) {
654                         /* More just got used, free them then recheck. */
655                         capacity += free_old_xmit_skbs(vi);
656                         if (capacity >= 2+MAX_SKB_FRAGS) {
657                                 netif_start_queue(dev);
658                                 virtqueue_disable_cb(vi->svq);
659                         }
660                 }
661         }
662
663         return NETDEV_TX_OK;
664 }
665
666 static int virtnet_set_mac_address(struct net_device *dev, void *p)
667 {
668         struct virtnet_info *vi = netdev_priv(dev);
669         struct virtio_device *vdev = vi->vdev;
670         int ret;
671
672         ret = eth_mac_addr(dev, p);
673         if (ret)
674                 return ret;
675
676         if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC))
677                 vdev->config->set(vdev, offsetof(struct virtio_net_config, mac),
678                                   dev->dev_addr, dev->addr_len);
679
680         return 0;
681 }
682
683 static struct rtnl_link_stats64 *virtnet_stats(struct net_device *dev,
684                                                struct rtnl_link_stats64 *tot)
685 {
686         struct virtnet_info *vi = netdev_priv(dev);
687         int cpu;
688         unsigned int start;
689
690         for_each_possible_cpu(cpu) {
691                 struct virtnet_stats __percpu *stats
692                         = per_cpu_ptr(vi->stats, cpu);
693                 u64 tpackets, tbytes, rpackets, rbytes;
694
695                 do {
696                         start = u64_stats_fetch_begin(&stats->syncp);
697                         tpackets = stats->tx_packets;
698                         tbytes   = stats->tx_bytes;
699                         rpackets = stats->rx_packets;
700                         rbytes   = stats->rx_bytes;
701                 } while (u64_stats_fetch_retry(&stats->syncp, start));
702
703                 tot->rx_packets += rpackets;
704                 tot->tx_packets += tpackets;
705                 tot->rx_bytes   += rbytes;
706                 tot->tx_bytes   += tbytes;
707         }
708
709         tot->tx_dropped = dev->stats.tx_dropped;
710         tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
711         tot->rx_dropped = dev->stats.rx_dropped;
712         tot->rx_length_errors = dev->stats.rx_length_errors;
713         tot->rx_frame_errors = dev->stats.rx_frame_errors;
714
715         return tot;
716 }
717
718 #ifdef CONFIG_NET_POLL_CONTROLLER
719 static void virtnet_netpoll(struct net_device *dev)
720 {
721         struct virtnet_info *vi = netdev_priv(dev);
722
723         napi_schedule(&vi->napi);
724 }
725 #endif
726
727 static int virtnet_open(struct net_device *dev)
728 {
729         struct virtnet_info *vi = netdev_priv(dev);
730
731         /* Make sure we have some buffers: if oom use wq. */
732         if (!try_fill_recv(vi, GFP_KERNEL))
733                 queue_delayed_work(system_nrt_wq, &vi->refill, 0);
734
735         virtnet_napi_enable(vi);
736         return 0;
737 }
738
739 /*
740  * Send command via the control virtqueue and check status.  Commands
741  * supported by the hypervisor, as indicated by feature bits, should
742  * never fail unless improperly formated.
743  */
744 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd,
745                                  struct scatterlist *data, int out, int in)
746 {
747         struct scatterlist *s, sg[VIRTNET_SEND_COMMAND_SG_MAX + 2];
748         struct virtio_net_ctrl_hdr ctrl;
749         virtio_net_ctrl_ack status = ~0;
750         unsigned int tmp;
751         int i;
752
753         /* Caller should know better */
754         BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ||
755                 (out + in > VIRTNET_SEND_COMMAND_SG_MAX));
756
757         out++; /* Add header */
758         in++; /* Add return status */
759
760         ctrl.class = class;
761         ctrl.cmd = cmd;
762
763         sg_init_table(sg, out + in);
764
765         sg_set_buf(&sg[0], &ctrl, sizeof(ctrl));
766         for_each_sg(data, s, out + in - 2, i)
767                 sg_set_buf(&sg[i + 1], sg_virt(s), s->length);
768         sg_set_buf(&sg[out + in - 1], &status, sizeof(status));
769
770         BUG_ON(virtqueue_add_buf(vi->cvq, sg, out, in, vi, GFP_ATOMIC) < 0);
771
772         virtqueue_kick(vi->cvq);
773
774         /*
775          * Spin for a response, the kick causes an ioport write, trapping
776          * into the hypervisor, so the request should be handled immediately.
777          */
778         while (!virtqueue_get_buf(vi->cvq, &tmp))
779                 cpu_relax();
780
781         return status == VIRTIO_NET_OK;
782 }
783
784 static int virtnet_close(struct net_device *dev)
785 {
786         struct virtnet_info *vi = netdev_priv(dev);
787
788         /* Make sure refill_work doesn't re-enable napi! */
789         cancel_delayed_work_sync(&vi->refill);
790         napi_disable(&vi->napi);
791
792         return 0;
793 }
794
795 static void virtnet_set_rx_mode(struct net_device *dev)
796 {
797         struct virtnet_info *vi = netdev_priv(dev);
798         struct scatterlist sg[2];
799         u8 promisc, allmulti;
800         struct virtio_net_ctrl_mac *mac_data;
801         struct netdev_hw_addr *ha;
802         int uc_count;
803         int mc_count;
804         void *buf;
805         int i;
806
807         /* We can't dynamicaly set ndo_set_rx_mode, so return gracefully */
808         if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX))
809                 return;
810
811         promisc = ((dev->flags & IFF_PROMISC) != 0);
812         allmulti = ((dev->flags & IFF_ALLMULTI) != 0);
813
814         sg_init_one(sg, &promisc, sizeof(promisc));
815
816         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
817                                   VIRTIO_NET_CTRL_RX_PROMISC,
818                                   sg, 1, 0))
819                 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n",
820                          promisc ? "en" : "dis");
821
822         sg_init_one(sg, &allmulti, sizeof(allmulti));
823
824         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX,
825                                   VIRTIO_NET_CTRL_RX_ALLMULTI,
826                                   sg, 1, 0))
827                 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n",
828                          allmulti ? "en" : "dis");
829
830         uc_count = netdev_uc_count(dev);
831         mc_count = netdev_mc_count(dev);
832         /* MAC filter - use one buffer for both lists */
833         buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) +
834                       (2 * sizeof(mac_data->entries)), GFP_ATOMIC);
835         mac_data = buf;
836         if (!buf) {
837                 dev_warn(&dev->dev, "No memory for MAC address buffer\n");
838                 return;
839         }
840
841         sg_init_table(sg, 2);
842
843         /* Store the unicast list and count in the front of the buffer */
844         mac_data->entries = uc_count;
845         i = 0;
846         netdev_for_each_uc_addr(ha, dev)
847                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
848
849         sg_set_buf(&sg[0], mac_data,
850                    sizeof(mac_data->entries) + (uc_count * ETH_ALEN));
851
852         /* multicast list and count fill the end */
853         mac_data = (void *)&mac_data->macs[uc_count][0];
854
855         mac_data->entries = mc_count;
856         i = 0;
857         netdev_for_each_mc_addr(ha, dev)
858                 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN);
859
860         sg_set_buf(&sg[1], mac_data,
861                    sizeof(mac_data->entries) + (mc_count * ETH_ALEN));
862
863         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC,
864                                   VIRTIO_NET_CTRL_MAC_TABLE_SET,
865                                   sg, 2, 0))
866                 dev_warn(&dev->dev, "Failed to set MAC fitler table.\n");
867
868         kfree(buf);
869 }
870
871 static int virtnet_vlan_rx_add_vid(struct net_device *dev, u16 vid)
872 {
873         struct virtnet_info *vi = netdev_priv(dev);
874         struct scatterlist sg;
875
876         sg_init_one(&sg, &vid, sizeof(vid));
877
878         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
879                                   VIRTIO_NET_CTRL_VLAN_ADD, &sg, 1, 0))
880                 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid);
881         return 0;
882 }
883
884 static int virtnet_vlan_rx_kill_vid(struct net_device *dev, u16 vid)
885 {
886         struct virtnet_info *vi = netdev_priv(dev);
887         struct scatterlist sg;
888
889         sg_init_one(&sg, &vid, sizeof(vid));
890
891         if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN,
892                                   VIRTIO_NET_CTRL_VLAN_DEL, &sg, 1, 0))
893                 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid);
894         return 0;
895 }
896
897 static void virtnet_get_ringparam(struct net_device *dev,
898                                 struct ethtool_ringparam *ring)
899 {
900         struct virtnet_info *vi = netdev_priv(dev);
901
902         ring->rx_max_pending = virtqueue_get_vring_size(vi->rvq);
903         ring->tx_max_pending = virtqueue_get_vring_size(vi->svq);
904         ring->rx_pending = ring->rx_max_pending;
905         ring->tx_pending = ring->tx_max_pending;
906
907 }
908
909
910 static void virtnet_get_drvinfo(struct net_device *dev,
911                                 struct ethtool_drvinfo *info)
912 {
913         struct virtnet_info *vi = netdev_priv(dev);
914         struct virtio_device *vdev = vi->vdev;
915
916         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
917         strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
918         strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
919
920 }
921
922 static const struct ethtool_ops virtnet_ethtool_ops = {
923         .get_drvinfo = virtnet_get_drvinfo,
924         .get_link = ethtool_op_get_link,
925         .get_ringparam = virtnet_get_ringparam,
926 };
927
928 #define MIN_MTU 68
929 #define MAX_MTU 65535
930
931 static int virtnet_change_mtu(struct net_device *dev, int new_mtu)
932 {
933         if (new_mtu < MIN_MTU || new_mtu > MAX_MTU)
934                 return -EINVAL;
935         dev->mtu = new_mtu;
936         return 0;
937 }
938
939 static const struct net_device_ops virtnet_netdev = {
940         .ndo_open            = virtnet_open,
941         .ndo_stop            = virtnet_close,
942         .ndo_start_xmit      = start_xmit,
943         .ndo_validate_addr   = eth_validate_addr,
944         .ndo_set_mac_address = virtnet_set_mac_address,
945         .ndo_set_rx_mode     = virtnet_set_rx_mode,
946         .ndo_change_mtu      = virtnet_change_mtu,
947         .ndo_get_stats64     = virtnet_stats,
948         .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid,
949         .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
950 #ifdef CONFIG_NET_POLL_CONTROLLER
951         .ndo_poll_controller = virtnet_netpoll,
952 #endif
953 };
954
955 static void virtnet_update_status(struct virtnet_info *vi)
956 {
957         u16 v;
958
959         if (virtio_config_val(vi->vdev, VIRTIO_NET_F_STATUS,
960                               offsetof(struct virtio_net_config, status),
961                               &v) < 0)
962                 return;
963
964         /* Ignore unknown (future) status bits */
965         v &= VIRTIO_NET_S_LINK_UP;
966
967         if (vi->status == v)
968                 return;
969
970         vi->status = v;
971
972         if (vi->status & VIRTIO_NET_S_LINK_UP) {
973                 netif_carrier_on(vi->dev);
974                 netif_wake_queue(vi->dev);
975         } else {
976                 netif_carrier_off(vi->dev);
977                 netif_stop_queue(vi->dev);
978         }
979 }
980
981 static void virtnet_config_changed(struct virtio_device *vdev)
982 {
983         struct virtnet_info *vi = vdev->priv;
984
985         virtnet_update_status(vi);
986 }
987
988 static int virtnet_probe(struct virtio_device *vdev)
989 {
990         int err;
991         struct net_device *dev;
992         struct virtnet_info *vi;
993         struct virtqueue *vqs[3];
994         vq_callback_t *callbacks[] = { skb_recv_done, skb_xmit_done, NULL};
995         const char *names[] = { "input", "output", "control" };
996         int nvqs;
997
998         /* Allocate ourselves a network device with room for our info */
999         dev = alloc_etherdev(sizeof(struct virtnet_info));
1000         if (!dev)
1001                 return -ENOMEM;
1002
1003         /* Set up network device as normal. */
1004         dev->priv_flags |= IFF_UNICAST_FLT;
1005         dev->netdev_ops = &virtnet_netdev;
1006         dev->features = NETIF_F_HIGHDMA;
1007
1008         SET_ETHTOOL_OPS(dev, &virtnet_ethtool_ops);
1009         SET_NETDEV_DEV(dev, &vdev->dev);
1010
1011         /* Do we support "hardware" checksums? */
1012         if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) {
1013                 /* This opens up the world of extra features. */
1014                 dev->hw_features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
1015                 if (csum)
1016                         dev->features |= NETIF_F_HW_CSUM|NETIF_F_SG|NETIF_F_FRAGLIST;
1017
1018                 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) {
1019                         dev->hw_features |= NETIF_F_TSO | NETIF_F_UFO
1020                                 | NETIF_F_TSO_ECN | NETIF_F_TSO6;
1021                 }
1022                 /* Individual feature bits: what can host handle? */
1023                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4))
1024                         dev->hw_features |= NETIF_F_TSO;
1025                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6))
1026                         dev->hw_features |= NETIF_F_TSO6;
1027                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN))
1028                         dev->hw_features |= NETIF_F_TSO_ECN;
1029                 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UFO))
1030                         dev->hw_features |= NETIF_F_UFO;
1031
1032                 if (gso)
1033                         dev->features |= dev->hw_features & (NETIF_F_ALL_TSO|NETIF_F_UFO);
1034                 /* (!csum && gso) case will be fixed by register_netdev() */
1035         }
1036
1037         /* Configuration may specify what MAC to use.  Otherwise random. */
1038         if (virtio_config_val_len(vdev, VIRTIO_NET_F_MAC,
1039                                   offsetof(struct virtio_net_config, mac),
1040                                   dev->dev_addr, dev->addr_len) < 0)
1041                 random_ether_addr(dev->dev_addr);
1042
1043         /* Set up our device-specific information */
1044         vi = netdev_priv(dev);
1045         netif_napi_add(dev, &vi->napi, virtnet_poll, napi_weight);
1046         vi->dev = dev;
1047         vi->vdev = vdev;
1048         vdev->priv = vi;
1049         vi->pages = NULL;
1050         vi->stats = alloc_percpu(struct virtnet_stats);
1051         err = -ENOMEM;
1052         if (vi->stats == NULL)
1053                 goto free;
1054
1055         INIT_DELAYED_WORK(&vi->refill, refill_work);
1056         sg_init_table(vi->rx_sg, ARRAY_SIZE(vi->rx_sg));
1057         sg_init_table(vi->tx_sg, ARRAY_SIZE(vi->tx_sg));
1058
1059         /* If we can receive ANY GSO packets, we must allocate large ones. */
1060         if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) ||
1061             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) ||
1062             virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN))
1063                 vi->big_packets = true;
1064
1065         if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
1066                 vi->mergeable_rx_bufs = true;
1067
1068         /* We expect two virtqueues, receive then send,
1069          * and optionally control. */
1070         nvqs = virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ) ? 3 : 2;
1071
1072         err = vdev->config->find_vqs(vdev, nvqs, vqs, callbacks, names);
1073         if (err)
1074                 goto free_stats;
1075
1076         vi->rvq = vqs[0];
1077         vi->svq = vqs[1];
1078
1079         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)) {
1080                 vi->cvq = vqs[2];
1081
1082                 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN))
1083                         dev->features |= NETIF_F_HW_VLAN_FILTER;
1084         }
1085
1086         err = register_netdev(dev);
1087         if (err) {
1088                 pr_debug("virtio_net: registering device failed\n");
1089                 goto free_vqs;
1090         }
1091
1092         /* Last of all, set up some receive buffers. */
1093         try_fill_recv(vi, GFP_KERNEL);
1094
1095         /* If we didn't even get one input buffer, we're useless. */
1096         if (vi->num == 0) {
1097                 err = -ENOMEM;
1098                 goto unregister;
1099         }
1100
1101         /* Assume link up if device can't report link status,
1102            otherwise get link status from config. */
1103         if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) {
1104                 netif_carrier_off(dev);
1105                 virtnet_update_status(vi);
1106         } else {
1107                 vi->status = VIRTIO_NET_S_LINK_UP;
1108                 netif_carrier_on(dev);
1109         }
1110
1111         pr_debug("virtnet: registered device %s\n", dev->name);
1112         return 0;
1113
1114 unregister:
1115         unregister_netdev(dev);
1116 free_vqs:
1117         vdev->config->del_vqs(vdev);
1118 free_stats:
1119         free_percpu(vi->stats);
1120 free:
1121         free_netdev(dev);
1122         return err;
1123 }
1124
1125 static void free_unused_bufs(struct virtnet_info *vi)
1126 {
1127         void *buf;
1128         while (1) {
1129                 buf = virtqueue_detach_unused_buf(vi->svq);
1130                 if (!buf)
1131                         break;
1132                 dev_kfree_skb(buf);
1133         }
1134         while (1) {
1135                 buf = virtqueue_detach_unused_buf(vi->rvq);
1136                 if (!buf)
1137                         break;
1138                 if (vi->mergeable_rx_bufs || vi->big_packets)
1139                         give_pages(vi, buf);
1140                 else
1141                         dev_kfree_skb(buf);
1142                 --vi->num;
1143         }
1144         BUG_ON(vi->num != 0);
1145 }
1146
1147 static void __devexit virtnet_remove(struct virtio_device *vdev)
1148 {
1149         struct virtnet_info *vi = vdev->priv;
1150
1151         /* Stop all the virtqueues. */
1152         vdev->config->reset(vdev);
1153
1154         unregister_netdev(vi->dev);
1155
1156         /* Free unused buffers in both send and recv, if any. */
1157         free_unused_bufs(vi);
1158
1159         vdev->config->del_vqs(vi->vdev);
1160
1161         while (vi->pages)
1162                 __free_pages(get_a_page(vi, GFP_KERNEL), 0);
1163
1164         free_percpu(vi->stats);
1165         free_netdev(vi->dev);
1166 }
1167
1168 static struct virtio_device_id id_table[] = {
1169         { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
1170         { 0 },
1171 };
1172
1173 static unsigned int features[] = {
1174         VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM,
1175         VIRTIO_NET_F_GSO, VIRTIO_NET_F_MAC,
1176         VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6,
1177         VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6,
1178         VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO,
1179         VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ,
1180         VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN,
1181 };
1182
1183 static struct virtio_driver virtio_net_driver = {
1184         .feature_table = features,
1185         .feature_table_size = ARRAY_SIZE(features),
1186         .driver.name =  KBUILD_MODNAME,
1187         .driver.owner = THIS_MODULE,
1188         .id_table =     id_table,
1189         .probe =        virtnet_probe,
1190         .remove =       __devexit_p(virtnet_remove),
1191         .config_changed = virtnet_config_changed,
1192 };
1193
1194 static int __init init(void)
1195 {
1196         return register_virtio_driver(&virtio_net_driver);
1197 }
1198
1199 static void __exit fini(void)
1200 {
1201         unregister_virtio_driver(&virtio_net_driver);
1202 }
1203 module_init(init);
1204 module_exit(fini);
1205
1206 MODULE_DEVICE_TABLE(virtio, id_table);
1207 MODULE_DESCRIPTION("Virtio network driver");
1208 MODULE_LICENSE("GPL");