ipvs: extend connection flags to 32 bits
[linux-2.6.git] / net / netfilter / ipvs / ip_vs_core.c
1 /*
2  * IPVS         An implementation of the IP virtual server support for the
3  *              LINUX operating system.  IPVS is now implemented as a module
4  *              over the Netfilter framework. IPVS can be used to build a
5  *              high-performance and highly available server based on a
6  *              cluster of servers.
7  *
8  * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
9  *              Peter Kese <peter.kese@ijs.si>
10  *              Julian Anastasov <ja@ssi.bg>
11  *
12  *              This program is free software; you can redistribute it and/or
13  *              modify it under the terms of the GNU General Public License
14  *              as published by the Free Software Foundation; either version
15  *              2 of the License, or (at your option) any later version.
16  *
17  * The IPVS code for kernel 2.2 was done by Wensong Zhang and Peter Kese,
18  * with changes/fixes from Julian Anastasov, Lars Marowsky-Bree, Horms
19  * and others.
20  *
21  * Changes:
22  *      Paul `Rusty' Russell            properly handle non-linear skbs
23  *      Harald Welte                    don't use nfcache
24  *
25  */
26
27 #define KMSG_COMPONENT "IPVS"
28 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
29
30 #include <linux/module.h>
31 #include <linux/kernel.h>
32 #include <linux/ip.h>
33 #include <linux/tcp.h>
34 #include <linux/sctp.h>
35 #include <linux/icmp.h>
36 #include <linux/slab.h>
37
38 #include <net/ip.h>
39 #include <net/tcp.h>
40 #include <net/udp.h>
41 #include <net/icmp.h>                   /* for icmp_send */
42 #include <net/route.h>
43 #include <net/ip6_checksum.h>
44
45 #include <linux/netfilter.h>
46 #include <linux/netfilter_ipv4.h>
47
48 #ifdef CONFIG_IP_VS_IPV6
49 #include <net/ipv6.h>
50 #include <linux/netfilter_ipv6.h>
51 #endif
52
53 #include <net/ip_vs.h>
54
55
56 EXPORT_SYMBOL(register_ip_vs_scheduler);
57 EXPORT_SYMBOL(unregister_ip_vs_scheduler);
58 EXPORT_SYMBOL(ip_vs_proto_name);
59 EXPORT_SYMBOL(ip_vs_conn_new);
60 EXPORT_SYMBOL(ip_vs_conn_in_get);
61 EXPORT_SYMBOL(ip_vs_conn_out_get);
62 #ifdef CONFIG_IP_VS_PROTO_TCP
63 EXPORT_SYMBOL(ip_vs_tcp_conn_listen);
64 #endif
65 EXPORT_SYMBOL(ip_vs_conn_put);
66 #ifdef CONFIG_IP_VS_DEBUG
67 EXPORT_SYMBOL(ip_vs_get_debug_level);
68 #endif
69
70
71 /* ID used in ICMP lookups */
72 #define icmp_id(icmph)          (((icmph)->un).echo.id)
73 #define icmpv6_id(icmph)        (icmph->icmp6_dataun.u_echo.identifier)
74
75 const char *ip_vs_proto_name(unsigned proto)
76 {
77         static char buf[20];
78
79         switch (proto) {
80         case IPPROTO_IP:
81                 return "IP";
82         case IPPROTO_UDP:
83                 return "UDP";
84         case IPPROTO_TCP:
85                 return "TCP";
86         case IPPROTO_SCTP:
87                 return "SCTP";
88         case IPPROTO_ICMP:
89                 return "ICMP";
90 #ifdef CONFIG_IP_VS_IPV6
91         case IPPROTO_ICMPV6:
92                 return "ICMPv6";
93 #endif
94         default:
95                 sprintf(buf, "IP_%d", proto);
96                 return buf;
97         }
98 }
99
100 void ip_vs_init_hash_table(struct list_head *table, int rows)
101 {
102         while (--rows >= 0)
103                 INIT_LIST_HEAD(&table[rows]);
104 }
105
106 static inline void
107 ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
108 {
109         struct ip_vs_dest *dest = cp->dest;
110         if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
111                 spin_lock(&dest->stats.lock);
112                 dest->stats.ustats.inpkts++;
113                 dest->stats.ustats.inbytes += skb->len;
114                 spin_unlock(&dest->stats.lock);
115
116                 spin_lock(&dest->svc->stats.lock);
117                 dest->svc->stats.ustats.inpkts++;
118                 dest->svc->stats.ustats.inbytes += skb->len;
119                 spin_unlock(&dest->svc->stats.lock);
120
121                 spin_lock(&ip_vs_stats.lock);
122                 ip_vs_stats.ustats.inpkts++;
123                 ip_vs_stats.ustats.inbytes += skb->len;
124                 spin_unlock(&ip_vs_stats.lock);
125         }
126 }
127
128
129 static inline void
130 ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
131 {
132         struct ip_vs_dest *dest = cp->dest;
133         if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
134                 spin_lock(&dest->stats.lock);
135                 dest->stats.ustats.outpkts++;
136                 dest->stats.ustats.outbytes += skb->len;
137                 spin_unlock(&dest->stats.lock);
138
139                 spin_lock(&dest->svc->stats.lock);
140                 dest->svc->stats.ustats.outpkts++;
141                 dest->svc->stats.ustats.outbytes += skb->len;
142                 spin_unlock(&dest->svc->stats.lock);
143
144                 spin_lock(&ip_vs_stats.lock);
145                 ip_vs_stats.ustats.outpkts++;
146                 ip_vs_stats.ustats.outbytes += skb->len;
147                 spin_unlock(&ip_vs_stats.lock);
148         }
149 }
150
151
152 static inline void
153 ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc)
154 {
155         spin_lock(&cp->dest->stats.lock);
156         cp->dest->stats.ustats.conns++;
157         spin_unlock(&cp->dest->stats.lock);
158
159         spin_lock(&svc->stats.lock);
160         svc->stats.ustats.conns++;
161         spin_unlock(&svc->stats.lock);
162
163         spin_lock(&ip_vs_stats.lock);
164         ip_vs_stats.ustats.conns++;
165         spin_unlock(&ip_vs_stats.lock);
166 }
167
168
169 static inline int
170 ip_vs_set_state(struct ip_vs_conn *cp, int direction,
171                 const struct sk_buff *skb,
172                 struct ip_vs_protocol *pp)
173 {
174         if (unlikely(!pp->state_transition))
175                 return 0;
176         return pp->state_transition(cp, direction, skb, pp);
177 }
178
179
180 /*
181  *  IPVS persistent scheduling function
182  *  It creates a connection entry according to its template if exists,
183  *  or selects a server and creates a connection entry plus a template.
184  *  Locking: we are svc user (svc->refcnt), so we hold all dests too
185  *  Protocols supported: TCP, UDP
186  */
187 static struct ip_vs_conn *
188 ip_vs_sched_persist(struct ip_vs_service *svc,
189                     const struct sk_buff *skb,
190                     __be16 ports[2])
191 {
192         struct ip_vs_conn *cp = NULL;
193         struct ip_vs_iphdr iph;
194         struct ip_vs_dest *dest;
195         struct ip_vs_conn *ct;
196         __be16  dport;                  /* destination port to forward */
197         unsigned int flags;
198         union nf_inet_addr snet;        /* source network of the client,
199                                            after masking */
200
201         ip_vs_fill_iphdr(svc->af, skb_network_header(skb), &iph);
202
203         /* Mask saddr with the netmask to adjust template granularity */
204 #ifdef CONFIG_IP_VS_IPV6
205         if (svc->af == AF_INET6)
206                 ipv6_addr_prefix(&snet.in6, &iph.saddr.in6, svc->netmask);
207         else
208 #endif
209                 snet.ip = iph.saddr.ip & svc->netmask;
210
211         IP_VS_DBG_BUF(6, "p-schedule: src %s:%u dest %s:%u "
212                       "mnet %s\n",
213                       IP_VS_DBG_ADDR(svc->af, &iph.saddr), ntohs(ports[0]),
214                       IP_VS_DBG_ADDR(svc->af, &iph.daddr), ntohs(ports[1]),
215                       IP_VS_DBG_ADDR(svc->af, &snet));
216
217         /*
218          * As far as we know, FTP is a very complicated network protocol, and
219          * it uses control connection and data connections. For active FTP,
220          * FTP server initialize data connection to the client, its source port
221          * is often 20. For passive FTP, FTP server tells the clients the port
222          * that it passively listens to,  and the client issues the data
223          * connection. In the tunneling or direct routing mode, the load
224          * balancer is on the client-to-server half of connection, the port
225          * number is unknown to the load balancer. So, a conn template like
226          * <caddr, 0, vaddr, 0, daddr, 0> is created for persistent FTP
227          * service, and a template like <caddr, 0, vaddr, vport, daddr, dport>
228          * is created for other persistent services.
229          */
230         if (ports[1] == svc->port) {
231                 /* Check if a template already exists */
232                 if (svc->port != FTPPORT)
233                         ct = ip_vs_ct_in_get(svc->af, iph.protocol, &snet, 0,
234                                              &iph.daddr, ports[1]);
235                 else
236                         ct = ip_vs_ct_in_get(svc->af, iph.protocol, &snet, 0,
237                                              &iph.daddr, 0);
238
239                 if (!ct || !ip_vs_check_template(ct)) {
240                         /*
241                          * No template found or the dest of the connection
242                          * template is not available.
243                          */
244                         dest = svc->scheduler->schedule(svc, skb);
245                         if (dest == NULL) {
246                                 IP_VS_DBG(1, "p-schedule: no dest found.\n");
247                                 return NULL;
248                         }
249
250                         /*
251                          * Create a template like <protocol,caddr,0,
252                          * vaddr,vport,daddr,dport> for non-ftp service,
253                          * and <protocol,caddr,0,vaddr,0,daddr,0>
254                          * for ftp service.
255                          */
256                         if (svc->port != FTPPORT)
257                                 ct = ip_vs_conn_new(svc->af, iph.protocol,
258                                                     &snet, 0,
259                                                     &iph.daddr,
260                                                     ports[1],
261                                                     &dest->addr, dest->port,
262                                                     IP_VS_CONN_F_TEMPLATE,
263                                                     dest);
264                         else
265                                 ct = ip_vs_conn_new(svc->af, iph.protocol,
266                                                     &snet, 0,
267                                                     &iph.daddr, 0,
268                                                     &dest->addr, 0,
269                                                     IP_VS_CONN_F_TEMPLATE,
270                                                     dest);
271                         if (ct == NULL)
272                                 return NULL;
273
274                         ct->timeout = svc->timeout;
275                 } else {
276                         /* set destination with the found template */
277                         dest = ct->dest;
278                 }
279                 dport = dest->port;
280         } else {
281                 /*
282                  * Note: persistent fwmark-based services and persistent
283                  * port zero service are handled here.
284                  * fwmark template: <IPPROTO_IP,caddr,0,fwmark,0,daddr,0>
285                  * port zero template: <protocol,caddr,0,vaddr,0,daddr,0>
286                  */
287                 if (svc->fwmark) {
288                         union nf_inet_addr fwmark = {
289                                 .ip = htonl(svc->fwmark)
290                         };
291
292                         ct = ip_vs_ct_in_get(svc->af, IPPROTO_IP, &snet, 0,
293                                              &fwmark, 0);
294                 } else
295                         ct = ip_vs_ct_in_get(svc->af, iph.protocol, &snet, 0,
296                                              &iph.daddr, 0);
297
298                 if (!ct || !ip_vs_check_template(ct)) {
299                         /*
300                          * If it is not persistent port zero, return NULL,
301                          * otherwise create a connection template.
302                          */
303                         if (svc->port)
304                                 return NULL;
305
306                         dest = svc->scheduler->schedule(svc, skb);
307                         if (dest == NULL) {
308                                 IP_VS_DBG(1, "p-schedule: no dest found.\n");
309                                 return NULL;
310                         }
311
312                         /*
313                          * Create a template according to the service
314                          */
315                         if (svc->fwmark) {
316                                 union nf_inet_addr fwmark = {
317                                         .ip = htonl(svc->fwmark)
318                                 };
319
320                                 ct = ip_vs_conn_new(svc->af, IPPROTO_IP,
321                                                     &snet, 0,
322                                                     &fwmark, 0,
323                                                     &dest->addr, 0,
324                                                     IP_VS_CONN_F_TEMPLATE,
325                                                     dest);
326                         } else
327                                 ct = ip_vs_conn_new(svc->af, iph.protocol,
328                                                     &snet, 0,
329                                                     &iph.daddr, 0,
330                                                     &dest->addr, 0,
331                                                     IP_VS_CONN_F_TEMPLATE,
332                                                     dest);
333                         if (ct == NULL)
334                                 return NULL;
335
336                         ct->timeout = svc->timeout;
337                 } else {
338                         /* set destination with the found template */
339                         dest = ct->dest;
340                 }
341                 dport = ports[1];
342         }
343
344         flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
345                  && iph.protocol == IPPROTO_UDP)?
346                 IP_VS_CONN_F_ONE_PACKET : 0;
347
348         /*
349          *    Create a new connection according to the template
350          */
351         cp = ip_vs_conn_new(svc->af, iph.protocol,
352                             &iph.saddr, ports[0],
353                             &iph.daddr, ports[1],
354                             &dest->addr, dport,
355                             flags,
356                             dest);
357         if (cp == NULL) {
358                 ip_vs_conn_put(ct);
359                 return NULL;
360         }
361
362         /*
363          *    Add its control
364          */
365         ip_vs_control_add(cp, ct);
366         ip_vs_conn_put(ct);
367
368         ip_vs_conn_stats(cp, svc);
369         return cp;
370 }
371
372
373 /*
374  *  IPVS main scheduling function
375  *  It selects a server according to the virtual service, and
376  *  creates a connection entry.
377  *  Protocols supported: TCP, UDP
378  */
379 struct ip_vs_conn *
380 ip_vs_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
381 {
382         struct ip_vs_conn *cp = NULL;
383         struct ip_vs_iphdr iph;
384         struct ip_vs_dest *dest;
385         __be16 _ports[2], *pptr;
386         unsigned int flags;
387
388         ip_vs_fill_iphdr(svc->af, skb_network_header(skb), &iph);
389         pptr = skb_header_pointer(skb, iph.len, sizeof(_ports), _ports);
390         if (pptr == NULL)
391                 return NULL;
392
393         /*
394          *    Persistent service
395          */
396         if (svc->flags & IP_VS_SVC_F_PERSISTENT)
397                 return ip_vs_sched_persist(svc, skb, pptr);
398
399         /*
400          *    Non-persistent service
401          */
402         if (!svc->fwmark && pptr[1] != svc->port) {
403                 if (!svc->port)
404                         pr_err("Schedule: port zero only supported "
405                                "in persistent services, "
406                                "check your ipvs configuration\n");
407                 return NULL;
408         }
409
410         dest = svc->scheduler->schedule(svc, skb);
411         if (dest == NULL) {
412                 IP_VS_DBG(1, "Schedule: no dest found.\n");
413                 return NULL;
414         }
415
416         flags = (svc->flags & IP_VS_SVC_F_ONEPACKET
417                  && iph.protocol == IPPROTO_UDP)?
418                 IP_VS_CONN_F_ONE_PACKET : 0;
419
420         /*
421          *    Create a connection entry.
422          */
423         cp = ip_vs_conn_new(svc->af, iph.protocol,
424                             &iph.saddr, pptr[0],
425                             &iph.daddr, pptr[1],
426                             &dest->addr, dest->port ? dest->port : pptr[1],
427                             flags,
428                             dest);
429         if (cp == NULL)
430                 return NULL;
431
432         IP_VS_DBG_BUF(6, "Schedule fwd:%c c:%s:%u v:%s:%u "
433                       "d:%s:%u conn->flags:%X conn->refcnt:%d\n",
434                       ip_vs_fwd_tag(cp),
435                       IP_VS_DBG_ADDR(svc->af, &cp->caddr), ntohs(cp->cport),
436                       IP_VS_DBG_ADDR(svc->af, &cp->vaddr), ntohs(cp->vport),
437                       IP_VS_DBG_ADDR(svc->af, &cp->daddr), ntohs(cp->dport),
438                       cp->flags, atomic_read(&cp->refcnt));
439
440         ip_vs_conn_stats(cp, svc);
441         return cp;
442 }
443
444
445 /*
446  *  Pass or drop the packet.
447  *  Called by ip_vs_in, when the virtual service is available but
448  *  no destination is available for a new connection.
449  */
450 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
451                 struct ip_vs_protocol *pp)
452 {
453         __be16 _ports[2], *pptr;
454         struct ip_vs_iphdr iph;
455         int unicast;
456         ip_vs_fill_iphdr(svc->af, skb_network_header(skb), &iph);
457
458         pptr = skb_header_pointer(skb, iph.len, sizeof(_ports), _ports);
459         if (pptr == NULL) {
460                 ip_vs_service_put(svc);
461                 return NF_DROP;
462         }
463
464 #ifdef CONFIG_IP_VS_IPV6
465         if (svc->af == AF_INET6)
466                 unicast = ipv6_addr_type(&iph.daddr.in6) & IPV6_ADDR_UNICAST;
467         else
468 #endif
469                 unicast = (inet_addr_type(&init_net, iph.daddr.ip) == RTN_UNICAST);
470
471         /* if it is fwmark-based service, the cache_bypass sysctl is up
472            and the destination is a non-local unicast, then create
473            a cache_bypass connection entry */
474         if (sysctl_ip_vs_cache_bypass && svc->fwmark && unicast) {
475                 int ret, cs;
476                 struct ip_vs_conn *cp;
477                 unsigned int flags = (svc->flags & IP_VS_SVC_F_ONEPACKET &&
478                                       iph.protocol == IPPROTO_UDP)?
479                                       IP_VS_CONN_F_ONE_PACKET : 0;
480                 union nf_inet_addr daddr =  { .all = { 0, 0, 0, 0 } };
481
482                 ip_vs_service_put(svc);
483
484                 /* create a new connection entry */
485                 IP_VS_DBG(6, "%s(): create a cache_bypass entry\n", __func__);
486                 cp = ip_vs_conn_new(svc->af, iph.protocol,
487                                     &iph.saddr, pptr[0],
488                                     &iph.daddr, pptr[1],
489                                     &daddr, 0,
490                                     IP_VS_CONN_F_BYPASS | flags,
491                                     NULL);
492                 if (cp == NULL)
493                         return NF_DROP;
494
495                 /* statistics */
496                 ip_vs_in_stats(cp, skb);
497
498                 /* set state */
499                 cs = ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pp);
500
501                 /* transmit the first SYN packet */
502                 ret = cp->packet_xmit(skb, cp, pp);
503                 /* do not touch skb anymore */
504
505                 atomic_inc(&cp->in_pkts);
506                 ip_vs_conn_put(cp);
507                 return ret;
508         }
509
510         /*
511          * When the virtual ftp service is presented, packets destined
512          * for other services on the VIP may get here (except services
513          * listed in the ipvs table), pass the packets, because it is
514          * not ipvs job to decide to drop the packets.
515          */
516         if ((svc->port == FTPPORT) && (pptr[1] != FTPPORT)) {
517                 ip_vs_service_put(svc);
518                 return NF_ACCEPT;
519         }
520
521         ip_vs_service_put(svc);
522
523         /*
524          * Notify the client that the destination is unreachable, and
525          * release the socket buffer.
526          * Since it is in IP layer, the TCP socket is not actually
527          * created, the TCP RST packet cannot be sent, instead that
528          * ICMP_PORT_UNREACH is sent here no matter it is TCP/UDP. --WZ
529          */
530 #ifdef CONFIG_IP_VS_IPV6
531         if (svc->af == AF_INET6)
532                 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_PORT_UNREACH, 0);
533         else
534 #endif
535                 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
536
537         return NF_DROP;
538 }
539
540 __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset)
541 {
542         return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0));
543 }
544
545 static inline int ip_vs_gather_frags(struct sk_buff *skb, u_int32_t user)
546 {
547         int err = ip_defrag(skb, user);
548
549         if (!err)
550                 ip_send_check(ip_hdr(skb));
551
552         return err;
553 }
554
555 #ifdef CONFIG_IP_VS_IPV6
556 static inline int ip_vs_gather_frags_v6(struct sk_buff *skb, u_int32_t user)
557 {
558         /* TODO IPv6: Find out what to do here for IPv6 */
559         return 0;
560 }
561 #endif
562
563 /*
564  * Packet has been made sufficiently writable in caller
565  * - inout: 1=in->out, 0=out->in
566  */
567 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
568                     struct ip_vs_conn *cp, int inout)
569 {
570         struct iphdr *iph        = ip_hdr(skb);
571         unsigned int icmp_offset = iph->ihl*4;
572         struct icmphdr *icmph    = (struct icmphdr *)(skb_network_header(skb) +
573                                                       icmp_offset);
574         struct iphdr *ciph       = (struct iphdr *)(icmph + 1);
575
576         if (inout) {
577                 iph->saddr = cp->vaddr.ip;
578                 ip_send_check(iph);
579                 ciph->daddr = cp->vaddr.ip;
580                 ip_send_check(ciph);
581         } else {
582                 iph->daddr = cp->daddr.ip;
583                 ip_send_check(iph);
584                 ciph->saddr = cp->daddr.ip;
585                 ip_send_check(ciph);
586         }
587
588         /* the TCP/UDP/SCTP port */
589         if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol ||
590             IPPROTO_SCTP == ciph->protocol) {
591                 __be16 *ports = (void *)ciph + ciph->ihl*4;
592
593                 if (inout)
594                         ports[1] = cp->vport;
595                 else
596                         ports[0] = cp->dport;
597         }
598
599         /* And finally the ICMP checksum */
600         icmph->checksum = 0;
601         icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
602         skb->ip_summed = CHECKSUM_UNNECESSARY;
603
604         if (inout)
605                 IP_VS_DBG_PKT(11, pp, skb, (void *)ciph - (void *)iph,
606                         "Forwarding altered outgoing ICMP");
607         else
608                 IP_VS_DBG_PKT(11, pp, skb, (void *)ciph - (void *)iph,
609                         "Forwarding altered incoming ICMP");
610 }
611
612 #ifdef CONFIG_IP_VS_IPV6
613 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp,
614                     struct ip_vs_conn *cp, int inout)
615 {
616         struct ipv6hdr *iph      = ipv6_hdr(skb);
617         unsigned int icmp_offset = sizeof(struct ipv6hdr);
618         struct icmp6hdr *icmph   = (struct icmp6hdr *)(skb_network_header(skb) +
619                                                       icmp_offset);
620         struct ipv6hdr *ciph     = (struct ipv6hdr *)(icmph + 1);
621
622         if (inout) {
623                 iph->saddr = cp->vaddr.in6;
624                 ciph->daddr = cp->vaddr.in6;
625         } else {
626                 iph->daddr = cp->daddr.in6;
627                 ciph->saddr = cp->daddr.in6;
628         }
629
630         /* the TCP/UDP/SCTP port */
631         if (IPPROTO_TCP == ciph->nexthdr || IPPROTO_UDP == ciph->nexthdr ||
632             IPPROTO_SCTP == ciph->nexthdr) {
633                 __be16 *ports = (void *)ciph + sizeof(struct ipv6hdr);
634
635                 if (inout)
636                         ports[1] = cp->vport;
637                 else
638                         ports[0] = cp->dport;
639         }
640
641         /* And finally the ICMP checksum */
642         icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr,
643                                               skb->len - icmp_offset,
644                                               IPPROTO_ICMPV6, 0);
645         skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset;
646         skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
647         skb->ip_summed = CHECKSUM_PARTIAL;
648
649         if (inout)
650                 IP_VS_DBG_PKT(11, pp, skb, (void *)ciph - (void *)iph,
651                         "Forwarding altered outgoing ICMPv6");
652         else
653                 IP_VS_DBG_PKT(11, pp, skb, (void *)ciph - (void *)iph,
654                         "Forwarding altered incoming ICMPv6");
655 }
656 #endif
657
658 /* Handle relevant response ICMP messages - forward to the right
659  * destination host. Used for NAT and local client.
660  */
661 static int handle_response_icmp(int af, struct sk_buff *skb,
662                                 union nf_inet_addr *snet,
663                                 __u8 protocol, struct ip_vs_conn *cp,
664                                 struct ip_vs_protocol *pp,
665                                 unsigned int offset, unsigned int ihl)
666 {
667         unsigned int verdict = NF_DROP;
668
669         if (IP_VS_FWD_METHOD(cp) != 0) {
670                 pr_err("shouldn't reach here, because the box is on the "
671                        "half connection in the tun/dr module.\n");
672         }
673
674         /* Ensure the checksum is correct */
675         if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
676                 /* Failed checksum! */
677                 IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n",
678                               IP_VS_DBG_ADDR(af, snet));
679                 goto out;
680         }
681
682         if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol ||
683             IPPROTO_SCTP == protocol)
684                 offset += 2 * sizeof(__u16);
685         if (!skb_make_writable(skb, offset))
686                 goto out;
687
688 #ifdef CONFIG_IP_VS_IPV6
689         if (af == AF_INET6)
690                 ip_vs_nat_icmp_v6(skb, pp, cp, 1);
691         else
692 #endif
693                 ip_vs_nat_icmp(skb, pp, cp, 1);
694
695         /* do the statistics and put it back */
696         ip_vs_out_stats(cp, skb);
697
698         skb->ipvs_property = 1;
699         verdict = NF_ACCEPT;
700
701 out:
702         __ip_vs_conn_put(cp);
703
704         return verdict;
705 }
706
707 /*
708  *      Handle ICMP messages in the inside-to-outside direction (outgoing).
709  *      Find any that might be relevant, check against existing connections.
710  *      Currently handles error types - unreachable, quench, ttl exceeded.
711  */
712 static int ip_vs_out_icmp(struct sk_buff *skb, int *related)
713 {
714         struct iphdr *iph;
715         struct icmphdr  _icmph, *ic;
716         struct iphdr    _ciph, *cih;    /* The ip header contained within the ICMP */
717         struct ip_vs_iphdr ciph;
718         struct ip_vs_conn *cp;
719         struct ip_vs_protocol *pp;
720         unsigned int offset, ihl;
721         union nf_inet_addr snet;
722
723         *related = 1;
724
725         /* reassemble IP fragments */
726         if (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) {
727                 if (ip_vs_gather_frags(skb, IP_DEFRAG_VS_OUT))
728                         return NF_STOLEN;
729         }
730
731         iph = ip_hdr(skb);
732         offset = ihl = iph->ihl * 4;
733         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
734         if (ic == NULL)
735                 return NF_DROP;
736
737         IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n",
738                   ic->type, ntohs(icmp_id(ic)),
739                   &iph->saddr, &iph->daddr);
740
741         /*
742          * Work through seeing if this is for us.
743          * These checks are supposed to be in an order that means easy
744          * things are checked first to speed up processing.... however
745          * this means that some packets will manage to get a long way
746          * down this stack and then be rejected, but that's life.
747          */
748         if ((ic->type != ICMP_DEST_UNREACH) &&
749             (ic->type != ICMP_SOURCE_QUENCH) &&
750             (ic->type != ICMP_TIME_EXCEEDED)) {
751                 *related = 0;
752                 return NF_ACCEPT;
753         }
754
755         /* Now find the contained IP header */
756         offset += sizeof(_icmph);
757         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
758         if (cih == NULL)
759                 return NF_ACCEPT; /* The packet looks wrong, ignore */
760
761         pp = ip_vs_proto_get(cih->protocol);
762         if (!pp)
763                 return NF_ACCEPT;
764
765         /* Is the embedded protocol header present? */
766         if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
767                      pp->dont_defrag))
768                 return NF_ACCEPT;
769
770         IP_VS_DBG_PKT(11, pp, skb, offset, "Checking outgoing ICMP for");
771
772         offset += cih->ihl * 4;
773
774         ip_vs_fill_iphdr(AF_INET, cih, &ciph);
775         /* The embedded headers contain source and dest in reverse order */
776         cp = pp->conn_out_get(AF_INET, skb, pp, &ciph, offset, 1);
777         if (!cp)
778                 return NF_ACCEPT;
779
780         snet.ip = iph->saddr;
781         return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp,
782                                     pp, offset, ihl);
783 }
784
785 #ifdef CONFIG_IP_VS_IPV6
786 static int ip_vs_out_icmp_v6(struct sk_buff *skb, int *related)
787 {
788         struct ipv6hdr *iph;
789         struct icmp6hdr _icmph, *ic;
790         struct ipv6hdr  _ciph, *cih;    /* The ip header contained
791                                            within the ICMP */
792         struct ip_vs_iphdr ciph;
793         struct ip_vs_conn *cp;
794         struct ip_vs_protocol *pp;
795         unsigned int offset;
796         union nf_inet_addr snet;
797
798         *related = 1;
799
800         /* reassemble IP fragments */
801         if (ipv6_hdr(skb)->nexthdr == IPPROTO_FRAGMENT) {
802                 if (ip_vs_gather_frags_v6(skb, IP_DEFRAG_VS_OUT))
803                         return NF_STOLEN;
804         }
805
806         iph = ipv6_hdr(skb);
807         offset = sizeof(struct ipv6hdr);
808         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
809         if (ic == NULL)
810                 return NF_DROP;
811
812         IP_VS_DBG(12, "Outgoing ICMPv6 (%d,%d) %pI6->%pI6\n",
813                   ic->icmp6_type, ntohs(icmpv6_id(ic)),
814                   &iph->saddr, &iph->daddr);
815
816         /*
817          * Work through seeing if this is for us.
818          * These checks are supposed to be in an order that means easy
819          * things are checked first to speed up processing.... however
820          * this means that some packets will manage to get a long way
821          * down this stack and then be rejected, but that's life.
822          */
823         if ((ic->icmp6_type != ICMPV6_DEST_UNREACH) &&
824             (ic->icmp6_type != ICMPV6_PKT_TOOBIG) &&
825             (ic->icmp6_type != ICMPV6_TIME_EXCEED)) {
826                 *related = 0;
827                 return NF_ACCEPT;
828         }
829
830         /* Now find the contained IP header */
831         offset += sizeof(_icmph);
832         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
833         if (cih == NULL)
834                 return NF_ACCEPT; /* The packet looks wrong, ignore */
835
836         pp = ip_vs_proto_get(cih->nexthdr);
837         if (!pp)
838                 return NF_ACCEPT;
839
840         /* Is the embedded protocol header present? */
841         /* TODO: we don't support fragmentation at the moment anyways */
842         if (unlikely(cih->nexthdr == IPPROTO_FRAGMENT && pp->dont_defrag))
843                 return NF_ACCEPT;
844
845         IP_VS_DBG_PKT(11, pp, skb, offset, "Checking outgoing ICMPv6 for");
846
847         offset += sizeof(struct ipv6hdr);
848
849         ip_vs_fill_iphdr(AF_INET6, cih, &ciph);
850         /* The embedded headers contain source and dest in reverse order */
851         cp = pp->conn_out_get(AF_INET6, skb, pp, &ciph, offset, 1);
852         if (!cp)
853                 return NF_ACCEPT;
854
855         ipv6_addr_copy(&snet.in6, &iph->saddr);
856         return handle_response_icmp(AF_INET6, skb, &snet, cih->nexthdr, cp,
857                                     pp, offset, sizeof(struct ipv6hdr));
858 }
859 #endif
860
861 /*
862  * Check if sctp chunc is ABORT chunk
863  */
864 static inline int is_sctp_abort(const struct sk_buff *skb, int nh_len)
865 {
866         sctp_chunkhdr_t *sch, schunk;
867         sch = skb_header_pointer(skb, nh_len + sizeof(sctp_sctphdr_t),
868                         sizeof(schunk), &schunk);
869         if (sch == NULL)
870                 return 0;
871         if (sch->type == SCTP_CID_ABORT)
872                 return 1;
873         return 0;
874 }
875
876 static inline int is_tcp_reset(const struct sk_buff *skb, int nh_len)
877 {
878         struct tcphdr _tcph, *th;
879
880         th = skb_header_pointer(skb, nh_len, sizeof(_tcph), &_tcph);
881         if (th == NULL)
882                 return 0;
883         return th->rst;
884 }
885
886 /* Handle response packets: rewrite addresses and send away...
887  * Used for NAT and local client.
888  */
889 static unsigned int
890 handle_response(int af, struct sk_buff *skb, struct ip_vs_protocol *pp,
891                 struct ip_vs_conn *cp, int ihl)
892 {
893         IP_VS_DBG_PKT(11, pp, skb, 0, "Outgoing packet");
894
895         if (!skb_make_writable(skb, ihl))
896                 goto drop;
897
898         /* mangle the packet */
899         if (pp->snat_handler && !pp->snat_handler(skb, pp, cp))
900                 goto drop;
901
902 #ifdef CONFIG_IP_VS_IPV6
903         if (af == AF_INET6)
904                 ipv6_hdr(skb)->saddr = cp->vaddr.in6;
905         else
906 #endif
907         {
908                 ip_hdr(skb)->saddr = cp->vaddr.ip;
909                 ip_send_check(ip_hdr(skb));
910         }
911
912         /* For policy routing, packets originating from this
913          * machine itself may be routed differently to packets
914          * passing through.  We want this packet to be routed as
915          * if it came from this machine itself.  So re-compute
916          * the routing information.
917          */
918 #ifdef CONFIG_IP_VS_IPV6
919         if (af == AF_INET6) {
920                 if (ip6_route_me_harder(skb) != 0)
921                         goto drop;
922         } else
923 #endif
924                 if (ip_route_me_harder(skb, RTN_LOCAL) != 0)
925                         goto drop;
926
927         IP_VS_DBG_PKT(10, pp, skb, 0, "After SNAT");
928
929         ip_vs_out_stats(cp, skb);
930         ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pp);
931         ip_vs_update_conntrack(skb, cp, 0);
932         ip_vs_conn_put(cp);
933
934         skb->ipvs_property = 1;
935
936         LeaveFunction(11);
937         return NF_ACCEPT;
938
939 drop:
940         ip_vs_conn_put(cp);
941         kfree_skb(skb);
942         return NF_STOLEN;
943 }
944
945 /*
946  *      It is hooked at the NF_INET_FORWARD chain, used only for VS/NAT.
947  *      Check if outgoing packet belongs to the established ip_vs_conn.
948  */
949 static unsigned int
950 ip_vs_out(unsigned int hooknum, struct sk_buff *skb,
951           const struct net_device *in, const struct net_device *out,
952           int (*okfn)(struct sk_buff *))
953 {
954         struct ip_vs_iphdr iph;
955         struct ip_vs_protocol *pp;
956         struct ip_vs_conn *cp;
957         int af;
958
959         EnterFunction(11);
960
961         af = (skb->protocol == htons(ETH_P_IP)) ? AF_INET : AF_INET6;
962
963         if (skb->ipvs_property)
964                 return NF_ACCEPT;
965
966         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
967 #ifdef CONFIG_IP_VS_IPV6
968         if (af == AF_INET6) {
969                 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
970                         int related, verdict = ip_vs_out_icmp_v6(skb, &related);
971
972                         if (related)
973                                 return verdict;
974                         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
975                 }
976         } else
977 #endif
978                 if (unlikely(iph.protocol == IPPROTO_ICMP)) {
979                         int related, verdict = ip_vs_out_icmp(skb, &related);
980
981                         if (related)
982                                 return verdict;
983                         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
984                 }
985
986         pp = ip_vs_proto_get(iph.protocol);
987         if (unlikely(!pp))
988                 return NF_ACCEPT;
989
990         /* reassemble IP fragments */
991 #ifdef CONFIG_IP_VS_IPV6
992         if (af == AF_INET6) {
993                 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
994                         int related, verdict = ip_vs_out_icmp_v6(skb, &related);
995
996                         if (related)
997                                 return verdict;
998
999                         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
1000                 }
1001         } else
1002 #endif
1003                 if (unlikely(ip_hdr(skb)->frag_off & htons(IP_MF|IP_OFFSET) &&
1004                              !pp->dont_defrag)) {
1005                         if (ip_vs_gather_frags(skb, IP_DEFRAG_VS_OUT))
1006                                 return NF_STOLEN;
1007
1008                         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
1009                 }
1010
1011         /*
1012          * Check if the packet belongs to an existing entry
1013          */
1014         cp = pp->conn_out_get(af, skb, pp, &iph, iph.len, 0);
1015
1016         if (unlikely(!cp)) {
1017                 if (sysctl_ip_vs_nat_icmp_send &&
1018                     (pp->protocol == IPPROTO_TCP ||
1019                      pp->protocol == IPPROTO_UDP ||
1020                      pp->protocol == IPPROTO_SCTP)) {
1021                         __be16 _ports[2], *pptr;
1022
1023                         pptr = skb_header_pointer(skb, iph.len,
1024                                                   sizeof(_ports), _ports);
1025                         if (pptr == NULL)
1026                                 return NF_ACCEPT;       /* Not for me */
1027                         if (ip_vs_lookup_real_service(af, iph.protocol,
1028                                                       &iph.saddr,
1029                                                       pptr[0])) {
1030                                 /*
1031                                  * Notify the real server: there is no
1032                                  * existing entry if it is not RST
1033                                  * packet or not TCP packet.
1034                                  */
1035                                 if ((iph.protocol != IPPROTO_TCP &&
1036                                      iph.protocol != IPPROTO_SCTP)
1037                                      || ((iph.protocol == IPPROTO_TCP
1038                                           && !is_tcp_reset(skb, iph.len))
1039                                          || (iph.protocol == IPPROTO_SCTP
1040                                                 && !is_sctp_abort(skb,
1041                                                         iph.len)))) {
1042 #ifdef CONFIG_IP_VS_IPV6
1043                                         if (af == AF_INET6)
1044                                                 icmpv6_send(skb,
1045                                                             ICMPV6_DEST_UNREACH,
1046                                                             ICMPV6_PORT_UNREACH,
1047                                                             0);
1048                                         else
1049 #endif
1050                                                 icmp_send(skb,
1051                                                           ICMP_DEST_UNREACH,
1052                                                           ICMP_PORT_UNREACH, 0);
1053                                         return NF_DROP;
1054                                 }
1055                         }
1056                 }
1057                 IP_VS_DBG_PKT(12, pp, skb, 0,
1058                               "packet continues traversal as normal");
1059                 return NF_ACCEPT;
1060         }
1061
1062         return handle_response(af, skb, pp, cp, iph.len);
1063 }
1064
1065
1066 /*
1067  *      Handle ICMP messages in the outside-to-inside direction (incoming).
1068  *      Find any that might be relevant, check against existing connections,
1069  *      forward to the right destination host if relevant.
1070  *      Currently handles error types - unreachable, quench, ttl exceeded.
1071  */
1072 static int
1073 ip_vs_in_icmp(struct sk_buff *skb, int *related, unsigned int hooknum)
1074 {
1075         struct iphdr *iph;
1076         struct icmphdr  _icmph, *ic;
1077         struct iphdr    _ciph, *cih;    /* The ip header contained within the ICMP */
1078         struct ip_vs_iphdr ciph;
1079         struct ip_vs_conn *cp;
1080         struct ip_vs_protocol *pp;
1081         unsigned int offset, ihl, verdict;
1082         union nf_inet_addr snet;
1083
1084         *related = 1;
1085
1086         /* reassemble IP fragments */
1087         if (ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)) {
1088                 if (ip_vs_gather_frags(skb, hooknum == NF_INET_LOCAL_IN ?
1089                                             IP_DEFRAG_VS_IN : IP_DEFRAG_VS_FWD))
1090                         return NF_STOLEN;
1091         }
1092
1093         iph = ip_hdr(skb);
1094         offset = ihl = iph->ihl * 4;
1095         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
1096         if (ic == NULL)
1097                 return NF_DROP;
1098
1099         IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n",
1100                   ic->type, ntohs(icmp_id(ic)),
1101                   &iph->saddr, &iph->daddr);
1102
1103         /*
1104          * Work through seeing if this is for us.
1105          * These checks are supposed to be in an order that means easy
1106          * things are checked first to speed up processing.... however
1107          * this means that some packets will manage to get a long way
1108          * down this stack and then be rejected, but that's life.
1109          */
1110         if ((ic->type != ICMP_DEST_UNREACH) &&
1111             (ic->type != ICMP_SOURCE_QUENCH) &&
1112             (ic->type != ICMP_TIME_EXCEEDED)) {
1113                 *related = 0;
1114                 return NF_ACCEPT;
1115         }
1116
1117         /* Now find the contained IP header */
1118         offset += sizeof(_icmph);
1119         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1120         if (cih == NULL)
1121                 return NF_ACCEPT; /* The packet looks wrong, ignore */
1122
1123         pp = ip_vs_proto_get(cih->protocol);
1124         if (!pp)
1125                 return NF_ACCEPT;
1126
1127         /* Is the embedded protocol header present? */
1128         if (unlikely(cih->frag_off & htons(IP_OFFSET) &&
1129                      pp->dont_defrag))
1130                 return NF_ACCEPT;
1131
1132         IP_VS_DBG_PKT(11, pp, skb, offset, "Checking incoming ICMP for");
1133
1134         offset += cih->ihl * 4;
1135
1136         ip_vs_fill_iphdr(AF_INET, cih, &ciph);
1137         /* The embedded headers contain source and dest in reverse order */
1138         cp = pp->conn_in_get(AF_INET, skb, pp, &ciph, offset, 1);
1139         if (!cp) {
1140                 /* The packet could also belong to a local client */
1141                 cp = pp->conn_out_get(AF_INET, skb, pp, &ciph, offset, 1);
1142                 if (cp) {
1143                         snet.ip = iph->saddr;
1144                         return handle_response_icmp(AF_INET, skb, &snet,
1145                                                     cih->protocol, cp, pp,
1146                                                     offset, ihl);
1147                 }
1148                 return NF_ACCEPT;
1149         }
1150
1151         verdict = NF_DROP;
1152
1153         /* Ensure the checksum is correct */
1154         if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) {
1155                 /* Failed checksum! */
1156                 IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n",
1157                           &iph->saddr);
1158                 goto out;
1159         }
1160
1161         /* do the statistics and put it back */
1162         ip_vs_in_stats(cp, skb);
1163         if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol)
1164                 offset += 2 * sizeof(__u16);
1165         verdict = ip_vs_icmp_xmit(skb, cp, pp, offset);
1166         /* do not touch skb anymore */
1167
1168   out:
1169         __ip_vs_conn_put(cp);
1170
1171         return verdict;
1172 }
1173
1174 #ifdef CONFIG_IP_VS_IPV6
1175 static int
1176 ip_vs_in_icmp_v6(struct sk_buff *skb, int *related, unsigned int hooknum)
1177 {
1178         struct ipv6hdr *iph;
1179         struct icmp6hdr _icmph, *ic;
1180         struct ipv6hdr  _ciph, *cih;    /* The ip header contained
1181                                            within the ICMP */
1182         struct ip_vs_iphdr ciph;
1183         struct ip_vs_conn *cp;
1184         struct ip_vs_protocol *pp;
1185         unsigned int offset, verdict;
1186         union nf_inet_addr snet;
1187
1188         *related = 1;
1189
1190         /* reassemble IP fragments */
1191         if (ipv6_hdr(skb)->nexthdr == IPPROTO_FRAGMENT) {
1192                 if (ip_vs_gather_frags_v6(skb, hooknum == NF_INET_LOCAL_IN ?
1193                                                IP_DEFRAG_VS_IN :
1194                                                IP_DEFRAG_VS_FWD))
1195                         return NF_STOLEN;
1196         }
1197
1198         iph = ipv6_hdr(skb);
1199         offset = sizeof(struct ipv6hdr);
1200         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
1201         if (ic == NULL)
1202                 return NF_DROP;
1203
1204         IP_VS_DBG(12, "Incoming ICMPv6 (%d,%d) %pI6->%pI6\n",
1205                   ic->icmp6_type, ntohs(icmpv6_id(ic)),
1206                   &iph->saddr, &iph->daddr);
1207
1208         /*
1209          * Work through seeing if this is for us.
1210          * These checks are supposed to be in an order that means easy
1211          * things are checked first to speed up processing.... however
1212          * this means that some packets will manage to get a long way
1213          * down this stack and then be rejected, but that's life.
1214          */
1215         if ((ic->icmp6_type != ICMPV6_DEST_UNREACH) &&
1216             (ic->icmp6_type != ICMPV6_PKT_TOOBIG) &&
1217             (ic->icmp6_type != ICMPV6_TIME_EXCEED)) {
1218                 *related = 0;
1219                 return NF_ACCEPT;
1220         }
1221
1222         /* Now find the contained IP header */
1223         offset += sizeof(_icmph);
1224         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
1225         if (cih == NULL)
1226                 return NF_ACCEPT; /* The packet looks wrong, ignore */
1227
1228         pp = ip_vs_proto_get(cih->nexthdr);
1229         if (!pp)
1230                 return NF_ACCEPT;
1231
1232         /* Is the embedded protocol header present? */
1233         /* TODO: we don't support fragmentation at the moment anyways */
1234         if (unlikely(cih->nexthdr == IPPROTO_FRAGMENT && pp->dont_defrag))
1235                 return NF_ACCEPT;
1236
1237         IP_VS_DBG_PKT(11, pp, skb, offset, "Checking incoming ICMPv6 for");
1238
1239         offset += sizeof(struct ipv6hdr);
1240
1241         ip_vs_fill_iphdr(AF_INET6, cih, &ciph);
1242         /* The embedded headers contain source and dest in reverse order */
1243         cp = pp->conn_in_get(AF_INET6, skb, pp, &ciph, offset, 1);
1244         if (!cp) {
1245                 /* The packet could also belong to a local client */
1246                 cp = pp->conn_out_get(AF_INET6, skb, pp, &ciph, offset, 1);
1247                 if (cp) {
1248                         ipv6_addr_copy(&snet.in6, &iph->saddr);
1249                         return handle_response_icmp(AF_INET6, skb, &snet,
1250                                                     cih->nexthdr,
1251                                                     cp, pp, offset,
1252                                                     sizeof(struct ipv6hdr));
1253                 }
1254                 return NF_ACCEPT;
1255         }
1256
1257         verdict = NF_DROP;
1258
1259         /* do the statistics and put it back */
1260         ip_vs_in_stats(cp, skb);
1261         if (IPPROTO_TCP == cih->nexthdr || IPPROTO_UDP == cih->nexthdr ||
1262             IPPROTO_SCTP == cih->nexthdr)
1263                 offset += 2 * sizeof(__u16);
1264         verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, offset);
1265         /* do not touch skb anymore */
1266
1267         __ip_vs_conn_put(cp);
1268
1269         return verdict;
1270 }
1271 #endif
1272
1273
1274 /*
1275  *      Check if it's for virtual services, look it up,
1276  *      and send it on its way...
1277  */
1278 static unsigned int
1279 ip_vs_in(unsigned int hooknum, struct sk_buff *skb,
1280          const struct net_device *in, const struct net_device *out,
1281          int (*okfn)(struct sk_buff *))
1282 {
1283         struct ip_vs_iphdr iph;
1284         struct ip_vs_protocol *pp;
1285         struct ip_vs_conn *cp;
1286         int ret, restart, af, pkts;
1287
1288         af = (skb->protocol == htons(ETH_P_IP)) ? AF_INET : AF_INET6;
1289
1290         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
1291
1292         /*
1293          *      Big tappo: only PACKET_HOST, including loopback for local client
1294          *      Don't handle local packets on IPv6 for now
1295          */
1296         if (unlikely(skb->pkt_type != PACKET_HOST)) {
1297                 IP_VS_DBG_BUF(12, "packet type=%d proto=%d daddr=%s ignored\n",
1298                               skb->pkt_type,
1299                               iph.protocol,
1300                               IP_VS_DBG_ADDR(af, &iph.daddr));
1301                 return NF_ACCEPT;
1302         }
1303
1304 #ifdef CONFIG_IP_VS_IPV6
1305         if (af == AF_INET6) {
1306                 if (unlikely(iph.protocol == IPPROTO_ICMPV6)) {
1307                         int related, verdict = ip_vs_in_icmp_v6(skb, &related, hooknum);
1308
1309                         if (related)
1310                                 return verdict;
1311                         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
1312                 }
1313         } else
1314 #endif
1315                 if (unlikely(iph.protocol == IPPROTO_ICMP)) {
1316                         int related, verdict = ip_vs_in_icmp(skb, &related, hooknum);
1317
1318                         if (related)
1319                                 return verdict;
1320                         ip_vs_fill_iphdr(af, skb_network_header(skb), &iph);
1321                 }
1322
1323         /* Protocol supported? */
1324         pp = ip_vs_proto_get(iph.protocol);
1325         if (unlikely(!pp))
1326                 return NF_ACCEPT;
1327
1328         /*
1329          * Check if the packet belongs to an existing connection entry
1330          */
1331         cp = pp->conn_in_get(af, skb, pp, &iph, iph.len, 0);
1332
1333         if (unlikely(!cp)) {
1334                 int v;
1335
1336                 /* For local client packets, it could be a response */
1337                 cp = pp->conn_out_get(af, skb, pp, &iph, iph.len, 0);
1338                 if (cp)
1339                         return handle_response(af, skb, pp, cp, iph.len);
1340
1341                 if (!pp->conn_schedule(af, skb, pp, &v, &cp))
1342                         return v;
1343         }
1344
1345         if (unlikely(!cp)) {
1346                 /* sorry, all this trouble for a no-hit :) */
1347                 IP_VS_DBG_PKT(12, pp, skb, 0,
1348                               "packet continues traversal as normal");
1349                 return NF_ACCEPT;
1350         }
1351
1352         IP_VS_DBG_PKT(11, pp, skb, 0, "Incoming packet");
1353
1354         /* Check the server status */
1355         if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
1356                 /* the destination server is not available */
1357
1358                 if (sysctl_ip_vs_expire_nodest_conn) {
1359                         /* try to expire the connection immediately */
1360                         ip_vs_conn_expire_now(cp);
1361                 }
1362                 /* don't restart its timer, and silently
1363                    drop the packet. */
1364                 __ip_vs_conn_put(cp);
1365                 return NF_DROP;
1366         }
1367
1368         ip_vs_in_stats(cp, skb);
1369         restart = ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pp);
1370         if (cp->packet_xmit)
1371                 ret = cp->packet_xmit(skb, cp, pp);
1372                 /* do not touch skb anymore */
1373         else {
1374                 IP_VS_DBG_RL("warning: packet_xmit is null");
1375                 ret = NF_ACCEPT;
1376         }
1377
1378         /* Increase its packet counter and check if it is needed
1379          * to be synchronized
1380          *
1381          * Sync connection if it is about to close to
1382          * encorage the standby servers to update the connections timeout
1383          */
1384         pkts = atomic_add_return(1, &cp->in_pkts);
1385         if (af == AF_INET && (ip_vs_sync_state & IP_VS_STATE_MASTER) &&
1386             cp->protocol == IPPROTO_SCTP) {
1387                 if ((cp->state == IP_VS_SCTP_S_ESTABLISHED &&
1388                         (pkts % sysctl_ip_vs_sync_threshold[1]
1389                          == sysctl_ip_vs_sync_threshold[0])) ||
1390                                 (cp->old_state != cp->state &&
1391                                  ((cp->state == IP_VS_SCTP_S_CLOSED) ||
1392                                   (cp->state == IP_VS_SCTP_S_SHUT_ACK_CLI) ||
1393                                   (cp->state == IP_VS_SCTP_S_SHUT_ACK_SER)))) {
1394                         ip_vs_sync_conn(cp);
1395                         goto out;
1396                 }
1397         }
1398
1399         /* Keep this block last: TCP and others with pp->num_states <= 1 */
1400         else if (af == AF_INET &&
1401             (ip_vs_sync_state & IP_VS_STATE_MASTER) &&
1402             (((cp->protocol != IPPROTO_TCP ||
1403                cp->state == IP_VS_TCP_S_ESTABLISHED) &&
1404               (pkts % sysctl_ip_vs_sync_threshold[1]
1405                == sysctl_ip_vs_sync_threshold[0])) ||
1406              ((cp->protocol == IPPROTO_TCP) && (cp->old_state != cp->state) &&
1407               ((cp->state == IP_VS_TCP_S_FIN_WAIT) ||
1408                (cp->state == IP_VS_TCP_S_CLOSE) ||
1409                (cp->state == IP_VS_TCP_S_CLOSE_WAIT) ||
1410                (cp->state == IP_VS_TCP_S_TIME_WAIT)))))
1411                 ip_vs_sync_conn(cp);
1412 out:
1413         cp->old_state = cp->state;
1414
1415         ip_vs_conn_put(cp);
1416         return ret;
1417 }
1418
1419
1420 /*
1421  *      It is hooked at the NF_INET_FORWARD chain, in order to catch ICMP
1422  *      related packets destined for 0.0.0.0/0.
1423  *      When fwmark-based virtual service is used, such as transparent
1424  *      cache cluster, TCP packets can be marked and routed to ip_vs_in,
1425  *      but ICMP destined for 0.0.0.0/0 cannot not be easily marked and
1426  *      sent to ip_vs_in_icmp. So, catch them at the NF_INET_FORWARD chain
1427  *      and send them to ip_vs_in_icmp.
1428  */
1429 static unsigned int
1430 ip_vs_forward_icmp(unsigned int hooknum, struct sk_buff *skb,
1431                    const struct net_device *in, const struct net_device *out,
1432                    int (*okfn)(struct sk_buff *))
1433 {
1434         int r;
1435
1436         if (ip_hdr(skb)->protocol != IPPROTO_ICMP)
1437                 return NF_ACCEPT;
1438
1439         return ip_vs_in_icmp(skb, &r, hooknum);
1440 }
1441
1442 #ifdef CONFIG_IP_VS_IPV6
1443 static unsigned int
1444 ip_vs_forward_icmp_v6(unsigned int hooknum, struct sk_buff *skb,
1445                       const struct net_device *in, const struct net_device *out,
1446                       int (*okfn)(struct sk_buff *))
1447 {
1448         int r;
1449
1450         if (ipv6_hdr(skb)->nexthdr != IPPROTO_ICMPV6)
1451                 return NF_ACCEPT;
1452
1453         return ip_vs_in_icmp_v6(skb, &r, hooknum);
1454 }
1455 #endif
1456
1457
1458 static struct nf_hook_ops ip_vs_ops[] __read_mostly = {
1459         /* After packet filtering, forward packet through VS/DR, VS/TUN,
1460          * or VS/NAT(change destination), so that filtering rules can be
1461          * applied to IPVS. */
1462         {
1463                 .hook           = ip_vs_in,
1464                 .owner          = THIS_MODULE,
1465                 .pf             = PF_INET,
1466                 .hooknum        = NF_INET_LOCAL_IN,
1467                 .priority       = 100,
1468         },
1469         /* After packet filtering, change source only for VS/NAT */
1470         {
1471                 .hook           = ip_vs_out,
1472                 .owner          = THIS_MODULE,
1473                 .pf             = PF_INET,
1474                 .hooknum        = NF_INET_FORWARD,
1475                 .priority       = 100,
1476         },
1477         /* After packet filtering (but before ip_vs_out_icmp), catch icmp
1478          * destined for 0.0.0.0/0, which is for incoming IPVS connections */
1479         {
1480                 .hook           = ip_vs_forward_icmp,
1481                 .owner          = THIS_MODULE,
1482                 .pf             = PF_INET,
1483                 .hooknum        = NF_INET_FORWARD,
1484                 .priority       = 99,
1485         },
1486 #ifdef CONFIG_IP_VS_IPV6
1487         /* After packet filtering, forward packet through VS/DR, VS/TUN,
1488          * or VS/NAT(change destination), so that filtering rules can be
1489          * applied to IPVS. */
1490         {
1491                 .hook           = ip_vs_in,
1492                 .owner          = THIS_MODULE,
1493                 .pf             = PF_INET6,
1494                 .hooknum        = NF_INET_LOCAL_IN,
1495                 .priority       = 100,
1496         },
1497         /* After packet filtering, change source only for VS/NAT */
1498         {
1499                 .hook           = ip_vs_out,
1500                 .owner          = THIS_MODULE,
1501                 .pf             = PF_INET6,
1502                 .hooknum        = NF_INET_FORWARD,
1503                 .priority       = 100,
1504         },
1505         /* After packet filtering (but before ip_vs_out_icmp), catch icmp
1506          * destined for 0.0.0.0/0, which is for incoming IPVS connections */
1507         {
1508                 .hook           = ip_vs_forward_icmp_v6,
1509                 .owner          = THIS_MODULE,
1510                 .pf             = PF_INET6,
1511                 .hooknum        = NF_INET_FORWARD,
1512                 .priority       = 99,
1513         },
1514 #endif
1515 };
1516
1517
1518 /*
1519  *      Initialize IP Virtual Server
1520  */
1521 static int __init ip_vs_init(void)
1522 {
1523         int ret;
1524
1525         ip_vs_estimator_init();
1526
1527         ret = ip_vs_control_init();
1528         if (ret < 0) {
1529                 pr_err("can't setup control.\n");
1530                 goto cleanup_estimator;
1531         }
1532
1533         ip_vs_protocol_init();
1534
1535         ret = ip_vs_app_init();
1536         if (ret < 0) {
1537                 pr_err("can't setup application helper.\n");
1538                 goto cleanup_protocol;
1539         }
1540
1541         ret = ip_vs_conn_init();
1542         if (ret < 0) {
1543                 pr_err("can't setup connection table.\n");
1544                 goto cleanup_app;
1545         }
1546
1547         ret = nf_register_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
1548         if (ret < 0) {
1549                 pr_err("can't register hooks.\n");
1550                 goto cleanup_conn;
1551         }
1552
1553         pr_info("ipvs loaded.\n");
1554         return ret;
1555
1556   cleanup_conn:
1557         ip_vs_conn_cleanup();
1558   cleanup_app:
1559         ip_vs_app_cleanup();
1560   cleanup_protocol:
1561         ip_vs_protocol_cleanup();
1562         ip_vs_control_cleanup();
1563   cleanup_estimator:
1564         ip_vs_estimator_cleanup();
1565         return ret;
1566 }
1567
1568 static void __exit ip_vs_cleanup(void)
1569 {
1570         nf_unregister_hooks(ip_vs_ops, ARRAY_SIZE(ip_vs_ops));
1571         ip_vs_conn_cleanup();
1572         ip_vs_app_cleanup();
1573         ip_vs_protocol_cleanup();
1574         ip_vs_control_cleanup();
1575         ip_vs_estimator_cleanup();
1576         pr_info("ipvs unloaded.\n");
1577 }
1578
1579 module_init(ip_vs_init);
1580 module_exit(ip_vs_cleanup);
1581 MODULE_LICENSE("GPL");