2 * xfrm_output.c - Common IPsec encapsulation code.
4 * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au>
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
12 #include <linux/errno.h>
13 #include <linux/module.h>
14 #include <linux/netdevice.h>
15 #include <linux/netfilter.h>
16 #include <linux/skbuff.h>
17 #include <linux/slab.h>
18 #include <linux/spinlock.h>
20 #include <net/inet_ecn.h>
23 #include "xfrm_inout.h"
25 static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb);
26 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb);
28 static int xfrm_skb_check_space(struct sk_buff *skb)
30 struct dst_entry *dst = skb_dst(skb);
31 int nhead = dst->header_len + LL_RESERVED_SPACE(dst->dev)
33 int ntail = dst->dev->needed_tailroom - skb_tailroom(skb);
42 return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC);
45 /* Children define the path of the packet through the
46 * Linux networking. Thus, destinations are stackable.
49 static struct dst_entry *skb_dst_pop(struct sk_buff *skb)
51 struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb)));
57 /* Add encapsulation header.
59 * The IP header will be moved forward to make space for the encapsulation
62 static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb)
64 struct iphdr *iph = ip_hdr(skb);
65 int ihl = iph->ihl * 4;
67 skb_set_inner_transport_header(skb, skb_transport_offset(skb));
69 skb_set_network_header(skb, -x->props.header_len);
70 skb->mac_header = skb->network_header +
71 offsetof(struct iphdr, protocol);
72 skb->transport_header = skb->network_header + ihl;
74 memmove(skb_network_header(skb), iph, ihl);
78 /* Add encapsulation header.
80 * The IP header and mutable extension headers will be moved forward to make
81 * space for the encapsulation header.
83 static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb)
85 #if IS_ENABLED(CONFIG_IPV6)
91 skb_set_inner_transport_header(skb, skb_transport_offset(skb));
93 hdr_len = x->type->hdr_offset(x, skb, &prevhdr);
96 skb_set_mac_header(skb,
97 (prevhdr - x->props.header_len) - skb->data);
98 skb_set_network_header(skb, -x->props.header_len);
99 skb->transport_header = skb->network_header + hdr_len;
100 __skb_pull(skb, hdr_len);
101 memmove(ipv6_hdr(skb), iph, hdr_len);
105 return -EAFNOSUPPORT;
109 /* Add route optimization header space.
111 * The IP header and mutable extension headers will be moved forward to make
112 * space for the route optimization header.
114 static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb)
116 #if IS_ENABLED(CONFIG_IPV6)
123 hdr_len = x->type->hdr_offset(x, skb, &prevhdr);
126 skb_set_mac_header(skb,
127 (prevhdr - x->props.header_len) - skb->data);
128 skb_set_network_header(skb, -x->props.header_len);
129 skb->transport_header = skb->network_header + hdr_len;
130 __skb_pull(skb, hdr_len);
131 memmove(ipv6_hdr(skb), iph, hdr_len);
133 x->lastused = ktime_get_real_seconds();
138 return -EAFNOSUPPORT;
142 /* Add encapsulation header.
144 * The top IP header will be constructed per draft-nikander-esp-beet-mode-06.txt.
146 static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
148 struct ip_beet_phdr *ph;
149 struct iphdr *top_iph;
153 optlen = XFRM_MODE_SKB_CB(skb)->optlen;
154 if (unlikely(optlen))
155 hdrlen += IPV4_BEET_PHMAXLEN - (optlen & 4);
157 skb_set_network_header(skb, -x->props.header_len - hdrlen +
158 (XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph)));
159 if (x->sel.family != AF_INET6)
160 skb->network_header += IPV4_BEET_PHMAXLEN;
161 skb->mac_header = skb->network_header +
162 offsetof(struct iphdr, protocol);
163 skb->transport_header = skb->network_header + sizeof(*top_iph);
165 xfrm4_beet_make_header(skb);
167 ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen);
169 top_iph = ip_hdr(skb);
171 if (unlikely(optlen)) {
172 if (WARN_ON(optlen < 0))
175 ph->padlen = 4 - (optlen & 4);
176 ph->hdrlen = optlen / 8;
177 ph->nexthdr = top_iph->protocol;
179 memset(ph + 1, IPOPT_NOP, ph->padlen);
181 top_iph->protocol = IPPROTO_BEETPH;
182 top_iph->ihl = sizeof(struct iphdr) / 4;
185 top_iph->saddr = x->props.saddr.a4;
186 top_iph->daddr = x->id.daddr.a4;
191 /* Add encapsulation header.
193 * The top IP header will be constructed per RFC 2401.
195 static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
197 struct dst_entry *dst = skb_dst(skb);
198 struct iphdr *top_iph;
201 skb_set_inner_network_header(skb, skb_network_offset(skb));
202 skb_set_inner_transport_header(skb, skb_transport_offset(skb));
204 skb_set_network_header(skb, -x->props.header_len);
205 skb->mac_header = skb->network_header +
206 offsetof(struct iphdr, protocol);
207 skb->transport_header = skb->network_header + sizeof(*top_iph);
208 top_iph = ip_hdr(skb);
211 top_iph->version = 4;
213 top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family);
215 /* DS disclosing depends on XFRM_SA_XFLAG_DONT_ENCAP_DSCP */
216 if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
219 top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos;
220 top_iph->tos = INET_ECN_encapsulate(top_iph->tos,
221 XFRM_MODE_SKB_CB(skb)->tos);
223 flags = x->props.flags;
224 if (flags & XFRM_STATE_NOECN)
225 IP_ECN_clear(top_iph);
227 top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) ?
228 0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF));
230 top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst));
232 top_iph->saddr = x->props.saddr.a4;
233 top_iph->daddr = x->id.daddr.a4;
234 ip_select_ident(dev_net(dst->dev), skb, NULL);
239 #if IS_ENABLED(CONFIG_IPV6)
240 static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb)
242 struct dst_entry *dst = skb_dst(skb);
243 struct ipv6hdr *top_iph;
246 skb_set_inner_network_header(skb, skb_network_offset(skb));
247 skb_set_inner_transport_header(skb, skb_transport_offset(skb));
249 skb_set_network_header(skb, -x->props.header_len);
250 skb->mac_header = skb->network_header +
251 offsetof(struct ipv6hdr, nexthdr);
252 skb->transport_header = skb->network_header + sizeof(*top_iph);
253 top_iph = ipv6_hdr(skb);
255 top_iph->version = 6;
257 memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl,
258 sizeof(top_iph->flow_lbl));
259 top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family);
261 if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP)
264 dsfield = XFRM_MODE_SKB_CB(skb)->tos;
265 dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos);
266 if (x->props.flags & XFRM_STATE_NOECN)
267 dsfield &= ~INET_ECN_MASK;
268 ipv6_change_dsfield(top_iph, 0, dsfield);
269 top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst));
270 top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
271 top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
275 static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb)
277 struct ipv6hdr *top_iph;
278 struct ip_beet_phdr *ph;
282 optlen = XFRM_MODE_SKB_CB(skb)->optlen;
283 if (unlikely(optlen))
284 hdr_len += IPV4_BEET_PHMAXLEN - (optlen & 4);
286 skb_set_network_header(skb, -x->props.header_len - hdr_len);
287 if (x->sel.family != AF_INET6)
288 skb->network_header += IPV4_BEET_PHMAXLEN;
289 skb->mac_header = skb->network_header +
290 offsetof(struct ipv6hdr, nexthdr);
291 skb->transport_header = skb->network_header + sizeof(*top_iph);
292 ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len);
294 xfrm6_beet_make_header(skb);
296 top_iph = ipv6_hdr(skb);
297 if (unlikely(optlen)) {
298 if (WARN_ON(optlen < 0))
301 ph->padlen = 4 - (optlen & 4);
302 ph->hdrlen = optlen / 8;
303 ph->nexthdr = top_iph->nexthdr;
305 memset(ph + 1, IPOPT_NOP, ph->padlen);
307 top_iph->nexthdr = IPPROTO_BEETPH;
310 top_iph->saddr = *(struct in6_addr *)&x->props.saddr;
311 top_iph->daddr = *(struct in6_addr *)&x->id.daddr;
316 /* Add encapsulation header.
318 * On exit, the transport header will be set to the start of the
319 * encapsulation header to be filled in by x->type->output and the mac
320 * header will be set to the nextheader (protocol for IPv4) field of the
321 * extension header directly preceding the encapsulation header, or in
322 * its absence, that of the top IP header.
323 * The value of the network header will always point to the top IP header
324 * while skb->data will point to the payload.
326 static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
330 err = xfrm_inner_extract_output(x, skb);
334 IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE;
335 skb->protocol = htons(ETH_P_IP);
337 switch (x->outer_mode.encap) {
339 return xfrm4_beet_encap_add(x, skb);
340 case XFRM_MODE_TUNNEL:
341 return xfrm4_tunnel_encap_add(x, skb);
348 static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb)
350 #if IS_ENABLED(CONFIG_IPV6)
353 err = xfrm_inner_extract_output(x, skb);
358 skb->protocol = htons(ETH_P_IPV6);
360 switch (x->outer_mode.encap) {
362 return xfrm6_beet_encap_add(x, skb);
363 case XFRM_MODE_TUNNEL:
364 return xfrm6_tunnel_encap_add(x, skb);
371 return -EAFNOSUPPORT;
374 static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
376 switch (x->outer_mode.encap) {
378 case XFRM_MODE_TUNNEL:
379 if (x->outer_mode.family == AF_INET)
380 return xfrm4_prepare_output(x, skb);
381 if (x->outer_mode.family == AF_INET6)
382 return xfrm6_prepare_output(x, skb);
384 case XFRM_MODE_TRANSPORT:
385 if (x->outer_mode.family == AF_INET)
386 return xfrm4_transport_output(x, skb);
387 if (x->outer_mode.family == AF_INET6)
388 return xfrm6_transport_output(x, skb);
390 case XFRM_MODE_ROUTEOPTIMIZATION:
391 if (x->outer_mode.family == AF_INET6)
392 return xfrm6_ro_output(x, skb);
403 #if IS_ENABLED(CONFIG_NET_PKTGEN)
404 int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb)
406 return xfrm_outer_mode_output(x, skb);
408 EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output);
411 static int xfrm_output_one(struct sk_buff *skb, int err)
413 struct dst_entry *dst = skb_dst(skb);
414 struct xfrm_state *x = dst->xfrm;
415 struct net *net = xs_net(x);
421 err = xfrm_skb_check_space(skb);
423 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
427 skb->mark = xfrm_smark_get(skb->mark, x);
429 err = xfrm_outer_mode_output(x, skb);
431 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
435 spin_lock_bh(&x->lock);
437 if (unlikely(x->km.state != XFRM_STATE_VALID)) {
438 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID);
443 err = xfrm_state_check_expire(x);
445 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEEXPIRED);
449 err = x->repl->overflow(x, skb);
451 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATESEQERROR);
455 x->curlft.bytes += skb->len;
458 spin_unlock_bh(&x->lock);
462 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
467 if (xfrm_offload(skb)) {
468 x->type_offload->encap(x, skb);
470 /* Inner headers are invalid now. */
471 skb->encapsulation = 0;
473 err = x->type->output(x, skb);
474 if (err == -EINPROGRESS)
480 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
484 dst = skb_dst_pop(skb);
486 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
490 skb_dst_set(skb, dst);
492 } while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL));
497 spin_unlock_bh(&x->lock);
504 int xfrm_output_resume(struct sk_buff *skb, int err)
506 struct net *net = xs_net(skb_dst(skb)->xfrm);
508 while (likely((err = xfrm_output_one(skb, err)) == 0)) {
511 err = skb_dst(skb)->ops->local_out(net, skb->sk, skb);
512 if (unlikely(err != 1))
515 if (!skb_dst(skb)->xfrm)
516 return dst_output(net, skb->sk, skb);
518 err = nf_hook(skb_dst(skb)->ops->family,
519 NF_INET_POST_ROUTING, net, skb->sk, skb,
520 NULL, skb_dst(skb)->dev, xfrm_output2);
521 if (unlikely(err != 1))
525 if (err == -EINPROGRESS)
531 EXPORT_SYMBOL_GPL(xfrm_output_resume);
533 static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
535 return xfrm_output_resume(skb, 1);
538 static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb)
540 struct sk_buff *segs;
542 BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_SGO_CB_OFFSET);
543 BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_SGO_CB_OFFSET);
544 segs = skb_gso_segment(skb, 0);
547 return PTR_ERR(segs);
552 struct sk_buff *nskb = segs->next;
555 skb_mark_not_on_list(segs);
556 err = xfrm_output2(net, sk, segs);
559 kfree_skb_list(nskb);
569 int xfrm_output(struct sock *sk, struct sk_buff *skb)
571 struct net *net = dev_net(skb_dst(skb)->dev);
572 struct xfrm_state *x = skb_dst(skb)->xfrm;
577 if (xfrm_dev_offload_ok(skb, x)) {
580 sp = secpath_set(skb);
582 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
586 skb->encapsulation = 1;
589 sp->xvec[sp->len++] = x;
592 if (skb_is_gso(skb)) {
593 skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
595 return xfrm_output2(net, sk, skb);
598 if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
603 return xfrm_output_gso(net, sk, skb);
605 if (skb->ip_summed == CHECKSUM_PARTIAL) {
606 err = skb_checksum_help(skb);
608 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
615 return xfrm_output2(net, sk, skb);
617 EXPORT_SYMBOL_GPL(xfrm_output);
619 static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb)
621 const struct xfrm_state_afinfo *afinfo;
622 const struct xfrm_mode *inner_mode;
623 int err = -EAFNOSUPPORT;
625 if (x->sel.family == AF_UNSPEC)
626 inner_mode = xfrm_ip2inner_mode(x,
627 xfrm_af2proto(skb_dst(skb)->ops->family));
629 inner_mode = &x->inner_mode;
631 if (inner_mode == NULL)
632 return -EAFNOSUPPORT;
635 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family);
637 err = afinfo->extract_output(x, skb);
643 void xfrm_local_error(struct sk_buff *skb, int mtu)
646 struct xfrm_state_afinfo *afinfo;
648 if (skb->protocol == htons(ETH_P_IP))
650 else if (skb->protocol == htons(ETH_P_IPV6))
655 afinfo = xfrm_state_get_afinfo(proto);
657 afinfo->local_error(skb, mtu);
661 EXPORT_SYMBOL_GPL(xfrm_local_error);