Merge tag 'icc-6.7-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc...
[linux-2.6-microblaze.git] / net / ipv4 / tcp_ao.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET         An implementation of the TCP Authentication Option (TCP-AO).
4  *              See RFC5925.
5  *
6  * Authors:     Dmitry Safonov <dima@arista.com>
7  *              Francesco Ruggeri <fruggeri@arista.com>
8  *              Salam Noureddine <noureddine@arista.com>
9  */
10 #define pr_fmt(fmt) "TCP: " fmt
11
12 #include <crypto/hash.h>
13 #include <linux/inetdevice.h>
14 #include <linux/tcp.h>
15
16 #include <net/tcp.h>
17 #include <net/ipv6.h>
18 #include <net/icmp.h>
19
20 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_ao_needed, HZ);
21
22 int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx,
23                             unsigned int len, struct tcp_sigpool *hp)
24 {
25         struct scatterlist sg;
26         int ret;
27
28         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp->req),
29                                 mkt->key, mkt->keylen))
30                 goto clear_hash;
31
32         ret = crypto_ahash_init(hp->req);
33         if (ret)
34                 goto clear_hash;
35
36         sg_init_one(&sg, ctx, len);
37         ahash_request_set_crypt(hp->req, &sg, key, len);
38         crypto_ahash_update(hp->req);
39
40         ret = crypto_ahash_final(hp->req);
41         if (ret)
42                 goto clear_hash;
43
44         return 0;
45 clear_hash:
46         memset(key, 0, tcp_ao_digest_size(mkt));
47         return 1;
48 }
49
50 bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code)
51 {
52         bool ignore_icmp = false;
53         struct tcp_ao_info *ao;
54
55         if (!static_branch_unlikely(&tcp_ao_needed.key))
56                 return false;
57
58         /* RFC5925, 7.8:
59          * >> A TCP-AO implementation MUST default to ignore incoming ICMPv4
60          * messages of Type 3 (destination unreachable), Codes 2-4 (protocol
61          * unreachable, port unreachable, and fragmentation needed -- ’hard
62          * errors’), and ICMPv6 Type 1 (destination unreachable), Code 1
63          * (administratively prohibited) and Code 4 (port unreachable) intended
64          * for connections in synchronized states (ESTABLISHED, FIN-WAIT-1, FIN-
65          * WAIT-2, CLOSE-WAIT, CLOSING, LAST-ACK, TIME-WAIT) that match MKTs.
66          */
67         if (family == AF_INET) {
68                 if (type != ICMP_DEST_UNREACH)
69                         return false;
70                 if (code < ICMP_PROT_UNREACH || code > ICMP_FRAG_NEEDED)
71                         return false;
72         } else {
73                 if (type != ICMPV6_DEST_UNREACH)
74                         return false;
75                 if (code != ICMPV6_ADM_PROHIBITED && code != ICMPV6_PORT_UNREACH)
76                         return false;
77         }
78
79         rcu_read_lock();
80         switch (sk->sk_state) {
81         case TCP_TIME_WAIT:
82                 ao = rcu_dereference(tcp_twsk(sk)->ao_info);
83                 break;
84         case TCP_SYN_SENT:
85         case TCP_SYN_RECV:
86         case TCP_LISTEN:
87         case TCP_NEW_SYN_RECV:
88                 /* RFC5925 specifies to ignore ICMPs *only* on connections
89                  * in synchronized states.
90                  */
91                 rcu_read_unlock();
92                 return false;
93         default:
94                 ao = rcu_dereference(tcp_sk(sk)->ao_info);
95         }
96
97         if (ao && !ao->accept_icmps) {
98                 ignore_icmp = true;
99                 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAODROPPEDICMPS);
100                 atomic64_inc(&ao->counters.dropped_icmp);
101         }
102         rcu_read_unlock();
103
104         return ignore_icmp;
105 }
106
107 /* Optimized version of tcp_ao_do_lookup(): only for sockets for which
108  * it's known that the keys in ao_info are matching peer's
109  * family/address/VRF/etc.
110  */
111 struct tcp_ao_key *tcp_ao_established_key(struct tcp_ao_info *ao,
112                                           int sndid, int rcvid)
113 {
114         struct tcp_ao_key *key;
115
116         hlist_for_each_entry_rcu(key, &ao->head, node) {
117                 if ((sndid >= 0 && key->sndid != sndid) ||
118                     (rcvid >= 0 && key->rcvid != rcvid))
119                         continue;
120                 return key;
121         }
122
123         return NULL;
124 }
125
126 static int ipv4_prefix_cmp(const struct in_addr *addr1,
127                            const struct in_addr *addr2,
128                            unsigned int prefixlen)
129 {
130         __be32 mask = inet_make_mask(prefixlen);
131         __be32 a1 = addr1->s_addr & mask;
132         __be32 a2 = addr2->s_addr & mask;
133
134         if (a1 == a2)
135                 return 0;
136         return memcmp(&a1, &a2, sizeof(a1));
137 }
138
139 static int __tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
140                             const union tcp_ao_addr *addr, u8 prefixlen,
141                             int family, int sndid, int rcvid)
142 {
143         if (sndid >= 0 && key->sndid != sndid)
144                 return (key->sndid > sndid) ? 1 : -1;
145         if (rcvid >= 0 && key->rcvid != rcvid)
146                 return (key->rcvid > rcvid) ? 1 : -1;
147         if (l3index >= 0 && (key->keyflags & TCP_AO_KEYF_IFINDEX)) {
148                 if (key->l3index != l3index)
149                         return (key->l3index > l3index) ? 1 : -1;
150         }
151
152         if (family == AF_UNSPEC)
153                 return 0;
154         if (key->family != family)
155                 return (key->family > family) ? 1 : -1;
156
157         if (family == AF_INET) {
158                 if (ntohl(key->addr.a4.s_addr) == INADDR_ANY)
159                         return 0;
160                 if (ntohl(addr->a4.s_addr) == INADDR_ANY)
161                         return 0;
162                 return ipv4_prefix_cmp(&key->addr.a4, &addr->a4, prefixlen);
163 #if IS_ENABLED(CONFIG_IPV6)
164         } else {
165                 if (ipv6_addr_any(&key->addr.a6) || ipv6_addr_any(&addr->a6))
166                         return 0;
167                 if (ipv6_prefix_equal(&key->addr.a6, &addr->a6, prefixlen))
168                         return 0;
169                 return memcmp(&key->addr.a6, &addr->a6, sizeof(addr->a6));
170 #endif
171         }
172         return -1;
173 }
174
175 static int tcp_ao_key_cmp(const struct tcp_ao_key *key, int l3index,
176                           const union tcp_ao_addr *addr, u8 prefixlen,
177                           int family, int sndid, int rcvid)
178 {
179 #if IS_ENABLED(CONFIG_IPV6)
180         if (family == AF_INET6 && ipv6_addr_v4mapped(&addr->a6)) {
181                 __be32 addr4 = addr->a6.s6_addr32[3];
182
183                 return __tcp_ao_key_cmp(key, l3index,
184                                         (union tcp_ao_addr *)&addr4,
185                                         prefixlen, AF_INET, sndid, rcvid);
186         }
187 #endif
188         return __tcp_ao_key_cmp(key, l3index, addr,
189                                 prefixlen, family, sndid, rcvid);
190 }
191
192 static struct tcp_ao_key *__tcp_ao_do_lookup(const struct sock *sk, int l3index,
193                 const union tcp_ao_addr *addr, int family, u8 prefix,
194                 int sndid, int rcvid)
195 {
196         struct tcp_ao_key *key;
197         struct tcp_ao_info *ao;
198
199         if (!static_branch_unlikely(&tcp_ao_needed.key))
200                 return NULL;
201
202         ao = rcu_dereference_check(tcp_sk(sk)->ao_info,
203                                    lockdep_sock_is_held(sk));
204         if (!ao)
205                 return NULL;
206
207         hlist_for_each_entry_rcu(key, &ao->head, node) {
208                 u8 prefixlen = min(prefix, key->prefixlen);
209
210                 if (!tcp_ao_key_cmp(key, l3index, addr, prefixlen,
211                                     family, sndid, rcvid))
212                         return key;
213         }
214         return NULL;
215 }
216
217 struct tcp_ao_key *tcp_ao_do_lookup(const struct sock *sk, int l3index,
218                                     const union tcp_ao_addr *addr,
219                                     int family, int sndid, int rcvid)
220 {
221         return __tcp_ao_do_lookup(sk, l3index, addr, family, U8_MAX, sndid, rcvid);
222 }
223
224 static struct tcp_ao_info *tcp_ao_alloc_info(gfp_t flags)
225 {
226         struct tcp_ao_info *ao;
227
228         ao = kzalloc(sizeof(*ao), flags);
229         if (!ao)
230                 return NULL;
231         INIT_HLIST_HEAD(&ao->head);
232         refcount_set(&ao->refcnt, 1);
233
234         return ao;
235 }
236
237 static void tcp_ao_link_mkt(struct tcp_ao_info *ao, struct tcp_ao_key *mkt)
238 {
239         hlist_add_head_rcu(&mkt->node, &ao->head);
240 }
241
242 static struct tcp_ao_key *tcp_ao_copy_key(struct sock *sk,
243                                           struct tcp_ao_key *key)
244 {
245         struct tcp_ao_key *new_key;
246
247         new_key = sock_kmalloc(sk, tcp_ao_sizeof_key(key),
248                                GFP_ATOMIC);
249         if (!new_key)
250                 return NULL;
251
252         *new_key = *key;
253         INIT_HLIST_NODE(&new_key->node);
254         tcp_sigpool_get(new_key->tcp_sigpool_id);
255         atomic64_set(&new_key->pkt_good, 0);
256         atomic64_set(&new_key->pkt_bad, 0);
257
258         return new_key;
259 }
260
261 static void tcp_ao_key_free_rcu(struct rcu_head *head)
262 {
263         struct tcp_ao_key *key = container_of(head, struct tcp_ao_key, rcu);
264
265         tcp_sigpool_release(key->tcp_sigpool_id);
266         kfree_sensitive(key);
267 }
268
269 void tcp_ao_destroy_sock(struct sock *sk, bool twsk)
270 {
271         struct tcp_ao_info *ao;
272         struct tcp_ao_key *key;
273         struct hlist_node *n;
274
275         if (twsk) {
276                 ao = rcu_dereference_protected(tcp_twsk(sk)->ao_info, 1);
277                 tcp_twsk(sk)->ao_info = NULL;
278         } else {
279                 ao = rcu_dereference_protected(tcp_sk(sk)->ao_info, 1);
280                 tcp_sk(sk)->ao_info = NULL;
281         }
282
283         if (!ao || !refcount_dec_and_test(&ao->refcnt))
284                 return;
285
286         hlist_for_each_entry_safe(key, n, &ao->head, node) {
287                 hlist_del_rcu(&key->node);
288                 if (!twsk)
289                         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
290                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
291         }
292
293         kfree_rcu(ao, rcu);
294         static_branch_slow_dec_deferred(&tcp_ao_needed);
295 }
296
297 void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp)
298 {
299         struct tcp_ao_info *ao_info = rcu_dereference_protected(tp->ao_info, 1);
300
301         if (ao_info) {
302                 struct tcp_ao_key *key;
303                 struct hlist_node *n;
304                 int omem = 0;
305
306                 hlist_for_each_entry_safe(key, n, &ao_info->head, node) {
307                         omem += tcp_ao_sizeof_key(key);
308                 }
309
310                 refcount_inc(&ao_info->refcnt);
311                 atomic_sub(omem, &(((struct sock *)tp)->sk_omem_alloc));
312                 rcu_assign_pointer(tcptw->ao_info, ao_info);
313         } else {
314                 tcptw->ao_info = NULL;
315         }
316 }
317
318 /* 4 tuple and ISNs are expected in NBO */
319 static int tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
320                               __be32 saddr, __be32 daddr,
321                               __be16 sport, __be16 dport,
322                               __be32 sisn,  __be32 disn)
323 {
324         /* See RFC5926 3.1.1 */
325         struct kdf_input_block {
326                 u8                      counter;
327                 u8                      label[6];
328                 struct tcp4_ao_context  ctx;
329                 __be16                  outlen;
330         } __packed * tmp;
331         struct tcp_sigpool hp;
332         int err;
333
334         err = tcp_sigpool_start(mkt->tcp_sigpool_id, &hp);
335         if (err)
336                 return err;
337
338         tmp = hp.scratch;
339         tmp->counter    = 1;
340         memcpy(tmp->label, "TCP-AO", 6);
341         tmp->ctx.saddr  = saddr;
342         tmp->ctx.daddr  = daddr;
343         tmp->ctx.sport  = sport;
344         tmp->ctx.dport  = dport;
345         tmp->ctx.sisn   = sisn;
346         tmp->ctx.disn   = disn;
347         tmp->outlen     = htons(tcp_ao_digest_size(mkt) * 8); /* in bits */
348
349         err = tcp_ao_calc_traffic_key(mkt, key, tmp, sizeof(*tmp), &hp);
350         tcp_sigpool_end(&hp);
351
352         return err;
353 }
354
355 int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
356                           const struct sock *sk,
357                           __be32 sisn, __be32 disn, bool send)
358 {
359         if (send)
360                 return tcp_v4_ao_calc_key(mkt, key, sk->sk_rcv_saddr,
361                                           sk->sk_daddr, htons(sk->sk_num),
362                                           sk->sk_dport, sisn, disn);
363         else
364                 return tcp_v4_ao_calc_key(mkt, key, sk->sk_daddr,
365                                           sk->sk_rcv_saddr, sk->sk_dport,
366                                           htons(sk->sk_num), disn, sisn);
367 }
368
369 static int tcp_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
370                               const struct sock *sk,
371                               __be32 sisn, __be32 disn, bool send)
372 {
373         if (mkt->family == AF_INET)
374                 return tcp_v4_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
375 #if IS_ENABLED(CONFIG_IPV6)
376         else if (mkt->family == AF_INET6)
377                 return tcp_v6_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
378 #endif
379         else
380                 return -EOPNOTSUPP;
381 }
382
383 int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
384                            struct request_sock *req)
385 {
386         struct inet_request_sock *ireq = inet_rsk(req);
387
388         return tcp_v4_ao_calc_key(mkt, key,
389                                   ireq->ir_loc_addr, ireq->ir_rmt_addr,
390                                   htons(ireq->ir_num), ireq->ir_rmt_port,
391                                   htonl(tcp_rsk(req)->snt_isn),
392                                   htonl(tcp_rsk(req)->rcv_isn));
393 }
394
395 static int tcp_v4_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
396                                   const struct sk_buff *skb,
397                                   __be32 sisn, __be32 disn)
398 {
399         const struct iphdr *iph = ip_hdr(skb);
400         const struct tcphdr *th = tcp_hdr(skb);
401
402         return tcp_v4_ao_calc_key(mkt, key, iph->saddr, iph->daddr,
403                                   th->source, th->dest, sisn, disn);
404 }
405
406 static int tcp_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
407                                const struct sk_buff *skb,
408                                __be32 sisn, __be32 disn, int family)
409 {
410         if (family == AF_INET)
411                 return tcp_v4_ao_calc_key_skb(mkt, key, skb, sisn, disn);
412 #if IS_ENABLED(CONFIG_IPV6)
413         else if (family == AF_INET6)
414                 return tcp_v6_ao_calc_key_skb(mkt, key, skb, sisn, disn);
415 #endif
416         return -EAFNOSUPPORT;
417 }
418
419 static int tcp_v4_ao_hash_pseudoheader(struct tcp_sigpool *hp,
420                                        __be32 daddr, __be32 saddr,
421                                        int nbytes)
422 {
423         struct tcp4_pseudohdr *bp;
424         struct scatterlist sg;
425
426         bp = hp->scratch;
427         bp->saddr = saddr;
428         bp->daddr = daddr;
429         bp->pad = 0;
430         bp->protocol = IPPROTO_TCP;
431         bp->len = cpu_to_be16(nbytes);
432
433         sg_init_one(&sg, bp, sizeof(*bp));
434         ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
435         return crypto_ahash_update(hp->req);
436 }
437
438 static int tcp_ao_hash_pseudoheader(unsigned short int family,
439                                     const struct sock *sk,
440                                     const struct sk_buff *skb,
441                                     struct tcp_sigpool *hp, int nbytes)
442 {
443         const struct tcphdr *th = tcp_hdr(skb);
444
445         /* TODO: Can we rely on checksum being zero to mean outbound pkt? */
446         if (!th->check) {
447                 if (family == AF_INET)
448                         return tcp_v4_ao_hash_pseudoheader(hp, sk->sk_daddr,
449                                         sk->sk_rcv_saddr, skb->len);
450 #if IS_ENABLED(CONFIG_IPV6)
451                 else if (family == AF_INET6)
452                         return tcp_v6_ao_hash_pseudoheader(hp, &sk->sk_v6_daddr,
453                                         &sk->sk_v6_rcv_saddr, skb->len);
454 #endif
455                 else
456                         return -EAFNOSUPPORT;
457         }
458
459         if (family == AF_INET) {
460                 const struct iphdr *iph = ip_hdr(skb);
461
462                 return tcp_v4_ao_hash_pseudoheader(hp, iph->daddr,
463                                 iph->saddr, skb->len);
464 #if IS_ENABLED(CONFIG_IPV6)
465         } else if (family == AF_INET6) {
466                 const struct ipv6hdr *iph = ipv6_hdr(skb);
467
468                 return tcp_v6_ao_hash_pseudoheader(hp, &iph->daddr,
469                                 &iph->saddr, skb->len);
470 #endif
471         }
472         return -EAFNOSUPPORT;
473 }
474
475 u32 tcp_ao_compute_sne(u32 next_sne, u32 next_seq, u32 seq)
476 {
477         u32 sne = next_sne;
478
479         if (before(seq, next_seq)) {
480                 if (seq > next_seq)
481                         sne--;
482         } else {
483                 if (seq < next_seq)
484                         sne++;
485         }
486
487         return sne;
488 }
489
490 /* tcp_ao_hash_sne(struct tcp_sigpool *hp)
491  * @hp  - used for hashing
492  * @sne - sne value
493  */
494 static int tcp_ao_hash_sne(struct tcp_sigpool *hp, u32 sne)
495 {
496         struct scatterlist sg;
497         __be32 *bp;
498
499         bp = (__be32 *)hp->scratch;
500         *bp = htonl(sne);
501
502         sg_init_one(&sg, bp, sizeof(*bp));
503         ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
504         return crypto_ahash_update(hp->req);
505 }
506
507 static int tcp_ao_hash_header(struct tcp_sigpool *hp,
508                               const struct tcphdr *th,
509                               bool exclude_options, u8 *hash,
510                               int hash_offset, int hash_len)
511 {
512         int err, len = th->doff << 2;
513         struct scatterlist sg;
514         u8 *hdr = hp->scratch;
515
516         /* We are not allowed to change tcphdr, make a local copy */
517         if (exclude_options) {
518                 len = sizeof(*th) + sizeof(struct tcp_ao_hdr) + hash_len;
519                 memcpy(hdr, th, sizeof(*th));
520                 memcpy(hdr + sizeof(*th),
521                        (u8 *)th + hash_offset - sizeof(struct tcp_ao_hdr),
522                        sizeof(struct tcp_ao_hdr));
523                 memset(hdr + sizeof(*th) + sizeof(struct tcp_ao_hdr),
524                        0, hash_len);
525                 ((struct tcphdr *)hdr)->check = 0;
526         } else {
527                 len = th->doff << 2;
528                 memcpy(hdr, th, len);
529                 /* zero out tcp-ao hash */
530                 ((struct tcphdr *)hdr)->check = 0;
531                 memset(hdr + hash_offset, 0, hash_len);
532         }
533
534         sg_init_one(&sg, hdr, len);
535         ahash_request_set_crypt(hp->req, &sg, NULL, len);
536         err = crypto_ahash_update(hp->req);
537         WARN_ON_ONCE(err != 0);
538         return err;
539 }
540
541 int tcp_ao_hash_hdr(unsigned short int family, char *ao_hash,
542                     struct tcp_ao_key *key, const u8 *tkey,
543                     const union tcp_ao_addr *daddr,
544                     const union tcp_ao_addr *saddr,
545                     const struct tcphdr *th, u32 sne)
546 {
547         int tkey_len = tcp_ao_digest_size(key);
548         int hash_offset = ao_hash - (char *)th;
549         struct tcp_sigpool hp;
550         void *hash_buf = NULL;
551
552         hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
553         if (!hash_buf)
554                 goto clear_hash_noput;
555
556         if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
557                 goto clear_hash_noput;
558
559         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
560                 goto clear_hash;
561
562         if (crypto_ahash_init(hp.req))
563                 goto clear_hash;
564
565         if (tcp_ao_hash_sne(&hp, sne))
566                 goto clear_hash;
567         if (family == AF_INET) {
568                 if (tcp_v4_ao_hash_pseudoheader(&hp, daddr->a4.s_addr,
569                                                 saddr->a4.s_addr, th->doff * 4))
570                         goto clear_hash;
571 #if IS_ENABLED(CONFIG_IPV6)
572         } else if (family == AF_INET6) {
573                 if (tcp_v6_ao_hash_pseudoheader(&hp, &daddr->a6,
574                                                 &saddr->a6, th->doff * 4))
575                         goto clear_hash;
576 #endif
577         } else {
578                 WARN_ON_ONCE(1);
579                 goto clear_hash;
580         }
581         if (tcp_ao_hash_header(&hp, th,
582                                !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
583                                ao_hash, hash_offset, tcp_ao_maclen(key)))
584                 goto clear_hash;
585         ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
586         if (crypto_ahash_final(hp.req))
587                 goto clear_hash;
588
589         memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
590         tcp_sigpool_end(&hp);
591         kfree(hash_buf);
592         return 0;
593
594 clear_hash:
595         tcp_sigpool_end(&hp);
596 clear_hash_noput:
597         memset(ao_hash, 0, tcp_ao_maclen(key));
598         kfree(hash_buf);
599         return 1;
600 }
601
602 int tcp_ao_hash_skb(unsigned short int family,
603                     char *ao_hash, struct tcp_ao_key *key,
604                     const struct sock *sk, const struct sk_buff *skb,
605                     const u8 *tkey, int hash_offset, u32 sne)
606 {
607         const struct tcphdr *th = tcp_hdr(skb);
608         int tkey_len = tcp_ao_digest_size(key);
609         struct tcp_sigpool hp;
610         void *hash_buf = NULL;
611
612         hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
613         if (!hash_buf)
614                 goto clear_hash_noput;
615
616         if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
617                 goto clear_hash_noput;
618
619         if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
620                 goto clear_hash;
621
622         /* For now use sha1 by default. Depends on alg in tcp_ao_key */
623         if (crypto_ahash_init(hp.req))
624                 goto clear_hash;
625
626         if (tcp_ao_hash_sne(&hp, sne))
627                 goto clear_hash;
628         if (tcp_ao_hash_pseudoheader(family, sk, skb, &hp, skb->len))
629                 goto clear_hash;
630         if (tcp_ao_hash_header(&hp, th,
631                                !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
632                                ao_hash, hash_offset, tcp_ao_maclen(key)))
633                 goto clear_hash;
634         if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
635                 goto clear_hash;
636         ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
637         if (crypto_ahash_final(hp.req))
638                 goto clear_hash;
639
640         memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
641         tcp_sigpool_end(&hp);
642         kfree(hash_buf);
643         return 0;
644
645 clear_hash:
646         tcp_sigpool_end(&hp);
647 clear_hash_noput:
648         memset(ao_hash, 0, tcp_ao_maclen(key));
649         kfree(hash_buf);
650         return 1;
651 }
652
653 int tcp_v4_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key,
654                        const struct sock *sk, const struct sk_buff *skb,
655                        const u8 *tkey, int hash_offset, u32 sne)
656 {
657         return tcp_ao_hash_skb(AF_INET, ao_hash, key, sk, skb,
658                                tkey, hash_offset, sne);
659 }
660
661 int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *ao_key,
662                           struct request_sock *req, const struct sk_buff *skb,
663                           int hash_offset, u32 sne)
664 {
665         void *hash_buf = NULL;
666         int err;
667
668         hash_buf = kmalloc(tcp_ao_digest_size(ao_key), GFP_ATOMIC);
669         if (!hash_buf)
670                 return -ENOMEM;
671
672         err = tcp_v4_ao_calc_key_rsk(ao_key, hash_buf, req);
673         if (err)
674                 goto out;
675
676         err = tcp_ao_hash_skb(AF_INET, ao_hash, ao_key, req_to_sk(req), skb,
677                               hash_buf, hash_offset, sne);
678 out:
679         kfree(hash_buf);
680         return err;
681 }
682
683 struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk,
684                                         struct request_sock *req,
685                                         int sndid, int rcvid)
686 {
687         struct inet_request_sock *ireq = inet_rsk(req);
688         union tcp_ao_addr *addr = (union tcp_ao_addr *)&ireq->ir_rmt_addr;
689         int l3index;
690
691         l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
692         return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
693 }
694
695 struct tcp_ao_key *tcp_v4_ao_lookup(const struct sock *sk, struct sock *addr_sk,
696                                     int sndid, int rcvid)
697 {
698         int l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
699                                                      addr_sk->sk_bound_dev_if);
700         union tcp_ao_addr *addr = (union tcp_ao_addr *)&addr_sk->sk_daddr;
701
702         return tcp_ao_do_lookup(sk, l3index, addr, AF_INET, sndid, rcvid);
703 }
704
705 int tcp_ao_prepare_reset(const struct sock *sk, struct sk_buff *skb,
706                          const struct tcp_ao_hdr *aoh, int l3index, u32 seq,
707                          struct tcp_ao_key **key, char **traffic_key,
708                          bool *allocated_traffic_key, u8 *keyid, u32 *sne)
709 {
710         const struct tcphdr *th = tcp_hdr(skb);
711         struct tcp_ao_info *ao_info;
712
713         *allocated_traffic_key = false;
714         /* If there's no socket - than initial sisn/disn are unknown.
715          * Drop the segment. RFC5925 (7.7) advises to require graceful
716          * restart [RFC4724]. Alternatively, the RFC5925 advises to
717          * save/restore traffic keys before/after reboot.
718          * Linux TCP-AO support provides TCP_AO_ADD_KEY and TCP_AO_REPAIR
719          * options to restore a socket post-reboot.
720          */
721         if (!sk)
722                 return -ENOTCONN;
723
724         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
725                 unsigned int family = READ_ONCE(sk->sk_family);
726                 union tcp_ao_addr *addr;
727                 __be32 disn, sisn;
728
729                 if (sk->sk_state == TCP_NEW_SYN_RECV) {
730                         struct request_sock *req = inet_reqsk(sk);
731
732                         sisn = htonl(tcp_rsk(req)->rcv_isn);
733                         disn = htonl(tcp_rsk(req)->snt_isn);
734                         *sne = tcp_ao_compute_sne(0, tcp_rsk(req)->snt_isn, seq);
735                 } else {
736                         sisn = th->seq;
737                         disn = 0;
738                 }
739                 if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
740                         addr = (union tcp_md5_addr *)&ipv6_hdr(skb)->saddr;
741                 else
742                         addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
743 #if IS_ENABLED(CONFIG_IPV6)
744                 if (family == AF_INET6 && ipv6_addr_v4mapped(&sk->sk_v6_daddr))
745                         family = AF_INET;
746 #endif
747
748                 sk = sk_const_to_full_sk(sk);
749                 ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
750                 if (!ao_info)
751                         return -ENOENT;
752                 *key = tcp_ao_do_lookup(sk, l3index, addr, family,
753                                         -1, aoh->rnext_keyid);
754                 if (!*key)
755                         return -ENOENT;
756                 *traffic_key = kmalloc(tcp_ao_digest_size(*key), GFP_ATOMIC);
757                 if (!*traffic_key)
758                         return -ENOMEM;
759                 *allocated_traffic_key = true;
760                 if (tcp_ao_calc_key_skb(*key, *traffic_key, skb,
761                                         sisn, disn, family))
762                         return -1;
763                 *keyid = (*key)->rcvid;
764         } else {
765                 struct tcp_ao_key *rnext_key;
766                 u32 snd_basis;
767
768                 if (sk->sk_state == TCP_TIME_WAIT) {
769                         ao_info = rcu_dereference(tcp_twsk(sk)->ao_info);
770                         snd_basis = tcp_twsk(sk)->tw_snd_nxt;
771                 } else {
772                         ao_info = rcu_dereference(tcp_sk(sk)->ao_info);
773                         snd_basis = tcp_sk(sk)->snd_una;
774                 }
775                 if (!ao_info)
776                         return -ENOENT;
777
778                 *key = tcp_ao_established_key(ao_info, aoh->rnext_keyid, -1);
779                 if (!*key)
780                         return -ENOENT;
781                 *traffic_key = snd_other_key(*key);
782                 rnext_key = READ_ONCE(ao_info->rnext_key);
783                 *keyid = rnext_key->rcvid;
784                 *sne = tcp_ao_compute_sne(READ_ONCE(ao_info->snd_sne),
785                                           snd_basis, seq);
786         }
787         return 0;
788 }
789
790 int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
791                         struct tcp_ao_key *key, struct tcphdr *th,
792                         __u8 *hash_location)
793 {
794         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
795         struct tcp_sock *tp = tcp_sk(sk);
796         struct tcp_ao_info *ao;
797         void *tkey_buf = NULL;
798         u8 *traffic_key;
799         u32 sne;
800
801         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
802                                        lockdep_sock_is_held(sk));
803         traffic_key = snd_other_key(key);
804         if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
805                 __be32 disn;
806
807                 if (!(tcb->tcp_flags & TCPHDR_ACK)) {
808                         disn = 0;
809                         tkey_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
810                         if (!tkey_buf)
811                                 return -ENOMEM;
812                         traffic_key = tkey_buf;
813                 } else {
814                         disn = ao->risn;
815                 }
816                 tp->af_specific->ao_calc_key_sk(key, traffic_key,
817                                                 sk, ao->lisn, disn, true);
818         }
819         sne = tcp_ao_compute_sne(READ_ONCE(ao->snd_sne), READ_ONCE(tp->snd_una),
820                                  ntohl(th->seq));
821         tp->af_specific->calc_ao_hash(hash_location, key, sk, skb, traffic_key,
822                                       hash_location - (u8 *)th, sne);
823         kfree(tkey_buf);
824         return 0;
825 }
826
827 static struct tcp_ao_key *tcp_ao_inbound_lookup(unsigned short int family,
828                 const struct sock *sk, const struct sk_buff *skb,
829                 int sndid, int rcvid, int l3index)
830 {
831         if (family == AF_INET) {
832                 const struct iphdr *iph = ip_hdr(skb);
833
834                 return tcp_ao_do_lookup(sk, l3index,
835                                         (union tcp_ao_addr *)&iph->saddr,
836                                         AF_INET, sndid, rcvid);
837         } else {
838                 const struct ipv6hdr *iph = ipv6_hdr(skb);
839
840                 return tcp_ao_do_lookup(sk, l3index,
841                                         (union tcp_ao_addr *)&iph->saddr,
842                                         AF_INET6, sndid, rcvid);
843         }
844 }
845
846 void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb,
847                       struct tcp_request_sock *treq,
848                       unsigned short int family, int l3index)
849 {
850         const struct tcphdr *th = tcp_hdr(skb);
851         const struct tcp_ao_hdr *aoh;
852         struct tcp_ao_key *key;
853
854         treq->used_tcp_ao = false;
855
856         if (tcp_parse_auth_options(th, NULL, &aoh) || !aoh)
857                 return;
858
859         key = tcp_ao_inbound_lookup(family, sk, skb, -1, aoh->keyid, l3index);
860         if (!key)
861                 /* Key not found, continue without TCP-AO */
862                 return;
863
864         treq->ao_rcv_next = aoh->keyid;
865         treq->ao_keyid = aoh->rnext_keyid;
866         treq->used_tcp_ao = true;
867 }
868
869 static enum skb_drop_reason
870 tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
871                    unsigned short int family, struct tcp_ao_info *info,
872                    const struct tcp_ao_hdr *aoh, struct tcp_ao_key *key,
873                    u8 *traffic_key, u8 *phash, u32 sne, int l3index)
874 {
875         u8 maclen = aoh->length - sizeof(struct tcp_ao_hdr);
876         const struct tcphdr *th = tcp_hdr(skb);
877         void *hash_buf = NULL;
878
879         if (maclen != tcp_ao_maclen(key)) {
880                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
881                 atomic64_inc(&info->counters.pkt_bad);
882                 atomic64_inc(&key->pkt_bad);
883                 tcp_hash_fail("AO hash wrong length", family, skb,
884                               "%u != %d L3index: %d", maclen,
885                               tcp_ao_maclen(key), l3index);
886                 return SKB_DROP_REASON_TCP_AOFAILURE;
887         }
888
889         hash_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
890         if (!hash_buf)
891                 return SKB_DROP_REASON_NOT_SPECIFIED;
892
893         /* XXX: make it per-AF callback? */
894         tcp_ao_hash_skb(family, hash_buf, key, sk, skb, traffic_key,
895                         (phash - (u8 *)th), sne);
896         if (memcmp(phash, hash_buf, maclen)) {
897                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
898                 atomic64_inc(&info->counters.pkt_bad);
899                 atomic64_inc(&key->pkt_bad);
900                 tcp_hash_fail("AO hash mismatch", family, skb,
901                               "L3index: %d", l3index);
902                 kfree(hash_buf);
903                 return SKB_DROP_REASON_TCP_AOFAILURE;
904         }
905         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOGOOD);
906         atomic64_inc(&info->counters.pkt_good);
907         atomic64_inc(&key->pkt_good);
908         kfree(hash_buf);
909         return SKB_NOT_DROPPED_YET;
910 }
911
912 enum skb_drop_reason
913 tcp_inbound_ao_hash(struct sock *sk, const struct sk_buff *skb,
914                     unsigned short int family, const struct request_sock *req,
915                     int l3index, const struct tcp_ao_hdr *aoh)
916 {
917         const struct tcphdr *th = tcp_hdr(skb);
918         u8 *phash = (u8 *)(aoh + 1); /* hash goes just after the header */
919         struct tcp_ao_info *info;
920         enum skb_drop_reason ret;
921         struct tcp_ao_key *key;
922         __be32 sisn, disn;
923         u8 *traffic_key;
924         u32 sne = 0;
925
926         info = rcu_dereference(tcp_sk(sk)->ao_info);
927         if (!info) {
928                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
929                 tcp_hash_fail("AO key not found", family, skb,
930                               "keyid: %u L3index: %d", aoh->keyid, l3index);
931                 return SKB_DROP_REASON_TCP_AOUNEXPECTED;
932         }
933
934         if (unlikely(th->syn)) {
935                 sisn = th->seq;
936                 disn = 0;
937         }
938
939         /* Fast-path */
940         if (likely((1 << sk->sk_state) & TCP_AO_ESTABLISHED)) {
941                 enum skb_drop_reason err;
942                 struct tcp_ao_key *current_key;
943
944                 /* Check if this socket's rnext_key matches the keyid in the
945                  * packet. If not we lookup the key based on the keyid
946                  * matching the rcvid in the mkt.
947                  */
948                 key = READ_ONCE(info->rnext_key);
949                 if (key->rcvid != aoh->keyid) {
950                         key = tcp_ao_established_key(info, -1, aoh->keyid);
951                         if (!key)
952                                 goto key_not_found;
953                 }
954
955                 /* Delayed retransmitted SYN */
956                 if (unlikely(th->syn && !th->ack))
957                         goto verify_hash;
958
959                 sne = tcp_ao_compute_sne(info->rcv_sne, tcp_sk(sk)->rcv_nxt,
960                                          ntohl(th->seq));
961                 /* Established socket, traffic key are cached */
962                 traffic_key = rcv_other_key(key);
963                 err = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
964                                          traffic_key, phash, sne, l3index);
965                 if (err)
966                         return err;
967                 current_key = READ_ONCE(info->current_key);
968                 /* Key rotation: the peer asks us to use new key (RNext) */
969                 if (unlikely(aoh->rnext_keyid != current_key->sndid)) {
970                         /* If the key is not found we do nothing. */
971                         key = tcp_ao_established_key(info, aoh->rnext_keyid, -1);
972                         if (key)
973                                 /* pairs with tcp_ao_del_cmd */
974                                 WRITE_ONCE(info->current_key, key);
975                 }
976                 return SKB_NOT_DROPPED_YET;
977         }
978
979         /* Lookup key based on peer address and keyid.
980          * current_key and rnext_key must not be used on tcp listen
981          * sockets as otherwise:
982          * - request sockets would race on those key pointers
983          * - tcp_ao_del_cmd() allows async key removal
984          */
985         key = tcp_ao_inbound_lookup(family, sk, skb, -1, aoh->keyid, l3index);
986         if (!key)
987                 goto key_not_found;
988
989         if (th->syn && !th->ack)
990                 goto verify_hash;
991
992         if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV)) {
993                 /* Make the initial syn the likely case here */
994                 if (unlikely(req)) {
995                         sne = tcp_ao_compute_sne(0, tcp_rsk(req)->rcv_isn,
996                                                  ntohl(th->seq));
997                         sisn = htonl(tcp_rsk(req)->rcv_isn);
998                         disn = htonl(tcp_rsk(req)->snt_isn);
999                 } else if (unlikely(th->ack && !th->syn)) {
1000                         /* Possible syncookie packet */
1001                         sisn = htonl(ntohl(th->seq) - 1);
1002                         disn = htonl(ntohl(th->ack_seq) - 1);
1003                         sne = tcp_ao_compute_sne(0, ntohl(sisn),
1004                                                  ntohl(th->seq));
1005                 } else if (unlikely(!th->syn)) {
1006                         /* no way to figure out initial sisn/disn - drop */
1007                         return SKB_DROP_REASON_TCP_FLAGS;
1008                 }
1009         } else if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
1010                 disn = info->lisn;
1011                 if (th->syn || th->rst)
1012                         sisn = th->seq;
1013                 else
1014                         sisn = info->risn;
1015         } else {
1016                 WARN_ONCE(1, "TCP-AO: Unexpected sk_state %d", sk->sk_state);
1017                 return SKB_DROP_REASON_TCP_AOFAILURE;
1018         }
1019 verify_hash:
1020         traffic_key = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
1021         if (!traffic_key)
1022                 return SKB_DROP_REASON_NOT_SPECIFIED;
1023         tcp_ao_calc_key_skb(key, traffic_key, skb, sisn, disn, family);
1024         ret = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
1025                                  traffic_key, phash, sne, l3index);
1026         kfree(traffic_key);
1027         return ret;
1028
1029 key_not_found:
1030         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
1031         atomic64_inc(&info->counters.key_not_found);
1032         tcp_hash_fail("Requested by the peer AO key id not found",
1033                       family, skb, "L3index: %d", l3index);
1034         return SKB_DROP_REASON_TCP_AOKEYNOTFOUND;
1035 }
1036
1037 static int tcp_ao_cache_traffic_keys(const struct sock *sk,
1038                                      struct tcp_ao_info *ao,
1039                                      struct tcp_ao_key *ao_key)
1040 {
1041         u8 *traffic_key = snd_other_key(ao_key);
1042         int ret;
1043
1044         ret = tcp_ao_calc_key_sk(ao_key, traffic_key, sk,
1045                                  ao->lisn, ao->risn, true);
1046         if (ret)
1047                 return ret;
1048
1049         traffic_key = rcv_other_key(ao_key);
1050         ret = tcp_ao_calc_key_sk(ao_key, traffic_key, sk,
1051                                  ao->lisn, ao->risn, false);
1052         return ret;
1053 }
1054
1055 void tcp_ao_connect_init(struct sock *sk)
1056 {
1057         struct tcp_sock *tp = tcp_sk(sk);
1058         struct tcp_ao_info *ao_info;
1059         union tcp_ao_addr *addr;
1060         struct tcp_ao_key *key;
1061         int family, l3index;
1062
1063         ao_info = rcu_dereference_protected(tp->ao_info,
1064                                             lockdep_sock_is_held(sk));
1065         if (!ao_info)
1066                 return;
1067
1068         /* Remove all keys that don't match the peer */
1069         family = sk->sk_family;
1070         if (family == AF_INET)
1071                 addr = (union tcp_ao_addr *)&sk->sk_daddr;
1072 #if IS_ENABLED(CONFIG_IPV6)
1073         else if (family == AF_INET6)
1074                 addr = (union tcp_ao_addr *)&sk->sk_v6_daddr;
1075 #endif
1076         else
1077                 return;
1078         l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1079                                                  sk->sk_bound_dev_if);
1080
1081         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
1082                 if (!tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1))
1083                         continue;
1084
1085                 if (key == ao_info->current_key)
1086                         ao_info->current_key = NULL;
1087                 if (key == ao_info->rnext_key)
1088                         ao_info->rnext_key = NULL;
1089                 hlist_del_rcu(&key->node);
1090                 atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1091                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1092         }
1093
1094         key = tp->af_specific->ao_lookup(sk, sk, -1, -1);
1095         if (key) {
1096                 /* if current_key or rnext_key were not provided,
1097                  * use the first key matching the peer
1098                  */
1099                 if (!ao_info->current_key)
1100                         ao_info->current_key = key;
1101                 if (!ao_info->rnext_key)
1102                         ao_info->rnext_key = key;
1103                 tp->tcp_header_len += tcp_ao_len_aligned(key);
1104
1105                 ao_info->lisn = htonl(tp->write_seq);
1106                 ao_info->snd_sne = 0;
1107         } else {
1108                 /* Can't happen: tcp_connect() verifies that there's
1109                  * at least one tcp-ao key that matches the remote peer.
1110                  */
1111                 WARN_ON_ONCE(1);
1112                 rcu_assign_pointer(tp->ao_info, NULL);
1113                 kfree(ao_info);
1114         }
1115 }
1116
1117 void tcp_ao_established(struct sock *sk)
1118 {
1119         struct tcp_ao_info *ao;
1120         struct tcp_ao_key *key;
1121
1122         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1123                                        lockdep_sock_is_held(sk));
1124         if (!ao)
1125                 return;
1126
1127         hlist_for_each_entry_rcu(key, &ao->head, node)
1128                 tcp_ao_cache_traffic_keys(sk, ao, key);
1129 }
1130
1131 void tcp_ao_finish_connect(struct sock *sk, struct sk_buff *skb)
1132 {
1133         struct tcp_ao_info *ao;
1134         struct tcp_ao_key *key;
1135
1136         ao = rcu_dereference_protected(tcp_sk(sk)->ao_info,
1137                                        lockdep_sock_is_held(sk));
1138         if (!ao)
1139                 return;
1140
1141         WRITE_ONCE(ao->risn, tcp_hdr(skb)->seq);
1142         ao->rcv_sne = 0;
1143
1144         hlist_for_each_entry_rcu(key, &ao->head, node)
1145                 tcp_ao_cache_traffic_keys(sk, ao, key);
1146 }
1147
1148 int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk,
1149                              struct request_sock *req, struct sk_buff *skb,
1150                              int family)
1151 {
1152         struct tcp_ao_key *key, *new_key, *first_key;
1153         struct tcp_ao_info *new_ao, *ao;
1154         struct hlist_node *key_head;
1155         int l3index, ret = -ENOMEM;
1156         union tcp_ao_addr *addr;
1157         bool match = false;
1158
1159         ao = rcu_dereference(tcp_sk(sk)->ao_info);
1160         if (!ao)
1161                 return 0;
1162
1163         /* New socket without TCP-AO on it */
1164         if (!tcp_rsk_used_ao(req))
1165                 return 0;
1166
1167         new_ao = tcp_ao_alloc_info(GFP_ATOMIC);
1168         if (!new_ao)
1169                 return -ENOMEM;
1170         new_ao->lisn = htonl(tcp_rsk(req)->snt_isn);
1171         new_ao->risn = htonl(tcp_rsk(req)->rcv_isn);
1172         new_ao->ao_required = ao->ao_required;
1173         new_ao->accept_icmps = ao->accept_icmps;
1174
1175         if (family == AF_INET) {
1176                 addr = (union tcp_ao_addr *)&newsk->sk_daddr;
1177 #if IS_ENABLED(CONFIG_IPV6)
1178         } else if (family == AF_INET6) {
1179                 addr = (union tcp_ao_addr *)&newsk->sk_v6_daddr;
1180 #endif
1181         } else {
1182                 ret = -EAFNOSUPPORT;
1183                 goto free_ao;
1184         }
1185         l3index = l3mdev_master_ifindex_by_index(sock_net(newsk),
1186                                                  newsk->sk_bound_dev_if);
1187
1188         hlist_for_each_entry_rcu(key, &ao->head, node) {
1189                 if (tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1))
1190                         continue;
1191
1192                 new_key = tcp_ao_copy_key(newsk, key);
1193                 if (!new_key)
1194                         goto free_and_exit;
1195
1196                 tcp_ao_cache_traffic_keys(newsk, new_ao, new_key);
1197                 tcp_ao_link_mkt(new_ao, new_key);
1198                 match = true;
1199         }
1200
1201         if (!match) {
1202                 /* RFC5925 (7.4.1) specifies that the TCP-AO status
1203                  * of a connection is determined on the initial SYN.
1204                  * At this point the connection was TCP-AO enabled, so
1205                  * it can't switch to being unsigned if peer's key
1206                  * disappears on the listening socket.
1207                  */
1208                 ret = -EKEYREJECTED;
1209                 goto free_and_exit;
1210         }
1211
1212         if (!static_key_fast_inc_not_disabled(&tcp_ao_needed.key.key)) {
1213                 ret = -EUSERS;
1214                 goto free_and_exit;
1215         }
1216
1217         key_head = rcu_dereference(hlist_first_rcu(&new_ao->head));
1218         first_key = hlist_entry_safe(key_head, struct tcp_ao_key, node);
1219
1220         key = tcp_ao_established_key(new_ao, tcp_rsk(req)->ao_keyid, -1);
1221         if (key)
1222                 new_ao->current_key = key;
1223         else
1224                 new_ao->current_key = first_key;
1225
1226         /* set rnext_key */
1227         key = tcp_ao_established_key(new_ao, -1, tcp_rsk(req)->ao_rcv_next);
1228         if (key)
1229                 new_ao->rnext_key = key;
1230         else
1231                 new_ao->rnext_key = first_key;
1232
1233         sk_gso_disable(newsk);
1234         rcu_assign_pointer(tcp_sk(newsk)->ao_info, new_ao);
1235
1236         return 0;
1237
1238 free_and_exit:
1239         hlist_for_each_entry_safe(key, key_head, &new_ao->head, node) {
1240                 hlist_del(&key->node);
1241                 tcp_sigpool_release(key->tcp_sigpool_id);
1242                 atomic_sub(tcp_ao_sizeof_key(key), &newsk->sk_omem_alloc);
1243                 kfree_sensitive(key);
1244         }
1245 free_ao:
1246         kfree(new_ao);
1247         return ret;
1248 }
1249
1250 static bool tcp_ao_can_set_current_rnext(struct sock *sk)
1251 {
1252         /* There aren't current/rnext keys on TCP_LISTEN sockets */
1253         if (sk->sk_state == TCP_LISTEN)
1254                 return false;
1255         return true;
1256 }
1257
1258 static int tcp_ao_verify_ipv4(struct sock *sk, struct tcp_ao_add *cmd,
1259                               union tcp_ao_addr **addr)
1260 {
1261         struct sockaddr_in *sin = (struct sockaddr_in *)&cmd->addr;
1262         struct inet_sock *inet = inet_sk(sk);
1263
1264         if (sin->sin_family != AF_INET)
1265                 return -EINVAL;
1266
1267         /* Currently matching is not performed on port (or port ranges) */
1268         if (sin->sin_port != 0)
1269                 return -EINVAL;
1270
1271         /* Check prefix and trailing 0's in addr */
1272         if (cmd->prefix != 0) {
1273                 __be32 mask;
1274
1275                 if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY)
1276                         return -EINVAL;
1277                 if (cmd->prefix > 32)
1278                         return -EINVAL;
1279
1280                 mask = inet_make_mask(cmd->prefix);
1281                 if (sin->sin_addr.s_addr & ~mask)
1282                         return -EINVAL;
1283
1284                 /* Check that MKT address is consistent with socket */
1285                 if (ntohl(inet->inet_daddr) != INADDR_ANY &&
1286                     (inet->inet_daddr & mask) != sin->sin_addr.s_addr)
1287                         return -EINVAL;
1288         } else {
1289                 if (ntohl(sin->sin_addr.s_addr) != INADDR_ANY)
1290                         return -EINVAL;
1291         }
1292
1293         *addr = (union tcp_ao_addr *)&sin->sin_addr;
1294         return 0;
1295 }
1296
1297 static int tcp_ao_parse_crypto(struct tcp_ao_add *cmd, struct tcp_ao_key *key)
1298 {
1299         unsigned int syn_tcp_option_space;
1300         bool is_kdf_aes_128_cmac = false;
1301         struct crypto_ahash *tfm;
1302         struct tcp_sigpool hp;
1303         void *tmp_key = NULL;
1304         int err;
1305
1306         /* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1307         if (!strcmp("cmac(aes128)", cmd->alg_name)) {
1308                 strscpy(cmd->alg_name, "cmac(aes)", sizeof(cmd->alg_name));
1309                 is_kdf_aes_128_cmac = (cmd->keylen != 16);
1310                 tmp_key = kmalloc(cmd->keylen, GFP_KERNEL);
1311                 if (!tmp_key)
1312                         return -ENOMEM;
1313         }
1314
1315         key->maclen = cmd->maclen ?: 12; /* 12 is the default in RFC5925 */
1316
1317         /* Check: maclen + tcp-ao header <= (MAX_TCP_OPTION_SPACE - mss
1318          *                                      - tstamp (including sackperm)
1319          *                                      - wscale),
1320          * see tcp_syn_options(), tcp_synack_options(), commit 33ad798c924b.
1321          *
1322          * In order to allow D-SACK with TCP-AO, the header size should be:
1323          * (MAX_TCP_OPTION_SPACE - TCPOLEN_TSTAMP_ALIGNED
1324          *                      - TCPOLEN_SACK_BASE_ALIGNED
1325          *                      - 2 * TCPOLEN_SACK_PERBLOCK) = 8 (maclen = 4),
1326          * see tcp_established_options().
1327          *
1328          * RFC5925, 2.2:
1329          * Typical MACs are 96-128 bits (12-16 bytes), but any length
1330          * that fits in the header of the segment being authenticated
1331          * is allowed.
1332          *
1333          * RFC5925, 7.6:
1334          * TCP-AO continues to consume 16 bytes in non-SYN segments,
1335          * leaving a total of 24 bytes for other options, of which
1336          * the timestamp consumes 10.  This leaves 14 bytes, of which 10
1337          * are used for a single SACK block. When two SACK blocks are used,
1338          * such as to handle D-SACK, a smaller TCP-AO MAC would be required
1339          * to make room for the additional SACK block (i.e., to leave 18
1340          * bytes for the D-SACK variant of the SACK option) [RFC2883].
1341          * Note that D-SACK is not supportable in TCP MD5 in the presence
1342          * of timestamps, because TCP MD5’s MAC length is fixed and too
1343          * large to leave sufficient option space.
1344          */
1345         syn_tcp_option_space = MAX_TCP_OPTION_SPACE;
1346         syn_tcp_option_space -= TCPOLEN_MSS_ALIGNED;
1347         syn_tcp_option_space -= TCPOLEN_TSTAMP_ALIGNED;
1348         syn_tcp_option_space -= TCPOLEN_WSCALE_ALIGNED;
1349         if (tcp_ao_len_aligned(key) > syn_tcp_option_space) {
1350                 err = -EMSGSIZE;
1351                 goto err_kfree;
1352         }
1353
1354         key->keylen = cmd->keylen;
1355         memcpy(key->key, cmd->key, cmd->keylen);
1356
1357         err = tcp_sigpool_start(key->tcp_sigpool_id, &hp);
1358         if (err)
1359                 goto err_kfree;
1360
1361         tfm = crypto_ahash_reqtfm(hp.req);
1362         if (is_kdf_aes_128_cmac) {
1363                 void *scratch = hp.scratch;
1364                 struct scatterlist sg;
1365
1366                 memcpy(tmp_key, cmd->key, cmd->keylen);
1367                 sg_init_one(&sg, tmp_key, cmd->keylen);
1368
1369                 /* Using zero-key of 16 bytes as described in RFC5926 */
1370                 memset(scratch, 0, 16);
1371                 err = crypto_ahash_setkey(tfm, scratch, 16);
1372                 if (err)
1373                         goto err_pool_end;
1374
1375                 err = crypto_ahash_init(hp.req);
1376                 if (err)
1377                         goto err_pool_end;
1378
1379                 ahash_request_set_crypt(hp.req, &sg, key->key, cmd->keylen);
1380                 err = crypto_ahash_update(hp.req);
1381                 if (err)
1382                         goto err_pool_end;
1383
1384                 err |= crypto_ahash_final(hp.req);
1385                 if (err)
1386                         goto err_pool_end;
1387                 key->keylen = 16;
1388         }
1389
1390         err = crypto_ahash_setkey(tfm, key->key, key->keylen);
1391         if (err)
1392                 goto err_pool_end;
1393
1394         tcp_sigpool_end(&hp);
1395         kfree_sensitive(tmp_key);
1396
1397         if (tcp_ao_maclen(key) > key->digest_size)
1398                 return -EINVAL;
1399
1400         return 0;
1401
1402 err_pool_end:
1403         tcp_sigpool_end(&hp);
1404 err_kfree:
1405         kfree_sensitive(tmp_key);
1406         return err;
1407 }
1408
1409 #if IS_ENABLED(CONFIG_IPV6)
1410 static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1411                               union tcp_ao_addr **paddr,
1412                               unsigned short int *family)
1413 {
1414         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd->addr;
1415         struct in6_addr *addr = &sin6->sin6_addr;
1416         u8 prefix = cmd->prefix;
1417
1418         if (sin6->sin6_family != AF_INET6)
1419                 return -EINVAL;
1420
1421         /* Currently matching is not performed on port (or port ranges) */
1422         if (sin6->sin6_port != 0)
1423                 return -EINVAL;
1424
1425         /* Check prefix and trailing 0's in addr */
1426         if (cmd->prefix != 0 && ipv6_addr_v4mapped(addr)) {
1427                 __be32 addr4 = addr->s6_addr32[3];
1428                 __be32 mask;
1429
1430                 if (prefix > 32 || ntohl(addr4) == INADDR_ANY)
1431                         return -EINVAL;
1432
1433                 mask = inet_make_mask(prefix);
1434                 if (addr4 & ~mask)
1435                         return -EINVAL;
1436
1437                 /* Check that MKT address is consistent with socket */
1438                 if (!ipv6_addr_any(&sk->sk_v6_daddr)) {
1439                         __be32 daddr4 = sk->sk_v6_daddr.s6_addr32[3];
1440
1441                         if (!ipv6_addr_v4mapped(&sk->sk_v6_daddr))
1442                                 return -EINVAL;
1443                         if ((daddr4 & mask) != addr4)
1444                                 return -EINVAL;
1445                 }
1446
1447                 *paddr = (union tcp_ao_addr *)&addr->s6_addr32[3];
1448                 *family = AF_INET;
1449                 return 0;
1450         } else if (cmd->prefix != 0) {
1451                 struct in6_addr pfx;
1452
1453                 if (ipv6_addr_any(addr) || prefix > 128)
1454                         return -EINVAL;
1455
1456                 ipv6_addr_prefix(&pfx, addr, prefix);
1457                 if (ipv6_addr_cmp(&pfx, addr))
1458                         return -EINVAL;
1459
1460                 /* Check that MKT address is consistent with socket */
1461                 if (!ipv6_addr_any(&sk->sk_v6_daddr) &&
1462                     !ipv6_prefix_equal(&sk->sk_v6_daddr, addr, prefix))
1463
1464                         return -EINVAL;
1465         } else {
1466                 if (!ipv6_addr_any(addr))
1467                         return -EINVAL;
1468         }
1469
1470         *paddr = (union tcp_ao_addr *)addr;
1471         return 0;
1472 }
1473 #else
1474 static int tcp_ao_verify_ipv6(struct sock *sk, struct tcp_ao_add *cmd,
1475                               union tcp_ao_addr **paddr,
1476                               unsigned short int *family)
1477 {
1478         return -EOPNOTSUPP;
1479 }
1480 #endif
1481
1482 static struct tcp_ao_info *setsockopt_ao_info(struct sock *sk)
1483 {
1484         if (sk_fullsock(sk)) {
1485                 return rcu_dereference_protected(tcp_sk(sk)->ao_info,
1486                                                  lockdep_sock_is_held(sk));
1487         } else if (sk->sk_state == TCP_TIME_WAIT) {
1488                 return rcu_dereference_protected(tcp_twsk(sk)->ao_info,
1489                                                  lockdep_sock_is_held(sk));
1490         }
1491         return ERR_PTR(-ESOCKTNOSUPPORT);
1492 }
1493
1494 static struct tcp_ao_info *getsockopt_ao_info(struct sock *sk)
1495 {
1496         if (sk_fullsock(sk))
1497                 return rcu_dereference(tcp_sk(sk)->ao_info);
1498         else if (sk->sk_state == TCP_TIME_WAIT)
1499                 return rcu_dereference(tcp_twsk(sk)->ao_info);
1500
1501         return ERR_PTR(-ESOCKTNOSUPPORT);
1502 }
1503
1504 #define TCP_AO_KEYF_ALL (TCP_AO_KEYF_IFINDEX | TCP_AO_KEYF_EXCLUDE_OPT)
1505 #define TCP_AO_GET_KEYF_VALID   (TCP_AO_KEYF_IFINDEX)
1506
1507 static struct tcp_ao_key *tcp_ao_key_alloc(struct sock *sk,
1508                                            struct tcp_ao_add *cmd)
1509 {
1510         const char *algo = cmd->alg_name;
1511         unsigned int digest_size;
1512         struct crypto_ahash *tfm;
1513         struct tcp_ao_key *key;
1514         struct tcp_sigpool hp;
1515         int err, pool_id;
1516         size_t size;
1517
1518         /* Force null-termination of alg_name */
1519         cmd->alg_name[ARRAY_SIZE(cmd->alg_name) - 1] = '\0';
1520
1521         /* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
1522         if (!strcmp("cmac(aes128)", algo))
1523                 algo = "cmac(aes)";
1524
1525         /* Full TCP header (th->doff << 2) should fit into scratch area,
1526          * see tcp_ao_hash_header().
1527          */
1528         pool_id = tcp_sigpool_alloc_ahash(algo, 60);
1529         if (pool_id < 0)
1530                 return ERR_PTR(pool_id);
1531
1532         err = tcp_sigpool_start(pool_id, &hp);
1533         if (err)
1534                 goto err_free_pool;
1535
1536         tfm = crypto_ahash_reqtfm(hp.req);
1537         digest_size = crypto_ahash_digestsize(tfm);
1538         tcp_sigpool_end(&hp);
1539
1540         size = sizeof(struct tcp_ao_key) + (digest_size << 1);
1541         key = sock_kmalloc(sk, size, GFP_KERNEL);
1542         if (!key) {
1543                 err = -ENOMEM;
1544                 goto err_free_pool;
1545         }
1546
1547         key->tcp_sigpool_id = pool_id;
1548         key->digest_size = digest_size;
1549         return key;
1550
1551 err_free_pool:
1552         tcp_sigpool_release(pool_id);
1553         return ERR_PTR(err);
1554 }
1555
1556 static int tcp_ao_add_cmd(struct sock *sk, unsigned short int family,
1557                           sockptr_t optval, int optlen)
1558 {
1559         struct tcp_ao_info *ao_info;
1560         union tcp_ao_addr *addr;
1561         struct tcp_ao_key *key;
1562         struct tcp_ao_add cmd;
1563         int ret, l3index = 0;
1564         bool first = false;
1565
1566         if (optlen < sizeof(cmd))
1567                 return -EINVAL;
1568
1569         ret = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1570         if (ret)
1571                 return ret;
1572
1573         if (cmd.keylen > TCP_AO_MAXKEYLEN)
1574                 return -EINVAL;
1575
1576         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1577                 return -EINVAL;
1578
1579         if (family == AF_INET)
1580                 ret = tcp_ao_verify_ipv4(sk, &cmd, &addr);
1581         else
1582                 ret = tcp_ao_verify_ipv6(sk, &cmd, &addr, &family);
1583         if (ret)
1584                 return ret;
1585
1586         if (cmd.keyflags & ~TCP_AO_KEYF_ALL)
1587                 return -EINVAL;
1588
1589         if (cmd.set_current || cmd.set_rnext) {
1590                 if (!tcp_ao_can_set_current_rnext(sk))
1591                         return -EINVAL;
1592         }
1593
1594         if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1595                 return -EINVAL;
1596
1597         /* For cmd.tcp_ifindex = 0 the key will apply to the default VRF */
1598         if (cmd.keyflags & TCP_AO_KEYF_IFINDEX && cmd.ifindex) {
1599                 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
1600                 struct net_device *dev;
1601
1602                 rcu_read_lock();
1603                 dev = dev_get_by_index_rcu(sock_net(sk), cmd.ifindex);
1604                 if (dev && netif_is_l3_master(dev))
1605                         l3index = dev->ifindex;
1606                 rcu_read_unlock();
1607
1608                 if (!dev || !l3index)
1609                         return -EINVAL;
1610
1611                 if (!bound_dev_if || bound_dev_if != cmd.ifindex) {
1612                         /* tcp_ao_established_key() doesn't expect having
1613                          * non peer-matching key on an established TCP-AO
1614                          * connection.
1615                          */
1616                         if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1617                                 return -EINVAL;
1618                 }
1619
1620                 /* It's still possible to bind after adding keys or even
1621                  * re-bind to a different dev (with CAP_NET_RAW).
1622                  * So, no reason to return error here, rather try to be
1623                  * nice and warn the user.
1624                  */
1625                 if (bound_dev_if && bound_dev_if != cmd.ifindex)
1626                         net_warn_ratelimited("AO key ifindex %d != sk bound ifindex %d\n",
1627                                              cmd.ifindex, bound_dev_if);
1628         }
1629
1630         /* Don't allow keys for peers that have a matching TCP-MD5 key */
1631         if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) {
1632                 /* Non-_exact version of tcp_md5_do_lookup() will
1633                  * as well match keys that aren't bound to a specific VRF
1634                  * (that will make them match AO key with
1635                  * sysctl_tcp_l3dev_accept = 1
1636                  */
1637                 if (tcp_md5_do_lookup(sk, l3index, addr, family))
1638                         return -EKEYREJECTED;
1639         } else {
1640                 if (tcp_md5_do_lookup_any_l3index(sk, addr, family))
1641                         return -EKEYREJECTED;
1642         }
1643
1644         ao_info = setsockopt_ao_info(sk);
1645         if (IS_ERR(ao_info))
1646                 return PTR_ERR(ao_info);
1647
1648         if (!ao_info) {
1649                 ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1650                 if (!ao_info)
1651                         return -ENOMEM;
1652                 first = true;
1653         } else {
1654                 /* Check that neither RecvID nor SendID match any
1655                  * existing key for the peer, RFC5925 3.1:
1656                  * > The IDs of MKTs MUST NOT overlap where their
1657                  * > TCP connection identifiers overlap.
1658                  */
1659                 if (__tcp_ao_do_lookup(sk, l3index, addr, family, cmd.prefix, -1, cmd.rcvid))
1660                         return -EEXIST;
1661                 if (__tcp_ao_do_lookup(sk, l3index, addr, family,
1662                                        cmd.prefix, cmd.sndid, -1))
1663                         return -EEXIST;
1664         }
1665
1666         key = tcp_ao_key_alloc(sk, &cmd);
1667         if (IS_ERR(key)) {
1668                 ret = PTR_ERR(key);
1669                 goto err_free_ao;
1670         }
1671
1672         INIT_HLIST_NODE(&key->node);
1673         memcpy(&key->addr, addr, (family == AF_INET) ? sizeof(struct in_addr) :
1674                                                        sizeof(struct in6_addr));
1675         key->prefixlen  = cmd.prefix;
1676         key->family     = family;
1677         key->keyflags   = cmd.keyflags;
1678         key->sndid      = cmd.sndid;
1679         key->rcvid      = cmd.rcvid;
1680         key->l3index    = l3index;
1681         atomic64_set(&key->pkt_good, 0);
1682         atomic64_set(&key->pkt_bad, 0);
1683
1684         ret = tcp_ao_parse_crypto(&cmd, key);
1685         if (ret < 0)
1686                 goto err_free_sock;
1687
1688         if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))) {
1689                 tcp_ao_cache_traffic_keys(sk, ao_info, key);
1690                 if (first) {
1691                         ao_info->current_key = key;
1692                         ao_info->rnext_key = key;
1693                 }
1694         }
1695
1696         tcp_ao_link_mkt(ao_info, key);
1697         if (first) {
1698                 if (!static_branch_inc(&tcp_ao_needed.key)) {
1699                         ret = -EUSERS;
1700                         goto err_free_sock;
1701                 }
1702                 sk_gso_disable(sk);
1703                 rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
1704         }
1705
1706         if (cmd.set_current)
1707                 WRITE_ONCE(ao_info->current_key, key);
1708         if (cmd.set_rnext)
1709                 WRITE_ONCE(ao_info->rnext_key, key);
1710         return 0;
1711
1712 err_free_sock:
1713         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1714         tcp_sigpool_release(key->tcp_sigpool_id);
1715         kfree_sensitive(key);
1716 err_free_ao:
1717         if (first)
1718                 kfree(ao_info);
1719         return ret;
1720 }
1721
1722 static int tcp_ao_delete_key(struct sock *sk, struct tcp_ao_info *ao_info,
1723                              bool del_async, struct tcp_ao_key *key,
1724                              struct tcp_ao_key *new_current,
1725                              struct tcp_ao_key *new_rnext)
1726 {
1727         int err;
1728
1729         hlist_del_rcu(&key->node);
1730
1731         /* Support for async delete on listening sockets: as they don't
1732          * need current_key/rnext_key maintaining, we don't need to check
1733          * them and we can just free all resources in RCU fashion.
1734          */
1735         if (del_async) {
1736                 atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1737                 call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1738                 return 0;
1739         }
1740
1741         /* At this moment another CPU could have looked this key up
1742          * while it was unlinked from the list. Wait for RCU grace period,
1743          * after which the key is off-list and can't be looked up again;
1744          * the rx path [just before RCU came] might have used it and set it
1745          * as current_key (very unlikely).
1746          * Free the key with next RCU grace period (in case it was
1747          * current_key before tcp_ao_current_rnext() might have
1748          * changed it in forced-delete).
1749          */
1750         synchronize_rcu();
1751         if (new_current)
1752                 WRITE_ONCE(ao_info->current_key, new_current);
1753         if (new_rnext)
1754                 WRITE_ONCE(ao_info->rnext_key, new_rnext);
1755
1756         if (unlikely(READ_ONCE(ao_info->current_key) == key ||
1757                      READ_ONCE(ao_info->rnext_key) == key)) {
1758                 err = -EBUSY;
1759                 goto add_key;
1760         }
1761
1762         atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
1763         call_rcu(&key->rcu, tcp_ao_key_free_rcu);
1764
1765         return 0;
1766 add_key:
1767         hlist_add_head_rcu(&key->node, &ao_info->head);
1768         return err;
1769 }
1770
1771 #define TCP_AO_DEL_KEYF_ALL (TCP_AO_KEYF_IFINDEX)
1772 static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family,
1773                           sockptr_t optval, int optlen)
1774 {
1775         struct tcp_ao_key *key, *new_current = NULL, *new_rnext = NULL;
1776         int err, addr_len, l3index = 0;
1777         struct tcp_ao_info *ao_info;
1778         union tcp_ao_addr *addr;
1779         struct tcp_ao_del cmd;
1780         __u8 prefix;
1781         u16 port;
1782
1783         if (optlen < sizeof(cmd))
1784                 return -EINVAL;
1785
1786         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1787         if (err)
1788                 return err;
1789
1790         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1791                 return -EINVAL;
1792
1793         if (cmd.set_current || cmd.set_rnext) {
1794                 if (!tcp_ao_can_set_current_rnext(sk))
1795                         return -EINVAL;
1796         }
1797
1798         if (cmd.keyflags & ~TCP_AO_DEL_KEYF_ALL)
1799                 return -EINVAL;
1800
1801         /* No sanity check for TCP_AO_KEYF_IFINDEX as if a VRF
1802          * was destroyed, there still should be a way to delete keys,
1803          * that were bound to that l3intf. So, fail late at lookup stage
1804          * if there is no key for that ifindex.
1805          */
1806         if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX))
1807                 return -EINVAL;
1808
1809         ao_info = setsockopt_ao_info(sk);
1810         if (IS_ERR(ao_info))
1811                 return PTR_ERR(ao_info);
1812         if (!ao_info)
1813                 return -ENOENT;
1814
1815         /* For sockets in TCP_CLOSED it's possible set keys that aren't
1816          * matching the future peer (address/VRF/etc),
1817          * tcp_ao_connect_init() will choose a correct matching MKT
1818          * if there's any.
1819          */
1820         if (cmd.set_current) {
1821                 new_current = tcp_ao_established_key(ao_info, cmd.current_key, -1);
1822                 if (!new_current)
1823                         return -ENOENT;
1824         }
1825         if (cmd.set_rnext) {
1826                 new_rnext = tcp_ao_established_key(ao_info, -1, cmd.rnext);
1827                 if (!new_rnext)
1828                         return -ENOENT;
1829         }
1830         if (cmd.del_async && sk->sk_state != TCP_LISTEN)
1831                 return -EINVAL;
1832
1833         if (family == AF_INET) {
1834                 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.addr;
1835
1836                 addr = (union tcp_ao_addr *)&sin->sin_addr;
1837                 addr_len = sizeof(struct in_addr);
1838                 port = ntohs(sin->sin_port);
1839         } else {
1840                 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&cmd.addr;
1841                 struct in6_addr *addr6 = &sin6->sin6_addr;
1842
1843                 if (ipv6_addr_v4mapped(addr6)) {
1844                         addr = (union tcp_ao_addr *)&addr6->s6_addr32[3];
1845                         addr_len = sizeof(struct in_addr);
1846                         family = AF_INET;
1847                 } else {
1848                         addr = (union tcp_ao_addr *)addr6;
1849                         addr_len = sizeof(struct in6_addr);
1850                 }
1851                 port = ntohs(sin6->sin6_port);
1852         }
1853         prefix = cmd.prefix;
1854
1855         /* Currently matching is not performed on port (or port ranges) */
1856         if (port != 0)
1857                 return -EINVAL;
1858
1859         /* We could choose random present key here for current/rnext
1860          * but that's less predictable. Let's be strict and don't
1861          * allow removing a key that's in use. RFC5925 doesn't
1862          * specify how-to coordinate key removal, but says:
1863          * "It is presumed that an MKT affecting a particular
1864          * connection cannot be destroyed during an active connection"
1865          */
1866         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
1867                 if (cmd.sndid != key->sndid ||
1868                     cmd.rcvid != key->rcvid)
1869                         continue;
1870
1871                 if (family != key->family ||
1872                     prefix != key->prefixlen ||
1873                     memcmp(addr, &key->addr, addr_len))
1874                         continue;
1875
1876                 if ((cmd.keyflags & TCP_AO_KEYF_IFINDEX) !=
1877                     (key->keyflags & TCP_AO_KEYF_IFINDEX))
1878                         continue;
1879
1880                 if (key->l3index != l3index)
1881                         continue;
1882
1883                 if (key == new_current || key == new_rnext)
1884                         continue;
1885
1886                 return tcp_ao_delete_key(sk, ao_info, cmd.del_async, key,
1887                                          new_current, new_rnext);
1888         }
1889         return -ENOENT;
1890 }
1891
1892 /* cmd.ao_required makes a socket TCP-AO only.
1893  * Don't allow any md5 keys for any l3intf on the socket together with it.
1894  * Restricting it early in setsockopt() removes a check for
1895  * ao_info->ao_required on inbound tcp segment fast-path.
1896  */
1897 static int tcp_ao_required_verify(struct sock *sk)
1898 {
1899 #ifdef CONFIG_TCP_MD5SIG
1900         const struct tcp_md5sig_info *md5sig;
1901
1902         if (!static_branch_unlikely(&tcp_md5_needed.key))
1903                 return 0;
1904
1905         md5sig = rcu_dereference_check(tcp_sk(sk)->md5sig_info,
1906                                        lockdep_sock_is_held(sk));
1907         if (!md5sig)
1908                 return 0;
1909
1910         if (rcu_dereference_check(hlist_first_rcu(&md5sig->head),
1911                                   lockdep_sock_is_held(sk)))
1912                 return 1;
1913 #endif
1914         return 0;
1915 }
1916
1917 static int tcp_ao_info_cmd(struct sock *sk, unsigned short int family,
1918                            sockptr_t optval, int optlen)
1919 {
1920         struct tcp_ao_key *new_current = NULL, *new_rnext = NULL;
1921         struct tcp_ao_info *ao_info;
1922         struct tcp_ao_info_opt cmd;
1923         bool first = false;
1924         int err;
1925
1926         if (optlen < sizeof(cmd))
1927                 return -EINVAL;
1928
1929         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
1930         if (err)
1931                 return err;
1932
1933         if (cmd.set_current || cmd.set_rnext) {
1934                 if (!tcp_ao_can_set_current_rnext(sk))
1935                         return -EINVAL;
1936         }
1937
1938         if (cmd.reserved != 0 || cmd.reserved2 != 0)
1939                 return -EINVAL;
1940
1941         ao_info = setsockopt_ao_info(sk);
1942         if (IS_ERR(ao_info))
1943                 return PTR_ERR(ao_info);
1944         if (!ao_info) {
1945                 if (!((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)))
1946                         return -EINVAL;
1947                 ao_info = tcp_ao_alloc_info(GFP_KERNEL);
1948                 if (!ao_info)
1949                         return -ENOMEM;
1950                 first = true;
1951         }
1952
1953         if (cmd.ao_required && tcp_ao_required_verify(sk))
1954                 return -EKEYREJECTED;
1955
1956         /* For sockets in TCP_CLOSED it's possible set keys that aren't
1957          * matching the future peer (address/port/VRF/etc),
1958          * tcp_ao_connect_init() will choose a correct matching MKT
1959          * if there's any.
1960          */
1961         if (cmd.set_current) {
1962                 new_current = tcp_ao_established_key(ao_info, cmd.current_key, -1);
1963                 if (!new_current) {
1964                         err = -ENOENT;
1965                         goto out;
1966                 }
1967         }
1968         if (cmd.set_rnext) {
1969                 new_rnext = tcp_ao_established_key(ao_info, -1, cmd.rnext);
1970                 if (!new_rnext) {
1971                         err = -ENOENT;
1972                         goto out;
1973                 }
1974         }
1975         if (cmd.set_counters) {
1976                 atomic64_set(&ao_info->counters.pkt_good, cmd.pkt_good);
1977                 atomic64_set(&ao_info->counters.pkt_bad, cmd.pkt_bad);
1978                 atomic64_set(&ao_info->counters.key_not_found, cmd.pkt_key_not_found);
1979                 atomic64_set(&ao_info->counters.ao_required, cmd.pkt_ao_required);
1980                 atomic64_set(&ao_info->counters.dropped_icmp, cmd.pkt_dropped_icmp);
1981         }
1982
1983         ao_info->ao_required = cmd.ao_required;
1984         ao_info->accept_icmps = cmd.accept_icmps;
1985         if (new_current)
1986                 WRITE_ONCE(ao_info->current_key, new_current);
1987         if (new_rnext)
1988                 WRITE_ONCE(ao_info->rnext_key, new_rnext);
1989         if (first) {
1990                 if (!static_branch_inc(&tcp_ao_needed.key)) {
1991                         err = -EUSERS;
1992                         goto out;
1993                 }
1994                 sk_gso_disable(sk);
1995                 rcu_assign_pointer(tcp_sk(sk)->ao_info, ao_info);
1996         }
1997         return 0;
1998 out:
1999         if (first)
2000                 kfree(ao_info);
2001         return err;
2002 }
2003
2004 int tcp_parse_ao(struct sock *sk, int cmd, unsigned short int family,
2005                  sockptr_t optval, int optlen)
2006 {
2007         if (WARN_ON_ONCE(family != AF_INET && family != AF_INET6))
2008                 return -EAFNOSUPPORT;
2009
2010         switch (cmd) {
2011         case TCP_AO_ADD_KEY:
2012                 return tcp_ao_add_cmd(sk, family, optval, optlen);
2013         case TCP_AO_DEL_KEY:
2014                 return tcp_ao_del_cmd(sk, family, optval, optlen);
2015         case TCP_AO_INFO:
2016                 return tcp_ao_info_cmd(sk, family, optval, optlen);
2017         default:
2018                 WARN_ON_ONCE(1);
2019                 return -EINVAL;
2020         }
2021 }
2022
2023 int tcp_v4_parse_ao(struct sock *sk, int cmd, sockptr_t optval, int optlen)
2024 {
2025         return tcp_parse_ao(sk, cmd, AF_INET, optval, optlen);
2026 }
2027
2028 /* tcp_ao_copy_mkts_to_user(ao_info, optval, optlen)
2029  *
2030  * @ao_info:    struct tcp_ao_info on the socket that
2031  *              socket getsockopt(TCP_AO_GET_KEYS) is executed on
2032  * @optval:     pointer to array of tcp_ao_getsockopt structures in user space.
2033  *              Must be != NULL.
2034  * @optlen:     pointer to size of tcp_ao_getsockopt structure.
2035  *              Must be != NULL.
2036  *
2037  * Return value: 0 on success, a negative error number otherwise.
2038  *
2039  * optval points to an array of tcp_ao_getsockopt structures in user space.
2040  * optval[0] is used as both input and output to getsockopt. It determines
2041  * which keys are returned by the kernel.
2042  * optval[0].nkeys is the size of the array in user space. On return it contains
2043  * the number of keys matching the search criteria.
2044  * If tcp_ao_getsockopt::get_all is set, then all keys in the socket are
2045  * returned, otherwise only keys matching <addr, prefix, sndid, rcvid>
2046  * in optval[0] are returned.
2047  * optlen is also used as both input and output. The user provides the size
2048  * of struct tcp_ao_getsockopt in user space, and the kernel returns the size
2049  * of the structure in kernel space.
2050  * The size of struct tcp_ao_getsockopt may differ between user and kernel.
2051  * There are three cases to consider:
2052  *  * If usize == ksize, then keys are copied verbatim.
2053  *  * If usize < ksize, then the userspace has passed an old struct to a
2054  *    newer kernel. The rest of the trailing bytes in optval[0]
2055  *    (ksize - usize) are interpreted as 0 by the kernel.
2056  *  * If usize > ksize, then the userspace has passed a new struct to an
2057  *    older kernel. The trailing bytes unknown to the kernel (usize - ksize)
2058  *    are checked to ensure they are zeroed, otherwise -E2BIG is returned.
2059  * On return the kernel fills in min(usize, ksize) in each entry of the array.
2060  * The layout of the fields in the user and kernel structures is expected to
2061  * be the same (including in the 32bit vs 64bit case).
2062  */
2063 static int tcp_ao_copy_mkts_to_user(struct tcp_ao_info *ao_info,
2064                                     sockptr_t optval, sockptr_t optlen)
2065 {
2066         struct tcp_ao_getsockopt opt_in, opt_out;
2067         struct tcp_ao_key *key, *current_key;
2068         bool do_address_matching = true;
2069         union tcp_ao_addr *addr = NULL;
2070         int err, l3index, user_len;
2071         unsigned int max_keys;  /* maximum number of keys to copy to user */
2072         size_t out_offset = 0;
2073         size_t bytes_to_write;  /* number of bytes to write to user level */
2074         u32 matched_keys;       /* keys from ao_info matched so far */
2075         int optlen_out;
2076         __be16 port = 0;
2077
2078         if (copy_from_sockptr(&user_len, optlen, sizeof(int)))
2079                 return -EFAULT;
2080
2081         if (user_len <= 0)
2082                 return -EINVAL;
2083
2084         memset(&opt_in, 0, sizeof(struct tcp_ao_getsockopt));
2085         err = copy_struct_from_sockptr(&opt_in, sizeof(opt_in),
2086                                        optval, user_len);
2087         if (err < 0)
2088                 return err;
2089
2090         if (opt_in.pkt_good || opt_in.pkt_bad)
2091                 return -EINVAL;
2092         if (opt_in.keyflags & ~TCP_AO_GET_KEYF_VALID)
2093                 return -EINVAL;
2094         if (opt_in.ifindex && !(opt_in.keyflags & TCP_AO_KEYF_IFINDEX))
2095                 return -EINVAL;
2096
2097         if (opt_in.reserved != 0)
2098                 return -EINVAL;
2099
2100         max_keys = opt_in.nkeys;
2101         l3index = (opt_in.keyflags & TCP_AO_KEYF_IFINDEX) ? opt_in.ifindex : -1;
2102
2103         if (opt_in.get_all || opt_in.is_current || opt_in.is_rnext) {
2104                 if (opt_in.get_all && (opt_in.is_current || opt_in.is_rnext))
2105                         return -EINVAL;
2106                 do_address_matching = false;
2107         }
2108
2109         switch (opt_in.addr.ss_family) {
2110         case AF_INET: {
2111                 struct sockaddr_in *sin;
2112                 __be32 mask;
2113
2114                 sin = (struct sockaddr_in *)&opt_in.addr;
2115                 port = sin->sin_port;
2116                 addr = (union tcp_ao_addr *)&sin->sin_addr;
2117
2118                 if (opt_in.prefix > 32)
2119                         return -EINVAL;
2120
2121                 if (ntohl(sin->sin_addr.s_addr) == INADDR_ANY &&
2122                     opt_in.prefix != 0)
2123                         return -EINVAL;
2124
2125                 mask = inet_make_mask(opt_in.prefix);
2126                 if (sin->sin_addr.s_addr & ~mask)
2127                         return -EINVAL;
2128
2129                 break;
2130         }
2131         case AF_INET6: {
2132                 struct sockaddr_in6 *sin6;
2133                 struct in6_addr *addr6;
2134
2135                 sin6 = (struct sockaddr_in6 *)&opt_in.addr;
2136                 addr = (union tcp_ao_addr *)&sin6->sin6_addr;
2137                 addr6 = &sin6->sin6_addr;
2138                 port = sin6->sin6_port;
2139
2140                 /* We don't have to change family and @addr here if
2141                  * ipv6_addr_v4mapped() like in key adding:
2142                  * tcp_ao_key_cmp() does it. Do the sanity checks though.
2143                  */
2144                 if (opt_in.prefix != 0) {
2145                         if (ipv6_addr_v4mapped(addr6)) {
2146                                 __be32 mask, addr4 = addr6->s6_addr32[3];
2147
2148                                 if (opt_in.prefix > 32 ||
2149                                     ntohl(addr4) == INADDR_ANY)
2150                                         return -EINVAL;
2151                                 mask = inet_make_mask(opt_in.prefix);
2152                                 if (addr4 & ~mask)
2153                                         return -EINVAL;
2154                         } else {
2155                                 struct in6_addr pfx;
2156
2157                                 if (ipv6_addr_any(addr6) ||
2158                                     opt_in.prefix > 128)
2159                                         return -EINVAL;
2160
2161                                 ipv6_addr_prefix(&pfx, addr6, opt_in.prefix);
2162                                 if (ipv6_addr_cmp(&pfx, addr6))
2163                                         return -EINVAL;
2164                         }
2165                 } else if (!ipv6_addr_any(addr6)) {
2166                         return -EINVAL;
2167                 }
2168                 break;
2169         }
2170         case 0:
2171                 if (!do_address_matching)
2172                         break;
2173                 fallthrough;
2174         default:
2175                 return -EAFNOSUPPORT;
2176         }
2177
2178         if (!do_address_matching) {
2179                 /* We could just ignore those, but let's do stricter checks */
2180                 if (addr || port)
2181                         return -EINVAL;
2182                 if (opt_in.prefix || opt_in.sndid || opt_in.rcvid)
2183                         return -EINVAL;
2184         }
2185
2186         bytes_to_write = min_t(int, user_len, sizeof(struct tcp_ao_getsockopt));
2187         matched_keys = 0;
2188         /* May change in RX, while we're dumping, pre-fetch it */
2189         current_key = READ_ONCE(ao_info->current_key);
2190
2191         hlist_for_each_entry_rcu(key, &ao_info->head, node) {
2192                 if (opt_in.get_all)
2193                         goto match;
2194
2195                 if (opt_in.is_current || opt_in.is_rnext) {
2196                         if (opt_in.is_current && key == current_key)
2197                                 goto match;
2198                         if (opt_in.is_rnext && key == ao_info->rnext_key)
2199                                 goto match;
2200                         continue;
2201                 }
2202
2203                 if (tcp_ao_key_cmp(key, l3index, addr, opt_in.prefix,
2204                                    opt_in.addr.ss_family,
2205                                    opt_in.sndid, opt_in.rcvid) != 0)
2206                         continue;
2207 match:
2208                 matched_keys++;
2209                 if (matched_keys > max_keys)
2210                         continue;
2211
2212                 memset(&opt_out, 0, sizeof(struct tcp_ao_getsockopt));
2213
2214                 if (key->family == AF_INET) {
2215                         struct sockaddr_in *sin_out = (struct sockaddr_in *)&opt_out.addr;
2216
2217                         sin_out->sin_family = key->family;
2218                         sin_out->sin_port = 0;
2219                         memcpy(&sin_out->sin_addr, &key->addr, sizeof(struct in_addr));
2220                 } else {
2221                         struct sockaddr_in6 *sin6_out = (struct sockaddr_in6 *)&opt_out.addr;
2222
2223                         sin6_out->sin6_family = key->family;
2224                         sin6_out->sin6_port = 0;
2225                         memcpy(&sin6_out->sin6_addr, &key->addr, sizeof(struct in6_addr));
2226                 }
2227                 opt_out.sndid = key->sndid;
2228                 opt_out.rcvid = key->rcvid;
2229                 opt_out.prefix = key->prefixlen;
2230                 opt_out.keyflags = key->keyflags;
2231                 opt_out.is_current = (key == current_key);
2232                 opt_out.is_rnext = (key == ao_info->rnext_key);
2233                 opt_out.nkeys = 0;
2234                 opt_out.maclen = key->maclen;
2235                 opt_out.keylen = key->keylen;
2236                 opt_out.ifindex = key->l3index;
2237                 opt_out.pkt_good = atomic64_read(&key->pkt_good);
2238                 opt_out.pkt_bad = atomic64_read(&key->pkt_bad);
2239                 memcpy(&opt_out.key, key->key, key->keylen);
2240                 tcp_sigpool_algo(key->tcp_sigpool_id, opt_out.alg_name, 64);
2241
2242                 /* Copy key to user */
2243                 if (copy_to_sockptr_offset(optval, out_offset,
2244                                            &opt_out, bytes_to_write))
2245                         return -EFAULT;
2246                 out_offset += user_len;
2247         }
2248
2249         optlen_out = (int)sizeof(struct tcp_ao_getsockopt);
2250         if (copy_to_sockptr(optlen, &optlen_out, sizeof(int)))
2251                 return -EFAULT;
2252
2253         out_offset = offsetof(struct tcp_ao_getsockopt, nkeys);
2254         if (copy_to_sockptr_offset(optval, out_offset,
2255                                    &matched_keys, sizeof(u32)))
2256                 return -EFAULT;
2257
2258         return 0;
2259 }
2260
2261 int tcp_ao_get_mkts(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2262 {
2263         struct tcp_ao_info *ao_info;
2264
2265         ao_info = setsockopt_ao_info(sk);
2266         if (IS_ERR(ao_info))
2267                 return PTR_ERR(ao_info);
2268         if (!ao_info)
2269                 return -ENOENT;
2270
2271         return tcp_ao_copy_mkts_to_user(ao_info, optval, optlen);
2272 }
2273
2274 int tcp_ao_get_sock_info(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2275 {
2276         struct tcp_ao_info_opt out, in = {};
2277         struct tcp_ao_key *current_key;
2278         struct tcp_ao_info *ao;
2279         int err, len;
2280
2281         if (copy_from_sockptr(&len, optlen, sizeof(int)))
2282                 return -EFAULT;
2283
2284         if (len <= 0)
2285                 return -EINVAL;
2286
2287         /* Copying this "in" only to check ::reserved, ::reserved2,
2288          * that may be needed to extend (struct tcp_ao_info_opt) and
2289          * what getsockopt() provides in future.
2290          */
2291         err = copy_struct_from_sockptr(&in, sizeof(in), optval, len);
2292         if (err)
2293                 return err;
2294
2295         if (in.reserved != 0 || in.reserved2 != 0)
2296                 return -EINVAL;
2297
2298         ao = setsockopt_ao_info(sk);
2299         if (IS_ERR(ao))
2300                 return PTR_ERR(ao);
2301         if (!ao)
2302                 return -ENOENT;
2303
2304         memset(&out, 0, sizeof(out));
2305         out.ao_required         = ao->ao_required;
2306         out.accept_icmps        = ao->accept_icmps;
2307         out.pkt_good            = atomic64_read(&ao->counters.pkt_good);
2308         out.pkt_bad             = atomic64_read(&ao->counters.pkt_bad);
2309         out.pkt_key_not_found   = atomic64_read(&ao->counters.key_not_found);
2310         out.pkt_ao_required     = atomic64_read(&ao->counters.ao_required);
2311         out.pkt_dropped_icmp    = atomic64_read(&ao->counters.dropped_icmp);
2312
2313         current_key = READ_ONCE(ao->current_key);
2314         if (current_key) {
2315                 out.set_current = 1;
2316                 out.current_key = current_key->sndid;
2317         }
2318         if (ao->rnext_key) {
2319                 out.set_rnext = 1;
2320                 out.rnext = ao->rnext_key->rcvid;
2321         }
2322
2323         if (copy_to_sockptr(optval, &out, min_t(int, len, sizeof(out))))
2324                 return -EFAULT;
2325
2326         return 0;
2327 }
2328
2329 int tcp_ao_set_repair(struct sock *sk, sockptr_t optval, unsigned int optlen)
2330 {
2331         struct tcp_sock *tp = tcp_sk(sk);
2332         struct tcp_ao_repair cmd;
2333         struct tcp_ao_key *key;
2334         struct tcp_ao_info *ao;
2335         int err;
2336
2337         if (optlen < sizeof(cmd))
2338                 return -EINVAL;
2339
2340         err = copy_struct_from_sockptr(&cmd, sizeof(cmd), optval, optlen);
2341         if (err)
2342                 return err;
2343
2344         if (!tp->repair)
2345                 return -EPERM;
2346
2347         ao = setsockopt_ao_info(sk);
2348         if (IS_ERR(ao))
2349                 return PTR_ERR(ao);
2350         if (!ao)
2351                 return -ENOENT;
2352
2353         WRITE_ONCE(ao->lisn, cmd.snt_isn);
2354         WRITE_ONCE(ao->risn, cmd.rcv_isn);
2355         WRITE_ONCE(ao->snd_sne, cmd.snd_sne);
2356         WRITE_ONCE(ao->rcv_sne, cmd.rcv_sne);
2357
2358         hlist_for_each_entry_rcu(key, &ao->head, node)
2359                 tcp_ao_cache_traffic_keys(sk, ao, key);
2360
2361         return 0;
2362 }
2363
2364 int tcp_ao_get_repair(struct sock *sk, sockptr_t optval, sockptr_t optlen)
2365 {
2366         struct tcp_sock *tp = tcp_sk(sk);
2367         struct tcp_ao_repair opt;
2368         struct tcp_ao_info *ao;
2369         int len;
2370
2371         if (copy_from_sockptr(&len, optlen, sizeof(int)))
2372                 return -EFAULT;
2373
2374         if (len <= 0)
2375                 return -EINVAL;
2376
2377         if (!tp->repair)
2378                 return -EPERM;
2379
2380         rcu_read_lock();
2381         ao = getsockopt_ao_info(sk);
2382         if (IS_ERR_OR_NULL(ao)) {
2383                 rcu_read_unlock();
2384                 return ao ? PTR_ERR(ao) : -ENOENT;
2385         }
2386
2387         opt.snt_isn     = ao->lisn;
2388         opt.rcv_isn     = ao->risn;
2389         opt.snd_sne     = READ_ONCE(ao->snd_sne);
2390         opt.rcv_sne     = READ_ONCE(ao->rcv_sne);
2391         rcu_read_unlock();
2392
2393         if (copy_to_sockptr(optval, &opt, min_t(int, len, sizeof(opt))))
2394                 return -EFAULT;
2395         return 0;
2396 }