1
2
3
4
5
6
7#define pr_fmt(fmt) "MPTCP: " fmt
8
9#include <linux/kernel.h>
10#include <linux/module.h>
11#include <linux/netdevice.h>
12#include <linux/sched/signal.h>
13#include <linux/atomic.h>
14#include <net/sock.h>
15#include <net/inet_common.h>
16#include <net/inet_hashtables.h>
17#include <net/protocol.h>
18#include <net/tcp.h>
19#include <net/tcp_states.h>
20#if IS_ENABLED(CONFIG_MPTCP_IPV6)
21#include <net/transp_v6.h>
22#endif
23#include <net/mptcp.h>
24#include <net/xfrm.h>
25#include "protocol.h"
26#include "mib.h"
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/mptcp.h>
30
31#if IS_ENABLED(CONFIG_MPTCP_IPV6)
32struct mptcp6_sock {
33 struct mptcp_sock msk;
34 struct ipv6_pinfo np;
35};
36#endif
37
38struct mptcp_skb_cb {
39 u64 map_seq;
40 u64 end_seq;
41 u32 offset;
42 u8 has_rxtstamp:1;
43};
44
45#define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0]))
46
47enum {
48 MPTCP_CMSG_TS = BIT(0),
49};
50
51static struct percpu_counter mptcp_sockets_allocated;
52
53static void __mptcp_destroy_sock(struct sock *sk);
54static void __mptcp_check_send_data_fin(struct sock *sk);
55
56DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions);
57static struct net_device mptcp_napi_dev;
58
59
60
61
62
63struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
64{
65 if (!msk->subflow || READ_ONCE(msk->can_ack))
66 return NULL;
67
68 return msk->subflow;
69}
70
71
72static u64 mptcp_wnd_end(const struct mptcp_sock *msk)
73{
74 return READ_ONCE(msk->wnd_end);
75}
76
77static bool mptcp_is_tcpsk(struct sock *sk)
78{
79 struct socket *sock = sk->sk_socket;
80
81 if (unlikely(sk->sk_prot == &tcp_prot)) {
82
83
84
85
86
87
88
89 sock->ops = &inet_stream_ops;
90 return true;
91#if IS_ENABLED(CONFIG_MPTCP_IPV6)
92 } else if (unlikely(sk->sk_prot == &tcpv6_prot)) {
93 sock->ops = &inet6_stream_ops;
94 return true;
95#endif
96 }
97
98 return false;
99}
100
101static int __mptcp_socket_create(struct mptcp_sock *msk)
102{
103 struct mptcp_subflow_context *subflow;
104 struct sock *sk = (struct sock *)msk;
105 struct socket *ssock;
106 int err;
107
108 err = mptcp_subflow_create_socket(sk, &ssock);
109 if (err)
110 return err;
111
112 msk->first = ssock->sk;
113 msk->subflow = ssock;
114 subflow = mptcp_subflow_ctx(ssock->sk);
115 list_add(&subflow->node, &msk->conn_list);
116 sock_hold(ssock->sk);
117 subflow->request_mptcp = 1;
118 mptcp_sock_graft(msk->first, sk->sk_socket);
119
120 return 0;
121}
122
123static void mptcp_drop(struct sock *sk, struct sk_buff *skb)
124{
125 sk_drops_add(sk, skb);
126 __kfree_skb(skb);
127}
128
129static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to,
130 struct sk_buff *from)
131{
132 bool fragstolen;
133 int delta;
134
135 if (MPTCP_SKB_CB(from)->offset ||
136 !skb_try_coalesce(to, from, &fragstolen, &delta))
137 return false;
138
139 pr_debug("colesced seq %llx into %llx new len %d new end seq %llx",
140 MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq,
141 to->len, MPTCP_SKB_CB(from)->end_seq);
142 MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq;
143 kfree_skb_partial(from, fragstolen);
144 atomic_add(delta, &sk->sk_rmem_alloc);
145 sk_mem_charge(sk, delta);
146 return true;
147}
148
149static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to,
150 struct sk_buff *from)
151{
152 if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq)
153 return false;
154
155 return mptcp_try_coalesce((struct sock *)msk, to, from);
156}
157
158
159
160
161
162static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb)
163{
164 struct sock *sk = (struct sock *)msk;
165 struct rb_node **p, *parent;
166 u64 seq, end_seq, max_seq;
167 struct sk_buff *skb1;
168
169 seq = MPTCP_SKB_CB(skb)->map_seq;
170 end_seq = MPTCP_SKB_CB(skb)->end_seq;
171 max_seq = READ_ONCE(msk->rcv_wnd_sent);
172
173 pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq,
174 RB_EMPTY_ROOT(&msk->out_of_order_queue));
175 if (after64(end_seq, max_seq)) {
176
177 mptcp_drop(sk, skb);
178 pr_debug("oow by %lld, rcv_wnd_sent %llu\n",
179 (unsigned long long)end_seq - (unsigned long)max_seq,
180 (unsigned long long)msk->rcv_wnd_sent);
181 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
182 return;
183 }
184
185 p = &msk->out_of_order_queue.rb_node;
186 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE);
187 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) {
188 rb_link_node(&skb->rbnode, NULL, p);
189 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
190 msk->ooo_last_skb = skb;
191 goto end;
192 }
193
194
195
196
197 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) {
198 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
199 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
200 return;
201 }
202
203
204 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) {
205 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
206 parent = &msk->ooo_last_skb->rbnode;
207 p = &parent->rb_right;
208 goto insert;
209 }
210
211
212 parent = NULL;
213 while (*p) {
214 parent = *p;
215 skb1 = rb_to_skb(parent);
216 if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
217 p = &parent->rb_left;
218 continue;
219 }
220 if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) {
221 if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) {
222
223 mptcp_drop(sk, skb);
224 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
225 return;
226 }
227 if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
228
229
230
231
232
233 } else {
234
235
236
237 rb_replace_node(&skb1->rbnode, &skb->rbnode,
238 &msk->out_of_order_queue);
239 mptcp_drop(sk, skb1);
240 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
241 goto merge_right;
242 }
243 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) {
244 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
245 return;
246 }
247 p = &parent->rb_right;
248 }
249
250insert:
251
252 rb_link_node(&skb->rbnode, parent, p);
253 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
254
255merge_right:
256
257 while ((skb1 = skb_rb_next(skb)) != NULL) {
258 if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq))
259 break;
260 rb_erase(&skb1->rbnode, &msk->out_of_order_queue);
261 mptcp_drop(sk, skb1);
262 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
263 }
264
265 if (!skb1)
266 msk->ooo_last_skb = skb;
267
268end:
269 skb_condense(skb);
270 skb_set_owner_r(skb, sk);
271}
272
273static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,
274 struct sk_buff *skb, unsigned int offset,
275 size_t copy_len)
276{
277 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
278 struct sock *sk = (struct sock *)msk;
279 struct sk_buff *tail;
280 bool has_rxtstamp;
281
282 __skb_unlink(skb, &ssk->sk_receive_queue);
283
284 skb_ext_reset(skb);
285 skb_orphan(skb);
286
287
288 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
289 int amount = sk_mem_pages(skb->truesize) << SK_MEM_QUANTUM_SHIFT;
290
291 if (ssk->sk_forward_alloc < amount)
292 goto drop;
293
294 ssk->sk_forward_alloc -= amount;
295 sk->sk_forward_alloc += amount;
296 }
297
298 has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp;
299
300
301
302
303
304 MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);
305 MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len;
306 MPTCP_SKB_CB(skb)->offset = offset;
307 MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp;
308
309 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) {
310
311 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
312 tail = skb_peek_tail(&sk->sk_receive_queue);
313 if (tail && mptcp_try_coalesce(sk, tail, skb))
314 return true;
315
316 skb_set_owner_r(skb, sk);
317 __skb_queue_tail(&sk->sk_receive_queue, skb);
318 return true;
319 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) {
320 mptcp_data_queue_ofo(msk, skb);
321 return false;
322 }
323
324
325
326
327 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
328drop:
329 mptcp_drop(sk, skb);
330 return false;
331}
332
333static void mptcp_stop_timer(struct sock *sk)
334{
335 struct inet_connection_sock *icsk = inet_csk(sk);
336
337 sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
338 mptcp_sk(sk)->timer_ival = 0;
339}
340
341static void mptcp_close_wake_up(struct sock *sk)
342{
343 if (sock_flag(sk, SOCK_DEAD))
344 return;
345
346 sk->sk_state_change(sk);
347 if (sk->sk_shutdown == SHUTDOWN_MASK ||
348 sk->sk_state == TCP_CLOSE)
349 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
350 else
351 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
352}
353
354static bool mptcp_pending_data_fin_ack(struct sock *sk)
355{
356 struct mptcp_sock *msk = mptcp_sk(sk);
357
358 return !__mptcp_check_fallback(msk) &&
359 ((1 << sk->sk_state) &
360 (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) &&
361 msk->write_seq == READ_ONCE(msk->snd_una);
362}
363
364static void mptcp_check_data_fin_ack(struct sock *sk)
365{
366 struct mptcp_sock *msk = mptcp_sk(sk);
367
368
369 if (mptcp_pending_data_fin_ack(sk)) {
370 WRITE_ONCE(msk->snd_data_fin_enable, 0);
371
372 switch (sk->sk_state) {
373 case TCP_FIN_WAIT1:
374 inet_sk_state_store(sk, TCP_FIN_WAIT2);
375 break;
376 case TCP_CLOSING:
377 case TCP_LAST_ACK:
378 inet_sk_state_store(sk, TCP_CLOSE);
379 break;
380 }
381
382 mptcp_close_wake_up(sk);
383 }
384}
385
386static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq)
387{
388 struct mptcp_sock *msk = mptcp_sk(sk);
389
390 if (READ_ONCE(msk->rcv_data_fin) &&
391 ((1 << sk->sk_state) &
392 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) {
393 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq);
394
395 if (msk->ack_seq == rcv_data_fin_seq) {
396 if (seq)
397 *seq = rcv_data_fin_seq;
398
399 return true;
400 }
401 }
402
403 return false;
404}
405
406static void mptcp_set_datafin_timeout(const struct sock *sk)
407{
408 struct inet_connection_sock *icsk = inet_csk(sk);
409
410 mptcp_sk(sk)->timer_ival = min(TCP_RTO_MAX,
411 TCP_RTO_MIN << icsk->icsk_retransmits);
412}
413
414static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk)
415{
416 long tout = ssk && inet_csk(ssk)->icsk_pending ?
417 inet_csk(ssk)->icsk_timeout - jiffies : 0;
418
419 if (tout <= 0)
420 tout = mptcp_sk(sk)->timer_ival;
421 mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
422}
423
424static bool tcp_can_send_ack(const struct sock *ssk)
425{
426 return !((1 << inet_sk_state_load(ssk)) &
427 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN));
428}
429
430static void mptcp_send_ack(struct mptcp_sock *msk)
431{
432 struct mptcp_subflow_context *subflow;
433
434 mptcp_for_each_subflow(msk, subflow) {
435 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
436 bool slow;
437
438 slow = lock_sock_fast(ssk);
439 if (tcp_can_send_ack(ssk))
440 tcp_send_ack(ssk);
441 unlock_sock_fast(ssk, slow);
442 }
443}
444
445static void mptcp_subflow_cleanup_rbuf(struct sock *ssk)
446{
447 bool slow;
448
449 slow = lock_sock_fast(ssk);
450 if (tcp_can_send_ack(ssk))
451 tcp_cleanup_rbuf(ssk, 1);
452 unlock_sock_fast(ssk, slow);
453}
454
455static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty)
456{
457 const struct inet_connection_sock *icsk = inet_csk(ssk);
458 u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending);
459 const struct tcp_sock *tp = tcp_sk(ssk);
460
461 return (ack_pending & ICSK_ACK_SCHED) &&
462 ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) >
463 READ_ONCE(icsk->icsk_ack.rcv_mss)) ||
464 (rx_empty && ack_pending &
465 (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED)));
466}
467
468static void mptcp_cleanup_rbuf(struct mptcp_sock *msk)
469{
470 int old_space = READ_ONCE(msk->old_wspace);
471 struct mptcp_subflow_context *subflow;
472 struct sock *sk = (struct sock *)msk;
473 int space = __mptcp_space(sk);
474 bool cleanup, rx_empty;
475
476 cleanup = (space > 0) && (space >= (old_space << 1));
477 rx_empty = !__mptcp_rmem(sk);
478
479 mptcp_for_each_subflow(msk, subflow) {
480 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
481
482 if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty))
483 mptcp_subflow_cleanup_rbuf(ssk);
484 }
485}
486
487static bool mptcp_check_data_fin(struct sock *sk)
488{
489 struct mptcp_sock *msk = mptcp_sk(sk);
490 u64 rcv_data_fin_seq;
491 bool ret = false;
492
493 if (__mptcp_check_fallback(msk))
494 return ret;
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509 if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {
510 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1);
511 WRITE_ONCE(msk->rcv_data_fin, 0);
512
513 sk->sk_shutdown |= RCV_SHUTDOWN;
514 smp_mb__before_atomic();
515 set_bit(MPTCP_DATA_READY, &msk->flags);
516
517 switch (sk->sk_state) {
518 case TCP_ESTABLISHED:
519 inet_sk_state_store(sk, TCP_CLOSE_WAIT);
520 break;
521 case TCP_FIN_WAIT1:
522 inet_sk_state_store(sk, TCP_CLOSING);
523 break;
524 case TCP_FIN_WAIT2:
525 inet_sk_state_store(sk, TCP_CLOSE);
526 break;
527 default:
528
529 WARN_ON_ONCE(1);
530 break;
531 }
532
533 ret = true;
534 mptcp_set_timeout(sk, NULL);
535 mptcp_send_ack(msk);
536 mptcp_close_wake_up(sk);
537 }
538 return ret;
539}
540
541static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
542 struct sock *ssk,
543 unsigned int *bytes)
544{
545 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
546 struct sock *sk = (struct sock *)msk;
547 unsigned int moved = 0;
548 bool more_data_avail;
549 struct tcp_sock *tp;
550 bool done = false;
551 int sk_rbuf;
552
553 sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
554
555 if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
556 int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf);
557
558 if (unlikely(ssk_rbuf > sk_rbuf)) {
559 WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf);
560 sk_rbuf = ssk_rbuf;
561 }
562 }
563
564 pr_debug("msk=%p ssk=%p", msk, ssk);
565 tp = tcp_sk(ssk);
566 do {
567 u32 map_remaining, offset;
568 u32 seq = tp->copied_seq;
569 struct sk_buff *skb;
570 bool fin;
571
572
573 map_remaining = subflow->map_data_len -
574 mptcp_subflow_get_map_offset(subflow);
575
576 skb = skb_peek(&ssk->sk_receive_queue);
577 if (!skb) {
578
579
580
581
582 if (!moved)
583 done = true;
584 break;
585 }
586
587 if (__mptcp_check_fallback(msk)) {
588
589
590
591
592 map_remaining = skb->len;
593 subflow->map_data_len = skb->len;
594 }
595
596 offset = seq - TCP_SKB_CB(skb)->seq;
597 fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
598 if (fin) {
599 done = true;
600 seq++;
601 }
602
603 if (offset < skb->len) {
604 size_t len = skb->len - offset;
605
606 if (tp->urg_data)
607 done = true;
608
609 if (__mptcp_move_skb(msk, ssk, skb, offset, len))
610 moved += len;
611 seq += len;
612
613 if (WARN_ON_ONCE(map_remaining < len))
614 break;
615 } else {
616 WARN_ON_ONCE(!fin);
617 sk_eat_skb(ssk, skb);
618 done = true;
619 }
620
621 WRITE_ONCE(tp->copied_seq, seq);
622 more_data_avail = mptcp_subflow_data_available(ssk);
623
624 if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) {
625 done = true;
626 break;
627 }
628 } while (more_data_avail);
629
630 *bytes += moved;
631 return done;
632}
633
634static bool __mptcp_ofo_queue(struct mptcp_sock *msk)
635{
636 struct sock *sk = (struct sock *)msk;
637 struct sk_buff *skb, *tail;
638 bool moved = false;
639 struct rb_node *p;
640 u64 end_seq;
641
642 p = rb_first(&msk->out_of_order_queue);
643 pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue));
644 while (p) {
645 skb = rb_to_skb(p);
646 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq))
647 break;
648
649 p = rb_next(p);
650 rb_erase(&skb->rbnode, &msk->out_of_order_queue);
651
652 if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq,
653 msk->ack_seq))) {
654 mptcp_drop(sk, skb);
655 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
656 continue;
657 }
658
659 end_seq = MPTCP_SKB_CB(skb)->end_seq;
660 tail = skb_peek_tail(&sk->sk_receive_queue);
661 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) {
662 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq;
663
664
665 pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d",
666 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq,
667 delta);
668 MPTCP_SKB_CB(skb)->offset += delta;
669 __skb_queue_tail(&sk->sk_receive_queue, skb);
670 }
671 msk->ack_seq = end_seq;
672 moved = true;
673 }
674 return moved;
675}
676
677
678
679
680static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)
681{
682 struct sock *sk = (struct sock *)msk;
683 unsigned int moved = 0;
684
685 __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
686 __mptcp_ofo_queue(msk);
687 if (unlikely(ssk->sk_err)) {
688 if (!sock_owned_by_user(sk))
689 __mptcp_error_report(sk);
690 else
691 set_bit(MPTCP_ERROR_REPORT, &msk->flags);
692 }
693
694
695
696
697
698
699 if (mptcp_pending_data_fin(sk, NULL))
700 mptcp_schedule_work(sk);
701 return moved > 0;
702}
703
704void mptcp_data_ready(struct sock *sk, struct sock *ssk)
705{
706 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
707 struct mptcp_sock *msk = mptcp_sk(sk);
708 int sk_rbuf, ssk_rbuf;
709
710
711
712
713
714 if (unlikely(subflow->disposable))
715 return;
716
717 ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf);
718 sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
719 if (unlikely(ssk_rbuf > sk_rbuf))
720 sk_rbuf = ssk_rbuf;
721
722
723 if (__mptcp_rmem(sk) > sk_rbuf) {
724 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED);
725 return;
726 }
727
728
729 mptcp_data_lock(sk);
730 if (move_skbs_to_msk(msk, ssk)) {
731 set_bit(MPTCP_DATA_READY, &msk->flags);
732 sk->sk_data_ready(sk);
733 }
734 mptcp_data_unlock(sk);
735}
736
737static bool mptcp_do_flush_join_list(struct mptcp_sock *msk)
738{
739 struct mptcp_subflow_context *subflow;
740 bool ret = false;
741
742 if (likely(list_empty(&msk->join_list)))
743 return false;
744
745 spin_lock_bh(&msk->join_list_lock);
746 list_for_each_entry(subflow, &msk->join_list, node) {
747 u32 sseq = READ_ONCE(subflow->setsockopt_seq);
748
749 mptcp_propagate_sndbuf((struct sock *)msk, mptcp_subflow_tcp_sock(subflow));
750 if (READ_ONCE(msk->setsockopt_seq) != sseq)
751 ret = true;
752 }
753 list_splice_tail_init(&msk->join_list, &msk->conn_list);
754 spin_unlock_bh(&msk->join_list_lock);
755
756 return ret;
757}
758
759void __mptcp_flush_join_list(struct mptcp_sock *msk)
760{
761 if (likely(!mptcp_do_flush_join_list(msk)))
762 return;
763
764 if (!test_and_set_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags))
765 mptcp_schedule_work((struct sock *)msk);
766}
767
768static void mptcp_flush_join_list(struct mptcp_sock *msk)
769{
770 bool sync_needed = test_and_clear_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags);
771
772 might_sleep();
773
774 if (!mptcp_do_flush_join_list(msk) && !sync_needed)
775 return;
776
777 mptcp_sockopt_sync_all(msk);
778}
779
780static bool mptcp_timer_pending(struct sock *sk)
781{
782 return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);
783}
784
785static void mptcp_reset_timer(struct sock *sk)
786{
787 struct inet_connection_sock *icsk = inet_csk(sk);
788 unsigned long tout;
789
790
791 if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE))
792 return;
793
794
795 tout = READ_ONCE(mptcp_sk(sk)->timer_ival);
796 if (WARN_ON_ONCE(!tout))
797 tout = TCP_RTO_MIN;
798 sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);
799}
800
801bool mptcp_schedule_work(struct sock *sk)
802{
803 if (inet_sk_state_load(sk) != TCP_CLOSE &&
804 schedule_work(&mptcp_sk(sk)->work)) {
805
806
807
808 sock_hold(sk);
809 return true;
810 }
811 return false;
812}
813
814void mptcp_subflow_eof(struct sock *sk)
815{
816 if (!test_and_set_bit(MPTCP_WORK_EOF, &mptcp_sk(sk)->flags))
817 mptcp_schedule_work(sk);
818}
819
820static void mptcp_check_for_eof(struct mptcp_sock *msk)
821{
822 struct mptcp_subflow_context *subflow;
823 struct sock *sk = (struct sock *)msk;
824 int receivers = 0;
825
826 mptcp_for_each_subflow(msk, subflow)
827 receivers += !subflow->rx_eof;
828 if (receivers)
829 return;
830
831 if (!(sk->sk_shutdown & RCV_SHUTDOWN)) {
832
833
834
835 sk->sk_shutdown |= RCV_SHUTDOWN;
836
837 smp_mb__before_atomic();
838 set_bit(MPTCP_DATA_READY, &msk->flags);
839 sk->sk_data_ready(sk);
840 }
841
842 switch (sk->sk_state) {
843 case TCP_ESTABLISHED:
844 inet_sk_state_store(sk, TCP_CLOSE_WAIT);
845 break;
846 case TCP_FIN_WAIT1:
847 inet_sk_state_store(sk, TCP_CLOSING);
848 break;
849 case TCP_FIN_WAIT2:
850 inet_sk_state_store(sk, TCP_CLOSE);
851 break;
852 default:
853 return;
854 }
855 mptcp_close_wake_up(sk);
856}
857
858static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk)
859{
860 struct mptcp_subflow_context *subflow;
861 struct sock *sk = (struct sock *)msk;
862
863 sock_owned_by_me(sk);
864
865 mptcp_for_each_subflow(msk, subflow) {
866 if (READ_ONCE(subflow->data_avail))
867 return mptcp_subflow_tcp_sock(subflow);
868 }
869
870 return NULL;
871}
872
873static bool mptcp_skb_can_collapse_to(u64 write_seq,
874 const struct sk_buff *skb,
875 const struct mptcp_ext *mpext)
876{
877 if (!tcp_skb_can_collapse_to(skb))
878 return false;
879
880
881
882
883 return mpext && mpext->data_seq + mpext->data_len == write_seq &&
884 !mpext->frozen;
885}
886
887
888
889
890
891
892static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk,
893 const struct page_frag *pfrag,
894 const struct mptcp_data_frag *df)
895{
896 return df && pfrag->page == df->page &&
897 pfrag->size - pfrag->offset > 0 &&
898 pfrag->offset == (df->offset + df->data_len) &&
899 df->data_seq + df->data_len == msk->write_seq;
900}
901
902static int mptcp_wmem_with_overhead(int size)
903{
904 return size + ((sizeof(struct mptcp_data_frag) * size) >> PAGE_SHIFT);
905}
906
907static void __mptcp_wmem_reserve(struct sock *sk, int size)
908{
909 int amount = mptcp_wmem_with_overhead(size);
910 struct mptcp_sock *msk = mptcp_sk(sk);
911
912 WARN_ON_ONCE(msk->wmem_reserved);
913 if (WARN_ON_ONCE(amount < 0))
914 amount = 0;
915
916 if (amount <= sk->sk_forward_alloc)
917 goto reserve;
918
919
920
921
922
923 if ((tcp_under_memory_pressure(sk) && amount > PAGE_SIZE) ||
924 !sk_wmem_schedule(sk, amount)) {
925 if (amount <= PAGE_SIZE)
926 goto nomem;
927
928 amount = PAGE_SIZE;
929 if (!sk_wmem_schedule(sk, amount))
930 goto nomem;
931 }
932
933reserve:
934 msk->wmem_reserved = amount;
935 sk->sk_forward_alloc -= amount;
936 return;
937
938nomem:
939
940 msk->wmem_reserved = -1;
941}
942
943static void __mptcp_update_wmem(struct sock *sk)
944{
945 struct mptcp_sock *msk = mptcp_sk(sk);
946
947#ifdef CONFIG_LOCKDEP
948 WARN_ON_ONCE(!lockdep_is_held(&sk->sk_lock.slock));
949#endif
950
951 if (!msk->wmem_reserved)
952 return;
953
954 if (msk->wmem_reserved < 0)
955 msk->wmem_reserved = 0;
956 if (msk->wmem_reserved > 0) {
957 sk->sk_forward_alloc += msk->wmem_reserved;
958 msk->wmem_reserved = 0;
959 }
960}
961
962static bool mptcp_wmem_alloc(struct sock *sk, int size)
963{
964 struct mptcp_sock *msk = mptcp_sk(sk);
965
966
967 if (msk->wmem_reserved < 0)
968 return false;
969
970 if (msk->wmem_reserved >= size)
971 goto account;
972
973 mptcp_data_lock(sk);
974 if (!sk_wmem_schedule(sk, size)) {
975 mptcp_data_unlock(sk);
976 return false;
977 }
978
979 sk->sk_forward_alloc -= size;
980 msk->wmem_reserved += size;
981 mptcp_data_unlock(sk);
982
983account:
984 msk->wmem_reserved -= size;
985 return true;
986}
987
988static void mptcp_wmem_uncharge(struct sock *sk, int size)
989{
990 struct mptcp_sock *msk = mptcp_sk(sk);
991
992 if (msk->wmem_reserved < 0)
993 msk->wmem_reserved = 0;
994 msk->wmem_reserved += size;
995}
996
997static void mptcp_mem_reclaim_partial(struct sock *sk)
998{
999 struct mptcp_sock *msk = mptcp_sk(sk);
1000
1001
1002
1003
1004
1005 if (msk->wmem_reserved < 0)
1006 return;
1007
1008 mptcp_data_lock(sk);
1009 sk->sk_forward_alloc += msk->wmem_reserved;
1010 sk_mem_reclaim_partial(sk);
1011 msk->wmem_reserved = sk->sk_forward_alloc;
1012 sk->sk_forward_alloc = 0;
1013 mptcp_data_unlock(sk);
1014}
1015
1016static void dfrag_uncharge(struct sock *sk, int len)
1017{
1018 sk_mem_uncharge(sk, len);
1019 sk_wmem_queued_add(sk, -len);
1020}
1021
1022static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)
1023{
1024 int len = dfrag->data_len + dfrag->overhead;
1025
1026 list_del(&dfrag->list);
1027 dfrag_uncharge(sk, len);
1028 put_page(dfrag->page);
1029}
1030
1031static void __mptcp_clean_una(struct sock *sk)
1032{
1033 struct mptcp_sock *msk = mptcp_sk(sk);
1034 struct mptcp_data_frag *dtmp, *dfrag;
1035 bool cleaned = false;
1036 u64 snd_una;
1037
1038
1039
1040
1041 if (__mptcp_check_fallback(msk))
1042 msk->snd_una = READ_ONCE(msk->snd_nxt);
1043
1044 snd_una = msk->snd_una;
1045 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {
1046 if (after64(dfrag->data_seq + dfrag->data_len, snd_una))
1047 break;
1048
1049 if (WARN_ON_ONCE(dfrag == msk->first_pending))
1050 break;
1051 dfrag_clear(sk, dfrag);
1052 cleaned = true;
1053 }
1054
1055 dfrag = mptcp_rtx_head(sk);
1056 if (dfrag && after64(snd_una, dfrag->data_seq)) {
1057 u64 delta = snd_una - dfrag->data_seq;
1058
1059 if (WARN_ON_ONCE(delta > dfrag->already_sent))
1060 goto out;
1061
1062 dfrag->data_seq += delta;
1063 dfrag->offset += delta;
1064 dfrag->data_len -= delta;
1065 dfrag->already_sent -= delta;
1066
1067 dfrag_uncharge(sk, delta);
1068 cleaned = true;
1069 }
1070
1071out:
1072 if (cleaned) {
1073 if (tcp_under_memory_pressure(sk)) {
1074 __mptcp_update_wmem(sk);
1075 sk_mem_reclaim_partial(sk);
1076 }
1077 }
1078
1079 if (snd_una == READ_ONCE(msk->snd_nxt)) {
1080 if (msk->timer_ival && !mptcp_data_fin_enabled(msk))
1081 mptcp_stop_timer(sk);
1082 } else {
1083 mptcp_reset_timer(sk);
1084 }
1085}
1086
1087static void __mptcp_clean_una_wakeup(struct sock *sk)
1088{
1089#ifdef CONFIG_LOCKDEP
1090 WARN_ON_ONCE(!lockdep_is_held(&sk->sk_lock.slock));
1091#endif
1092 __mptcp_clean_una(sk);
1093 mptcp_write_space(sk);
1094}
1095
1096static void mptcp_clean_una_wakeup(struct sock *sk)
1097{
1098 mptcp_data_lock(sk);
1099 __mptcp_clean_una_wakeup(sk);
1100 mptcp_data_unlock(sk);
1101}
1102
1103static void mptcp_enter_memory_pressure(struct sock *sk)
1104{
1105 struct mptcp_subflow_context *subflow;
1106 struct mptcp_sock *msk = mptcp_sk(sk);
1107 bool first = true;
1108
1109 sk_stream_moderate_sndbuf(sk);
1110 mptcp_for_each_subflow(msk, subflow) {
1111 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
1112
1113 if (first)
1114 tcp_enter_memory_pressure(ssk);
1115 sk_stream_moderate_sndbuf(ssk);
1116 first = false;
1117 }
1118}
1119
1120
1121
1122
1123static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
1124{
1125 if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag),
1126 pfrag, sk->sk_allocation)))
1127 return true;
1128
1129 mptcp_enter_memory_pressure(sk);
1130 return false;
1131}
1132
1133static struct mptcp_data_frag *
1134mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag,
1135 int orig_offset)
1136{
1137 int offset = ALIGN(orig_offset, sizeof(long));
1138 struct mptcp_data_frag *dfrag;
1139
1140 dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);
1141 dfrag->data_len = 0;
1142 dfrag->data_seq = msk->write_seq;
1143 dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);
1144 dfrag->offset = offset + sizeof(struct mptcp_data_frag);
1145 dfrag->already_sent = 0;
1146 dfrag->page = pfrag->page;
1147
1148 return dfrag;
1149}
1150
1151struct mptcp_sendmsg_info {
1152 int mss_now;
1153 int size_goal;
1154 u16 limit;
1155 u16 sent;
1156 unsigned int flags;
1157};
1158
1159static int mptcp_check_allowed_size(struct mptcp_sock *msk, u64 data_seq,
1160 int avail_size)
1161{
1162 u64 window_end = mptcp_wnd_end(msk);
1163
1164 if (__mptcp_check_fallback(msk))
1165 return avail_size;
1166
1167 if (!before64(data_seq + avail_size, window_end)) {
1168 u64 allowed_size = window_end - data_seq;
1169
1170 return min_t(unsigned int, allowed_size, avail_size);
1171 }
1172
1173 return avail_size;
1174}
1175
1176static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp)
1177{
1178 struct skb_ext *mpext = __skb_ext_alloc(gfp);
1179
1180 if (!mpext)
1181 return false;
1182 __skb_ext_set(skb, SKB_EXT_MPTCP, mpext);
1183 return true;
1184}
1185
1186static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp)
1187{
1188 struct sk_buff *skb;
1189
1190 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
1191 if (likely(skb)) {
1192 if (likely(__mptcp_add_ext(skb, gfp))) {
1193 skb_reserve(skb, MAX_TCP_HEADER);
1194 skb->reserved_tailroom = skb->end - skb->tail;
1195 return skb;
1196 }
1197 __kfree_skb(skb);
1198 } else {
1199 mptcp_enter_memory_pressure(sk);
1200 }
1201 return NULL;
1202}
1203
1204static bool __mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp)
1205{
1206 struct sk_buff *skb;
1207
1208 if (ssk->sk_tx_skb_cache) {
1209 skb = ssk->sk_tx_skb_cache;
1210 if (unlikely(!skb_ext_find(skb, SKB_EXT_MPTCP) &&
1211 !__mptcp_add_ext(skb, gfp)))
1212 return false;
1213 return true;
1214 }
1215
1216 skb = __mptcp_do_alloc_tx_skb(sk, gfp);
1217 if (!skb)
1218 return false;
1219
1220 if (likely(sk_wmem_schedule(ssk, skb->truesize))) {
1221 ssk->sk_tx_skb_cache = skb;
1222 return true;
1223 }
1224 kfree_skb(skb);
1225 return false;
1226}
1227
1228static bool mptcp_must_reclaim_memory(struct sock *sk, struct sock *ssk)
1229{
1230 return !ssk->sk_tx_skb_cache &&
1231 tcp_under_memory_pressure(sk);
1232}
1233
1234static bool mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk)
1235{
1236 if (unlikely(mptcp_must_reclaim_memory(sk, ssk)))
1237 mptcp_mem_reclaim_partial(sk);
1238 return __mptcp_alloc_tx_skb(sk, ssk, sk->sk_allocation);
1239}
1240
1241
1242
1243
1244static void mptcp_update_data_checksum(struct sk_buff *skb, int added)
1245{
1246 struct mptcp_ext *mpext = mptcp_get_ext(skb);
1247 __wsum csum = ~csum_unfold(mpext->csum);
1248 int offset = skb->len - added;
1249
1250 mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset));
1251}
1252
1253static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1254 struct mptcp_data_frag *dfrag,
1255 struct mptcp_sendmsg_info *info)
1256{
1257 u64 data_seq = dfrag->data_seq + info->sent;
1258 struct mptcp_sock *msk = mptcp_sk(sk);
1259 bool zero_window_probe = false;
1260 struct mptcp_ext *mpext = NULL;
1261 struct sk_buff *skb, *tail;
1262 bool can_collapse = false;
1263 int size_bias = 0;
1264 int avail_size;
1265 size_t ret = 0;
1266
1267 pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u",
1268 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);
1269
1270
1271 info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags);
1272 avail_size = info->size_goal;
1273 skb = tcp_write_queue_tail(ssk);
1274 if (skb) {
1275
1276
1277
1278
1279
1280
1281 mpext = skb_ext_find(skb, SKB_EXT_MPTCP);
1282 can_collapse = (info->size_goal - skb->len > 0) &&
1283 mptcp_skb_can_collapse_to(data_seq, skb, mpext);
1284 if (!can_collapse) {
1285 TCP_SKB_CB(skb)->eor = 1;
1286 } else {
1287 size_bias = skb->len;
1288 avail_size = info->size_goal - skb->len;
1289 }
1290 }
1291
1292
1293 avail_size = mptcp_check_allowed_size(msk, data_seq, avail_size);
1294 if (avail_size == 0) {
1295 u64 snd_una = READ_ONCE(msk->snd_una);
1296
1297 if (skb || snd_una != msk->snd_nxt)
1298 return 0;
1299 zero_window_probe = true;
1300 data_seq = snd_una - 1;
1301 avail_size = 1;
1302 }
1303
1304 if (WARN_ON_ONCE(info->sent > info->limit ||
1305 info->limit > dfrag->data_len))
1306 return 0;
1307
1308 ret = info->limit - info->sent;
1309 tail = tcp_build_frag(ssk, avail_size + size_bias, info->flags,
1310 dfrag->page, dfrag->offset + info->sent, &ret);
1311 if (!tail) {
1312 tcp_remove_empty_skb(sk, tcp_write_queue_tail(ssk));
1313 return -ENOMEM;
1314 }
1315
1316
1317
1318 if (skb == tail) {
1319 TCP_SKB_CB(tail)->tcp_flags &= ~TCPHDR_PSH;
1320 mpext->data_len += ret;
1321 WARN_ON_ONCE(!can_collapse);
1322 WARN_ON_ONCE(zero_window_probe);
1323 goto out;
1324 }
1325
1326 mpext = skb_ext_find(tail, SKB_EXT_MPTCP);
1327 if (WARN_ON_ONCE(!mpext)) {
1328
1329 return -EINVAL;
1330 }
1331
1332 memset(mpext, 0, sizeof(*mpext));
1333 mpext->data_seq = data_seq;
1334 mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;
1335 mpext->data_len = ret;
1336 mpext->use_map = 1;
1337 mpext->dsn64 = 1;
1338
1339 pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d",
1340 mpext->data_seq, mpext->subflow_seq, mpext->data_len,
1341 mpext->dsn64);
1342
1343 if (zero_window_probe) {
1344 mptcp_subflow_ctx(ssk)->rel_write_seq += ret;
1345 mpext->frozen = 1;
1346 if (READ_ONCE(msk->csum_enabled))
1347 mptcp_update_data_checksum(tail, ret);
1348 tcp_push_pending_frames(ssk);
1349 return 0;
1350 }
1351out:
1352 if (READ_ONCE(msk->csum_enabled))
1353 mptcp_update_data_checksum(tail, ret);
1354 mptcp_subflow_ctx(ssk)->rel_write_seq += ret;
1355 return ret;
1356}
1357
1358#define MPTCP_SEND_BURST_SIZE ((1 << 16) - \
1359 sizeof(struct tcphdr) - \
1360 MAX_TCP_OPTION_SPACE - \
1361 sizeof(struct ipv6hdr) - \
1362 sizeof(struct frag_hdr))
1363
1364struct subflow_send_info {
1365 struct sock *ssk;
1366 u64 ratio;
1367};
1368
1369static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
1370{
1371 struct subflow_send_info send_info[2];
1372 struct mptcp_subflow_context *subflow;
1373 int i, nr_active = 0;
1374 struct sock *ssk;
1375 u64 ratio;
1376 u32 pace;
1377
1378 sock_owned_by_me((struct sock *)msk);
1379
1380 if (__mptcp_check_fallback(msk)) {
1381 if (!msk->first)
1382 return NULL;
1383 return sk_stream_memory_free(msk->first) ? msk->first : NULL;
1384 }
1385
1386
1387 if (msk->last_snd && msk->snd_burst > 0 &&
1388 sk_stream_memory_free(msk->last_snd) &&
1389 mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd)))
1390 return msk->last_snd;
1391
1392
1393 for (i = 0; i < 2; ++i) {
1394 send_info[i].ssk = NULL;
1395 send_info[i].ratio = -1;
1396 }
1397 mptcp_for_each_subflow(msk, subflow) {
1398 trace_mptcp_subflow_get_send(subflow);
1399 ssk = mptcp_subflow_tcp_sock(subflow);
1400 if (!mptcp_subflow_active(subflow))
1401 continue;
1402
1403 nr_active += !subflow->backup;
1404 if (!sk_stream_memory_free(subflow->tcp_sock) || !tcp_sk(ssk)->snd_wnd)
1405 continue;
1406
1407 pace = READ_ONCE(ssk->sk_pacing_rate);
1408 if (!pace)
1409 continue;
1410
1411 ratio = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32,
1412 pace);
1413 if (ratio < send_info[subflow->backup].ratio) {
1414 send_info[subflow->backup].ssk = ssk;
1415 send_info[subflow->backup].ratio = ratio;
1416 }
1417 }
1418
1419
1420 if (!nr_active)
1421 send_info[0].ssk = send_info[1].ssk;
1422
1423 if (send_info[0].ssk) {
1424 msk->last_snd = send_info[0].ssk;
1425 msk->snd_burst = min_t(int, MPTCP_SEND_BURST_SIZE,
1426 tcp_sk(msk->last_snd)->snd_wnd);
1427 return msk->last_snd;
1428 }
1429
1430 return NULL;
1431}
1432
1433static void mptcp_push_release(struct sock *sk, struct sock *ssk,
1434 struct mptcp_sendmsg_info *info)
1435{
1436 mptcp_set_timeout(sk, ssk);
1437 tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal);
1438 release_sock(ssk);
1439}
1440
1441static void __mptcp_push_pending(struct sock *sk, unsigned int flags)
1442{
1443 struct sock *prev_ssk = NULL, *ssk = NULL;
1444 struct mptcp_sock *msk = mptcp_sk(sk);
1445 struct mptcp_sendmsg_info info = {
1446 .flags = flags,
1447 };
1448 struct mptcp_data_frag *dfrag;
1449 int len, copied = 0;
1450
1451 while ((dfrag = mptcp_send_head(sk))) {
1452 info.sent = dfrag->already_sent;
1453 info.limit = dfrag->data_len;
1454 len = dfrag->data_len - dfrag->already_sent;
1455 while (len > 0) {
1456 int ret = 0;
1457
1458 prev_ssk = ssk;
1459 mptcp_flush_join_list(msk);
1460 ssk = mptcp_subflow_get_send(msk);
1461
1462
1463
1464
1465 if (ssk != prev_ssk && prev_ssk)
1466 mptcp_push_release(sk, prev_ssk, &info);
1467 if (!ssk)
1468 goto out;
1469
1470 if (ssk != prev_ssk || !prev_ssk)
1471 lock_sock(ssk);
1472
1473
1474
1475
1476
1477 if (!mptcp_alloc_tx_skb(sk, ssk)) {
1478 mptcp_push_release(sk, ssk, &info);
1479 goto out;
1480 }
1481
1482 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
1483 if (ret <= 0) {
1484 mptcp_push_release(sk, ssk, &info);
1485 goto out;
1486 }
1487
1488 info.sent += ret;
1489 dfrag->already_sent += ret;
1490 msk->snd_nxt += ret;
1491 msk->snd_burst -= ret;
1492 msk->tx_pending_data -= ret;
1493 copied += ret;
1494 len -= ret;
1495 }
1496 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
1497 }
1498
1499
1500 if (ssk)
1501 mptcp_push_release(sk, ssk, &info);
1502
1503out:
1504 if (copied) {
1505
1506 if (!mptcp_timer_pending(sk))
1507 mptcp_reset_timer(sk);
1508 __mptcp_check_send_data_fin(sk);
1509 }
1510}
1511
1512static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk)
1513{
1514 struct mptcp_sock *msk = mptcp_sk(sk);
1515 struct mptcp_sendmsg_info info;
1516 struct mptcp_data_frag *dfrag;
1517 struct sock *xmit_ssk;
1518 int len, copied = 0;
1519 bool first = true;
1520
1521 info.flags = 0;
1522 while ((dfrag = mptcp_send_head(sk))) {
1523 info.sent = dfrag->already_sent;
1524 info.limit = dfrag->data_len;
1525 len = dfrag->data_len - dfrag->already_sent;
1526 while (len > 0) {
1527 int ret = 0;
1528
1529
1530
1531
1532
1533 xmit_ssk = first ? ssk : mptcp_subflow_get_send(mptcp_sk(sk));
1534 if (!xmit_ssk)
1535 goto out;
1536 if (xmit_ssk != ssk) {
1537 mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk));
1538 goto out;
1539 }
1540
1541 if (unlikely(mptcp_must_reclaim_memory(sk, ssk))) {
1542 __mptcp_update_wmem(sk);
1543 sk_mem_reclaim_partial(sk);
1544 }
1545 if (!__mptcp_alloc_tx_skb(sk, ssk, GFP_ATOMIC))
1546 goto out;
1547
1548 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
1549 if (ret <= 0)
1550 goto out;
1551
1552 info.sent += ret;
1553 dfrag->already_sent += ret;
1554 msk->snd_nxt += ret;
1555 msk->snd_burst -= ret;
1556 msk->tx_pending_data -= ret;
1557 copied += ret;
1558 len -= ret;
1559 first = false;
1560 }
1561 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
1562 }
1563
1564out:
1565
1566
1567
1568 __mptcp_update_wmem(sk);
1569 if (copied) {
1570 mptcp_set_timeout(sk, ssk);
1571 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,
1572 info.size_goal);
1573 if (!mptcp_timer_pending(sk))
1574 mptcp_reset_timer(sk);
1575
1576 if (msk->snd_data_fin_enable &&
1577 msk->snd_nxt + 1 == msk->write_seq)
1578 mptcp_schedule_work(sk);
1579 }
1580}
1581
1582static void mptcp_set_nospace(struct sock *sk)
1583{
1584
1585 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1586
1587
1588 set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags);
1589}
1590
1591static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
1592{
1593 struct mptcp_sock *msk = mptcp_sk(sk);
1594 struct page_frag *pfrag;
1595 size_t copied = 0;
1596 int ret = 0;
1597 long timeo;
1598
1599
1600 if (msg->msg_flags & MSG_FASTOPEN)
1601 return -EOPNOTSUPP;
1602
1603
1604 msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL;
1605
1606 mptcp_lock_sock(sk, __mptcp_wmem_reserve(sk, min_t(size_t, 1 << 20, len)));
1607
1608 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
1609
1610 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
1611 ret = sk_stream_wait_connect(sk, &timeo);
1612 if (ret)
1613 goto out;
1614 }
1615
1616 pfrag = sk_page_frag(sk);
1617
1618 while (msg_data_left(msg)) {
1619 int total_ts, frag_truesize = 0;
1620 struct mptcp_data_frag *dfrag;
1621 bool dfrag_collapsed;
1622 size_t psize, offset;
1623
1624 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) {
1625 ret = -EPIPE;
1626 goto out;
1627 }
1628
1629
1630
1631
1632 dfrag = mptcp_pending_tail(sk);
1633 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);
1634 if (!dfrag_collapsed) {
1635 if (!sk_stream_memory_free(sk))
1636 goto wait_for_memory;
1637
1638 if (!mptcp_page_frag_refill(sk, pfrag))
1639 goto wait_for_memory;
1640
1641 dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset);
1642 frag_truesize = dfrag->overhead;
1643 }
1644
1645
1646
1647
1648
1649 offset = dfrag->offset + dfrag->data_len;
1650 psize = pfrag->size - offset;
1651 psize = min_t(size_t, psize, msg_data_left(msg));
1652 total_ts = psize + frag_truesize;
1653
1654 if (!mptcp_wmem_alloc(sk, total_ts))
1655 goto wait_for_memory;
1656
1657 if (copy_page_from_iter(dfrag->page, offset, psize,
1658 &msg->msg_iter) != psize) {
1659 mptcp_wmem_uncharge(sk, psize + frag_truesize);
1660 ret = -EFAULT;
1661 goto out;
1662 }
1663
1664
1665 copied += psize;
1666 dfrag->data_len += psize;
1667 frag_truesize += psize;
1668 pfrag->offset += frag_truesize;
1669 WRITE_ONCE(msk->write_seq, msk->write_seq + psize);
1670 msk->tx_pending_data += psize;
1671
1672
1673
1674
1675 sk_wmem_queued_add(sk, frag_truesize);
1676 if (!dfrag_collapsed) {
1677 get_page(dfrag->page);
1678 list_add_tail(&dfrag->list, &msk->rtx_queue);
1679 if (!msk->first_pending)
1680 WRITE_ONCE(msk->first_pending, dfrag);
1681 }
1682 pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk,
1683 dfrag->data_seq, dfrag->data_len, dfrag->already_sent,
1684 !dfrag_collapsed);
1685
1686 continue;
1687
1688wait_for_memory:
1689 mptcp_set_nospace(sk);
1690 __mptcp_push_pending(sk, msg->msg_flags);
1691 ret = sk_stream_wait_memory(sk, &timeo);
1692 if (ret)
1693 goto out;
1694 }
1695
1696 if (copied)
1697 __mptcp_push_pending(sk, msg->msg_flags);
1698
1699out:
1700 release_sock(sk);
1701 return copied ? : ret;
1702}
1703
1704static void mptcp_wait_data(struct sock *sk, long *timeo)
1705{
1706 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1707 struct mptcp_sock *msk = mptcp_sk(sk);
1708
1709 add_wait_queue(sk_sleep(sk), &wait);
1710 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1711
1712 sk_wait_event(sk, timeo,
1713 test_bit(MPTCP_DATA_READY, &msk->flags), &wait);
1714
1715 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1716 remove_wait_queue(sk_sleep(sk), &wait);
1717}
1718
1719static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
1720 struct msghdr *msg,
1721 size_t len, int flags,
1722 struct scm_timestamping_internal *tss,
1723 int *cmsg_flags)
1724{
1725 struct sk_buff *skb, *tmp;
1726 int copied = 0;
1727
1728 skb_queue_walk_safe(&msk->receive_queue, skb, tmp) {
1729 u32 offset = MPTCP_SKB_CB(skb)->offset;
1730 u32 data_len = skb->len - offset;
1731 u32 count = min_t(size_t, len - copied, data_len);
1732 int err;
1733
1734 if (!(flags & MSG_TRUNC)) {
1735 err = skb_copy_datagram_msg(skb, offset, msg, count);
1736 if (unlikely(err < 0)) {
1737 if (!copied)
1738 return err;
1739 break;
1740 }
1741 }
1742
1743 if (MPTCP_SKB_CB(skb)->has_rxtstamp) {
1744 tcp_update_recv_tstamps(skb, tss);
1745 *cmsg_flags |= MPTCP_CMSG_TS;
1746 }
1747
1748 copied += count;
1749
1750 if (count < data_len) {
1751 if (!(flags & MSG_PEEK))
1752 MPTCP_SKB_CB(skb)->offset += count;
1753 break;
1754 }
1755
1756 if (!(flags & MSG_PEEK)) {
1757
1758 skb->destructor = NULL;
1759 WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize);
1760 __skb_unlink(skb, &msk->receive_queue);
1761 __kfree_skb(skb);
1762 }
1763
1764 if (copied >= len)
1765 break;
1766 }
1767
1768 return copied;
1769}
1770
1771
1772
1773
1774
1775static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied)
1776{
1777 struct mptcp_subflow_context *subflow;
1778 struct sock *sk = (struct sock *)msk;
1779 u32 time, advmss = 1;
1780 u64 rtt_us, mstamp;
1781
1782 sock_owned_by_me(sk);
1783
1784 if (copied <= 0)
1785 return;
1786
1787 msk->rcvq_space.copied += copied;
1788
1789 mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC);
1790 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time);
1791
1792 rtt_us = msk->rcvq_space.rtt_us;
1793 if (rtt_us && time < (rtt_us >> 3))
1794 return;
1795
1796 rtt_us = 0;
1797 mptcp_for_each_subflow(msk, subflow) {
1798 const struct tcp_sock *tp;
1799 u64 sf_rtt_us;
1800 u32 sf_advmss;
1801
1802 tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));
1803
1804 sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us);
1805 sf_advmss = READ_ONCE(tp->advmss);
1806
1807 rtt_us = max(sf_rtt_us, rtt_us);
1808 advmss = max(sf_advmss, advmss);
1809 }
1810
1811 msk->rcvq_space.rtt_us = rtt_us;
1812 if (time < (rtt_us >> 3) || rtt_us == 0)
1813 return;
1814
1815 if (msk->rcvq_space.copied <= msk->rcvq_space.space)
1816 goto new_measure;
1817
1818 if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf &&
1819 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
1820 int rcvmem, rcvbuf;
1821 u64 rcvwin, grow;
1822
1823 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss;
1824
1825 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space);
1826
1827 do_div(grow, msk->rcvq_space.space);
1828 rcvwin += (grow << 1);
1829
1830 rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER);
1831 while (tcp_win_from_space(sk, rcvmem) < advmss)
1832 rcvmem += 128;
1833
1834 do_div(rcvwin, advmss);
1835 rcvbuf = min_t(u64, rcvwin * rcvmem,
1836 sock_net(sk)->ipv4.sysctl_tcp_rmem[2]);
1837
1838 if (rcvbuf > sk->sk_rcvbuf) {
1839 u32 window_clamp;
1840
1841 window_clamp = tcp_win_from_space(sk, rcvbuf);
1842 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
1843
1844
1845
1846
1847
1848
1849 mptcp_for_each_subflow(msk, subflow) {
1850 struct sock *ssk;
1851 bool slow;
1852
1853 ssk = mptcp_subflow_tcp_sock(subflow);
1854 slow = lock_sock_fast(ssk);
1855 WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf);
1856 tcp_sk(ssk)->window_clamp = window_clamp;
1857 tcp_cleanup_rbuf(ssk, 1);
1858 unlock_sock_fast(ssk, slow);
1859 }
1860 }
1861 }
1862
1863 msk->rcvq_space.space = msk->rcvq_space.copied;
1864new_measure:
1865 msk->rcvq_space.copied = 0;
1866 msk->rcvq_space.time = mstamp;
1867}
1868
1869static void __mptcp_update_rmem(struct sock *sk)
1870{
1871 struct mptcp_sock *msk = mptcp_sk(sk);
1872
1873 if (!msk->rmem_released)
1874 return;
1875
1876 atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc);
1877 sk_mem_uncharge(sk, msk->rmem_released);
1878 WRITE_ONCE(msk->rmem_released, 0);
1879}
1880
1881static void __mptcp_splice_receive_queue(struct sock *sk)
1882{
1883 struct mptcp_sock *msk = mptcp_sk(sk);
1884
1885 skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue);
1886}
1887
1888static bool __mptcp_move_skbs(struct mptcp_sock *msk)
1889{
1890 struct sock *sk = (struct sock *)msk;
1891 unsigned int moved = 0;
1892 bool ret, done;
1893
1894 mptcp_flush_join_list(msk);
1895 do {
1896 struct sock *ssk = mptcp_subflow_recv_lookup(msk);
1897 bool slowpath;
1898
1899
1900
1901
1902
1903 if (likely(!ssk))
1904 break;
1905
1906 slowpath = lock_sock_fast(ssk);
1907 mptcp_data_lock(sk);
1908 __mptcp_update_rmem(sk);
1909 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
1910 mptcp_data_unlock(sk);
1911
1912 if (unlikely(ssk->sk_err))
1913 __mptcp_error_report(sk);
1914 unlock_sock_fast(ssk, slowpath);
1915 } while (!done);
1916
1917
1918 ret = moved > 0;
1919 if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) ||
1920 !skb_queue_empty_lockless(&sk->sk_receive_queue)) {
1921 mptcp_data_lock(sk);
1922 __mptcp_update_rmem(sk);
1923 ret |= __mptcp_ofo_queue(msk);
1924 __mptcp_splice_receive_queue(sk);
1925 mptcp_data_unlock(sk);
1926 }
1927 if (ret)
1928 mptcp_check_data_fin((struct sock *)msk);
1929 return !skb_queue_empty(&msk->receive_queue);
1930}
1931
1932static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
1933 int nonblock, int flags, int *addr_len)
1934{
1935 struct mptcp_sock *msk = mptcp_sk(sk);
1936 struct scm_timestamping_internal tss;
1937 int copied = 0, cmsg_flags = 0;
1938 int target;
1939 long timeo;
1940
1941
1942 if (unlikely(flags & MSG_ERRQUEUE))
1943 return inet_recv_error(sk, msg, len, addr_len);
1944
1945 mptcp_lock_sock(sk, __mptcp_splice_receive_queue(sk));
1946 if (unlikely(sk->sk_state == TCP_LISTEN)) {
1947 copied = -ENOTCONN;
1948 goto out_err;
1949 }
1950
1951 timeo = sock_rcvtimeo(sk, nonblock);
1952
1953 len = min_t(size_t, len, INT_MAX);
1954 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1955
1956 while (copied < len) {
1957 int bytes_read;
1958
1959 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags);
1960 if (unlikely(bytes_read < 0)) {
1961 if (!copied)
1962 copied = bytes_read;
1963 goto out_err;
1964 }
1965
1966 copied += bytes_read;
1967
1968
1969 mptcp_cleanup_rbuf(msk);
1970
1971 if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk))
1972 continue;
1973
1974
1975
1976
1977 if (copied >= target)
1978 break;
1979
1980 if (copied) {
1981 if (sk->sk_err ||
1982 sk->sk_state == TCP_CLOSE ||
1983 (sk->sk_shutdown & RCV_SHUTDOWN) ||
1984 !timeo ||
1985 signal_pending(current))
1986 break;
1987 } else {
1988 if (sk->sk_err) {
1989 copied = sock_error(sk);
1990 break;
1991 }
1992
1993 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags))
1994 mptcp_check_for_eof(msk);
1995
1996 if (sk->sk_shutdown & RCV_SHUTDOWN) {
1997
1998
1999
2000 if (__mptcp_move_skbs(msk))
2001 continue;
2002 break;
2003 }
2004
2005 if (sk->sk_state == TCP_CLOSE) {
2006 copied = -ENOTCONN;
2007 break;
2008 }
2009
2010 if (!timeo) {
2011 copied = -EAGAIN;
2012 break;
2013 }
2014
2015 if (signal_pending(current)) {
2016 copied = sock_intr_errno(timeo);
2017 break;
2018 }
2019 }
2020
2021 pr_debug("block timeout %ld", timeo);
2022 mptcp_wait_data(sk, &timeo);
2023 }
2024
2025 if (skb_queue_empty_lockless(&sk->sk_receive_queue) &&
2026 skb_queue_empty(&msk->receive_queue)) {
2027
2028 clear_bit(MPTCP_DATA_READY, &msk->flags);
2029
2030
2031
2032
2033 if (unlikely(__mptcp_move_skbs(msk)))
2034 set_bit(MPTCP_DATA_READY, &msk->flags);
2035 }
2036
2037out_err:
2038 if (cmsg_flags && copied >= 0) {
2039 if (cmsg_flags & MPTCP_CMSG_TS)
2040 tcp_recv_timestamp(msg, sk, &tss);
2041 }
2042
2043 pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d",
2044 msk, test_bit(MPTCP_DATA_READY, &msk->flags),
2045 skb_queue_empty_lockless(&sk->sk_receive_queue), copied);
2046 if (!(flags & MSG_PEEK))
2047 mptcp_rcv_space_adjust(msk, copied);
2048
2049 release_sock(sk);
2050 return copied;
2051}
2052
2053static void mptcp_retransmit_timer(struct timer_list *t)
2054{
2055 struct inet_connection_sock *icsk = from_timer(icsk, t,
2056 icsk_retransmit_timer);
2057 struct sock *sk = &icsk->icsk_inet.sk;
2058 struct mptcp_sock *msk = mptcp_sk(sk);
2059
2060 bh_lock_sock(sk);
2061 if (!sock_owned_by_user(sk)) {
2062
2063 if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags))
2064 mptcp_schedule_work(sk);
2065 } else {
2066
2067 set_bit(MPTCP_RETRANSMIT, &msk->flags);
2068 }
2069 bh_unlock_sock(sk);
2070 sock_put(sk);
2071}
2072
2073static void mptcp_timeout_timer(struct timer_list *t)
2074{
2075 struct sock *sk = from_timer(sk, t, sk_timer);
2076
2077 mptcp_schedule_work(sk);
2078 sock_put(sk);
2079}
2080
2081
2082
2083
2084
2085
2086static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk)
2087{
2088 struct mptcp_subflow_context *subflow;
2089 struct sock *backup = NULL;
2090
2091 sock_owned_by_me((const struct sock *)msk);
2092
2093 if (__mptcp_check_fallback(msk))
2094 return NULL;
2095
2096 mptcp_for_each_subflow(msk, subflow) {
2097 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
2098
2099 if (!mptcp_subflow_active(subflow))
2100 continue;
2101
2102
2103 if (!tcp_write_queue_empty(ssk)) {
2104 if (inet_csk(ssk)->icsk_ca_state >= TCP_CA_Loss)
2105 continue;
2106 return NULL;
2107 }
2108
2109 if (subflow->backup) {
2110 if (!backup)
2111 backup = ssk;
2112 continue;
2113 }
2114
2115 return ssk;
2116 }
2117
2118 return backup;
2119}
2120
2121static void mptcp_dispose_initial_subflow(struct mptcp_sock *msk)
2122{
2123 if (msk->subflow) {
2124 iput(SOCK_INODE(msk->subflow));
2125 msk->subflow = NULL;
2126 }
2127}
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
2138 struct mptcp_subflow_context *subflow)
2139{
2140 struct mptcp_sock *msk = mptcp_sk(sk);
2141
2142 list_del(&subflow->node);
2143
2144 lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);
2145
2146
2147
2148
2149 if (ssk->sk_socket)
2150 sock_orphan(ssk);
2151
2152 subflow->disposable = 1;
2153
2154
2155
2156
2157
2158 if (!inet_csk(ssk)->icsk_ulp_ops) {
2159 kfree_rcu(subflow, rcu);
2160 } else {
2161
2162 __tcp_close(ssk, 0);
2163
2164
2165 __sock_put(ssk);
2166 }
2167 release_sock(ssk);
2168
2169 sock_put(ssk);
2170
2171 if (ssk == msk->last_snd)
2172 msk->last_snd = NULL;
2173
2174 if (ssk == msk->first)
2175 msk->first = NULL;
2176
2177 if (msk->subflow && ssk == msk->subflow->sk)
2178 mptcp_dispose_initial_subflow(msk);
2179}
2180
2181void mptcp_close_ssk(struct sock *sk, struct sock *ssk,
2182 struct mptcp_subflow_context *subflow)
2183{
2184 if (sk->sk_state == TCP_ESTABLISHED)
2185 mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL);
2186 __mptcp_close_ssk(sk, ssk, subflow);
2187}
2188
2189static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu)
2190{
2191 return 0;
2192}
2193
2194static void __mptcp_close_subflow(struct mptcp_sock *msk)
2195{
2196 struct mptcp_subflow_context *subflow, *tmp;
2197
2198 might_sleep();
2199
2200 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
2201 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
2202
2203 if (inet_sk_state_load(ssk) != TCP_CLOSE)
2204 continue;
2205
2206
2207 if (!skb_queue_empty_lockless(&ssk->sk_receive_queue))
2208 continue;
2209
2210 mptcp_close_ssk((struct sock *)msk, ssk, subflow);
2211 }
2212}
2213
2214static bool mptcp_check_close_timeout(const struct sock *sk)
2215{
2216 s32 delta = tcp_jiffies32 - inet_csk(sk)->icsk_mtup.probe_timestamp;
2217 struct mptcp_subflow_context *subflow;
2218
2219 if (delta >= TCP_TIMEWAIT_LEN)
2220 return true;
2221
2222
2223
2224
2225 mptcp_for_each_subflow(mptcp_sk(sk), subflow) {
2226 if (inet_sk_state_load(mptcp_subflow_tcp_sock(subflow)) !=
2227 TCP_CLOSE)
2228 return false;
2229 }
2230 return true;
2231}
2232
2233static void mptcp_check_fastclose(struct mptcp_sock *msk)
2234{
2235 struct mptcp_subflow_context *subflow, *tmp;
2236 struct sock *sk = &msk->sk.icsk_inet.sk;
2237
2238 if (likely(!READ_ONCE(msk->rcv_fastclose)))
2239 return;
2240
2241 mptcp_token_destroy(msk);
2242
2243 list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
2244 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
2245 bool slow;
2246
2247 slow = lock_sock_fast(tcp_sk);
2248 if (tcp_sk->sk_state != TCP_CLOSE) {
2249 tcp_send_active_reset(tcp_sk, GFP_ATOMIC);
2250 tcp_set_state(tcp_sk, TCP_CLOSE);
2251 }
2252 unlock_sock_fast(tcp_sk, slow);
2253 }
2254
2255 inet_sk_state_store(sk, TCP_CLOSE);
2256 sk->sk_shutdown = SHUTDOWN_MASK;
2257 smp_mb__before_atomic();
2258 set_bit(MPTCP_DATA_READY, &msk->flags);
2259 set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags);
2260
2261 mptcp_close_wake_up(sk);
2262}
2263
2264static void __mptcp_retrans(struct sock *sk)
2265{
2266 struct mptcp_sock *msk = mptcp_sk(sk);
2267 struct mptcp_sendmsg_info info = {};
2268 struct mptcp_data_frag *dfrag;
2269 size_t copied = 0;
2270 struct sock *ssk;
2271 int ret;
2272
2273 mptcp_clean_una_wakeup(sk);
2274 dfrag = mptcp_rtx_head(sk);
2275 if (!dfrag) {
2276 if (mptcp_data_fin_enabled(msk)) {
2277 struct inet_connection_sock *icsk = inet_csk(sk);
2278
2279 icsk->icsk_retransmits++;
2280 mptcp_set_datafin_timeout(sk);
2281 mptcp_send_ack(msk);
2282
2283 goto reset_timer;
2284 }
2285
2286 return;
2287 }
2288
2289 ssk = mptcp_subflow_get_retrans(msk);
2290 if (!ssk)
2291 goto reset_timer;
2292
2293 lock_sock(ssk);
2294
2295
2296 info.sent = 0;
2297 info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent;
2298 while (info.sent < info.limit) {
2299 if (!mptcp_alloc_tx_skb(sk, ssk))
2300 break;
2301
2302 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
2303 if (ret <= 0)
2304 break;
2305
2306 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS);
2307 copied += ret;
2308 info.sent += ret;
2309 }
2310 if (copied) {
2311 dfrag->already_sent = max(dfrag->already_sent, info.sent);
2312 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,
2313 info.size_goal);
2314 }
2315
2316 mptcp_set_timeout(sk, ssk);
2317 release_sock(ssk);
2318
2319reset_timer:
2320 if (!mptcp_timer_pending(sk))
2321 mptcp_reset_timer(sk);
2322}
2323
2324static void mptcp_worker(struct work_struct *work)
2325{
2326 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
2327 struct sock *sk = &msk->sk.icsk_inet.sk;
2328 int state;
2329
2330 lock_sock(sk);
2331 state = sk->sk_state;
2332 if (unlikely(state == TCP_CLOSE))
2333 goto unlock;
2334
2335 mptcp_check_data_fin_ack(sk);
2336 mptcp_flush_join_list(msk);
2337
2338 mptcp_check_fastclose(msk);
2339
2340 if (msk->pm.status)
2341 mptcp_pm_nl_work(msk);
2342
2343 if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags))
2344 mptcp_check_for_eof(msk);
2345
2346 __mptcp_check_send_data_fin(sk);
2347 mptcp_check_data_fin(sk);
2348
2349
2350
2351
2352
2353 if (sock_flag(sk, SOCK_DEAD) &&
2354 (mptcp_check_close_timeout(sk) || sk->sk_state == TCP_CLOSE)) {
2355 inet_sk_state_store(sk, TCP_CLOSE);
2356 __mptcp_destroy_sock(sk);
2357 goto unlock;
2358 }
2359
2360 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
2361 __mptcp_close_subflow(msk);
2362
2363 if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags))
2364 __mptcp_retrans(sk);
2365
2366unlock:
2367 release_sock(sk);
2368 sock_put(sk);
2369}
2370
2371static int __mptcp_init_sock(struct sock *sk)
2372{
2373 struct mptcp_sock *msk = mptcp_sk(sk);
2374
2375 spin_lock_init(&msk->join_list_lock);
2376
2377 INIT_LIST_HEAD(&msk->conn_list);
2378 INIT_LIST_HEAD(&msk->join_list);
2379 INIT_LIST_HEAD(&msk->rtx_queue);
2380 INIT_WORK(&msk->work, mptcp_worker);
2381 __skb_queue_head_init(&msk->receive_queue);
2382 msk->out_of_order_queue = RB_ROOT;
2383 msk->first_pending = NULL;
2384 msk->wmem_reserved = 0;
2385 WRITE_ONCE(msk->rmem_released, 0);
2386 msk->tx_pending_data = 0;
2387
2388 msk->first = NULL;
2389 inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
2390 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk)));
2391
2392 mptcp_pm_data_init(msk);
2393
2394
2395 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);
2396 timer_setup(&sk->sk_timer, mptcp_timeout_timer, 0);
2397
2398 return 0;
2399}
2400
2401static int mptcp_init_sock(struct sock *sk)
2402{
2403 struct inet_connection_sock *icsk = inet_csk(sk);
2404 struct net *net = sock_net(sk);
2405 int ret;
2406
2407 ret = __mptcp_init_sock(sk);
2408 if (ret)
2409 return ret;
2410
2411 if (!mptcp_is_enabled(net))
2412 return -ENOPROTOOPT;
2413
2414 if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net))
2415 return -ENOMEM;
2416
2417 ret = __mptcp_socket_create(mptcp_sk(sk));
2418 if (ret)
2419 return ret;
2420
2421
2422
2423
2424 tcp_assign_congestion_control(sk);
2425 strcpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name);
2426
2427
2428 tcp_cleanup_congestion_control(sk);
2429 icsk->icsk_ca_ops = NULL;
2430
2431 sk_sockets_allocated_inc(sk);
2432 sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1];
2433 sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1];
2434
2435 return 0;
2436}
2437
2438static void __mptcp_clear_xmit(struct sock *sk)
2439{
2440 struct mptcp_sock *msk = mptcp_sk(sk);
2441 struct mptcp_data_frag *dtmp, *dfrag;
2442
2443 WRITE_ONCE(msk->first_pending, NULL);
2444 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)
2445 dfrag_clear(sk, dfrag);
2446}
2447
2448static void mptcp_cancel_work(struct sock *sk)
2449{
2450 struct mptcp_sock *msk = mptcp_sk(sk);
2451
2452 if (cancel_work_sync(&msk->work))
2453 __sock_put(sk);
2454}
2455
2456void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)
2457{
2458 lock_sock(ssk);
2459
2460 switch (ssk->sk_state) {
2461 case TCP_LISTEN:
2462 if (!(how & RCV_SHUTDOWN))
2463 break;
2464 fallthrough;
2465 case TCP_SYN_SENT:
2466 tcp_disconnect(ssk, O_NONBLOCK);
2467 break;
2468 default:
2469 if (__mptcp_check_fallback(mptcp_sk(sk))) {
2470 pr_debug("Fallback");
2471 ssk->sk_shutdown |= how;
2472 tcp_shutdown(ssk, how);
2473 } else {
2474 pr_debug("Sending DATA_FIN on subflow %p", ssk);
2475 mptcp_set_timeout(sk, ssk);
2476 tcp_send_ack(ssk);
2477 if (!mptcp_timer_pending(sk))
2478 mptcp_reset_timer(sk);
2479 }
2480 break;
2481 }
2482
2483 release_sock(ssk);
2484}
2485
2486static const unsigned char new_state[16] = {
2487
2488 [0 ] = TCP_CLOSE,
2489 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2490 [TCP_SYN_SENT] = TCP_CLOSE,
2491 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2492 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
2493 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
2494 [TCP_TIME_WAIT] = TCP_CLOSE,
2495 [TCP_CLOSE] = TCP_CLOSE,
2496 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
2497 [TCP_LAST_ACK] = TCP_LAST_ACK,
2498 [TCP_LISTEN] = TCP_CLOSE,
2499 [TCP_CLOSING] = TCP_CLOSING,
2500 [TCP_NEW_SYN_RECV] = TCP_CLOSE,
2501};
2502
2503static int mptcp_close_state(struct sock *sk)
2504{
2505 int next = (int)new_state[sk->sk_state];
2506 int ns = next & TCP_STATE_MASK;
2507
2508 inet_sk_state_store(sk, ns);
2509
2510 return next & TCP_ACTION_FIN;
2511}
2512
2513static void __mptcp_check_send_data_fin(struct sock *sk)
2514{
2515 struct mptcp_subflow_context *subflow;
2516 struct mptcp_sock *msk = mptcp_sk(sk);
2517
2518 pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu",
2519 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk),
2520 msk->snd_nxt, msk->write_seq);
2521
2522
2523
2524
2525 if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq ||
2526 mptcp_send_head(sk))
2527 return;
2528
2529 WRITE_ONCE(msk->snd_nxt, msk->write_seq);
2530
2531
2532
2533
2534 if (__mptcp_check_fallback(msk)) {
2535 if ((1 << sk->sk_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) {
2536 inet_sk_state_store(sk, TCP_CLOSE);
2537 mptcp_close_wake_up(sk);
2538 } else if (sk->sk_state == TCP_FIN_WAIT1) {
2539 inet_sk_state_store(sk, TCP_FIN_WAIT2);
2540 }
2541 }
2542
2543 mptcp_flush_join_list(msk);
2544 mptcp_for_each_subflow(msk, subflow) {
2545 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
2546
2547 mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN);
2548 }
2549}
2550
2551static void __mptcp_wr_shutdown(struct sock *sk)
2552{
2553 struct mptcp_sock *msk = mptcp_sk(sk);
2554
2555 pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d",
2556 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state,
2557 !!mptcp_send_head(sk));
2558
2559
2560 WRITE_ONCE(msk->write_seq, msk->write_seq + 1);
2561 WRITE_ONCE(msk->snd_data_fin_enable, 1);
2562
2563 __mptcp_check_send_data_fin(sk);
2564}
2565
2566static void __mptcp_destroy_sock(struct sock *sk)
2567{
2568 struct mptcp_subflow_context *subflow, *tmp;
2569 struct mptcp_sock *msk = mptcp_sk(sk);
2570 LIST_HEAD(conn_list);
2571
2572 pr_debug("msk=%p", msk);
2573
2574 might_sleep();
2575
2576
2577
2578
2579 spin_lock_bh(&msk->join_list_lock);
2580 list_splice_tail_init(&msk->join_list, &msk->conn_list);
2581 spin_unlock_bh(&msk->join_list_lock);
2582 list_splice_init(&msk->conn_list, &conn_list);
2583
2584 sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer);
2585 sk_stop_timer(sk, &sk->sk_timer);
2586 msk->pm.status = 0;
2587
2588 list_for_each_entry_safe(subflow, tmp, &conn_list, node) {
2589 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
2590 __mptcp_close_ssk(sk, ssk, subflow);
2591 }
2592
2593 sk->sk_prot->destroy(sk);
2594
2595 WARN_ON_ONCE(msk->wmem_reserved);
2596 WARN_ON_ONCE(msk->rmem_released);
2597 sk_stream_kill_queues(sk);
2598 xfrm_sk_free_policy(sk);
2599
2600 sk_refcnt_debug_release(sk);
2601 mptcp_dispose_initial_subflow(msk);
2602 sock_put(sk);
2603}
2604
2605static void mptcp_close(struct sock *sk, long timeout)
2606{
2607 struct mptcp_subflow_context *subflow;
2608 bool do_cancel_work = false;
2609
2610 lock_sock(sk);
2611 sk->sk_shutdown = SHUTDOWN_MASK;
2612
2613 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) {
2614 inet_sk_state_store(sk, TCP_CLOSE);
2615 goto cleanup;
2616 }
2617
2618 if (mptcp_close_state(sk))
2619 __mptcp_wr_shutdown(sk);
2620
2621 sk_stream_wait_close(sk, timeout);
2622
2623cleanup:
2624
2625 inet_csk(sk)->icsk_mtup.probe_timestamp = tcp_jiffies32;
2626 mptcp_for_each_subflow(mptcp_sk(sk), subflow) {
2627 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
2628 bool slow = lock_sock_fast(ssk);
2629
2630 sock_orphan(ssk);
2631 unlock_sock_fast(ssk, slow);
2632 }
2633 sock_orphan(sk);
2634
2635 sock_hold(sk);
2636 pr_debug("msk=%p state=%d", sk, sk->sk_state);
2637 if (sk->sk_state == TCP_CLOSE) {
2638 __mptcp_destroy_sock(sk);
2639 do_cancel_work = true;
2640 } else {
2641 sk_reset_timer(sk, &sk->sk_timer, jiffies + TCP_TIMEWAIT_LEN);
2642 }
2643 release_sock(sk);
2644 if (do_cancel_work)
2645 mptcp_cancel_work(sk);
2646
2647 if (mptcp_sk(sk)->token)
2648 mptcp_event(MPTCP_EVENT_CLOSED, mptcp_sk(sk), NULL, GFP_KERNEL);
2649
2650 sock_put(sk);
2651}
2652
2653static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
2654{
2655#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2656 const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
2657 struct ipv6_pinfo *msk6 = inet6_sk(msk);
2658
2659 msk->sk_v6_daddr = ssk->sk_v6_daddr;
2660 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;
2661
2662 if (msk6 && ssk6) {
2663 msk6->saddr = ssk6->saddr;
2664 msk6->flow_label = ssk6->flow_label;
2665 }
2666#endif
2667
2668 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
2669 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
2670 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
2671 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
2672 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
2673 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
2674}
2675
2676static int mptcp_disconnect(struct sock *sk, int flags)
2677{
2678 struct mptcp_subflow_context *subflow;
2679 struct mptcp_sock *msk = mptcp_sk(sk);
2680
2681 mptcp_do_flush_join_list(msk);
2682
2683 mptcp_for_each_subflow(msk, subflow) {
2684 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
2685
2686 lock_sock(ssk);
2687 tcp_disconnect(ssk, flags);
2688 release_sock(ssk);
2689 }
2690 return 0;
2691}
2692
2693#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2694static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
2695{
2696 unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo);
2697
2698 return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
2699}
2700#endif
2701
2702struct sock *mptcp_sk_clone(const struct sock *sk,
2703 const struct mptcp_options_received *mp_opt,
2704 struct request_sock *req)
2705{
2706 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
2707 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
2708 struct mptcp_sock *msk;
2709 u64 ack_seq;
2710
2711 if (!nsk)
2712 return NULL;
2713
2714#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2715 if (nsk->sk_family == AF_INET6)
2716 inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk);
2717#endif
2718
2719 __mptcp_init_sock(nsk);
2720
2721 msk = mptcp_sk(nsk);
2722 msk->local_key = subflow_req->local_key;
2723 msk->token = subflow_req->token;
2724 msk->subflow = NULL;
2725 WRITE_ONCE(msk->fully_established, false);
2726 if (mp_opt->csum_reqd)
2727 WRITE_ONCE(msk->csum_enabled, true);
2728
2729 msk->write_seq = subflow_req->idsn + 1;
2730 msk->snd_nxt = msk->write_seq;
2731 msk->snd_una = msk->write_seq;
2732 msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd;
2733 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq;
2734
2735 if (mp_opt->mp_capable) {
2736 msk->can_ack = true;
2737 msk->remote_key = mp_opt->sndr_key;
2738 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq);
2739 ack_seq++;
2740 WRITE_ONCE(msk->ack_seq, ack_seq);
2741 WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
2742 }
2743
2744 sock_reset_flag(nsk, SOCK_RCU_FREE);
2745
2746 inet_sk_state_store(nsk, TCP_SYN_RECV);
2747
2748 security_inet_csk_clone(nsk, req);
2749 bh_unlock_sock(nsk);
2750
2751
2752 __sock_put(nsk);
2753 return nsk;
2754}
2755
2756void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)
2757{
2758 const struct tcp_sock *tp = tcp_sk(ssk);
2759
2760 msk->rcvq_space.copied = 0;
2761 msk->rcvq_space.rtt_us = 0;
2762
2763 msk->rcvq_space.time = tp->tcp_mstamp;
2764
2765
2766 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd,
2767 TCP_INIT_CWND * tp->advmss);
2768 if (msk->rcvq_space.space == 0)
2769 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT;
2770
2771 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd);
2772}
2773
2774static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
2775 bool kern)
2776{
2777 struct mptcp_sock *msk = mptcp_sk(sk);
2778 struct socket *listener;
2779 struct sock *newsk;
2780
2781 listener = __mptcp_nmpc_socket(msk);
2782 if (WARN_ON_ONCE(!listener)) {
2783 *err = -EINVAL;
2784 return NULL;
2785 }
2786
2787 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
2788 newsk = inet_csk_accept(listener->sk, flags, err, kern);
2789 if (!newsk)
2790 return NULL;
2791
2792 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));
2793 if (sk_is_mptcp(newsk)) {
2794 struct mptcp_subflow_context *subflow;
2795 struct sock *new_mptcp_sock;
2796
2797 subflow = mptcp_subflow_ctx(newsk);
2798 new_mptcp_sock = subflow->conn;
2799
2800
2801
2802
2803 if (WARN_ON_ONCE(!new_mptcp_sock)) {
2804 tcp_sk(newsk)->is_mptcp = 0;
2805 return newsk;
2806 }
2807
2808
2809 sock_hold(new_mptcp_sock);
2810 newsk = new_mptcp_sock;
2811 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK);
2812 } else {
2813 MPTCP_INC_STATS(sock_net(sk),
2814 MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK);
2815 }
2816
2817 return newsk;
2818}
2819
2820void mptcp_destroy_common(struct mptcp_sock *msk)
2821{
2822 struct sock *sk = (struct sock *)msk;
2823
2824 __mptcp_clear_xmit(sk);
2825
2826
2827 skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue);
2828
2829 skb_rbtree_purge(&msk->out_of_order_queue);
2830 mptcp_token_destroy(msk);
2831 mptcp_pm_free_anno_list(msk);
2832}
2833
2834static void mptcp_destroy(struct sock *sk)
2835{
2836 struct mptcp_sock *msk = mptcp_sk(sk);
2837
2838 mptcp_destroy_common(msk);
2839 sk_sockets_allocated_dec(sk);
2840}
2841
2842void __mptcp_data_acked(struct sock *sk)
2843{
2844 if (!sock_owned_by_user(sk))
2845 __mptcp_clean_una(sk);
2846 else
2847 set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags);
2848
2849 if (mptcp_pending_data_fin_ack(sk))
2850 mptcp_schedule_work(sk);
2851}
2852
2853void __mptcp_check_push(struct sock *sk, struct sock *ssk)
2854{
2855 if (!mptcp_send_head(sk))
2856 return;
2857
2858 if (!sock_owned_by_user(sk)) {
2859 struct sock *xmit_ssk = mptcp_subflow_get_send(mptcp_sk(sk));
2860
2861 if (xmit_ssk == ssk)
2862 __mptcp_subflow_push_pending(sk, ssk);
2863 else if (xmit_ssk)
2864 mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk));
2865 } else {
2866 set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags);
2867 }
2868}
2869
2870
2871static void mptcp_release_cb(struct sock *sk)
2872{
2873 for (;;) {
2874 unsigned long flags = 0;
2875
2876 if (test_and_clear_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags))
2877 flags |= BIT(MPTCP_PUSH_PENDING);
2878 if (test_and_clear_bit(MPTCP_RETRANSMIT, &mptcp_sk(sk)->flags))
2879 flags |= BIT(MPTCP_RETRANSMIT);
2880 if (!flags)
2881 break;
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891 spin_unlock_bh(&sk->sk_lock.slock);
2892 if (flags & BIT(MPTCP_PUSH_PENDING))
2893 __mptcp_push_pending(sk, 0);
2894 if (flags & BIT(MPTCP_RETRANSMIT))
2895 __mptcp_retrans(sk);
2896
2897 cond_resched();
2898 spin_lock_bh(&sk->sk_lock.slock);
2899 }
2900
2901
2902
2903
2904 if (test_and_clear_bit(MPTCP_CONNECTED, &mptcp_sk(sk)->flags))
2905 __mptcp_set_connected(sk);
2906 if (test_and_clear_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->flags))
2907 __mptcp_clean_una_wakeup(sk);
2908 if (test_and_clear_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->flags))
2909 __mptcp_error_report(sk);
2910
2911
2912
2913
2914 __mptcp_update_wmem(sk);
2915 __mptcp_update_rmem(sk);
2916}
2917
2918void mptcp_subflow_process_delegated(struct sock *ssk)
2919{
2920 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
2921 struct sock *sk = subflow->conn;
2922
2923 mptcp_data_lock(sk);
2924 if (!sock_owned_by_user(sk))
2925 __mptcp_subflow_push_pending(sk, ssk);
2926 else
2927 set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags);
2928 mptcp_data_unlock(sk);
2929 mptcp_subflow_delegated_done(subflow);
2930}
2931
2932static int mptcp_hash(struct sock *sk)
2933{
2934
2935
2936
2937 WARN_ON_ONCE(1);
2938 return 0;
2939}
2940
2941static void mptcp_unhash(struct sock *sk)
2942{
2943
2944}
2945
2946static int mptcp_get_port(struct sock *sk, unsigned short snum)
2947{
2948 struct mptcp_sock *msk = mptcp_sk(sk);
2949 struct socket *ssock;
2950
2951 ssock = __mptcp_nmpc_socket(msk);
2952 pr_debug("msk=%p, subflow=%p", msk, ssock);
2953 if (WARN_ON_ONCE(!ssock))
2954 return -EINVAL;
2955
2956 return inet_csk_get_port(ssock->sk, snum);
2957}
2958
2959void mptcp_finish_connect(struct sock *ssk)
2960{
2961 struct mptcp_subflow_context *subflow;
2962 struct mptcp_sock *msk;
2963 struct sock *sk;
2964 u64 ack_seq;
2965
2966 subflow = mptcp_subflow_ctx(ssk);
2967 sk = subflow->conn;
2968 msk = mptcp_sk(sk);
2969
2970 pr_debug("msk=%p, token=%u", sk, subflow->token);
2971
2972 mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq);
2973 ack_seq++;
2974 subflow->map_seq = ack_seq;
2975 subflow->map_subflow_seq = 1;
2976
2977
2978
2979
2980 WRITE_ONCE(msk->remote_key, subflow->remote_key);
2981 WRITE_ONCE(msk->local_key, subflow->local_key);
2982 WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
2983 WRITE_ONCE(msk->snd_nxt, msk->write_seq);
2984 WRITE_ONCE(msk->ack_seq, ack_seq);
2985 WRITE_ONCE(msk->rcv_wnd_sent, ack_seq);
2986 WRITE_ONCE(msk->can_ack, 1);
2987 WRITE_ONCE(msk->snd_una, msk->write_seq);
2988
2989 mptcp_pm_new_connection(msk, ssk, 0);
2990
2991 mptcp_rcv_space_init(msk, ssk);
2992}
2993
2994void mptcp_sock_graft(struct sock *sk, struct socket *parent)
2995{
2996 write_lock_bh(&sk->sk_callback_lock);
2997 rcu_assign_pointer(sk->sk_wq, &parent->wq);
2998 sk_set_socket(sk, parent);
2999 sk->sk_uid = SOCK_INODE(parent)->i_uid;
3000 write_unlock_bh(&sk->sk_callback_lock);
3001}
3002
3003bool mptcp_finish_join(struct sock *ssk)
3004{
3005 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
3006 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
3007 struct sock *parent = (void *)msk;
3008 struct socket *parent_sock;
3009 bool ret;
3010
3011 pr_debug("msk=%p, subflow=%p", msk, subflow);
3012
3013
3014 if (!mptcp_is_fully_established(parent)) {
3015 subflow->reset_reason = MPTCP_RST_EMPTCP;
3016 return false;
3017 }
3018
3019 if (!msk->pm.server_side)
3020 goto out;
3021
3022 if (!mptcp_pm_allow_new_subflow(msk)) {
3023 subflow->reset_reason = MPTCP_RST_EPROHIBIT;
3024 return false;
3025 }
3026
3027
3028
3029
3030
3031
3032 spin_lock_bh(&msk->join_list_lock);
3033 ret = inet_sk_state_load(parent) == TCP_ESTABLISHED;
3034 if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node))) {
3035 list_add_tail(&subflow->node, &msk->join_list);
3036 sock_hold(ssk);
3037 }
3038 spin_unlock_bh(&msk->join_list_lock);
3039 if (!ret) {
3040 subflow->reset_reason = MPTCP_RST_EPROHIBIT;
3041 return false;
3042 }
3043
3044
3045
3046
3047 parent_sock = READ_ONCE(parent->sk_socket);
3048 if (parent_sock && !ssk->sk_socket)
3049 mptcp_sock_graft(ssk, parent_sock);
3050 subflow->map_seq = READ_ONCE(msk->ack_seq);
3051out:
3052 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC);
3053 return true;
3054}
3055
3056static void mptcp_shutdown(struct sock *sk, int how)
3057{
3058 pr_debug("sk=%p, how=%d", sk, how);
3059
3060 if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk))
3061 __mptcp_wr_shutdown(sk);
3062}
3063
3064static struct proto mptcp_prot = {
3065 .name = "MPTCP",
3066 .owner = THIS_MODULE,
3067 .init = mptcp_init_sock,
3068 .disconnect = mptcp_disconnect,
3069 .close = mptcp_close,
3070 .accept = mptcp_accept,
3071 .setsockopt = mptcp_setsockopt,
3072 .getsockopt = mptcp_getsockopt,
3073 .shutdown = mptcp_shutdown,
3074 .destroy = mptcp_destroy,
3075 .sendmsg = mptcp_sendmsg,
3076 .recvmsg = mptcp_recvmsg,
3077 .release_cb = mptcp_release_cb,
3078 .hash = mptcp_hash,
3079 .unhash = mptcp_unhash,
3080 .get_port = mptcp_get_port,
3081 .sockets_allocated = &mptcp_sockets_allocated,
3082 .memory_allocated = &tcp_memory_allocated,
3083 .memory_pressure = &tcp_memory_pressure,
3084 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
3085 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
3086 .sysctl_mem = sysctl_tcp_mem,
3087 .obj_size = sizeof(struct mptcp_sock),
3088 .slab_flags = SLAB_TYPESAFE_BY_RCU,
3089 .no_autobind = true,
3090};
3091
3092static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
3093{
3094 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3095 struct socket *ssock;
3096 int err;
3097
3098 lock_sock(sock->sk);
3099 ssock = __mptcp_nmpc_socket(msk);
3100 if (!ssock) {
3101 err = -EINVAL;
3102 goto unlock;
3103 }
3104
3105 err = ssock->ops->bind(ssock, uaddr, addr_len);
3106 if (!err)
3107 mptcp_copy_inaddrs(sock->sk, ssock->sk);
3108
3109unlock:
3110 release_sock(sock->sk);
3111 return err;
3112}
3113
3114static void mptcp_subflow_early_fallback(struct mptcp_sock *msk,
3115 struct mptcp_subflow_context *subflow)
3116{
3117 subflow->request_mptcp = 0;
3118 __mptcp_do_fallback(msk);
3119}
3120
3121static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
3122 int addr_len, int flags)
3123{
3124 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3125 struct mptcp_subflow_context *subflow;
3126 struct socket *ssock;
3127 int err;
3128
3129 lock_sock(sock->sk);
3130 if (sock->state != SS_UNCONNECTED && msk->subflow) {
3131
3132
3133
3134 ssock = msk->subflow;
3135 goto do_connect;
3136 }
3137
3138 ssock = __mptcp_nmpc_socket(msk);
3139 if (!ssock) {
3140 err = -EINVAL;
3141 goto unlock;
3142 }
3143
3144 mptcp_token_destroy(msk);
3145 inet_sk_state_store(sock->sk, TCP_SYN_SENT);
3146 subflow = mptcp_subflow_ctx(ssock->sk);
3147#ifdef CONFIG_TCP_MD5SIG
3148
3149
3150
3151 if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
3152 mptcp_subflow_early_fallback(msk, subflow);
3153#endif
3154 if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) {
3155 MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT);
3156 mptcp_subflow_early_fallback(msk, subflow);
3157 }
3158 if (likely(!__mptcp_check_fallback(msk)))
3159 MPTCP_INC_STATS(sock_net(sock->sk), MPTCP_MIB_MPCAPABLEACTIVE);
3160
3161do_connect:
3162 err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
3163 sock->state = ssock->state;
3164
3165
3166
3167
3168 if (!err || err == -EINPROGRESS)
3169 mptcp_copy_inaddrs(sock->sk, ssock->sk);
3170 else
3171 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
3172
3173unlock:
3174 release_sock(sock->sk);
3175 return err;
3176}
3177
3178static int mptcp_listen(struct socket *sock, int backlog)
3179{
3180 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3181 struct socket *ssock;
3182 int err;
3183
3184 pr_debug("msk=%p", msk);
3185
3186 lock_sock(sock->sk);
3187 ssock = __mptcp_nmpc_socket(msk);
3188 if (!ssock) {
3189 err = -EINVAL;
3190 goto unlock;
3191 }
3192
3193 mptcp_token_destroy(msk);
3194 inet_sk_state_store(sock->sk, TCP_LISTEN);
3195 sock_set_flag(sock->sk, SOCK_RCU_FREE);
3196
3197 err = ssock->ops->listen(ssock, backlog);
3198 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
3199 if (!err)
3200 mptcp_copy_inaddrs(sock->sk, ssock->sk);
3201
3202unlock:
3203 release_sock(sock->sk);
3204 return err;
3205}
3206
3207static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
3208 int flags, bool kern)
3209{
3210 struct mptcp_sock *msk = mptcp_sk(sock->sk);
3211 struct socket *ssock;
3212 int err;
3213
3214 pr_debug("msk=%p", msk);
3215
3216 lock_sock(sock->sk);
3217 if (sock->sk->sk_state != TCP_LISTEN)
3218 goto unlock_fail;
3219
3220 ssock = __mptcp_nmpc_socket(msk);
3221 if (!ssock)
3222 goto unlock_fail;
3223
3224 clear_bit(MPTCP_DATA_READY, &msk->flags);
3225 sock_hold(ssock->sk);
3226 release_sock(sock->sk);
3227
3228 err = ssock->ops->accept(sock, newsock, flags, kern);
3229 if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) {
3230 struct mptcp_sock *msk = mptcp_sk(newsock->sk);
3231 struct mptcp_subflow_context *subflow;
3232 struct sock *newsk = newsock->sk;
3233
3234 lock_sock(newsk);
3235
3236
3237
3238
3239
3240
3241
3242
3243 subflow = mptcp_subflow_ctx(msk->first);
3244 list_add(&subflow->node, &msk->conn_list);
3245 sock_hold(msk->first);
3246 if (mptcp_is_fully_established(newsk))
3247 mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL);
3248
3249 mptcp_copy_inaddrs(newsk, msk->first);
3250 mptcp_rcv_space_init(msk, msk->first);
3251 mptcp_propagate_sndbuf(newsk, msk->first);
3252
3253
3254
3255
3256 mptcp_flush_join_list(msk);
3257 mptcp_for_each_subflow(msk, subflow) {
3258 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
3259
3260 if (!ssk->sk_socket)
3261 mptcp_sock_graft(ssk, newsock);
3262 }
3263 release_sock(newsk);
3264 }
3265
3266 if (inet_csk_listen_poll(ssock->sk))
3267 set_bit(MPTCP_DATA_READY, &msk->flags);
3268 sock_put(ssock->sk);
3269 return err;
3270
3271unlock_fail:
3272 release_sock(sock->sk);
3273 return -EINVAL;
3274}
3275
3276static __poll_t mptcp_check_readable(struct mptcp_sock *msk)
3277{
3278 return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM :
3279 0;
3280}
3281
3282static __poll_t mptcp_check_writeable(struct mptcp_sock *msk)
3283{
3284 struct sock *sk = (struct sock *)msk;
3285
3286 if (unlikely(sk->sk_shutdown & SEND_SHUTDOWN))
3287 return EPOLLOUT | EPOLLWRNORM;
3288
3289 if (sk_stream_is_writeable(sk))
3290 return EPOLLOUT | EPOLLWRNORM;
3291
3292 mptcp_set_nospace(sk);
3293 smp_mb__after_atomic();
3294 if (sk_stream_is_writeable(sk))
3295 return EPOLLOUT | EPOLLWRNORM;
3296
3297 return 0;
3298}
3299
3300static __poll_t mptcp_poll(struct file *file, struct socket *sock,
3301 struct poll_table_struct *wait)
3302{
3303 struct sock *sk = sock->sk;
3304 struct mptcp_sock *msk;
3305 __poll_t mask = 0;
3306 int state;
3307
3308 msk = mptcp_sk(sk);
3309 sock_poll_wait(file, sock, wait);
3310
3311 state = inet_sk_state_load(sk);
3312 pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags);
3313 if (state == TCP_LISTEN)
3314 return mptcp_check_readable(msk);
3315
3316 if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) {
3317 mask |= mptcp_check_readable(msk);
3318 mask |= mptcp_check_writeable(msk);
3319 }
3320 if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
3321 mask |= EPOLLHUP;
3322 if (sk->sk_shutdown & RCV_SHUTDOWN)
3323 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
3324
3325
3326 smp_rmb();
3327 if (sk->sk_err)
3328 mask |= EPOLLERR;
3329
3330 return mask;
3331}
3332
3333static const struct proto_ops mptcp_stream_ops = {
3334 .family = PF_INET,
3335 .owner = THIS_MODULE,
3336 .release = inet_release,
3337 .bind = mptcp_bind,
3338 .connect = mptcp_stream_connect,
3339 .socketpair = sock_no_socketpair,
3340 .accept = mptcp_stream_accept,
3341 .getname = inet_getname,
3342 .poll = mptcp_poll,
3343 .ioctl = inet_ioctl,
3344 .gettstamp = sock_gettstamp,
3345 .listen = mptcp_listen,
3346 .shutdown = inet_shutdown,
3347 .setsockopt = sock_common_setsockopt,
3348 .getsockopt = sock_common_getsockopt,
3349 .sendmsg = inet_sendmsg,
3350 .recvmsg = inet_recvmsg,
3351 .mmap = sock_no_mmap,
3352 .sendpage = inet_sendpage,
3353};
3354
3355static struct inet_protosw mptcp_protosw = {
3356 .type = SOCK_STREAM,
3357 .protocol = IPPROTO_MPTCP,
3358 .prot = &mptcp_prot,
3359 .ops = &mptcp_stream_ops,
3360 .flags = INET_PROTOSW_ICSK,
3361};
3362
3363static int mptcp_napi_poll(struct napi_struct *napi, int budget)
3364{
3365 struct mptcp_delegated_action *delegated;
3366 struct mptcp_subflow_context *subflow;
3367 int work_done = 0;
3368
3369 delegated = container_of(napi, struct mptcp_delegated_action, napi);
3370 while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) {
3371 struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
3372
3373 bh_lock_sock_nested(ssk);
3374 if (!sock_owned_by_user(ssk) &&
3375 mptcp_subflow_has_delegated_action(subflow))
3376 mptcp_subflow_process_delegated(ssk);
3377
3378
3379
3380
3381
3382 bh_unlock_sock(ssk);
3383 sock_put(ssk);
3384
3385 if (++work_done == budget)
3386 return budget;
3387 }
3388
3389
3390
3391
3392 napi_complete_done(napi, 0);
3393 return work_done;
3394}
3395
3396void __init mptcp_proto_init(void)
3397{
3398 struct mptcp_delegated_action *delegated;
3399 int cpu;
3400
3401 mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
3402
3403 if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL))
3404 panic("Failed to allocate MPTCP pcpu counter\n");
3405
3406 init_dummy_netdev(&mptcp_napi_dev);
3407 for_each_possible_cpu(cpu) {
3408 delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu);
3409 INIT_LIST_HEAD(&delegated->head);
3410 netif_tx_napi_add(&mptcp_napi_dev, &delegated->napi, mptcp_napi_poll,
3411 NAPI_POLL_WEIGHT);
3412 napi_enable(&delegated->napi);
3413 }
3414
3415 mptcp_subflow_init();
3416 mptcp_pm_init();
3417 mptcp_token_init();
3418
3419 if (proto_register(&mptcp_prot, 1) != 0)
3420 panic("Failed to register MPTCP proto.\n");
3421
3422 inet_register_protosw(&mptcp_protosw);
3423
3424 BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb));
3425}
3426
3427#if IS_ENABLED(CONFIG_MPTCP_IPV6)
3428static const struct proto_ops mptcp_v6_stream_ops = {
3429 .family = PF_INET6,
3430 .owner = THIS_MODULE,
3431 .release = inet6_release,
3432 .bind = mptcp_bind,
3433 .connect = mptcp_stream_connect,
3434 .socketpair = sock_no_socketpair,
3435 .accept = mptcp_stream_accept,
3436 .getname = inet6_getname,
3437 .poll = mptcp_poll,
3438 .ioctl = inet6_ioctl,
3439 .gettstamp = sock_gettstamp,
3440 .listen = mptcp_listen,
3441 .shutdown = inet_shutdown,
3442 .setsockopt = sock_common_setsockopt,
3443 .getsockopt = sock_common_getsockopt,
3444 .sendmsg = inet6_sendmsg,
3445 .recvmsg = inet6_recvmsg,
3446 .mmap = sock_no_mmap,
3447 .sendpage = inet_sendpage,
3448#ifdef CONFIG_COMPAT
3449 .compat_ioctl = inet6_compat_ioctl,
3450#endif
3451};
3452
3453static struct proto mptcp_v6_prot;
3454
3455static void mptcp_v6_destroy(struct sock *sk)
3456{
3457 mptcp_destroy(sk);
3458 inet6_destroy_sock(sk);
3459}
3460
3461static struct inet_protosw mptcp_v6_protosw = {
3462 .type = SOCK_STREAM,
3463 .protocol = IPPROTO_MPTCP,
3464 .prot = &mptcp_v6_prot,
3465 .ops = &mptcp_v6_stream_ops,
3466 .flags = INET_PROTOSW_ICSK,
3467};
3468
3469int __init mptcp_proto_v6_init(void)
3470{
3471 int err;
3472
3473 mptcp_v6_prot = mptcp_prot;
3474 strcpy(mptcp_v6_prot.name, "MPTCPv6");
3475 mptcp_v6_prot.slab = NULL;
3476 mptcp_v6_prot.destroy = mptcp_v6_destroy;
3477 mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock);
3478
3479 err = proto_register(&mptcp_v6_prot, 1);
3480 if (err)
3481 return err;
3482
3483 err = inet6_register_protosw(&mptcp_v6_protosw);
3484 if (err)
3485 proto_unregister(&mptcp_v6_prot);
3486
3487 return err;
3488}
3489#endif
3490