#include "protocol.h"
#include "mib.h"
+#define CREATE_TRACE_POINTS
+#include <trace/events/mptcp.h>
+
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
struct mptcp6_sock {
struct mptcp_sock msk;
return false;
}
-static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk)
-{
- sock_owned_by_me((const struct sock *)msk);
-
- if (likely(!__mptcp_check_fallback(msk)))
- return NULL;
-
- return msk->first;
-}
-
static int __mptcp_socket_create(struct mptcp_sock *msk)
{
struct mptcp_subflow_context *subflow;
return false;
}
+static void mptcp_set_datafin_timeout(const struct sock *sk)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ mptcp_sk(sk)->timer_ival = min(TCP_RTO_MAX,
+ TCP_RTO_MIN << icsk->icsk_retransmits);
+}
+
static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk)
{
long tout = ssk && inet_csk(ssk)->icsk_pending ?
mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
}
-static bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
-{
- struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
-
- /* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */
- if (subflow->request_join && !subflow->fully_established)
- return false;
-
- /* only send if our side has not closed yet */
- return ((1 << ssk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT));
-}
-
static bool tcp_can_send_ack(const struct sock *ssk)
{
return !((1 << inet_sk_state_load(ssk)) &
u64 rcv_data_fin_seq;
bool ret = false;
- if (__mptcp_check_fallback(msk) || !msk->first)
+ if (__mptcp_check_fallback(msk))
return ret;
/* Need to ack a DATA_FIN received from a peer while this side
sk->sk_data_ready(sk);
}
-void __mptcp_flush_join_list(struct mptcp_sock *msk)
+static bool mptcp_do_flush_join_list(struct mptcp_sock *msk)
{
struct mptcp_subflow_context *subflow;
+ bool ret = false;
if (likely(list_empty(&msk->join_list)))
- return;
+ return false;
spin_lock_bh(&msk->join_list_lock);
- list_for_each_entry(subflow, &msk->join_list, node)
+ list_for_each_entry(subflow, &msk->join_list, node) {
+ u32 sseq = READ_ONCE(subflow->setsockopt_seq);
+
mptcp_propagate_sndbuf((struct sock *)msk, mptcp_subflow_tcp_sock(subflow));
+ if (READ_ONCE(msk->setsockopt_seq) != sseq)
+ ret = true;
+ }
list_splice_tail_init(&msk->join_list, &msk->conn_list);
spin_unlock_bh(&msk->join_list_lock);
+
+ return ret;
+}
+
+void __mptcp_flush_join_list(struct mptcp_sock *msk)
+{
+ if (likely(!mptcp_do_flush_join_list(msk)))
+ return;
+
+ if (!test_and_set_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags))
+ mptcp_schedule_work((struct sock *)msk);
+}
+
+static void mptcp_flush_join_list(struct mptcp_sock *msk)
+{
+ bool sync_needed = test_and_clear_bit(MPTCP_WORK_SYNC_SETSOCKOPT, &msk->flags);
+
+ might_sleep();
+
+ if (!mptcp_do_flush_join_list(msk) && !sync_needed)
+ return;
+
+ mptcp_sockopt_sync_all(msk);
}
static bool mptcp_timer_pending(struct sock *sk)
}
if (snd_una == READ_ONCE(msk->snd_nxt)) {
- if (msk->timer_ival)
+ if (msk->timer_ival && !mptcp_data_fin_enabled(msk))
mptcp_stop_timer(sk);
} else {
mptcp_reset_timer(sk);
int avail_size;
size_t ret = 0;
- pr_debug("msk=%p ssk=%p sending dfrag at seq=%lld len=%d already sent=%d",
+ pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u",
msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);
/* compute send limit */
send_info[i].ratio = -1;
}
mptcp_for_each_subflow(msk, subflow) {
+ trace_mptcp_subflow_get_send(subflow);
ssk = mptcp_subflow_tcp_sock(subflow);
if (!mptcp_subflow_active(subflow))
continue;
}
}
- pr_debug("msk=%p nr_active=%d ssk=%p:%lld backup=%p:%lld",
- msk, nr_active, send_info[0].ssk, send_info[0].ratio,
- send_info[1].ssk, send_info[1].ratio);
-
/* pick the best backup if no other subflow is active */
if (!nr_active)
send_info[0].ssk = send_info[1].ssk;
int ret = 0;
prev_ssk = ssk;
- __mptcp_flush_join_list(msk);
+ mptcp_flush_join_list(msk);
ssk = mptcp_subflow_get_send(msk);
/* try to keep the subflow socket lock across
int ret = 0;
long timeo;
- if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
+ /* we don't support FASTOPEN yet */
+ if (msg->msg_flags & MSG_FASTOPEN)
return -EOPNOTSUPP;
+ /* silently ignore everything else */
+ msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL;
+
mptcp_lock_sock(sk, __mptcp_wmem_reserve(sk, min_t(size_t, 1 << 20, len)));
timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
if (!msk->first_pending)
WRITE_ONCE(msk->first_pending, dfrag);
}
- pr_debug("msk=%p dfrag at seq=%lld len=%d sent=%d new=%d", msk,
+ pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk,
dfrag->data_seq, dfrag->data_len, dfrag->already_sent,
!dfrag_collapsed);
static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
struct msghdr *msg,
- size_t len)
+ size_t len, int flags)
{
- struct sk_buff *skb;
+ struct sk_buff *skb, *tmp;
int copied = 0;
- while ((skb = skb_peek(&msk->receive_queue)) != NULL) {
+ skb_queue_walk_safe(&msk->receive_queue, skb, tmp) {
u32 offset = MPTCP_SKB_CB(skb)->offset;
u32 data_len = skb->len - offset;
u32 count = min_t(size_t, len - copied, data_len);
int err;
- err = skb_copy_datagram_msg(skb, offset, msg, count);
- if (unlikely(err < 0)) {
- if (!copied)
- return err;
- break;
+ if (!(flags & MSG_TRUNC)) {
+ err = skb_copy_datagram_msg(skb, offset, msg, count);
+ if (unlikely(err < 0)) {
+ if (!copied)
+ return err;
+ break;
+ }
}
copied += count;
if (count < data_len) {
- MPTCP_SKB_CB(skb)->offset += count;
+ if (!(flags & MSG_PEEK))
+ MPTCP_SKB_CB(skb)->offset += count;
break;
}
- /* we will bulk release the skb memory later */
- skb->destructor = NULL;
- msk->rmem_released += skb->truesize;
- __skb_unlink(skb, &msk->receive_queue);
- __kfree_skb(skb);
+ if (!(flags & MSG_PEEK)) {
+ /* we will bulk release the skb memory later */
+ skb->destructor = NULL;
+ msk->rmem_released += skb->truesize;
+ __skb_unlink(skb, &msk->receive_queue);
+ __kfree_skb(skb);
+ }
if (copied >= len)
break;
unsigned int moved = 0;
bool ret, done;
- __mptcp_flush_join_list(msk);
+ mptcp_flush_join_list(msk);
do {
struct sock *ssk = mptcp_subflow_recv_lookup(msk);
bool slowpath;
int target;
long timeo;
- if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT))
- return -EOPNOTSUPP;
+ /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */
+ if (unlikely(flags & MSG_ERRQUEUE))
+ return inet_recv_error(sk, msg, len, addr_len);
mptcp_lock_sock(sk, __mptcp_splice_receive_queue(sk));
if (unlikely(sk->sk_state == TCP_LISTEN)) {
while (copied < len) {
int bytes_read;
- bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied);
+ bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags);
if (unlikely(bytes_read < 0)) {
if (!copied)
copied = bytes_read;
pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d",
msk, test_bit(MPTCP_DATA_READY, &msk->flags),
skb_queue_empty_lockless(&sk->sk_receive_queue), copied);
- mptcp_rcv_space_adjust(msk, copied);
+ if (!(flags & MSG_PEEK))
+ mptcp_rcv_space_adjust(msk, copied);
release_sock(sk);
return copied;
}
-static void mptcp_retransmit_handler(struct sock *sk)
-{
- struct mptcp_sock *msk = mptcp_sk(sk);
-
- set_bit(MPTCP_WORK_RTX, &msk->flags);
- mptcp_schedule_work(sk);
-}
-
static void mptcp_retransmit_timer(struct timer_list *t)
{
struct inet_connection_sock *icsk = from_timer(icsk, t,
icsk_retransmit_timer);
struct sock *sk = &icsk->icsk_inet.sk;
+ struct mptcp_sock *msk = mptcp_sk(sk);
bh_lock_sock(sk);
if (!sock_owned_by_user(sk)) {
- mptcp_retransmit_handler(sk);
+ /* we need a process context to retransmit */
+ if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags))
+ mptcp_schedule_work(sk);
} else {
/* delegate our work to tcp_release_cb() */
- if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED,
- &sk->sk_tsq_flags))
- sock_hold(sk);
+ set_bit(MPTCP_RETRANSMIT, &msk->flags);
}
bh_unlock_sock(sk);
sock_put(sk);
__mptcp_clean_una_wakeup(sk);
dfrag = mptcp_rtx_head(sk);
- if (!dfrag)
+ if (!dfrag) {
+ if (mptcp_data_fin_enabled(msk)) {
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ icsk->icsk_retransmits++;
+ mptcp_set_datafin_timeout(sk);
+ mptcp_send_ack(msk);
+
+ goto reset_timer;
+ }
+
return;
+ }
ssk = mptcp_subflow_get_retrans(msk);
if (!ssk)
goto unlock;
mptcp_check_data_fin_ack(sk);
- __mptcp_flush_join_list(msk);
+ mptcp_flush_join_list(msk);
mptcp_check_fastclose(msk);
/* re-use the csk retrans timer for MPTCP-level retrans */
timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);
timer_setup(&sk->sk_timer, mptcp_timeout_timer, 0);
+
+ tcp_assign_congestion_control(sk);
+
return 0;
}
pr_debug("Sending DATA_FIN on subflow %p", ssk);
mptcp_set_timeout(sk, ssk);
tcp_send_ack(ssk);
+ if (!mptcp_timer_pending(sk))
+ mptcp_reset_timer(sk);
}
break;
}
}
}
- __mptcp_flush_join_list(msk);
+ mptcp_flush_join_list(msk);
mptcp_for_each_subflow(msk, subflow) {
struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
WARN_ON_ONCE(msk->rmem_released);
sk_stream_kill_queues(sk);
xfrm_sk_free_policy(sk);
+
+ tcp_cleanup_congestion_control(sk);
sk_refcnt_debug_release(sk);
mptcp_dispose_initial_subflow(msk);
sock_put(sk);
cleanup:
/* orphan all the subflows */
inet_csk(sk)->icsk_mtup.probe_timestamp = tcp_jiffies32;
- list_for_each_entry(subflow, &mptcp_sk(sk)->conn_list, node) {
+ mptcp_for_each_subflow(mptcp_sk(sk), subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
bool slow = lock_sock_fast(ssk);
struct mptcp_subflow_context *subflow;
struct mptcp_sock *msk = mptcp_sk(sk);
- __mptcp_flush_join_list(msk);
+ mptcp_do_flush_join_list(msk);
+
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
msk->snd_nxt = msk->write_seq;
msk->snd_una = msk->write_seq;
msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd;
+ msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq;
if (mp_opt->mp_capable) {
msk->can_ack = true;
sk_sockets_allocated_dec(sk);
}
-static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname,
- sockptr_t optval, unsigned int optlen)
-{
- struct sock *sk = (struct sock *)msk;
- struct socket *ssock;
- int ret;
-
- switch (optname) {
- case SO_REUSEPORT:
- case SO_REUSEADDR:
- lock_sock(sk);
- ssock = __mptcp_nmpc_socket(msk);
- if (!ssock) {
- release_sock(sk);
- return -EINVAL;
- }
-
- ret = sock_setsockopt(ssock, SOL_SOCKET, optname, optval, optlen);
- if (ret == 0) {
- if (optname == SO_REUSEPORT)
- sk->sk_reuseport = ssock->sk->sk_reuseport;
- else if (optname == SO_REUSEADDR)
- sk->sk_reuse = ssock->sk->sk_reuse;
- }
- release_sock(sk);
- return ret;
- }
-
- return sock_setsockopt(sk->sk_socket, SOL_SOCKET, optname, optval, optlen);
-}
-
-static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname,
- sockptr_t optval, unsigned int optlen)
-{
- struct sock *sk = (struct sock *)msk;
- int ret = -EOPNOTSUPP;
- struct socket *ssock;
-
- switch (optname) {
- case IPV6_V6ONLY:
- lock_sock(sk);
- ssock = __mptcp_nmpc_socket(msk);
- if (!ssock) {
- release_sock(sk);
- return -EINVAL;
- }
-
- ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen);
- if (ret == 0)
- sk->sk_ipv6only = ssock->sk->sk_ipv6only;
-
- release_sock(sk);
- break;
- }
-
- return ret;
-}
-
-static bool mptcp_unsupported(int level, int optname)
-{
- if (level == SOL_IP) {
- switch (optname) {
- case IP_ADD_MEMBERSHIP:
- case IP_ADD_SOURCE_MEMBERSHIP:
- case IP_DROP_MEMBERSHIP:
- case IP_DROP_SOURCE_MEMBERSHIP:
- case IP_BLOCK_SOURCE:
- case IP_UNBLOCK_SOURCE:
- case MCAST_JOIN_GROUP:
- case MCAST_LEAVE_GROUP:
- case MCAST_JOIN_SOURCE_GROUP:
- case MCAST_LEAVE_SOURCE_GROUP:
- case MCAST_BLOCK_SOURCE:
- case MCAST_UNBLOCK_SOURCE:
- case MCAST_MSFILTER:
- return true;
- }
- return false;
- }
- if (level == SOL_IPV6) {
- switch (optname) {
- case IPV6_ADDRFORM:
- case IPV6_ADD_MEMBERSHIP:
- case IPV6_DROP_MEMBERSHIP:
- case IPV6_JOIN_ANYCAST:
- case IPV6_LEAVE_ANYCAST:
- case MCAST_JOIN_GROUP:
- case MCAST_LEAVE_GROUP:
- case MCAST_JOIN_SOURCE_GROUP:
- case MCAST_LEAVE_SOURCE_GROUP:
- case MCAST_BLOCK_SOURCE:
- case MCAST_UNBLOCK_SOURCE:
- case MCAST_MSFILTER:
- return true;
- }
- return false;
- }
- return false;
-}
-
-static int mptcp_setsockopt(struct sock *sk, int level, int optname,
- sockptr_t optval, unsigned int optlen)
-{
- struct mptcp_sock *msk = mptcp_sk(sk);
- struct sock *ssk;
-
- pr_debug("msk=%p", msk);
-
- if (mptcp_unsupported(level, optname))
- return -ENOPROTOOPT;
-
- if (level == SOL_SOCKET)
- return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen);
-
- /* @@ the meaning of setsockopt() when the socket is connected and
- * there are multiple subflows is not yet defined. It is up to the
- * MPTCP-level socket to configure the subflows until the subflow
- * is in TCP fallback, when TCP socket options are passed through
- * to the one remaining subflow.
- */
- lock_sock(sk);
- ssk = __mptcp_tcp_fallback(msk);
- release_sock(sk);
- if (ssk)
- return tcp_setsockopt(ssk, level, optname, optval, optlen);
-
- if (level == SOL_IPV6)
- return mptcp_setsockopt_v6(msk, optname, optval, optlen);
-
- return -EOPNOTSUPP;
-}
-
-static int mptcp_getsockopt(struct sock *sk, int level, int optname,
- char __user *optval, int __user *option)
-{
- struct mptcp_sock *msk = mptcp_sk(sk);
- struct sock *ssk;
-
- pr_debug("msk=%p", msk);
-
- /* @@ the meaning of setsockopt() when the socket is connected and
- * there are multiple subflows is not yet defined. It is up to the
- * MPTCP-level socket to configure the subflows until the subflow
- * is in TCP fallback, when socket options are passed through
- * to the one remaining subflow.
- */
- lock_sock(sk);
- ssk = __mptcp_tcp_fallback(msk);
- release_sock(sk);
- if (ssk)
- return tcp_getsockopt(ssk, level, optname, optval, option);
-
- return -EOPNOTSUPP;
-}
-
void __mptcp_data_acked(struct sock *sk)
{
if (!sock_owned_by_user(sk))
}
}
-#define MPTCP_DEFERRED_ALL (TCPF_WRITE_TIMER_DEFERRED)
-
/* processes deferred events and flush wmem */
static void mptcp_release_cb(struct sock *sk)
{
- unsigned long flags, nflags;
-
for (;;) {
- flags = 0;
+ unsigned long flags = 0;
+
if (test_and_clear_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags))
flags |= BIT(MPTCP_PUSH_PENDING);
+ if (test_and_clear_bit(MPTCP_RETRANSMIT, &mptcp_sk(sk)->flags))
+ flags |= BIT(MPTCP_RETRANSMIT);
if (!flags)
break;
spin_unlock_bh(&sk->sk_lock.slock);
if (flags & BIT(MPTCP_PUSH_PENDING))
__mptcp_push_pending(sk, 0);
+ if (flags & BIT(MPTCP_RETRANSMIT))
+ __mptcp_retrans(sk);
cond_resched();
spin_lock_bh(&sk->sk_lock.slock);
*/
__mptcp_update_wmem(sk);
__mptcp_update_rmem(sk);
-
- do {
- flags = sk->sk_tsq_flags;
- if (!(flags & MPTCP_DEFERRED_ALL))
- return;
- nflags = flags & ~MPTCP_DEFERRED_ALL;
- } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags);
-
- sock_release_ownership(sk);
-
- if (flags & TCPF_WRITE_TIMER_DEFERRED) {
- mptcp_retransmit_handler(sk);
- __sock_put(sk);
- }
}
void mptcp_subflow_process_delegated(struct sock *ssk)
pr_debug("msk=%p, subflow=%p", msk, subflow);
/* mptcp socket already closing? */
- if (!mptcp_is_fully_established(parent))
+ if (!mptcp_is_fully_established(parent)) {
+ subflow->reset_reason = MPTCP_RST_EMPTCP;
return false;
+ }
if (!msk->pm.server_side)
goto out;
- if (!mptcp_pm_allow_new_subflow(msk))
+ if (!mptcp_pm_allow_new_subflow(msk)) {
+ subflow->reset_reason = MPTCP_RST_EPROHIBIT;
return false;
+ }
/* active connections are already on conn_list, and we can't acquire
* msk lock here.
sock_hold(ssk);
}
spin_unlock_bh(&msk->join_list_lock);
- if (!ret)
+ if (!ret) {
+ subflow->reset_reason = MPTCP_RST_EPROHIBIT;
return false;
+ }
/* attach to msk socket only after we are sure he will deal with us
* at close time
if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
mptcp_subflow_early_fallback(msk, subflow);
#endif
- if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk))
+ if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) {
+ MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT);
mptcp_subflow_early_fallback(msk, subflow);
+ }
+ if (likely(!__mptcp_check_fallback(msk)))
+ MPTCP_INC_STATS(sock_net(sock->sk), MPTCP_MIB_MPCAPABLEACTIVE);
do_connect:
err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
/* set ssk->sk_socket of accept()ed flows to mptcp socket.
* This is needed so NOSPACE flag can be set from tcp stack.
*/
- __mptcp_flush_join_list(msk);
+ mptcp_flush_join_list(msk);
mptcp_for_each_subflow(msk, subflow) {
struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
projects / linux-2.6-microblaze.git / blobdiff