Merge tag 'mlx5-updates-2022-07-17' of git://git.kernel.org/pub/scm/linux/kernel...

[linux-2.6-microblaze.git] / net / core / skbuff.c

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 5b3559c..974bbbb 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -91,6 +91,9 @@ static struct kmem_cache *skbuff_ext_cache __ro_after_init;
 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
 EXPORT_SYMBOL(sysctl_max_skb_frags);
 
+/* The array 'drop_reasons' is auto-generated in dropreason_str.c */
+EXPORT_SYMBOL(drop_reasons);
+
 /**
  *     skb_panic - private function for out-of-line support
  *     @skb:   buffer
@@ -172,13 +175,14 @@ static struct sk_buff *napi_skb_cache_get(void)
        struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
        struct sk_buff *skb;
 
-       if (unlikely(!nc->skb_count))
+       if (unlikely(!nc->skb_count)) {
                nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache,
                                                      GFP_ATOMIC,
                                                      NAPI_SKB_CACHE_BULK,
                                                      nc->skb_cache);
-       if (unlikely(!nc->skb_count))
-               return NULL;
+               if (unlikely(!nc->skb_count))
+                       return NULL;
+       }
 
        skb = nc->skb_cache[--nc->skb_count];
        kasan_unpoison_object_data(skbuff_head_cache, skb);
@@ -450,8 +454,6 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
 
                skb->fclone = SKB_FCLONE_ORIG;
                refcount_set(&fclones->fclone_ref, 1);
-
-               fclones->skb2.fclone = SKB_FCLONE_CLONE;
        }
 
        return skb;
@@ -557,6 +559,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
        struct sk_buff *skb;
        void *data;
 
+       DEBUG_NET_WARN_ON_ONCE(!in_softirq());
        len += NET_SKB_PAD + NET_IP_ALIGN;
 
        /* If requested length is either too small or too big,
@@ -666,11 +669,18 @@ static void skb_release_data(struct sk_buff *skb)
                              &shinfo->dataref))
                goto exit;
 
-       skb_zcopy_clear(skb, true);
+       if (skb_zcopy(skb)) {
+               bool skip_unref = shinfo->flags & SKBFL_MANAGED_FRAG_REFS;
+
+               skb_zcopy_clear(skb, true);
+               if (skip_unref)
+                       goto free_head;
+       }
 
        for (i = 0; i < shinfo->nr_frags; i++)
                __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle);
 
+free_head:
        if (shinfo->frag_list)
                kfree_skb_list(shinfo->frag_list);
 
@@ -725,7 +735,7 @@ void skb_release_head_state(struct sk_buff *skb)
 {
        skb_dst_drop(skb);
        if (skb->destructor) {
-               WARN_ON(in_hardirq());
+               DEBUG_NET_WARN_ON_ONCE(in_hardirq());
                skb->destructor(skb);
        }
 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
@@ -895,7 +905,10 @@ EXPORT_SYMBOL(skb_dump);
  */
 void skb_tx_error(struct sk_buff *skb)
 {
-       skb_zcopy_clear(skb, true);
+       if (skb) {
+               skb_zcopy_downgrade_managed(skb);
+               skb_zcopy_clear(skb, true);
+       }
 }
 EXPORT_SYMBOL(skb_tx_error);
 
@@ -978,7 +991,7 @@ void napi_consume_skb(struct sk_buff *skb, int budget)
                return;
        }
 
-       lockdep_assert_in_softirq();
+       DEBUG_NET_WARN_ON_ONCE(!in_softirq());
 
        if (!skb_unref(skb))
                return;
@@ -1193,7 +1206,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
        uarg->len = 1;
        uarg->bytelen = size;
        uarg->zerocopy = 1;
-       uarg->flags = SKBFL_ZEROCOPY_FRAG;
+       uarg->flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
        refcount_set(&uarg->refcnt, 1);
        sock_hold(sk);
 
@@ -1212,6 +1225,10 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
                const u32 byte_limit = 1 << 19;         /* limit to a few TSO */
                u32 bytelen, next;
 
+               /* there might be non MSG_ZEROCOPY users */
+               if (uarg->callback != msg_zerocopy_callback)
+                       return NULL;
+
                /* realloc only when socket is locked (TCP, UDP cork),
                 * so uarg->len and sk_zckey access is serialized
                 */
@@ -1354,7 +1371,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
        if (orig_uarg && uarg != orig_uarg)
                return -EEXIST;
 
-       err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len);
+       err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
        if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
                struct sock *save_sk = skb->sk;
 
@@ -1371,6 +1388,16 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
 }
 EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
 
+void __skb_zcopy_downgrade_managed(struct sk_buff *skb)
+{
+       int i;
+
+       skb_shinfo(skb)->flags &= ~SKBFL_MANAGED_FRAG_REFS;
+       for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
+               skb_frag_ref(skb, i);
+}
+EXPORT_SYMBOL_GPL(__skb_zcopy_downgrade_managed);
+
 static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig,
                              gfp_t gfp_mask)
 {
@@ -1508,6 +1535,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
            refcount_read(&fclones->fclone_ref) == 1) {
                n = &fclones->skb2;
                refcount_set(&fclones->fclone_ref, 2);
+               n->fclone = SKB_FCLONE_CLONE;
        } else {
                if (skb_pfmemalloc(skb))
                        gfp_mask |= __GFP_MEMALLOC;
@@ -1688,6 +1716,8 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
 
        BUG_ON(skb_shared(skb));
 
+       skb_zcopy_downgrade_managed(skb);
+
        size = SKB_DATA_ALIGN(size);
 
        if (skb_pfmemalloc(skb))
@@ -3190,9 +3220,7 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen)
                }
        }
 
-       to->truesize += len + plen;
-       to->len += len + plen;
-       to->data_len += len + plen;
+       skb_len_add(to, len + plen);
 
        if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) {
                skb_tx_error(from);
@@ -3484,6 +3512,8 @@ void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
        int pos = skb_headlen(skb);
        const int zc_flags = SKBFL_SHARED_FRAG | SKBFL_PURE_ZEROCOPY;
 
+       skb_zcopy_downgrade_managed(skb);
+
        skb_shinfo(skb1)->flags |= skb_shinfo(skb)->flags & zc_flags;
        skb_zerocopy_clone(skb1, skb, 0);
        if (len < pos)  /* Split line is inside header. */
@@ -3629,13 +3659,8 @@ onlymerged:
        tgt->ip_summed = CHECKSUM_PARTIAL;
        skb->ip_summed = CHECKSUM_PARTIAL;
 
-       /* Yak, is it really working this way? Some helper please? */
-       skb->len -= shiftlen;
-       skb->data_len -= shiftlen;
-       skb->truesize -= shiftlen;
-       tgt->len += shiftlen;
-       tgt->data_len += shiftlen;
-       tgt->truesize += shiftlen;
+       skb_len_add(skb, -shiftlen);
+       skb_len_add(tgt, shiftlen);
 
        return shiftlen;
 }
@@ -3837,6 +3862,7 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
        if (skb_can_coalesce(skb, i, page, offset)) {
                skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
        } else if (i < MAX_SKB_FRAGS) {
+               skb_zcopy_downgrade_managed(skb);
                get_page(page);
                skb_fill_page_desc(skb, i, page, offset, size);
        } else {