netfilter: nft_set_pipapo: speed up bulk element insertions

[linux-2.6-microblaze.git] / net / netfilter / nft_set_pipapo.c

diff --git a/net/netfilter/nft_set_pipapo.c b/net/netfilter/nft_set_pipapo.c
index b63278b..6118e4b 100644 (file)
--- a/net/netfilter/nft_set_pipapo.c
+++ b/net/netfilter/nft_set_pipapo.c
@@ -621,6 +621,65 @@ nft_pipapo_get(const struct net *net, const struct nft_set *set,
        return &e->priv;
 }
 
+/**
+ * pipapo_realloc_mt() - Reallocate mapping table if needed upon resize
+ * @f:         Field containing mapping table
+ * @old_rules: Amount of existing mapped rules
+ * @rules:     Amount of new rules to map
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+static int pipapo_realloc_mt(struct nft_pipapo_field *f,
+                            unsigned int old_rules, unsigned int rules)
+{
+       union nft_pipapo_map_bucket *new_mt = NULL, *old_mt = f->mt;
+       const unsigned int extra = PAGE_SIZE / sizeof(*new_mt);
+       unsigned int rules_alloc = rules;
+
+       might_sleep();
+
+       if (unlikely(rules == 0))
+               goto out_free;
+
+       /* growing and enough space left, no action needed */
+       if (rules > old_rules && f->rules_alloc > rules)
+               return 0;
+
+       /* downsize and extra slack has not grown too large */
+       if (rules < old_rules) {
+               unsigned int remove = f->rules_alloc - rules;
+
+               if (remove < (2u * extra))
+                       return 0;
+       }
+
+       /* If set needs more than one page of memory for rules then
+        * allocate another extra page to avoid frequent reallocation.
+        */
+       if (rules > extra &&
+           check_add_overflow(rules, extra, &rules_alloc))
+               return -EOVERFLOW;
+
+       new_mt = kvmalloc_array(rules_alloc, sizeof(*new_mt), GFP_KERNEL);
+       if (!new_mt)
+               return -ENOMEM;
+
+       if (old_mt)
+               memcpy(new_mt, old_mt, min(old_rules, rules) * sizeof(*new_mt));
+
+       if (rules > old_rules) {
+               memset(new_mt + old_rules, 0,
+                      (rules - old_rules) * sizeof(*new_mt));
+       }
+out_free:
+       f->rules_alloc = rules_alloc;
+       f->mt = new_mt;
+
+       kvfree(old_mt);
+
+       return 0;
+}
+
 /**
  * pipapo_resize() - Resize lookup or mapping table, or both
  * @f:         Field containing lookup and mapping tables
@@ -637,9 +696,8 @@ static int pipapo_resize(struct nft_pipapo_field *f,
                         unsigned int old_rules, unsigned int rules)
 {
        long *new_lt = NULL, *new_p, *old_lt = f->lt, *old_p;
-       union nft_pipapo_map_bucket *new_mt, *old_mt = f->mt;
        unsigned int new_bucket_size, copy;
-       int group, bucket;
+       int group, bucket, err;
 
        if (rules >= NFT_PIPAPO_RULE0_MAX)
                return -ENOSPC;
@@ -682,16 +740,10 @@ static int pipapo_resize(struct nft_pipapo_field *f,
        }
 
 mt:
-       new_mt = kvmalloc(rules * sizeof(*new_mt), GFP_KERNEL);
-       if (!new_mt) {
+       err = pipapo_realloc_mt(f, old_rules, rules);
+       if (err) {
                kvfree(new_lt);
-               return -ENOMEM;
-       }
-
-       memcpy(new_mt, f->mt, min(old_rules, rules) * sizeof(*new_mt));
-       if (rules > old_rules) {
-               memset(new_mt + old_rules, 0,
-                      (rules - old_rules) * sizeof(*new_mt));
+               return err;
        }
 
        if (new_lt) {
@@ -700,9 +752,6 @@ mt:
                kvfree(old_lt);
        }
 
-       f->mt = new_mt;
-       kvfree(old_mt);
-
        return 0;
 }
 
@@ -1382,14 +1431,15 @@ static struct nft_pipapo_match *pipapo_clone(struct nft_pipapo_match *old)
                       src->groups * NFT_PIPAPO_BUCKETS(src->bb));
 
                if (src->rules > 0) {
-                       dst->mt = kvmalloc_array(src->rules, sizeof(*src->mt),
-                                                GFP_KERNEL);
+                       dst->mt = kvmalloc_array(src->rules_alloc,
+                                                sizeof(*src->mt), GFP_KERNEL);
                        if (!dst->mt)
                                goto out_mt;
 
                        memcpy(dst->mt, src->mt, src->rules * sizeof(*src->mt));
                } else {
                        dst->mt = NULL;
+                       dst->rules_alloc = 0;
                }
 
                src++;
@@ -2205,6 +2255,7 @@ static int nft_pipapo_init(const struct nft_set *set,
 
                f->bsize = 0;
                f->rules = 0;
+               f->rules_alloc = 0;
                f->lt = NULL;
                f->mt = NULL;
        }