maple_tree: optimise mas_wr_node_store() when not in rcu mode

Clearing the entire node on the stack is unnecessary since most of the
node will be overwritten anyways.  Just clear what isn't used after the
data is in place.

Benchmarking shows a speedup of 0.67% on a height 4 tree with 2048
entries.

Link: https://lore.kernel.org/20260821192627.4085470-10-liam@infradead.org
Signed-off-by: Liam R. Howlett (Oracle) <liam@infradead.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Breno Leitao <leitao@debian.org>
Cc: Chris Mason <clm@meta.com>
Cc: Chuck Lever <cel@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Joe Perches <joe@perches.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@surriel.com>
Cc: Waiman Long <longman@redhat.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index d5fa85b..56812db 100644 (file)
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -3187,7 +3187,7 @@ static void mas_wr_spanning_store(struct ma_wr_state *wr_mas)
 static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
 {
        unsigned char dst_offset, offset_end;
-       unsigned char copy_size, node_pivots;
+       unsigned char copy_size, node_pivots, node_slots;
        struct maple_node reuse, *newnode;
        unsigned long *dst_pivots;
        void __rcu **dst_slots;
@@ -3200,6 +3200,7 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
        in_rcu = mt_in_rcu(mas->tree);
        offset_end = wr_mas->offset_end;
        node_pivots = mt_pivots[wr_mas->type];
+       node_slots = mt_slots[wr_mas->type];
        /* Assume last adds an entry */
        new_end = mas->end + 1 - offset_end + mas->offset;
        if (mas->last == wr_mas->end_piv) {
@@ -3211,7 +3212,6 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
        if (in_rcu) {
                newnode = mas_pop_node(mas);
        } else {
-               memset(&reuse, 0, sizeof(struct maple_node));
                newnode = &reuse;
        }
 
@@ -3255,7 +3255,21 @@ static inline void mas_wr_node_store(struct ma_wr_state *wr_mas)
                dst_pivots[new_end] = mas->max;
 
 done:
-       mas_leaf_set_meta(newnode, maple_leaf_64, new_end);
+       if (!in_rcu && new_end + 2 < node_slots) {
+               unsigned char clear_from = new_end + 1;
+
+               /*
+                * Note that the last slot is never cleared, since the metadata
+                * will be stored there or it has a value.
+                */
+               memset(dst_slots + clear_from, 0,
+                      sizeof(void __rcu *) * (node_slots - clear_from));
+               if (clear_from < node_pivots)
+                       memset(dst_pivots + clear_from, 0,
+                              sizeof(unsigned long) * (node_pivots - clear_from));
+       }
+
+       mas_leaf_set_meta(newnode, wr_mas->type, new_end);
        if (in_rcu) {
                struct maple_enode *old_enode = mas->node;