xfs: optimise xfs_buf_item_size/format for contiguous regions

We process the buf_log_item bitmap one set bit at a time with
xfs_next_bit() so we can detect if a region crosses a memcpy
discontinuity in the buffer data address. This has massive overhead
on large buffers (e.g. 64k directory blocks) because we do a lot of
unnecessary checks and xfs_buf_offset() calls.

For example, 16-way concurrent create workload on debug kernel
running CPU bound has this at the top of the profile at ~120k
create/s on 64kb directory block size:

  20.66%  [kernel]  [k] xfs_dir3_leaf_check_int
   7.10%  [kernel]  [k] memcpy
   6.22%  [kernel]  [k] xfs_next_bit
   3.55%  [kernel]  [k] xfs_buf_offset
   3.53%  [kernel]  [k] xfs_buf_item_format
   3.34%  [kernel]  [k] __pv_queued_spin_lock_slowpath
   3.04%  [kernel]  [k] do_raw_spin_lock
   2.84%  [kernel]  [k] xfs_buf_item_size_segment.isra.0
   2.31%  [kernel]  [k] __raw_callee_save___pv_queued_spin_unlock
   1.36%  [kernel]  [k] xfs_log_commit_cil

(debug checks hurt large blocks)

The only buffers with discontinuities in the data address are
unmapped buffers, and they are only used for inode cluster buffers
and only for logging unlinked pointers. IOWs, it is -rare- that we
even need to detect a discontinuity in the buffer item formatting
code.

Optimise all this by using xfs_contig_bits() to find the size of
the contiguous regions, then test for a discontiunity inside it. If
we find one, do the slow "bit at a time" method we do now. If we
don't, then just copy the entire contiguous range in one go.

Profile now looks like:

  25.26%  [kernel]  [k] xfs_dir3_leaf_check_int
   9.25%  [kernel]  [k] memcpy
   5.01%  [kernel]  [k] __pv_queued_spin_lock_slowpath
   2.84%  [kernel]  [k] do_raw_spin_lock
   2.22%  [kernel]  [k] __raw_callee_save___pv_queued_spin_unlock
   1.88%  [kernel]  [k] xfs_buf_find
   1.53%  [kernel]  [k] memmove
   1.47%  [kernel]  [k] xfs_log_commit_cil
....
   0.34%  [kernel]  [k] xfs_buf_item_format
....
   0.21%  [kernel]  [k] xfs_buf_offset
....
   0.16%  [kernel]  [k] xfs_contig_bits
....
   0.13%  [kernel]  [k] xfs_buf_item_size_segment.isra.0

So the bit scanning over for the dirty region tracking for the
buffer log items is basically gone. Debug overhead hurts even more
now...

Perf comparison

		dir block	 creates		unlink
		size (kb)	time	rate		time

Original	 4		4m08s	220k		 5m13s
Original	64		7m21s	115k		13m25s
Patched		 4		3m59s	230k		 5m03s
Patched		64		6m23s	143k		12m33s

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>

diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
index 189a453..fb69879 100644 (file)
--- a/fs/xfs/xfs_buf_item.c
+++ b/fs/xfs/xfs_buf_item.c
@@ -59,12 +59,18 @@ static inline bool
 xfs_buf_item_straddle(
        struct xfs_buf          *bp,
        uint                    offset,
-       int                     next_bit,
-       int                     last_bit)
+       int                     first_bit,
+       int                     nbits)
 {
-       return xfs_buf_offset(bp, offset + (next_bit << XFS_BLF_SHIFT)) !=
-               (xfs_buf_offset(bp, offset + (last_bit << XFS_BLF_SHIFT)) +
-                XFS_BLF_CHUNK);
+       void                    *first, *last;
+
+       first = xfs_buf_offset(bp, offset + (first_bit << XFS_BLF_SHIFT));
+       last = xfs_buf_offset(bp,
+                       offset + ((first_bit + nbits) << XFS_BLF_SHIFT));
+
+       if (last - first != nbits * XFS_BLF_CHUNK)
+               return true;
+       return false;
 }
 
 /*
@@ -86,20 +92,51 @@ xfs_buf_item_size_segment(
        int                             *nbytes)
 {
        struct xfs_buf                  *bp = bip->bli_buf;
+       int                             first_bit;
+       int                             nbits;
        int                             next_bit;
        int                             last_bit;
 
-       last_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
-       if (last_bit == -1)
+       first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0);
+       if (first_bit == -1)
                return;
 
-       /*
-        * initial count for a dirty buffer is 2 vectors - the format structure
-        * and the first dirty region.
-        */
-       *nvecs += 2;
-       *nbytes += xfs_buf_log_format_size(blfp) + XFS_BLF_CHUNK;
+       (*nvecs)++;
+       *nbytes += xfs_buf_log_format_size(blfp);
+
+       do {
+               nbits = xfs_contig_bits(blfp->blf_data_map,
+                                       blfp->blf_map_size, first_bit);
+               ASSERT(nbits > 0);
+
+               /*
+                * Straddling a page is rare because we don't log contiguous
+                * chunks of unmapped buffers anywhere.
+                */
+               if (nbits > 1 &&
+                   xfs_buf_item_straddle(bp, offset, first_bit, nbits))
+                       goto slow_scan;
+
+               (*nvecs)++;
+               *nbytes += nbits * XFS_BLF_CHUNK;
+
+               /*
+                * This takes the bit number to start looking from and
+                * returns the next set bit from there.  It returns -1
+                * if there are no more bits set or the start bit is
+                * beyond the end of the bitmap.
+                */
+               first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
+                                       (uint)first_bit + nbits + 1);
+       } while (first_bit != -1);
 
+       return;
+
+slow_scan:
+       /* Count the first bit we jumped out of the above loop from */
+       (*nvecs)++;
+       *nbytes += XFS_BLF_CHUNK;
+       last_bit = first_bit;
        while (last_bit != -1) {
                /*
                 * This takes the bit number to start looking from and
@@ -117,11 +154,14 @@ xfs_buf_item_size_segment(
                if (next_bit == -1) {
                        break;
                } else if (next_bit != last_bit + 1 ||
-                          xfs_buf_item_straddle(bp, offset, next_bit, last_bit)) {
+                          xfs_buf_item_straddle(bp, offset, first_bit, nbits)) {
                        last_bit = next_bit;
+                       first_bit = next_bit;
                        (*nvecs)++;
+                       nbits = 1;
                } else {
                        last_bit++;
+                       nbits++;
                }
                *nbytes += XFS_BLF_CHUNK;
        }
@@ -277,6 +317,38 @@ xfs_buf_item_format_segment(
        /*
         * Fill in an iovec for each set of contiguous chunks.
         */
+       do {
+               ASSERT(first_bit >= 0);
+               nbits = xfs_contig_bits(blfp->blf_data_map,
+                                       blfp->blf_map_size, first_bit);
+               ASSERT(nbits > 0);
+
+               /*
+                * Straddling a page is rare because we don't log contiguous
+                * chunks of unmapped buffers anywhere.
+                */
+               if (nbits > 1 &&
+                   xfs_buf_item_straddle(bp, offset, first_bit, nbits))
+                       goto slow_scan;
+
+               xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
+                                       first_bit, nbits);
+               blfp->blf_size++;
+
+               /*
+                * This takes the bit number to start looking from and
+                * returns the next set bit from there.  It returns -1
+                * if there are no more bits set or the start bit is
+                * beyond the end of the bitmap.
+                */
+               first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size,
+                                       (uint)first_bit + nbits + 1);
+       } while (first_bit != -1);
+
+       return;
+
+slow_scan:
+       ASSERT(bp->b_addr == NULL);
        last_bit = first_bit;
        nbits = 1;
        for (;;) {
@@ -301,7 +373,7 @@ xfs_buf_item_format_segment(
                        blfp->blf_size++;
                        break;
                } else if (next_bit != last_bit + 1 ||
-                          xfs_buf_item_straddle(bp, offset, next_bit, last_bit)) {
+                          xfs_buf_item_straddle(bp, offset, first_bit, nbits)) {
                        xfs_buf_item_copy_iovec(lv, vecp, bp, offset,
                                                first_bit, nbits);
                        blfp->blf_size++;