f2fs: add ckpt_valid_blocks to the section entry

when performing buffered writes in a large section,
overhead is incurred due to the iteration through
ckpt_valid_blocks within the section.
when SEGS_PER_SEC is 128, this overhead accounts for 20% within
the f2fs_write_single_data_page routine.
as the size of the section increases, the overhead also grows.
to handle this problem ckpt_valid_blocks is
added within the section entries.

Test
insmod null_blk.ko nr_devices=1 completion_nsec=1  submit_queues=8
hw_queue_depth=64 max_sectors=512 bs=4096 memory_backed=1
make_f2fs /dev/block/nullb0
make_f2fs -s 128 /dev/block/nullb0
fio --bs=512k --size=1536M --rw=write --name=1
--filename=/mnt/test_dir/seq_write
--ioengine=io_uring --iodepth=64 --end_fsync=1

before
SEGS_PER_SEC 1
2556MiB/s
SEGS_PER_SEC 128
2145MiB/s

after
SEGS_PER_SEC 1
2556MiB/s
SEGS_PER_SEC 128
2556MiB/s

Signed-off-by: yohan.joung <yohan.joung@sk.com>
Reviewed-by: Chao Yu <chao@kernel.org>
Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>

diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index 671bc5a..b77b5de 100644 (file)
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -2447,7 +2447,7 @@ static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
  * that the consecutive input blocks belong to the same segment.
  */
 static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_entry *se,
-                               block_t blkaddr, unsigned int offset, int del)
+                               unsigned int segno, block_t blkaddr, unsigned int offset, int del)
 {
        bool exist;
 #ifdef CONFIG_F2FS_CHECK_FS
@@ -2492,15 +2492,21 @@ static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_ent
                                f2fs_test_and_clear_bit(offset + i, se->discard_map))
                        sbi->discard_blks++;
 
-               if (!f2fs_test_bit(offset + i, se->ckpt_valid_map))
+               if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) {
                        se->ckpt_valid_blocks -= 1;
+                       if (__is_large_section(sbi))
+                               get_sec_entry(sbi, segno)->ckpt_valid_blocks -= 1;
+               }
        }
 
+       if (__is_large_section(sbi))
+               sanity_check_valid_blocks(sbi, segno);
+
        return del;
 }
 
 static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry *se,
-                               block_t blkaddr, unsigned int offset, int del)
+                               unsigned int segno, block_t blkaddr, unsigned int offset, int del)
 {
        bool exist;
 #ifdef CONFIG_F2FS_CHECK_FS
@@ -2533,12 +2539,21 @@ static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry
         * or newly invalidated.
         */
        if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
-               if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
+               if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) {
                        se->ckpt_valid_blocks++;
+                       if (__is_large_section(sbi))
+                               get_sec_entry(sbi, segno)->ckpt_valid_blocks++;
+               }
        }
 
-       if (!f2fs_test_bit(offset, se->ckpt_valid_map))
+       if (!f2fs_test_bit(offset, se->ckpt_valid_map)) {
                se->ckpt_valid_blocks += del;
+               if (__is_large_section(sbi))
+                       get_sec_entry(sbi, segno)->ckpt_valid_blocks += del;
+       }
+
+       if (__is_large_section(sbi))
+               sanity_check_valid_blocks(sbi, segno);
 
        return del;
 }
@@ -2569,9 +2584,9 @@ static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
 
        /* Update valid block bitmap */
        if (del > 0) {
-               del = update_sit_entry_for_alloc(sbi, se, blkaddr, offset, del);
+               del = update_sit_entry_for_alloc(sbi, se, segno, blkaddr, offset, del);
        } else {
-               del = update_sit_entry_for_release(sbi, se, blkaddr, offset, del);
+               del = update_sit_entry_for_release(sbi, se, segno, blkaddr, offset, del);
        }
 
        __mark_sit_entry_dirty(sbi, segno);
@@ -4708,6 +4723,12 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
                                                &raw_sit->entries[sit_offset]);
                        }
 
+                       /* update ckpt_valid_block */
+                       if (__is_large_section(sbi)) {
+                               set_ckpt_valid_blocks(sbi, segno);
+                               sanity_check_valid_blocks(sbi, segno);
+                       }
+
                        __clear_bit(segno, bitmap);
                        sit_i->dirty_sentries--;
                        ses->entry_cnt--;
@@ -5029,6 +5050,16 @@ init_discard_map_done:
        }
        up_read(&curseg->journal_rwsem);
 
+       /* update ckpt_valid_block */
+       if (__is_large_section(sbi)) {
+               unsigned int segno;
+
+               for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
+                       set_ckpt_valid_blocks(sbi, segno);
+                       sanity_check_valid_blocks(sbi, segno);
+               }
+       }
+
        if (err)
                return err;
 

diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
index 5777b38..db619fd 100644 (file)
--- a/fs/f2fs/segment.h
+++ b/fs/f2fs/segment.h
@@ -211,6 +211,7 @@ struct seg_entry {
 
 struct sec_entry {
        unsigned int valid_blocks;      /* # of valid blocks in a section */
+       unsigned int ckpt_valid_blocks; /* # of valid blocks last cp in a section */
 };
 
 #define MAX_SKIP_GC_COUNT                      16
@@ -347,22 +348,57 @@ static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi,
 static inline unsigned int get_ckpt_valid_blocks(struct f2fs_sb_info *sbi,
                                unsigned int segno, bool use_section)
 {
-       if (use_section && __is_large_section(sbi)) {
-               unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
-               unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno);
-               unsigned int blocks = 0;
-               int i;
+       if (use_section && __is_large_section(sbi))
+               return get_sec_entry(sbi, segno)->ckpt_valid_blocks;
+       else
+               return get_seg_entry(sbi, segno)->ckpt_valid_blocks;
+}
 
-               for (i = 0; i < SEGS_PER_SEC(sbi); i++, start_segno++) {
-                       struct seg_entry *se = get_seg_entry(sbi, start_segno);
+static inline void set_ckpt_valid_blocks(struct f2fs_sb_info *sbi,
+               unsigned int segno)
+{
+       unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
+       unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno);
+       unsigned int blocks = 0;
+       int i;
 
-                       blocks += se->ckpt_valid_blocks;
-               }
-               return blocks;
+       for (i = 0; i < SEGS_PER_SEC(sbi); i++, start_segno++) {
+               struct seg_entry *se = get_seg_entry(sbi, start_segno);
+
+               blocks += se->ckpt_valid_blocks;
        }
-       return get_seg_entry(sbi, segno)->ckpt_valid_blocks;
+       get_sec_entry(sbi, segno)->ckpt_valid_blocks = blocks;
 }
 
+#ifdef CONFIG_F2FS_CHECK_FS
+static inline void sanity_check_valid_blocks(struct f2fs_sb_info *sbi,
+               unsigned int segno)
+{
+       unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
+       unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno);
+       unsigned int blocks = 0;
+       int i;
+
+       for (i = 0; i < SEGS_PER_SEC(sbi); i++, start_segno++) {
+               struct seg_entry *se = get_seg_entry(sbi, start_segno);
+
+               blocks += se->ckpt_valid_blocks;
+       }
+
+       if (blocks != get_sec_entry(sbi, segno)->ckpt_valid_blocks) {
+               f2fs_err(sbi,
+                       "Inconsistent ckpt valid blocks: "
+                       "seg entry(%d) vs sec entry(%d) at secno %d",
+                       blocks, get_sec_entry(sbi, segno)->ckpt_valid_blocks, secno);
+               f2fs_bug_on(sbi, 1);
+       }
+}
+#else
+static inline void sanity_check_valid_blocks(struct f2fs_sb_info *sbi,
+                       unsigned int segno)
+{
+}
+#endif
 static inline void seg_info_from_raw_sit(struct seg_entry *se,
                                        struct f2fs_sit_entry *rs)
 {