fs/f2fs/checkpoint.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/checkpoint.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/bio.h>
  10 #include <linux/mpage.h>
  11 #include <linux/writeback.h>
  12 #include <linux/blkdev.h>
  13 #include <linux/f2fs_fs.h>
  14 #include <linux/pagevec.h>
  15 #include <linux/swap.h>
  16 #include <linux/kthread.h>
  17
  18 #include "f2fs.h"
  19 #include "node.h"
  20 #include "segment.h"
  21 #include "iostat.h"
  22 #include <trace/events/f2fs.h>
  23
  24 #define DEFAULT_CHECKPOINT_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3))
  25
  26 static struct kmem_cache *ino_entry_slab;
  27 struct kmem_cache *f2fs_inode_entry_slab;
  28
  29 void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
  30                                                 unsigned char reason)
  31 {
  32         f2fs_build_fault_attr(sbi, 0, 0);
  33         set_ckpt_flags(sbi, CP_ERROR_FLAG);
  34         if (!end_io) {
  35                 f2fs_flush_merged_writes(sbi);
  36
  37                 f2fs_handle_stop(sbi, reason);
  38         }
  39 }
  40
  41 /*
  42  * We guarantee no failure on the returned page.
  43  */
  44 struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
  45 {
  46         struct address_space *mapping = META_MAPPING(sbi);
  47         struct page *page;
  48 repeat:
  49         page = f2fs_grab_cache_page(mapping, index, false);
  50         if (!page) {
  51                 cond_resched();
  52                 goto repeat;
  53         }
  54         f2fs_wait_on_page_writeback(page, META, true, true);
  55         if (!PageUptodate(page))
  56                 SetPageUptodate(page);
  57         return page;
  58 }
  59
  60 static struct page *__get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index,
  61                                                         bool is_meta)
  62 {
  63         struct address_space *mapping = META_MAPPING(sbi);
  64         struct page *page;
  65         struct f2fs_io_info fio = {
  66                 .sbi = sbi,
  67                 .type = META,
  68                 .op = REQ_OP_READ,
  69                 .op_flags = REQ_META | REQ_PRIO,
  70                 .old_blkaddr = index,
  71                 .new_blkaddr = index,
  72                 .encrypted_page = NULL,
  73                 .is_por = !is_meta ? 1 : 0,
  74         };
  75         int err;
  76
  77         if (unlikely(!is_meta))
  78                 fio.op_flags &= ~REQ_META;
  79 repeat:
  80         page = f2fs_grab_cache_page(mapping, index, false);
  81         if (!page) {
  82                 cond_resched();
  83                 goto repeat;
  84         }
  85         if (PageUptodate(page))
  86                 goto out;
  87
  88         fio.page = page;
  89
  90         err = f2fs_submit_page_bio(&fio);
  91         if (err) {
  92                 f2fs_put_page(page, 1);
  93                 return ERR_PTR(err);
  94         }
  95
  96         f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE);
  97
  98         lock_page(page);
  99         if (unlikely(page->mapping != mapping)) {
 100                 f2fs_put_page(page, 1);
 101                 goto repeat;
 102         }
 103
 104         if (unlikely(!PageUptodate(page))) {
 105                 f2fs_handle_page_eio(sbi, page->index, META);
 106                 f2fs_put_page(page, 1);
 107                 return ERR_PTR(-EIO);
 108         }
 109 out:
 110         return page;
 111 }
 112
 113 struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
 114 {
 115         return __get_meta_page(sbi, index, true);
 116 }
 117
 118 struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index)
 119 {
 120         struct page *page;
 121         int count = 0;
 122
 123 retry:
 124         page = __get_meta_page(sbi, index, true);
 125         if (IS_ERR(page)) {
 126                 if (PTR_ERR(page) == -EIO &&
 127                                 ++count <= DEFAULT_RETRY_IO_COUNT)
 128                         goto retry;
 129                 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE);
 130         }
 131         return page;
 132 }
 133
 134 /* for POR only */
 135 struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index)
 136 {
 137         return __get_meta_page(sbi, index, false);
 138 }
 139
 140 static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr,
 141                                                         int type)
 142 {
 143         struct seg_entry *se;
 144         unsigned int segno, offset;
 145         bool exist;
 146
 147         if (type == DATA_GENERIC)
 148                 return true;
 149
 150         segno = GET_SEGNO(sbi, blkaddr);
 151         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 152         se = get_seg_entry(sbi, segno);
 153
 154         exist = f2fs_test_bit(offset, se->cur_valid_map);
 155
 156         /* skip data, if we already have an error in checkpoint. */
 157         if (unlikely(f2fs_cp_error(sbi)))
 158                 return exist;
 159
 160         if (exist && type == DATA_GENERIC_ENHANCE_UPDATE) {
 161                 f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
 162                          blkaddr, exist);
 163                 set_sbi_flag(sbi, SBI_NEED_FSCK);
 164                 return exist;
 165         }
 166
 167         if (!exist && type == DATA_GENERIC_ENHANCE) {
 168                 f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
 169                          blkaddr, exist);
 170                 set_sbi_flag(sbi, SBI_NEED_FSCK);
 171                 dump_stack();
 172         }
 173         return exist;
 174 }
 175
 176 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
 177                                         block_t blkaddr, int type)
 178 {
 179         if (time_to_inject(sbi, FAULT_BLKADDR))
 180                 return false;
 181
 182         switch (type) {
 183         case META_NAT:
 184                 break;
 185         case META_SIT:
 186                 if (unlikely(blkaddr >= SIT_BLK_CNT(sbi)))
 187                         return false;
 188                 break;
 189         case META_SSA:
 190                 if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) ||
 191                         blkaddr < SM_I(sbi)->ssa_blkaddr))
 192                         return false;
 193                 break;
 194         case META_CP:
 195                 if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr ||
 196                         blkaddr < __start_cp_addr(sbi)))
 197                         return false;
 198                 break;
 199         case META_POR:
 200                 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
 201                         blkaddr < MAIN_BLKADDR(sbi)))
 202                         return false;
 203                 break;
 204         case DATA_GENERIC:
 205         case DATA_GENERIC_ENHANCE:
 206         case DATA_GENERIC_ENHANCE_READ:
 207         case DATA_GENERIC_ENHANCE_UPDATE:
 208                 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
 209                                 blkaddr < MAIN_BLKADDR(sbi))) {
 210
 211                         /* Skip to emit an error message. */
 212                         if (unlikely(f2fs_cp_error(sbi)))
 213                                 return false;
 214
 215                         f2fs_warn(sbi, "access invalid blkaddr:%u",
 216                                   blkaddr);
 217                         set_sbi_flag(sbi, SBI_NEED_FSCK);
 218                         dump_stack();
 219                         return false;
 220                 } else {
 221                         return __is_bitmap_valid(sbi, blkaddr, type);
 222                 }
 223                 break;
 224         case META_GENERIC:
 225                 if (unlikely(blkaddr < SEG0_BLKADDR(sbi) ||
 226                         blkaddr >= MAIN_BLKADDR(sbi)))
 227                         return false;
 228                 break;
 229         default:
 230                 BUG();
 231         }
 232
 233         return true;
 234 }
 235
 236 /*
 237  * Readahead CP/NAT/SIT/SSA/POR pages
 238  */
 239 int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 240                                                         int type, bool sync)
 241 {
 242         struct page *page;
 243         block_t blkno = start;
 244         struct f2fs_io_info fio = {
 245                 .sbi = sbi,
 246                 .type = META,
 247                 .op = REQ_OP_READ,
 248                 .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
 249                 .encrypted_page = NULL,
 250                 .in_list = 0,
 251                 .is_por = (type == META_POR) ? 1 : 0,
 252         };
 253         struct blk_plug plug;
 254         int err;
 255
 256         if (unlikely(type == META_POR))
 257                 fio.op_flags &= ~REQ_META;
 258
 259         blk_start_plug(&plug);
 260         for (; nrpages-- > 0; blkno++) {
 261
 262                 if (!f2fs_is_valid_blkaddr(sbi, blkno, type))
 263                         goto out;
 264
 265                 switch (type) {
 266                 case META_NAT:
 267                         if (unlikely(blkno >=
 268                                         NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid)))
 269                                 blkno = 0;
 270                         /* get nat block addr */
 271                         fio.new_blkaddr = current_nat_addr(sbi,
 272                                         blkno * NAT_ENTRY_PER_BLOCK);
 273                         break;
 274                 case META_SIT:
 275                         if (unlikely(blkno >= TOTAL_SEGS(sbi)))
 276                                 goto out;
 277                         /* get sit block addr */
 278                         fio.new_blkaddr = current_sit_addr(sbi,
 279                                         blkno * SIT_ENTRY_PER_BLOCK);
 280                         break;
 281                 case META_SSA:
 282                 case META_CP:
 283                 case META_POR:
 284                         fio.new_blkaddr = blkno;
 285                         break;
 286                 default:
 287                         BUG();
 288                 }
 289
 290                 page = f2fs_grab_cache_page(META_MAPPING(sbi),
 291                                                 fio.new_blkaddr, false);
 292                 if (!page)
 293                         continue;
 294                 if (PageUptodate(page)) {
 295                         f2fs_put_page(page, 1);
 296                         continue;
 297                 }
 298
 299                 fio.page = page;
 300                 err = f2fs_submit_page_bio(&fio);
 301                 f2fs_put_page(page, err ? 1 : 0);
 302
 303                 if (!err)
 304                         f2fs_update_iostat(sbi, NULL, FS_META_READ_IO,
 305                                                         F2FS_BLKSIZE);
 306         }
 307 out:
 308         blk_finish_plug(&plug);
 309         return blkno - start;
 310 }
 311
 312 void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
 313                                                         unsigned int ra_blocks)
 314 {
 315         struct page *page;
 316         bool readahead = false;
 317
 318         if (ra_blocks == RECOVERY_MIN_RA_BLOCKS)
 319                 return;
 320
 321         page = find_get_page(META_MAPPING(sbi), index);
 322         if (!page || !PageUptodate(page))
 323                 readahead = true;
 324         f2fs_put_page(page, 0);
 325
 326         if (readahead)
 327                 f2fs_ra_meta_pages(sbi, index, ra_blocks, META_POR, true);
 328 }
 329
 330 static int __f2fs_write_meta_page(struct page *page,
 331                                 struct writeback_control *wbc,
 332                                 enum iostat_type io_type)
 333 {
 334         struct f2fs_sb_info *sbi = F2FS_P_SB(page);
 335
 336         trace_f2fs_writepage(page, META);
 337
 338         if (unlikely(f2fs_cp_error(sbi))) {
 339                 if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) {
 340                         ClearPageUptodate(page);
 341                         dec_page_count(sbi, F2FS_DIRTY_META);
 342                         unlock_page(page);
 343                         return 0;
 344                 }
 345                 goto redirty_out;
 346         }
 347         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 348                 goto redirty_out;
 349         if (wbc->for_reclaim && page->index < GET_SUM_BLOCK(sbi, 0))
 350                 goto redirty_out;
 351
 352         f2fs_do_write_meta_page(sbi, page, io_type);
 353         dec_page_count(sbi, F2FS_DIRTY_META);
 354
 355         if (wbc->for_reclaim)
 356                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, META);
 357
 358         unlock_page(page);
 359
 360         if (unlikely(f2fs_cp_error(sbi)))
 361                 f2fs_submit_merged_write(sbi, META);
 362
 363         return 0;
 364
 365 redirty_out:
 366         redirty_page_for_writepage(wbc, page);
 367         return AOP_WRITEPAGE_ACTIVATE;
 368 }
 369
 370 static int f2fs_write_meta_page(struct page *page,
 371                                 struct writeback_control *wbc)
 372 {
 373         return __f2fs_write_meta_page(page, wbc, FS_META_IO);
 374 }
 375
 376 static int f2fs_write_meta_pages(struct address_space *mapping,
 377                                 struct writeback_control *wbc)
 378 {
 379         struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
 380         long diff, written;
 381
 382         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 383                 goto skip_write;
 384
 385         /* collect a number of dirty meta pages and write together */
 386         if (wbc->sync_mode != WB_SYNC_ALL &&
 387                         get_pages(sbi, F2FS_DIRTY_META) <
 388                                         nr_pages_to_skip(sbi, META))
 389                 goto skip_write;
 390
 391         /* if locked failed, cp will flush dirty pages instead */
 392         if (!f2fs_down_write_trylock(&sbi->cp_global_sem))
 393                 goto skip_write;
 394
 395         trace_f2fs_writepages(mapping->host, wbc, META);
 396         diff = nr_pages_to_write(sbi, META, wbc);
 397         written = f2fs_sync_meta_pages(sbi, META, wbc->nr_to_write, FS_META_IO);
 398         f2fs_up_write(&sbi->cp_global_sem);
 399         wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff);
 400         return 0;
 401
 402 skip_write:
 403         wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
 404         trace_f2fs_writepages(mapping->host, wbc, META);
 405         return 0;
 406 }
 407
 408 long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
 409                                 long nr_to_write, enum iostat_type io_type)
 410 {
 411         struct address_space *mapping = META_MAPPING(sbi);
 412         pgoff_t index = 0, prev = ULONG_MAX;
 413         struct folio_batch fbatch;
 414         long nwritten = 0;
 415         int nr_folios;
 416         struct writeback_control wbc = {
 417                 .for_reclaim = 0,
 418         };
 419         struct blk_plug plug;
 420
 421         folio_batch_init(&fbatch);
 422
 423         blk_start_plug(&plug);
 424
 425         while ((nr_folios = filemap_get_folios_tag(mapping, &index,
 426                                         (pgoff_t)-1,
 427                                         PAGECACHE_TAG_DIRTY, &fbatch))) {
 428                 int i;
 429
 430                 for (i = 0; i < nr_folios; i++) {
 431                         struct folio *folio = fbatch.folios[i];
 432
 433                         if (nr_to_write != LONG_MAX && i != 0 &&
 434                                         folio->index != prev +
 435                                         folio_nr_pages(fbatch.folios[i-1])) {
 436                                 folio_batch_release(&fbatch);
 437                                 goto stop;
 438                         }
 439
 440                         folio_lock(folio);
 441
 442                         if (unlikely(folio->mapping != mapping)) {
 443 continue_unlock:
 444                                 folio_unlock(folio);
 445                                 continue;
 446                         }
 447                         if (!folio_test_dirty(folio)) {
 448                                 /* someone wrote it for us */
 449                                 goto continue_unlock;
 450                         }
 451
 452                         f2fs_wait_on_page_writeback(&folio->page, META,
 453                                         true, true);
 454
 455                         if (!folio_clear_dirty_for_io(folio))
 456                                 goto continue_unlock;
 457
 458                         if (__f2fs_write_meta_page(&folio->page, &wbc,
 459                                                 io_type)) {
 460                                 folio_unlock(folio);
 461                                 break;
 462                         }
 463                         nwritten += folio_nr_pages(folio);
 464                         prev = folio->index;
 465                         if (unlikely(nwritten >= nr_to_write))
 466                                 break;
 467                 }
 468                 folio_batch_release(&fbatch);
 469                 cond_resched();
 470         }
 471 stop:
 472         if (nwritten)
 473                 f2fs_submit_merged_write(sbi, type);
 474
 475         blk_finish_plug(&plug);
 476
 477         return nwritten;
 478 }
 479
 480 static bool f2fs_dirty_meta_folio(struct address_space *mapping,
 481                 struct folio *folio)
 482 {
 483         trace_f2fs_set_page_dirty(&folio->page, META);
 484
 485         if (!folio_test_uptodate(folio))
 486                 folio_mark_uptodate(folio);
 487         if (filemap_dirty_folio(mapping, folio)) {
 488                 inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META);
 489                 set_page_private_reference(&folio->page);
 490                 return true;
 491         }
 492         return false;
 493 }
 494
 495 const struct address_space_operations f2fs_meta_aops = {
 496         .writepage      = f2fs_write_meta_page,
 497         .writepages     = f2fs_write_meta_pages,
 498         .dirty_folio    = f2fs_dirty_meta_folio,
 499         .invalidate_folio = f2fs_invalidate_folio,
 500         .release_folio  = f2fs_release_folio,
 501         .migrate_folio  = filemap_migrate_folio,
 502 };
 503
 504 static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino,
 505                                                 unsigned int devidx, int type)
 506 {
 507         struct inode_management *im = &sbi->im[type];
 508         struct ino_entry *e = NULL, *new = NULL;
 509
 510         if (type == FLUSH_INO) {
 511                 rcu_read_lock();
 512                 e = radix_tree_lookup(&im->ino_root, ino);
 513                 rcu_read_unlock();
 514         }
 515
 516 retry:
 517         if (!e)
 518                 new = f2fs_kmem_cache_alloc(ino_entry_slab,
 519                                                 GFP_NOFS, true, NULL);
 520
 521         radix_tree_preload(GFP_NOFS | __GFP_NOFAIL);
 522
 523         spin_lock(&im->ino_lock);
 524         e = radix_tree_lookup(&im->ino_root, ino);
 525         if (!e) {
 526                 if (!new) {
 527                         spin_unlock(&im->ino_lock);
 528                         radix_tree_preload_end();
 529                         goto retry;
 530                 }
 531                 e = new;
 532                 if (unlikely(radix_tree_insert(&im->ino_root, ino, e)))
 533                         f2fs_bug_on(sbi, 1);
 534
 535                 memset(e, 0, sizeof(struct ino_entry));
 536                 e->ino = ino;
 537
 538                 list_add_tail(&e->list, &im->ino_list);
 539                 if (type != ORPHAN_INO)
 540                         im->ino_num++;
 541         }
 542
 543         if (type == FLUSH_INO)
 544                 f2fs_set_bit(devidx, (char *)&e->dirty_device);
 545
 546         spin_unlock(&im->ino_lock);
 547         radix_tree_preload_end();
 548
 549         if (new && e != new)
 550                 kmem_cache_free(ino_entry_slab, new);
 551 }
 552
 553 static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 554 {
 555         struct inode_management *im = &sbi->im[type];
 556         struct ino_entry *e;
 557
 558         spin_lock(&im->ino_lock);
 559         e = radix_tree_lookup(&im->ino_root, ino);
 560         if (e) {
 561                 list_del(&e->list);
 562                 radix_tree_delete(&im->ino_root, ino);
 563                 im->ino_num--;
 564                 spin_unlock(&im->ino_lock);
 565                 kmem_cache_free(ino_entry_slab, e);
 566                 return;
 567         }
 568         spin_unlock(&im->ino_lock);
 569 }
 570
 571 void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 572 {
 573         /* add new dirty ino entry into list */
 574         __add_ino_entry(sbi, ino, 0, type);
 575 }
 576
 577 void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 578 {
 579         /* remove dirty ino entry from list */
 580         __remove_ino_entry(sbi, ino, type);
 581 }
 582
 583 /* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */
 584 bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
 585 {
 586         struct inode_management *im = &sbi->im[mode];
 587         struct ino_entry *e;
 588
 589         spin_lock(&im->ino_lock);
 590         e = radix_tree_lookup(&im->ino_root, ino);
 591         spin_unlock(&im->ino_lock);
 592         return e ? true : false;
 593 }
 594
 595 void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all)
 596 {
 597         struct ino_entry *e, *tmp;
 598         int i;
 599
 600         for (i = all ? ORPHAN_INO : APPEND_INO; i < MAX_INO_ENTRY; i++) {
 601                 struct inode_management *im = &sbi->im[i];
 602
 603                 spin_lock(&im->ino_lock);
 604                 list_for_each_entry_safe(e, tmp, &im->ino_list, list) {
 605                         list_del(&e->list);
 606                         radix_tree_delete(&im->ino_root, e->ino);
 607                         kmem_cache_free(ino_entry_slab, e);
 608                         im->ino_num--;
 609                 }
 610                 spin_unlock(&im->ino_lock);
 611         }
 612 }
 613
 614 void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
 615                                         unsigned int devidx, int type)
 616 {
 617         __add_ino_entry(sbi, ino, devidx, type);
 618 }
 619
 620 bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
 621                                         unsigned int devidx, int type)
 622 {
 623         struct inode_management *im = &sbi->im[type];
 624         struct ino_entry *e;
 625         bool is_dirty = false;
 626
 627         spin_lock(&im->ino_lock);
 628         e = radix_tree_lookup(&im->ino_root, ino);
 629         if (e && f2fs_test_bit(devidx, (char *)&e->dirty_device))
 630                 is_dirty = true;
 631         spin_unlock(&im->ino_lock);
 632         return is_dirty;
 633 }
 634
 635 int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi)
 636 {
 637         struct inode_management *im = &sbi->im[ORPHAN_INO];
 638         int err = 0;
 639
 640         spin_lock(&im->ino_lock);
 641
 642         if (time_to_inject(sbi, FAULT_ORPHAN)) {
 643                 spin_unlock(&im->ino_lock);
 644                 return -ENOSPC;
 645         }
 646
 647         if (unlikely(im->ino_num >= sbi->max_orphans))
 648                 err = -ENOSPC;
 649         else
 650                 im->ino_num++;
 651         spin_unlock(&im->ino_lock);
 652
 653         return err;
 654 }
 655
 656 void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi)
 657 {
 658         struct inode_management *im = &sbi->im[ORPHAN_INO];
 659
 660         spin_lock(&im->ino_lock);
 661         f2fs_bug_on(sbi, im->ino_num == 0);
 662         im->ino_num--;
 663         spin_unlock(&im->ino_lock);
 664 }
 665
 666 void f2fs_add_orphan_inode(struct inode *inode)
 667 {
 668         /* add new orphan ino entry into list */
 669         __add_ino_entry(F2FS_I_SB(inode), inode->i_ino, 0, ORPHAN_INO);
 670         f2fs_update_inode_page(inode);
 671 }
 672
 673 void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
 674 {
 675         /* remove orphan entry from orphan list */
 676         __remove_ino_entry(sbi, ino, ORPHAN_INO);
 677 }
 678
 679 static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
 680 {
 681         struct inode *inode;
 682         struct node_info ni;
 683         int err;
 684
 685         inode = f2fs_iget_retry(sbi->sb, ino);
 686         if (IS_ERR(inode)) {
 687                 /*
 688                  * there should be a bug that we can't find the entry
 689                  * to orphan inode.
 690                  */
 691                 f2fs_bug_on(sbi, PTR_ERR(inode) == -ENOENT);
 692                 return PTR_ERR(inode);
 693         }
 694
 695         err = f2fs_dquot_initialize(inode);
 696         if (err) {
 697                 iput(inode);
 698                 goto err_out;
 699         }
 700
 701         clear_nlink(inode);
 702
 703         /* truncate all the data during iput */
 704         iput(inode);
 705
 706         err = f2fs_get_node_info(sbi, ino, &ni, false);
 707         if (err)
 708                 goto err_out;
 709
 710         /* ENOMEM was fully retried in f2fs_evict_inode. */
 711         if (ni.blk_addr != NULL_ADDR) {
 712                 err = -EIO;
 713                 goto err_out;
 714         }
 715         return 0;
 716
 717 err_out:
 718         set_sbi_flag(sbi, SBI_NEED_FSCK);
 719         f2fs_warn(sbi, "%s: orphan failed (ino=%x), run fsck to fix.",
 720                   __func__, ino);
 721         return err;
 722 }
 723
 724 int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi)
 725 {
 726         block_t start_blk, orphan_blocks, i, j;
 727         int err = 0;
 728
 729         if (!is_set_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG))
 730                 return 0;
 731
 732         if (f2fs_hw_is_readonly(sbi)) {
 733                 f2fs_info(sbi, "write access unavailable, skipping orphan cleanup");
 734                 return 0;
 735         }
 736
 737         if (is_sbi_flag_set(sbi, SBI_IS_WRITABLE))
 738                 f2fs_info(sbi, "orphan cleanup on readonly fs");
 739
 740         start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi);
 741         orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi);
 742
 743         f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true);
 744
 745         for (i = 0; i < orphan_blocks; i++) {
 746                 struct page *page;
 747                 struct f2fs_orphan_block *orphan_blk;
 748
 749                 page = f2fs_get_meta_page(sbi, start_blk + i);
 750                 if (IS_ERR(page)) {
 751                         err = PTR_ERR(page);
 752                         goto out;
 753                 }
 754
 755                 orphan_blk = (struct f2fs_orphan_block *)page_address(page);
 756                 for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) {
 757                         nid_t ino = le32_to_cpu(orphan_blk->ino[j]);
 758
 759                         err = recover_orphan_inode(sbi, ino);
 760                         if (err) {
 761                                 f2fs_put_page(page, 1);
 762                                 goto out;
 763                         }
 764                 }
 765                 f2fs_put_page(page, 1);
 766         }
 767         /* clear Orphan Flag */
 768         clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG);
 769 out:
 770         set_sbi_flag(sbi, SBI_IS_RECOVERED);
 771
 772         return err;
 773 }
 774
 775 static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
 776 {
 777         struct list_head *head;
 778         struct f2fs_orphan_block *orphan_blk = NULL;
 779         unsigned int nentries = 0;
 780         unsigned short index = 1;
 781         unsigned short orphan_blocks;
 782         struct page *page = NULL;
 783         struct ino_entry *orphan = NULL;
 784         struct inode_management *im = &sbi->im[ORPHAN_INO];
 785
 786         orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num);
 787
 788         /*
 789          * we don't need to do spin_lock(&im->ino_lock) here, since all the
 790          * orphan inode operations are covered under f2fs_lock_op().
 791          * And, spin_lock should be avoided due to page operations below.
 792          */
 793         head = &im->ino_list;
 794
 795         /* loop for each orphan inode entry and write them in journal block */
 796         list_for_each_entry(orphan, head, list) {
 797                 if (!page) {
 798                         page = f2fs_grab_meta_page(sbi, start_blk++);
 799                         orphan_blk =
 800                                 (struct f2fs_orphan_block *)page_address(page);
 801                         memset(orphan_blk, 0, sizeof(*orphan_blk));
 802                 }
 803
 804                 orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
 805
 806                 if (nentries == F2FS_ORPHANS_PER_BLOCK) {
 807                         /*
 808                          * an orphan block is full of 1020 entries,
 809                          * then we need to flush current orphan blocks
 810                          * and bring another one in memory
 811                          */
 812                         orphan_blk->blk_addr = cpu_to_le16(index);
 813                         orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
 814                         orphan_blk->entry_count = cpu_to_le32(nentries);
 815                         set_page_dirty(page);
 816                         f2fs_put_page(page, 1);
 817                         index++;
 818                         nentries = 0;
 819                         page = NULL;
 820                 }
 821         }
 822
 823         if (page) {
 824                 orphan_blk->blk_addr = cpu_to_le16(index);
 825                 orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
 826                 orphan_blk->entry_count = cpu_to_le32(nentries);
 827                 set_page_dirty(page);
 828                 f2fs_put_page(page, 1);
 829         }
 830 }
 831
 832 static __u32 f2fs_checkpoint_chksum(struct f2fs_sb_info *sbi,
 833                                                 struct f2fs_checkpoint *ckpt)
 834 {
 835         unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset);
 836         __u32 chksum;
 837
 838         chksum = f2fs_crc32(sbi, ckpt, chksum_ofs);
 839         if (chksum_ofs < CP_CHKSUM_OFFSET) {
 840                 chksum_ofs += sizeof(chksum);
 841                 chksum = f2fs_chksum(sbi, chksum, (__u8 *)ckpt + chksum_ofs,
 842                                                 F2FS_BLKSIZE - chksum_ofs);
 843         }
 844         return chksum;
 845 }
 846
 847 static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr,
 848                 struct f2fs_checkpoint **cp_block, struct page **cp_page,
 849                 unsigned long long *version)
 850 {
 851         size_t crc_offset = 0;
 852         __u32 crc;
 853
 854         *cp_page = f2fs_get_meta_page(sbi, cp_addr);
 855         if (IS_ERR(*cp_page))
 856                 return PTR_ERR(*cp_page);
 857
 858         *cp_block = (struct f2fs_checkpoint *)page_address(*cp_page);
 859
 860         crc_offset = le32_to_cpu((*cp_block)->checksum_offset);
 861         if (crc_offset < CP_MIN_CHKSUM_OFFSET ||
 862                         crc_offset > CP_CHKSUM_OFFSET) {
 863                 f2fs_put_page(*cp_page, 1);
 864                 f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset);
 865                 return -EINVAL;
 866         }
 867
 868         crc = f2fs_checkpoint_chksum(sbi, *cp_block);
 869         if (crc != cur_cp_crc(*cp_block)) {
 870                 f2fs_put_page(*cp_page, 1);
 871                 f2fs_warn(sbi, "invalid crc value");
 872                 return -EINVAL;
 873         }
 874
 875         *version = cur_cp_version(*cp_block);
 876         return 0;
 877 }
 878
 879 static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
 880                                 block_t cp_addr, unsigned long long *version)
 881 {
 882         struct page *cp_page_1 = NULL, *cp_page_2 = NULL;
 883         struct f2fs_checkpoint *cp_block = NULL;
 884         unsigned long long cur_version = 0, pre_version = 0;
 885         unsigned int cp_blocks;
 886         int err;
 887
 888         err = get_checkpoint_version(sbi, cp_addr, &cp_block,
 889                                         &cp_page_1, version);
 890         if (err)
 891                 return NULL;
 892
 893         cp_blocks = le32_to_cpu(cp_block->cp_pack_total_block_count);
 894
 895         if (cp_blocks > sbi->blocks_per_seg || cp_blocks <= F2FS_CP_PACKS) {
 896                 f2fs_warn(sbi, "invalid cp_pack_total_block_count:%u",
 897                           le32_to_cpu(cp_block->cp_pack_total_block_count));
 898                 goto invalid_cp;
 899         }
 900         pre_version = *version;
 901
 902         cp_addr += cp_blocks - 1;
 903         err = get_checkpoint_version(sbi, cp_addr, &cp_block,
 904                                         &cp_page_2, version);
 905         if (err)
 906                 goto invalid_cp;
 907         cur_version = *version;
 908
 909         if (cur_version == pre_version) {
 910                 *version = cur_version;
 911                 f2fs_put_page(cp_page_2, 1);
 912                 return cp_page_1;
 913         }
 914         f2fs_put_page(cp_page_2, 1);
 915 invalid_cp:
 916         f2fs_put_page(cp_page_1, 1);
 917         return NULL;
 918 }
 919
 920 int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
 921 {
 922         struct f2fs_checkpoint *cp_block;
 923         struct f2fs_super_block *fsb = sbi->raw_super;
 924         struct page *cp1, *cp2, *cur_page;
 925         unsigned long blk_size = sbi->blocksize;
 926         unsigned long long cp1_version = 0, cp2_version = 0;
 927         unsigned long long cp_start_blk_no;
 928         unsigned int cp_blks = 1 + __cp_payload(sbi);
 929         block_t cp_blk_no;
 930         int i;
 931         int err;
 932
 933         sbi->ckpt = f2fs_kvzalloc(sbi, array_size(blk_size, cp_blks),
 934                                   GFP_KERNEL);
 935         if (!sbi->ckpt)
 936                 return -ENOMEM;
 937         /*
 938          * Finding out valid cp block involves read both
 939          * sets( cp pack 1 and cp pack 2)
 940          */
 941         cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr);
 942         cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version);
 943
 944         /* The second checkpoint pack should start at the next segment */
 945         cp_start_blk_no += ((unsigned long long)1) <<
 946                                 le32_to_cpu(fsb->log_blocks_per_seg);
 947         cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version);
 948
 949         if (cp1 && cp2) {
 950                 if (ver_after(cp2_version, cp1_version))
 951                         cur_page = cp2;
 952                 else
 953                         cur_page = cp1;
 954         } else if (cp1) {
 955                 cur_page = cp1;
 956         } else if (cp2) {
 957                 cur_page = cp2;
 958         } else {
 959                 err = -EFSCORRUPTED;
 960                 goto fail_no_cp;
 961         }
 962
 963         cp_block = (struct f2fs_checkpoint *)page_address(cur_page);
 964         memcpy(sbi->ckpt, cp_block, blk_size);
 965
 966         if (cur_page == cp1)
 967                 sbi->cur_cp_pack = 1;
 968         else
 969                 sbi->cur_cp_pack = 2;
 970
 971         /* Sanity checking of checkpoint */
 972         if (f2fs_sanity_check_ckpt(sbi)) {
 973                 err = -EFSCORRUPTED;
 974                 goto free_fail_no_cp;
 975         }
 976
 977         if (cp_blks <= 1)
 978                 goto done;
 979
 980         cp_blk_no = le32_to_cpu(fsb->cp_blkaddr);
 981         if (cur_page == cp2)
 982                 cp_blk_no += BIT(le32_to_cpu(fsb->log_blocks_per_seg));
 983
 984         for (i = 1; i < cp_blks; i++) {
 985                 void *sit_bitmap_ptr;
 986                 unsigned char *ckpt = (unsigned char *)sbi->ckpt;
 987
 988                 cur_page = f2fs_get_meta_page(sbi, cp_blk_no + i);
 989                 if (IS_ERR(cur_page)) {
 990                         err = PTR_ERR(cur_page);
 991                         goto free_fail_no_cp;
 992                 }
 993                 sit_bitmap_ptr = page_address(cur_page);
 994                 memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size);
 995                 f2fs_put_page(cur_page, 1);
 996         }
 997 done:
 998         f2fs_put_page(cp1, 1);
 999         f2fs_put_page(cp2, 1);
1000         return 0;
1001
1002 free_fail_no_cp:
1003         f2fs_put_page(cp1, 1);
1004         f2fs_put_page(cp2, 1);
1005 fail_no_cp:
1006         kvfree(sbi->ckpt);
1007         return err;
1008 }
1009
1010 static void __add_dirty_inode(struct inode *inode, enum inode_type type)
1011 {
1012         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1013         int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1014
1015         if (is_inode_flag_set(inode, flag))
1016                 return;
1017
1018         set_inode_flag(inode, flag);
1019         list_add_tail(&F2FS_I(inode)->dirty_list, &sbi->inode_list[type]);
1020         stat_inc_dirty_inode(sbi, type);
1021 }
1022
1023 static void __remove_dirty_inode(struct inode *inode, enum inode_type type)
1024 {
1025         int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1026
1027         if (get_dirty_pages(inode) || !is_inode_flag_set(inode, flag))
1028                 return;
1029
1030         list_del_init(&F2FS_I(inode)->dirty_list);
1031         clear_inode_flag(inode, flag);
1032         stat_dec_dirty_inode(F2FS_I_SB(inode), type);
1033 }
1034
1035 void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio)
1036 {
1037         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1038         enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1039
1040         if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1041                         !S_ISLNK(inode->i_mode))
1042                 return;
1043
1044         spin_lock(&sbi->inode_lock[type]);
1045         if (type != FILE_INODE || test_opt(sbi, DATA_FLUSH))
1046                 __add_dirty_inode(inode, type);
1047         inode_inc_dirty_pages(inode);
1048         spin_unlock(&sbi->inode_lock[type]);
1049
1050         set_page_private_reference(&folio->page);
1051 }
1052
1053 void f2fs_remove_dirty_inode(struct inode *inode)
1054 {
1055         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1056         enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1057
1058         if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1059                         !S_ISLNK(inode->i_mode))
1060                 return;
1061
1062         if (type == FILE_INODE && !test_opt(sbi, DATA_FLUSH))
1063                 return;
1064
1065         spin_lock(&sbi->inode_lock[type]);
1066         __remove_dirty_inode(inode, type);
1067         spin_unlock(&sbi->inode_lock[type]);
1068 }
1069
1070 int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
1071                                                 bool from_cp)
1072 {
1073         struct list_head *head;
1074         struct inode *inode;
1075         struct f2fs_inode_info *fi;
1076         bool is_dir = (type == DIR_INODE);
1077         unsigned long ino = 0;
1078
1079         trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir,
1080                                 get_pages(sbi, is_dir ?
1081                                 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1082 retry:
1083         if (unlikely(f2fs_cp_error(sbi))) {
1084                 trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1085                                 get_pages(sbi, is_dir ?
1086                                 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1087                 return -EIO;
1088         }
1089
1090         spin_lock(&sbi->inode_lock[type]);
1091
1092         head = &sbi->inode_list[type];
1093         if (list_empty(head)) {
1094                 spin_unlock(&sbi->inode_lock[type]);
1095                 trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1096                                 get_pages(sbi, is_dir ?
1097                                 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1098                 return 0;
1099         }
1100         fi = list_first_entry(head, struct f2fs_inode_info, dirty_list);
1101         inode = igrab(&fi->vfs_inode);
1102         spin_unlock(&sbi->inode_lock[type]);
1103         if (inode) {
1104                 unsigned long cur_ino = inode->i_ino;
1105
1106                 if (from_cp)
1107                         F2FS_I(inode)->cp_task = current;
1108                 F2FS_I(inode)->wb_task = current;
1109
1110                 filemap_fdatawrite(inode->i_mapping);
1111
1112                 F2FS_I(inode)->wb_task = NULL;
1113                 if (from_cp)
1114                         F2FS_I(inode)->cp_task = NULL;
1115
1116                 iput(inode);
1117                 /* We need to give cpu to another writers. */
1118                 if (ino == cur_ino)
1119                         cond_resched();
1120                 else
1121                         ino = cur_ino;
1122         } else {
1123                 /*
1124                  * We should submit bio, since it exists several
1125                  * writebacking dentry pages in the freeing inode.
1126                  */
1127                 f2fs_submit_merged_write(sbi, DATA);
1128                 cond_resched();
1129         }
1130         goto retry;
1131 }
1132
1133 static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
1134 {
1135         struct list_head *head = &sbi->inode_list[DIRTY_META];
1136         struct inode *inode;
1137         struct f2fs_inode_info *fi;
1138         s64 total = get_pages(sbi, F2FS_DIRTY_IMETA);
1139
1140         while (total--) {
1141                 if (unlikely(f2fs_cp_error(sbi)))
1142                         return -EIO;
1143
1144                 spin_lock(&sbi->inode_lock[DIRTY_META]);
1145                 if (list_empty(head)) {
1146                         spin_unlock(&sbi->inode_lock[DIRTY_META]);
1147                         return 0;
1148                 }
1149                 fi = list_first_entry(head, struct f2fs_inode_info,
1150                                                         gdirty_list);
1151                 inode = igrab(&fi->vfs_inode);
1152                 spin_unlock(&sbi->inode_lock[DIRTY_META]);
1153                 if (inode) {
1154                         sync_inode_metadata(inode, 0);
1155
1156                         /* it's on eviction */
1157                         if (is_inode_flag_set(inode, FI_DIRTY_INODE))
1158                                 f2fs_update_inode_page(inode);
1159                         iput(inode);
1160                 }
1161         }
1162         return 0;
1163 }
1164
1165 static void __prepare_cp_block(struct f2fs_sb_info *sbi)
1166 {
1167         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1168         struct f2fs_nm_info *nm_i = NM_I(sbi);
1169         nid_t last_nid = nm_i->next_scan_nid;
1170
1171         next_free_nid(sbi, &last_nid);
1172         ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi));
1173         ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi));
1174         ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi));
1175         ckpt->next_free_nid = cpu_to_le32(last_nid);
1176 }
1177
1178 static bool __need_flush_quota(struct f2fs_sb_info *sbi)
1179 {
1180         bool ret = false;
1181
1182         if (!is_journalled_quota(sbi))
1183                 return false;
1184
1185         if (!f2fs_down_write_trylock(&sbi->quota_sem))
1186                 return true;
1187         if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
1188                 ret = false;
1189         } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
1190                 ret = false;
1191         } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
1192                 clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1193                 ret = true;
1194         } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
1195                 ret = true;
1196         }
1197         f2fs_up_write(&sbi->quota_sem);
1198         return ret;
1199 }
1200
1201 /*
1202  * Freeze all the FS-operations for checkpoint.
1203  */
1204 static int block_operations(struct f2fs_sb_info *sbi)
1205 {
1206         struct writeback_control wbc = {
1207                 .sync_mode = WB_SYNC_ALL,
1208                 .nr_to_write = LONG_MAX,
1209                 .for_reclaim = 0,
1210         };
1211         int err = 0, cnt = 0;
1212
1213         /*
1214          * Let's flush inline_data in dirty node pages.
1215          */
1216         f2fs_flush_inline_data(sbi);
1217
1218 retry_flush_quotas:
1219         f2fs_lock_all(sbi);
1220         if (__need_flush_quota(sbi)) {
1221                 int locked;
1222
1223                 if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) {
1224                         set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1225                         set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1226                         goto retry_flush_dents;
1227                 }
1228                 f2fs_unlock_all(sbi);
1229
1230                 /* only failed during mount/umount/freeze/quotactl */
1231                 locked = down_read_trylock(&sbi->sb->s_umount);
1232                 f2fs_quota_sync(sbi->sb, -1);
1233                 if (locked)
1234                         up_read(&sbi->sb->s_umount);
1235                 cond_resched();
1236                 goto retry_flush_quotas;
1237         }
1238
1239 retry_flush_dents:
1240         /* write all the dirty dentry pages */
1241         if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
1242                 f2fs_unlock_all(sbi);
1243                 err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true);
1244                 if (err)
1245                         return err;
1246                 cond_resched();
1247                 goto retry_flush_quotas;
1248         }
1249
1250         /*
1251          * POR: we should ensure that there are no dirty node pages
1252          * until finishing nat/sit flush. inode->i_blocks can be updated.
1253          */
1254         f2fs_down_write(&sbi->node_change);
1255
1256         if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
1257                 f2fs_up_write(&sbi->node_change);
1258                 f2fs_unlock_all(sbi);
1259                 err = f2fs_sync_inode_meta(sbi);
1260                 if (err)
1261                         return err;
1262                 cond_resched();
1263                 goto retry_flush_quotas;
1264         }
1265
1266 retry_flush_nodes:
1267         f2fs_down_write(&sbi->node_write);
1268
1269         if (get_pages(sbi, F2FS_DIRTY_NODES)) {
1270                 f2fs_up_write(&sbi->node_write);
1271                 atomic_inc(&sbi->wb_sync_req[NODE]);
1272                 err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO);
1273                 atomic_dec(&sbi->wb_sync_req[NODE]);
1274                 if (err) {
1275                         f2fs_up_write(&sbi->node_change);
1276                         f2fs_unlock_all(sbi);
1277                         return err;
1278                 }
1279                 cond_resched();
1280                 goto retry_flush_nodes;
1281         }
1282
1283         /*
1284          * sbi->node_change is used only for AIO write_begin path which produces
1285          * dirty node blocks and some checkpoint values by block allocation.
1286          */
1287         __prepare_cp_block(sbi);
1288         f2fs_up_write(&sbi->node_change);
1289         return err;
1290 }
1291
1292 static void unblock_operations(struct f2fs_sb_info *sbi)
1293 {
1294         f2fs_up_write(&sbi->node_write);
1295         f2fs_unlock_all(sbi);
1296 }
1297
1298 void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type)
1299 {
1300         DEFINE_WAIT(wait);
1301
1302         for (;;) {
1303                 if (!get_pages(sbi, type))
1304                         break;
1305
1306                 if (unlikely(f2fs_cp_error(sbi) &&
1307                         !is_sbi_flag_set(sbi, SBI_IS_CLOSE)))
1308                         break;
1309
1310                 if (type == F2FS_DIRTY_META)
1311                         f2fs_sync_meta_pages(sbi, META, LONG_MAX,
1312                                                         FS_CP_META_IO);
1313                 else if (type == F2FS_WB_CP_DATA)
1314                         f2fs_submit_merged_write(sbi, DATA);
1315
1316                 prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);
1317                 io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1318         }
1319         finish_wait(&sbi->cp_wait, &wait);
1320 }
1321
1322 static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1323 {
1324         unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num;
1325         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1326         unsigned long flags;
1327
1328         if (cpc->reason & CP_UMOUNT) {
1329                 if (le32_to_cpu(ckpt->cp_pack_total_block_count) +
1330                         NM_I(sbi)->nat_bits_blocks > sbi->blocks_per_seg) {
1331                         clear_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
1332                         f2fs_notice(sbi, "Disable nat_bits due to no space");
1333                 } else if (!is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG) &&
1334                                                 f2fs_nat_bitmap_enabled(sbi)) {
1335                         f2fs_enable_nat_bits(sbi);
1336                         set_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
1337                         f2fs_notice(sbi, "Rebuild and enable nat_bits");
1338                 }
1339         }
1340
1341         spin_lock_irqsave(&sbi->cp_lock, flags);
1342
1343         if (cpc->reason & CP_TRIMMED)
1344                 __set_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1345         else
1346                 __clear_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1347
1348         if (cpc->reason & CP_UMOUNT)
1349                 __set_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1350         else
1351                 __clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1352
1353         if (cpc->reason & CP_FASTBOOT)
1354                 __set_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1355         else
1356                 __clear_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1357
1358         if (orphan_num)
1359                 __set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1360         else
1361                 __clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1362
1363         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1364                 __set_ckpt_flags(ckpt, CP_FSCK_FLAG);
1365
1366         if (is_sbi_flag_set(sbi, SBI_IS_RESIZEFS))
1367                 __set_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1368         else
1369                 __clear_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1370
1371         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED))
1372                 __set_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1373         else
1374                 __clear_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1375
1376         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK))
1377                 __set_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1378         else
1379                 __clear_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1380
1381         if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH))
1382                 __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1383         else
1384                 __clear_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1385
1386         if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR))
1387                 __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1388
1389         /* set this flag to activate crc|cp_ver for recovery */
1390         __set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG);
1391         __clear_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG);
1392
1393         spin_unlock_irqrestore(&sbi->cp_lock, flags);
1394 }
1395
1396 static void commit_checkpoint(struct f2fs_sb_info *sbi,
1397         void *src, block_t blk_addr)
1398 {
1399         struct writeback_control wbc = {
1400                 .for_reclaim = 0,
1401         };
1402
1403         /*
1404          * filemap_get_folios_tag and lock_page again will take
1405          * some extra time. Therefore, f2fs_update_meta_pages and
1406          * f2fs_sync_meta_pages are combined in this function.
1407          */
1408         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
1409         int err;
1410
1411         f2fs_wait_on_page_writeback(page, META, true, true);
1412
1413         memcpy(page_address(page), src, PAGE_SIZE);
1414
1415         set_page_dirty(page);
1416         if (unlikely(!clear_page_dirty_for_io(page)))
1417                 f2fs_bug_on(sbi, 1);
1418
1419         /* writeout cp pack 2 page */
1420         err = __f2fs_write_meta_page(page, &wbc, FS_CP_META_IO);
1421         if (unlikely(err && f2fs_cp_error(sbi))) {
1422                 f2fs_put_page(page, 1);
1423                 return;
1424         }
1425
1426         f2fs_bug_on(sbi, err);
1427         f2fs_put_page(page, 0);
1428
1429         /* submit checkpoint (with barrier if NOBARRIER is not set) */
1430         f2fs_submit_merged_write(sbi, META_FLUSH);
1431 }
1432
1433 static inline u64 get_sectors_written(struct block_device *bdev)
1434 {
1435         return (u64)part_stat_read(bdev, sectors[STAT_WRITE]);
1436 }
1437
1438 u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi)
1439 {
1440         if (f2fs_is_multi_device(sbi)) {
1441                 u64 sectors = 0;
1442                 int i;
1443
1444                 for (i = 0; i < sbi->s_ndevs; i++)
1445                         sectors += get_sectors_written(FDEV(i).bdev);
1446
1447                 return sectors;
1448         }
1449
1450         return get_sectors_written(sbi->sb->s_bdev);
1451 }
1452
1453 static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1454 {
1455         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1456         struct f2fs_nm_info *nm_i = NM_I(sbi);
1457         unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num, flags;
1458         block_t start_blk;
1459         unsigned int data_sum_blocks, orphan_blocks;
1460         __u32 crc32 = 0;
1461         int i;
1462         int cp_payload_blks = __cp_payload(sbi);
1463         struct curseg_info *seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE);
1464         u64 kbytes_written;
1465         int err;
1466
1467         /* Flush all the NAT/SIT pages */
1468         f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1469
1470         /* start to update checkpoint, cp ver is already updated previously */
1471         ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi, true));
1472         ckpt->free_segment_count = cpu_to_le32(free_segments(sbi));
1473         for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) {
1474                 struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_NODE);
1475
1476                 ckpt->cur_node_segno[i] = cpu_to_le32(curseg->segno);
1477                 ckpt->cur_node_blkoff[i] = cpu_to_le16(curseg->next_blkoff);
1478                 ckpt->alloc_type[i + CURSEG_HOT_NODE] = curseg->alloc_type;
1479         }
1480         for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) {
1481                 struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_DATA);
1482
1483                 ckpt->cur_data_segno[i] = cpu_to_le32(curseg->segno);
1484                 ckpt->cur_data_blkoff[i] = cpu_to_le16(curseg->next_blkoff);
1485                 ckpt->alloc_type[i + CURSEG_HOT_DATA] = curseg->alloc_type;
1486         }
1487
1488         /* 2 cp + n data seg summary + orphan inode blocks */
1489         data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false);
1490         spin_lock_irqsave(&sbi->cp_lock, flags);
1491         if (data_sum_blocks < NR_CURSEG_DATA_TYPE)
1492                 __set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1493         else
1494                 __clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1495         spin_unlock_irqrestore(&sbi->cp_lock, flags);
1496
1497         orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num);
1498         ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
1499                         orphan_blocks);
1500
1501         if (__remain_node_summaries(cpc->reason))
1502                 ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1503                                 cp_payload_blks + data_sum_blocks +
1504                                 orphan_blocks + NR_CURSEG_NODE_TYPE);
1505         else
1506                 ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1507                                 cp_payload_blks + data_sum_blocks +
1508                                 orphan_blocks);
1509
1510         /* update ckpt flag for checkpoint */
1511         update_ckpt_flags(sbi, cpc);
1512
1513         /* update SIT/NAT bitmap */
1514         get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
1515         get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));
1516
1517         crc32 = f2fs_checkpoint_chksum(sbi, ckpt);
1518         *((__le32 *)((unsigned char *)ckpt +
1519                                 le32_to_cpu(ckpt->checksum_offset)))
1520                                 = cpu_to_le32(crc32);
1521
1522         start_blk = __start_cp_next_addr(sbi);
1523
1524         /* write nat bits */
1525         if ((cpc->reason & CP_UMOUNT) &&
1526                         is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG)) {
1527                 __u64 cp_ver = cur_cp_version(ckpt);
1528                 block_t blk;
1529
1530                 cp_ver |= ((__u64)crc32 << 32);
1531                 *(__le64 *)nm_i->nat_bits = cpu_to_le64(cp_ver);
1532
1533                 blk = start_blk + sbi->blocks_per_seg - nm_i->nat_bits_blocks;
1534                 for (i = 0; i < nm_i->nat_bits_blocks; i++)
1535                         f2fs_update_meta_page(sbi, nm_i->nat_bits +
1536                                         (i << F2FS_BLKSIZE_BITS), blk + i);
1537         }
1538
1539         /* write out checkpoint buffer at block 0 */
1540         f2fs_update_meta_page(sbi, ckpt, start_blk++);
1541
1542         for (i = 1; i < 1 + cp_payload_blks; i++)
1543                 f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE,
1544                                                         start_blk++);
1545
1546         if (orphan_num) {
1547                 write_orphan_inodes(sbi, start_blk);
1548                 start_blk += orphan_blocks;
1549         }
1550
1551         f2fs_write_data_summaries(sbi, start_blk);
1552         start_blk += data_sum_blocks;
1553
1554         /* Record write statistics in the hot node summary */
1555         kbytes_written = sbi->kbytes_written;
1556         kbytes_written += (f2fs_get_sectors_written(sbi) -
1557                                 sbi->sectors_written_start) >> 1;
1558         seg_i->journal->info.kbytes_written = cpu_to_le64(kbytes_written);
1559
1560         if (__remain_node_summaries(cpc->reason)) {
1561                 f2fs_write_node_summaries(sbi, start_blk);
1562                 start_blk += NR_CURSEG_NODE_TYPE;
1563         }
1564
1565         /* update user_block_counts */
1566         sbi->last_valid_block_count = sbi->total_valid_block_count;
1567         percpu_counter_set(&sbi->alloc_valid_block_count, 0);
1568         percpu_counter_set(&sbi->rf_node_block_count, 0);
1569
1570         /* Here, we have one bio having CP pack except cp pack 2 page */
1571         f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1572         /* Wait for all dirty meta pages to be submitted for IO */
1573         f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META);
1574
1575         /* wait for previous submitted meta pages writeback */
1576         f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1577
1578         /* flush all device cache */
1579         err = f2fs_flush_device_cache(sbi);
1580         if (err)
1581                 return err;
1582
1583         /* barrier and flush checkpoint cp pack 2 page if it can */
1584         commit_checkpoint(sbi, ckpt, start_blk);
1585         f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1586
1587         /*
1588          * invalidate intermediate page cache borrowed from meta inode which are
1589          * used for migration of encrypted, verity or compressed inode's blocks.
1590          */
1591         if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) ||
1592                 f2fs_sb_has_compression(sbi))
1593                 invalidate_mapping_pages(META_MAPPING(sbi),
1594                                 MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1);
1595
1596         f2fs_release_ino_entry(sbi, false);
1597
1598         f2fs_reset_fsync_node_info(sbi);
1599
1600         clear_sbi_flag(sbi, SBI_IS_DIRTY);
1601         clear_sbi_flag(sbi, SBI_NEED_CP);
1602         clear_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1603
1604         spin_lock(&sbi->stat_lock);
1605         sbi->unusable_block_count = 0;
1606         spin_unlock(&sbi->stat_lock);
1607
1608         __set_cp_next_pack(sbi);
1609
1610         /*
1611          * redirty superblock if metadata like node page or inode cache is
1612          * updated during writing checkpoint.
1613          */
1614         if (get_pages(sbi, F2FS_DIRTY_NODES) ||
1615                         get_pages(sbi, F2FS_DIRTY_IMETA))
1616                 set_sbi_flag(sbi, SBI_IS_DIRTY);
1617
1618         f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS));
1619
1620         return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0;
1621 }
1622
1623 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1624 {
1625         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1626         unsigned long long ckpt_ver;
1627         int err = 0;
1628
1629         if (f2fs_readonly(sbi->sb) || f2fs_hw_is_readonly(sbi))
1630                 return -EROFS;
1631
1632         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
1633                 if (cpc->reason != CP_PAUSE)
1634                         return 0;
1635                 f2fs_warn(sbi, "Start checkpoint disabled!");
1636         }
1637         if (cpc->reason != CP_RESIZE)
1638                 f2fs_down_write(&sbi->cp_global_sem);
1639
1640         if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) &&
1641                 ((cpc->reason & CP_FASTBOOT) || (cpc->reason & CP_SYNC) ||
1642                 ((cpc->reason & CP_DISCARD) && !sbi->discard_blks)))
1643                 goto out;
1644         if (unlikely(f2fs_cp_error(sbi))) {
1645                 err = -EIO;
1646                 goto out;
1647         }
1648
1649         trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "start block_ops");
1650
1651         err = block_operations(sbi);
1652         if (err)
1653                 goto out;
1654
1655         trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish block_ops");
1656
1657         f2fs_flush_merged_writes(sbi);
1658
1659         /* this is the case of multiple fstrims without any changes */
1660         if (cpc->reason & CP_DISCARD) {
1661                 if (!f2fs_exist_trim_candidates(sbi, cpc)) {
1662                         unblock_operations(sbi);
1663                         goto out;
1664                 }
1665
1666                 if (NM_I(sbi)->nat_cnt[DIRTY_NAT] == 0 &&
1667                                 SIT_I(sbi)->dirty_sentries == 0 &&
1668                                 prefree_segments(sbi) == 0) {
1669                         f2fs_flush_sit_entries(sbi, cpc);
1670                         f2fs_clear_prefree_segments(sbi, cpc);
1671                         unblock_operations(sbi);
1672                         goto out;
1673                 }
1674         }
1675
1676         /*
1677          * update checkpoint pack index
1678          * Increase the version number so that
1679          * SIT entries and seg summaries are written at correct place
1680          */
1681         ckpt_ver = cur_cp_version(ckpt);
1682         ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver);
1683
1684         /* write cached NAT/SIT entries to NAT/SIT area */
1685         err = f2fs_flush_nat_entries(sbi, cpc);
1686         if (err) {
1687                 f2fs_err(sbi, "f2fs_flush_nat_entries failed err:%d, stop checkpoint", err);
1688                 f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1689                 goto stop;
1690         }
1691
1692         f2fs_flush_sit_entries(sbi, cpc);
1693
1694         /* save inmem log status */
1695         f2fs_save_inmem_curseg(sbi);
1696
1697         err = do_checkpoint(sbi, cpc);
1698         if (err) {
1699                 f2fs_err(sbi, "do_checkpoint failed err:%d, stop checkpoint", err);
1700                 f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1701                 f2fs_release_discard_addrs(sbi);
1702         } else {
1703                 f2fs_clear_prefree_segments(sbi, cpc);
1704         }
1705
1706         f2fs_restore_inmem_curseg(sbi);
1707 stop:
1708         unblock_operations(sbi);
1709         stat_inc_cp_count(sbi->stat_info);
1710
1711         if (cpc->reason & CP_RECOVERY)
1712                 f2fs_notice(sbi, "checkpoint: version = %llx", ckpt_ver);
1713
1714         /* update CP_TIME to trigger checkpoint periodically */
1715         f2fs_update_time(sbi, CP_TIME);
1716         trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint");
1717 out:
1718         if (cpc->reason != CP_RESIZE)
1719                 f2fs_up_write(&sbi->cp_global_sem);
1720         return err;
1721 }
1722
1723 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi)
1724 {
1725         int i;
1726
1727         for (i = 0; i < MAX_INO_ENTRY; i++) {
1728                 struct inode_management *im = &sbi->im[i];
1729
1730                 INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC);
1731                 spin_lock_init(&im->ino_lock);
1732                 INIT_LIST_HEAD(&im->ino_list);
1733                 im->ino_num = 0;
1734         }
1735
1736         sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS -
1737                         NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) *
1738                                 F2FS_ORPHANS_PER_BLOCK;
1739 }
1740
1741 int __init f2fs_create_checkpoint_caches(void)
1742 {
1743         ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry",
1744                         sizeof(struct ino_entry));
1745         if (!ino_entry_slab)
1746                 return -ENOMEM;
1747         f2fs_inode_entry_slab = f2fs_kmem_cache_create("f2fs_inode_entry",
1748                         sizeof(struct inode_entry));
1749         if (!f2fs_inode_entry_slab) {
1750                 kmem_cache_destroy(ino_entry_slab);
1751                 return -ENOMEM;
1752         }
1753         return 0;
1754 }
1755
1756 void f2fs_destroy_checkpoint_caches(void)
1757 {
1758         kmem_cache_destroy(ino_entry_slab);
1759         kmem_cache_destroy(f2fs_inode_entry_slab);
1760 }
1761
1762 static int __write_checkpoint_sync(struct f2fs_sb_info *sbi)
1763 {
1764         struct cp_control cpc = { .reason = CP_SYNC, };
1765         int err;
1766
1767         f2fs_down_write(&sbi->gc_lock);
1768         err = f2fs_write_checkpoint(sbi, &cpc);
1769         f2fs_up_write(&sbi->gc_lock);
1770
1771         return err;
1772 }
1773
1774 static void __checkpoint_and_complete_reqs(struct f2fs_sb_info *sbi)
1775 {
1776         struct ckpt_req_control *cprc = &sbi->cprc_info;
1777         struct ckpt_req *req, *next;
1778         struct llist_node *dispatch_list;
1779         u64 sum_diff = 0, diff, count = 0;
1780         int ret;
1781
1782         dispatch_list = llist_del_all(&cprc->issue_list);
1783         if (!dispatch_list)
1784                 return;
1785         dispatch_list = llist_reverse_order(dispatch_list);
1786
1787         ret = __write_checkpoint_sync(sbi);
1788         atomic_inc(&cprc->issued_ckpt);
1789
1790         llist_for_each_entry_safe(req, next, dispatch_list, llnode) {
1791                 diff = (u64)ktime_ms_delta(ktime_get(), req->queue_time);
1792                 req->ret = ret;
1793                 complete(&req->wait);
1794
1795                 sum_diff += diff;
1796                 count++;
1797         }
1798         atomic_sub(count, &cprc->queued_ckpt);
1799         atomic_add(count, &cprc->total_ckpt);
1800
1801         spin_lock(&cprc->stat_lock);
1802         cprc->cur_time = (unsigned int)div64_u64(sum_diff, count);
1803         if (cprc->peak_time < cprc->cur_time)
1804                 cprc->peak_time = cprc->cur_time;
1805         spin_unlock(&cprc->stat_lock);
1806 }
1807
1808 static int issue_checkpoint_thread(void *data)
1809 {
1810         struct f2fs_sb_info *sbi = data;
1811         struct ckpt_req_control *cprc = &sbi->cprc_info;
1812         wait_queue_head_t *q = &cprc->ckpt_wait_queue;
1813 repeat:
1814         if (kthread_should_stop())
1815                 return 0;
1816
1817         if (!llist_empty(&cprc->issue_list))
1818                 __checkpoint_and_complete_reqs(sbi);
1819
1820         wait_event_interruptible(*q,
1821                 kthread_should_stop() || !llist_empty(&cprc->issue_list));
1822         goto repeat;
1823 }
1824
1825 static void flush_remained_ckpt_reqs(struct f2fs_sb_info *sbi,
1826                 struct ckpt_req *wait_req)
1827 {
1828         struct ckpt_req_control *cprc = &sbi->cprc_info;
1829
1830         if (!llist_empty(&cprc->issue_list)) {
1831                 __checkpoint_and_complete_reqs(sbi);
1832         } else {
1833                 /* already dispatched by issue_checkpoint_thread */
1834                 if (wait_req)
1835                         wait_for_completion(&wait_req->wait);
1836         }
1837 }
1838
1839 static void init_ckpt_req(struct ckpt_req *req)
1840 {
1841         memset(req, 0, sizeof(struct ckpt_req));
1842
1843         init_completion(&req->wait);
1844         req->queue_time = ktime_get();
1845 }
1846
1847 int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi)
1848 {
1849         struct ckpt_req_control *cprc = &sbi->cprc_info;
1850         struct ckpt_req req;
1851         struct cp_control cpc;
1852
1853         cpc.reason = __get_cp_reason(sbi);
1854         if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC) {
1855                 int ret;
1856
1857                 f2fs_down_write(&sbi->gc_lock);
1858                 ret = f2fs_write_checkpoint(sbi, &cpc);
1859                 f2fs_up_write(&sbi->gc_lock);
1860
1861                 return ret;
1862         }
1863
1864         if (!cprc->f2fs_issue_ckpt)
1865                 return __write_checkpoint_sync(sbi);
1866
1867         init_ckpt_req(&req);
1868
1869         llist_add(&req.llnode, &cprc->issue_list);
1870         atomic_inc(&cprc->queued_ckpt);
1871
1872         /*
1873          * update issue_list before we wake up issue_checkpoint thread,
1874          * this smp_mb() pairs with another barrier in ___wait_event(),
1875          * see more details in comments of waitqueue_active().
1876          */
1877         smp_mb();
1878
1879         if (waitqueue_active(&cprc->ckpt_wait_queue))
1880                 wake_up(&cprc->ckpt_wait_queue);
1881
1882         if (cprc->f2fs_issue_ckpt)
1883                 wait_for_completion(&req.wait);
1884         else
1885                 flush_remained_ckpt_reqs(sbi, &req);
1886
1887         return req.ret;
1888 }
1889
1890 int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi)
1891 {
1892         dev_t dev = sbi->sb->s_bdev->bd_dev;
1893         struct ckpt_req_control *cprc = &sbi->cprc_info;
1894
1895         if (cprc->f2fs_issue_ckpt)
1896                 return 0;
1897
1898         cprc->f2fs_issue_ckpt = kthread_run(issue_checkpoint_thread, sbi,
1899                         "f2fs_ckpt-%u:%u", MAJOR(dev), MINOR(dev));
1900         if (IS_ERR(cprc->f2fs_issue_ckpt)) {
1901                 int err = PTR_ERR(cprc->f2fs_issue_ckpt);
1902
1903                 cprc->f2fs_issue_ckpt = NULL;
1904                 return err;
1905         }
1906
1907         set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio);
1908
1909         return 0;
1910 }
1911
1912 void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi)
1913 {
1914         struct ckpt_req_control *cprc = &sbi->cprc_info;
1915         struct task_struct *ckpt_task;
1916
1917         if (!cprc->f2fs_issue_ckpt)
1918                 return;
1919
1920         ckpt_task = cprc->f2fs_issue_ckpt;
1921         cprc->f2fs_issue_ckpt = NULL;
1922         kthread_stop(ckpt_task);
1923
1924         f2fs_flush_ckpt_thread(sbi);
1925 }
1926
1927 void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi)
1928 {
1929         struct ckpt_req_control *cprc = &sbi->cprc_info;
1930
1931         flush_remained_ckpt_reqs(sbi, NULL);
1932
1933         /* Let's wait for the previous dispatched checkpoint. */
1934         while (atomic_read(&cprc->queued_ckpt))
1935                 io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1936 }
1937
1938 void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi)
1939 {
1940         struct ckpt_req_control *cprc = &sbi->cprc_info;
1941
1942         atomic_set(&cprc->issued_ckpt, 0);
1943         atomic_set(&cprc->total_ckpt, 0);
1944         atomic_set(&cprc->queued_ckpt, 0);
1945         cprc->ckpt_thread_ioprio = DEFAULT_CHECKPOINT_IOPRIO;
1946         init_waitqueue_head(&cprc->ckpt_wait_queue);
1947         init_llist_head(&cprc->issue_list);
1948         spin_lock_init(&cprc->stat_lock);
1949 }