fs/f2fs/segment.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/segment.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/f2fs_fs.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/prefetch.h>
  14 #include <linux/kthread.h>
  15 #include <linux/swap.h>
  16 #include <linux/timer.h>
  17 #include <linux/freezer.h>
  18 #include <linux/sched/signal.h>
  19 #include <linux/random.h>
  20
  21 #include "f2fs.h"
  22 #include "segment.h"
  23 #include "node.h"
  24 #include "gc.h"
  25 #include "iostat.h"
  26 #include <trace/events/f2fs.h>
  27
  28 #define __reverse_ffz(x) __reverse_ffs(~(x))
  29
  30 static struct kmem_cache *discard_entry_slab;
  31 static struct kmem_cache *discard_cmd_slab;
  32 static struct kmem_cache *sit_entry_set_slab;
  33 static struct kmem_cache *revoke_entry_slab;
  34
  35 static unsigned long __reverse_ulong(unsigned char *str)
  36 {
  37         unsigned long tmp = 0;
  38         int shift = 24, idx = 0;
  39
  40 #if BITS_PER_LONG == 64
  41         shift = 56;
  42 #endif
  43         while (shift >= 0) {
  44                 tmp |= (unsigned long)str[idx++] << shift;
  45                 shift -= BITS_PER_BYTE;
  46         }
  47         return tmp;
  48 }
  49
  50 /*
  51  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  52  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  53  */
  54 static inline unsigned long __reverse_ffs(unsigned long word)
  55 {
  56         int num = 0;
  57
  58 #if BITS_PER_LONG == 64
  59         if ((word & 0xffffffff00000000UL) == 0)
  60                 num += 32;
  61         else
  62                 word >>= 32;
  63 #endif
  64         if ((word & 0xffff0000) == 0)
  65                 num += 16;
  66         else
  67                 word >>= 16;
  68
  69         if ((word & 0xff00) == 0)
  70                 num += 8;
  71         else
  72                 word >>= 8;
  73
  74         if ((word & 0xf0) == 0)
  75                 num += 4;
  76         else
  77                 word >>= 4;
  78
  79         if ((word & 0xc) == 0)
  80                 num += 2;
  81         else
  82                 word >>= 2;
  83
  84         if ((word & 0x2) == 0)
  85                 num += 1;
  86         return num;
  87 }
  88
  89 /*
  90  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  91  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  92  * @size must be integral times of unsigned long.
  93  * Example:
  94  *                             MSB <--> LSB
  95  *   f2fs_set_bit(0, bitmap) => 1000 0000
  96  *   f2fs_set_bit(7, bitmap) => 0000 0001
  97  */
  98 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  99                         unsigned long size, unsigned long offset)
 100 {
 101         const unsigned long *p = addr + BIT_WORD(offset);
 102         unsigned long result = size;
 103         unsigned long tmp;
 104
 105         if (offset >= size)
 106                 return size;
 107
 108         size -= (offset & ~(BITS_PER_LONG - 1));
 109         offset %= BITS_PER_LONG;
 110
 111         while (1) {
 112                 if (*p == 0)
 113                         goto pass;
 114
 115                 tmp = __reverse_ulong((unsigned char *)p);
 116
 117                 tmp &= ~0UL >> offset;
 118                 if (size < BITS_PER_LONG)
 119                         tmp &= (~0UL << (BITS_PER_LONG - size));
 120                 if (tmp)
 121                         goto found;
 122 pass:
 123                 if (size <= BITS_PER_LONG)
 124                         break;
 125                 size -= BITS_PER_LONG;
 126                 offset = 0;
 127                 p++;
 128         }
 129         return result;
 130 found:
 131         return result - size + __reverse_ffs(tmp);
 132 }
 133
 134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 135                         unsigned long size, unsigned long offset)
 136 {
 137         const unsigned long *p = addr + BIT_WORD(offset);
 138         unsigned long result = size;
 139         unsigned long tmp;
 140
 141         if (offset >= size)
 142                 return size;
 143
 144         size -= (offset & ~(BITS_PER_LONG - 1));
 145         offset %= BITS_PER_LONG;
 146
 147         while (1) {
 148                 if (*p == ~0UL)
 149                         goto pass;
 150
 151                 tmp = __reverse_ulong((unsigned char *)p);
 152
 153                 if (offset)
 154                         tmp |= ~0UL << (BITS_PER_LONG - offset);
 155                 if (size < BITS_PER_LONG)
 156                         tmp |= ~0UL >> size;
 157                 if (tmp != ~0UL)
 158                         goto found;
 159 pass:
 160                 if (size <= BITS_PER_LONG)
 161                         break;
 162                 size -= BITS_PER_LONG;
 163                 offset = 0;
 164                 p++;
 165         }
 166         return result;
 167 found:
 168         return result - size + __reverse_ffz(tmp);
 169 }
 170
 171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
 172 {
 173         int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 174         int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
 175         int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
 176
 177         if (f2fs_lfs_mode(sbi))
 178                 return false;
 179         if (sbi->gc_mode == GC_URGENT_HIGH)
 180                 return true;
 181         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
 182                 return true;
 183
 184         return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
 185                         SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
 186 }
 187
 188 void f2fs_abort_atomic_write(struct inode *inode, bool clean)
 189 {
 190         struct f2fs_inode_info *fi = F2FS_I(inode);
 191
 192         if (!f2fs_is_atomic_file(inode))
 193                 return;
 194
 195         release_atomic_write_cnt(inode);
 196         clear_inode_flag(inode, FI_ATOMIC_COMMITTED);
 197         clear_inode_flag(inode, FI_ATOMIC_REPLACE);
 198         clear_inode_flag(inode, FI_ATOMIC_FILE);
 199         stat_dec_atomic_inode(inode);
 200
 201         F2FS_I(inode)->atomic_write_task = NULL;
 202
 203         if (clean) {
 204                 truncate_inode_pages_final(inode->i_mapping);
 205                 f2fs_i_size_write(inode, fi->original_i_size);
 206                 fi->original_i_size = 0;
 207         }
 208 }
 209
 210 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
 211                         block_t new_addr, block_t *old_addr, bool recover)
 212 {
 213         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 214         struct dnode_of_data dn;
 215         struct node_info ni;
 216         int err;
 217
 218 retry:
 219         set_new_dnode(&dn, inode, NULL, NULL, 0);
 220         err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE_RA);
 221         if (err) {
 222                 if (err == -ENOMEM) {
 223                         f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
 224                         goto retry;
 225                 }
 226                 return err;
 227         }
 228
 229         err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
 230         if (err) {
 231                 f2fs_put_dnode(&dn);
 232                 return err;
 233         }
 234
 235         if (recover) {
 236                 /* dn.data_blkaddr is always valid */
 237                 if (!__is_valid_data_blkaddr(new_addr)) {
 238                         if (new_addr == NULL_ADDR)
 239                                 dec_valid_block_count(sbi, inode, 1);
 240                         f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
 241                         f2fs_update_data_blkaddr(&dn, new_addr);
 242                 } else {
 243                         f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
 244                                 new_addr, ni.version, true, true);
 245                 }
 246         } else {
 247                 blkcnt_t count = 1;
 248
 249                 *old_addr = dn.data_blkaddr;
 250                 f2fs_truncate_data_blocks_range(&dn, 1);
 251                 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);
 252                 inc_valid_block_count(sbi, inode, &count);
 253                 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,
 254                                         ni.version, true, false);
 255         }
 256
 257         f2fs_put_dnode(&dn);
 258
 259         trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode,
 260                         index, old_addr ? *old_addr : 0, new_addr, recover);
 261         return 0;
 262 }
 263
 264 static void __complete_revoke_list(struct inode *inode, struct list_head *head,
 265                                         bool revoke)
 266 {
 267         struct revoke_entry *cur, *tmp;
 268         pgoff_t start_index = 0;
 269         bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE);
 270
 271         list_for_each_entry_safe(cur, tmp, head, list) {
 272                 if (revoke) {
 273                         __replace_atomic_write_block(inode, cur->index,
 274                                                 cur->old_addr, NULL, true);
 275                 } else if (truncate) {
 276                         f2fs_truncate_hole(inode, start_index, cur->index);
 277                         start_index = cur->index + 1;
 278                 }
 279
 280                 list_del(&cur->list);
 281                 kmem_cache_free(revoke_entry_slab, cur);
 282         }
 283
 284         if (!revoke && truncate)
 285                 f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false);
 286 }
 287
 288 static int __f2fs_commit_atomic_write(struct inode *inode)
 289 {
 290         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 291         struct f2fs_inode_info *fi = F2FS_I(inode);
 292         struct inode *cow_inode = fi->cow_inode;
 293         struct revoke_entry *new;
 294         struct list_head revoke_list;
 295         block_t blkaddr;
 296         struct dnode_of_data dn;
 297         pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
 298         pgoff_t off = 0, blen, index;
 299         int ret = 0, i;
 300
 301         INIT_LIST_HEAD(&revoke_list);
 302
 303         while (len) {
 304                 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len);
 305
 306                 set_new_dnode(&dn, cow_inode, NULL, NULL, 0);
 307                 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA);
 308                 if (ret && ret != -ENOENT) {
 309                         goto out;
 310                 } else if (ret == -ENOENT) {
 311                         ret = 0;
 312                         if (dn.max_level == 0)
 313                                 goto out;
 314                         goto next;
 315                 }
 316
 317                 blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_page, cow_inode),
 318                                 len);
 319                 index = off;
 320                 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
 321                         blkaddr = f2fs_data_blkaddr(&dn);
 322
 323                         if (!__is_valid_data_blkaddr(blkaddr)) {
 324                                 continue;
 325                         } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
 326                                         DATA_GENERIC_ENHANCE)) {
 327                                 f2fs_put_dnode(&dn);
 328                                 ret = -EFSCORRUPTED;
 329                                 f2fs_handle_error(sbi,
 330                                                 ERROR_INVALID_BLKADDR);
 331                                 goto out;
 332                         }
 333
 334                         new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS,
 335                                                         true, NULL);
 336
 337                         ret = __replace_atomic_write_block(inode, index, blkaddr,
 338                                                         &new->old_addr, false);
 339                         if (ret) {
 340                                 f2fs_put_dnode(&dn);
 341                                 kmem_cache_free(revoke_entry_slab, new);
 342                                 goto out;
 343                         }
 344
 345                         f2fs_update_data_blkaddr(&dn, NULL_ADDR);
 346                         new->index = index;
 347                         list_add_tail(&new->list, &revoke_list);
 348                 }
 349                 f2fs_put_dnode(&dn);
 350 next:
 351                 off += blen;
 352                 len -= blen;
 353         }
 354
 355 out:
 356         if (ret) {
 357                 sbi->revoked_atomic_block += fi->atomic_write_cnt;
 358         } else {
 359                 sbi->committed_atomic_block += fi->atomic_write_cnt;
 360                 set_inode_flag(inode, FI_ATOMIC_COMMITTED);
 361         }
 362
 363         __complete_revoke_list(inode, &revoke_list, ret ? true : false);
 364
 365         return ret;
 366 }
 367
 368 int f2fs_commit_atomic_write(struct inode *inode)
 369 {
 370         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 371         struct f2fs_inode_info *fi = F2FS_I(inode);
 372         int err;
 373
 374         err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
 375         if (err)
 376                 return err;
 377
 378         f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
 379         f2fs_lock_op(sbi);
 380
 381         err = __f2fs_commit_atomic_write(inode);
 382
 383         f2fs_unlock_op(sbi);
 384         f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
 385
 386         return err;
 387 }
 388
 389 /*
 390  * This function balances dirty node and dentry pages.
 391  * In addition, it controls garbage collection.
 392  */
 393 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 394 {
 395         if (time_to_inject(sbi, FAULT_CHECKPOINT))
 396                 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
 397
 398         /* balance_fs_bg is able to be pending */
 399         if (need && excess_cached_nats(sbi))
 400                 f2fs_balance_fs_bg(sbi, false);
 401
 402         if (!f2fs_is_checkpoint_ready(sbi))
 403                 return;
 404
 405         /*
 406          * We should do GC or end up with checkpoint, if there are so many dirty
 407          * dir/node pages without enough free segments.
 408          */
 409         if (has_not_enough_free_secs(sbi, 0, 0)) {
 410                 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
 411                                         sbi->gc_thread->f2fs_gc_task) {
 412                         DEFINE_WAIT(wait);
 413
 414                         prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
 415                                                 TASK_UNINTERRUPTIBLE);
 416                         wake_up(&sbi->gc_thread->gc_wait_queue_head);
 417                         io_schedule();
 418                         finish_wait(&sbi->gc_thread->fggc_wq, &wait);
 419                 } else {
 420                         struct f2fs_gc_control gc_control = {
 421                                 .victim_segno = NULL_SEGNO,
 422                                 .init_gc_type = BG_GC,
 423                                 .no_bg_gc = true,
 424                                 .should_migrate_blocks = false,
 425                                 .err_gc_skipped = false,
 426                                 .nr_free_secs = 1 };
 427                         f2fs_down_write(&sbi->gc_lock);
 428                         f2fs_gc(sbi, &gc_control);
 429                 }
 430         }
 431 }
 432
 433 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
 434 {
 435         int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
 436         unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
 437         unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
 438         unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
 439         unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
 440         unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
 441         unsigned int threshold = sbi->blocks_per_seg * factor *
 442                                         DEFAULT_DIRTY_THRESHOLD;
 443         unsigned int global_threshold = threshold * 3 / 2;
 444
 445         if (dents >= threshold || qdata >= threshold ||
 446                 nodes >= threshold || meta >= threshold ||
 447                 imeta >= threshold)
 448                 return true;
 449         return dents + qdata + nodes + meta + imeta >  global_threshold;
 450 }
 451
 452 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
 453 {
 454         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 455                 return;
 456
 457         /* try to shrink extent cache when there is no enough memory */
 458         if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))
 459                 f2fs_shrink_read_extent_tree(sbi,
 460                                 READ_EXTENT_CACHE_SHRINK_NUMBER);
 461
 462         /* try to shrink age extent cache when there is no enough memory */
 463         if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))
 464                 f2fs_shrink_age_extent_tree(sbi,
 465                                 AGE_EXTENT_CACHE_SHRINK_NUMBER);
 466
 467         /* check the # of cached NAT entries */
 468         if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
 469                 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
 470
 471         if (!f2fs_available_free_memory(sbi, FREE_NIDS))
 472                 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
 473         else
 474                 f2fs_build_free_nids(sbi, false, false);
 475
 476         if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
 477                 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
 478                 goto do_sync;
 479
 480         /* there is background inflight IO or foreground operation recently */
 481         if (is_inflight_io(sbi, REQ_TIME) ||
 482                 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
 483                 return;
 484
 485         /* exceed periodical checkpoint timeout threshold */
 486         if (f2fs_time_over(sbi, CP_TIME))
 487                 goto do_sync;
 488
 489         /* checkpoint is the only way to shrink partial cached entries */
 490         if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
 491                 f2fs_available_free_memory(sbi, INO_ENTRIES))
 492                 return;
 493
 494 do_sync:
 495         if (test_opt(sbi, DATA_FLUSH) && from_bg) {
 496                 struct blk_plug plug;
 497
 498                 mutex_lock(&sbi->flush_lock);
 499
 500                 blk_start_plug(&plug);
 501                 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
 502                 blk_finish_plug(&plug);
 503
 504                 mutex_unlock(&sbi->flush_lock);
 505         }
 506         f2fs_sync_fs(sbi->sb, 1);
 507         stat_inc_bg_cp_count(sbi->stat_info);
 508 }
 509
 510 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
 511                                 struct block_device *bdev)
 512 {
 513         int ret = blkdev_issue_flush(bdev);
 514
 515         trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
 516                                 test_opt(sbi, FLUSH_MERGE), ret);
 517         if (!ret)
 518                 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0);
 519         return ret;
 520 }
 521
 522 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
 523 {
 524         int ret = 0;
 525         int i;
 526
 527         if (!f2fs_is_multi_device(sbi))
 528                 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
 529
 530         for (i = 0; i < sbi->s_ndevs; i++) {
 531                 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
 532                         continue;
 533                 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 534                 if (ret)
 535                         break;
 536         }
 537         return ret;
 538 }
 539
 540 static int issue_flush_thread(void *data)
 541 {
 542         struct f2fs_sb_info *sbi = data;
 543         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 544         wait_queue_head_t *q = &fcc->flush_wait_queue;
 545 repeat:
 546         if (kthread_should_stop())
 547                 return 0;
 548
 549         if (!llist_empty(&fcc->issue_list)) {
 550                 struct flush_cmd *cmd, *next;
 551                 int ret;
 552
 553                 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 554                 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 555
 556                 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
 557
 558                 ret = submit_flush_wait(sbi, cmd->ino);
 559                 atomic_inc(&fcc->issued_flush);
 560
 561                 llist_for_each_entry_safe(cmd, next,
 562                                           fcc->dispatch_list, llnode) {
 563                         cmd->ret = ret;
 564                         complete(&cmd->wait);
 565                 }
 566                 fcc->dispatch_list = NULL;
 567         }
 568
 569         wait_event_interruptible(*q,
 570                 kthread_should_stop() || !llist_empty(&fcc->issue_list));
 571         goto repeat;
 572 }
 573
 574 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
 575 {
 576         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 577         struct flush_cmd cmd;
 578         int ret;
 579
 580         if (test_opt(sbi, NOBARRIER))
 581                 return 0;
 582
 583         if (!test_opt(sbi, FLUSH_MERGE)) {
 584                 atomic_inc(&fcc->queued_flush);
 585                 ret = submit_flush_wait(sbi, ino);
 586                 atomic_dec(&fcc->queued_flush);
 587                 atomic_inc(&fcc->issued_flush);
 588                 return ret;
 589         }
 590
 591         if (atomic_inc_return(&fcc->queued_flush) == 1 ||
 592             f2fs_is_multi_device(sbi)) {
 593                 ret = submit_flush_wait(sbi, ino);
 594                 atomic_dec(&fcc->queued_flush);
 595
 596                 atomic_inc(&fcc->issued_flush);
 597                 return ret;
 598         }
 599
 600         cmd.ino = ino;
 601         init_completion(&cmd.wait);
 602
 603         llist_add(&cmd.llnode, &fcc->issue_list);
 604
 605         /*
 606          * update issue_list before we wake up issue_flush thread, this
 607          * smp_mb() pairs with another barrier in ___wait_event(), see
 608          * more details in comments of waitqueue_active().
 609          */
 610         smp_mb();
 611
 612         if (waitqueue_active(&fcc->flush_wait_queue))
 613                 wake_up(&fcc->flush_wait_queue);
 614
 615         if (fcc->f2fs_issue_flush) {
 616                 wait_for_completion(&cmd.wait);
 617                 atomic_dec(&fcc->queued_flush);
 618         } else {
 619                 struct llist_node *list;
 620
 621                 list = llist_del_all(&fcc->issue_list);
 622                 if (!list) {
 623                         wait_for_completion(&cmd.wait);
 624                         atomic_dec(&fcc->queued_flush);
 625                 } else {
 626                         struct flush_cmd *tmp, *next;
 627
 628                         ret = submit_flush_wait(sbi, ino);
 629
 630                         llist_for_each_entry_safe(tmp, next, list, llnode) {
 631                                 if (tmp == &cmd) {
 632                                         cmd.ret = ret;
 633                                         atomic_dec(&fcc->queued_flush);
 634                                         continue;
 635                                 }
 636                                 tmp->ret = ret;
 637                                 complete(&tmp->wait);
 638                         }
 639                 }
 640         }
 641
 642         return cmd.ret;
 643 }
 644
 645 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
 646 {
 647         dev_t dev = sbi->sb->s_bdev->bd_dev;
 648         struct flush_cmd_control *fcc;
 649
 650         if (SM_I(sbi)->fcc_info) {
 651                 fcc = SM_I(sbi)->fcc_info;
 652                 if (fcc->f2fs_issue_flush)
 653                         return 0;
 654                 goto init_thread;
 655         }
 656
 657         fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
 658         if (!fcc)
 659                 return -ENOMEM;
 660         atomic_set(&fcc->issued_flush, 0);
 661         atomic_set(&fcc->queued_flush, 0);
 662         init_waitqueue_head(&fcc->flush_wait_queue);
 663         init_llist_head(&fcc->issue_list);
 664         SM_I(sbi)->fcc_info = fcc;
 665         if (!test_opt(sbi, FLUSH_MERGE))
 666                 return 0;
 667
 668 init_thread:
 669         fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 670                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 671         if (IS_ERR(fcc->f2fs_issue_flush)) {
 672                 int err = PTR_ERR(fcc->f2fs_issue_flush);
 673
 674                 fcc->f2fs_issue_flush = NULL;
 675                 return err;
 676         }
 677
 678         return 0;
 679 }
 680
 681 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
 682 {
 683         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 684
 685         if (fcc && fcc->f2fs_issue_flush) {
 686                 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
 687
 688                 fcc->f2fs_issue_flush = NULL;
 689                 kthread_stop(flush_thread);
 690         }
 691         if (free) {
 692                 kfree(fcc);
 693                 SM_I(sbi)->fcc_info = NULL;
 694         }
 695 }
 696
 697 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
 698 {
 699         int ret = 0, i;
 700
 701         if (!f2fs_is_multi_device(sbi))
 702                 return 0;
 703
 704         if (test_opt(sbi, NOBARRIER))
 705                 return 0;
 706
 707         for (i = 1; i < sbi->s_ndevs; i++) {
 708                 int count = DEFAULT_RETRY_IO_COUNT;
 709
 710                 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
 711                         continue;
 712
 713                 do {
 714                         ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 715                         if (ret)
 716                                 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
 717                 } while (ret && --count);
 718
 719                 if (ret) {
 720                         f2fs_stop_checkpoint(sbi, false,
 721                                         STOP_CP_REASON_FLUSH_FAIL);
 722                         break;
 723                 }
 724
 725                 spin_lock(&sbi->dev_lock);
 726                 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
 727                 spin_unlock(&sbi->dev_lock);
 728         }
 729
 730         return ret;
 731 }
 732
 733 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 734                 enum dirty_type dirty_type)
 735 {
 736         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 737
 738         /* need not be added */
 739         if (IS_CURSEG(sbi, segno))
 740                 return;
 741
 742         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 743                 dirty_i->nr_dirty[dirty_type]++;
 744
 745         if (dirty_type == DIRTY) {
 746                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 747                 enum dirty_type t = sentry->type;
 748
 749                 if (unlikely(t >= DIRTY)) {
 750                         f2fs_bug_on(sbi, 1);
 751                         return;
 752                 }
 753                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 754                         dirty_i->nr_dirty[t]++;
 755
 756                 if (__is_large_section(sbi)) {
 757                         unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
 758                         block_t valid_blocks =
 759                                 get_valid_blocks(sbi, segno, true);
 760
 761                         f2fs_bug_on(sbi, unlikely(!valid_blocks ||
 762                                         valid_blocks == CAP_BLKS_PER_SEC(sbi)));
 763
 764                         if (!IS_CURSEC(sbi, secno))
 765                                 set_bit(secno, dirty_i->dirty_secmap);
 766                 }
 767         }
 768 }
 769
 770 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 771                 enum dirty_type dirty_type)
 772 {
 773         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 774         block_t valid_blocks;
 775
 776         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 777                 dirty_i->nr_dirty[dirty_type]--;
 778
 779         if (dirty_type == DIRTY) {
 780                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 781                 enum dirty_type t = sentry->type;
 782
 783                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 784                         dirty_i->nr_dirty[t]--;
 785
 786                 valid_blocks = get_valid_blocks(sbi, segno, true);
 787                 if (valid_blocks == 0) {
 788                         clear_bit(GET_SEC_FROM_SEG(sbi, segno),
 789                                                 dirty_i->victim_secmap);
 790 #ifdef CONFIG_F2FS_CHECK_FS
 791                         clear_bit(segno, SIT_I(sbi)->invalid_segmap);
 792 #endif
 793                 }
 794                 if (__is_large_section(sbi)) {
 795                         unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
 796
 797                         if (!valid_blocks ||
 798                                         valid_blocks == CAP_BLKS_PER_SEC(sbi)) {
 799                                 clear_bit(secno, dirty_i->dirty_secmap);
 800                                 return;
 801                         }
 802
 803                         if (!IS_CURSEC(sbi, secno))
 804                                 set_bit(secno, dirty_i->dirty_secmap);
 805                 }
 806         }
 807 }
 808
 809 /*
 810  * Should not occur error such as -ENOMEM.
 811  * Adding dirty entry into seglist is not critical operation.
 812  * If a given segment is one of current working segments, it won't be added.
 813  */
 814 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 815 {
 816         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 817         unsigned short valid_blocks, ckpt_valid_blocks;
 818         unsigned int usable_blocks;
 819
 820         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 821                 return;
 822
 823         usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
 824         mutex_lock(&dirty_i->seglist_lock);
 825
 826         valid_blocks = get_valid_blocks(sbi, segno, false);
 827         ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
 828
 829         if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
 830                 ckpt_valid_blocks == usable_blocks)) {
 831                 __locate_dirty_segment(sbi, segno, PRE);
 832                 __remove_dirty_segment(sbi, segno, DIRTY);
 833         } else if (valid_blocks < usable_blocks) {
 834                 __locate_dirty_segment(sbi, segno, DIRTY);
 835         } else {
 836                 /* Recovery routine with SSR needs this */
 837                 __remove_dirty_segment(sbi, segno, DIRTY);
 838         }
 839
 840         mutex_unlock(&dirty_i->seglist_lock);
 841 }
 842
 843 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
 844 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
 845 {
 846         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 847         unsigned int segno;
 848
 849         mutex_lock(&dirty_i->seglist_lock);
 850         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 851                 if (get_valid_blocks(sbi, segno, false))
 852                         continue;
 853                 if (IS_CURSEG(sbi, segno))
 854                         continue;
 855                 __locate_dirty_segment(sbi, segno, PRE);
 856                 __remove_dirty_segment(sbi, segno, DIRTY);
 857         }
 858         mutex_unlock(&dirty_i->seglist_lock);
 859 }
 860
 861 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
 862 {
 863         int ovp_hole_segs =
 864                 (overprovision_segments(sbi) - reserved_segments(sbi));
 865         block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
 866         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 867         block_t holes[2] = {0, 0};      /* DATA and NODE */
 868         block_t unusable;
 869         struct seg_entry *se;
 870         unsigned int segno;
 871
 872         mutex_lock(&dirty_i->seglist_lock);
 873         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 874                 se = get_seg_entry(sbi, segno);
 875                 if (IS_NODESEG(se->type))
 876                         holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
 877                                                         se->valid_blocks;
 878                 else
 879                         holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
 880                                                         se->valid_blocks;
 881         }
 882         mutex_unlock(&dirty_i->seglist_lock);
 883
 884         unusable = max(holes[DATA], holes[NODE]);
 885         if (unusable > ovp_holes)
 886                 return unusable - ovp_holes;
 887         return 0;
 888 }
 889
 890 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
 891 {
 892         int ovp_hole_segs =
 893                 (overprovision_segments(sbi) - reserved_segments(sbi));
 894         if (unusable > F2FS_OPTION(sbi).unusable_cap)
 895                 return -EAGAIN;
 896         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
 897                 dirty_segments(sbi) > ovp_hole_segs)
 898                 return -EAGAIN;
 899         return 0;
 900 }
 901
 902 /* This is only used by SBI_CP_DISABLED */
 903 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
 904 {
 905         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 906         unsigned int segno = 0;
 907
 908         mutex_lock(&dirty_i->seglist_lock);
 909         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 910                 if (get_valid_blocks(sbi, segno, false))
 911                         continue;
 912                 if (get_ckpt_valid_blocks(sbi, segno, false))
 913                         continue;
 914                 mutex_unlock(&dirty_i->seglist_lock);
 915                 return segno;
 916         }
 917         mutex_unlock(&dirty_i->seglist_lock);
 918         return NULL_SEGNO;
 919 }
 920
 921 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
 922                 struct block_device *bdev, block_t lstart,
 923                 block_t start, block_t len)
 924 {
 925         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 926         struct list_head *pend_list;
 927         struct discard_cmd *dc;
 928
 929         f2fs_bug_on(sbi, !len);
 930
 931         pend_list = &dcc->pend_list[plist_idx(len)];
 932
 933         dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
 934         INIT_LIST_HEAD(&dc->list);
 935         dc->bdev = bdev;
 936         dc->di.lstart = lstart;
 937         dc->di.start = start;
 938         dc->di.len = len;
 939         dc->ref = 0;
 940         dc->state = D_PREP;
 941         dc->queued = 0;
 942         dc->error = 0;
 943         init_completion(&dc->wait);
 944         list_add_tail(&dc->list, pend_list);
 945         spin_lock_init(&dc->lock);
 946         dc->bio_ref = 0;
 947         atomic_inc(&dcc->discard_cmd_cnt);
 948         dcc->undiscard_blks += len;
 949
 950         return dc;
 951 }
 952
 953 static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)
 954 {
 955 #ifdef CONFIG_F2FS_CHECK_FS
 956         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 957         struct rb_node *cur = rb_first_cached(&dcc->root), *next;
 958         struct discard_cmd *cur_dc, *next_dc;
 959
 960         while (cur) {
 961                 next = rb_next(cur);
 962                 if (!next)
 963                         return true;
 964
 965                 cur_dc = rb_entry(cur, struct discard_cmd, rb_node);
 966                 next_dc = rb_entry(next, struct discard_cmd, rb_node);
 967
 968                 if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) {
 969                         f2fs_info(sbi, "broken discard_rbtree, "
 970                                 "cur(%u, %u) next(%u, %u)",
 971                                 cur_dc->di.lstart, cur_dc->di.len,
 972                                 next_dc->di.lstart, next_dc->di.len);
 973                         return false;
 974                 }
 975                 cur = next;
 976         }
 977 #endif
 978         return true;
 979 }
 980
 981 static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,
 982                                                 block_t blkaddr)
 983 {
 984         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 985         struct rb_node *node = dcc->root.rb_root.rb_node;
 986         struct discard_cmd *dc;
 987
 988         while (node) {
 989                 dc = rb_entry(node, struct discard_cmd, rb_node);
 990
 991                 if (blkaddr < dc->di.lstart)
 992                         node = node->rb_left;
 993                 else if (blkaddr >= dc->di.lstart + dc->di.len)
 994                         node = node->rb_right;
 995                 else
 996                         return dc;
 997         }
 998         return NULL;
 999 }
1000
1001 static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root,
1002                                 block_t blkaddr,
1003                                 struct discard_cmd **prev_entry,
1004                                 struct discard_cmd **next_entry,
1005                                 struct rb_node ***insert_p,
1006                                 struct rb_node **insert_parent)
1007 {
1008         struct rb_node **pnode = &root->rb_root.rb_node;
1009         struct rb_node *parent = NULL, *tmp_node;
1010         struct discard_cmd *dc;
1011
1012         *insert_p = NULL;
1013         *insert_parent = NULL;
1014         *prev_entry = NULL;
1015         *next_entry = NULL;
1016
1017         if (RB_EMPTY_ROOT(&root->rb_root))
1018                 return NULL;
1019
1020         while (*pnode) {
1021                 parent = *pnode;
1022                 dc = rb_entry(*pnode, struct discard_cmd, rb_node);
1023
1024                 if (blkaddr < dc->di.lstart)
1025                         pnode = &(*pnode)->rb_left;
1026                 else if (blkaddr >= dc->di.lstart + dc->di.len)
1027                         pnode = &(*pnode)->rb_right;
1028                 else
1029                         goto lookup_neighbors;
1030         }
1031
1032         *insert_p = pnode;
1033         *insert_parent = parent;
1034
1035         dc = rb_entry(parent, struct discard_cmd, rb_node);
1036         tmp_node = parent;
1037         if (parent && blkaddr > dc->di.lstart)
1038                 tmp_node = rb_next(parent);
1039         *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1040
1041         tmp_node = parent;
1042         if (parent && blkaddr < dc->di.lstart)
1043                 tmp_node = rb_prev(parent);
1044         *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1045         return NULL;
1046
1047 lookup_neighbors:
1048         /* lookup prev node for merging backward later */
1049         tmp_node = rb_prev(&dc->rb_node);
1050         *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1051
1052         /* lookup next node for merging frontward later */
1053         tmp_node = rb_next(&dc->rb_node);
1054         *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1055         return dc;
1056 }
1057
1058 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1059                                                         struct discard_cmd *dc)
1060 {
1061         if (dc->state == D_DONE)
1062                 atomic_sub(dc->queued, &dcc->queued_discard);
1063
1064         list_del(&dc->list);
1065         rb_erase_cached(&dc->rb_node, &dcc->root);
1066         dcc->undiscard_blks -= dc->di.len;
1067
1068         kmem_cache_free(discard_cmd_slab, dc);
1069
1070         atomic_dec(&dcc->discard_cmd_cnt);
1071 }
1072
1073 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1074                                                         struct discard_cmd *dc)
1075 {
1076         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1077         unsigned long flags;
1078
1079         trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len);
1080
1081         spin_lock_irqsave(&dc->lock, flags);
1082         if (dc->bio_ref) {
1083                 spin_unlock_irqrestore(&dc->lock, flags);
1084                 return;
1085         }
1086         spin_unlock_irqrestore(&dc->lock, flags);
1087
1088         f2fs_bug_on(sbi, dc->ref);
1089
1090         if (dc->error == -EOPNOTSUPP)
1091                 dc->error = 0;
1092
1093         if (dc->error)
1094                 printk_ratelimited(
1095                         "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
1096                         KERN_INFO, sbi->sb->s_id,
1097                         dc->di.lstart, dc->di.start, dc->di.len, dc->error);
1098         __detach_discard_cmd(dcc, dc);
1099 }
1100
1101 static void f2fs_submit_discard_endio(struct bio *bio)
1102 {
1103         struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1104         unsigned long flags;
1105
1106         spin_lock_irqsave(&dc->lock, flags);
1107         if (!dc->error)
1108                 dc->error = blk_status_to_errno(bio->bi_status);
1109         dc->bio_ref--;
1110         if (!dc->bio_ref && dc->state == D_SUBMIT) {
1111                 dc->state = D_DONE;
1112                 complete_all(&dc->wait);
1113         }
1114         spin_unlock_irqrestore(&dc->lock, flags);
1115         bio_put(bio);
1116 }
1117
1118 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1119                                 block_t start, block_t end)
1120 {
1121 #ifdef CONFIG_F2FS_CHECK_FS
1122         struct seg_entry *sentry;
1123         unsigned int segno;
1124         block_t blk = start;
1125         unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1126         unsigned long *map;
1127
1128         while (blk < end) {
1129                 segno = GET_SEGNO(sbi, blk);
1130                 sentry = get_seg_entry(sbi, segno);
1131                 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1132
1133                 if (end < START_BLOCK(sbi, segno + 1))
1134                         size = GET_BLKOFF_FROM_SEG0(sbi, end);
1135                 else
1136                         size = max_blocks;
1137                 map = (unsigned long *)(sentry->cur_valid_map);
1138                 offset = __find_rev_next_bit(map, size, offset);
1139                 f2fs_bug_on(sbi, offset != size);
1140                 blk = START_BLOCK(sbi, segno + 1);
1141         }
1142 #endif
1143 }
1144
1145 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1146                                 struct discard_policy *dpolicy,
1147                                 int discard_type, unsigned int granularity)
1148 {
1149         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1150
1151         /* common policy */
1152         dpolicy->type = discard_type;
1153         dpolicy->sync = true;
1154         dpolicy->ordered = false;
1155         dpolicy->granularity = granularity;
1156
1157         dpolicy->max_requests = dcc->max_discard_request;
1158         dpolicy->io_aware_gran = dcc->discard_io_aware_gran;
1159         dpolicy->timeout = false;
1160
1161         if (discard_type == DPOLICY_BG) {
1162                 dpolicy->min_interval = dcc->min_discard_issue_time;
1163                 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1164                 dpolicy->max_interval = dcc->max_discard_issue_time;
1165                 dpolicy->io_aware = true;
1166                 dpolicy->sync = false;
1167                 dpolicy->ordered = true;
1168                 if (utilization(sbi) > dcc->discard_urgent_util) {
1169                         dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1170                         if (atomic_read(&dcc->discard_cmd_cnt))
1171                                 dpolicy->max_interval =
1172                                         dcc->min_discard_issue_time;
1173                 }
1174         } else if (discard_type == DPOLICY_FORCE) {
1175                 dpolicy->min_interval = dcc->min_discard_issue_time;
1176                 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1177                 dpolicy->max_interval = dcc->max_discard_issue_time;
1178                 dpolicy->io_aware = false;
1179         } else if (discard_type == DPOLICY_FSTRIM) {
1180                 dpolicy->io_aware = false;
1181         } else if (discard_type == DPOLICY_UMOUNT) {
1182                 dpolicy->io_aware = false;
1183                 /* we need to issue all to keep CP_TRIMMED_FLAG */
1184                 dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1185                 dpolicy->timeout = true;
1186         }
1187 }
1188
1189 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1190                                 struct block_device *bdev, block_t lstart,
1191                                 block_t start, block_t len);
1192 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1193 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1194                                 struct discard_policy *dpolicy,
1195                                 struct discard_cmd *dc, int *issued)
1196 {
1197         struct block_device *bdev = dc->bdev;
1198         unsigned int max_discard_blocks =
1199                         SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1200         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1201         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1202                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1203         blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
1204         block_t lstart, start, len, total_len;
1205         int err = 0;
1206
1207         if (dc->state != D_PREP)
1208                 return 0;
1209
1210         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1211                 return 0;
1212
1213         trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len);
1214
1215         lstart = dc->di.lstart;
1216         start = dc->di.start;
1217         len = dc->di.len;
1218         total_len = len;
1219
1220         dc->di.len = 0;
1221
1222         while (total_len && *issued < dpolicy->max_requests && !err) {
1223                 struct bio *bio = NULL;
1224                 unsigned long flags;
1225                 bool last = true;
1226
1227                 if (len > max_discard_blocks) {
1228                         len = max_discard_blocks;
1229                         last = false;
1230                 }
1231
1232                 (*issued)++;
1233                 if (*issued == dpolicy->max_requests)
1234                         last = true;
1235
1236                 dc->di.len += len;
1237
1238                 if (time_to_inject(sbi, FAULT_DISCARD)) {
1239                         err = -EIO;
1240                 } else {
1241                         err = __blkdev_issue_discard(bdev,
1242                                         SECTOR_FROM_BLOCK(start),
1243                                         SECTOR_FROM_BLOCK(len),
1244                                         GFP_NOFS, &bio);
1245                 }
1246                 if (err) {
1247                         spin_lock_irqsave(&dc->lock, flags);
1248                         if (dc->state == D_PARTIAL)
1249                                 dc->state = D_SUBMIT;
1250                         spin_unlock_irqrestore(&dc->lock, flags);
1251
1252                         break;
1253                 }
1254
1255                 f2fs_bug_on(sbi, !bio);
1256
1257                 /*
1258                  * should keep before submission to avoid D_DONE
1259                  * right away
1260                  */
1261                 spin_lock_irqsave(&dc->lock, flags);
1262                 if (last)
1263                         dc->state = D_SUBMIT;
1264                 else
1265                         dc->state = D_PARTIAL;
1266                 dc->bio_ref++;
1267                 spin_unlock_irqrestore(&dc->lock, flags);
1268
1269                 atomic_inc(&dcc->queued_discard);
1270                 dc->queued++;
1271                 list_move_tail(&dc->list, wait_list);
1272
1273                 /* sanity check on discard range */
1274                 __check_sit_bitmap(sbi, lstart, lstart + len);
1275
1276                 bio->bi_private = dc;
1277                 bio->bi_end_io = f2fs_submit_discard_endio;
1278                 bio->bi_opf |= flag;
1279                 submit_bio(bio);
1280
1281                 atomic_inc(&dcc->issued_discard);
1282
1283                 f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE);
1284
1285                 lstart += len;
1286                 start += len;
1287                 total_len -= len;
1288                 len = total_len;
1289         }
1290
1291         if (!err && len) {
1292                 dcc->undiscard_blks -= len;
1293                 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1294         }
1295         return err;
1296 }
1297
1298 static void __insert_discard_cmd(struct f2fs_sb_info *sbi,
1299                                 struct block_device *bdev, block_t lstart,
1300                                 block_t start, block_t len)
1301 {
1302         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1303         struct rb_node **p = &dcc->root.rb_root.rb_node;
1304         struct rb_node *parent = NULL;
1305         struct discard_cmd *dc;
1306         bool leftmost = true;
1307
1308         /* look up rb tree to find parent node */
1309         while (*p) {
1310                 parent = *p;
1311                 dc = rb_entry(parent, struct discard_cmd, rb_node);
1312
1313                 if (lstart < dc->di.lstart) {
1314                         p = &(*p)->rb_left;
1315                 } else if (lstart >= dc->di.lstart + dc->di.len) {
1316                         p = &(*p)->rb_right;
1317                         leftmost = false;
1318                 } else {
1319                         f2fs_bug_on(sbi, 1);
1320                 }
1321         }
1322
1323         dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1324
1325         rb_link_node(&dc->rb_node, parent, p);
1326         rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1327 }
1328
1329 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1330                                                 struct discard_cmd *dc)
1331 {
1332         list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]);
1333 }
1334
1335 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1336                                 struct discard_cmd *dc, block_t blkaddr)
1337 {
1338         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1339         struct discard_info di = dc->di;
1340         bool modified = false;
1341
1342         if (dc->state == D_DONE || dc->di.len == 1) {
1343                 __remove_discard_cmd(sbi, dc);
1344                 return;
1345         }
1346
1347         dcc->undiscard_blks -= di.len;
1348
1349         if (blkaddr > di.lstart) {
1350                 dc->di.len = blkaddr - dc->di.lstart;
1351                 dcc->undiscard_blks += dc->di.len;
1352                 __relocate_discard_cmd(dcc, dc);
1353                 modified = true;
1354         }
1355
1356         if (blkaddr < di.lstart + di.len - 1) {
1357                 if (modified) {
1358                         __insert_discard_cmd(sbi, dc->bdev, blkaddr + 1,
1359                                         di.start + blkaddr + 1 - di.lstart,
1360                                         di.lstart + di.len - 1 - blkaddr);
1361                 } else {
1362                         dc->di.lstart++;
1363                         dc->di.len--;
1364                         dc->di.start++;
1365                         dcc->undiscard_blks += dc->di.len;
1366                         __relocate_discard_cmd(dcc, dc);
1367                 }
1368         }
1369 }
1370
1371 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1372                                 struct block_device *bdev, block_t lstart,
1373                                 block_t start, block_t len)
1374 {
1375         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1376         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1377         struct discard_cmd *dc;
1378         struct discard_info di = {0};
1379         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1380         unsigned int max_discard_blocks =
1381                         SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1382         block_t end = lstart + len;
1383
1384         dc = __lookup_discard_cmd_ret(&dcc->root, lstart,
1385                                 &prev_dc, &next_dc, &insert_p, &insert_parent);
1386         if (dc)
1387                 prev_dc = dc;
1388
1389         if (!prev_dc) {
1390                 di.lstart = lstart;
1391                 di.len = next_dc ? next_dc->di.lstart - lstart : len;
1392                 di.len = min(di.len, len);
1393                 di.start = start;
1394         }
1395
1396         while (1) {
1397                 struct rb_node *node;
1398                 bool merged = false;
1399                 struct discard_cmd *tdc = NULL;
1400
1401                 if (prev_dc) {
1402                         di.lstart = prev_dc->di.lstart + prev_dc->di.len;
1403                         if (di.lstart < lstart)
1404                                 di.lstart = lstart;
1405                         if (di.lstart >= end)
1406                                 break;
1407
1408                         if (!next_dc || next_dc->di.lstart > end)
1409                                 di.len = end - di.lstart;
1410                         else
1411                                 di.len = next_dc->di.lstart - di.lstart;
1412                         di.start = start + di.lstart - lstart;
1413                 }
1414
1415                 if (!di.len)
1416                         goto next;
1417
1418                 if (prev_dc && prev_dc->state == D_PREP &&
1419                         prev_dc->bdev == bdev &&
1420                         __is_discard_back_mergeable(&di, &prev_dc->di,
1421                                                         max_discard_blocks)) {
1422                         prev_dc->di.len += di.len;
1423                         dcc->undiscard_blks += di.len;
1424                         __relocate_discard_cmd(dcc, prev_dc);
1425                         di = prev_dc->di;
1426                         tdc = prev_dc;
1427                         merged = true;
1428                 }
1429
1430                 if (next_dc && next_dc->state == D_PREP &&
1431                         next_dc->bdev == bdev &&
1432                         __is_discard_front_mergeable(&di, &next_dc->di,
1433                                                         max_discard_blocks)) {
1434                         next_dc->di.lstart = di.lstart;
1435                         next_dc->di.len += di.len;
1436                         next_dc->di.start = di.start;
1437                         dcc->undiscard_blks += di.len;
1438                         __relocate_discard_cmd(dcc, next_dc);
1439                         if (tdc)
1440                                 __remove_discard_cmd(sbi, tdc);
1441                         merged = true;
1442                 }
1443
1444                 if (!merged)
1445                         __insert_discard_cmd(sbi, bdev,
1446                                                 di.lstart, di.start, di.len);
1447  next:
1448                 prev_dc = next_dc;
1449                 if (!prev_dc)
1450                         break;
1451
1452                 node = rb_next(&prev_dc->rb_node);
1453                 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1454         }
1455 }
1456
1457 static void __queue_discard_cmd(struct f2fs_sb_info *sbi,
1458                 struct block_device *bdev, block_t blkstart, block_t blklen)
1459 {
1460         block_t lblkstart = blkstart;
1461
1462         if (!f2fs_bdev_support_discard(bdev))
1463                 return;
1464
1465         trace_f2fs_queue_discard(bdev, blkstart, blklen);
1466
1467         if (f2fs_is_multi_device(sbi)) {
1468                 int devi = f2fs_target_device_index(sbi, blkstart);
1469
1470                 blkstart -= FDEV(devi).start_blk;
1471         }
1472         mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1473         __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1474         mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1475 }
1476
1477 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1478                 struct discard_policy *dpolicy, int *issued)
1479 {
1480         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1481         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1482         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1483         struct discard_cmd *dc;
1484         struct blk_plug plug;
1485         bool io_interrupted = false;
1486
1487         mutex_lock(&dcc->cmd_lock);
1488         dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos,
1489                                 &prev_dc, &next_dc, &insert_p, &insert_parent);
1490         if (!dc)
1491                 dc = next_dc;
1492
1493         blk_start_plug(&plug);
1494
1495         while (dc) {
1496                 struct rb_node *node;
1497                 int err = 0;
1498
1499                 if (dc->state != D_PREP)
1500                         goto next;
1501
1502                 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1503                         io_interrupted = true;
1504                         break;
1505                 }
1506
1507                 dcc->next_pos = dc->di.lstart + dc->di.len;
1508                 err = __submit_discard_cmd(sbi, dpolicy, dc, issued);
1509
1510                 if (*issued >= dpolicy->max_requests)
1511                         break;
1512 next:
1513                 node = rb_next(&dc->rb_node);
1514                 if (err)
1515                         __remove_discard_cmd(sbi, dc);
1516                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1517         }
1518
1519         blk_finish_plug(&plug);
1520
1521         if (!dc)
1522                 dcc->next_pos = 0;
1523
1524         mutex_unlock(&dcc->cmd_lock);
1525
1526         if (!(*issued) && io_interrupted)
1527                 *issued = -1;
1528 }
1529 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1530                                         struct discard_policy *dpolicy);
1531
1532 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1533                                         struct discard_policy *dpolicy)
1534 {
1535         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1536         struct list_head *pend_list;
1537         struct discard_cmd *dc, *tmp;
1538         struct blk_plug plug;
1539         int i, issued;
1540         bool io_interrupted = false;
1541
1542         if (dpolicy->timeout)
1543                 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1544
1545 retry:
1546         issued = 0;
1547         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1548                 if (dpolicy->timeout &&
1549                                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1550                         break;
1551
1552                 if (i + 1 < dpolicy->granularity)
1553                         break;
1554
1555                 if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) {
1556                         __issue_discard_cmd_orderly(sbi, dpolicy, &issued);
1557                         return issued;
1558                 }
1559
1560                 pend_list = &dcc->pend_list[i];
1561
1562                 mutex_lock(&dcc->cmd_lock);
1563                 if (list_empty(pend_list))
1564                         goto next;
1565                 if (unlikely(dcc->rbtree_check))
1566                         f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
1567                 blk_start_plug(&plug);
1568                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1569                         f2fs_bug_on(sbi, dc->state != D_PREP);
1570
1571                         if (dpolicy->timeout &&
1572                                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1573                                 break;
1574
1575                         if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1576                                                 !is_idle(sbi, DISCARD_TIME)) {
1577                                 io_interrupted = true;
1578                                 break;
1579                         }
1580
1581                         __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1582
1583                         if (issued >= dpolicy->max_requests)
1584                                 break;
1585                 }
1586                 blk_finish_plug(&plug);
1587 next:
1588                 mutex_unlock(&dcc->cmd_lock);
1589
1590                 if (issued >= dpolicy->max_requests || io_interrupted)
1591                         break;
1592         }
1593
1594         if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1595                 __wait_all_discard_cmd(sbi, dpolicy);
1596                 goto retry;
1597         }
1598
1599         if (!issued && io_interrupted)
1600                 issued = -1;
1601
1602         return issued;
1603 }
1604
1605 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1606 {
1607         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1608         struct list_head *pend_list;
1609         struct discard_cmd *dc, *tmp;
1610         int i;
1611         bool dropped = false;
1612
1613         mutex_lock(&dcc->cmd_lock);
1614         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1615                 pend_list = &dcc->pend_list[i];
1616                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1617                         f2fs_bug_on(sbi, dc->state != D_PREP);
1618                         __remove_discard_cmd(sbi, dc);
1619                         dropped = true;
1620                 }
1621         }
1622         mutex_unlock(&dcc->cmd_lock);
1623
1624         return dropped;
1625 }
1626
1627 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1628 {
1629         __drop_discard_cmd(sbi);
1630 }
1631
1632 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1633                                                         struct discard_cmd *dc)
1634 {
1635         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1636         unsigned int len = 0;
1637
1638         wait_for_completion_io(&dc->wait);
1639         mutex_lock(&dcc->cmd_lock);
1640         f2fs_bug_on(sbi, dc->state != D_DONE);
1641         dc->ref--;
1642         if (!dc->ref) {
1643                 if (!dc->error)
1644                         len = dc->di.len;
1645                 __remove_discard_cmd(sbi, dc);
1646         }
1647         mutex_unlock(&dcc->cmd_lock);
1648
1649         return len;
1650 }
1651
1652 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1653                                                 struct discard_policy *dpolicy,
1654                                                 block_t start, block_t end)
1655 {
1656         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1657         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1658                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1659         struct discard_cmd *dc = NULL, *iter, *tmp;
1660         unsigned int trimmed = 0;
1661
1662 next:
1663         dc = NULL;
1664
1665         mutex_lock(&dcc->cmd_lock);
1666         list_for_each_entry_safe(iter, tmp, wait_list, list) {
1667                 if (iter->di.lstart + iter->di.len <= start ||
1668                                         end <= iter->di.lstart)
1669                         continue;
1670                 if (iter->di.len < dpolicy->granularity)
1671                         continue;
1672                 if (iter->state == D_DONE && !iter->ref) {
1673                         wait_for_completion_io(&iter->wait);
1674                         if (!iter->error)
1675                                 trimmed += iter->di.len;
1676                         __remove_discard_cmd(sbi, iter);
1677                 } else {
1678                         iter->ref++;
1679                         dc = iter;
1680                         break;
1681                 }
1682         }
1683         mutex_unlock(&dcc->cmd_lock);
1684
1685         if (dc) {
1686                 trimmed += __wait_one_discard_bio(sbi, dc);
1687                 goto next;
1688         }
1689
1690         return trimmed;
1691 }
1692
1693 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1694                                                 struct discard_policy *dpolicy)
1695 {
1696         struct discard_policy dp;
1697         unsigned int discard_blks;
1698
1699         if (dpolicy)
1700                 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1701
1702         /* wait all */
1703         __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY);
1704         discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1705         __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY);
1706         discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1707
1708         return discard_blks;
1709 }
1710
1711 /* This should be covered by global mutex, &sit_i->sentry_lock */
1712 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1713 {
1714         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1715         struct discard_cmd *dc;
1716         bool need_wait = false;
1717
1718         mutex_lock(&dcc->cmd_lock);
1719         dc = __lookup_discard_cmd(sbi, blkaddr);
1720         if (dc) {
1721                 if (dc->state == D_PREP) {
1722                         __punch_discard_cmd(sbi, dc, blkaddr);
1723                 } else {
1724                         dc->ref++;
1725                         need_wait = true;
1726                 }
1727         }
1728         mutex_unlock(&dcc->cmd_lock);
1729
1730         if (need_wait)
1731                 __wait_one_discard_bio(sbi, dc);
1732 }
1733
1734 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1735 {
1736         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1737
1738         if (dcc && dcc->f2fs_issue_discard) {
1739                 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1740
1741                 dcc->f2fs_issue_discard = NULL;
1742                 kthread_stop(discard_thread);
1743         }
1744 }
1745
1746 /**
1747  * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT
1748  * @sbi: the f2fs_sb_info data for discard cmd to issue
1749  *
1750  * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped
1751  *
1752  * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false.
1753  */
1754 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1755 {
1756         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1757         struct discard_policy dpolicy;
1758         bool dropped;
1759
1760         if (!atomic_read(&dcc->discard_cmd_cnt))
1761                 return true;
1762
1763         __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1764                                         dcc->discard_granularity);
1765         __issue_discard_cmd(sbi, &dpolicy);
1766         dropped = __drop_discard_cmd(sbi);
1767
1768         /* just to make sure there is no pending discard commands */
1769         __wait_all_discard_cmd(sbi, NULL);
1770
1771         f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1772         return !dropped;
1773 }
1774
1775 static int issue_discard_thread(void *data)
1776 {
1777         struct f2fs_sb_info *sbi = data;
1778         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1779         wait_queue_head_t *q = &dcc->discard_wait_queue;
1780         struct discard_policy dpolicy;
1781         unsigned int wait_ms = dcc->min_discard_issue_time;
1782         int issued;
1783
1784         set_freezable();
1785
1786         do {
1787                 wait_event_interruptible_timeout(*q,
1788                                 kthread_should_stop() || freezing(current) ||
1789                                 dcc->discard_wake,
1790                                 msecs_to_jiffies(wait_ms));
1791
1792                 if (sbi->gc_mode == GC_URGENT_HIGH ||
1793                         !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1794                         __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE,
1795                                                 MIN_DISCARD_GRANULARITY);
1796                 else
1797                         __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1798                                                 dcc->discard_granularity);
1799
1800                 if (dcc->discard_wake)
1801                         dcc->discard_wake = false;
1802
1803                 /* clean up pending candidates before going to sleep */
1804                 if (atomic_read(&dcc->queued_discard))
1805                         __wait_all_discard_cmd(sbi, NULL);
1806
1807                 if (try_to_freeze())
1808                         continue;
1809                 if (f2fs_readonly(sbi->sb))
1810                         continue;
1811                 if (kthread_should_stop())
1812                         return 0;
1813                 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) ||
1814                         !atomic_read(&dcc->discard_cmd_cnt)) {
1815                         wait_ms = dpolicy.max_interval;
1816                         continue;
1817                 }
1818
1819                 sb_start_intwrite(sbi->sb);
1820
1821                 issued = __issue_discard_cmd(sbi, &dpolicy);
1822                 if (issued > 0) {
1823                         __wait_all_discard_cmd(sbi, &dpolicy);
1824                         wait_ms = dpolicy.min_interval;
1825                 } else if (issued == -1) {
1826                         wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1827                         if (!wait_ms)
1828                                 wait_ms = dpolicy.mid_interval;
1829                 } else {
1830                         wait_ms = dpolicy.max_interval;
1831                 }
1832                 if (!atomic_read(&dcc->discard_cmd_cnt))
1833                         wait_ms = dpolicy.max_interval;
1834
1835                 sb_end_intwrite(sbi->sb);
1836
1837         } while (!kthread_should_stop());
1838         return 0;
1839 }
1840
1841 #ifdef CONFIG_BLK_DEV_ZONED
1842 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1843                 struct block_device *bdev, block_t blkstart, block_t blklen)
1844 {
1845         sector_t sector, nr_sects;
1846         block_t lblkstart = blkstart;
1847         int devi = 0;
1848         u64 remainder = 0;
1849
1850         if (f2fs_is_multi_device(sbi)) {
1851                 devi = f2fs_target_device_index(sbi, blkstart);
1852                 if (blkstart < FDEV(devi).start_blk ||
1853                     blkstart > FDEV(devi).end_blk) {
1854                         f2fs_err(sbi, "Invalid block %x", blkstart);
1855                         return -EIO;
1856                 }
1857                 blkstart -= FDEV(devi).start_blk;
1858         }
1859
1860         /* For sequential zones, reset the zone write pointer */
1861         if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1862                 sector = SECTOR_FROM_BLOCK(blkstart);
1863                 nr_sects = SECTOR_FROM_BLOCK(blklen);
1864                 div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder);
1865
1866                 if (remainder || nr_sects != bdev_zone_sectors(bdev)) {
1867                         f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1868                                  devi, sbi->s_ndevs ? FDEV(devi).path : "",
1869                                  blkstart, blklen);
1870                         return -EIO;
1871                 }
1872                 trace_f2fs_issue_reset_zone(bdev, blkstart);
1873                 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1874                                         sector, nr_sects, GFP_NOFS);
1875         }
1876
1877         /* For conventional zones, use regular discard if supported */
1878         __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1879         return 0;
1880 }
1881 #endif
1882
1883 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1884                 struct block_device *bdev, block_t blkstart, block_t blklen)
1885 {
1886 #ifdef CONFIG_BLK_DEV_ZONED
1887         if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1888                 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1889 #endif
1890         __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1891         return 0;
1892 }
1893
1894 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1895                                 block_t blkstart, block_t blklen)
1896 {
1897         sector_t start = blkstart, len = 0;
1898         struct block_device *bdev;
1899         struct seg_entry *se;
1900         unsigned int offset;
1901         block_t i;
1902         int err = 0;
1903
1904         bdev = f2fs_target_device(sbi, blkstart, NULL);
1905
1906         for (i = blkstart; i < blkstart + blklen; i++, len++) {
1907                 if (i != start) {
1908                         struct block_device *bdev2 =
1909                                 f2fs_target_device(sbi, i, NULL);
1910
1911                         if (bdev2 != bdev) {
1912                                 err = __issue_discard_async(sbi, bdev,
1913                                                 start, len);
1914                                 if (err)
1915                                         return err;
1916                                 bdev = bdev2;
1917                                 start = i;
1918                                 len = 0;
1919                         }
1920                 }
1921
1922                 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1923                 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1924
1925                 if (f2fs_block_unit_discard(sbi) &&
1926                                 !f2fs_test_and_set_bit(offset, se->discard_map))
1927                         sbi->discard_blks--;
1928         }
1929
1930         if (len)
1931                 err = __issue_discard_async(sbi, bdev, start, len);
1932         return err;
1933 }
1934
1935 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1936                                                         bool check_only)
1937 {
1938         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1939         int max_blocks = sbi->blocks_per_seg;
1940         struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1941         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1942         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1943         unsigned long *discard_map = (unsigned long *)se->discard_map;
1944         unsigned long *dmap = SIT_I(sbi)->tmp_map;
1945         unsigned int start = 0, end = -1;
1946         bool force = (cpc->reason & CP_DISCARD);
1947         struct discard_entry *de = NULL;
1948         struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1949         int i;
1950
1951         if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1952                         !f2fs_block_unit_discard(sbi))
1953                 return false;
1954
1955         if (!force) {
1956                 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1957                         SM_I(sbi)->dcc_info->nr_discards >=
1958                                 SM_I(sbi)->dcc_info->max_discards)
1959                         return false;
1960         }
1961
1962         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
1963         for (i = 0; i < entries; i++)
1964                 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1965                                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1966
1967         while (force || SM_I(sbi)->dcc_info->nr_discards <=
1968                                 SM_I(sbi)->dcc_info->max_discards) {
1969                 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1970                 if (start >= max_blocks)
1971                         break;
1972
1973                 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1974                 if (force && start && end != max_blocks
1975                                         && (end - start) < cpc->trim_minlen)
1976                         continue;
1977
1978                 if (check_only)
1979                         return true;
1980
1981                 if (!de) {
1982                         de = f2fs_kmem_cache_alloc(discard_entry_slab,
1983                                                 GFP_F2FS_ZERO, true, NULL);
1984                         de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1985                         list_add_tail(&de->list, head);
1986                 }
1987
1988                 for (i = start; i < end; i++)
1989                         __set_bit_le(i, (void *)de->discard_map);
1990
1991                 SM_I(sbi)->dcc_info->nr_discards += end - start;
1992         }
1993         return false;
1994 }
1995
1996 static void release_discard_addr(struct discard_entry *entry)
1997 {
1998         list_del(&entry->list);
1999         kmem_cache_free(discard_entry_slab, entry);
2000 }
2001
2002 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
2003 {
2004         struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
2005         struct discard_entry *entry, *this;
2006
2007         /* drop caches */
2008         list_for_each_entry_safe(entry, this, head, list)
2009                 release_discard_addr(entry);
2010 }
2011
2012 /*
2013  * Should call f2fs_clear_prefree_segments after checkpoint is done.
2014  */
2015 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
2016 {
2017         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2018         unsigned int segno;
2019
2020         mutex_lock(&dirty_i->seglist_lock);
2021         for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
2022                 __set_test_and_free(sbi, segno, false);
2023         mutex_unlock(&dirty_i->seglist_lock);
2024 }
2025
2026 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
2027                                                 struct cp_control *cpc)
2028 {
2029         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2030         struct list_head *head = &dcc->entry_list;
2031         struct discard_entry *entry, *this;
2032         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2033         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2034         unsigned int start = 0, end = -1;
2035         unsigned int secno, start_segno;
2036         bool force = (cpc->reason & CP_DISCARD);
2037         bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2038                                                 DISCARD_UNIT_SECTION;
2039
2040         if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2041                 section_alignment = true;
2042
2043         mutex_lock(&dirty_i->seglist_lock);
2044
2045         while (1) {
2046                 int i;
2047
2048                 if (section_alignment && end != -1)
2049                         end--;
2050                 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2051                 if (start >= MAIN_SEGS(sbi))
2052                         break;
2053                 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2054                                                                 start + 1);
2055
2056                 if (section_alignment) {
2057                         start = rounddown(start, sbi->segs_per_sec);
2058                         end = roundup(end, sbi->segs_per_sec);
2059                 }
2060
2061                 for (i = start; i < end; i++) {
2062                         if (test_and_clear_bit(i, prefree_map))
2063                                 dirty_i->nr_dirty[PRE]--;
2064                 }
2065
2066                 if (!f2fs_realtime_discard_enable(sbi))
2067                         continue;
2068
2069                 if (force && start >= cpc->trim_start &&
2070                                         (end - 1) <= cpc->trim_end)
2071                                 continue;
2072
2073                 /* Should cover 2MB zoned device for zone-based reset */
2074                 if (!f2fs_sb_has_blkzoned(sbi) &&
2075                     (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) {
2076                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2077                                 (end - start) << sbi->log_blocks_per_seg);
2078                         continue;
2079                 }
2080 next:
2081                 secno = GET_SEC_FROM_SEG(sbi, start);
2082                 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2083                 if (!IS_CURSEC(sbi, secno) &&
2084                         !get_valid_blocks(sbi, start, true))
2085                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2086                                 sbi->segs_per_sec << sbi->log_blocks_per_seg);
2087
2088                 start = start_segno + sbi->segs_per_sec;
2089                 if (start < end)
2090                         goto next;
2091                 else
2092                         end = start - 1;
2093         }
2094         mutex_unlock(&dirty_i->seglist_lock);
2095
2096         if (!f2fs_block_unit_discard(sbi))
2097                 goto wakeup;
2098
2099         /* send small discards */
2100         list_for_each_entry_safe(entry, this, head, list) {
2101                 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2102                 bool is_valid = test_bit_le(0, entry->discard_map);
2103
2104 find_next:
2105                 if (is_valid) {
2106                         next_pos = find_next_zero_bit_le(entry->discard_map,
2107                                         sbi->blocks_per_seg, cur_pos);
2108                         len = next_pos - cur_pos;
2109
2110                         if (f2fs_sb_has_blkzoned(sbi) ||
2111                             (force && len < cpc->trim_minlen))
2112                                 goto skip;
2113
2114                         f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2115                                                                         len);
2116                         total_len += len;
2117                 } else {
2118                         next_pos = find_next_bit_le(entry->discard_map,
2119                                         sbi->blocks_per_seg, cur_pos);
2120                 }
2121 skip:
2122                 cur_pos = next_pos;
2123                 is_valid = !is_valid;
2124
2125                 if (cur_pos < sbi->blocks_per_seg)
2126                         goto find_next;
2127
2128                 release_discard_addr(entry);
2129                 dcc->nr_discards -= total_len;
2130         }
2131
2132 wakeup:
2133         wake_up_discard_thread(sbi, false);
2134 }
2135
2136 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2137 {
2138         dev_t dev = sbi->sb->s_bdev->bd_dev;
2139         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2140         int err = 0;
2141
2142         if (!f2fs_realtime_discard_enable(sbi))
2143                 return 0;
2144
2145         dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2146                                 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2147         if (IS_ERR(dcc->f2fs_issue_discard)) {
2148                 err = PTR_ERR(dcc->f2fs_issue_discard);
2149                 dcc->f2fs_issue_discard = NULL;
2150         }
2151
2152         return err;
2153 }
2154
2155 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2156 {
2157         struct discard_cmd_control *dcc;
2158         int err = 0, i;
2159
2160         if (SM_I(sbi)->dcc_info) {
2161                 dcc = SM_I(sbi)->dcc_info;
2162                 goto init_thread;
2163         }
2164
2165         dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2166         if (!dcc)
2167                 return -ENOMEM;
2168
2169         dcc->discard_io_aware_gran = MAX_PLIST_NUM;
2170         dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2171         dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY;
2172         if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2173                 dcc->discard_granularity = sbi->blocks_per_seg;
2174         else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2175                 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2176
2177         INIT_LIST_HEAD(&dcc->entry_list);
2178         for (i = 0; i < MAX_PLIST_NUM; i++)
2179                 INIT_LIST_HEAD(&dcc->pend_list[i]);
2180         INIT_LIST_HEAD(&dcc->wait_list);
2181         INIT_LIST_HEAD(&dcc->fstrim_list);
2182         mutex_init(&dcc->cmd_lock);
2183         atomic_set(&dcc->issued_discard, 0);
2184         atomic_set(&dcc->queued_discard, 0);
2185         atomic_set(&dcc->discard_cmd_cnt, 0);
2186         dcc->nr_discards = 0;
2187         dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2188         dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2189         dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2190         dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2191         dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2192         dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL;
2193         dcc->undiscard_blks = 0;
2194         dcc->next_pos = 0;
2195         dcc->root = RB_ROOT_CACHED;
2196         dcc->rbtree_check = false;
2197
2198         init_waitqueue_head(&dcc->discard_wait_queue);
2199         SM_I(sbi)->dcc_info = dcc;
2200 init_thread:
2201         err = f2fs_start_discard_thread(sbi);
2202         if (err) {
2203                 kfree(dcc);
2204                 SM_I(sbi)->dcc_info = NULL;
2205         }
2206
2207         return err;
2208 }
2209
2210 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2211 {
2212         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2213
2214         if (!dcc)
2215                 return;
2216
2217         f2fs_stop_discard_thread(sbi);
2218
2219         /*
2220          * Recovery can cache discard commands, so in error path of
2221          * fill_super(), it needs to give a chance to handle them.
2222          */
2223         f2fs_issue_discard_timeout(sbi);
2224
2225         kfree(dcc);
2226         SM_I(sbi)->dcc_info = NULL;
2227 }
2228
2229 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2230 {
2231         struct sit_info *sit_i = SIT_I(sbi);
2232
2233         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2234                 sit_i->dirty_sentries++;
2235                 return false;
2236         }
2237
2238         return true;
2239 }
2240
2241 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2242                                         unsigned int segno, int modified)
2243 {
2244         struct seg_entry *se = get_seg_entry(sbi, segno);
2245
2246         se->type = type;
2247         if (modified)
2248                 __mark_sit_entry_dirty(sbi, segno);
2249 }
2250
2251 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2252                                                                 block_t blkaddr)
2253 {
2254         unsigned int segno = GET_SEGNO(sbi, blkaddr);
2255
2256         if (segno == NULL_SEGNO)
2257                 return 0;
2258         return get_seg_entry(sbi, segno)->mtime;
2259 }
2260
2261 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2262                                                 unsigned long long old_mtime)
2263 {
2264         struct seg_entry *se;
2265         unsigned int segno = GET_SEGNO(sbi, blkaddr);
2266         unsigned long long ctime = get_mtime(sbi, false);
2267         unsigned long long mtime = old_mtime ? old_mtime : ctime;
2268
2269         if (segno == NULL_SEGNO)
2270                 return;
2271
2272         se = get_seg_entry(sbi, segno);
2273
2274         if (!se->mtime)
2275                 se->mtime = mtime;
2276         else
2277                 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2278                                                 se->valid_blocks + 1);
2279
2280         if (ctime > SIT_I(sbi)->max_mtime)
2281                 SIT_I(sbi)->max_mtime = ctime;
2282 }
2283
2284 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2285 {
2286         struct seg_entry *se;
2287         unsigned int segno, offset;
2288         long int new_vblocks;
2289         bool exist;
2290 #ifdef CONFIG_F2FS_CHECK_FS
2291         bool mir_exist;
2292 #endif
2293
2294         segno = GET_SEGNO(sbi, blkaddr);
2295
2296         se = get_seg_entry(sbi, segno);
2297         new_vblocks = se->valid_blocks + del;
2298         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2299
2300         f2fs_bug_on(sbi, (new_vblocks < 0 ||
2301                         (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2302
2303         se->valid_blocks = new_vblocks;
2304
2305         /* Update valid block bitmap */
2306         if (del > 0) {
2307                 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2308 #ifdef CONFIG_F2FS_CHECK_FS
2309                 mir_exist = f2fs_test_and_set_bit(offset,
2310                                                 se->cur_valid_map_mir);
2311                 if (unlikely(exist != mir_exist)) {
2312                         f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2313                                  blkaddr, exist);
2314                         f2fs_bug_on(sbi, 1);
2315                 }
2316 #endif
2317                 if (unlikely(exist)) {
2318                         f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2319                                  blkaddr);
2320                         f2fs_bug_on(sbi, 1);
2321                         se->valid_blocks--;
2322                         del = 0;
2323                 }
2324
2325                 if (f2fs_block_unit_discard(sbi) &&
2326                                 !f2fs_test_and_set_bit(offset, se->discard_map))
2327                         sbi->discard_blks--;
2328
2329                 /*
2330                  * SSR should never reuse block which is checkpointed
2331                  * or newly invalidated.
2332                  */
2333                 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2334                         if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2335                                 se->ckpt_valid_blocks++;
2336                 }
2337         } else {
2338                 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2339 #ifdef CONFIG_F2FS_CHECK_FS
2340                 mir_exist = f2fs_test_and_clear_bit(offset,
2341                                                 se->cur_valid_map_mir);
2342                 if (unlikely(exist != mir_exist)) {
2343                         f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2344                                  blkaddr, exist);
2345                         f2fs_bug_on(sbi, 1);
2346                 }
2347 #endif
2348                 if (unlikely(!exist)) {
2349                         f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2350                                  blkaddr);
2351                         f2fs_bug_on(sbi, 1);
2352                         se->valid_blocks++;
2353                         del = 0;
2354                 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2355                         /*
2356                          * If checkpoints are off, we must not reuse data that
2357                          * was used in the previous checkpoint. If it was used
2358                          * before, we must track that to know how much space we
2359                          * really have.
2360                          */
2361                         if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2362                                 spin_lock(&sbi->stat_lock);
2363                                 sbi->unusable_block_count++;
2364                                 spin_unlock(&sbi->stat_lock);
2365                         }
2366                 }
2367
2368                 if (f2fs_block_unit_discard(sbi) &&
2369                         f2fs_test_and_clear_bit(offset, se->discard_map))
2370                         sbi->discard_blks++;
2371         }
2372         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2373                 se->ckpt_valid_blocks += del;
2374
2375         __mark_sit_entry_dirty(sbi, segno);
2376
2377         /* update total number of valid blocks to be written in ckpt area */
2378         SIT_I(sbi)->written_valid_blocks += del;
2379
2380         if (__is_large_section(sbi))
2381                 get_sec_entry(sbi, segno)->valid_blocks += del;
2382 }
2383
2384 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2385 {
2386         unsigned int segno = GET_SEGNO(sbi, addr);
2387         struct sit_info *sit_i = SIT_I(sbi);
2388
2389         f2fs_bug_on(sbi, addr == NULL_ADDR);
2390         if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2391                 return;
2392
2393         invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2394         f2fs_invalidate_compress_page(sbi, addr);
2395
2396         /* add it into sit main buffer */
2397         down_write(&sit_i->sentry_lock);
2398
2399         update_segment_mtime(sbi, addr, 0);
2400         update_sit_entry(sbi, addr, -1);
2401
2402         /* add it into dirty seglist */
2403         locate_dirty_segment(sbi, segno);
2404
2405         up_write(&sit_i->sentry_lock);
2406 }
2407
2408 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2409 {
2410         struct sit_info *sit_i = SIT_I(sbi);
2411         unsigned int segno, offset;
2412         struct seg_entry *se;
2413         bool is_cp = false;
2414
2415         if (!__is_valid_data_blkaddr(blkaddr))
2416                 return true;
2417
2418         down_read(&sit_i->sentry_lock);
2419
2420         segno = GET_SEGNO(sbi, blkaddr);
2421         se = get_seg_entry(sbi, segno);
2422         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2423
2424         if (f2fs_test_bit(offset, se->ckpt_valid_map))
2425                 is_cp = true;
2426
2427         up_read(&sit_i->sentry_lock);
2428
2429         return is_cp;
2430 }
2431
2432 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)
2433 {
2434         struct curseg_info *curseg = CURSEG_I(sbi, type);
2435
2436         if (sbi->ckpt->alloc_type[type] == SSR)
2437                 return sbi->blocks_per_seg;
2438         return curseg->next_blkoff;
2439 }
2440
2441 /*
2442  * Calculate the number of current summary pages for writing
2443  */
2444 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2445 {
2446         int valid_sum_count = 0;
2447         int i, sum_in_page;
2448
2449         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2450                 if (sbi->ckpt->alloc_type[i] != SSR && for_ra)
2451                         valid_sum_count +=
2452                                 le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2453                 else
2454                         valid_sum_count += f2fs_curseg_valid_blocks(sbi, i);
2455         }
2456
2457         sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2458                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2459         if (valid_sum_count <= sum_in_page)
2460                 return 1;
2461         else if ((valid_sum_count - sum_in_page) <=
2462                 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2463                 return 2;
2464         return 3;
2465 }
2466
2467 /*
2468  * Caller should put this summary page
2469  */
2470 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2471 {
2472         if (unlikely(f2fs_cp_error(sbi)))
2473                 return ERR_PTR(-EIO);
2474         return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2475 }
2476
2477 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2478                                         void *src, block_t blk_addr)
2479 {
2480         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2481
2482         memcpy(page_address(page), src, PAGE_SIZE);
2483         set_page_dirty(page);
2484         f2fs_put_page(page, 1);
2485 }
2486
2487 static void write_sum_page(struct f2fs_sb_info *sbi,
2488                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
2489 {
2490         f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2491 }
2492
2493 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2494                                                 int type, block_t blk_addr)
2495 {
2496         struct curseg_info *curseg = CURSEG_I(sbi, type);
2497         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2498         struct f2fs_summary_block *src = curseg->sum_blk;
2499         struct f2fs_summary_block *dst;
2500
2501         dst = (struct f2fs_summary_block *)page_address(page);
2502         memset(dst, 0, PAGE_SIZE);
2503
2504         mutex_lock(&curseg->curseg_mutex);
2505
2506         down_read(&curseg->journal_rwsem);
2507         memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2508         up_read(&curseg->journal_rwsem);
2509
2510         memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2511         memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2512
2513         mutex_unlock(&curseg->curseg_mutex);
2514
2515         set_page_dirty(page);
2516         f2fs_put_page(page, 1);
2517 }
2518
2519 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2520                                 struct curseg_info *curseg, int type)
2521 {
2522         unsigned int segno = curseg->segno + 1;
2523         struct free_segmap_info *free_i = FREE_I(sbi);
2524
2525         if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2526                 return !test_bit(segno, free_i->free_segmap);
2527         return 0;
2528 }
2529
2530 /*
2531  * Find a new segment from the free segments bitmap to right order
2532  * This function should be returned with success, otherwise BUG
2533  */
2534 static void get_new_segment(struct f2fs_sb_info *sbi,
2535                         unsigned int *newseg, bool new_sec, int dir)
2536 {
2537         struct free_segmap_info *free_i = FREE_I(sbi);
2538         unsigned int segno, secno, zoneno;
2539         unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2540         unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2541         unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2542         unsigned int left_start = hint;
2543         bool init = true;
2544         int go_left = 0;
2545         int i;
2546
2547         spin_lock(&free_i->segmap_lock);
2548
2549         if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2550                 segno = find_next_zero_bit(free_i->free_segmap,
2551                         GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2552                 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2553                         goto got_it;
2554         }
2555 find_other_zone:
2556         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2557         if (secno >= MAIN_SECS(sbi)) {
2558                 if (dir == ALLOC_RIGHT) {
2559                         secno = find_first_zero_bit(free_i->free_secmap,
2560                                                         MAIN_SECS(sbi));
2561                         f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2562                 } else {
2563                         go_left = 1;
2564                         left_start = hint - 1;
2565                 }
2566         }
2567         if (go_left == 0)
2568                 goto skip_left;
2569
2570         while (test_bit(left_start, free_i->free_secmap)) {
2571                 if (left_start > 0) {
2572                         left_start--;
2573                         continue;
2574                 }
2575                 left_start = find_first_zero_bit(free_i->free_secmap,
2576                                                         MAIN_SECS(sbi));
2577                 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2578                 break;
2579         }
2580         secno = left_start;
2581 skip_left:
2582         segno = GET_SEG_FROM_SEC(sbi, secno);
2583         zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2584
2585         /* give up on finding another zone */
2586         if (!init)
2587                 goto got_it;
2588         if (sbi->secs_per_zone == 1)
2589                 goto got_it;
2590         if (zoneno == old_zoneno)
2591                 goto got_it;
2592         if (dir == ALLOC_LEFT) {
2593                 if (!go_left && zoneno + 1 >= total_zones)
2594                         goto got_it;
2595                 if (go_left && zoneno == 0)
2596                         goto got_it;
2597         }
2598         for (i = 0; i < NR_CURSEG_TYPE; i++)
2599                 if (CURSEG_I(sbi, i)->zone == zoneno)
2600                         break;
2601
2602         if (i < NR_CURSEG_TYPE) {
2603                 /* zone is in user, try another */
2604                 if (go_left)
2605                         hint = zoneno * sbi->secs_per_zone - 1;
2606                 else if (zoneno + 1 >= total_zones)
2607                         hint = 0;
2608                 else
2609                         hint = (zoneno + 1) * sbi->secs_per_zone;
2610                 init = false;
2611                 goto find_other_zone;
2612         }
2613 got_it:
2614         /* set it as dirty segment in free segmap */
2615         f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2616         __set_inuse(sbi, segno);
2617         *newseg = segno;
2618         spin_unlock(&free_i->segmap_lock);
2619 }
2620
2621 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2622 {
2623         struct curseg_info *curseg = CURSEG_I(sbi, type);
2624         struct summary_footer *sum_footer;
2625         unsigned short seg_type = curseg->seg_type;
2626
2627         curseg->inited = true;
2628         curseg->segno = curseg->next_segno;
2629         curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2630         curseg->next_blkoff = 0;
2631         curseg->next_segno = NULL_SEGNO;
2632
2633         sum_footer = &(curseg->sum_blk->footer);
2634         memset(sum_footer, 0, sizeof(struct summary_footer));
2635
2636         sanity_check_seg_type(sbi, seg_type);
2637
2638         if (IS_DATASEG(seg_type))
2639                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2640         if (IS_NODESEG(seg_type))
2641                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2642         __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2643 }
2644
2645 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2646 {
2647         struct curseg_info *curseg = CURSEG_I(sbi, type);
2648         unsigned short seg_type = curseg->seg_type;
2649
2650         sanity_check_seg_type(sbi, seg_type);
2651         if (f2fs_need_rand_seg(sbi))
2652                 return get_random_u32_below(MAIN_SECS(sbi) * sbi->segs_per_sec);
2653
2654         /* if segs_per_sec is large than 1, we need to keep original policy. */
2655         if (__is_large_section(sbi))
2656                 return curseg->segno;
2657
2658         /* inmem log may not locate on any segment after mount */
2659         if (!curseg->inited)
2660                 return 0;
2661
2662         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2663                 return 0;
2664
2665         if (test_opt(sbi, NOHEAP) &&
2666                 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2667                 return 0;
2668
2669         if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2670                 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2671
2672         /* find segments from 0 to reuse freed segments */
2673         if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2674                 return 0;
2675
2676         return curseg->segno;
2677 }
2678
2679 /*
2680  * Allocate a current working segment.
2681  * This function always allocates a free segment in LFS manner.
2682  */
2683 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2684 {
2685         struct curseg_info *curseg = CURSEG_I(sbi, type);
2686         unsigned short seg_type = curseg->seg_type;
2687         unsigned int segno = curseg->segno;
2688         int dir = ALLOC_LEFT;
2689
2690         if (curseg->inited)
2691                 write_sum_page(sbi, curseg->sum_blk,
2692                                 GET_SUM_BLOCK(sbi, segno));
2693         if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2694                 dir = ALLOC_RIGHT;
2695
2696         if (test_opt(sbi, NOHEAP))
2697                 dir = ALLOC_RIGHT;
2698
2699         segno = __get_next_segno(sbi, type);
2700         get_new_segment(sbi, &segno, new_sec, dir);
2701         curseg->next_segno = segno;
2702         reset_curseg(sbi, type, 1);
2703         curseg->alloc_type = LFS;
2704         if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2705                 curseg->fragment_remained_chunk =
2706                                 get_random_u32_inclusive(1, sbi->max_fragment_chunk);
2707 }
2708
2709 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2710                                         int segno, block_t start)
2711 {
2712         struct seg_entry *se = get_seg_entry(sbi, segno);
2713         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2714         unsigned long *target_map = SIT_I(sbi)->tmp_map;
2715         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2716         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2717         int i;
2718
2719         for (i = 0; i < entries; i++)
2720                 target_map[i] = ckpt_map[i] | cur_map[i];
2721
2722         return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2723 }
2724
2725 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,
2726                 struct curseg_info *seg)
2727 {
2728         return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1);
2729 }
2730
2731 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2732 {
2733         return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2734 }
2735
2736 /*
2737  * This function always allocates a used segment(from dirty seglist) by SSR
2738  * manner, so it should recover the existing segment information of valid blocks
2739  */
2740 static void change_curseg(struct f2fs_sb_info *sbi, int type)
2741 {
2742         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2743         struct curseg_info *curseg = CURSEG_I(sbi, type);
2744         unsigned int new_segno = curseg->next_segno;
2745         struct f2fs_summary_block *sum_node;
2746         struct page *sum_page;
2747
2748         write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno));
2749
2750         __set_test_and_inuse(sbi, new_segno);
2751
2752         mutex_lock(&dirty_i->seglist_lock);
2753         __remove_dirty_segment(sbi, new_segno, PRE);
2754         __remove_dirty_segment(sbi, new_segno, DIRTY);
2755         mutex_unlock(&dirty_i->seglist_lock);
2756
2757         reset_curseg(sbi, type, 1);
2758         curseg->alloc_type = SSR;
2759         curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2760
2761         sum_page = f2fs_get_sum_page(sbi, new_segno);
2762         if (IS_ERR(sum_page)) {
2763                 /* GC won't be able to use stale summary pages by cp_error */
2764                 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2765                 return;
2766         }
2767         sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2768         memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2769         f2fs_put_page(sum_page, 1);
2770 }
2771
2772 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2773                                 int alloc_mode, unsigned long long age);
2774
2775 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2776                                         int target_type, int alloc_mode,
2777                                         unsigned long long age)
2778 {
2779         struct curseg_info *curseg = CURSEG_I(sbi, type);
2780
2781         curseg->seg_type = target_type;
2782
2783         if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2784                 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2785
2786                 curseg->seg_type = se->type;
2787                 change_curseg(sbi, type);
2788         } else {
2789                 /* allocate cold segment by default */
2790                 curseg->seg_type = CURSEG_COLD_DATA;
2791                 new_curseg(sbi, type, true);
2792         }
2793         stat_inc_seg_type(sbi, curseg);
2794 }
2795
2796 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2797 {
2798         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2799
2800         if (!sbi->am.atgc_enabled)
2801                 return;
2802
2803         f2fs_down_read(&SM_I(sbi)->curseg_lock);
2804
2805         mutex_lock(&curseg->curseg_mutex);
2806         down_write(&SIT_I(sbi)->sentry_lock);
2807
2808         get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2809
2810         up_write(&SIT_I(sbi)->sentry_lock);
2811         mutex_unlock(&curseg->curseg_mutex);
2812
2813         f2fs_up_read(&SM_I(sbi)->curseg_lock);
2814
2815 }
2816 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2817 {
2818         __f2fs_init_atgc_curseg(sbi);
2819 }
2820
2821 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2822 {
2823         struct curseg_info *curseg = CURSEG_I(sbi, type);
2824
2825         mutex_lock(&curseg->curseg_mutex);
2826         if (!curseg->inited)
2827                 goto out;
2828
2829         if (get_valid_blocks(sbi, curseg->segno, false)) {
2830                 write_sum_page(sbi, curseg->sum_blk,
2831                                 GET_SUM_BLOCK(sbi, curseg->segno));
2832         } else {
2833                 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2834                 __set_test_and_free(sbi, curseg->segno, true);
2835                 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2836         }
2837 out:
2838         mutex_unlock(&curseg->curseg_mutex);
2839 }
2840
2841 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2842 {
2843         __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2844
2845         if (sbi->am.atgc_enabled)
2846                 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2847 }
2848
2849 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2850 {
2851         struct curseg_info *curseg = CURSEG_I(sbi, type);
2852
2853         mutex_lock(&curseg->curseg_mutex);
2854         if (!curseg->inited)
2855                 goto out;
2856         if (get_valid_blocks(sbi, curseg->segno, false))
2857                 goto out;
2858
2859         mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2860         __set_test_and_inuse(sbi, curseg->segno);
2861         mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2862 out:
2863         mutex_unlock(&curseg->curseg_mutex);
2864 }
2865
2866 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2867 {
2868         __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2869
2870         if (sbi->am.atgc_enabled)
2871                 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2872 }
2873
2874 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2875                                 int alloc_mode, unsigned long long age)
2876 {
2877         struct curseg_info *curseg = CURSEG_I(sbi, type);
2878         unsigned segno = NULL_SEGNO;
2879         unsigned short seg_type = curseg->seg_type;
2880         int i, cnt;
2881         bool reversed = false;
2882
2883         sanity_check_seg_type(sbi, seg_type);
2884
2885         /* f2fs_need_SSR() already forces to do this */
2886         if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2887                 curseg->next_segno = segno;
2888                 return 1;
2889         }
2890
2891         /* For node segments, let's do SSR more intensively */
2892         if (IS_NODESEG(seg_type)) {
2893                 if (seg_type >= CURSEG_WARM_NODE) {
2894                         reversed = true;
2895                         i = CURSEG_COLD_NODE;
2896                 } else {
2897                         i = CURSEG_HOT_NODE;
2898                 }
2899                 cnt = NR_CURSEG_NODE_TYPE;
2900         } else {
2901                 if (seg_type >= CURSEG_WARM_DATA) {
2902                         reversed = true;
2903                         i = CURSEG_COLD_DATA;
2904                 } else {
2905                         i = CURSEG_HOT_DATA;
2906                 }
2907                 cnt = NR_CURSEG_DATA_TYPE;
2908         }
2909
2910         for (; cnt-- > 0; reversed ? i-- : i++) {
2911                 if (i == seg_type)
2912                         continue;
2913                 if (!f2fs_get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2914                         curseg->next_segno = segno;
2915                         return 1;
2916                 }
2917         }
2918
2919         /* find valid_blocks=0 in dirty list */
2920         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2921                 segno = get_free_segment(sbi);
2922                 if (segno != NULL_SEGNO) {
2923                         curseg->next_segno = segno;
2924                         return 1;
2925                 }
2926         }
2927         return 0;
2928 }
2929
2930 static bool need_new_seg(struct f2fs_sb_info *sbi, int type)
2931 {
2932         struct curseg_info *curseg = CURSEG_I(sbi, type);
2933
2934         if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2935             curseg->seg_type == CURSEG_WARM_NODE)
2936                 return true;
2937         if (curseg->alloc_type == LFS &&
2938             is_next_segment_free(sbi, curseg, type) &&
2939             likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2940                 return true;
2941         if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0))
2942                 return true;
2943         return false;
2944 }
2945
2946 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2947                                         unsigned int start, unsigned int end)
2948 {
2949         struct curseg_info *curseg = CURSEG_I(sbi, type);
2950         unsigned int segno;
2951
2952         f2fs_down_read(&SM_I(sbi)->curseg_lock);
2953         mutex_lock(&curseg->curseg_mutex);
2954         down_write(&SIT_I(sbi)->sentry_lock);
2955
2956         segno = CURSEG_I(sbi, type)->segno;
2957         if (segno < start || segno > end)
2958                 goto unlock;
2959
2960         if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
2961                 change_curseg(sbi, type);
2962         else
2963                 new_curseg(sbi, type, true);
2964
2965         stat_inc_seg_type(sbi, curseg);
2966
2967         locate_dirty_segment(sbi, segno);
2968 unlock:
2969         up_write(&SIT_I(sbi)->sentry_lock);
2970
2971         if (segno != curseg->segno)
2972                 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
2973                             type, segno, curseg->segno);
2974
2975         mutex_unlock(&curseg->curseg_mutex);
2976         f2fs_up_read(&SM_I(sbi)->curseg_lock);
2977 }
2978
2979 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
2980                                                 bool new_sec, bool force)
2981 {
2982         struct curseg_info *curseg = CURSEG_I(sbi, type);
2983         unsigned int old_segno;
2984
2985         if (!force && curseg->inited &&
2986             !curseg->next_blkoff &&
2987             !get_valid_blocks(sbi, curseg->segno, new_sec) &&
2988             !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
2989                 return;
2990
2991         old_segno = curseg->segno;
2992         new_curseg(sbi, type, true);
2993         stat_inc_seg_type(sbi, curseg);
2994         locate_dirty_segment(sbi, old_segno);
2995 }
2996
2997 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
2998 {
2999         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3000         down_write(&SIT_I(sbi)->sentry_lock);
3001         __allocate_new_segment(sbi, type, true, force);
3002         up_write(&SIT_I(sbi)->sentry_lock);
3003         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3004 }
3005
3006 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
3007 {
3008         int i;
3009
3010         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3011         down_write(&SIT_I(sbi)->sentry_lock);
3012         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
3013                 __allocate_new_segment(sbi, i, false, false);
3014         up_write(&SIT_I(sbi)->sentry_lock);
3015         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3016 }
3017
3018 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3019                                                 struct cp_control *cpc)
3020 {
3021         __u64 trim_start = cpc->trim_start;
3022         bool has_candidate = false;
3023
3024         down_write(&SIT_I(sbi)->sentry_lock);
3025         for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3026                 if (add_discard_addrs(sbi, cpc, true)) {
3027                         has_candidate = true;
3028                         break;
3029                 }
3030         }
3031         up_write(&SIT_I(sbi)->sentry_lock);
3032
3033         cpc->trim_start = trim_start;
3034         return has_candidate;
3035 }
3036
3037 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3038                                         struct discard_policy *dpolicy,
3039                                         unsigned int start, unsigned int end)
3040 {
3041         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3042         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3043         struct rb_node **insert_p = NULL, *insert_parent = NULL;
3044         struct discard_cmd *dc;
3045         struct blk_plug plug;
3046         int issued;
3047         unsigned int trimmed = 0;
3048
3049 next:
3050         issued = 0;
3051
3052         mutex_lock(&dcc->cmd_lock);
3053         if (unlikely(dcc->rbtree_check))
3054                 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
3055
3056         dc = __lookup_discard_cmd_ret(&dcc->root, start,
3057                                 &prev_dc, &next_dc, &insert_p, &insert_parent);
3058         if (!dc)
3059                 dc = next_dc;
3060
3061         blk_start_plug(&plug);
3062
3063         while (dc && dc->di.lstart <= end) {
3064                 struct rb_node *node;
3065                 int err = 0;
3066
3067                 if (dc->di.len < dpolicy->granularity)
3068                         goto skip;
3069
3070                 if (dc->state != D_PREP) {
3071                         list_move_tail(&dc->list, &dcc->fstrim_list);
3072                         goto skip;
3073                 }
3074
3075                 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3076
3077                 if (issued >= dpolicy->max_requests) {
3078                         start = dc->di.lstart + dc->di.len;
3079
3080                         if (err)
3081                                 __remove_discard_cmd(sbi, dc);
3082
3083                         blk_finish_plug(&plug);
3084                         mutex_unlock(&dcc->cmd_lock);
3085                         trimmed += __wait_all_discard_cmd(sbi, NULL);
3086                         f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
3087                         goto next;
3088                 }
3089 skip:
3090                 node = rb_next(&dc->rb_node);
3091                 if (err)
3092                         __remove_discard_cmd(sbi, dc);
3093                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3094
3095                 if (fatal_signal_pending(current))
3096                         break;
3097         }
3098
3099         blk_finish_plug(&plug);
3100         mutex_unlock(&dcc->cmd_lock);
3101
3102         return trimmed;
3103 }
3104
3105 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3106 {
3107         __u64 start = F2FS_BYTES_TO_BLK(range->start);
3108         __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3109         unsigned int start_segno, end_segno;
3110         block_t start_block, end_block;
3111         struct cp_control cpc;
3112         struct discard_policy dpolicy;
3113         unsigned long long trimmed = 0;
3114         int err = 0;
3115         bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3116
3117         if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3118                 return -EINVAL;
3119
3120         if (end < MAIN_BLKADDR(sbi))
3121                 goto out;
3122
3123         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3124                 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3125                 return -EFSCORRUPTED;
3126         }
3127
3128         /* start/end segment number in main_area */
3129         start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3130         end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3131                                                 GET_SEGNO(sbi, end);
3132         if (need_align) {
3133                 start_segno = rounddown(start_segno, sbi->segs_per_sec);
3134                 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3135         }
3136
3137         cpc.reason = CP_DISCARD;
3138         cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3139         cpc.trim_start = start_segno;
3140         cpc.trim_end = end_segno;
3141
3142         if (sbi->discard_blks == 0)
3143                 goto out;
3144
3145         f2fs_down_write(&sbi->gc_lock);
3146         err = f2fs_write_checkpoint(sbi, &cpc);
3147         f2fs_up_write(&sbi->gc_lock);
3148         if (err)
3149                 goto out;
3150
3151         /*
3152          * We filed discard candidates, but actually we don't need to wait for
3153          * all of them, since they'll be issued in idle time along with runtime
3154          * discard option. User configuration looks like using runtime discard
3155          * or periodic fstrim instead of it.
3156          */
3157         if (f2fs_realtime_discard_enable(sbi))
3158                 goto out;
3159
3160         start_block = START_BLOCK(sbi, start_segno);
3161         end_block = START_BLOCK(sbi, end_segno + 1);
3162
3163         __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3164         trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3165                                         start_block, end_block);
3166
3167         trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3168                                         start_block, end_block);
3169 out:
3170         if (!err)
3171                 range->len = F2FS_BLK_TO_BYTES(trimmed);
3172         return err;
3173 }
3174
3175 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3176 {
3177         switch (hint) {
3178         case WRITE_LIFE_SHORT:
3179                 return CURSEG_HOT_DATA;
3180         case WRITE_LIFE_EXTREME:
3181                 return CURSEG_COLD_DATA;
3182         default:
3183                 return CURSEG_WARM_DATA;
3184         }
3185 }
3186
3187 static int __get_segment_type_2(struct f2fs_io_info *fio)
3188 {
3189         if (fio->type == DATA)
3190                 return CURSEG_HOT_DATA;
3191         else
3192                 return CURSEG_HOT_NODE;
3193 }
3194
3195 static int __get_segment_type_4(struct f2fs_io_info *fio)
3196 {
3197         if (fio->type == DATA) {
3198                 struct inode *inode = fio->page->mapping->host;
3199
3200                 if (S_ISDIR(inode->i_mode))
3201                         return CURSEG_HOT_DATA;
3202                 else
3203                         return CURSEG_COLD_DATA;
3204         } else {
3205                 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3206                         return CURSEG_WARM_NODE;
3207                 else
3208                         return CURSEG_COLD_NODE;
3209         }
3210 }
3211
3212 static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
3213 {
3214         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3215         struct extent_info ei = {};
3216
3217         if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) {
3218                 if (!ei.age)
3219                         return NO_CHECK_TYPE;
3220                 if (ei.age <= sbi->hot_data_age_threshold)
3221                         return CURSEG_HOT_DATA;
3222                 if (ei.age <= sbi->warm_data_age_threshold)
3223                         return CURSEG_WARM_DATA;
3224                 return CURSEG_COLD_DATA;
3225         }
3226         return NO_CHECK_TYPE;
3227 }
3228
3229 static int __get_segment_type_6(struct f2fs_io_info *fio)
3230 {
3231         if (fio->type == DATA) {
3232                 struct inode *inode = fio->page->mapping->host;
3233                 int type;
3234
3235                 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3236                         return CURSEG_COLD_DATA_PINNED;
3237
3238                 if (page_private_gcing(fio->page)) {
3239                         if (fio->sbi->am.atgc_enabled &&
3240                                 (fio->io_type == FS_DATA_IO) &&
3241                                 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3242                                 return CURSEG_ALL_DATA_ATGC;
3243                         else
3244                                 return CURSEG_COLD_DATA;
3245                 }
3246                 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3247                         return CURSEG_COLD_DATA;
3248
3249                 type = __get_age_segment_type(inode, fio->page->index);
3250                 if (type != NO_CHECK_TYPE)
3251                         return type;
3252
3253                 if (file_is_hot(inode) ||
3254                                 is_inode_flag_set(inode, FI_HOT_DATA) ||
3255                                 f2fs_is_cow_file(inode))
3256                         return CURSEG_HOT_DATA;
3257                 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3258         } else {
3259                 if (IS_DNODE(fio->page))
3260                         return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3261                                                 CURSEG_HOT_NODE;
3262                 return CURSEG_COLD_NODE;
3263         }
3264 }
3265
3266 static int __get_segment_type(struct f2fs_io_info *fio)
3267 {
3268         int type = 0;
3269
3270         switch (F2FS_OPTION(fio->sbi).active_logs) {
3271         case 2:
3272                 type = __get_segment_type_2(fio);
3273                 break;
3274         case 4:
3275                 type = __get_segment_type_4(fio);
3276                 break;
3277         case 6:
3278                 type = __get_segment_type_6(fio);
3279                 break;
3280         default:
3281                 f2fs_bug_on(fio->sbi, true);
3282         }
3283
3284         if (IS_HOT(type))
3285                 fio->temp = HOT;
3286         else if (IS_WARM(type))
3287                 fio->temp = WARM;
3288         else
3289                 fio->temp = COLD;
3290         return type;
3291 }
3292
3293 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
3294                 struct curseg_info *seg)
3295 {
3296         /* To allocate block chunks in different sizes, use random number */
3297         if (--seg->fragment_remained_chunk > 0)
3298                 return;
3299
3300         seg->fragment_remained_chunk =
3301                 get_random_u32_inclusive(1, sbi->max_fragment_chunk);
3302         seg->next_blkoff +=
3303                 get_random_u32_inclusive(1, sbi->max_fragment_hole);
3304 }
3305
3306 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3307                 block_t old_blkaddr, block_t *new_blkaddr,
3308                 struct f2fs_summary *sum, int type,
3309                 struct f2fs_io_info *fio)
3310 {
3311         struct sit_info *sit_i = SIT_I(sbi);
3312         struct curseg_info *curseg = CURSEG_I(sbi, type);
3313         unsigned long long old_mtime;
3314         bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3315         struct seg_entry *se = NULL;
3316         bool segment_full = false;
3317
3318         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3319
3320         mutex_lock(&curseg->curseg_mutex);
3321         down_write(&sit_i->sentry_lock);
3322
3323         if (from_gc) {
3324                 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3325                 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3326                 sanity_check_seg_type(sbi, se->type);
3327                 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3328         }
3329         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3330
3331         f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3332
3333         f2fs_wait_discard_bio(sbi, *new_blkaddr);
3334
3335         curseg->sum_blk->entries[curseg->next_blkoff] = *sum;
3336         if (curseg->alloc_type == SSR) {
3337                 curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg);
3338         } else {
3339                 curseg->next_blkoff++;
3340                 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
3341                         f2fs_randomize_chunk(sbi, curseg);
3342         }
3343         if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno))
3344                 segment_full = true;
3345         stat_inc_block_count(sbi, curseg);
3346
3347         if (from_gc) {
3348                 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3349         } else {
3350                 update_segment_mtime(sbi, old_blkaddr, 0);
3351                 old_mtime = 0;
3352         }
3353         update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3354
3355         /*
3356          * SIT information should be updated before segment allocation,
3357          * since SSR needs latest valid block information.
3358          */
3359         update_sit_entry(sbi, *new_blkaddr, 1);
3360         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3361                 update_sit_entry(sbi, old_blkaddr, -1);
3362
3363         /*
3364          * If the current segment is full, flush it out and replace it with a
3365          * new segment.
3366          */
3367         if (segment_full) {
3368                 if (from_gc) {
3369                         get_atssr_segment(sbi, type, se->type,
3370                                                 AT_SSR, se->mtime);
3371                 } else {
3372                         if (need_new_seg(sbi, type))
3373                                 new_curseg(sbi, type, false);
3374                         else
3375                                 change_curseg(sbi, type);
3376                         stat_inc_seg_type(sbi, curseg);
3377                 }
3378         }
3379         /*
3380          * segment dirty status should be updated after segment allocation,
3381          * so we just need to update status only one time after previous
3382          * segment being closed.
3383          */
3384         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3385         locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3386
3387         if (IS_DATASEG(type))
3388                 atomic64_inc(&sbi->allocated_data_blocks);
3389
3390         up_write(&sit_i->sentry_lock);
3391
3392         if (page && IS_NODESEG(type)) {
3393                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3394
3395                 f2fs_inode_chksum_set(sbi, page);
3396         }
3397
3398         if (fio) {
3399                 struct f2fs_bio_info *io;
3400
3401                 if (F2FS_IO_ALIGNED(sbi))
3402                         fio->retry = 0;
3403
3404                 INIT_LIST_HEAD(&fio->list);
3405                 fio->in_list = 1;
3406                 io = sbi->write_io[fio->type] + fio->temp;
3407                 spin_lock(&io->io_lock);
3408                 list_add_tail(&fio->list, &io->io_list);
3409                 spin_unlock(&io->io_lock);
3410         }
3411
3412         mutex_unlock(&curseg->curseg_mutex);
3413
3414         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3415 }
3416
3417 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3418                                         block_t blkaddr, unsigned int blkcnt)
3419 {
3420         if (!f2fs_is_multi_device(sbi))
3421                 return;
3422
3423         while (1) {
3424                 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3425                 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3426
3427                 /* update device state for fsync */
3428                 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3429
3430                 /* update device state for checkpoint */
3431                 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3432                         spin_lock(&sbi->dev_lock);
3433                         f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3434                         spin_unlock(&sbi->dev_lock);
3435                 }
3436
3437                 if (blkcnt <= blks)
3438                         break;
3439                 blkcnt -= blks;
3440                 blkaddr += blks;
3441         }
3442 }
3443
3444 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3445 {
3446         int type = __get_segment_type(fio);
3447         bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3448
3449         if (keep_order)
3450                 f2fs_down_read(&fio->sbi->io_order_lock);
3451 reallocate:
3452         f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3453                         &fio->new_blkaddr, sum, type, fio);
3454         if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3455                 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3456                                         fio->old_blkaddr, fio->old_blkaddr);
3457                 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
3458         }
3459
3460         /* writeout dirty page into bdev */
3461         f2fs_submit_page_write(fio);
3462         if (fio->retry) {
3463                 fio->old_blkaddr = fio->new_blkaddr;
3464                 goto reallocate;
3465         }
3466
3467         f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3468
3469         if (keep_order)
3470                 f2fs_up_read(&fio->sbi->io_order_lock);
3471 }
3472
3473 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3474                                         enum iostat_type io_type)
3475 {
3476         struct f2fs_io_info fio = {
3477                 .sbi = sbi,
3478                 .type = META,
3479                 .temp = HOT,
3480                 .op = REQ_OP_WRITE,
3481                 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3482                 .old_blkaddr = page->index,
3483                 .new_blkaddr = page->index,
3484                 .page = page,
3485                 .encrypted_page = NULL,
3486                 .in_list = 0,
3487         };
3488
3489         if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3490                 fio.op_flags &= ~REQ_META;
3491
3492         set_page_writeback(page);
3493         f2fs_submit_page_write(&fio);
3494
3495         stat_inc_meta_count(sbi, page->index);
3496         f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
3497 }
3498
3499 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3500 {
3501         struct f2fs_summary sum;
3502
3503         set_summary(&sum, nid, 0, 0);
3504         do_write_page(&sum, fio);
3505
3506         f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
3507 }
3508
3509 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3510                                         struct f2fs_io_info *fio)
3511 {
3512         struct f2fs_sb_info *sbi = fio->sbi;
3513         struct f2fs_summary sum;
3514
3515         f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3516         if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO)
3517                 f2fs_update_age_extent_cache(dn);
3518         set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3519         do_write_page(&sum, fio);
3520         f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3521
3522         f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
3523 }
3524
3525 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3526 {
3527         int err;
3528         struct f2fs_sb_info *sbi = fio->sbi;
3529         unsigned int segno;
3530
3531         fio->new_blkaddr = fio->old_blkaddr;
3532         /* i/o temperature is needed for passing down write hints */
3533         __get_segment_type(fio);
3534
3535         segno = GET_SEGNO(sbi, fio->new_blkaddr);
3536
3537         if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3538                 set_sbi_flag(sbi, SBI_NEED_FSCK);
3539                 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3540                           __func__, segno);
3541                 err = -EFSCORRUPTED;
3542                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
3543                 goto drop_bio;
3544         }
3545
3546         if (f2fs_cp_error(sbi)) {
3547                 err = -EIO;
3548                 goto drop_bio;
3549         }
3550
3551         if (fio->post_read)
3552                 invalidate_mapping_pages(META_MAPPING(sbi),
3553                                 fio->new_blkaddr, fio->new_blkaddr);
3554
3555         stat_inc_inplace_blocks(fio->sbi);
3556
3557         if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi))
3558                 err = f2fs_merge_page_bio(fio);
3559         else
3560                 err = f2fs_submit_page_bio(fio);
3561         if (!err) {
3562                 f2fs_update_device_state(fio->sbi, fio->ino,
3563                                                 fio->new_blkaddr, 1);
3564                 f2fs_update_iostat(fio->sbi, fio->page->mapping->host,
3565                                                 fio->io_type, F2FS_BLKSIZE);
3566         }
3567
3568         return err;
3569 drop_bio:
3570         if (fio->bio && *(fio->bio)) {
3571                 struct bio *bio = *(fio->bio);
3572
3573                 bio->bi_status = BLK_STS_IOERR;
3574                 bio_endio(bio);
3575                 *(fio->bio) = NULL;
3576         }
3577         return err;
3578 }
3579
3580 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3581                                                 unsigned int segno)
3582 {
3583         int i;
3584
3585         for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3586                 if (CURSEG_I(sbi, i)->segno == segno)
3587                         break;
3588         }
3589         return i;
3590 }
3591
3592 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3593                                 block_t old_blkaddr, block_t new_blkaddr,
3594                                 bool recover_curseg, bool recover_newaddr,
3595                                 bool from_gc)
3596 {
3597         struct sit_info *sit_i = SIT_I(sbi);
3598         struct curseg_info *curseg;
3599         unsigned int segno, old_cursegno;
3600         struct seg_entry *se;
3601         int type;
3602         unsigned short old_blkoff;
3603         unsigned char old_alloc_type;
3604
3605         segno = GET_SEGNO(sbi, new_blkaddr);
3606         se = get_seg_entry(sbi, segno);
3607         type = se->type;
3608
3609         f2fs_down_write(&SM_I(sbi)->curseg_lock);
3610
3611         if (!recover_curseg) {
3612                 /* for recovery flow */
3613                 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3614                         if (old_blkaddr == NULL_ADDR)
3615                                 type = CURSEG_COLD_DATA;
3616                         else
3617                                 type = CURSEG_WARM_DATA;
3618                 }
3619         } else {
3620                 if (IS_CURSEG(sbi, segno)) {
3621                         /* se->type is volatile as SSR allocation */
3622                         type = __f2fs_get_curseg(sbi, segno);
3623                         f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3624                 } else {
3625                         type = CURSEG_WARM_DATA;
3626                 }
3627         }
3628
3629         f2fs_bug_on(sbi, !IS_DATASEG(type));
3630         curseg = CURSEG_I(sbi, type);
3631
3632         mutex_lock(&curseg->curseg_mutex);
3633         down_write(&sit_i->sentry_lock);
3634
3635         old_cursegno = curseg->segno;
3636         old_blkoff = curseg->next_blkoff;
3637         old_alloc_type = curseg->alloc_type;
3638
3639         /* change the current segment */
3640         if (segno != curseg->segno) {
3641                 curseg->next_segno = segno;
3642                 change_curseg(sbi, type);
3643         }
3644
3645         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3646         curseg->sum_blk->entries[curseg->next_blkoff] = *sum;
3647
3648         if (!recover_curseg || recover_newaddr) {
3649                 if (!from_gc)
3650                         update_segment_mtime(sbi, new_blkaddr, 0);
3651                 update_sit_entry(sbi, new_blkaddr, 1);
3652         }
3653         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3654                 invalidate_mapping_pages(META_MAPPING(sbi),
3655                                         old_blkaddr, old_blkaddr);
3656                 f2fs_invalidate_compress_page(sbi, old_blkaddr);
3657                 if (!from_gc)
3658                         update_segment_mtime(sbi, old_blkaddr, 0);
3659                 update_sit_entry(sbi, old_blkaddr, -1);
3660         }
3661
3662         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3663         locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3664
3665         locate_dirty_segment(sbi, old_cursegno);
3666
3667         if (recover_curseg) {
3668                 if (old_cursegno != curseg->segno) {
3669                         curseg->next_segno = old_cursegno;
3670                         change_curseg(sbi, type);
3671                 }
3672                 curseg->next_blkoff = old_blkoff;
3673                 curseg->alloc_type = old_alloc_type;
3674         }
3675
3676         up_write(&sit_i->sentry_lock);
3677         mutex_unlock(&curseg->curseg_mutex);
3678         f2fs_up_write(&SM_I(sbi)->curseg_lock);
3679 }
3680
3681 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3682                                 block_t old_addr, block_t new_addr,
3683                                 unsigned char version, bool recover_curseg,
3684                                 bool recover_newaddr)
3685 {
3686         struct f2fs_summary sum;
3687
3688         set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3689
3690         f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3691                                         recover_curseg, recover_newaddr, false);
3692
3693         f2fs_update_data_blkaddr(dn, new_addr);
3694 }
3695
3696 void f2fs_wait_on_page_writeback(struct page *page,
3697                                 enum page_type type, bool ordered, bool locked)
3698 {
3699         if (PageWriteback(page)) {
3700                 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3701
3702                 /* submit cached LFS IO */
3703                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3704                 /* submit cached IPU IO */
3705                 f2fs_submit_merged_ipu_write(sbi, NULL, page);
3706                 if (ordered) {
3707                         wait_on_page_writeback(page);
3708                         f2fs_bug_on(sbi, locked && PageWriteback(page));
3709                 } else {
3710                         wait_for_stable_page(page);
3711                 }
3712         }
3713 }
3714
3715 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3716 {
3717         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3718         struct page *cpage;
3719
3720         if (!f2fs_post_read_required(inode))
3721                 return;
3722
3723         if (!__is_valid_data_blkaddr(blkaddr))
3724                 return;
3725
3726         cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3727         if (cpage) {
3728                 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3729                 f2fs_put_page(cpage, 1);
3730         }
3731 }
3732
3733 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3734                                                                 block_t len)
3735 {
3736         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3737         block_t i;
3738
3739         if (!f2fs_post_read_required(inode))
3740                 return;
3741
3742         for (i = 0; i < len; i++)
3743                 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3744
3745         invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1);
3746 }
3747
3748 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3749 {
3750         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3751         struct curseg_info *seg_i;
3752         unsigned char *kaddr;
3753         struct page *page;
3754         block_t start;
3755         int i, j, offset;
3756
3757         start = start_sum_block(sbi);
3758
3759         page = f2fs_get_meta_page(sbi, start++);
3760         if (IS_ERR(page))
3761                 return PTR_ERR(page);
3762         kaddr = (unsigned char *)page_address(page);
3763
3764         /* Step 1: restore nat cache */
3765         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3766         memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3767
3768         /* Step 2: restore sit cache */
3769         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3770         memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3771         offset = 2 * SUM_JOURNAL_SIZE;
3772
3773         /* Step 3: restore summary entries */
3774         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3775                 unsigned short blk_off;
3776                 unsigned int segno;
3777
3778                 seg_i = CURSEG_I(sbi, i);
3779                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3780                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3781                 seg_i->next_segno = segno;
3782                 reset_curseg(sbi, i, 0);
3783                 seg_i->alloc_type = ckpt->alloc_type[i];
3784                 seg_i->next_blkoff = blk_off;
3785
3786                 if (seg_i->alloc_type == SSR)
3787                         blk_off = sbi->blocks_per_seg;
3788
3789                 for (j = 0; j < blk_off; j++) {
3790                         struct f2fs_summary *s;
3791
3792                         s = (struct f2fs_summary *)(kaddr + offset);
3793                         seg_i->sum_blk->entries[j] = *s;
3794                         offset += SUMMARY_SIZE;
3795                         if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3796                                                 SUM_FOOTER_SIZE)
3797                                 continue;
3798
3799                         f2fs_put_page(page, 1);
3800                         page = NULL;
3801
3802                         page = f2fs_get_meta_page(sbi, start++);
3803                         if (IS_ERR(page))
3804                                 return PTR_ERR(page);
3805                         kaddr = (unsigned char *)page_address(page);
3806                         offset = 0;
3807                 }
3808         }
3809         f2fs_put_page(page, 1);
3810         return 0;
3811 }
3812
3813 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3814 {
3815         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3816         struct f2fs_summary_block *sum;
3817         struct curseg_info *curseg;
3818         struct page *new;
3819         unsigned short blk_off;
3820         unsigned int segno = 0;
3821         block_t blk_addr = 0;
3822         int err = 0;
3823
3824         /* get segment number and block addr */
3825         if (IS_DATASEG(type)) {
3826                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3827                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3828                                                         CURSEG_HOT_DATA]);
3829                 if (__exist_node_summaries(sbi))
3830                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3831                 else
3832                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3833         } else {
3834                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3835                                                         CURSEG_HOT_NODE]);
3836                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3837                                                         CURSEG_HOT_NODE]);
3838                 if (__exist_node_summaries(sbi))
3839                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3840                                                         type - CURSEG_HOT_NODE);
3841                 else
3842                         blk_addr = GET_SUM_BLOCK(sbi, segno);
3843         }
3844
3845         new = f2fs_get_meta_page(sbi, blk_addr);
3846         if (IS_ERR(new))
3847                 return PTR_ERR(new);
3848         sum = (struct f2fs_summary_block *)page_address(new);
3849
3850         if (IS_NODESEG(type)) {
3851                 if (__exist_node_summaries(sbi)) {
3852                         struct f2fs_summary *ns = &sum->entries[0];
3853                         int i;
3854
3855                         for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3856                                 ns->version = 0;
3857                                 ns->ofs_in_node = 0;
3858                         }
3859                 } else {
3860                         err = f2fs_restore_node_summary(sbi, segno, sum);
3861                         if (err)
3862                                 goto out;
3863                 }
3864         }
3865
3866         /* set uncompleted segment to curseg */
3867         curseg = CURSEG_I(sbi, type);
3868         mutex_lock(&curseg->curseg_mutex);
3869
3870         /* update journal info */
3871         down_write(&curseg->journal_rwsem);
3872         memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3873         up_write(&curseg->journal_rwsem);
3874
3875         memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3876         memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3877         curseg->next_segno = segno;
3878         reset_curseg(sbi, type, 0);
3879         curseg->alloc_type = ckpt->alloc_type[type];
3880         curseg->next_blkoff = blk_off;
3881         mutex_unlock(&curseg->curseg_mutex);
3882 out:
3883         f2fs_put_page(new, 1);
3884         return err;
3885 }
3886
3887 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3888 {
3889         struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3890         struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3891         int type = CURSEG_HOT_DATA;
3892         int err;
3893
3894         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3895                 int npages = f2fs_npages_for_summary_flush(sbi, true);
3896
3897                 if (npages >= 2)
3898                         f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3899                                                         META_CP, true);
3900
3901                 /* restore for compacted data summary */
3902                 err = read_compacted_summaries(sbi);
3903                 if (err)
3904                         return err;
3905                 type = CURSEG_HOT_NODE;
3906         }
3907
3908         if (__exist_node_summaries(sbi))
3909                 f2fs_ra_meta_pages(sbi,
3910                                 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
3911                                 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
3912
3913         for (; type <= CURSEG_COLD_NODE; type++) {
3914                 err = read_normal_summaries(sbi, type);
3915                 if (err)
3916                         return err;
3917         }
3918
3919         /* sanity check for summary blocks */
3920         if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3921                         sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3922                 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
3923                          nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3924                 return -EINVAL;
3925         }
3926
3927         return 0;
3928 }
3929
3930 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3931 {
3932         struct page *page;
3933         unsigned char *kaddr;
3934         struct f2fs_summary *summary;
3935         struct curseg_info *seg_i;
3936         int written_size = 0;
3937         int i, j;
3938
3939         page = f2fs_grab_meta_page(sbi, blkaddr++);
3940         kaddr = (unsigned char *)page_address(page);
3941         memset(kaddr, 0, PAGE_SIZE);
3942
3943         /* Step 1: write nat cache */
3944         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3945         memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3946         written_size += SUM_JOURNAL_SIZE;
3947
3948         /* Step 2: write sit cache */
3949         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3950         memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3951         written_size += SUM_JOURNAL_SIZE;
3952
3953         /* Step 3: write summary entries */
3954         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3955                 seg_i = CURSEG_I(sbi, i);
3956                 for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) {
3957                         if (!page) {
3958                                 page = f2fs_grab_meta_page(sbi, blkaddr++);
3959                                 kaddr = (unsigned char *)page_address(page);
3960                                 memset(kaddr, 0, PAGE_SIZE);
3961                                 written_size = 0;
3962                         }
3963                         summary = (struct f2fs_summary *)(kaddr + written_size);
3964                         *summary = seg_i->sum_blk->entries[j];
3965                         written_size += SUMMARY_SIZE;
3966
3967                         if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
3968                                                         SUM_FOOTER_SIZE)
3969                                 continue;
3970
3971                         set_page_dirty(page);
3972                         f2fs_put_page(page, 1);
3973                         page = NULL;
3974                 }
3975         }
3976         if (page) {
3977                 set_page_dirty(page);
3978                 f2fs_put_page(page, 1);
3979         }
3980 }
3981
3982 static void write_normal_summaries(struct f2fs_sb_info *sbi,
3983                                         block_t blkaddr, int type)
3984 {
3985         int i, end;
3986
3987         if (IS_DATASEG(type))
3988                 end = type + NR_CURSEG_DATA_TYPE;
3989         else
3990                 end = type + NR_CURSEG_NODE_TYPE;
3991
3992         for (i = type; i < end; i++)
3993                 write_current_sum_page(sbi, i, blkaddr + (i - type));
3994 }
3995
3996 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3997 {
3998         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
3999                 write_compacted_summaries(sbi, start_blk);
4000         else
4001                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4002 }
4003
4004 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4005 {
4006         write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4007 }
4008
4009 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4010                                         unsigned int val, int alloc)
4011 {
4012         int i;
4013
4014         if (type == NAT_JOURNAL) {
4015                 for (i = 0; i < nats_in_cursum(journal); i++) {
4016                         if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4017                                 return i;
4018                 }
4019                 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4020                         return update_nats_in_cursum(journal, 1);
4021         } else if (type == SIT_JOURNAL) {
4022                 for (i = 0; i < sits_in_cursum(journal); i++)
4023                         if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4024                                 return i;
4025                 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4026                         return update_sits_in_cursum(journal, 1);
4027         }
4028         return -1;
4029 }
4030
4031 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4032                                         unsigned int segno)
4033 {
4034         return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4035 }
4036
4037 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4038                                         unsigned int start)
4039 {
4040         struct sit_info *sit_i = SIT_I(sbi);
4041         struct page *page;
4042         pgoff_t src_off, dst_off;
4043
4044         src_off = current_sit_addr(sbi, start);
4045         dst_off = next_sit_addr(sbi, src_off);
4046
4047         page = f2fs_grab_meta_page(sbi, dst_off);
4048         seg_info_to_sit_page(sbi, page, start);
4049
4050         set_page_dirty(page);
4051         set_to_next_sit(sit_i, start);
4052
4053         return page;
4054 }
4055
4056 static struct sit_entry_set *grab_sit_entry_set(void)
4057 {
4058         struct sit_entry_set *ses =
4059                         f2fs_kmem_cache_alloc(sit_entry_set_slab,
4060                                                 GFP_NOFS, true, NULL);
4061
4062         ses->entry_cnt = 0;
4063         INIT_LIST_HEAD(&ses->set_list);
4064         return ses;
4065 }
4066
4067 static void release_sit_entry_set(struct sit_entry_set *ses)
4068 {
4069         list_del(&ses->set_list);
4070         kmem_cache_free(sit_entry_set_slab, ses);
4071 }
4072
4073 static void adjust_sit_entry_set(struct sit_entry_set *ses,
4074                                                 struct list_head *head)
4075 {
4076         struct sit_entry_set *next = ses;
4077
4078         if (list_is_last(&ses->set_list, head))
4079                 return;
4080
4081         list_for_each_entry_continue(next, head, set_list)
4082                 if (ses->entry_cnt <= next->entry_cnt) {
4083                         list_move_tail(&ses->set_list, &next->set_list);
4084                         return;
4085                 }
4086
4087         list_move_tail(&ses->set_list, head);
4088 }
4089
4090 static void add_sit_entry(unsigned int segno, struct list_head *head)
4091 {
4092         struct sit_entry_set *ses;
4093         unsigned int start_segno = START_SEGNO(segno);
4094
4095         list_for_each_entry(ses, head, set_list) {
4096                 if (ses->start_segno == start_segno) {
4097                         ses->entry_cnt++;
4098                         adjust_sit_entry_set(ses, head);
4099                         return;
4100                 }
4101         }
4102
4103         ses = grab_sit_entry_set();
4104
4105         ses->start_segno = start_segno;
4106         ses->entry_cnt++;
4107         list_add(&ses->set_list, head);
4108 }
4109
4110 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4111 {
4112         struct f2fs_sm_info *sm_info = SM_I(sbi);
4113         struct list_head *set_list = &sm_info->sit_entry_set;
4114         unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4115         unsigned int segno;
4116
4117         for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4118                 add_sit_entry(segno, set_list);
4119 }
4120
4121 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4122 {
4123         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4124         struct f2fs_journal *journal = curseg->journal;
4125         int i;
4126
4127         down_write(&curseg->journal_rwsem);
4128         for (i = 0; i < sits_in_cursum(journal); i++) {
4129                 unsigned int segno;
4130                 bool dirtied;
4131
4132                 segno = le32_to_cpu(segno_in_journal(journal, i));
4133                 dirtied = __mark_sit_entry_dirty(sbi, segno);
4134
4135                 if (!dirtied)
4136                         add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4137         }
4138         update_sits_in_cursum(journal, -i);
4139         up_write(&curseg->journal_rwsem);
4140 }
4141
4142 /*
4143  * CP calls this function, which flushes SIT entries including sit_journal,
4144  * and moves prefree segs to free segs.
4145  */
4146 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4147 {
4148         struct sit_info *sit_i = SIT_I(sbi);
4149         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4150         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4151         struct f2fs_journal *journal = curseg->journal;
4152         struct sit_entry_set *ses, *tmp;
4153         struct list_head *head = &SM_I(sbi)->sit_entry_set;
4154         bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4155         struct seg_entry *se;
4156
4157         down_write(&sit_i->sentry_lock);
4158
4159         if (!sit_i->dirty_sentries)
4160                 goto out;
4161
4162         /*
4163          * add and account sit entries of dirty bitmap in sit entry
4164          * set temporarily
4165          */
4166         add_sits_in_set(sbi);
4167
4168         /*
4169          * if there are no enough space in journal to store dirty sit
4170          * entries, remove all entries from journal and add and account
4171          * them in sit entry set.
4172          */
4173         if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4174                                                                 !to_journal)
4175                 remove_sits_in_journal(sbi);
4176
4177         /*
4178          * there are two steps to flush sit entries:
4179          * #1, flush sit entries to journal in current cold data summary block.
4180          * #2, flush sit entries to sit page.
4181          */
4182         list_for_each_entry_safe(ses, tmp, head, set_list) {
4183                 struct page *page = NULL;
4184                 struct f2fs_sit_block *raw_sit = NULL;
4185                 unsigned int start_segno = ses->start_segno;
4186                 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4187                                                 (unsigned long)MAIN_SEGS(sbi));
4188                 unsigned int segno = start_segno;
4189
4190                 if (to_journal &&
4191                         !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4192                         to_journal = false;
4193
4194                 if (to_journal) {
4195                         down_write(&curseg->journal_rwsem);
4196                 } else {
4197                         page = get_next_sit_page(sbi, start_segno);
4198                         raw_sit = page_address(page);
4199                 }
4200
4201                 /* flush dirty sit entries in region of current sit set */
4202                 for_each_set_bit_from(segno, bitmap, end) {
4203                         int offset, sit_offset;
4204
4205                         se = get_seg_entry(sbi, segno);
4206 #ifdef CONFIG_F2FS_CHECK_FS
4207                         if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4208                                                 SIT_VBLOCK_MAP_SIZE))
4209                                 f2fs_bug_on(sbi, 1);
4210 #endif
4211
4212                         /* add discard candidates */
4213                         if (!(cpc->reason & CP_DISCARD)) {
4214                                 cpc->trim_start = segno;
4215                                 add_discard_addrs(sbi, cpc, false);
4216                         }
4217
4218                         if (to_journal) {
4219                                 offset = f2fs_lookup_journal_in_cursum(journal,
4220                                                         SIT_JOURNAL, segno, 1);
4221                                 f2fs_bug_on(sbi, offset < 0);
4222                                 segno_in_journal(journal, offset) =
4223                                                         cpu_to_le32(segno);
4224                                 seg_info_to_raw_sit(se,
4225                                         &sit_in_journal(journal, offset));
4226                                 check_block_count(sbi, segno,
4227                                         &sit_in_journal(journal, offset));
4228                         } else {
4229                                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4230                                 seg_info_to_raw_sit(se,
4231                                                 &raw_sit->entries[sit_offset]);
4232                                 check_block_count(sbi, segno,
4233                                                 &raw_sit->entries[sit_offset]);
4234                         }
4235
4236                         __clear_bit(segno, bitmap);
4237                         sit_i->dirty_sentries--;
4238                         ses->entry_cnt--;
4239                 }
4240
4241                 if (to_journal)
4242                         up_write(&curseg->journal_rwsem);
4243                 else
4244                         f2fs_put_page(page, 1);
4245
4246                 f2fs_bug_on(sbi, ses->entry_cnt);
4247                 release_sit_entry_set(ses);
4248         }
4249
4250         f2fs_bug_on(sbi, !list_empty(head));
4251         f2fs_bug_on(sbi, sit_i->dirty_sentries);
4252 out:
4253         if (cpc->reason & CP_DISCARD) {
4254                 __u64 trim_start = cpc->trim_start;
4255
4256                 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4257                         add_discard_addrs(sbi, cpc, false);
4258
4259                 cpc->trim_start = trim_start;
4260         }
4261         up_write(&sit_i->sentry_lock);
4262
4263         set_prefree_as_free_segments(sbi);
4264 }
4265
4266 static int build_sit_info(struct f2fs_sb_info *sbi)
4267 {
4268         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4269         struct sit_info *sit_i;
4270         unsigned int sit_segs, start;
4271         char *src_bitmap, *bitmap;
4272         unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4273         unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4274
4275         /* allocate memory for SIT information */
4276         sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4277         if (!sit_i)
4278                 return -ENOMEM;
4279
4280         SM_I(sbi)->sit_info = sit_i;
4281
4282         sit_i->sentries =
4283                 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4284                                               MAIN_SEGS(sbi)),
4285                               GFP_KERNEL);
4286         if (!sit_i->sentries)
4287                 return -ENOMEM;
4288
4289         main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4290         sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4291                                                                 GFP_KERNEL);
4292         if (!sit_i->dirty_sentries_bitmap)
4293                 return -ENOMEM;
4294
4295 #ifdef CONFIG_F2FS_CHECK_FS
4296         bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4297 #else
4298         bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4299 #endif
4300         sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4301         if (!sit_i->bitmap)
4302                 return -ENOMEM;
4303
4304         bitmap = sit_i->bitmap;
4305
4306         for (start = 0; start < MAIN_SEGS(sbi); start++) {
4307                 sit_i->sentries[start].cur_valid_map = bitmap;
4308                 bitmap += SIT_VBLOCK_MAP_SIZE;
4309
4310                 sit_i->sentries[start].ckpt_valid_map = bitmap;
4311                 bitmap += SIT_VBLOCK_MAP_SIZE;
4312
4313 #ifdef CONFIG_F2FS_CHECK_FS
4314                 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4315                 bitmap += SIT_VBLOCK_MAP_SIZE;
4316 #endif
4317
4318                 if (discard_map) {
4319                         sit_i->sentries[start].discard_map = bitmap;
4320                         bitmap += SIT_VBLOCK_MAP_SIZE;
4321                 }
4322         }
4323
4324         sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4325         if (!sit_i->tmp_map)
4326                 return -ENOMEM;
4327
4328         if (__is_large_section(sbi)) {
4329                 sit_i->sec_entries =
4330                         f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4331                                                       MAIN_SECS(sbi)),
4332                                       GFP_KERNEL);
4333                 if (!sit_i->sec_entries)
4334                         return -ENOMEM;
4335         }
4336
4337         /* get information related with SIT */
4338         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4339
4340         /* setup SIT bitmap from ckeckpoint pack */
4341         sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4342         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4343
4344         sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4345         if (!sit_i->sit_bitmap)
4346                 return -ENOMEM;
4347
4348 #ifdef CONFIG_F2FS_CHECK_FS
4349         sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4350                                         sit_bitmap_size, GFP_KERNEL);
4351         if (!sit_i->sit_bitmap_mir)
4352                 return -ENOMEM;
4353
4354         sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4355                                         main_bitmap_size, GFP_KERNEL);
4356         if (!sit_i->invalid_segmap)
4357                 return -ENOMEM;
4358 #endif
4359
4360         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4361         sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4362         sit_i->written_valid_blocks = 0;
4363         sit_i->bitmap_size = sit_bitmap_size;
4364         sit_i->dirty_sentries = 0;
4365         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4366         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4367         sit_i->mounted_time = ktime_get_boottime_seconds();
4368         init_rwsem(&sit_i->sentry_lock);
4369         return 0;
4370 }
4371
4372 static int build_free_segmap(struct f2fs_sb_info *sbi)
4373 {
4374         struct free_segmap_info *free_i;
4375         unsigned int bitmap_size, sec_bitmap_size;
4376
4377         /* allocate memory for free segmap information */
4378         free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4379         if (!free_i)
4380                 return -ENOMEM;
4381
4382         SM_I(sbi)->free_info = free_i;
4383
4384         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4385         free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4386         if (!free_i->free_segmap)
4387                 return -ENOMEM;
4388
4389         sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4390         free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4391         if (!free_i->free_secmap)
4392                 return -ENOMEM;
4393
4394         /* set all segments as dirty temporarily */
4395         memset(free_i->free_segmap, 0xff, bitmap_size);
4396         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4397
4398         /* init free segmap information */
4399         free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4400         free_i->free_segments = 0;
4401         free_i->free_sections = 0;
4402         spin_lock_init(&free_i->segmap_lock);
4403         return 0;
4404 }
4405
4406 static int build_curseg(struct f2fs_sb_info *sbi)
4407 {
4408         struct curseg_info *array;
4409         int i;
4410
4411         array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4412                                         sizeof(*array)), GFP_KERNEL);
4413         if (!array)
4414                 return -ENOMEM;
4415
4416         SM_I(sbi)->curseg_array = array;
4417
4418         for (i = 0; i < NO_CHECK_TYPE; i++) {
4419                 mutex_init(&array[i].curseg_mutex);
4420                 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4421                 if (!array[i].sum_blk)
4422                         return -ENOMEM;
4423                 init_rwsem(&array[i].journal_rwsem);
4424                 array[i].journal = f2fs_kzalloc(sbi,
4425                                 sizeof(struct f2fs_journal), GFP_KERNEL);
4426                 if (!array[i].journal)
4427                         return -ENOMEM;
4428                 if (i < NR_PERSISTENT_LOG)
4429                         array[i].seg_type = CURSEG_HOT_DATA + i;
4430                 else if (i == CURSEG_COLD_DATA_PINNED)
4431                         array[i].seg_type = CURSEG_COLD_DATA;
4432                 else if (i == CURSEG_ALL_DATA_ATGC)
4433                         array[i].seg_type = CURSEG_COLD_DATA;
4434                 array[i].segno = NULL_SEGNO;
4435                 array[i].next_blkoff = 0;
4436                 array[i].inited = false;
4437         }
4438         return restore_curseg_summaries(sbi);
4439 }
4440
4441 static int build_sit_entries(struct f2fs_sb_info *sbi)
4442 {
4443         struct sit_info *sit_i = SIT_I(sbi);
4444         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4445         struct f2fs_journal *journal = curseg->journal;
4446         struct seg_entry *se;
4447         struct f2fs_sit_entry sit;
4448         int sit_blk_cnt = SIT_BLK_CNT(sbi);
4449         unsigned int i, start, end;
4450         unsigned int readed, start_blk = 0;
4451         int err = 0;
4452         block_t sit_valid_blocks[2] = {0, 0};
4453
4454         do {
4455                 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4456                                                         META_SIT, true);
4457
4458                 start = start_blk * sit_i->sents_per_block;
4459                 end = (start_blk + readed) * sit_i->sents_per_block;
4460
4461                 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4462                         struct f2fs_sit_block *sit_blk;
4463                         struct page *page;
4464
4465                         se = &sit_i->sentries[start];
4466                         page = get_current_sit_page(sbi, start);
4467                         if (IS_ERR(page))
4468                                 return PTR_ERR(page);
4469                         sit_blk = (struct f2fs_sit_block *)page_address(page);
4470                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4471                         f2fs_put_page(page, 1);
4472
4473                         err = check_block_count(sbi, start, &sit);
4474                         if (err)
4475                                 return err;
4476                         seg_info_from_raw_sit(se, &sit);
4477
4478                         if (se->type >= NR_PERSISTENT_LOG) {
4479                                 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4480                                                         se->type, start);
4481                                 f2fs_handle_error(sbi,
4482                                                 ERROR_INCONSISTENT_SUM_TYPE);
4483                                 return -EFSCORRUPTED;
4484                         }
4485
4486                         sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4487
4488                         if (f2fs_block_unit_discard(sbi)) {
4489                                 /* build discard map only one time */
4490                                 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4491                                         memset(se->discard_map, 0xff,
4492                                                 SIT_VBLOCK_MAP_SIZE);
4493                                 } else {
4494                                         memcpy(se->discard_map,
4495                                                 se->cur_valid_map,
4496                                                 SIT_VBLOCK_MAP_SIZE);
4497                                         sbi->discard_blks +=
4498                                                 sbi->blocks_per_seg -
4499                                                 se->valid_blocks;
4500                                 }
4501                         }
4502
4503                         if (__is_large_section(sbi))
4504                                 get_sec_entry(sbi, start)->valid_blocks +=
4505                                                         se->valid_blocks;
4506                 }
4507                 start_blk += readed;
4508         } while (start_blk < sit_blk_cnt);
4509
4510         down_read(&curseg->journal_rwsem);
4511         for (i = 0; i < sits_in_cursum(journal); i++) {
4512                 unsigned int old_valid_blocks;
4513
4514                 start = le32_to_cpu(segno_in_journal(journal, i));
4515                 if (start >= MAIN_SEGS(sbi)) {
4516                         f2fs_err(sbi, "Wrong journal entry on segno %u",
4517                                  start);
4518                         err = -EFSCORRUPTED;
4519                         f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);
4520                         break;
4521                 }
4522
4523                 se = &sit_i->sentries[start];
4524                 sit = sit_in_journal(journal, i);
4525
4526                 old_valid_blocks = se->valid_blocks;
4527
4528                 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4529
4530                 err = check_block_count(sbi, start, &sit);
4531                 if (err)
4532                         break;
4533                 seg_info_from_raw_sit(se, &sit);
4534
4535                 if (se->type >= NR_PERSISTENT_LOG) {
4536                         f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4537                                                         se->type, start);
4538                         err = -EFSCORRUPTED;
4539                         f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
4540                         break;
4541                 }
4542
4543                 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4544
4545                 if (f2fs_block_unit_discard(sbi)) {
4546                         if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4547                                 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4548                         } else {
4549                                 memcpy(se->discard_map, se->cur_valid_map,
4550                                                         SIT_VBLOCK_MAP_SIZE);
4551                                 sbi->discard_blks += old_valid_blocks;
4552                                 sbi->discard_blks -= se->valid_blocks;
4553                         }
4554                 }
4555
4556                 if (__is_large_section(sbi)) {
4557                         get_sec_entry(sbi, start)->valid_blocks +=
4558                                                         se->valid_blocks;
4559                         get_sec_entry(sbi, start)->valid_blocks -=
4560                                                         old_valid_blocks;
4561                 }
4562         }
4563         up_read(&curseg->journal_rwsem);
4564
4565         if (err)
4566                 return err;
4567
4568         if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
4569                 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4570                          sit_valid_blocks[NODE], valid_node_count(sbi));
4571                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);
4572                 return -EFSCORRUPTED;
4573         }
4574
4575         if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
4576                                 valid_user_blocks(sbi)) {
4577                 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
4578                          sit_valid_blocks[DATA], sit_valid_blocks[NODE],
4579                          valid_user_blocks(sbi));
4580                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);
4581                 return -EFSCORRUPTED;
4582         }
4583
4584         return 0;
4585 }
4586
4587 static void init_free_segmap(struct f2fs_sb_info *sbi)
4588 {
4589         unsigned int start;
4590         int type;
4591         struct seg_entry *sentry;
4592
4593         for (start = 0; start < MAIN_SEGS(sbi); start++) {
4594                 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4595                         continue;
4596                 sentry = get_seg_entry(sbi, start);
4597                 if (!sentry->valid_blocks)
4598                         __set_free(sbi, start);
4599                 else
4600                         SIT_I(sbi)->written_valid_blocks +=
4601                                                 sentry->valid_blocks;
4602         }
4603
4604         /* set use the current segments */
4605         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4606                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4607
4608                 __set_test_and_inuse(sbi, curseg_t->segno);
4609         }
4610 }
4611
4612 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4613 {
4614         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4615         struct free_segmap_info *free_i = FREE_I(sbi);
4616         unsigned int segno = 0, offset = 0, secno;
4617         block_t valid_blocks, usable_blks_in_seg;
4618
4619         while (1) {
4620                 /* find dirty segment based on free segmap */
4621                 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4622                 if (segno >= MAIN_SEGS(sbi))
4623                         break;
4624                 offset = segno + 1;
4625                 valid_blocks = get_valid_blocks(sbi, segno, false);
4626                 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4627                 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4628                         continue;
4629                 if (valid_blocks > usable_blks_in_seg) {
4630                         f2fs_bug_on(sbi, 1);
4631                         continue;
4632                 }
4633                 mutex_lock(&dirty_i->seglist_lock);
4634                 __locate_dirty_segment(sbi, segno, DIRTY);
4635                 mutex_unlock(&dirty_i->seglist_lock);
4636         }
4637
4638         if (!__is_large_section(sbi))
4639                 return;
4640
4641         mutex_lock(&dirty_i->seglist_lock);
4642         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4643                 valid_blocks = get_valid_blocks(sbi, segno, true);
4644                 secno = GET_SEC_FROM_SEG(sbi, segno);
4645
4646                 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))
4647                         continue;
4648                 if (IS_CURSEC(sbi, secno))
4649                         continue;
4650                 set_bit(secno, dirty_i->dirty_secmap);
4651         }
4652         mutex_unlock(&dirty_i->seglist_lock);
4653 }
4654
4655 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4656 {
4657         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4658         unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4659
4660         dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4661         if (!dirty_i->victim_secmap)
4662                 return -ENOMEM;
4663
4664         dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4665         if (!dirty_i->pinned_secmap)
4666                 return -ENOMEM;
4667
4668         dirty_i->pinned_secmap_cnt = 0;
4669         dirty_i->enable_pin_section = true;
4670         return 0;
4671 }
4672
4673 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4674 {
4675         struct dirty_seglist_info *dirty_i;
4676         unsigned int bitmap_size, i;
4677
4678         /* allocate memory for dirty segments list information */
4679         dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4680                                                                 GFP_KERNEL);
4681         if (!dirty_i)
4682                 return -ENOMEM;
4683
4684         SM_I(sbi)->dirty_info = dirty_i;
4685         mutex_init(&dirty_i->seglist_lock);
4686
4687         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4688
4689         for (i = 0; i < NR_DIRTY_TYPE; i++) {
4690                 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4691                                                                 GFP_KERNEL);
4692                 if (!dirty_i->dirty_segmap[i])
4693                         return -ENOMEM;
4694         }
4695
4696         if (__is_large_section(sbi)) {
4697                 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4698                 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4699                                                 bitmap_size, GFP_KERNEL);
4700                 if (!dirty_i->dirty_secmap)
4701                         return -ENOMEM;
4702         }
4703
4704         init_dirty_segmap(sbi);
4705         return init_victim_secmap(sbi);
4706 }
4707
4708 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4709 {
4710         int i;
4711
4712         /*
4713          * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4714          * In LFS curseg, all blkaddr after .next_blkoff should be unused.
4715          */
4716         for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4717                 struct curseg_info *curseg = CURSEG_I(sbi, i);
4718                 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4719                 unsigned int blkofs = curseg->next_blkoff;
4720
4721                 if (f2fs_sb_has_readonly(sbi) &&
4722                         i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4723                         continue;
4724
4725                 sanity_check_seg_type(sbi, curseg->seg_type);
4726
4727                 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
4728                         f2fs_err(sbi,
4729                                  "Current segment has invalid alloc_type:%d",
4730                                  curseg->alloc_type);
4731                         f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
4732                         return -EFSCORRUPTED;
4733                 }
4734
4735                 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4736                         goto out;
4737
4738                 if (curseg->alloc_type == SSR)
4739                         continue;
4740
4741                 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4742                         if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4743                                 continue;
4744 out:
4745                         f2fs_err(sbi,
4746                                  "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4747                                  i, curseg->segno, curseg->alloc_type,
4748                                  curseg->next_blkoff, blkofs);
4749                         f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
4750                         return -EFSCORRUPTED;
4751                 }
4752         }
4753         return 0;
4754 }
4755
4756 #ifdef CONFIG_BLK_DEV_ZONED
4757
4758 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4759                                     struct f2fs_dev_info *fdev,
4760                                     struct blk_zone *zone)
4761 {
4762         unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4763         block_t zone_block, wp_block, last_valid_block;
4764         unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4765         int i, s, b, ret;
4766         struct seg_entry *se;
4767
4768         if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4769                 return 0;
4770
4771         wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4772         wp_segno = GET_SEGNO(sbi, wp_block);
4773         wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4774         zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4775         zone_segno = GET_SEGNO(sbi, zone_block);
4776         zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4777
4778         if (zone_segno >= MAIN_SEGS(sbi))
4779                 return 0;
4780
4781         /*
4782          * Skip check of zones cursegs point to, since
4783          * fix_curseg_write_pointer() checks them.
4784          */
4785         for (i = 0; i < NO_CHECK_TYPE; i++)
4786                 if (zone_secno == GET_SEC_FROM_SEG(sbi,
4787                                                    CURSEG_I(sbi, i)->segno))
4788                         return 0;
4789
4790         /*
4791          * Get last valid block of the zone.
4792          */
4793         last_valid_block = zone_block - 1;
4794         for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4795                 segno = zone_segno + s;
4796                 se = get_seg_entry(sbi, segno);
4797                 for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4798                         if (f2fs_test_bit(b, se->cur_valid_map)) {
4799                                 last_valid_block = START_BLOCK(sbi, segno) + b;
4800                                 break;
4801                         }
4802                 if (last_valid_block >= zone_block)
4803                         break;
4804         }
4805
4806         /*
4807          * If last valid block is beyond the write pointer, report the
4808          * inconsistency. This inconsistency does not cause write error
4809          * because the zone will not be selected for write operation until
4810          * it get discarded. Just report it.
4811          */
4812         if (last_valid_block >= wp_block) {
4813                 f2fs_notice(sbi, "Valid block beyond write pointer: "
4814                             "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4815                             GET_SEGNO(sbi, last_valid_block),
4816                             GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4817                             wp_segno, wp_blkoff);
4818                 return 0;
4819         }
4820
4821         /*
4822          * If there is no valid block in the zone and if write pointer is
4823          * not at zone start, reset the write pointer.
4824          */
4825         if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4826                 f2fs_notice(sbi,
4827                             "Zone without valid block has non-zero write "
4828                             "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4829                             wp_segno, wp_blkoff);
4830                 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4831                                         zone->len >> log_sectors_per_block);
4832                 if (ret) {
4833                         f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4834                                  fdev->path, ret);
4835                         return ret;
4836                 }
4837         }
4838
4839         return 0;
4840 }
4841
4842 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4843                                                   block_t zone_blkaddr)
4844 {
4845         int i;
4846
4847         for (i = 0; i < sbi->s_ndevs; i++) {
4848                 if (!bdev_is_zoned(FDEV(i).bdev))
4849                         continue;
4850                 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4851                                 zone_blkaddr <= FDEV(i).end_blk))
4852                         return &FDEV(i);
4853         }
4854
4855         return NULL;
4856 }
4857
4858 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4859                               void *data)
4860 {
4861         memcpy(data, zone, sizeof(struct blk_zone));
4862         return 0;
4863 }
4864
4865 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4866 {
4867         struct curseg_info *cs = CURSEG_I(sbi, type);
4868         struct f2fs_dev_info *zbd;
4869         struct blk_zone zone;
4870         unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4871         block_t cs_zone_block, wp_block;
4872         unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4873         sector_t zone_sector;
4874         int err;
4875
4876         cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4877         cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4878
4879         zbd = get_target_zoned_dev(sbi, cs_zone_block);
4880         if (!zbd)
4881                 return 0;
4882
4883         /* report zone for the sector the curseg points to */
4884         zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4885                 << log_sectors_per_block;
4886         err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4887                                   report_one_zone_cb, &zone);
4888         if (err != 1) {
4889                 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4890                          zbd->path, err);
4891                 return err;
4892         }
4893
4894         if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4895                 return 0;
4896
4897         wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
4898         wp_segno = GET_SEGNO(sbi, wp_block);
4899         wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4900         wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
4901
4902         if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
4903                 wp_sector_off == 0)
4904                 return 0;
4905
4906         f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
4907                     "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
4908                     type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
4909
4910         f2fs_notice(sbi, "Assign new section to curseg[%d]: "
4911                     "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
4912
4913         f2fs_allocate_new_section(sbi, type, true);
4914
4915         /* check consistency of the zone curseg pointed to */
4916         if (check_zone_write_pointer(sbi, zbd, &zone))
4917                 return -EIO;
4918
4919         /* check newly assigned zone */
4920         cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4921         cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4922
4923         zbd = get_target_zoned_dev(sbi, cs_zone_block);
4924         if (!zbd)
4925                 return 0;
4926
4927         zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4928                 << log_sectors_per_block;
4929         err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4930                                   report_one_zone_cb, &zone);
4931         if (err != 1) {
4932                 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4933                          zbd->path, err);
4934                 return err;
4935         }
4936
4937         if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4938                 return 0;
4939
4940         if (zone.wp != zone.start) {
4941                 f2fs_notice(sbi,
4942                             "New zone for curseg[%d] is not yet discarded. "
4943                             "Reset the zone: curseg[0x%x,0x%x]",
4944                             type, cs->segno, cs->next_blkoff);
4945                 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, cs_zone_block,
4946                                         zone.len >> log_sectors_per_block);
4947                 if (err) {
4948                         f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4949                                  zbd->path, err);
4950                         return err;
4951                 }
4952         }
4953
4954         return 0;
4955 }
4956
4957 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4958 {
4959         int i, ret;
4960
4961         for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4962                 ret = fix_curseg_write_pointer(sbi, i);
4963                 if (ret)
4964                         return ret;
4965         }
4966
4967         return 0;
4968 }
4969
4970 struct check_zone_write_pointer_args {
4971         struct f2fs_sb_info *sbi;
4972         struct f2fs_dev_info *fdev;
4973 };
4974
4975 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
4976                                       void *data)
4977 {
4978         struct check_zone_write_pointer_args *args;
4979
4980         args = (struct check_zone_write_pointer_args *)data;
4981
4982         return check_zone_write_pointer(args->sbi, args->fdev, zone);
4983 }
4984
4985 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4986 {
4987         int i, ret;
4988         struct check_zone_write_pointer_args args;
4989
4990         for (i = 0; i < sbi->s_ndevs; i++) {
4991                 if (!bdev_is_zoned(FDEV(i).bdev))
4992                         continue;
4993
4994                 args.sbi = sbi;
4995                 args.fdev = &FDEV(i);
4996                 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
4997                                           check_zone_write_pointer_cb, &args);
4998                 if (ret < 0)
4999                         return ret;
5000         }
5001
5002         return 0;
5003 }
5004
5005 /*
5006  * Return the number of usable blocks in a segment. The number of blocks
5007  * returned is always equal to the number of blocks in a segment for
5008  * segments fully contained within a sequential zone capacity or a
5009  * conventional zone. For segments partially contained in a sequential
5010  * zone capacity, the number of usable blocks up to the zone capacity
5011  * is returned. 0 is returned in all other cases.
5012  */
5013 static inline unsigned int f2fs_usable_zone_blks_in_seg(
5014                         struct f2fs_sb_info *sbi, unsigned int segno)
5015 {
5016         block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5017         unsigned int secno;
5018
5019         if (!sbi->unusable_blocks_per_sec)
5020                 return sbi->blocks_per_seg;
5021
5022         secno = GET_SEC_FROM_SEG(sbi, segno);
5023         seg_start = START_BLOCK(sbi, segno);
5024         sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5025         sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
5026
5027         /*
5028          * If segment starts before zone capacity and spans beyond
5029          * zone capacity, then usable blocks are from seg start to
5030          * zone capacity. If the segment starts after the zone capacity,
5031          * then there are no usable blocks.
5032          */
5033         if (seg_start >= sec_cap_blkaddr)
5034                 return 0;
5035         if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
5036                 return sec_cap_blkaddr - seg_start;
5037
5038         return sbi->blocks_per_seg;
5039 }
5040 #else
5041 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5042 {
5043         return 0;
5044 }
5045
5046 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5047 {
5048         return 0;
5049 }
5050
5051 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5052                                                         unsigned int segno)
5053 {
5054         return 0;
5055 }
5056
5057 #endif
5058 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5059                                         unsigned int segno)
5060 {
5061         if (f2fs_sb_has_blkzoned(sbi))
5062                 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5063
5064         return sbi->blocks_per_seg;
5065 }
5066
5067 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5068                                         unsigned int segno)
5069 {
5070         if (f2fs_sb_has_blkzoned(sbi))
5071                 return CAP_SEGS_PER_SEC(sbi);
5072
5073         return sbi->segs_per_sec;
5074 }
5075
5076 /*
5077  * Update min, max modified time for cost-benefit GC algorithm
5078  */
5079 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5080 {
5081         struct sit_info *sit_i = SIT_I(sbi);
5082         unsigned int segno;
5083
5084         down_write(&sit_i->sentry_lock);
5085
5086         sit_i->min_mtime = ULLONG_MAX;
5087
5088         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5089                 unsigned int i;
5090                 unsigned long long mtime = 0;
5091
5092                 for (i = 0; i < sbi->segs_per_sec; i++)
5093                         mtime += get_seg_entry(sbi, segno + i)->mtime;
5094
5095                 mtime = div_u64(mtime, sbi->segs_per_sec);
5096
5097                 if (sit_i->min_mtime > mtime)
5098                         sit_i->min_mtime = mtime;
5099         }
5100         sit_i->max_mtime = get_mtime(sbi, false);
5101         sit_i->dirty_max_mtime = 0;
5102         up_write(&sit_i->sentry_lock);
5103 }
5104
5105 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5106 {
5107         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5108         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5109         struct f2fs_sm_info *sm_info;
5110         int err;
5111
5112         sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5113         if (!sm_info)
5114                 return -ENOMEM;
5115
5116         /* init sm info */
5117         sbi->sm_info = sm_info;
5118         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5119         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5120         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5121         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5122         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5123         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5124         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5125         sm_info->rec_prefree_segments = sm_info->main_segments *
5126                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5127         if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5128                 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5129
5130         if (!f2fs_lfs_mode(sbi))
5131                 sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC);
5132         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5133         sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5134         sm_info->min_seq_blocks = sbi->blocks_per_seg;
5135         sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5136         sm_info->min_ssr_sections = reserved_sections(sbi);
5137
5138         INIT_LIST_HEAD(&sm_info->sit_entry_set);
5139
5140         init_f2fs_rwsem(&sm_info->curseg_lock);
5141
5142         err = f2fs_create_flush_cmd_control(sbi);
5143         if (err)
5144                 return err;
5145
5146         err = create_discard_cmd_control(sbi);
5147         if (err)
5148                 return err;
5149
5150         err = build_sit_info(sbi);
5151         if (err)
5152                 return err;
5153         err = build_free_segmap(sbi);
5154         if (err)
5155                 return err;
5156         err = build_curseg(sbi);
5157         if (err)
5158                 return err;
5159
5160         /* reinit free segmap based on SIT */
5161         err = build_sit_entries(sbi);
5162         if (err)
5163                 return err;
5164
5165         init_free_segmap(sbi);
5166         err = build_dirty_segmap(sbi);
5167         if (err)
5168                 return err;
5169
5170         err = sanity_check_curseg(sbi);
5171         if (err)
5172                 return err;
5173
5174         init_min_max_mtime(sbi);
5175         return 0;
5176 }
5177
5178 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5179                 enum dirty_type dirty_type)
5180 {
5181         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5182
5183         mutex_lock(&dirty_i->seglist_lock);
5184         kvfree(dirty_i->dirty_segmap[dirty_type]);
5185         dirty_i->nr_dirty[dirty_type] = 0;
5186         mutex_unlock(&dirty_i->seglist_lock);
5187 }
5188
5189 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5190 {
5191         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5192
5193         kvfree(dirty_i->pinned_secmap);
5194         kvfree(dirty_i->victim_secmap);
5195 }
5196
5197 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5198 {
5199         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5200         int i;
5201
5202         if (!dirty_i)
5203                 return;
5204
5205         /* discard pre-free/dirty segments list */
5206         for (i = 0; i < NR_DIRTY_TYPE; i++)
5207                 discard_dirty_segmap(sbi, i);
5208
5209         if (__is_large_section(sbi)) {
5210                 mutex_lock(&dirty_i->seglist_lock);
5211                 kvfree(dirty_i->dirty_secmap);
5212                 mutex_unlock(&dirty_i->seglist_lock);
5213         }
5214
5215         destroy_victim_secmap(sbi);
5216         SM_I(sbi)->dirty_info = NULL;
5217         kfree(dirty_i);
5218 }
5219
5220 static void destroy_curseg(struct f2fs_sb_info *sbi)
5221 {
5222         struct curseg_info *array = SM_I(sbi)->curseg_array;
5223         int i;
5224
5225         if (!array)
5226                 return;
5227         SM_I(sbi)->curseg_array = NULL;
5228         for (i = 0; i < NR_CURSEG_TYPE; i++) {
5229                 kfree(array[i].sum_blk);
5230                 kfree(array[i].journal);
5231         }
5232         kfree(array);
5233 }
5234
5235 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5236 {
5237         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5238
5239         if (!free_i)
5240                 return;
5241         SM_I(sbi)->free_info = NULL;
5242         kvfree(free_i->free_segmap);
5243         kvfree(free_i->free_secmap);
5244         kfree(free_i);
5245 }
5246
5247 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5248 {
5249         struct sit_info *sit_i = SIT_I(sbi);
5250
5251         if (!sit_i)
5252                 return;
5253
5254         if (sit_i->sentries)
5255                 kvfree(sit_i->bitmap);
5256         kfree(sit_i->tmp_map);
5257
5258         kvfree(sit_i->sentries);
5259         kvfree(sit_i->sec_entries);
5260         kvfree(sit_i->dirty_sentries_bitmap);
5261
5262         SM_I(sbi)->sit_info = NULL;
5263         kvfree(sit_i->sit_bitmap);
5264 #ifdef CONFIG_F2FS_CHECK_FS
5265         kvfree(sit_i->sit_bitmap_mir);
5266         kvfree(sit_i->invalid_segmap);
5267 #endif
5268         kfree(sit_i);
5269 }
5270
5271 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5272 {
5273         struct f2fs_sm_info *sm_info = SM_I(sbi);
5274
5275         if (!sm_info)
5276                 return;
5277         f2fs_destroy_flush_cmd_control(sbi, true);
5278         destroy_discard_cmd_control(sbi);
5279         destroy_dirty_segmap(sbi);
5280         destroy_curseg(sbi);
5281         destroy_free_segmap(sbi);
5282         destroy_sit_info(sbi);
5283         sbi->sm_info = NULL;
5284         kfree(sm_info);
5285 }
5286
5287 int __init f2fs_create_segment_manager_caches(void)
5288 {
5289         discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5290                         sizeof(struct discard_entry));
5291         if (!discard_entry_slab)
5292                 goto fail;
5293
5294         discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5295                         sizeof(struct discard_cmd));
5296         if (!discard_cmd_slab)
5297                 goto destroy_discard_entry;
5298
5299         sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5300                         sizeof(struct sit_entry_set));
5301         if (!sit_entry_set_slab)
5302                 goto destroy_discard_cmd;
5303
5304         revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry",
5305                         sizeof(struct revoke_entry));
5306         if (!revoke_entry_slab)
5307                 goto destroy_sit_entry_set;
5308         return 0;
5309
5310 destroy_sit_entry_set:
5311         kmem_cache_destroy(sit_entry_set_slab);
5312 destroy_discard_cmd:
5313         kmem_cache_destroy(discard_cmd_slab);
5314 destroy_discard_entry:
5315         kmem_cache_destroy(discard_entry_slab);
5316 fail:
5317         return -ENOMEM;
5318 }
5319
5320 void f2fs_destroy_segment_manager_caches(void)
5321 {
5322         kmem_cache_destroy(sit_entry_set_slab);
5323         kmem_cache_destroy(discard_cmd_slab);
5324         kmem_cache_destroy(discard_entry_slab);
5325         kmem_cache_destroy(revoke_entry_slab);
5326 }