fs/f2fs/compress.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * f2fs compress support
   4  *
   5  * Copyright (c) 2019 Chao Yu <chao@kernel.org>
   6  */
   7
   8 #include <linux/fs.h>
   9 #include <linux/f2fs_fs.h>
  10 #include <linux/writeback.h>
  11 #include <linux/backing-dev.h>
  12 #include <linux/lzo.h>
  13 #include <linux/lz4.h>
  14 #include <linux/zstd.h>
  15
  16 #include "f2fs.h"
  17 #include "node.h"
  18 #include <trace/events/f2fs.h>
  19
  20 struct f2fs_compress_ops {
  21         int (*init_compress_ctx)(struct compress_ctx *cc);
  22         void (*destroy_compress_ctx)(struct compress_ctx *cc);
  23         int (*compress_pages)(struct compress_ctx *cc);
  24         int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
  25         void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
  26         int (*decompress_pages)(struct decompress_io_ctx *dic);
  27 };
  28
  29 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)
  30 {
  31         return index & (cc->cluster_size - 1);
  32 }
  33
  34 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index)
  35 {
  36         return index >> cc->log_cluster_size;
  37 }
  38
  39 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc)
  40 {
  41         return cc->cluster_idx << cc->log_cluster_size;
  42 }
  43
  44 bool f2fs_is_compressed_page(struct page *page)
  45 {
  46         if (!PagePrivate(page))
  47                 return false;
  48         if (!page_private(page))
  49                 return false;
  50         if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page))
  51                 return false;
  52         f2fs_bug_on(F2FS_M_SB(page->mapping),
  53                 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC);
  54         return true;
  55 }
  56
  57 static void f2fs_set_compressed_page(struct page *page,
  58                 struct inode *inode, pgoff_t index, void *data)
  59 {
  60         SetPagePrivate(page);
  61         set_page_private(page, (unsigned long)data);
  62
  63         /* i_crypto_info and iv index */
  64         page->index = index;
  65         page->mapping = inode->i_mapping;
  66 }
  67
  68 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)
  69 {
  70         int i;
  71
  72         for (i = 0; i < len; i++) {
  73                 if (!cc->rpages[i])
  74                         continue;
  75                 if (unlock)
  76                         unlock_page(cc->rpages[i]);
  77                 else
  78                         put_page(cc->rpages[i]);
  79         }
  80 }
  81
  82 static void f2fs_put_rpages(struct compress_ctx *cc)
  83 {
  84         f2fs_drop_rpages(cc, cc->cluster_size, false);
  85 }
  86
  87 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len)
  88 {
  89         f2fs_drop_rpages(cc, len, true);
  90 }
  91
  92 static void f2fs_put_rpages_mapping(struct address_space *mapping,
  93                                 pgoff_t start, int len)
  94 {
  95         int i;
  96
  97         for (i = 0; i < len; i++) {
  98                 struct page *page = find_get_page(mapping, start + i);
  99
 100                 put_page(page);
 101                 put_page(page);
 102         }
 103 }
 104
 105 static void f2fs_put_rpages_wbc(struct compress_ctx *cc,
 106                 struct writeback_control *wbc, bool redirty, int unlock)
 107 {
 108         unsigned int i;
 109
 110         for (i = 0; i < cc->cluster_size; i++) {
 111                 if (!cc->rpages[i])
 112                         continue;
 113                 if (redirty)
 114                         redirty_page_for_writepage(wbc, cc->rpages[i]);
 115                 f2fs_put_page(cc->rpages[i], unlock);
 116         }
 117 }
 118
 119 struct page *f2fs_compress_control_page(struct page *page)
 120 {
 121         return ((struct compress_io_ctx *)page_private(page))->rpages[0];
 122 }
 123
 124 int f2fs_init_compress_ctx(struct compress_ctx *cc)
 125 {
 126         struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
 127
 128         if (cc->nr_rpages)
 129                 return 0;
 130
 131         cc->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
 132                                         cc->log_cluster_size, GFP_NOFS);
 133         return cc->rpages ? 0 : -ENOMEM;
 134 }
 135
 136 void f2fs_destroy_compress_ctx(struct compress_ctx *cc)
 137 {
 138         kfree(cc->rpages);
 139         cc->rpages = NULL;
 140         cc->nr_rpages = 0;
 141         cc->nr_cpages = 0;
 142         cc->cluster_idx = NULL_CLUSTER;
 143 }
 144
 145 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)
 146 {
 147         unsigned int cluster_ofs;
 148
 149         if (!f2fs_cluster_can_merge_page(cc, page->index))
 150                 f2fs_bug_on(F2FS_I_SB(cc->inode), 1);
 151
 152         cluster_ofs = offset_in_cluster(cc, page->index);
 153         cc->rpages[cluster_ofs] = page;
 154         cc->nr_rpages++;
 155         cc->cluster_idx = cluster_idx(cc, page->index);
 156 }
 157
 158 #ifdef CONFIG_F2FS_FS_LZO
 159 static int lzo_init_compress_ctx(struct compress_ctx *cc)
 160 {
 161         cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
 162                                 LZO1X_MEM_COMPRESS, GFP_NOFS);
 163         if (!cc->private)
 164                 return -ENOMEM;
 165
 166         cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size);
 167         return 0;
 168 }
 169
 170 static void lzo_destroy_compress_ctx(struct compress_ctx *cc)
 171 {
 172         kvfree(cc->private);
 173         cc->private = NULL;
 174 }
 175
 176 static int lzo_compress_pages(struct compress_ctx *cc)
 177 {
 178         int ret;
 179
 180         ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
 181                                         &cc->clen, cc->private);
 182         if (ret != LZO_E_OK) {
 183                 printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n",
 184                                 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
 185                 return -EIO;
 186         }
 187         return 0;
 188 }
 189
 190 static int lzo_decompress_pages(struct decompress_io_ctx *dic)
 191 {
 192         int ret;
 193
 194         ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,
 195                                                 dic->rbuf, &dic->rlen);
 196         if (ret != LZO_E_OK) {
 197                 printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n",
 198                                 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
 199                 return -EIO;
 200         }
 201
 202         if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {
 203                 printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, "
 204                                         "expected:%lu\n", KERN_ERR,
 205                                         F2FS_I_SB(dic->inode)->sb->s_id,
 206                                         dic->rlen,
 207                                         PAGE_SIZE << dic->log_cluster_size);
 208                 return -EIO;
 209         }
 210         return 0;
 211 }
 212
 213 static const struct f2fs_compress_ops f2fs_lzo_ops = {
 214         .init_compress_ctx      = lzo_init_compress_ctx,
 215         .destroy_compress_ctx   = lzo_destroy_compress_ctx,
 216         .compress_pages         = lzo_compress_pages,
 217         .decompress_pages       = lzo_decompress_pages,
 218 };
 219 #endif
 220
 221 #ifdef CONFIG_F2FS_FS_LZ4
 222 static int lz4_init_compress_ctx(struct compress_ctx *cc)
 223 {
 224         cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
 225                                 LZ4_MEM_COMPRESS, GFP_NOFS);
 226         if (!cc->private)
 227                 return -ENOMEM;
 228
 229         /*
 230          * we do not change cc->clen to LZ4_compressBound(inputsize) to
 231          * adapt worst compress case, because lz4 compressor can handle
 232          * output budget properly.
 233          */
 234         cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
 235         return 0;
 236 }
 237
 238 static void lz4_destroy_compress_ctx(struct compress_ctx *cc)
 239 {
 240         kvfree(cc->private);
 241         cc->private = NULL;
 242 }
 243
 244 static int lz4_compress_pages(struct compress_ctx *cc)
 245 {
 246         int len;
 247
 248         len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
 249                                                 cc->clen, cc->private);
 250         if (!len)
 251                 return -EAGAIN;
 252
 253         cc->clen = len;
 254         return 0;
 255 }
 256
 257 static int lz4_decompress_pages(struct decompress_io_ctx *dic)
 258 {
 259         int ret;
 260
 261         ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,
 262                                                 dic->clen, dic->rlen);
 263         if (ret < 0) {
 264                 printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n",
 265                                 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
 266                 return -EIO;
 267         }
 268
 269         if (ret != PAGE_SIZE << dic->log_cluster_size) {
 270                 printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, "
 271                                         "expected:%lu\n", KERN_ERR,
 272                                         F2FS_I_SB(dic->inode)->sb->s_id,
 273                                         dic->rlen,
 274                                         PAGE_SIZE << dic->log_cluster_size);
 275                 return -EIO;
 276         }
 277         return 0;
 278 }
 279
 280 static const struct f2fs_compress_ops f2fs_lz4_ops = {
 281         .init_compress_ctx      = lz4_init_compress_ctx,
 282         .destroy_compress_ctx   = lz4_destroy_compress_ctx,
 283         .compress_pages         = lz4_compress_pages,
 284         .decompress_pages       = lz4_decompress_pages,
 285 };
 286 #endif
 287
 288 #ifdef CONFIG_F2FS_FS_ZSTD
 289 #define F2FS_ZSTD_DEFAULT_CLEVEL        1
 290
 291 static int zstd_init_compress_ctx(struct compress_ctx *cc)
 292 {
 293         ZSTD_parameters params;
 294         ZSTD_CStream *stream;
 295         void *workspace;
 296         unsigned int workspace_size;
 297
 298         params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0);
 299         workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams);
 300
 301         workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
 302                                         workspace_size, GFP_NOFS);
 303         if (!workspace)
 304                 return -ENOMEM;
 305
 306         stream = ZSTD_initCStream(params, 0, workspace, workspace_size);
 307         if (!stream) {
 308                 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n",
 309                                 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
 310                                 __func__);
 311                 kvfree(workspace);
 312                 return -EIO;
 313         }
 314
 315         cc->private = workspace;
 316         cc->private2 = stream;
 317
 318         cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
 319         return 0;
 320 }
 321
 322 static void zstd_destroy_compress_ctx(struct compress_ctx *cc)
 323 {
 324         kvfree(cc->private);
 325         cc->private = NULL;
 326         cc->private2 = NULL;
 327 }
 328
 329 static int zstd_compress_pages(struct compress_ctx *cc)
 330 {
 331         ZSTD_CStream *stream = cc->private2;
 332         ZSTD_inBuffer inbuf;
 333         ZSTD_outBuffer outbuf;
 334         int src_size = cc->rlen;
 335         int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE;
 336         int ret;
 337
 338         inbuf.pos = 0;
 339         inbuf.src = cc->rbuf;
 340         inbuf.size = src_size;
 341
 342         outbuf.pos = 0;
 343         outbuf.dst = cc->cbuf->cdata;
 344         outbuf.size = dst_size;
 345
 346         ret = ZSTD_compressStream(stream, &outbuf, &inbuf);
 347         if (ZSTD_isError(ret)) {
 348                 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
 349                                 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
 350                                 __func__, ZSTD_getErrorCode(ret));
 351                 return -EIO;
 352         }
 353
 354         ret = ZSTD_endStream(stream, &outbuf);
 355         if (ZSTD_isError(ret)) {
 356                 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n",
 357                                 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
 358                                 __func__, ZSTD_getErrorCode(ret));
 359                 return -EIO;
 360         }
 361
 362         /*
 363          * there is compressed data remained in intermediate buffer due to
 364          * no more space in cbuf.cdata
 365          */
 366         if (ret)
 367                 return -EAGAIN;
 368
 369         cc->clen = outbuf.pos;
 370         return 0;
 371 }
 372
 373 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)
 374 {
 375         ZSTD_DStream *stream;
 376         void *workspace;
 377         unsigned int workspace_size;
 378
 379         workspace_size = ZSTD_DStreamWorkspaceBound(MAX_COMPRESS_WINDOW_SIZE);
 380
 381         workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode),
 382                                         workspace_size, GFP_NOFS);
 383         if (!workspace)
 384                 return -ENOMEM;
 385
 386         stream = ZSTD_initDStream(MAX_COMPRESS_WINDOW_SIZE,
 387                                         workspace, workspace_size);
 388         if (!stream) {
 389                 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n",
 390                                 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
 391                                 __func__);
 392                 kvfree(workspace);
 393                 return -EIO;
 394         }
 395
 396         dic->private = workspace;
 397         dic->private2 = stream;
 398
 399         return 0;
 400 }
 401
 402 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)
 403 {
 404         kvfree(dic->private);
 405         dic->private = NULL;
 406         dic->private2 = NULL;
 407 }
 408
 409 static int zstd_decompress_pages(struct decompress_io_ctx *dic)
 410 {
 411         ZSTD_DStream *stream = dic->private2;
 412         ZSTD_inBuffer inbuf;
 413         ZSTD_outBuffer outbuf;
 414         int ret;
 415
 416         inbuf.pos = 0;
 417         inbuf.src = dic->cbuf->cdata;
 418         inbuf.size = dic->clen;
 419
 420         outbuf.pos = 0;
 421         outbuf.dst = dic->rbuf;
 422         outbuf.size = dic->rlen;
 423
 424         ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
 425         if (ZSTD_isError(ret)) {
 426                 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
 427                                 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
 428                                 __func__, ZSTD_getErrorCode(ret));
 429                 return -EIO;
 430         }
 431
 432         if (dic->rlen != outbuf.pos) {
 433                 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, "
 434                                 "expected:%lu\n", KERN_ERR,
 435                                 F2FS_I_SB(dic->inode)->sb->s_id,
 436                                 __func__, dic->rlen,
 437                                 PAGE_SIZE << dic->log_cluster_size);
 438                 return -EIO;
 439         }
 440
 441         return 0;
 442 }
 443
 444 static const struct f2fs_compress_ops f2fs_zstd_ops = {
 445         .init_compress_ctx      = zstd_init_compress_ctx,
 446         .destroy_compress_ctx   = zstd_destroy_compress_ctx,
 447         .compress_pages         = zstd_compress_pages,
 448         .init_decompress_ctx    = zstd_init_decompress_ctx,
 449         .destroy_decompress_ctx = zstd_destroy_decompress_ctx,
 450         .decompress_pages       = zstd_decompress_pages,
 451 };
 452 #endif
 453
 454 #ifdef CONFIG_F2FS_FS_LZO
 455 #ifdef CONFIG_F2FS_FS_LZORLE
 456 static int lzorle_compress_pages(struct compress_ctx *cc)
 457 {
 458         int ret;
 459
 460         ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
 461                                         &cc->clen, cc->private);
 462         if (ret != LZO_E_OK) {
 463                 printk_ratelimited("%sF2FS-fs (%s): lzo-rle compress failed, ret:%d\n",
 464                                 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
 465                 return -EIO;
 466         }
 467         return 0;
 468 }
 469
 470 static const struct f2fs_compress_ops f2fs_lzorle_ops = {
 471         .init_compress_ctx      = lzo_init_compress_ctx,
 472         .destroy_compress_ctx   = lzo_destroy_compress_ctx,
 473         .compress_pages         = lzorle_compress_pages,
 474         .decompress_pages       = lzo_decompress_pages,
 475 };
 476 #endif
 477 #endif
 478
 479 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
 480 #ifdef CONFIG_F2FS_FS_LZO
 481         &f2fs_lzo_ops,
 482 #else
 483         NULL,
 484 #endif
 485 #ifdef CONFIG_F2FS_FS_LZ4
 486         &f2fs_lz4_ops,
 487 #else
 488         NULL,
 489 #endif
 490 #ifdef CONFIG_F2FS_FS_ZSTD
 491         &f2fs_zstd_ops,
 492 #else
 493         NULL,
 494 #endif
 495 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE)
 496         &f2fs_lzorle_ops,
 497 #else
 498         NULL,
 499 #endif
 500 };
 501
 502 bool f2fs_is_compress_backend_ready(struct inode *inode)
 503 {
 504         if (!f2fs_compressed_file(inode))
 505                 return true;
 506         return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];
 507 }
 508
 509 static mempool_t *compress_page_pool = NULL;
 510 static int num_compress_pages = 512;
 511 module_param(num_compress_pages, uint, 0444);
 512 MODULE_PARM_DESC(num_compress_pages,
 513                 "Number of intermediate compress pages to preallocate");
 514
 515 int f2fs_init_compress_mempool(void)
 516 {
 517         compress_page_pool = mempool_create_page_pool(num_compress_pages, 0);
 518         if (!compress_page_pool)
 519                 return -ENOMEM;
 520
 521         return 0;
 522 }
 523
 524 void f2fs_destroy_compress_mempool(void)
 525 {
 526         mempool_destroy(compress_page_pool);
 527 }
 528
 529 static struct page *f2fs_compress_alloc_page(void)
 530 {
 531         struct page *page;
 532
 533         page = mempool_alloc(compress_page_pool, GFP_NOFS);
 534         lock_page(page);
 535
 536         return page;
 537 }
 538
 539 static void f2fs_compress_free_page(struct page *page)
 540 {
 541         if (!page)
 542                 return;
 543         set_page_private(page, (unsigned long)NULL);
 544         ClearPagePrivate(page);
 545         page->mapping = NULL;
 546         unlock_page(page);
 547         mempool_free(page, compress_page_pool);
 548 }
 549
 550 static int f2fs_compress_pages(struct compress_ctx *cc)
 551 {
 552         struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
 553         struct f2fs_inode_info *fi = F2FS_I(cc->inode);
 554         const struct f2fs_compress_ops *cops =
 555                                 f2fs_cops[fi->i_compress_algorithm];
 556         unsigned int max_len, nr_cpages;
 557         int i, ret;
 558
 559         trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,
 560                                 cc->cluster_size, fi->i_compress_algorithm);
 561
 562         if (cops->init_compress_ctx) {
 563                 ret = cops->init_compress_ctx(cc);
 564                 if (ret)
 565                         goto out;
 566         }
 567
 568         max_len = COMPRESS_HEADER_SIZE + cc->clen;
 569         cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);
 570
 571         cc->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
 572                                         cc->nr_cpages, GFP_NOFS);
 573         if (!cc->cpages) {
 574                 ret = -ENOMEM;
 575                 goto destroy_compress_ctx;
 576         }
 577
 578         for (i = 0; i < cc->nr_cpages; i++) {
 579                 cc->cpages[i] = f2fs_compress_alloc_page();
 580                 if (!cc->cpages[i]) {
 581                         ret = -ENOMEM;
 582                         goto out_free_cpages;
 583                 }
 584         }
 585
 586         cc->rbuf = vmap(cc->rpages, cc->cluster_size, VM_MAP, PAGE_KERNEL_RO);
 587         if (!cc->rbuf) {
 588                 ret = -ENOMEM;
 589                 goto out_free_cpages;
 590         }
 591
 592         cc->cbuf = vmap(cc->cpages, cc->nr_cpages, VM_MAP, PAGE_KERNEL);
 593         if (!cc->cbuf) {
 594                 ret = -ENOMEM;
 595                 goto out_vunmap_rbuf;
 596         }
 597
 598         ret = cops->compress_pages(cc);
 599         if (ret)
 600                 goto out_vunmap_cbuf;
 601
 602         max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE;
 603
 604         if (cc->clen > max_len) {
 605                 ret = -EAGAIN;
 606                 goto out_vunmap_cbuf;
 607         }
 608
 609         cc->cbuf->clen = cpu_to_le32(cc->clen);
 610
 611         for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)
 612                 cc->cbuf->reserved[i] = cpu_to_le32(0);
 613
 614         nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);
 615
 616         /* zero out any unused part of the last page */
 617         memset(&cc->cbuf->cdata[cc->clen], 0,
 618                (nr_cpages * PAGE_SIZE) - (cc->clen + COMPRESS_HEADER_SIZE));
 619
 620         vunmap(cc->cbuf);
 621         vunmap(cc->rbuf);
 622
 623         for (i = nr_cpages; i < cc->nr_cpages; i++) {
 624                 f2fs_compress_free_page(cc->cpages[i]);
 625                 cc->cpages[i] = NULL;
 626         }
 627
 628         if (cops->destroy_compress_ctx)
 629                 cops->destroy_compress_ctx(cc);
 630
 631         cc->nr_cpages = nr_cpages;
 632
 633         trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
 634                                                         cc->clen, ret);
 635         return 0;
 636
 637 out_vunmap_cbuf:
 638         vunmap(cc->cbuf);
 639 out_vunmap_rbuf:
 640         vunmap(cc->rbuf);
 641 out_free_cpages:
 642         for (i = 0; i < cc->nr_cpages; i++) {
 643                 if (cc->cpages[i])
 644                         f2fs_compress_free_page(cc->cpages[i]);
 645         }
 646         kfree(cc->cpages);
 647         cc->cpages = NULL;
 648 destroy_compress_ctx:
 649         if (cops->destroy_compress_ctx)
 650                 cops->destroy_compress_ctx(cc);
 651 out:
 652         trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
 653                                                         cc->clen, ret);
 654         return ret;
 655 }
 656
 657 void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity)
 658 {
 659         struct decompress_io_ctx *dic =
 660                         (struct decompress_io_ctx *)page_private(page);
 661         struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
 662         struct f2fs_inode_info *fi= F2FS_I(dic->inode);
 663         const struct f2fs_compress_ops *cops =
 664                         f2fs_cops[fi->i_compress_algorithm];
 665         int ret;
 666
 667         dec_page_count(sbi, F2FS_RD_DATA);
 668
 669         if (bio->bi_status || PageError(page))
 670                 dic->failed = true;
 671
 672         if (refcount_dec_not_one(&dic->ref))
 673                 return;
 674
 675         trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,
 676                                 dic->cluster_size, fi->i_compress_algorithm);
 677
 678         /* submit partial compressed pages */
 679         if (dic->failed) {
 680                 ret = -EIO;
 681                 goto out_free_dic;
 682         }
 683
 684         if (cops->init_decompress_ctx) {
 685                 ret = cops->init_decompress_ctx(dic);
 686                 if (ret)
 687                         goto out_free_dic;
 688         }
 689
 690         dic->rbuf = vmap(dic->tpages, dic->cluster_size, VM_MAP, PAGE_KERNEL);
 691         if (!dic->rbuf) {
 692                 ret = -ENOMEM;
 693                 goto destroy_decompress_ctx;
 694         }
 695
 696         dic->cbuf = vmap(dic->cpages, dic->nr_cpages, VM_MAP, PAGE_KERNEL_RO);
 697         if (!dic->cbuf) {
 698                 ret = -ENOMEM;
 699                 goto out_vunmap_rbuf;
 700         }
 701
 702         dic->clen = le32_to_cpu(dic->cbuf->clen);
 703         dic->rlen = PAGE_SIZE << dic->log_cluster_size;
 704
 705         if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
 706                 ret = -EFSCORRUPTED;
 707                 goto out_vunmap_cbuf;
 708         }
 709
 710         ret = cops->decompress_pages(dic);
 711
 712 out_vunmap_cbuf:
 713         vunmap(dic->cbuf);
 714 out_vunmap_rbuf:
 715         vunmap(dic->rbuf);
 716 destroy_decompress_ctx:
 717         if (cops->destroy_decompress_ctx)
 718                 cops->destroy_decompress_ctx(dic);
 719 out_free_dic:
 720         if (verity)
 721                 refcount_set(&dic->ref, dic->nr_cpages);
 722         if (!verity)
 723                 f2fs_decompress_end_io(dic->rpages, dic->cluster_size,
 724                                                                 ret, false);
 725
 726         trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,
 727                                                         dic->clen, ret);
 728         if (!verity)
 729                 f2fs_free_dic(dic);
 730 }
 731
 732 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)
 733 {
 734         if (cc->cluster_idx == NULL_CLUSTER)
 735                 return true;
 736         return cc->cluster_idx == cluster_idx(cc, index);
 737 }
 738
 739 bool f2fs_cluster_is_empty(struct compress_ctx *cc)
 740 {
 741         return cc->nr_rpages == 0;
 742 }
 743
 744 static bool f2fs_cluster_is_full(struct compress_ctx *cc)
 745 {
 746         return cc->cluster_size == cc->nr_rpages;
 747 }
 748
 749 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index)
 750 {
 751         if (f2fs_cluster_is_empty(cc))
 752                 return true;
 753         return is_page_in_cluster(cc, index);
 754 }
 755
 756 static bool __cluster_may_compress(struct compress_ctx *cc)
 757 {
 758         struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
 759         loff_t i_size = i_size_read(cc->inode);
 760         unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);
 761         int i;
 762
 763         for (i = 0; i < cc->cluster_size; i++) {
 764                 struct page *page = cc->rpages[i];
 765
 766                 f2fs_bug_on(sbi, !page);
 767
 768                 if (unlikely(f2fs_cp_error(sbi)))
 769                         return false;
 770                 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 771                         return false;
 772
 773                 /* beyond EOF */
 774                 if (page->index >= nr_pages)
 775                         return false;
 776         }
 777         return true;
 778 }
 779
 780 static int __f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
 781 {
 782         struct dnode_of_data dn;
 783         int ret;
 784
 785         set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
 786         ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc),
 787                                                         LOOKUP_NODE);
 788         if (ret) {
 789                 if (ret == -ENOENT)
 790                         ret = 0;
 791                 goto fail;
 792         }
 793
 794         if (dn.data_blkaddr == COMPRESS_ADDR) {
 795                 int i;
 796
 797                 ret = 1;
 798                 for (i = 1; i < cc->cluster_size; i++) {
 799                         block_t blkaddr;
 800
 801                         blkaddr = data_blkaddr(dn.inode,
 802                                         dn.node_page, dn.ofs_in_node + i);
 803                         if (compr) {
 804                                 if (__is_valid_data_blkaddr(blkaddr))
 805                                         ret++;
 806                         } else {
 807                                 if (blkaddr != NULL_ADDR)
 808                                         ret++;
 809                         }
 810                 }
 811         }
 812 fail:
 813         f2fs_put_dnode(&dn);
 814         return ret;
 815 }
 816
 817 /* return # of compressed blocks in compressed cluster */
 818 static int f2fs_compressed_blocks(struct compress_ctx *cc)
 819 {
 820         return __f2fs_cluster_blocks(cc, true);
 821 }
 822
 823 /* return # of valid blocks in compressed cluster */
 824 static int f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
 825 {
 826         return __f2fs_cluster_blocks(cc, false);
 827 }
 828
 829 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)
 830 {
 831         struct compress_ctx cc = {
 832                 .inode = inode,
 833                 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
 834                 .cluster_size = F2FS_I(inode)->i_cluster_size,
 835                 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
 836         };
 837
 838         return f2fs_cluster_blocks(&cc, false);
 839 }
 840
 841 static bool cluster_may_compress(struct compress_ctx *cc)
 842 {
 843         if (!f2fs_compressed_file(cc->inode))
 844                 return false;
 845         if (f2fs_is_atomic_file(cc->inode))
 846                 return false;
 847         if (f2fs_is_mmap_file(cc->inode))
 848                 return false;
 849         if (!f2fs_cluster_is_full(cc))
 850                 return false;
 851         if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode))))
 852                 return false;
 853         return __cluster_may_compress(cc);
 854 }
 855
 856 static void set_cluster_writeback(struct compress_ctx *cc)
 857 {
 858         int i;
 859
 860         for (i = 0; i < cc->cluster_size; i++) {
 861                 if (cc->rpages[i])
 862                         set_page_writeback(cc->rpages[i]);
 863         }
 864 }
 865
 866 static void set_cluster_dirty(struct compress_ctx *cc)
 867 {
 868         int i;
 869
 870         for (i = 0; i < cc->cluster_size; i++)
 871                 if (cc->rpages[i])
 872                         set_page_dirty(cc->rpages[i]);
 873 }
 874
 875 static int prepare_compress_overwrite(struct compress_ctx *cc,
 876                 struct page **pagep, pgoff_t index, void **fsdata)
 877 {
 878         struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
 879         struct address_space *mapping = cc->inode->i_mapping;
 880         struct page *page;
 881         struct dnode_of_data dn;
 882         sector_t last_block_in_bio;
 883         unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;
 884         pgoff_t start_idx = start_idx_of_cluster(cc);
 885         int i, ret;
 886         bool prealloc;
 887
 888 retry:
 889         ret = f2fs_cluster_blocks(cc, false);
 890         if (ret <= 0)
 891                 return ret;
 892
 893         /* compressed case */
 894         prealloc = (ret < cc->cluster_size);
 895
 896         ret = f2fs_init_compress_ctx(cc);
 897         if (ret)
 898                 return ret;
 899
 900         /* keep page reference to avoid page reclaim */
 901         for (i = 0; i < cc->cluster_size; i++) {
 902                 page = f2fs_pagecache_get_page(mapping, start_idx + i,
 903                                                         fgp_flag, GFP_NOFS);
 904                 if (!page) {
 905                         ret = -ENOMEM;
 906                         goto unlock_pages;
 907                 }
 908
 909                 if (PageUptodate(page))
 910                         unlock_page(page);
 911                 else
 912                         f2fs_compress_ctx_add_page(cc, page);
 913         }
 914
 915         if (!f2fs_cluster_is_empty(cc)) {
 916                 struct bio *bio = NULL;
 917
 918                 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,
 919                                         &last_block_in_bio, false, true);
 920                 f2fs_destroy_compress_ctx(cc);
 921                 if (ret)
 922                         goto release_pages;
 923                 if (bio)
 924                         f2fs_submit_bio(sbi, bio, DATA);
 925
 926                 ret = f2fs_init_compress_ctx(cc);
 927                 if (ret)
 928                         goto release_pages;
 929         }
 930
 931         for (i = 0; i < cc->cluster_size; i++) {
 932                 f2fs_bug_on(sbi, cc->rpages[i]);
 933
 934                 page = find_lock_page(mapping, start_idx + i);
 935                 f2fs_bug_on(sbi, !page);
 936
 937                 f2fs_wait_on_page_writeback(page, DATA, true, true);
 938
 939                 f2fs_compress_ctx_add_page(cc, page);
 940                 f2fs_put_page(page, 0);
 941
 942                 if (!PageUptodate(page)) {
 943                         f2fs_unlock_rpages(cc, i + 1);
 944                         f2fs_put_rpages_mapping(mapping, start_idx,
 945                                         cc->cluster_size);
 946                         f2fs_destroy_compress_ctx(cc);
 947                         goto retry;
 948                 }
 949         }
 950
 951         if (prealloc) {
 952                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
 953
 954                 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
 955
 956                 for (i = cc->cluster_size - 1; i > 0; i--) {
 957                         ret = f2fs_get_block(&dn, start_idx + i);
 958                         if (ret) {
 959                                 i = cc->cluster_size;
 960                                 break;
 961                         }
 962
 963                         if (dn.data_blkaddr != NEW_ADDR)
 964                                 break;
 965                 }
 966
 967                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
 968         }
 969
 970         if (likely(!ret)) {
 971                 *fsdata = cc->rpages;
 972                 *pagep = cc->rpages[offset_in_cluster(cc, index)];
 973                 return cc->cluster_size;
 974         }
 975
 976 unlock_pages:
 977         f2fs_unlock_rpages(cc, i);
 978 release_pages:
 979         f2fs_put_rpages_mapping(mapping, start_idx, i);
 980         f2fs_destroy_compress_ctx(cc);
 981         return ret;
 982 }
 983
 984 int f2fs_prepare_compress_overwrite(struct inode *inode,
 985                 struct page **pagep, pgoff_t index, void **fsdata)
 986 {
 987         struct compress_ctx cc = {
 988                 .inode = inode,
 989                 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
 990                 .cluster_size = F2FS_I(inode)->i_cluster_size,
 991                 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
 992                 .rpages = NULL,
 993                 .nr_rpages = 0,
 994         };
 995
 996         return prepare_compress_overwrite(&cc, pagep, index, fsdata);
 997 }
 998
 999 bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
1000                                         pgoff_t index, unsigned copied)
1001
1002 {
1003         struct compress_ctx cc = {
1004                 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
1005                 .cluster_size = F2FS_I(inode)->i_cluster_size,
1006                 .rpages = fsdata,
1007         };
1008         bool first_index = (index == cc.rpages[0]->index);
1009
1010         if (copied)
1011                 set_cluster_dirty(&cc);
1012
1013         f2fs_put_rpages_wbc(&cc, NULL, false, 1);
1014         f2fs_destroy_compress_ctx(&cc);
1015
1016         return first_index;
1017 }
1018
1019 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock)
1020 {
1021         void *fsdata = NULL;
1022         struct page *pagep;
1023         int log_cluster_size = F2FS_I(inode)->i_log_cluster_size;
1024         pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) <<
1025                                                         log_cluster_size;
1026         int err;
1027
1028         err = f2fs_is_compressed_cluster(inode, start_idx);
1029         if (err < 0)
1030                 return err;
1031
1032         /* truncate normal cluster */
1033         if (!err)
1034                 return f2fs_do_truncate_blocks(inode, from, lock);
1035
1036         /* truncate compressed cluster */
1037         err = f2fs_prepare_compress_overwrite(inode, &pagep,
1038                                                 start_idx, &fsdata);
1039
1040         /* should not be a normal cluster */
1041         f2fs_bug_on(F2FS_I_SB(inode), err == 0);
1042
1043         if (err <= 0)
1044                 return err;
1045
1046         if (err > 0) {
1047                 struct page **rpages = fsdata;
1048                 int cluster_size = F2FS_I(inode)->i_cluster_size;
1049                 int i;
1050
1051                 for (i = cluster_size - 1; i >= 0; i--) {
1052                         loff_t start = rpages[i]->index << PAGE_SHIFT;
1053
1054                         if (from <= start) {
1055                                 zero_user_segment(rpages[i], 0, PAGE_SIZE);
1056                         } else {
1057                                 zero_user_segment(rpages[i], from - start,
1058                                                                 PAGE_SIZE);
1059                                 break;
1060                         }
1061                 }
1062
1063                 f2fs_compress_write_end(inode, fsdata, start_idx, true);
1064         }
1065         return 0;
1066 }
1067
1068 static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1069                                         int *submitted,
1070                                         struct writeback_control *wbc,
1071                                         enum iostat_type io_type)
1072 {
1073         struct inode *inode = cc->inode;
1074         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1075         struct f2fs_inode_info *fi = F2FS_I(inode);
1076         struct f2fs_io_info fio = {
1077                 .sbi = sbi,
1078                 .ino = cc->inode->i_ino,
1079                 .type = DATA,
1080                 .op = REQ_OP_WRITE,
1081                 .op_flags = wbc_to_write_flags(wbc),
1082                 .old_blkaddr = NEW_ADDR,
1083                 .page = NULL,
1084                 .encrypted_page = NULL,
1085                 .compressed_page = NULL,
1086                 .submitted = false,
1087                 .io_type = io_type,
1088                 .io_wbc = wbc,
1089                 .encrypted = f2fs_encrypted_file(cc->inode),
1090         };
1091         struct dnode_of_data dn;
1092         struct node_info ni;
1093         struct compress_io_ctx *cic;
1094         pgoff_t start_idx = start_idx_of_cluster(cc);
1095         unsigned int last_index = cc->cluster_size - 1;
1096         loff_t psize;
1097         int i, err;
1098
1099         if (!IS_NOQUOTA(inode) && !f2fs_trylock_op(sbi))
1100                 return -EAGAIN;
1101
1102         set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
1103
1104         err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
1105         if (err)
1106                 goto out_unlock_op;
1107
1108         for (i = 0; i < cc->cluster_size; i++) {
1109                 if (data_blkaddr(dn.inode, dn.node_page,
1110                                         dn.ofs_in_node + i) == NULL_ADDR)
1111                         goto out_put_dnode;
1112         }
1113
1114         psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT;
1115
1116         err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
1117         if (err)
1118                 goto out_put_dnode;
1119
1120         fio.version = ni.version;
1121
1122         cic = f2fs_kzalloc(sbi, sizeof(struct compress_io_ctx), GFP_NOFS);
1123         if (!cic)
1124                 goto out_put_dnode;
1125
1126         cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1127         cic->inode = inode;
1128         refcount_set(&cic->ref, cc->nr_cpages);
1129         cic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1130                         cc->log_cluster_size, GFP_NOFS);
1131         if (!cic->rpages)
1132                 goto out_put_cic;
1133
1134         cic->nr_rpages = cc->cluster_size;
1135
1136         for (i = 0; i < cc->nr_cpages; i++) {
1137                 f2fs_set_compressed_page(cc->cpages[i], inode,
1138                                         cc->rpages[i + 1]->index, cic);
1139                 fio.compressed_page = cc->cpages[i];
1140                 if (fio.encrypted) {
1141                         fio.page = cc->rpages[i + 1];
1142                         err = f2fs_encrypt_one_page(&fio);
1143                         if (err)
1144                                 goto out_destroy_crypt;
1145                         cc->cpages[i] = fio.encrypted_page;
1146                 }
1147         }
1148
1149         set_cluster_writeback(cc);
1150
1151         for (i = 0; i < cc->cluster_size; i++)
1152                 cic->rpages[i] = cc->rpages[i];
1153
1154         for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {
1155                 block_t blkaddr;
1156
1157                 blkaddr = f2fs_data_blkaddr(&dn);
1158                 fio.page = cc->rpages[i];
1159                 fio.old_blkaddr = blkaddr;
1160
1161                 /* cluster header */
1162                 if (i == 0) {
1163                         if (blkaddr == COMPRESS_ADDR)
1164                                 fio.compr_blocks++;
1165                         if (__is_valid_data_blkaddr(blkaddr))
1166                                 f2fs_invalidate_blocks(sbi, blkaddr);
1167                         f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR);
1168                         goto unlock_continue;
1169                 }
1170
1171                 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))
1172                         fio.compr_blocks++;
1173
1174                 if (i > cc->nr_cpages) {
1175                         if (__is_valid_data_blkaddr(blkaddr)) {
1176                                 f2fs_invalidate_blocks(sbi, blkaddr);
1177                                 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
1178                         }
1179                         goto unlock_continue;
1180                 }
1181
1182                 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR);
1183
1184                 if (fio.encrypted)
1185                         fio.encrypted_page = cc->cpages[i - 1];
1186                 else
1187                         fio.compressed_page = cc->cpages[i - 1];
1188
1189                 cc->cpages[i - 1] = NULL;
1190                 f2fs_outplace_write_data(&dn, &fio);
1191                 (*submitted)++;
1192 unlock_continue:
1193                 inode_dec_dirty_pages(cc->inode);
1194                 unlock_page(fio.page);
1195         }
1196
1197         if (fio.compr_blocks)
1198                 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
1199         f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true);
1200
1201         set_inode_flag(cc->inode, FI_APPEND_WRITE);
1202         if (cc->cluster_idx == 0)
1203                 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1204
1205         f2fs_put_dnode(&dn);
1206         if (!IS_NOQUOTA(inode))
1207                 f2fs_unlock_op(sbi);
1208
1209         spin_lock(&fi->i_size_lock);
1210         if (fi->last_disk_size < psize)
1211                 fi->last_disk_size = psize;
1212         spin_unlock(&fi->i_size_lock);
1213
1214         f2fs_put_rpages(cc);
1215         f2fs_destroy_compress_ctx(cc);
1216         return 0;
1217
1218 out_destroy_crypt:
1219         kfree(cic->rpages);
1220
1221         for (--i; i >= 0; i--)
1222                 fscrypt_finalize_bounce_page(&cc->cpages[i]);
1223         for (i = 0; i < cc->nr_cpages; i++) {
1224                 if (!cc->cpages[i])
1225                         continue;
1226                 f2fs_put_page(cc->cpages[i], 1);
1227         }
1228 out_put_cic:
1229         kfree(cic);
1230 out_put_dnode:
1231         f2fs_put_dnode(&dn);
1232 out_unlock_op:
1233         if (!IS_NOQUOTA(inode))
1234                 f2fs_unlock_op(sbi);
1235         return -EAGAIN;
1236 }
1237
1238 void f2fs_compress_write_end_io(struct bio *bio, struct page *page)
1239 {
1240         struct f2fs_sb_info *sbi = bio->bi_private;
1241         struct compress_io_ctx *cic =
1242                         (struct compress_io_ctx *)page_private(page);
1243         int i;
1244
1245         if (unlikely(bio->bi_status))
1246                 mapping_set_error(cic->inode->i_mapping, -EIO);
1247
1248         f2fs_compress_free_page(page);
1249
1250         dec_page_count(sbi, F2FS_WB_DATA);
1251
1252         if (refcount_dec_not_one(&cic->ref))
1253                 return;
1254
1255         for (i = 0; i < cic->nr_rpages; i++) {
1256                 WARN_ON(!cic->rpages[i]);
1257                 clear_cold_data(cic->rpages[i]);
1258                 end_page_writeback(cic->rpages[i]);
1259         }
1260
1261         kfree(cic->rpages);
1262         kfree(cic);
1263 }
1264
1265 static int f2fs_write_raw_pages(struct compress_ctx *cc,
1266                                         int *submitted,
1267                                         struct writeback_control *wbc,
1268                                         enum iostat_type io_type)
1269 {
1270         struct address_space *mapping = cc->inode->i_mapping;
1271         int _submitted, compr_blocks, ret;
1272         int i = -1, err = 0;
1273
1274         compr_blocks = f2fs_compressed_blocks(cc);
1275         if (compr_blocks < 0) {
1276                 err = compr_blocks;
1277                 goto out_err;
1278         }
1279
1280         for (i = 0; i < cc->cluster_size; i++) {
1281                 if (!cc->rpages[i])
1282                         continue;
1283 retry_write:
1284                 if (cc->rpages[i]->mapping != mapping) {
1285                         unlock_page(cc->rpages[i]);
1286                         continue;
1287                 }
1288
1289                 BUG_ON(!PageLocked(cc->rpages[i]));
1290
1291                 ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted,
1292                                                 NULL, NULL, wbc, io_type,
1293                                                 compr_blocks);
1294                 if (ret) {
1295                         if (ret == AOP_WRITEPAGE_ACTIVATE) {
1296                                 unlock_page(cc->rpages[i]);
1297                                 ret = 0;
1298                         } else if (ret == -EAGAIN) {
1299                                 /*
1300                                  * for quota file, just redirty left pages to
1301                                  * avoid deadlock caused by cluster update race
1302                                  * from foreground operation.
1303                                  */
1304                                 if (IS_NOQUOTA(cc->inode)) {
1305                                         err = 0;
1306                                         goto out_err;
1307                                 }
1308                                 ret = 0;
1309                                 cond_resched();
1310                                 congestion_wait(BLK_RW_ASYNC,
1311                                                 DEFAULT_IO_TIMEOUT);
1312                                 lock_page(cc->rpages[i]);
1313                                 clear_page_dirty_for_io(cc->rpages[i]);
1314                                 goto retry_write;
1315                         }
1316                         err = ret;
1317                         goto out_err;
1318                 }
1319
1320                 *submitted += _submitted;
1321         }
1322         return 0;
1323 out_err:
1324         for (++i; i < cc->cluster_size; i++) {
1325                 if (!cc->rpages[i])
1326                         continue;
1327                 redirty_page_for_writepage(wbc, cc->rpages[i]);
1328                 unlock_page(cc->rpages[i]);
1329         }
1330         return err;
1331 }
1332
1333 int f2fs_write_multi_pages(struct compress_ctx *cc,
1334                                         int *submitted,
1335                                         struct writeback_control *wbc,
1336                                         enum iostat_type io_type)
1337 {
1338         struct f2fs_inode_info *fi = F2FS_I(cc->inode);
1339         const struct f2fs_compress_ops *cops =
1340                         f2fs_cops[fi->i_compress_algorithm];
1341         int err;
1342
1343         *submitted = 0;
1344         if (cluster_may_compress(cc)) {
1345                 err = f2fs_compress_pages(cc);
1346                 if (err == -EAGAIN) {
1347                         goto write;
1348                 } else if (err) {
1349                         f2fs_put_rpages_wbc(cc, wbc, true, 1);
1350                         goto destroy_out;
1351                 }
1352
1353                 err = f2fs_write_compressed_pages(cc, submitted,
1354                                                         wbc, io_type);
1355                 cops->destroy_compress_ctx(cc);
1356                 if (!err)
1357                         return 0;
1358                 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);
1359         }
1360 write:
1361         f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted);
1362
1363         err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);
1364         f2fs_put_rpages_wbc(cc, wbc, false, 0);
1365 destroy_out:
1366         f2fs_destroy_compress_ctx(cc);
1367         return err;
1368 }
1369
1370 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
1371 {
1372         struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1373         struct decompress_io_ctx *dic;
1374         pgoff_t start_idx = start_idx_of_cluster(cc);
1375         int i;
1376
1377         dic = f2fs_kzalloc(sbi, sizeof(struct decompress_io_ctx), GFP_NOFS);
1378         if (!dic)
1379                 return ERR_PTR(-ENOMEM);
1380
1381         dic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1382                         cc->log_cluster_size, GFP_NOFS);
1383         if (!dic->rpages) {
1384                 kfree(dic);
1385                 return ERR_PTR(-ENOMEM);
1386         }
1387
1388         dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1389         dic->inode = cc->inode;
1390         refcount_set(&dic->ref, cc->nr_cpages);
1391         dic->cluster_idx = cc->cluster_idx;
1392         dic->cluster_size = cc->cluster_size;
1393         dic->log_cluster_size = cc->log_cluster_size;
1394         dic->nr_cpages = cc->nr_cpages;
1395         dic->failed = false;
1396
1397         for (i = 0; i < dic->cluster_size; i++)
1398                 dic->rpages[i] = cc->rpages[i];
1399         dic->nr_rpages = cc->cluster_size;
1400
1401         dic->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1402                                         dic->nr_cpages, GFP_NOFS);
1403         if (!dic->cpages)
1404                 goto out_free;
1405
1406         for (i = 0; i < dic->nr_cpages; i++) {
1407                 struct page *page;
1408
1409                 page = f2fs_compress_alloc_page();
1410                 if (!page)
1411                         goto out_free;
1412
1413                 f2fs_set_compressed_page(page, cc->inode,
1414                                         start_idx + i + 1, dic);
1415                 dic->cpages[i] = page;
1416         }
1417
1418         dic->tpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1419                                         dic->cluster_size, GFP_NOFS);
1420         if (!dic->tpages)
1421                 goto out_free;
1422
1423         for (i = 0; i < dic->cluster_size; i++) {
1424                 if (cc->rpages[i]) {
1425                         dic->tpages[i] = cc->rpages[i];
1426                         continue;
1427                 }
1428
1429                 dic->tpages[i] = f2fs_compress_alloc_page();
1430                 if (!dic->tpages[i])
1431                         goto out_free;
1432         }
1433
1434         return dic;
1435
1436 out_free:
1437         f2fs_free_dic(dic);
1438         return ERR_PTR(-ENOMEM);
1439 }
1440
1441 void f2fs_free_dic(struct decompress_io_ctx *dic)
1442 {
1443         int i;
1444
1445         if (dic->tpages) {
1446                 for (i = 0; i < dic->cluster_size; i++) {
1447                         if (dic->rpages[i])
1448                                 continue;
1449                         if (!dic->tpages[i])
1450                                 continue;
1451                         f2fs_compress_free_page(dic->tpages[i]);
1452                 }
1453                 kfree(dic->tpages);
1454         }
1455
1456         if (dic->cpages) {
1457                 for (i = 0; i < dic->nr_cpages; i++) {
1458                         if (!dic->cpages[i])
1459                                 continue;
1460                         f2fs_compress_free_page(dic->cpages[i]);
1461                 }
1462                 kfree(dic->cpages);
1463         }
1464
1465         kfree(dic->rpages);
1466         kfree(dic);
1467 }
1468
1469 void f2fs_decompress_end_io(struct page **rpages,
1470                         unsigned int cluster_size, bool err, bool verity)
1471 {
1472         int i;
1473
1474         for (i = 0; i < cluster_size; i++) {
1475                 struct page *rpage = rpages[i];
1476
1477                 if (!rpage)
1478                         continue;
1479
1480                 if (err || PageError(rpage))
1481                         goto clear_uptodate;
1482
1483                 if (!verity || fsverity_verify_page(rpage)) {
1484                         SetPageUptodate(rpage);
1485                         goto unlock;
1486                 }
1487 clear_uptodate:
1488                 ClearPageUptodate(rpage);
1489                 ClearPageError(rpage);
1490 unlock:
1491                 unlock_page(rpage);
1492         }
1493 }