drivers/lightnvm/pblk-read.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2016 CNEX Labs
   4  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
   5  *                  Matias Bjorling <matias@cnexlabs.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License version
   9  * 2 as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful, but
  12  * WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * General Public License for more details.
  15  *
  16  * pblk-read.c - pblk's read path
  17  */
  18
  19 #include "pblk.h"
  20
  21 /*
  22  * There is no guarantee that the value read from cache has not been updated and
  23  * resides at another location in the cache. We guarantee though that if the
  24  * value is read from the cache, it belongs to the mapped lba. In order to
  25  * guarantee and order between writes and reads are ordered, a flush must be
  26  * issued.
  27  */
  28 static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio,
  29                                 sector_t lba, struct ppa_addr ppa,
  30                                 int bio_iter, bool advanced_bio)
  31 {
  32 #ifdef CONFIG_NVM_PBLK_DEBUG
  33         /* Callers must ensure that the ppa points to a cache address */
  34         BUG_ON(pblk_ppa_empty(ppa));
  35         BUG_ON(!pblk_addr_in_cache(ppa));
  36 #endif
  37
  38         return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa,
  39                                                 bio_iter, advanced_bio);
  40 }
  41
  42 static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd,
  43                                  struct bio *bio, sector_t blba,
  44                                  unsigned long *read_bitmap)
  45 {
  46         void *meta_list = rqd->meta_list;
  47         struct ppa_addr ppas[NVM_MAX_VLBA];
  48         int nr_secs = rqd->nr_ppas;
  49         bool advanced_bio = false;
  50         int i, j = 0;
  51
  52         pblk_lookup_l2p_seq(pblk, ppas, blba, nr_secs);
  53
  54         for (i = 0; i < nr_secs; i++) {
  55                 struct ppa_addr p = ppas[i];
  56                 struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
  57                 sector_t lba = blba + i;
  58
  59 retry:
  60                 if (pblk_ppa_empty(p)) {
  61                         __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
  62
  63                         WARN_ON(test_and_set_bit(i, read_bitmap));
  64                         meta->lba = addr_empty;
  65
  66                         if (unlikely(!advanced_bio)) {
  67                                 bio_advance(bio, (i) * PBLK_EXPOSED_PAGE_SIZE);
  68                                 advanced_bio = true;
  69                         }
  70
  71                         goto next;
  72                 }
  73
  74                 /* Try to read from write buffer. The address is later checked
  75                  * on the write buffer to prevent retrieving overwritten data.
  76                  */
  77                 if (pblk_addr_in_cache(p)) {
  78                         if (!pblk_read_from_cache(pblk, bio, lba, p, i,
  79                                                                 advanced_bio)) {
  80                                 pblk_lookup_l2p_seq(pblk, &p, lba, 1);
  81                                 goto retry;
  82                         }
  83                         WARN_ON(test_and_set_bit(i, read_bitmap));
  84                         meta->lba = cpu_to_le64(lba);
  85                         advanced_bio = true;
  86 #ifdef CONFIG_NVM_PBLK_DEBUG
  87                         atomic_long_inc(&pblk->cache_reads);
  88 #endif
  89                 } else {
  90                         /* Read from media non-cached sectors */
  91                         rqd->ppa_list[j++] = p;
  92                 }
  93
  94 next:
  95                 if (advanced_bio)
  96                         bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
  97         }
  98
  99         if (pblk_io_aligned(pblk, nr_secs))
 100                 rqd->is_seq = 1;
 101
 102 #ifdef CONFIG_NVM_PBLK_DEBUG
 103         atomic_long_add(nr_secs, &pblk->inflight_reads);
 104 #endif
 105 }
 106
 107
 108 static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd,
 109                                 sector_t blba)
 110 {
 111         void *meta_list = rqd->meta_list;
 112         int nr_lbas = rqd->nr_ppas;
 113         int i;
 114
 115         if (!pblk_is_oob_meta_supported(pblk))
 116                 return;
 117
 118         for (i = 0; i < nr_lbas; i++) {
 119                 struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
 120                 u64 lba = le64_to_cpu(meta->lba);
 121
 122                 if (lba == ADDR_EMPTY)
 123                         continue;
 124
 125                 if (lba != blba + i) {
 126 #ifdef CONFIG_NVM_PBLK_DEBUG
 127                         struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
 128
 129                         print_ppa(pblk, &ppa_list[i], "seq", i);
 130 #endif
 131                         pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
 132                                                         lba, (u64)blba + i);
 133                         WARN_ON(1);
 134                 }
 135         }
 136 }
 137
 138 /*
 139  * There can be holes in the lba list.
 140  */
 141 static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd,
 142                                  u64 *lba_list, int nr_lbas)
 143 {
 144         void *meta_lba_list = rqd->meta_list;
 145         int i, j;
 146
 147         if (!pblk_is_oob_meta_supported(pblk))
 148                 return;
 149
 150         for (i = 0, j = 0; i < nr_lbas; i++) {
 151                 struct pblk_sec_meta *meta = pblk_get_meta(pblk,
 152                                                            meta_lba_list, j);
 153                 u64 lba = lba_list[i];
 154                 u64 meta_lba;
 155
 156                 if (lba == ADDR_EMPTY)
 157                         continue;
 158
 159                 meta_lba = le64_to_cpu(meta->lba);
 160
 161                 if (lba != meta_lba) {
 162 #ifdef CONFIG_NVM_PBLK_DEBUG
 163                         struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
 164
 165                         print_ppa(pblk, &ppa_list[j], "rnd", j);
 166 #endif
 167                         pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
 168                                                         meta_lba, lba);
 169                         WARN_ON(1);
 170                 }
 171
 172                 j++;
 173         }
 174
 175         WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n");
 176 }
 177
 178 static void pblk_end_user_read(struct bio *bio)
 179 {
 180 #ifdef CONFIG_NVM_PBLK_DEBUG
 181         WARN_ONCE(bio->bi_status, "pblk: corrupted read bio\n");
 182 #endif
 183         bio_endio(bio);
 184 }
 185
 186 static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,
 187                                bool put_line)
 188 {
 189         struct nvm_tgt_dev *dev = pblk->dev;
 190         struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
 191         struct bio *int_bio = rqd->bio;
 192         unsigned long start_time = r_ctx->start_time;
 193
 194         generic_end_io_acct(dev->q, REQ_OP_READ, &pblk->disk->part0, start_time);
 195
 196         if (rqd->error)
 197                 pblk_log_read_err(pblk, rqd);
 198
 199         pblk_read_check_seq(pblk, rqd, r_ctx->lba);
 200
 201         if (int_bio)
 202                 bio_put(int_bio);
 203
 204         if (put_line)
 205                 pblk_rq_to_line_put(pblk, rqd);
 206
 207 #ifdef CONFIG_NVM_PBLK_DEBUG
 208         atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);
 209         atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);
 210 #endif
 211
 212         pblk_free_rqd(pblk, rqd, PBLK_READ);
 213         atomic_dec(&pblk->inflight_io);
 214 }
 215
 216 static void pblk_end_io_read(struct nvm_rq *rqd)
 217 {
 218         struct pblk *pblk = rqd->private;
 219         struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
 220         struct bio *bio = (struct bio *)r_ctx->private;
 221
 222         pblk_end_user_read(bio);
 223         __pblk_end_io_read(pblk, rqd, true);
 224 }
 225
 226 static void pblk_end_partial_read(struct nvm_rq *rqd)
 227 {
 228         struct pblk *pblk = rqd->private;
 229         struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
 230         struct pblk_pr_ctx *pr_ctx = r_ctx->private;
 231         struct pblk_sec_meta *meta;
 232         struct bio *new_bio = rqd->bio;
 233         struct bio *bio = pr_ctx->orig_bio;
 234         void *meta_list = rqd->meta_list;
 235         unsigned long *read_bitmap = pr_ctx->bitmap;
 236         struct bvec_iter orig_iter = BVEC_ITER_ALL_INIT;
 237         struct bvec_iter new_iter = BVEC_ITER_ALL_INIT;
 238         int nr_secs = pr_ctx->orig_nr_secs;
 239         int nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs);
 240         void *src_p, *dst_p;
 241         int bit, i;
 242
 243         if (unlikely(nr_holes == 1)) {
 244                 struct ppa_addr ppa;
 245
 246                 ppa = rqd->ppa_addr;
 247                 rqd->ppa_list = pr_ctx->ppa_ptr;
 248                 rqd->dma_ppa_list = pr_ctx->dma_ppa_list;
 249                 rqd->ppa_list[0] = ppa;
 250         }
 251
 252         for (i = 0; i < nr_secs; i++) {
 253                 meta = pblk_get_meta(pblk, meta_list, i);
 254                 pr_ctx->lba_list_media[i] = le64_to_cpu(meta->lba);
 255                 meta->lba = cpu_to_le64(pr_ctx->lba_list_mem[i]);
 256         }
 257
 258         /* Fill the holes in the original bio */
 259         i = 0;
 260         for (bit = 0; bit < nr_secs; bit++) {
 261                 if (!test_bit(bit, read_bitmap)) {
 262                         struct bio_vec dst_bv, src_bv;
 263                         struct pblk_line *line;
 264
 265                         line = pblk_ppa_to_line(pblk, rqd->ppa_list[i]);
 266                         kref_put(&line->ref, pblk_line_put);
 267
 268                         meta = pblk_get_meta(pblk, meta_list, bit);
 269                         meta->lba = cpu_to_le64(pr_ctx->lba_list_media[i]);
 270
 271                         dst_bv = bio_iter_iovec(bio, orig_iter);
 272                         src_bv = bio_iter_iovec(new_bio, new_iter);
 273
 274                         src_p = kmap_atomic(src_bv.bv_page);
 275                         dst_p = kmap_atomic(dst_bv.bv_page);
 276
 277                         memcpy(dst_p + dst_bv.bv_offset,
 278                                 src_p + src_bv.bv_offset,
 279                                 PBLK_EXPOSED_PAGE_SIZE);
 280
 281                         kunmap_atomic(src_p);
 282                         kunmap_atomic(dst_p);
 283
 284                         flush_dcache_page(dst_bv.bv_page);
 285                         mempool_free(src_bv.bv_page, &pblk->page_bio_pool);
 286
 287                         bio_advance_iter(new_bio, &new_iter,
 288                                         PBLK_EXPOSED_PAGE_SIZE);
 289                         i++;
 290                 }
 291                 bio_advance_iter(bio, &orig_iter, PBLK_EXPOSED_PAGE_SIZE);
 292         }
 293
 294         bio_put(new_bio);
 295         kfree(pr_ctx);
 296
 297         /* restore original request */
 298         rqd->bio = NULL;
 299         rqd->nr_ppas = nr_secs;
 300
 301         bio_endio(bio);
 302         __pblk_end_io_read(pblk, rqd, false);
 303 }
 304
 305 static int pblk_setup_partial_read(struct pblk *pblk, struct nvm_rq *rqd,
 306                             unsigned int bio_init_idx,
 307                             unsigned long *read_bitmap,
 308                             int nr_holes)
 309 {
 310         void *meta_list = rqd->meta_list;
 311         struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
 312         struct pblk_pr_ctx *pr_ctx;
 313         struct bio *new_bio, *bio = r_ctx->private;
 314         int nr_secs = rqd->nr_ppas;
 315         int i;
 316
 317         new_bio = bio_alloc(GFP_KERNEL, nr_holes);
 318
 319         if (pblk_bio_add_pages(pblk, new_bio, GFP_KERNEL, nr_holes))
 320                 goto fail_bio_put;
 321
 322         if (nr_holes != new_bio->bi_vcnt) {
 323                 WARN_ONCE(1, "pblk: malformed bio\n");
 324                 goto fail_free_pages;
 325         }
 326
 327         pr_ctx = kzalloc(sizeof(struct pblk_pr_ctx), GFP_KERNEL);
 328         if (!pr_ctx)
 329                 goto fail_free_pages;
 330
 331         for (i = 0; i < nr_secs; i++) {
 332                 struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
 333
 334                 pr_ctx->lba_list_mem[i] = le64_to_cpu(meta->lba);
 335         }
 336
 337         new_bio->bi_iter.bi_sector = 0; /* internal bio */
 338         bio_set_op_attrs(new_bio, REQ_OP_READ, 0);
 339
 340         rqd->bio = new_bio;
 341         rqd->nr_ppas = nr_holes;
 342
 343         pr_ctx->orig_bio = bio;
 344         bitmap_copy(pr_ctx->bitmap, read_bitmap, NVM_MAX_VLBA);
 345         pr_ctx->bio_init_idx = bio_init_idx;
 346         pr_ctx->orig_nr_secs = nr_secs;
 347         r_ctx->private = pr_ctx;
 348
 349         if (unlikely(nr_holes == 1)) {
 350                 pr_ctx->ppa_ptr = rqd->ppa_list;
 351                 pr_ctx->dma_ppa_list = rqd->dma_ppa_list;
 352                 rqd->ppa_addr = rqd->ppa_list[0];
 353         }
 354         return 0;
 355
 356 fail_free_pages:
 357         pblk_bio_free_pages(pblk, new_bio, 0, new_bio->bi_vcnt);
 358 fail_bio_put:
 359         bio_put(new_bio);
 360
 361         return -ENOMEM;
 362 }
 363
 364 static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
 365                                  unsigned int bio_init_idx,
 366                                  unsigned long *read_bitmap, int nr_secs)
 367 {
 368         int nr_holes;
 369         int ret;
 370
 371         nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs);
 372
 373         if (pblk_setup_partial_read(pblk, rqd, bio_init_idx, read_bitmap,
 374                                     nr_holes))
 375                 return NVM_IO_ERR;
 376
 377         rqd->end_io = pblk_end_partial_read;
 378
 379         ret = pblk_submit_io(pblk, rqd);
 380         if (ret) {
 381                 bio_put(rqd->bio);
 382                 pblk_err(pblk, "partial read IO submission failed\n");
 383                 goto err;
 384         }
 385
 386         return NVM_IO_OK;
 387
 388 err:
 389         pblk_err(pblk, "failed to perform partial read\n");
 390
 391         /* Free allocated pages in new bio */
 392         pblk_bio_free_pages(pblk, rqd->bio, 0, rqd->bio->bi_vcnt);
 393         __pblk_end_io_read(pblk, rqd, false);
 394         return NVM_IO_ERR;
 395 }
 396
 397 static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio,
 398                          sector_t lba, unsigned long *read_bitmap)
 399 {
 400         struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0);
 401         struct ppa_addr ppa;
 402
 403         pblk_lookup_l2p_seq(pblk, &ppa, lba, 1);
 404
 405 #ifdef CONFIG_NVM_PBLK_DEBUG
 406         atomic_long_inc(&pblk->inflight_reads);
 407 #endif
 408
 409 retry:
 410         if (pblk_ppa_empty(ppa)) {
 411                 __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
 412
 413                 WARN_ON(test_and_set_bit(0, read_bitmap));
 414                 meta->lba = addr_empty;
 415                 return;
 416         }
 417
 418         /* Try to read from write buffer. The address is later checked on the
 419          * write buffer to prevent retrieving overwritten data.
 420          */
 421         if (pblk_addr_in_cache(ppa)) {
 422                 if (!pblk_read_from_cache(pblk, bio, lba, ppa, 0, 1)) {
 423                         pblk_lookup_l2p_seq(pblk, &ppa, lba, 1);
 424                         goto retry;
 425                 }
 426
 427                 WARN_ON(test_and_set_bit(0, read_bitmap));
 428                 meta->lba = cpu_to_le64(lba);
 429
 430 #ifdef CONFIG_NVM_PBLK_DEBUG
 431                 atomic_long_inc(&pblk->cache_reads);
 432 #endif
 433         } else {
 434                 rqd->ppa_addr = ppa;
 435         }
 436 }
 437
 438 int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 439 {
 440         struct nvm_tgt_dev *dev = pblk->dev;
 441         struct request_queue *q = dev->q;
 442         sector_t blba = pblk_get_lba(bio);
 443         unsigned int nr_secs = pblk_get_secs(bio);
 444         struct pblk_g_ctx *r_ctx;
 445         struct nvm_rq *rqd;
 446         unsigned int bio_init_idx;
 447         DECLARE_BITMAP(read_bitmap, NVM_MAX_VLBA);
 448         int ret = NVM_IO_ERR;
 449
 450         generic_start_io_acct(q, REQ_OP_READ, bio_sectors(bio),
 451                               &pblk->disk->part0);
 452
 453         bitmap_zero(read_bitmap, nr_secs);
 454
 455         rqd = pblk_alloc_rqd(pblk, PBLK_READ);
 456
 457         rqd->opcode = NVM_OP_PREAD;
 458         rqd->nr_ppas = nr_secs;
 459         rqd->bio = NULL; /* cloned bio if needed */
 460         rqd->private = pblk;
 461         rqd->end_io = pblk_end_io_read;
 462
 463         r_ctx = nvm_rq_to_pdu(rqd);
 464         r_ctx->start_time = jiffies;
 465         r_ctx->lba = blba;
 466         r_ctx->private = bio; /* original bio */
 467
 468         /* Save the index for this bio's start. This is needed in case
 469          * we need to fill a partial read.
 470          */
 471         bio_init_idx = pblk_get_bi_idx(bio);
 472
 473         if (pblk_alloc_rqd_meta(pblk, rqd))
 474                 goto fail_rqd_free;
 475
 476         if (nr_secs > 1)
 477                 pblk_read_ppalist_rq(pblk, rqd, bio, blba, read_bitmap);
 478         else
 479                 pblk_read_rq(pblk, rqd, bio, blba, read_bitmap);
 480
 481         if (bitmap_full(read_bitmap, nr_secs)) {
 482                 atomic_inc(&pblk->inflight_io);
 483                 __pblk_end_io_read(pblk, rqd, false);
 484                 return NVM_IO_DONE;
 485         }
 486
 487         /* All sectors are to be read from the device */
 488         if (bitmap_empty(read_bitmap, rqd->nr_ppas)) {
 489                 struct bio *int_bio = NULL;
 490
 491                 /* Clone read bio to deal with read errors internally */
 492                 int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
 493                 if (!int_bio) {
 494                         pblk_err(pblk, "could not clone read bio\n");
 495                         goto fail_end_io;
 496                 }
 497
 498                 rqd->bio = int_bio;
 499
 500                 if (pblk_submit_io(pblk, rqd)) {
 501                         pblk_err(pblk, "read IO submission failed\n");
 502                         ret = NVM_IO_ERR;
 503                         goto fail_end_io;
 504                 }
 505
 506                 return NVM_IO_OK;
 507         }
 508
 509         /* The read bio request could be partially filled by the write buffer,
 510          * but there are some holes that need to be read from the drive.
 511          */
 512         ret = pblk_partial_read_bio(pblk, rqd, bio_init_idx, read_bitmap,
 513                                     nr_secs);
 514         if (ret)
 515                 goto fail_meta_free;
 516
 517         return NVM_IO_OK;
 518
 519 fail_meta_free:
 520         nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
 521 fail_rqd_free:
 522         pblk_free_rqd(pblk, rqd, PBLK_READ);
 523         return ret;
 524 fail_end_io:
 525         __pblk_end_io_read(pblk, rqd, false);
 526         return ret;
 527 }
 528
 529 static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
 530                               struct pblk_line *line, u64 *lba_list,
 531                               u64 *paddr_list_gc, unsigned int nr_secs)
 532 {
 533         struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA];
 534         struct ppa_addr ppa_gc;
 535         int valid_secs = 0;
 536         int i;
 537
 538         pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);
 539
 540         for (i = 0; i < nr_secs; i++) {
 541                 if (lba_list[i] == ADDR_EMPTY)
 542                         continue;
 543
 544                 ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);
 545                 if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) {
 546                         paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY;
 547                         continue;
 548                 }
 549
 550                 rqd->ppa_list[valid_secs++] = ppa_list_l2p[i];
 551         }
 552
 553 #ifdef CONFIG_NVM_PBLK_DEBUG
 554         atomic_long_add(valid_secs, &pblk->inflight_reads);
 555 #endif
 556
 557         return valid_secs;
 558 }
 559
 560 static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
 561                       struct pblk_line *line, sector_t lba,
 562                       u64 paddr_gc)
 563 {
 564         struct ppa_addr ppa_l2p, ppa_gc;
 565         int valid_secs = 0;
 566
 567         if (lba == ADDR_EMPTY)
 568                 goto out;
 569
 570         /* logic error: lba out-of-bounds */
 571         if (lba >= pblk->rl.nr_secs) {
 572                 WARN(1, "pblk: read lba out of bounds\n");
 573                 goto out;
 574         }
 575
 576         spin_lock(&pblk->trans_lock);
 577         ppa_l2p = pblk_trans_map_get(pblk, lba);
 578         spin_unlock(&pblk->trans_lock);
 579
 580         ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);
 581         if (!pblk_ppa_comp(ppa_l2p, ppa_gc))
 582                 goto out;
 583
 584         rqd->ppa_addr = ppa_l2p;
 585         valid_secs = 1;
 586
 587 #ifdef CONFIG_NVM_PBLK_DEBUG
 588         atomic_long_inc(&pblk->inflight_reads);
 589 #endif
 590
 591 out:
 592         return valid_secs;
 593 }
 594
 595 int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
 596 {
 597         struct nvm_tgt_dev *dev = pblk->dev;
 598         struct nvm_geo *geo = &dev->geo;
 599         struct bio *bio;
 600         struct nvm_rq rqd;
 601         int data_len;
 602         int ret = NVM_IO_OK;
 603
 604         memset(&rqd, 0, sizeof(struct nvm_rq));
 605
 606         ret = pblk_alloc_rqd_meta(pblk, &rqd);
 607         if (ret)
 608                 return ret;
 609
 610         if (gc_rq->nr_secs > 1) {
 611                 gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,
 612                                                         gc_rq->lba_list,
 613                                                         gc_rq->paddr_list,
 614                                                         gc_rq->nr_secs);
 615                 if (gc_rq->secs_to_gc == 1)
 616                         rqd.ppa_addr = rqd.ppa_list[0];
 617         } else {
 618                 gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,
 619                                                         gc_rq->lba_list[0],
 620                                                         gc_rq->paddr_list[0]);
 621         }
 622
 623         if (!(gc_rq->secs_to_gc))
 624                 goto out;
 625
 626         data_len = (gc_rq->secs_to_gc) * geo->csecs;
 627         bio = pblk_bio_map_addr(pblk, gc_rq->data, gc_rq->secs_to_gc, data_len,
 628                                                 PBLK_VMALLOC_META, GFP_KERNEL);
 629         if (IS_ERR(bio)) {
 630                 pblk_err(pblk, "could not allocate GC bio (%lu)\n",
 631                                                                 PTR_ERR(bio));
 632                 ret = PTR_ERR(bio);
 633                 goto err_free_dma;
 634         }
 635
 636         bio->bi_iter.bi_sector = 0; /* internal bio */
 637         bio_set_op_attrs(bio, REQ_OP_READ, 0);
 638
 639         rqd.opcode = NVM_OP_PREAD;
 640         rqd.nr_ppas = gc_rq->secs_to_gc;
 641         rqd.bio = bio;
 642
 643         if (pblk_submit_io_sync(pblk, &rqd)) {
 644                 ret = -EIO;
 645                 pblk_err(pblk, "GC read request failed\n");
 646                 goto err_free_bio;
 647         }
 648
 649         pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);
 650
 651         atomic_dec(&pblk->inflight_io);
 652
 653         if (rqd.error) {
 654                 atomic_long_inc(&pblk->read_failed_gc);
 655 #ifdef CONFIG_NVM_PBLK_DEBUG
 656                 pblk_print_failed_rqd(pblk, &rqd, rqd.error);
 657 #endif
 658         }
 659
 660 #ifdef CONFIG_NVM_PBLK_DEBUG
 661         atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);
 662         atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);
 663         atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);
 664 #endif
 665
 666 out:
 667         pblk_free_rqd_meta(pblk, &rqd);
 668         return ret;
 669
 670 err_free_bio:
 671         bio_put(bio);
 672 err_free_dma:
 673         pblk_free_rqd_meta(pblk, &rqd);
 674         return ret;
 675 }