Merge branch 'vmwgfx-fixes-5.1' of git://people.freedesktop.org/~thomash/linux into...
[linux-2.6-microblaze.git] / drivers / lightnvm / pblk-rb.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2016 CNEX Labs
4  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
5  *
6  * Based upon the circular ringbuffer.
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License version
10  * 2 as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License for more details.
16  *
17  * pblk-rb.c - pblk's write buffer
18  */
19
20 #include <linux/circ_buf.h>
21
22 #include "pblk.h"
23
24 static DECLARE_RWSEM(pblk_rb_lock);
25
26 static void pblk_rb_data_free(struct pblk_rb *rb)
27 {
28         struct pblk_rb_pages *p, *t;
29
30         down_write(&pblk_rb_lock);
31         list_for_each_entry_safe(p, t, &rb->pages, list) {
32                 free_pages((unsigned long)page_address(p->pages), p->order);
33                 list_del(&p->list);
34                 kfree(p);
35         }
36         up_write(&pblk_rb_lock);
37 }
38
39 void pblk_rb_free(struct pblk_rb *rb)
40 {
41         pblk_rb_data_free(rb);
42         vfree(rb->entries);
43 }
44
45 /*
46  * pblk_rb_calculate_size -- calculate the size of the write buffer
47  */
48 static unsigned int pblk_rb_calculate_size(unsigned int nr_entries,
49                                            unsigned int threshold)
50 {
51         unsigned int thr_sz = 1 << (get_count_order(threshold + NVM_MAX_VLBA));
52         unsigned int max_sz = max(thr_sz, nr_entries);
53         unsigned int max_io;
54
55         /* Alloc a write buffer that can (i) fit at least two split bios
56          * (considering max I/O size NVM_MAX_VLBA, and (ii) guarantee that the
57          * threshold will be respected
58          */
59         max_io = (1 << max((int)(get_count_order(max_sz)),
60                                 (int)(get_count_order(NVM_MAX_VLBA << 1))));
61         if ((threshold + NVM_MAX_VLBA) >= max_io)
62                 max_io <<= 1;
63
64         return max_io;
65 }
66
67 /*
68  * Initialize ring buffer. The data and metadata buffers must be previously
69  * allocated and their size must be a power of two
70  * (Documentation/core-api/circular-buffers.rst)
71  */
72 int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold,
73                  unsigned int seg_size)
74 {
75         struct pblk *pblk = container_of(rb, struct pblk, rwb);
76         struct pblk_rb_entry *entries;
77         unsigned int init_entry = 0;
78         unsigned int max_order = MAX_ORDER - 1;
79         unsigned int power_size, power_seg_sz;
80         unsigned int alloc_order, order, iter;
81         unsigned int nr_entries;
82
83         nr_entries = pblk_rb_calculate_size(size, threshold);
84         entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry)));
85         if (!entries)
86                 return -ENOMEM;
87
88         power_size = get_count_order(nr_entries);
89         power_seg_sz = get_count_order(seg_size);
90
91         down_write(&pblk_rb_lock);
92         rb->entries = entries;
93         rb->seg_size = (1 << power_seg_sz);
94         rb->nr_entries = (1 << power_size);
95         rb->mem = rb->subm = rb->sync = rb->l2p_update = 0;
96         rb->back_thres = threshold;
97         rb->flush_point = EMPTY_ENTRY;
98
99         spin_lock_init(&rb->w_lock);
100         spin_lock_init(&rb->s_lock);
101
102         INIT_LIST_HEAD(&rb->pages);
103
104         alloc_order = power_size;
105         if (alloc_order >= max_order) {
106                 order = max_order;
107                 iter = (1 << (alloc_order - max_order));
108         } else {
109                 order = alloc_order;
110                 iter = 1;
111         }
112
113         do {
114                 struct pblk_rb_entry *entry;
115                 struct pblk_rb_pages *page_set;
116                 void *kaddr;
117                 unsigned long set_size;
118                 int i;
119
120                 page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL);
121                 if (!page_set) {
122                         up_write(&pblk_rb_lock);
123                         vfree(entries);
124                         return -ENOMEM;
125                 }
126
127                 page_set->order = order;
128                 page_set->pages = alloc_pages(GFP_KERNEL, order);
129                 if (!page_set->pages) {
130                         kfree(page_set);
131                         pblk_rb_data_free(rb);
132                         up_write(&pblk_rb_lock);
133                         vfree(entries);
134                         return -ENOMEM;
135                 }
136                 kaddr = page_address(page_set->pages);
137
138                 entry = &rb->entries[init_entry];
139                 entry->data = kaddr;
140                 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
141                 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
142
143                 set_size = (1 << order);
144                 for (i = 1; i < set_size; i++) {
145                         entry = &rb->entries[init_entry];
146                         entry->cacheline = pblk_cacheline_to_addr(init_entry++);
147                         entry->data = kaddr + (i * rb->seg_size);
148                         entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
149                         bio_list_init(&entry->w_ctx.bios);
150                 }
151
152                 list_add_tail(&page_set->list, &rb->pages);
153                 iter--;
154         } while (iter > 0);
155         up_write(&pblk_rb_lock);
156
157 #ifdef CONFIG_NVM_PBLK_DEBUG
158         atomic_set(&rb->inflight_flush_point, 0);
159 #endif
160
161         /*
162          * Initialize rate-limiter, which controls access to the write buffer
163          * by user and GC I/O
164          */
165         pblk_rl_init(&pblk->rl, rb->nr_entries, threshold);
166
167         return 0;
168 }
169
170 static void clean_wctx(struct pblk_w_ctx *w_ctx)
171 {
172         int flags;
173
174         flags = READ_ONCE(w_ctx->flags);
175         WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY),
176                         "pblk: overwriting unsubmitted data\n");
177
178         /* Release flags on context. Protect from writes and reads */
179         smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY);
180         pblk_ppa_set_empty(&w_ctx->ppa);
181         w_ctx->lba = ADDR_EMPTY;
182 }
183
184 #define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size)
185 #define pblk_rb_ring_space(rb, head, tail, size) \
186                                         (CIRC_SPACE(head, tail, size))
187
188 /*
189  * Buffer space is calculated with respect to the back pointer signaling
190  * synchronized entries to the media.
191  */
192 static unsigned int pblk_rb_space(struct pblk_rb *rb)
193 {
194         unsigned int mem = READ_ONCE(rb->mem);
195         unsigned int sync = READ_ONCE(rb->sync);
196
197         return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries);
198 }
199
200 unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p,
201                               unsigned int nr_entries)
202 {
203         return (p + nr_entries) & (rb->nr_entries - 1);
204 }
205
206 /*
207  * Buffer count is calculated with respect to the submission entry signaling the
208  * entries that are available to send to the media
209  */
210 unsigned int pblk_rb_read_count(struct pblk_rb *rb)
211 {
212         unsigned int mem = READ_ONCE(rb->mem);
213         unsigned int subm = READ_ONCE(rb->subm);
214
215         return pblk_rb_ring_count(mem, subm, rb->nr_entries);
216 }
217
218 unsigned int pblk_rb_sync_count(struct pblk_rb *rb)
219 {
220         unsigned int mem = READ_ONCE(rb->mem);
221         unsigned int sync = READ_ONCE(rb->sync);
222
223         return pblk_rb_ring_count(mem, sync, rb->nr_entries);
224 }
225
226 unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries)
227 {
228         unsigned int subm;
229
230         subm = READ_ONCE(rb->subm);
231         /* Commit read means updating submission pointer */
232         smp_store_release(&rb->subm, pblk_rb_ptr_wrap(rb, subm, nr_entries));
233
234         return subm;
235 }
236
237 static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update)
238 {
239         struct pblk *pblk = container_of(rb, struct pblk, rwb);
240         struct pblk_line *line;
241         struct pblk_rb_entry *entry;
242         struct pblk_w_ctx *w_ctx;
243         unsigned int user_io = 0, gc_io = 0;
244         unsigned int i;
245         int flags;
246
247         for (i = 0; i < to_update; i++) {
248                 entry = &rb->entries[rb->l2p_update];
249                 w_ctx = &entry->w_ctx;
250
251                 flags = READ_ONCE(entry->w_ctx.flags);
252                 if (flags & PBLK_IOTYPE_USER)
253                         user_io++;
254                 else if (flags & PBLK_IOTYPE_GC)
255                         gc_io++;
256                 else
257                         WARN(1, "pblk: unknown IO type\n");
258
259                 pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa,
260                                                         entry->cacheline);
261
262                 line = pblk_ppa_to_line(pblk, w_ctx->ppa);
263                 atomic_dec(&line->sec_to_update);
264                 kref_put(&line->ref, pblk_line_put);
265                 clean_wctx(w_ctx);
266                 rb->l2p_update = pblk_rb_ptr_wrap(rb, rb->l2p_update, 1);
267         }
268
269         pblk_rl_out(&pblk->rl, user_io, gc_io);
270
271         return 0;
272 }
273
274 /*
275  * When we move the l2p_update pointer, we update the l2p table - lookups will
276  * point to the physical address instead of to the cacheline in the write buffer
277  * from this moment on.
278  */
279 static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries,
280                               unsigned int mem, unsigned int sync)
281 {
282         unsigned int space, count;
283         int ret = 0;
284
285         lockdep_assert_held(&rb->w_lock);
286
287         /* Update l2p only as buffer entries are being overwritten */
288         space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries);
289         if (space > nr_entries)
290                 goto out;
291
292         count = nr_entries - space;
293         /* l2p_update used exclusively under rb->w_lock */
294         ret = __pblk_rb_update_l2p(rb, count);
295
296 out:
297         return ret;
298 }
299
300 /*
301  * Update the l2p entry for all sectors stored on the write buffer. This means
302  * that all future lookups to the l2p table will point to a device address, not
303  * to the cacheline in the write buffer.
304  */
305 void pblk_rb_sync_l2p(struct pblk_rb *rb)
306 {
307         unsigned int sync;
308         unsigned int to_update;
309
310         spin_lock(&rb->w_lock);
311
312         /* Protect from reads and writes */
313         sync = smp_load_acquire(&rb->sync);
314
315         to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries);
316         __pblk_rb_update_l2p(rb, to_update);
317
318         spin_unlock(&rb->w_lock);
319 }
320
321 /*
322  * Write @nr_entries to ring buffer from @data buffer if there is enough space.
323  * Typically, 4KB data chunks coming from a bio will be copied to the ring
324  * buffer, thus the write will fail if not all incoming data can be copied.
325  *
326  */
327 static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data,
328                                   struct pblk_w_ctx w_ctx,
329                                   struct pblk_rb_entry *entry)
330 {
331         memcpy(entry->data, data, rb->seg_size);
332
333         entry->w_ctx.lba = w_ctx.lba;
334         entry->w_ctx.ppa = w_ctx.ppa;
335 }
336
337 void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data,
338                               struct pblk_w_ctx w_ctx, unsigned int ring_pos)
339 {
340         struct pblk *pblk = container_of(rb, struct pblk, rwb);
341         struct pblk_rb_entry *entry;
342         int flags;
343
344         entry = &rb->entries[ring_pos];
345         flags = READ_ONCE(entry->w_ctx.flags);
346 #ifdef CONFIG_NVM_PBLK_DEBUG
347         /* Caller must guarantee that the entry is free */
348         BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
349 #endif
350
351         __pblk_rb_write_entry(rb, data, w_ctx, entry);
352
353         pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline);
354         flags = w_ctx.flags | PBLK_WRITTEN_DATA;
355
356         /* Release flags on write context. Protect from writes */
357         smp_store_release(&entry->w_ctx.flags, flags);
358 }
359
360 void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
361                             struct pblk_w_ctx w_ctx, struct pblk_line *line,
362                             u64 paddr, unsigned int ring_pos)
363 {
364         struct pblk *pblk = container_of(rb, struct pblk, rwb);
365         struct pblk_rb_entry *entry;
366         int flags;
367
368         entry = &rb->entries[ring_pos];
369         flags = READ_ONCE(entry->w_ctx.flags);
370 #ifdef CONFIG_NVM_PBLK_DEBUG
371         /* Caller must guarantee that the entry is free */
372         BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
373 #endif
374
375         __pblk_rb_write_entry(rb, data, w_ctx, entry);
376
377         if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr))
378                 entry->w_ctx.lba = ADDR_EMPTY;
379
380         flags = w_ctx.flags | PBLK_WRITTEN_DATA;
381
382         /* Release flags on write context. Protect from writes */
383         smp_store_release(&entry->w_ctx.flags, flags);
384 }
385
386 static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
387                                    unsigned int pos)
388 {
389         struct pblk_rb_entry *entry;
390         unsigned int sync, flush_point;
391
392         pblk_rb_sync_init(rb, NULL);
393         sync = READ_ONCE(rb->sync);
394
395         if (pos == sync) {
396                 pblk_rb_sync_end(rb, NULL);
397                 return 0;
398         }
399
400 #ifdef CONFIG_NVM_PBLK_DEBUG
401         atomic_inc(&rb->inflight_flush_point);
402 #endif
403
404         flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
405         entry = &rb->entries[flush_point];
406
407         /* Protect flush points */
408         smp_store_release(&rb->flush_point, flush_point);
409
410         if (bio)
411                 bio_list_add(&entry->w_ctx.bios, bio);
412
413         pblk_rb_sync_end(rb, NULL);
414
415         return bio ? 1 : 0;
416 }
417
418 static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
419                                unsigned int *pos)
420 {
421         unsigned int mem;
422         unsigned int sync;
423         unsigned int threshold;
424
425         sync = READ_ONCE(rb->sync);
426         mem = READ_ONCE(rb->mem);
427
428         threshold = nr_entries + rb->back_thres;
429
430         if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < threshold)
431                 return 0;
432
433         if (pblk_rb_update_l2p(rb, nr_entries, mem, sync))
434                 return 0;
435
436         *pos = mem;
437
438         return 1;
439 }
440
441 static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
442                              unsigned int *pos)
443 {
444         if (!__pblk_rb_may_write(rb, nr_entries, pos))
445                 return 0;
446
447         /* Protect from read count */
448         smp_store_release(&rb->mem, pblk_rb_ptr_wrap(rb, *pos, nr_entries));
449         return 1;
450 }
451
452 void pblk_rb_flush(struct pblk_rb *rb)
453 {
454         struct pblk *pblk = container_of(rb, struct pblk, rwb);
455         unsigned int mem = READ_ONCE(rb->mem);
456
457         if (pblk_rb_flush_point_set(rb, NULL, mem))
458                 return;
459
460         pblk_write_kick(pblk);
461 }
462
463 static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries,
464                                    unsigned int *pos, struct bio *bio,
465                                    int *io_ret)
466 {
467         unsigned int mem;
468
469         if (!__pblk_rb_may_write(rb, nr_entries, pos))
470                 return 0;
471
472         mem = pblk_rb_ptr_wrap(rb, *pos, nr_entries);
473         *io_ret = NVM_IO_DONE;
474
475         if (bio->bi_opf & REQ_PREFLUSH) {
476                 struct pblk *pblk = container_of(rb, struct pblk, rwb);
477
478                 atomic64_inc(&pblk->nr_flush);
479                 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem))
480                         *io_ret = NVM_IO_OK;
481         }
482
483         /* Protect from read count */
484         smp_store_release(&rb->mem, mem);
485
486         return 1;
487 }
488
489 /*
490  * Atomically check that (i) there is space on the write buffer for the
491  * incoming I/O, and (ii) the current I/O type has enough budget in the write
492  * buffer (rate-limiter).
493  */
494 int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio,
495                            unsigned int nr_entries, unsigned int *pos)
496 {
497         struct pblk *pblk = container_of(rb, struct pblk, rwb);
498         int io_ret;
499
500         spin_lock(&rb->w_lock);
501         io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries);
502         if (io_ret) {
503                 spin_unlock(&rb->w_lock);
504                 return io_ret;
505         }
506
507         if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) {
508                 spin_unlock(&rb->w_lock);
509                 return NVM_IO_REQUEUE;
510         }
511
512         pblk_rl_user_in(&pblk->rl, nr_entries);
513         spin_unlock(&rb->w_lock);
514
515         return io_ret;
516 }
517
518 /*
519  * Look at pblk_rb_may_write_user comment
520  */
521 int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries,
522                          unsigned int *pos)
523 {
524         struct pblk *pblk = container_of(rb, struct pblk, rwb);
525
526         spin_lock(&rb->w_lock);
527         if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) {
528                 spin_unlock(&rb->w_lock);
529                 return 0;
530         }
531
532         if (!pblk_rb_may_write(rb, nr_entries, pos)) {
533                 spin_unlock(&rb->w_lock);
534                 return 0;
535         }
536
537         pblk_rl_gc_in(&pblk->rl, nr_entries);
538         spin_unlock(&rb->w_lock);
539
540         return 1;
541 }
542
543 /*
544  * Read available entries on rb and add them to the given bio. To avoid a memory
545  * copy, a page reference to the write buffer is used to be added to the bio.
546  *
547  * This function is used by the write thread to form the write bio that will
548  * persist data on the write buffer to the media.
549  */
550 unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
551                                  unsigned int pos, unsigned int nr_entries,
552                                  unsigned int count)
553 {
554         struct pblk *pblk = container_of(rb, struct pblk, rwb);
555         struct request_queue *q = pblk->dev->q;
556         struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
557         struct bio *bio = rqd->bio;
558         struct pblk_rb_entry *entry;
559         struct page *page;
560         unsigned int pad = 0, to_read = nr_entries;
561         unsigned int i;
562         int flags;
563
564         if (count < nr_entries) {
565                 pad = nr_entries - count;
566                 to_read = count;
567         }
568
569         /* Add space for packed metadata if in use*/
570         pad += (pblk->min_write_pgs - pblk->min_write_pgs_data);
571
572         c_ctx->sentry = pos;
573         c_ctx->nr_valid = to_read;
574         c_ctx->nr_padded = pad;
575
576         for (i = 0; i < to_read; i++) {
577                 entry = &rb->entries[pos];
578
579                 /* A write has been allowed into the buffer, but data is still
580                  * being copied to it. It is ok to busy wait.
581                  */
582 try:
583                 flags = READ_ONCE(entry->w_ctx.flags);
584                 if (!(flags & PBLK_WRITTEN_DATA)) {
585                         io_schedule();
586                         goto try;
587                 }
588
589                 page = virt_to_page(entry->data);
590                 if (!page) {
591                         pblk_err(pblk, "could not allocate write bio page\n");
592                         flags &= ~PBLK_WRITTEN_DATA;
593                         flags |= PBLK_SUBMITTED_ENTRY;
594                         /* Release flags on context. Protect from writes */
595                         smp_store_release(&entry->w_ctx.flags, flags);
596                         return NVM_IO_ERR;
597                 }
598
599                 if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) !=
600                                                                 rb->seg_size) {
601                         pblk_err(pblk, "could not add page to write bio\n");
602                         flags &= ~PBLK_WRITTEN_DATA;
603                         flags |= PBLK_SUBMITTED_ENTRY;
604                         /* Release flags on context. Protect from writes */
605                         smp_store_release(&entry->w_ctx.flags, flags);
606                         return NVM_IO_ERR;
607                 }
608
609                 flags &= ~PBLK_WRITTEN_DATA;
610                 flags |= PBLK_SUBMITTED_ENTRY;
611
612                 /* Release flags on context. Protect from writes */
613                 smp_store_release(&entry->w_ctx.flags, flags);
614
615                 pos = pblk_rb_ptr_wrap(rb, pos, 1);
616         }
617
618         if (pad) {
619                 if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) {
620                         pblk_err(pblk, "could not pad page in write bio\n");
621                         return NVM_IO_ERR;
622                 }
623
624                 if (pad < pblk->min_write_pgs)
625                         atomic64_inc(&pblk->pad_dist[pad - 1]);
626                 else
627                         pblk_warn(pblk, "padding more than min. sectors\n");
628
629                 atomic64_add(pad, &pblk->pad_wa);
630         }
631
632 #ifdef CONFIG_NVM_PBLK_DEBUG
633         atomic_long_add(pad, &pblk->padded_writes);
634 #endif
635
636         return NVM_IO_OK;
637 }
638
639 /*
640  * Copy to bio only if the lba matches the one on the given cache entry.
641  * Otherwise, it means that the entry has been overwritten, and the bio should
642  * be directed to disk.
643  */
644 int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba,
645                         struct ppa_addr ppa, int bio_iter, bool advanced_bio)
646 {
647         struct pblk *pblk = container_of(rb, struct pblk, rwb);
648         struct pblk_rb_entry *entry;
649         struct pblk_w_ctx *w_ctx;
650         struct ppa_addr l2p_ppa;
651         u64 pos = pblk_addr_to_cacheline(ppa);
652         void *data;
653         int flags;
654         int ret = 1;
655
656
657 #ifdef CONFIG_NVM_PBLK_DEBUG
658         /* Caller must ensure that the access will not cause an overflow */
659         BUG_ON(pos >= rb->nr_entries);
660 #endif
661         entry = &rb->entries[pos];
662         w_ctx = &entry->w_ctx;
663         flags = READ_ONCE(w_ctx->flags);
664
665         spin_lock(&rb->w_lock);
666         spin_lock(&pblk->trans_lock);
667         l2p_ppa = pblk_trans_map_get(pblk, lba);
668         spin_unlock(&pblk->trans_lock);
669
670         /* Check if the entry has been overwritten or is scheduled to be */
671         if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba ||
672                                                 flags & PBLK_WRITABLE_ENTRY) {
673                 ret = 0;
674                 goto out;
675         }
676
677         /* Only advance the bio if it hasn't been advanced already. If advanced,
678          * this bio is at least a partial bio (i.e., it has partially been
679          * filled with data from the cache). If part of the data resides on the
680          * media, we will read later on
681          */
682         if (unlikely(!advanced_bio))
683                 bio_advance(bio, bio_iter * PBLK_EXPOSED_PAGE_SIZE);
684
685         data = bio_data(bio);
686         memcpy(data, entry->data, rb->seg_size);
687
688 out:
689         spin_unlock(&rb->w_lock);
690         return ret;
691 }
692
693 struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos)
694 {
695         unsigned int entry = pblk_rb_ptr_wrap(rb, pos, 0);
696
697         return &rb->entries[entry].w_ctx;
698 }
699
700 unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags)
701         __acquires(&rb->s_lock)
702 {
703         if (flags)
704                 spin_lock_irqsave(&rb->s_lock, *flags);
705         else
706                 spin_lock_irq(&rb->s_lock);
707
708         return rb->sync;
709 }
710
711 void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
712         __releases(&rb->s_lock)
713 {
714         lockdep_assert_held(&rb->s_lock);
715
716         if (flags)
717                 spin_unlock_irqrestore(&rb->s_lock, *flags);
718         else
719                 spin_unlock_irq(&rb->s_lock);
720 }
721
722 unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
723 {
724         unsigned int sync, flush_point;
725         lockdep_assert_held(&rb->s_lock);
726
727         sync = READ_ONCE(rb->sync);
728         flush_point = READ_ONCE(rb->flush_point);
729
730         if (flush_point != EMPTY_ENTRY) {
731                 unsigned int secs_to_flush;
732
733                 secs_to_flush = pblk_rb_ring_count(flush_point, sync,
734                                         rb->nr_entries);
735                 if (secs_to_flush < nr_entries) {
736                         /* Protect flush points */
737                         smp_store_release(&rb->flush_point, EMPTY_ENTRY);
738                 }
739         }
740
741         sync = pblk_rb_ptr_wrap(rb, sync, nr_entries);
742
743         /* Protect from counts */
744         smp_store_release(&rb->sync, sync);
745
746         return sync;
747 }
748
749 /* Calculate how many sectors to submit up to the current flush point. */
750 unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
751 {
752         unsigned int subm, sync, flush_point;
753         unsigned int submitted, to_flush;
754
755         /* Protect flush points */
756         flush_point = smp_load_acquire(&rb->flush_point);
757         if (flush_point == EMPTY_ENTRY)
758                 return 0;
759
760         /* Protect syncs */
761         sync = smp_load_acquire(&rb->sync);
762
763         subm = READ_ONCE(rb->subm);
764         submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
765
766         /* The sync point itself counts as a sector to sync */
767         to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
768
769         return (submitted < to_flush) ? (to_flush - submitted) : 0;
770 }
771
772 int pblk_rb_tear_down_check(struct pblk_rb *rb)
773 {
774         struct pblk_rb_entry *entry;
775         int i;
776         int ret = 0;
777
778         spin_lock(&rb->w_lock);
779         spin_lock_irq(&rb->s_lock);
780
781         if ((rb->mem == rb->subm) && (rb->subm == rb->sync) &&
782                                 (rb->sync == rb->l2p_update) &&
783                                 (rb->flush_point == EMPTY_ENTRY)) {
784                 goto out;
785         }
786
787         if (!rb->entries) {
788                 ret = 1;
789                 goto out;
790         }
791
792         for (i = 0; i < rb->nr_entries; i++) {
793                 entry = &rb->entries[i];
794
795                 if (!entry->data) {
796                         ret = 1;
797                         goto out;
798                 }
799         }
800
801 out:
802         spin_unlock(&rb->w_lock);
803         spin_unlock_irq(&rb->s_lock);
804
805         return ret;
806 }
807
808 unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos)
809 {
810         return (pos & (rb->nr_entries - 1));
811 }
812
813 int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos)
814 {
815         return (pos >= rb->nr_entries);
816 }
817
818 ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
819 {
820         struct pblk *pblk = container_of(rb, struct pblk, rwb);
821         struct pblk_c_ctx *c;
822         ssize_t offset;
823         int queued_entries = 0;
824
825         spin_lock_irq(&rb->s_lock);
826         list_for_each_entry(c, &pblk->compl_list, list)
827                 queued_entries++;
828         spin_unlock_irq(&rb->s_lock);
829
830         if (rb->flush_point != EMPTY_ENTRY)
831                 offset = scnprintf(buf, PAGE_SIZE,
832                         "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
833                         rb->nr_entries,
834                         rb->mem,
835                         rb->subm,
836                         rb->sync,
837                         rb->l2p_update,
838 #ifdef CONFIG_NVM_PBLK_DEBUG
839                         atomic_read(&rb->inflight_flush_point),
840 #else
841                         0,
842 #endif
843                         rb->flush_point,
844                         pblk_rb_read_count(rb),
845                         pblk_rb_space(rb),
846                         pblk_rb_flush_point_count(rb),
847                         queued_entries);
848         else
849                 offset = scnprintf(buf, PAGE_SIZE,
850                         "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n",
851                         rb->nr_entries,
852                         rb->mem,
853                         rb->subm,
854                         rb->sync,
855                         rb->l2p_update,
856 #ifdef CONFIG_NVM_PBLK_DEBUG
857                         atomic_read(&rb->inflight_flush_point),
858 #else
859                         0,
860 #endif
861                         pblk_rb_read_count(rb),
862                         pblk_rb_space(rb),
863                         pblk_rb_flush_point_count(rb),
864                         queued_entries);
865
866         return offset;
867 }