Merge branch 'pm-pci'
[linux-2.6-microblaze.git] / mm / page_io.c
1 /*
2  *  linux/mm/page_io.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, 
7  *  Asynchronous swapping added 30.12.95. Stephen Tweedie
8  *  Removed race in async swapping. 14.4.1996. Bruno Haible
9  *  Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
10  *  Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
11  */
12
13 #include <linux/mm.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/gfp.h>
16 #include <linux/pagemap.h>
17 #include <linux/swap.h>
18 #include <linux/bio.h>
19 #include <linux/swapops.h>
20 #include <linux/buffer_head.h>
21 #include <linux/writeback.h>
22 #include <linux/frontswap.h>
23 #include <linux/blkdev.h>
24 #include <linux/uio.h>
25 #include <asm/pgtable.h>
26
27 static struct bio *get_swap_bio(gfp_t gfp_flags,
28                                 struct page *page, bio_end_io_t end_io)
29 {
30         struct bio *bio;
31
32         bio = bio_alloc(gfp_flags, 1);
33         if (bio) {
34                 bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev);
35                 bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
36                 bio->bi_end_io = end_io;
37
38                 bio_add_page(bio, page, PAGE_SIZE, 0);
39                 BUG_ON(bio->bi_iter.bi_size != PAGE_SIZE);
40         }
41         return bio;
42 }
43
44 void end_swap_bio_write(struct bio *bio)
45 {
46         struct page *page = bio->bi_io_vec[0].bv_page;
47
48         if (bio->bi_status) {
49                 SetPageError(page);
50                 /*
51                  * We failed to write the page out to swap-space.
52                  * Re-dirty the page in order to avoid it being reclaimed.
53                  * Also print a dire warning that things will go BAD (tm)
54                  * very quickly.
55                  *
56                  * Also clear PG_reclaim to avoid rotate_reclaimable_page()
57                  */
58                 set_page_dirty(page);
59                 pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
60                          imajor(bio->bi_bdev->bd_inode),
61                          iminor(bio->bi_bdev->bd_inode),
62                          (unsigned long long)bio->bi_iter.bi_sector);
63                 ClearPageReclaim(page);
64         }
65         end_page_writeback(page);
66         bio_put(bio);
67 }
68
69 static void swap_slot_free_notify(struct page *page)
70 {
71         struct swap_info_struct *sis;
72         struct gendisk *disk;
73
74         /*
75          * There is no guarantee that the page is in swap cache - the software
76          * suspend code (at least) uses end_swap_bio_read() against a non-
77          * swapcache page.  So we must check PG_swapcache before proceeding with
78          * this optimization.
79          */
80         if (unlikely(!PageSwapCache(page)))
81                 return;
82
83         sis = page_swap_info(page);
84         if (!(sis->flags & SWP_BLKDEV))
85                 return;
86
87         /*
88          * The swap subsystem performs lazy swap slot freeing,
89          * expecting that the page will be swapped out again.
90          * So we can avoid an unnecessary write if the page
91          * isn't redirtied.
92          * This is good for real swap storage because we can
93          * reduce unnecessary I/O and enhance wear-leveling
94          * if an SSD is used as the as swap device.
95          * But if in-memory swap device (eg zram) is used,
96          * this causes a duplicated copy between uncompressed
97          * data in VM-owned memory and compressed data in
98          * zram-owned memory.  So let's free zram-owned memory
99          * and make the VM-owned decompressed page *dirty*,
100          * so the page should be swapped out somewhere again if
101          * we again wish to reclaim it.
102          */
103         disk = sis->bdev->bd_disk;
104         if (disk->fops->swap_slot_free_notify) {
105                 swp_entry_t entry;
106                 unsigned long offset;
107
108                 entry.val = page_private(page);
109                 offset = swp_offset(entry);
110
111                 SetPageDirty(page);
112                 disk->fops->swap_slot_free_notify(sis->bdev,
113                                 offset);
114         }
115 }
116
117 static void end_swap_bio_read(struct bio *bio)
118 {
119         struct page *page = bio->bi_io_vec[0].bv_page;
120
121         if (bio->bi_status) {
122                 SetPageError(page);
123                 ClearPageUptodate(page);
124                 pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
125                          imajor(bio->bi_bdev->bd_inode),
126                          iminor(bio->bi_bdev->bd_inode),
127                          (unsigned long long)bio->bi_iter.bi_sector);
128                 goto out;
129         }
130
131         SetPageUptodate(page);
132         swap_slot_free_notify(page);
133 out:
134         unlock_page(page);
135         bio_put(bio);
136 }
137
138 int generic_swapfile_activate(struct swap_info_struct *sis,
139                                 struct file *swap_file,
140                                 sector_t *span)
141 {
142         struct address_space *mapping = swap_file->f_mapping;
143         struct inode *inode = mapping->host;
144         unsigned blocks_per_page;
145         unsigned long page_no;
146         unsigned blkbits;
147         sector_t probe_block;
148         sector_t last_block;
149         sector_t lowest_block = -1;
150         sector_t highest_block = 0;
151         int nr_extents = 0;
152         int ret;
153
154         blkbits = inode->i_blkbits;
155         blocks_per_page = PAGE_SIZE >> blkbits;
156
157         /*
158          * Map all the blocks into the extent list.  This code doesn't try
159          * to be very smart.
160          */
161         probe_block = 0;
162         page_no = 0;
163         last_block = i_size_read(inode) >> blkbits;
164         while ((probe_block + blocks_per_page) <= last_block &&
165                         page_no < sis->max) {
166                 unsigned block_in_page;
167                 sector_t first_block;
168
169                 cond_resched();
170
171                 first_block = bmap(inode, probe_block);
172                 if (first_block == 0)
173                         goto bad_bmap;
174
175                 /*
176                  * It must be PAGE_SIZE aligned on-disk
177                  */
178                 if (first_block & (blocks_per_page - 1)) {
179                         probe_block++;
180                         goto reprobe;
181                 }
182
183                 for (block_in_page = 1; block_in_page < blocks_per_page;
184                                         block_in_page++) {
185                         sector_t block;
186
187                         block = bmap(inode, probe_block + block_in_page);
188                         if (block == 0)
189                                 goto bad_bmap;
190                         if (block != first_block + block_in_page) {
191                                 /* Discontiguity */
192                                 probe_block++;
193                                 goto reprobe;
194                         }
195                 }
196
197                 first_block >>= (PAGE_SHIFT - blkbits);
198                 if (page_no) {  /* exclude the header page */
199                         if (first_block < lowest_block)
200                                 lowest_block = first_block;
201                         if (first_block > highest_block)
202                                 highest_block = first_block;
203                 }
204
205                 /*
206                  * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
207                  */
208                 ret = add_swap_extent(sis, page_no, 1, first_block);
209                 if (ret < 0)
210                         goto out;
211                 nr_extents += ret;
212                 page_no++;
213                 probe_block += blocks_per_page;
214 reprobe:
215                 continue;
216         }
217         ret = nr_extents;
218         *span = 1 + highest_block - lowest_block;
219         if (page_no == 0)
220                 page_no = 1;    /* force Empty message */
221         sis->max = page_no;
222         sis->pages = page_no - 1;
223         sis->highest_bit = page_no - 1;
224 out:
225         return ret;
226 bad_bmap:
227         pr_err("swapon: swapfile has holes\n");
228         ret = -EINVAL;
229         goto out;
230 }
231
232 /*
233  * We may have stale swap cache pages in memory: notice
234  * them here and get rid of the unnecessary final write.
235  */
236 int swap_writepage(struct page *page, struct writeback_control *wbc)
237 {
238         int ret = 0;
239
240         if (try_to_free_swap(page)) {
241                 unlock_page(page);
242                 goto out;
243         }
244         if (frontswap_store(page) == 0) {
245                 set_page_writeback(page);
246                 unlock_page(page);
247                 end_page_writeback(page);
248                 goto out;
249         }
250         ret = __swap_writepage(page, wbc, end_swap_bio_write);
251 out:
252         return ret;
253 }
254
255 static sector_t swap_page_sector(struct page *page)
256 {
257         return (sector_t)__page_file_index(page) << (PAGE_SHIFT - 9);
258 }
259
260 int __swap_writepage(struct page *page, struct writeback_control *wbc,
261                 bio_end_io_t end_write_func)
262 {
263         struct bio *bio;
264         int ret;
265         struct swap_info_struct *sis = page_swap_info(page);
266
267         VM_BUG_ON_PAGE(!PageSwapCache(page), page);
268         if (sis->flags & SWP_FILE) {
269                 struct kiocb kiocb;
270                 struct file *swap_file = sis->swap_file;
271                 struct address_space *mapping = swap_file->f_mapping;
272                 struct bio_vec bv = {
273                         .bv_page = page,
274                         .bv_len  = PAGE_SIZE,
275                         .bv_offset = 0
276                 };
277                 struct iov_iter from;
278
279                 iov_iter_bvec(&from, ITER_BVEC | WRITE, &bv, 1, PAGE_SIZE);
280                 init_sync_kiocb(&kiocb, swap_file);
281                 kiocb.ki_pos = page_file_offset(page);
282
283                 set_page_writeback(page);
284                 unlock_page(page);
285                 ret = mapping->a_ops->direct_IO(&kiocb, &from);
286                 if (ret == PAGE_SIZE) {
287                         count_vm_event(PSWPOUT);
288                         ret = 0;
289                 } else {
290                         /*
291                          * In the case of swap-over-nfs, this can be a
292                          * temporary failure if the system has limited
293                          * memory for allocating transmit buffers.
294                          * Mark the page dirty and avoid
295                          * rotate_reclaimable_page but rate-limit the
296                          * messages but do not flag PageError like
297                          * the normal direct-to-bio case as it could
298                          * be temporary.
299                          */
300                         set_page_dirty(page);
301                         ClearPageReclaim(page);
302                         pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
303                                            page_file_offset(page));
304                 }
305                 end_page_writeback(page);
306                 return ret;
307         }
308
309         ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
310         if (!ret) {
311                 count_vm_event(PSWPOUT);
312                 return 0;
313         }
314
315         ret = 0;
316         bio = get_swap_bio(GFP_NOIO, page, end_write_func);
317         if (bio == NULL) {
318                 set_page_dirty(page);
319                 unlock_page(page);
320                 ret = -ENOMEM;
321                 goto out;
322         }
323         bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
324         count_vm_event(PSWPOUT);
325         set_page_writeback(page);
326         unlock_page(page);
327         submit_bio(bio);
328 out:
329         return ret;
330 }
331
332 int swap_readpage(struct page *page)
333 {
334         struct bio *bio;
335         int ret = 0;
336         struct swap_info_struct *sis = page_swap_info(page);
337
338         VM_BUG_ON_PAGE(!PageSwapCache(page), page);
339         VM_BUG_ON_PAGE(!PageLocked(page), page);
340         VM_BUG_ON_PAGE(PageUptodate(page), page);
341         if (frontswap_load(page) == 0) {
342                 SetPageUptodate(page);
343                 unlock_page(page);
344                 goto out;
345         }
346
347         if (sis->flags & SWP_FILE) {
348                 struct file *swap_file = sis->swap_file;
349                 struct address_space *mapping = swap_file->f_mapping;
350
351                 ret = mapping->a_ops->readpage(swap_file, page);
352                 if (!ret)
353                         count_vm_event(PSWPIN);
354                 return ret;
355         }
356
357         ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
358         if (!ret) {
359                 if (trylock_page(page)) {
360                         swap_slot_free_notify(page);
361                         unlock_page(page);
362                 }
363
364                 count_vm_event(PSWPIN);
365                 return 0;
366         }
367
368         ret = 0;
369         bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);
370         if (bio == NULL) {
371                 unlock_page(page);
372                 ret = -ENOMEM;
373                 goto out;
374         }
375         bio_set_op_attrs(bio, REQ_OP_READ, 0);
376         count_vm_event(PSWPIN);
377         submit_bio(bio);
378 out:
379         return ret;
380 }
381
382 int swap_set_page_dirty(struct page *page)
383 {
384         struct swap_info_struct *sis = page_swap_info(page);
385
386         if (sis->flags & SWP_FILE) {
387                 struct address_space *mapping = sis->swap_file->f_mapping;
388
389                 VM_BUG_ON_PAGE(!PageSwapCache(page), page);
390                 return mapping->a_ops->set_page_dirty(page);
391         } else {
392                 return __set_page_dirty_no_writeback(page);
393         }
394 }