printk: introduce dump_stack_lvl()
[linux-2.6-microblaze.git] / lib / iov_iter.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <crypto/hash.h>
3 #include <linux/export.h>
4 #include <linux/bvec.h>
5 #include <linux/fault-inject-usercopy.h>
6 #include <linux/uio.h>
7 #include <linux/pagemap.h>
8 #include <linux/highmem.h>
9 #include <linux/slab.h>
10 #include <linux/vmalloc.h>
11 #include <linux/splice.h>
12 #include <linux/compat.h>
13 #include <net/checksum.h>
14 #include <linux/scatterlist.h>
15 #include <linux/instrumented.h>
16
17 #define PIPE_PARANOIA /* for now */
18
19 #define iterate_iovec(i, n, __v, __p, skip, STEP) {     \
20         size_t left;                                    \
21         size_t wanted = n;                              \
22         __p = i->iov;                                   \
23         __v.iov_len = min(n, __p->iov_len - skip);      \
24         if (likely(__v.iov_len)) {                      \
25                 __v.iov_base = __p->iov_base + skip;    \
26                 left = (STEP);                          \
27                 __v.iov_len -= left;                    \
28                 skip += __v.iov_len;                    \
29                 n -= __v.iov_len;                       \
30         } else {                                        \
31                 left = 0;                               \
32         }                                               \
33         while (unlikely(!left && n)) {                  \
34                 __p++;                                  \
35                 __v.iov_len = min(n, __p->iov_len);     \
36                 if (unlikely(!__v.iov_len))             \
37                         continue;                       \
38                 __v.iov_base = __p->iov_base;           \
39                 left = (STEP);                          \
40                 __v.iov_len -= left;                    \
41                 skip = __v.iov_len;                     \
42                 n -= __v.iov_len;                       \
43         }                                               \
44         n = wanted - n;                                 \
45 }
46
47 #define iterate_kvec(i, n, __v, __p, skip, STEP) {      \
48         size_t wanted = n;                              \
49         __p = i->kvec;                                  \
50         __v.iov_len = min(n, __p->iov_len - skip);      \
51         if (likely(__v.iov_len)) {                      \
52                 __v.iov_base = __p->iov_base + skip;    \
53                 (void)(STEP);                           \
54                 skip += __v.iov_len;                    \
55                 n -= __v.iov_len;                       \
56         }                                               \
57         while (unlikely(n)) {                           \
58                 __p++;                                  \
59                 __v.iov_len = min(n, __p->iov_len);     \
60                 if (unlikely(!__v.iov_len))             \
61                         continue;                       \
62                 __v.iov_base = __p->iov_base;           \
63                 (void)(STEP);                           \
64                 skip = __v.iov_len;                     \
65                 n -= __v.iov_len;                       \
66         }                                               \
67         n = wanted;                                     \
68 }
69
70 #define iterate_bvec(i, n, __v, __bi, skip, STEP) {     \
71         struct bvec_iter __start;                       \
72         __start.bi_size = n;                            \
73         __start.bi_bvec_done = skip;                    \
74         __start.bi_idx = 0;                             \
75         for_each_bvec(__v, i->bvec, __bi, __start) {    \
76                 (void)(STEP);                           \
77         }                                               \
78 }
79
80 #define iterate_xarray(i, n, __v, skip, STEP) {         \
81         struct page *head = NULL;                               \
82         size_t wanted = n, seg, offset;                         \
83         loff_t start = i->xarray_start + skip;                  \
84         pgoff_t index = start >> PAGE_SHIFT;                    \
85         int j;                                                  \
86                                                                 \
87         XA_STATE(xas, i->xarray, index);                        \
88                                                                 \
89         rcu_read_lock();                                                \
90         xas_for_each(&xas, head, ULONG_MAX) {                           \
91                 if (xas_retry(&xas, head))                              \
92                         continue;                                       \
93                 if (WARN_ON(xa_is_value(head)))                         \
94                         break;                                          \
95                 if (WARN_ON(PageHuge(head)))                            \
96                         break;                                          \
97                 for (j = (head->index < index) ? index - head->index : 0; \
98                      j < thp_nr_pages(head); j++) {                     \
99                         __v.bv_page = head + j;                         \
100                         offset = (i->xarray_start + skip) & ~PAGE_MASK; \
101                         seg = PAGE_SIZE - offset;                       \
102                         __v.bv_offset = offset;                         \
103                         __v.bv_len = min(n, seg);                       \
104                         (void)(STEP);                                   \
105                         n -= __v.bv_len;                                \
106                         skip += __v.bv_len;                             \
107                         if (n == 0)                                     \
108                                 break;                                  \
109                 }                                                       \
110                 if (n == 0)                                             \
111                         break;                                          \
112         }                                                       \
113         rcu_read_unlock();                                      \
114         n = wanted - n;                                         \
115 }
116
117 #define iterate_all_kinds(i, n, v, I, B, K, X) {                \
118         if (likely(n)) {                                        \
119                 size_t skip = i->iov_offset;                    \
120                 if (unlikely(i->type & ITER_BVEC)) {            \
121                         struct bio_vec v;                       \
122                         struct bvec_iter __bi;                  \
123                         iterate_bvec(i, n, v, __bi, skip, (B))  \
124                 } else if (unlikely(i->type & ITER_KVEC)) {     \
125                         const struct kvec *kvec;                \
126                         struct kvec v;                          \
127                         iterate_kvec(i, n, v, kvec, skip, (K))  \
128                 } else if (unlikely(i->type & ITER_DISCARD)) {  \
129                 } else if (unlikely(i->type & ITER_XARRAY)) {   \
130                         struct bio_vec v;                       \
131                         iterate_xarray(i, n, v, skip, (X));     \
132                 } else {                                        \
133                         const struct iovec *iov;                \
134                         struct iovec v;                         \
135                         iterate_iovec(i, n, v, iov, skip, (I))  \
136                 }                                               \
137         }                                                       \
138 }
139
140 #define iterate_and_advance(i, n, v, I, B, K, X) {              \
141         if (unlikely(i->count < n))                             \
142                 n = i->count;                                   \
143         if (i->count) {                                         \
144                 size_t skip = i->iov_offset;                    \
145                 if (unlikely(i->type & ITER_BVEC)) {            \
146                         const struct bio_vec *bvec = i->bvec;   \
147                         struct bio_vec v;                       \
148                         struct bvec_iter __bi;                  \
149                         iterate_bvec(i, n, v, __bi, skip, (B))  \
150                         i->bvec = __bvec_iter_bvec(i->bvec, __bi);      \
151                         i->nr_segs -= i->bvec - bvec;           \
152                         skip = __bi.bi_bvec_done;               \
153                 } else if (unlikely(i->type & ITER_KVEC)) {     \
154                         const struct kvec *kvec;                \
155                         struct kvec v;                          \
156                         iterate_kvec(i, n, v, kvec, skip, (K))  \
157                         if (skip == kvec->iov_len) {            \
158                                 kvec++;                         \
159                                 skip = 0;                       \
160                         }                                       \
161                         i->nr_segs -= kvec - i->kvec;           \
162                         i->kvec = kvec;                         \
163                 } else if (unlikely(i->type & ITER_DISCARD)) {  \
164                         skip += n;                              \
165                 } else if (unlikely(i->type & ITER_XARRAY)) {   \
166                         struct bio_vec v;                       \
167                         iterate_xarray(i, n, v, skip, (X))      \
168                 } else {                                        \
169                         const struct iovec *iov;                \
170                         struct iovec v;                         \
171                         iterate_iovec(i, n, v, iov, skip, (I))  \
172                         if (skip == iov->iov_len) {             \
173                                 iov++;                          \
174                                 skip = 0;                       \
175                         }                                       \
176                         i->nr_segs -= iov - i->iov;             \
177                         i->iov = iov;                           \
178                 }                                               \
179                 i->count -= n;                                  \
180                 i->iov_offset = skip;                           \
181         }                                                       \
182 }
183
184 static int copyout(void __user *to, const void *from, size_t n)
185 {
186         if (should_fail_usercopy())
187                 return n;
188         if (access_ok(to, n)) {
189                 instrument_copy_to_user(to, from, n);
190                 n = raw_copy_to_user(to, from, n);
191         }
192         return n;
193 }
194
195 static int copyin(void *to, const void __user *from, size_t n)
196 {
197         if (should_fail_usercopy())
198                 return n;
199         if (access_ok(from, n)) {
200                 instrument_copy_from_user(to, from, n);
201                 n = raw_copy_from_user(to, from, n);
202         }
203         return n;
204 }
205
206 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
207                          struct iov_iter *i)
208 {
209         size_t skip, copy, left, wanted;
210         const struct iovec *iov;
211         char __user *buf;
212         void *kaddr, *from;
213
214         if (unlikely(bytes > i->count))
215                 bytes = i->count;
216
217         if (unlikely(!bytes))
218                 return 0;
219
220         might_fault();
221         wanted = bytes;
222         iov = i->iov;
223         skip = i->iov_offset;
224         buf = iov->iov_base + skip;
225         copy = min(bytes, iov->iov_len - skip);
226
227         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
228                 kaddr = kmap_atomic(page);
229                 from = kaddr + offset;
230
231                 /* first chunk, usually the only one */
232                 left = copyout(buf, from, copy);
233                 copy -= left;
234                 skip += copy;
235                 from += copy;
236                 bytes -= copy;
237
238                 while (unlikely(!left && bytes)) {
239                         iov++;
240                         buf = iov->iov_base;
241                         copy = min(bytes, iov->iov_len);
242                         left = copyout(buf, from, copy);
243                         copy -= left;
244                         skip = copy;
245                         from += copy;
246                         bytes -= copy;
247                 }
248                 if (likely(!bytes)) {
249                         kunmap_atomic(kaddr);
250                         goto done;
251                 }
252                 offset = from - kaddr;
253                 buf += copy;
254                 kunmap_atomic(kaddr);
255                 copy = min(bytes, iov->iov_len - skip);
256         }
257         /* Too bad - revert to non-atomic kmap */
258
259         kaddr = kmap(page);
260         from = kaddr + offset;
261         left = copyout(buf, from, copy);
262         copy -= left;
263         skip += copy;
264         from += copy;
265         bytes -= copy;
266         while (unlikely(!left && bytes)) {
267                 iov++;
268                 buf = iov->iov_base;
269                 copy = min(bytes, iov->iov_len);
270                 left = copyout(buf, from, copy);
271                 copy -= left;
272                 skip = copy;
273                 from += copy;
274                 bytes -= copy;
275         }
276         kunmap(page);
277
278 done:
279         if (skip == iov->iov_len) {
280                 iov++;
281                 skip = 0;
282         }
283         i->count -= wanted - bytes;
284         i->nr_segs -= iov - i->iov;
285         i->iov = iov;
286         i->iov_offset = skip;
287         return wanted - bytes;
288 }
289
290 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
291                          struct iov_iter *i)
292 {
293         size_t skip, copy, left, wanted;
294         const struct iovec *iov;
295         char __user *buf;
296         void *kaddr, *to;
297
298         if (unlikely(bytes > i->count))
299                 bytes = i->count;
300
301         if (unlikely(!bytes))
302                 return 0;
303
304         might_fault();
305         wanted = bytes;
306         iov = i->iov;
307         skip = i->iov_offset;
308         buf = iov->iov_base + skip;
309         copy = min(bytes, iov->iov_len - skip);
310
311         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
312                 kaddr = kmap_atomic(page);
313                 to = kaddr + offset;
314
315                 /* first chunk, usually the only one */
316                 left = copyin(to, buf, copy);
317                 copy -= left;
318                 skip += copy;
319                 to += copy;
320                 bytes -= copy;
321
322                 while (unlikely(!left && bytes)) {
323                         iov++;
324                         buf = iov->iov_base;
325                         copy = min(bytes, iov->iov_len);
326                         left = copyin(to, buf, copy);
327                         copy -= left;
328                         skip = copy;
329                         to += copy;
330                         bytes -= copy;
331                 }
332                 if (likely(!bytes)) {
333                         kunmap_atomic(kaddr);
334                         goto done;
335                 }
336                 offset = to - kaddr;
337                 buf += copy;
338                 kunmap_atomic(kaddr);
339                 copy = min(bytes, iov->iov_len - skip);
340         }
341         /* Too bad - revert to non-atomic kmap */
342
343         kaddr = kmap(page);
344         to = kaddr + offset;
345         left = copyin(to, buf, copy);
346         copy -= left;
347         skip += copy;
348         to += copy;
349         bytes -= copy;
350         while (unlikely(!left && bytes)) {
351                 iov++;
352                 buf = iov->iov_base;
353                 copy = min(bytes, iov->iov_len);
354                 left = copyin(to, buf, copy);
355                 copy -= left;
356                 skip = copy;
357                 to += copy;
358                 bytes -= copy;
359         }
360         kunmap(page);
361
362 done:
363         if (skip == iov->iov_len) {
364                 iov++;
365                 skip = 0;
366         }
367         i->count -= wanted - bytes;
368         i->nr_segs -= iov - i->iov;
369         i->iov = iov;
370         i->iov_offset = skip;
371         return wanted - bytes;
372 }
373
374 #ifdef PIPE_PARANOIA
375 static bool sanity(const struct iov_iter *i)
376 {
377         struct pipe_inode_info *pipe = i->pipe;
378         unsigned int p_head = pipe->head;
379         unsigned int p_tail = pipe->tail;
380         unsigned int p_mask = pipe->ring_size - 1;
381         unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
382         unsigned int i_head = i->head;
383         unsigned int idx;
384
385         if (i->iov_offset) {
386                 struct pipe_buffer *p;
387                 if (unlikely(p_occupancy == 0))
388                         goto Bad;       // pipe must be non-empty
389                 if (unlikely(i_head != p_head - 1))
390                         goto Bad;       // must be at the last buffer...
391
392                 p = &pipe->bufs[i_head & p_mask];
393                 if (unlikely(p->offset + p->len != i->iov_offset))
394                         goto Bad;       // ... at the end of segment
395         } else {
396                 if (i_head != p_head)
397                         goto Bad;       // must be right after the last buffer
398         }
399         return true;
400 Bad:
401         printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
402         printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
403                         p_head, p_tail, pipe->ring_size);
404         for (idx = 0; idx < pipe->ring_size; idx++)
405                 printk(KERN_ERR "[%p %p %d %d]\n",
406                         pipe->bufs[idx].ops,
407                         pipe->bufs[idx].page,
408                         pipe->bufs[idx].offset,
409                         pipe->bufs[idx].len);
410         WARN_ON(1);
411         return false;
412 }
413 #else
414 #define sanity(i) true
415 #endif
416
417 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
418                          struct iov_iter *i)
419 {
420         struct pipe_inode_info *pipe = i->pipe;
421         struct pipe_buffer *buf;
422         unsigned int p_tail = pipe->tail;
423         unsigned int p_mask = pipe->ring_size - 1;
424         unsigned int i_head = i->head;
425         size_t off;
426
427         if (unlikely(bytes > i->count))
428                 bytes = i->count;
429
430         if (unlikely(!bytes))
431                 return 0;
432
433         if (!sanity(i))
434                 return 0;
435
436         off = i->iov_offset;
437         buf = &pipe->bufs[i_head & p_mask];
438         if (off) {
439                 if (offset == off && buf->page == page) {
440                         /* merge with the last one */
441                         buf->len += bytes;
442                         i->iov_offset += bytes;
443                         goto out;
444                 }
445                 i_head++;
446                 buf = &pipe->bufs[i_head & p_mask];
447         }
448         if (pipe_full(i_head, p_tail, pipe->max_usage))
449                 return 0;
450
451         buf->ops = &page_cache_pipe_buf_ops;
452         get_page(page);
453         buf->page = page;
454         buf->offset = offset;
455         buf->len = bytes;
456
457         pipe->head = i_head + 1;
458         i->iov_offset = offset + bytes;
459         i->head = i_head;
460 out:
461         i->count -= bytes;
462         return bytes;
463 }
464
465 /*
466  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
467  * bytes.  For each iovec, fault in each page that constitutes the iovec.
468  *
469  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
470  * because it is an invalid address).
471  */
472 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
473 {
474         size_t skip = i->iov_offset;
475         const struct iovec *iov;
476         int err;
477         struct iovec v;
478
479         if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
480                 iterate_iovec(i, bytes, v, iov, skip, ({
481                         err = fault_in_pages_readable(v.iov_base, v.iov_len);
482                         if (unlikely(err))
483                         return err;
484                 0;}))
485         }
486         return 0;
487 }
488 EXPORT_SYMBOL(iov_iter_fault_in_readable);
489
490 void iov_iter_init(struct iov_iter *i, unsigned int direction,
491                         const struct iovec *iov, unsigned long nr_segs,
492                         size_t count)
493 {
494         WARN_ON(direction & ~(READ | WRITE));
495         direction &= READ | WRITE;
496
497         /* It will get better.  Eventually... */
498         if (uaccess_kernel()) {
499                 i->type = ITER_KVEC | direction;
500                 i->kvec = (struct kvec *)iov;
501         } else {
502                 i->type = ITER_IOVEC | direction;
503                 i->iov = iov;
504         }
505         i->nr_segs = nr_segs;
506         i->iov_offset = 0;
507         i->count = count;
508 }
509 EXPORT_SYMBOL(iov_iter_init);
510
511 static inline bool allocated(struct pipe_buffer *buf)
512 {
513         return buf->ops == &default_pipe_buf_ops;
514 }
515
516 static inline void data_start(const struct iov_iter *i,
517                               unsigned int *iter_headp, size_t *offp)
518 {
519         unsigned int p_mask = i->pipe->ring_size - 1;
520         unsigned int iter_head = i->head;
521         size_t off = i->iov_offset;
522
523         if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
524                     off == PAGE_SIZE)) {
525                 iter_head++;
526                 off = 0;
527         }
528         *iter_headp = iter_head;
529         *offp = off;
530 }
531
532 static size_t push_pipe(struct iov_iter *i, size_t size,
533                         int *iter_headp, size_t *offp)
534 {
535         struct pipe_inode_info *pipe = i->pipe;
536         unsigned int p_tail = pipe->tail;
537         unsigned int p_mask = pipe->ring_size - 1;
538         unsigned int iter_head;
539         size_t off;
540         ssize_t left;
541
542         if (unlikely(size > i->count))
543                 size = i->count;
544         if (unlikely(!size))
545                 return 0;
546
547         left = size;
548         data_start(i, &iter_head, &off);
549         *iter_headp = iter_head;
550         *offp = off;
551         if (off) {
552                 left -= PAGE_SIZE - off;
553                 if (left <= 0) {
554                         pipe->bufs[iter_head & p_mask].len += size;
555                         return size;
556                 }
557                 pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
558                 iter_head++;
559         }
560         while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
561                 struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
562                 struct page *page = alloc_page(GFP_USER);
563                 if (!page)
564                         break;
565
566                 buf->ops = &default_pipe_buf_ops;
567                 buf->page = page;
568                 buf->offset = 0;
569                 buf->len = min_t(ssize_t, left, PAGE_SIZE);
570                 left -= buf->len;
571                 iter_head++;
572                 pipe->head = iter_head;
573
574                 if (left == 0)
575                         return size;
576         }
577         return size - left;
578 }
579
580 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
581                                 struct iov_iter *i)
582 {
583         struct pipe_inode_info *pipe = i->pipe;
584         unsigned int p_mask = pipe->ring_size - 1;
585         unsigned int i_head;
586         size_t n, off;
587
588         if (!sanity(i))
589                 return 0;
590
591         bytes = n = push_pipe(i, bytes, &i_head, &off);
592         if (unlikely(!n))
593                 return 0;
594         do {
595                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
596                 memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
597                 i->head = i_head;
598                 i->iov_offset = off + chunk;
599                 n -= chunk;
600                 addr += chunk;
601                 off = 0;
602                 i_head++;
603         } while (n);
604         i->count -= bytes;
605         return bytes;
606 }
607
608 static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
609                               __wsum sum, size_t off)
610 {
611         __wsum next = csum_partial_copy_nocheck(from, to, len);
612         return csum_block_add(sum, next, off);
613 }
614
615 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
616                                          struct csum_state *csstate,
617                                          struct iov_iter *i)
618 {
619         struct pipe_inode_info *pipe = i->pipe;
620         unsigned int p_mask = pipe->ring_size - 1;
621         __wsum sum = csstate->csum;
622         size_t off = csstate->off;
623         unsigned int i_head;
624         size_t n, r;
625
626         if (!sanity(i))
627                 return 0;
628
629         bytes = n = push_pipe(i, bytes, &i_head, &r);
630         if (unlikely(!n))
631                 return 0;
632         do {
633                 size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
634                 char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
635                 sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
636                 kunmap_atomic(p);
637                 i->head = i_head;
638                 i->iov_offset = r + chunk;
639                 n -= chunk;
640                 off += chunk;
641                 addr += chunk;
642                 r = 0;
643                 i_head++;
644         } while (n);
645         i->count -= bytes;
646         csstate->csum = sum;
647         csstate->off = off;
648         return bytes;
649 }
650
651 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
652 {
653         const char *from = addr;
654         if (unlikely(iov_iter_is_pipe(i)))
655                 return copy_pipe_to_iter(addr, bytes, i);
656         if (iter_is_iovec(i))
657                 might_fault();
658         iterate_and_advance(i, bytes, v,
659                 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
660                 memcpy_to_page(v.bv_page, v.bv_offset,
661                                (from += v.bv_len) - v.bv_len, v.bv_len),
662                 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
663                 memcpy_to_page(v.bv_page, v.bv_offset,
664                                (from += v.bv_len) - v.bv_len, v.bv_len)
665         )
666
667         return bytes;
668 }
669 EXPORT_SYMBOL(_copy_to_iter);
670
671 #ifdef CONFIG_ARCH_HAS_COPY_MC
672 static int copyout_mc(void __user *to, const void *from, size_t n)
673 {
674         if (access_ok(to, n)) {
675                 instrument_copy_to_user(to, from, n);
676                 n = copy_mc_to_user((__force void *) to, from, n);
677         }
678         return n;
679 }
680
681 static unsigned long copy_mc_to_page(struct page *page, size_t offset,
682                 const char *from, size_t len)
683 {
684         unsigned long ret;
685         char *to;
686
687         to = kmap_atomic(page);
688         ret = copy_mc_to_kernel(to + offset, from, len);
689         kunmap_atomic(to);
690
691         return ret;
692 }
693
694 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
695                                 struct iov_iter *i)
696 {
697         struct pipe_inode_info *pipe = i->pipe;
698         unsigned int p_mask = pipe->ring_size - 1;
699         unsigned int i_head;
700         size_t n, off, xfer = 0;
701
702         if (!sanity(i))
703                 return 0;
704
705         bytes = n = push_pipe(i, bytes, &i_head, &off);
706         if (unlikely(!n))
707                 return 0;
708         do {
709                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
710                 unsigned long rem;
711
712                 rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
713                                             off, addr, chunk);
714                 i->head = i_head;
715                 i->iov_offset = off + chunk - rem;
716                 xfer += chunk - rem;
717                 if (rem)
718                         break;
719                 n -= chunk;
720                 addr += chunk;
721                 off = 0;
722                 i_head++;
723         } while (n);
724         i->count -= xfer;
725         return xfer;
726 }
727
728 /**
729  * _copy_mc_to_iter - copy to iter with source memory error exception handling
730  * @addr: source kernel address
731  * @bytes: total transfer length
732  * @iter: destination iterator
733  *
734  * The pmem driver deploys this for the dax operation
735  * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
736  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
737  * successfully copied.
738  *
739  * The main differences between this and typical _copy_to_iter().
740  *
741  * * Typical tail/residue handling after a fault retries the copy
742  *   byte-by-byte until the fault happens again. Re-triggering machine
743  *   checks is potentially fatal so the implementation uses source
744  *   alignment and poison alignment assumptions to avoid re-triggering
745  *   hardware exceptions.
746  *
747  * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
748  *   Compare to copy_to_iter() where only ITER_IOVEC attempts might return
749  *   a short copy.
750  */
751 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
752 {
753         const char *from = addr;
754         unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
755
756         if (unlikely(iov_iter_is_pipe(i)))
757                 return copy_mc_pipe_to_iter(addr, bytes, i);
758         if (iter_is_iovec(i))
759                 might_fault();
760         iterate_and_advance(i, bytes, v,
761                 copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
762                            v.iov_len),
763                 ({
764                 rem = copy_mc_to_page(v.bv_page, v.bv_offset,
765                                       (from += v.bv_len) - v.bv_len, v.bv_len);
766                 if (rem) {
767                         curr_addr = (unsigned long) from;
768                         bytes = curr_addr - s_addr - rem;
769                         return bytes;
770                 }
771                 }),
772                 ({
773                 rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
774                                         - v.iov_len, v.iov_len);
775                 if (rem) {
776                         curr_addr = (unsigned long) from;
777                         bytes = curr_addr - s_addr - rem;
778                         return bytes;
779                 }
780                 }),
781                 ({
782                 rem = copy_mc_to_page(v.bv_page, v.bv_offset,
783                                       (from += v.bv_len) - v.bv_len, v.bv_len);
784                 if (rem) {
785                         curr_addr = (unsigned long) from;
786                         bytes = curr_addr - s_addr - rem;
787                         rcu_read_unlock();
788                         i->iov_offset += bytes;
789                         i->count -= bytes;
790                         return bytes;
791                 }
792                 })
793         )
794
795         return bytes;
796 }
797 EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
798 #endif /* CONFIG_ARCH_HAS_COPY_MC */
799
800 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
801 {
802         char *to = addr;
803         if (unlikely(iov_iter_is_pipe(i))) {
804                 WARN_ON(1);
805                 return 0;
806         }
807         if (iter_is_iovec(i))
808                 might_fault();
809         iterate_and_advance(i, bytes, v,
810                 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
811                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
812                                  v.bv_offset, v.bv_len),
813                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
814                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
815                                  v.bv_offset, v.bv_len)
816         )
817
818         return bytes;
819 }
820 EXPORT_SYMBOL(_copy_from_iter);
821
822 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
823 {
824         char *to = addr;
825         if (unlikely(iov_iter_is_pipe(i))) {
826                 WARN_ON(1);
827                 return false;
828         }
829         if (unlikely(i->count < bytes))
830                 return false;
831
832         if (iter_is_iovec(i))
833                 might_fault();
834         iterate_all_kinds(i, bytes, v, ({
835                 if (copyin((to += v.iov_len) - v.iov_len,
836                                       v.iov_base, v.iov_len))
837                         return false;
838                 0;}),
839                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
840                                  v.bv_offset, v.bv_len),
841                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
842                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
843                                  v.bv_offset, v.bv_len)
844         )
845
846         iov_iter_advance(i, bytes);
847         return true;
848 }
849 EXPORT_SYMBOL(_copy_from_iter_full);
850
851 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
852 {
853         char *to = addr;
854         if (unlikely(iov_iter_is_pipe(i))) {
855                 WARN_ON(1);
856                 return 0;
857         }
858         iterate_and_advance(i, bytes, v,
859                 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
860                                          v.iov_base, v.iov_len),
861                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
862                                  v.bv_offset, v.bv_len),
863                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
864                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
865                                  v.bv_offset, v.bv_len)
866         )
867
868         return bytes;
869 }
870 EXPORT_SYMBOL(_copy_from_iter_nocache);
871
872 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
873 /**
874  * _copy_from_iter_flushcache - write destination through cpu cache
875  * @addr: destination kernel address
876  * @bytes: total transfer length
877  * @iter: source iterator
878  *
879  * The pmem driver arranges for filesystem-dax to use this facility via
880  * dax_copy_from_iter() for ensuring that writes to persistent memory
881  * are flushed through the CPU cache. It is differentiated from
882  * _copy_from_iter_nocache() in that guarantees all data is flushed for
883  * all iterator types. The _copy_from_iter_nocache() only attempts to
884  * bypass the cache for the ITER_IOVEC case, and on some archs may use
885  * instructions that strand dirty-data in the cache.
886  */
887 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
888 {
889         char *to = addr;
890         if (unlikely(iov_iter_is_pipe(i))) {
891                 WARN_ON(1);
892                 return 0;
893         }
894         iterate_and_advance(i, bytes, v,
895                 __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
896                                          v.iov_base, v.iov_len),
897                 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
898                                  v.bv_offset, v.bv_len),
899                 memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
900                         v.iov_len),
901                 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
902                                  v.bv_offset, v.bv_len)
903         )
904
905         return bytes;
906 }
907 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
908 #endif
909
910 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
911 {
912         char *to = addr;
913         if (unlikely(iov_iter_is_pipe(i))) {
914                 WARN_ON(1);
915                 return false;
916         }
917         if (unlikely(i->count < bytes))
918                 return false;
919         iterate_all_kinds(i, bytes, v, ({
920                 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
921                                              v.iov_base, v.iov_len))
922                         return false;
923                 0;}),
924                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
925                                  v.bv_offset, v.bv_len),
926                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
927                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
928                                  v.bv_offset, v.bv_len)
929         )
930
931         iov_iter_advance(i, bytes);
932         return true;
933 }
934 EXPORT_SYMBOL(_copy_from_iter_full_nocache);
935
936 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
937 {
938         struct page *head;
939         size_t v = n + offset;
940
941         /*
942          * The general case needs to access the page order in order
943          * to compute the page size.
944          * However, we mostly deal with order-0 pages and thus can
945          * avoid a possible cache line miss for requests that fit all
946          * page orders.
947          */
948         if (n <= v && v <= PAGE_SIZE)
949                 return true;
950
951         head = compound_head(page);
952         v += (page - head) << PAGE_SHIFT;
953
954         if (likely(n <= v && v <= (page_size(head))))
955                 return true;
956         WARN_ON(1);
957         return false;
958 }
959
960 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
961                          struct iov_iter *i)
962 {
963         if (unlikely(!page_copy_sane(page, offset, bytes)))
964                 return 0;
965         if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {
966                 void *kaddr = kmap_atomic(page);
967                 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
968                 kunmap_atomic(kaddr);
969                 return wanted;
970         } else if (unlikely(iov_iter_is_discard(i)))
971                 return bytes;
972         else if (likely(!iov_iter_is_pipe(i)))
973                 return copy_page_to_iter_iovec(page, offset, bytes, i);
974         else
975                 return copy_page_to_iter_pipe(page, offset, bytes, i);
976 }
977 EXPORT_SYMBOL(copy_page_to_iter);
978
979 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
980                          struct iov_iter *i)
981 {
982         if (unlikely(!page_copy_sane(page, offset, bytes)))
983                 return 0;
984         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
985                 WARN_ON(1);
986                 return 0;
987         }
988         if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {
989                 void *kaddr = kmap_atomic(page);
990                 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
991                 kunmap_atomic(kaddr);
992                 return wanted;
993         } else
994                 return copy_page_from_iter_iovec(page, offset, bytes, i);
995 }
996 EXPORT_SYMBOL(copy_page_from_iter);
997
998 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
999 {
1000         struct pipe_inode_info *pipe = i->pipe;
1001         unsigned int p_mask = pipe->ring_size - 1;
1002         unsigned int i_head;
1003         size_t n, off;
1004
1005         if (!sanity(i))
1006                 return 0;
1007
1008         bytes = n = push_pipe(i, bytes, &i_head, &off);
1009         if (unlikely(!n))
1010                 return 0;
1011
1012         do {
1013                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
1014                 memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
1015                 i->head = i_head;
1016                 i->iov_offset = off + chunk;
1017                 n -= chunk;
1018                 off = 0;
1019                 i_head++;
1020         } while (n);
1021         i->count -= bytes;
1022         return bytes;
1023 }
1024
1025 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
1026 {
1027         if (unlikely(iov_iter_is_pipe(i)))
1028                 return pipe_zero(bytes, i);
1029         iterate_and_advance(i, bytes, v,
1030                 clear_user(v.iov_base, v.iov_len),
1031                 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
1032                 memset(v.iov_base, 0, v.iov_len),
1033                 memzero_page(v.bv_page, v.bv_offset, v.bv_len)
1034         )
1035
1036         return bytes;
1037 }
1038 EXPORT_SYMBOL(iov_iter_zero);
1039
1040 size_t iov_iter_copy_from_user_atomic(struct page *page,
1041                 struct iov_iter *i, unsigned long offset, size_t bytes)
1042 {
1043         char *kaddr = kmap_atomic(page), *p = kaddr + offset;
1044         if (unlikely(!page_copy_sane(page, offset, bytes))) {
1045                 kunmap_atomic(kaddr);
1046                 return 0;
1047         }
1048         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1049                 kunmap_atomic(kaddr);
1050                 WARN_ON(1);
1051                 return 0;
1052         }
1053         iterate_all_kinds(i, bytes, v,
1054                 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
1055                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
1056                                  v.bv_offset, v.bv_len),
1057                 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
1058                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
1059                                  v.bv_offset, v.bv_len)
1060         )
1061         kunmap_atomic(kaddr);
1062         return bytes;
1063 }
1064 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
1065
1066 static inline void pipe_truncate(struct iov_iter *i)
1067 {
1068         struct pipe_inode_info *pipe = i->pipe;
1069         unsigned int p_tail = pipe->tail;
1070         unsigned int p_head = pipe->head;
1071         unsigned int p_mask = pipe->ring_size - 1;
1072
1073         if (!pipe_empty(p_head, p_tail)) {
1074                 struct pipe_buffer *buf;
1075                 unsigned int i_head = i->head;
1076                 size_t off = i->iov_offset;
1077
1078                 if (off) {
1079                         buf = &pipe->bufs[i_head & p_mask];
1080                         buf->len = off - buf->offset;
1081                         i_head++;
1082                 }
1083                 while (p_head != i_head) {
1084                         p_head--;
1085                         pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
1086                 }
1087
1088                 pipe->head = p_head;
1089         }
1090 }
1091
1092 static void pipe_advance(struct iov_iter *i, size_t size)
1093 {
1094         struct pipe_inode_info *pipe = i->pipe;
1095         if (unlikely(i->count < size))
1096                 size = i->count;
1097         if (size) {
1098                 struct pipe_buffer *buf;
1099                 unsigned int p_mask = pipe->ring_size - 1;
1100                 unsigned int i_head = i->head;
1101                 size_t off = i->iov_offset, left = size;
1102
1103                 if (off) /* make it relative to the beginning of buffer */
1104                         left += off - pipe->bufs[i_head & p_mask].offset;
1105                 while (1) {
1106                         buf = &pipe->bufs[i_head & p_mask];
1107                         if (left <= buf->len)
1108                                 break;
1109                         left -= buf->len;
1110                         i_head++;
1111                 }
1112                 i->head = i_head;
1113                 i->iov_offset = buf->offset + left;
1114         }
1115         i->count -= size;
1116         /* ... and discard everything past that point */
1117         pipe_truncate(i);
1118 }
1119
1120 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
1121 {
1122         struct bvec_iter bi;
1123
1124         bi.bi_size = i->count;
1125         bi.bi_bvec_done = i->iov_offset;
1126         bi.bi_idx = 0;
1127         bvec_iter_advance(i->bvec, &bi, size);
1128
1129         i->bvec += bi.bi_idx;
1130         i->nr_segs -= bi.bi_idx;
1131         i->count = bi.bi_size;
1132         i->iov_offset = bi.bi_bvec_done;
1133 }
1134
1135 void iov_iter_advance(struct iov_iter *i, size_t size)
1136 {
1137         if (unlikely(iov_iter_is_pipe(i))) {
1138                 pipe_advance(i, size);
1139                 return;
1140         }
1141         if (unlikely(iov_iter_is_discard(i))) {
1142                 i->count -= size;
1143                 return;
1144         }
1145         if (unlikely(iov_iter_is_xarray(i))) {
1146                 size = min(size, i->count);
1147                 i->iov_offset += size;
1148                 i->count -= size;
1149                 return;
1150         }
1151         if (iov_iter_is_bvec(i)) {
1152                 iov_iter_bvec_advance(i, size);
1153                 return;
1154         }
1155         iterate_and_advance(i, size, v, 0, 0, 0, 0)
1156 }
1157 EXPORT_SYMBOL(iov_iter_advance);
1158
1159 void iov_iter_revert(struct iov_iter *i, size_t unroll)
1160 {
1161         if (!unroll)
1162                 return;
1163         if (WARN_ON(unroll > MAX_RW_COUNT))
1164                 return;
1165         i->count += unroll;
1166         if (unlikely(iov_iter_is_pipe(i))) {
1167                 struct pipe_inode_info *pipe = i->pipe;
1168                 unsigned int p_mask = pipe->ring_size - 1;
1169                 unsigned int i_head = i->head;
1170                 size_t off = i->iov_offset;
1171                 while (1) {
1172                         struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1173                         size_t n = off - b->offset;
1174                         if (unroll < n) {
1175                                 off -= unroll;
1176                                 break;
1177                         }
1178                         unroll -= n;
1179                         if (!unroll && i_head == i->start_head) {
1180                                 off = 0;
1181                                 break;
1182                         }
1183                         i_head--;
1184                         b = &pipe->bufs[i_head & p_mask];
1185                         off = b->offset + b->len;
1186                 }
1187                 i->iov_offset = off;
1188                 i->head = i_head;
1189                 pipe_truncate(i);
1190                 return;
1191         }
1192         if (unlikely(iov_iter_is_discard(i)))
1193                 return;
1194         if (unroll <= i->iov_offset) {
1195                 i->iov_offset -= unroll;
1196                 return;
1197         }
1198         unroll -= i->iov_offset;
1199         if (iov_iter_is_xarray(i)) {
1200                 BUG(); /* We should never go beyond the start of the specified
1201                         * range since we might then be straying into pages that
1202                         * aren't pinned.
1203                         */
1204         } else if (iov_iter_is_bvec(i)) {
1205                 const struct bio_vec *bvec = i->bvec;
1206                 while (1) {
1207                         size_t n = (--bvec)->bv_len;
1208                         i->nr_segs++;
1209                         if (unroll <= n) {
1210                                 i->bvec = bvec;
1211                                 i->iov_offset = n - unroll;
1212                                 return;
1213                         }
1214                         unroll -= n;
1215                 }
1216         } else { /* same logics for iovec and kvec */
1217                 const struct iovec *iov = i->iov;
1218                 while (1) {
1219                         size_t n = (--iov)->iov_len;
1220                         i->nr_segs++;
1221                         if (unroll <= n) {
1222                                 i->iov = iov;
1223                                 i->iov_offset = n - unroll;
1224                                 return;
1225                         }
1226                         unroll -= n;
1227                 }
1228         }
1229 }
1230 EXPORT_SYMBOL(iov_iter_revert);
1231
1232 /*
1233  * Return the count of just the current iov_iter segment.
1234  */
1235 size_t iov_iter_single_seg_count(const struct iov_iter *i)
1236 {
1237         if (unlikely(iov_iter_is_pipe(i)))
1238                 return i->count;        // it is a silly place, anyway
1239         if (i->nr_segs == 1)
1240                 return i->count;
1241         if (unlikely(iov_iter_is_discard(i) || iov_iter_is_xarray(i)))
1242                 return i->count;
1243         if (iov_iter_is_bvec(i))
1244                 return min(i->count, i->bvec->bv_len - i->iov_offset);
1245         else
1246                 return min(i->count, i->iov->iov_len - i->iov_offset);
1247 }
1248 EXPORT_SYMBOL(iov_iter_single_seg_count);
1249
1250 void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
1251                         const struct kvec *kvec, unsigned long nr_segs,
1252                         size_t count)
1253 {
1254         WARN_ON(direction & ~(READ | WRITE));
1255         i->type = ITER_KVEC | (direction & (READ | WRITE));
1256         i->kvec = kvec;
1257         i->nr_segs = nr_segs;
1258         i->iov_offset = 0;
1259         i->count = count;
1260 }
1261 EXPORT_SYMBOL(iov_iter_kvec);
1262
1263 void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
1264                         const struct bio_vec *bvec, unsigned long nr_segs,
1265                         size_t count)
1266 {
1267         WARN_ON(direction & ~(READ | WRITE));
1268         i->type = ITER_BVEC | (direction & (READ | WRITE));
1269         i->bvec = bvec;
1270         i->nr_segs = nr_segs;
1271         i->iov_offset = 0;
1272         i->count = count;
1273 }
1274 EXPORT_SYMBOL(iov_iter_bvec);
1275
1276 void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
1277                         struct pipe_inode_info *pipe,
1278                         size_t count)
1279 {
1280         BUG_ON(direction != READ);
1281         WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1282         i->type = ITER_PIPE | READ;
1283         i->pipe = pipe;
1284         i->head = pipe->head;
1285         i->iov_offset = 0;
1286         i->count = count;
1287         i->start_head = i->head;
1288 }
1289 EXPORT_SYMBOL(iov_iter_pipe);
1290
1291 /**
1292  * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
1293  * @i: The iterator to initialise.
1294  * @direction: The direction of the transfer.
1295  * @xarray: The xarray to access.
1296  * @start: The start file position.
1297  * @count: The size of the I/O buffer in bytes.
1298  *
1299  * Set up an I/O iterator to either draw data out of the pages attached to an
1300  * inode or to inject data into those pages.  The pages *must* be prevented
1301  * from evaporation, either by taking a ref on them or locking them by the
1302  * caller.
1303  */
1304 void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
1305                      struct xarray *xarray, loff_t start, size_t count)
1306 {
1307         BUG_ON(direction & ~1);
1308         i->type = ITER_XARRAY | (direction & (READ | WRITE));
1309         i->xarray = xarray;
1310         i->xarray_start = start;
1311         i->count = count;
1312         i->iov_offset = 0;
1313 }
1314 EXPORT_SYMBOL(iov_iter_xarray);
1315
1316 /**
1317  * iov_iter_discard - Initialise an I/O iterator that discards data
1318  * @i: The iterator to initialise.
1319  * @direction: The direction of the transfer.
1320  * @count: The size of the I/O buffer in bytes.
1321  *
1322  * Set up an I/O iterator that just discards everything that's written to it.
1323  * It's only available as a READ iterator.
1324  */
1325 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1326 {
1327         BUG_ON(direction != READ);
1328         i->type = ITER_DISCARD | READ;
1329         i->count = count;
1330         i->iov_offset = 0;
1331 }
1332 EXPORT_SYMBOL(iov_iter_discard);
1333
1334 unsigned long iov_iter_alignment(const struct iov_iter *i)
1335 {
1336         unsigned long res = 0;
1337         size_t size = i->count;
1338
1339         if (unlikely(iov_iter_is_pipe(i))) {
1340                 unsigned int p_mask = i->pipe->ring_size - 1;
1341
1342                 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
1343                         return size | i->iov_offset;
1344                 return size;
1345         }
1346         if (unlikely(iov_iter_is_xarray(i)))
1347                 return (i->xarray_start + i->iov_offset) | i->count;
1348         iterate_all_kinds(i, size, v,
1349                 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
1350                 res |= v.bv_offset | v.bv_len,
1351                 res |= (unsigned long)v.iov_base | v.iov_len,
1352                 res |= v.bv_offset | v.bv_len
1353         )
1354         return res;
1355 }
1356 EXPORT_SYMBOL(iov_iter_alignment);
1357
1358 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1359 {
1360         unsigned long res = 0;
1361         size_t size = i->count;
1362
1363         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1364                 WARN_ON(1);
1365                 return ~0U;
1366         }
1367
1368         iterate_all_kinds(i, size, v,
1369                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1370                         (size != v.iov_len ? size : 0), 0),
1371                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1372                         (size != v.bv_len ? size : 0)),
1373                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1374                         (size != v.iov_len ? size : 0)),
1375                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1376                         (size != v.bv_len ? size : 0))
1377                 );
1378         return res;
1379 }
1380 EXPORT_SYMBOL(iov_iter_gap_alignment);
1381
1382 static inline ssize_t __pipe_get_pages(struct iov_iter *i,
1383                                 size_t maxsize,
1384                                 struct page **pages,
1385                                 int iter_head,
1386                                 size_t *start)
1387 {
1388         struct pipe_inode_info *pipe = i->pipe;
1389         unsigned int p_mask = pipe->ring_size - 1;
1390         ssize_t n = push_pipe(i, maxsize, &iter_head, start);
1391         if (!n)
1392                 return -EFAULT;
1393
1394         maxsize = n;
1395         n += *start;
1396         while (n > 0) {
1397                 get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1398                 iter_head++;
1399                 n -= PAGE_SIZE;
1400         }
1401
1402         return maxsize;
1403 }
1404
1405 static ssize_t pipe_get_pages(struct iov_iter *i,
1406                    struct page **pages, size_t maxsize, unsigned maxpages,
1407                    size_t *start)
1408 {
1409         unsigned int iter_head, npages;
1410         size_t capacity;
1411
1412         if (!maxsize)
1413                 return 0;
1414
1415         if (!sanity(i))
1416                 return -EFAULT;
1417
1418         data_start(i, &iter_head, start);
1419         /* Amount of free space: some of this one + all after this one */
1420         npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1421         capacity = min(npages, maxpages) * PAGE_SIZE - *start;
1422
1423         return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
1424 }
1425
1426 static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,
1427                                           pgoff_t index, unsigned int nr_pages)
1428 {
1429         XA_STATE(xas, xa, index);
1430         struct page *page;
1431         unsigned int ret = 0;
1432
1433         rcu_read_lock();
1434         for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1435                 if (xas_retry(&xas, page))
1436                         continue;
1437
1438                 /* Has the page moved or been split? */
1439                 if (unlikely(page != xas_reload(&xas))) {
1440                         xas_reset(&xas);
1441                         continue;
1442                 }
1443
1444                 pages[ret] = find_subpage(page, xas.xa_index);
1445                 get_page(pages[ret]);
1446                 if (++ret == nr_pages)
1447                         break;
1448         }
1449         rcu_read_unlock();
1450         return ret;
1451 }
1452
1453 static ssize_t iter_xarray_get_pages(struct iov_iter *i,
1454                                      struct page **pages, size_t maxsize,
1455                                      unsigned maxpages, size_t *_start_offset)
1456 {
1457         unsigned nr, offset;
1458         pgoff_t index, count;
1459         size_t size = maxsize, actual;
1460         loff_t pos;
1461
1462         if (!size || !maxpages)
1463                 return 0;
1464
1465         pos = i->xarray_start + i->iov_offset;
1466         index = pos >> PAGE_SHIFT;
1467         offset = pos & ~PAGE_MASK;
1468         *_start_offset = offset;
1469
1470         count = 1;
1471         if (size > PAGE_SIZE - offset) {
1472                 size -= PAGE_SIZE - offset;
1473                 count += size >> PAGE_SHIFT;
1474                 size &= ~PAGE_MASK;
1475                 if (size)
1476                         count++;
1477         }
1478
1479         if (count > maxpages)
1480                 count = maxpages;
1481
1482         nr = iter_xarray_populate_pages(pages, i->xarray, index, count);
1483         if (nr == 0)
1484                 return 0;
1485
1486         actual = PAGE_SIZE * nr;
1487         actual -= offset;
1488         if (nr == count && size > 0) {
1489                 unsigned last_offset = (nr > 1) ? 0 : offset;
1490                 actual -= PAGE_SIZE - (last_offset + size);
1491         }
1492         return actual;
1493 }
1494
1495 ssize_t iov_iter_get_pages(struct iov_iter *i,
1496                    struct page **pages, size_t maxsize, unsigned maxpages,
1497                    size_t *start)
1498 {
1499         if (maxsize > i->count)
1500                 maxsize = i->count;
1501
1502         if (unlikely(iov_iter_is_pipe(i)))
1503                 return pipe_get_pages(i, pages, maxsize, maxpages, start);
1504         if (unlikely(iov_iter_is_xarray(i)))
1505                 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
1506         if (unlikely(iov_iter_is_discard(i)))
1507                 return -EFAULT;
1508
1509         iterate_all_kinds(i, maxsize, v, ({
1510                 unsigned long addr = (unsigned long)v.iov_base;
1511                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1512                 int n;
1513                 int res;
1514
1515                 if (len > maxpages * PAGE_SIZE)
1516                         len = maxpages * PAGE_SIZE;
1517                 addr &= ~(PAGE_SIZE - 1);
1518                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1519                 res = get_user_pages_fast(addr, n,
1520                                 iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0,
1521                                 pages);
1522                 if (unlikely(res < 0))
1523                         return res;
1524                 return (res == n ? len : res * PAGE_SIZE) - *start;
1525         0;}),({
1526                 /* can't be more than PAGE_SIZE */
1527                 *start = v.bv_offset;
1528                 get_page(*pages = v.bv_page);
1529                 return v.bv_len;
1530         }),({
1531                 return -EFAULT;
1532         }),
1533         0
1534         )
1535         return 0;
1536 }
1537 EXPORT_SYMBOL(iov_iter_get_pages);
1538
1539 static struct page **get_pages_array(size_t n)
1540 {
1541         return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1542 }
1543
1544 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1545                    struct page ***pages, size_t maxsize,
1546                    size_t *start)
1547 {
1548         struct page **p;
1549         unsigned int iter_head, npages;
1550         ssize_t n;
1551
1552         if (!maxsize)
1553                 return 0;
1554
1555         if (!sanity(i))
1556                 return -EFAULT;
1557
1558         data_start(i, &iter_head, start);
1559         /* Amount of free space: some of this one + all after this one */
1560         npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1561         n = npages * PAGE_SIZE - *start;
1562         if (maxsize > n)
1563                 maxsize = n;
1564         else
1565                 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1566         p = get_pages_array(npages);
1567         if (!p)
1568                 return -ENOMEM;
1569         n = __pipe_get_pages(i, maxsize, p, iter_head, start);
1570         if (n > 0)
1571                 *pages = p;
1572         else
1573                 kvfree(p);
1574         return n;
1575 }
1576
1577 static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i,
1578                                            struct page ***pages, size_t maxsize,
1579                                            size_t *_start_offset)
1580 {
1581         struct page **p;
1582         unsigned nr, offset;
1583         pgoff_t index, count;
1584         size_t size = maxsize, actual;
1585         loff_t pos;
1586
1587         if (!size)
1588                 return 0;
1589
1590         pos = i->xarray_start + i->iov_offset;
1591         index = pos >> PAGE_SHIFT;
1592         offset = pos & ~PAGE_MASK;
1593         *_start_offset = offset;
1594
1595         count = 1;
1596         if (size > PAGE_SIZE - offset) {
1597                 size -= PAGE_SIZE - offset;
1598                 count += size >> PAGE_SHIFT;
1599                 size &= ~PAGE_MASK;
1600                 if (size)
1601                         count++;
1602         }
1603
1604         p = get_pages_array(count);
1605         if (!p)
1606                 return -ENOMEM;
1607         *pages = p;
1608
1609         nr = iter_xarray_populate_pages(p, i->xarray, index, count);
1610         if (nr == 0)
1611                 return 0;
1612
1613         actual = PAGE_SIZE * nr;
1614         actual -= offset;
1615         if (nr == count && size > 0) {
1616                 unsigned last_offset = (nr > 1) ? 0 : offset;
1617                 actual -= PAGE_SIZE - (last_offset + size);
1618         }
1619         return actual;
1620 }
1621
1622 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1623                    struct page ***pages, size_t maxsize,
1624                    size_t *start)
1625 {
1626         struct page **p;
1627
1628         if (maxsize > i->count)
1629                 maxsize = i->count;
1630
1631         if (unlikely(iov_iter_is_pipe(i)))
1632                 return pipe_get_pages_alloc(i, pages, maxsize, start);
1633         if (unlikely(iov_iter_is_xarray(i)))
1634                 return iter_xarray_get_pages_alloc(i, pages, maxsize, start);
1635         if (unlikely(iov_iter_is_discard(i)))
1636                 return -EFAULT;
1637
1638         iterate_all_kinds(i, maxsize, v, ({
1639                 unsigned long addr = (unsigned long)v.iov_base;
1640                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1641                 int n;
1642                 int res;
1643
1644                 addr &= ~(PAGE_SIZE - 1);
1645                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1646                 p = get_pages_array(n);
1647                 if (!p)
1648                         return -ENOMEM;
1649                 res = get_user_pages_fast(addr, n,
1650                                 iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0, p);
1651                 if (unlikely(res < 0)) {
1652                         kvfree(p);
1653                         return res;
1654                 }
1655                 *pages = p;
1656                 return (res == n ? len : res * PAGE_SIZE) - *start;
1657         0;}),({
1658                 /* can't be more than PAGE_SIZE */
1659                 *start = v.bv_offset;
1660                 *pages = p = get_pages_array(1);
1661                 if (!p)
1662                         return -ENOMEM;
1663                 get_page(*p = v.bv_page);
1664                 return v.bv_len;
1665         }),({
1666                 return -EFAULT;
1667         }), 0
1668         )
1669         return 0;
1670 }
1671 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1672
1673 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1674                                struct iov_iter *i)
1675 {
1676         char *to = addr;
1677         __wsum sum, next;
1678         size_t off = 0;
1679         sum = *csum;
1680         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1681                 WARN_ON(1);
1682                 return 0;
1683         }
1684         iterate_and_advance(i, bytes, v, ({
1685                 next = csum_and_copy_from_user(v.iov_base,
1686                                                (to += v.iov_len) - v.iov_len,
1687                                                v.iov_len);
1688                 if (next) {
1689                         sum = csum_block_add(sum, next, off);
1690                         off += v.iov_len;
1691                 }
1692                 next ? 0 : v.iov_len;
1693         }), ({
1694                 char *p = kmap_atomic(v.bv_page);
1695                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1696                                       p + v.bv_offset, v.bv_len,
1697                                       sum, off);
1698                 kunmap_atomic(p);
1699                 off += v.bv_len;
1700         }),({
1701                 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1702                                       v.iov_base, v.iov_len,
1703                                       sum, off);
1704                 off += v.iov_len;
1705         }), ({
1706                 char *p = kmap_atomic(v.bv_page);
1707                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1708                                       p + v.bv_offset, v.bv_len,
1709                                       sum, off);
1710                 kunmap_atomic(p);
1711                 off += v.bv_len;
1712         })
1713         )
1714         *csum = sum;
1715         return bytes;
1716 }
1717 EXPORT_SYMBOL(csum_and_copy_from_iter);
1718
1719 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1720                                struct iov_iter *i)
1721 {
1722         char *to = addr;
1723         __wsum sum, next;
1724         size_t off = 0;
1725         sum = *csum;
1726         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1727                 WARN_ON(1);
1728                 return false;
1729         }
1730         if (unlikely(i->count < bytes))
1731                 return false;
1732         iterate_all_kinds(i, bytes, v, ({
1733                 next = csum_and_copy_from_user(v.iov_base,
1734                                                (to += v.iov_len) - v.iov_len,
1735                                                v.iov_len);
1736                 if (!next)
1737                         return false;
1738                 sum = csum_block_add(sum, next, off);
1739                 off += v.iov_len;
1740                 0;
1741         }), ({
1742                 char *p = kmap_atomic(v.bv_page);
1743                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1744                                       p + v.bv_offset, v.bv_len,
1745                                       sum, off);
1746                 kunmap_atomic(p);
1747                 off += v.bv_len;
1748         }),({
1749                 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1750                                       v.iov_base, v.iov_len,
1751                                       sum, off);
1752                 off += v.iov_len;
1753         }), ({
1754                 char *p = kmap_atomic(v.bv_page);
1755                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1756                                       p + v.bv_offset, v.bv_len,
1757                                       sum, off);
1758                 kunmap_atomic(p);
1759                 off += v.bv_len;
1760         })
1761         )
1762         *csum = sum;
1763         iov_iter_advance(i, bytes);
1764         return true;
1765 }
1766 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1767
1768 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
1769                              struct iov_iter *i)
1770 {
1771         struct csum_state *csstate = _csstate;
1772         const char *from = addr;
1773         __wsum sum, next;
1774         size_t off;
1775
1776         if (unlikely(iov_iter_is_pipe(i)))
1777                 return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);
1778
1779         sum = csstate->csum;
1780         off = csstate->off;
1781         if (unlikely(iov_iter_is_discard(i))) {
1782                 WARN_ON(1);     /* for now */
1783                 return 0;
1784         }
1785         iterate_and_advance(i, bytes, v, ({
1786                 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1787                                              v.iov_base,
1788                                              v.iov_len);
1789                 if (next) {
1790                         sum = csum_block_add(sum, next, off);
1791                         off += v.iov_len;
1792                 }
1793                 next ? 0 : v.iov_len;
1794         }), ({
1795                 char *p = kmap_atomic(v.bv_page);
1796                 sum = csum_and_memcpy(p + v.bv_offset,
1797                                       (from += v.bv_len) - v.bv_len,
1798                                       v.bv_len, sum, off);
1799                 kunmap_atomic(p);
1800                 off += v.bv_len;
1801         }),({
1802                 sum = csum_and_memcpy(v.iov_base,
1803                                      (from += v.iov_len) - v.iov_len,
1804                                      v.iov_len, sum, off);
1805                 off += v.iov_len;
1806         }), ({
1807                 char *p = kmap_atomic(v.bv_page);
1808                 sum = csum_and_memcpy(p + v.bv_offset,
1809                                       (from += v.bv_len) - v.bv_len,
1810                                       v.bv_len, sum, off);
1811                 kunmap_atomic(p);
1812                 off += v.bv_len;
1813         })
1814         )
1815         csstate->csum = sum;
1816         csstate->off = off;
1817         return bytes;
1818 }
1819 EXPORT_SYMBOL(csum_and_copy_to_iter);
1820
1821 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1822                 struct iov_iter *i)
1823 {
1824 #ifdef CONFIG_CRYPTO_HASH
1825         struct ahash_request *hash = hashp;
1826         struct scatterlist sg;
1827         size_t copied;
1828
1829         copied = copy_to_iter(addr, bytes, i);
1830         sg_init_one(&sg, addr, copied);
1831         ahash_request_set_crypt(hash, &sg, NULL, copied);
1832         crypto_ahash_update(hash);
1833         return copied;
1834 #else
1835         return 0;
1836 #endif
1837 }
1838 EXPORT_SYMBOL(hash_and_copy_to_iter);
1839
1840 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1841 {
1842         size_t size = i->count;
1843         int npages = 0;
1844
1845         if (!size)
1846                 return 0;
1847         if (unlikely(iov_iter_is_discard(i)))
1848                 return 0;
1849
1850         if (unlikely(iov_iter_is_pipe(i))) {
1851                 struct pipe_inode_info *pipe = i->pipe;
1852                 unsigned int iter_head;
1853                 size_t off;
1854
1855                 if (!sanity(i))
1856                         return 0;
1857
1858                 data_start(i, &iter_head, &off);
1859                 /* some of this one + all after this one */
1860                 npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
1861                 if (npages >= maxpages)
1862                         return maxpages;
1863         } else if (unlikely(iov_iter_is_xarray(i))) {
1864                 unsigned offset;
1865
1866                 offset = (i->xarray_start + i->iov_offset) & ~PAGE_MASK;
1867
1868                 npages = 1;
1869                 if (size > PAGE_SIZE - offset) {
1870                         size -= PAGE_SIZE - offset;
1871                         npages += size >> PAGE_SHIFT;
1872                         size &= ~PAGE_MASK;
1873                         if (size)
1874                                 npages++;
1875                 }
1876                 if (npages >= maxpages)
1877                         return maxpages;
1878         } else iterate_all_kinds(i, size, v, ({
1879                 unsigned long p = (unsigned long)v.iov_base;
1880                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1881                         - p / PAGE_SIZE;
1882                 if (npages >= maxpages)
1883                         return maxpages;
1884         0;}),({
1885                 npages++;
1886                 if (npages >= maxpages)
1887                         return maxpages;
1888         }),({
1889                 unsigned long p = (unsigned long)v.iov_base;
1890                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1891                         - p / PAGE_SIZE;
1892                 if (npages >= maxpages)
1893                         return maxpages;
1894         }),
1895         0
1896         )
1897         return npages;
1898 }
1899 EXPORT_SYMBOL(iov_iter_npages);
1900
1901 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1902 {
1903         *new = *old;
1904         if (unlikely(iov_iter_is_pipe(new))) {
1905                 WARN_ON(1);
1906                 return NULL;
1907         }
1908         if (unlikely(iov_iter_is_discard(new) || iov_iter_is_xarray(new)))
1909                 return NULL;
1910         if (iov_iter_is_bvec(new))
1911                 return new->bvec = kmemdup(new->bvec,
1912                                     new->nr_segs * sizeof(struct bio_vec),
1913                                     flags);
1914         else
1915                 /* iovec and kvec have identical layout */
1916                 return new->iov = kmemdup(new->iov,
1917                                    new->nr_segs * sizeof(struct iovec),
1918                                    flags);
1919 }
1920 EXPORT_SYMBOL(dup_iter);
1921
1922 static int copy_compat_iovec_from_user(struct iovec *iov,
1923                 const struct iovec __user *uvec, unsigned long nr_segs)
1924 {
1925         const struct compat_iovec __user *uiov =
1926                 (const struct compat_iovec __user *)uvec;
1927         int ret = -EFAULT, i;
1928
1929         if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1930                 return -EFAULT;
1931
1932         for (i = 0; i < nr_segs; i++) {
1933                 compat_uptr_t buf;
1934                 compat_ssize_t len;
1935
1936                 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1937                 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1938
1939                 /* check for compat_size_t not fitting in compat_ssize_t .. */
1940                 if (len < 0) {
1941                         ret = -EINVAL;
1942                         goto uaccess_end;
1943                 }
1944                 iov[i].iov_base = compat_ptr(buf);
1945                 iov[i].iov_len = len;
1946         }
1947
1948         ret = 0;
1949 uaccess_end:
1950         user_access_end();
1951         return ret;
1952 }
1953
1954 static int copy_iovec_from_user(struct iovec *iov,
1955                 const struct iovec __user *uvec, unsigned long nr_segs)
1956 {
1957         unsigned long seg;
1958
1959         if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
1960                 return -EFAULT;
1961         for (seg = 0; seg < nr_segs; seg++) {
1962                 if ((ssize_t)iov[seg].iov_len < 0)
1963                         return -EINVAL;
1964         }
1965
1966         return 0;
1967 }
1968
1969 struct iovec *iovec_from_user(const struct iovec __user *uvec,
1970                 unsigned long nr_segs, unsigned long fast_segs,
1971                 struct iovec *fast_iov, bool compat)
1972 {
1973         struct iovec *iov = fast_iov;
1974         int ret;
1975
1976         /*
1977          * SuS says "The readv() function *may* fail if the iovcnt argument was
1978          * less than or equal to 0, or greater than {IOV_MAX}.  Linux has
1979          * traditionally returned zero for zero segments, so...
1980          */
1981         if (nr_segs == 0)
1982                 return iov;
1983         if (nr_segs > UIO_MAXIOV)
1984                 return ERR_PTR(-EINVAL);
1985         if (nr_segs > fast_segs) {
1986                 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
1987                 if (!iov)
1988                         return ERR_PTR(-ENOMEM);
1989         }
1990
1991         if (compat)
1992                 ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
1993         else
1994                 ret = copy_iovec_from_user(iov, uvec, nr_segs);
1995         if (ret) {
1996                 if (iov != fast_iov)
1997                         kfree(iov);
1998                 return ERR_PTR(ret);
1999         }
2000
2001         return iov;
2002 }
2003
2004 ssize_t __import_iovec(int type, const struct iovec __user *uvec,
2005                  unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
2006                  struct iov_iter *i, bool compat)
2007 {
2008         ssize_t total_len = 0;
2009         unsigned long seg;
2010         struct iovec *iov;
2011
2012         iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
2013         if (IS_ERR(iov)) {
2014                 *iovp = NULL;
2015                 return PTR_ERR(iov);
2016         }
2017
2018         /*
2019          * According to the Single Unix Specification we should return EINVAL if
2020          * an element length is < 0 when cast to ssize_t or if the total length
2021          * would overflow the ssize_t return value of the system call.
2022          *
2023          * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
2024          * overflow case.
2025          */
2026         for (seg = 0; seg < nr_segs; seg++) {
2027                 ssize_t len = (ssize_t)iov[seg].iov_len;
2028
2029                 if (!access_ok(iov[seg].iov_base, len)) {
2030                         if (iov != *iovp)
2031                                 kfree(iov);
2032                         *iovp = NULL;
2033                         return -EFAULT;
2034                 }
2035
2036                 if (len > MAX_RW_COUNT - total_len) {
2037                         len = MAX_RW_COUNT - total_len;
2038                         iov[seg].iov_len = len;
2039                 }
2040                 total_len += len;
2041         }
2042
2043         iov_iter_init(i, type, iov, nr_segs, total_len);
2044         if (iov == *iovp)
2045                 *iovp = NULL;
2046         else
2047                 *iovp = iov;
2048         return total_len;
2049 }
2050
2051 /**
2052  * import_iovec() - Copy an array of &struct iovec from userspace
2053  *     into the kernel, check that it is valid, and initialize a new
2054  *     &struct iov_iter iterator to access it.
2055  *
2056  * @type: One of %READ or %WRITE.
2057  * @uvec: Pointer to the userspace array.
2058  * @nr_segs: Number of elements in userspace array.
2059  * @fast_segs: Number of elements in @iov.
2060  * @iovp: (input and output parameter) Pointer to pointer to (usually small
2061  *     on-stack) kernel array.
2062  * @i: Pointer to iterator that will be initialized on success.
2063  *
2064  * If the array pointed to by *@iov is large enough to hold all @nr_segs,
2065  * then this function places %NULL in *@iov on return. Otherwise, a new
2066  * array will be allocated and the result placed in *@iov. This means that
2067  * the caller may call kfree() on *@iov regardless of whether the small
2068  * on-stack array was used or not (and regardless of whether this function
2069  * returns an error or not).
2070  *
2071  * Return: Negative error code on error, bytes imported on success
2072  */
2073 ssize_t import_iovec(int type, const struct iovec __user *uvec,
2074                  unsigned nr_segs, unsigned fast_segs,
2075                  struct iovec **iovp, struct iov_iter *i)
2076 {
2077         return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
2078                               in_compat_syscall());
2079 }
2080 EXPORT_SYMBOL(import_iovec);
2081
2082 int import_single_range(int rw, void __user *buf, size_t len,
2083                  struct iovec *iov, struct iov_iter *i)
2084 {
2085         if (len > MAX_RW_COUNT)
2086                 len = MAX_RW_COUNT;
2087         if (unlikely(!access_ok(buf, len)))
2088                 return -EFAULT;
2089
2090         iov->iov_base = buf;
2091         iov->iov_len = len;
2092         iov_iter_init(i, rw, iov, 1, len);
2093         return 0;
2094 }
2095 EXPORT_SYMBOL(import_single_range);
2096
2097 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
2098                             int (*f)(struct kvec *vec, void *context),
2099                             void *context)
2100 {
2101         struct kvec w;
2102         int err = -EINVAL;
2103         if (!bytes)
2104                 return 0;
2105
2106         iterate_all_kinds(i, bytes, v, -EINVAL, ({
2107                 w.iov_base = kmap(v.bv_page) + v.bv_offset;
2108                 w.iov_len = v.bv_len;
2109                 err = f(&w, context);
2110                 kunmap(v.bv_page);
2111                 err;}), ({
2112                 w = v;
2113                 err = f(&w, context);}), ({
2114                 w.iov_base = kmap(v.bv_page) + v.bv_offset;
2115                 w.iov_len = v.bv_len;
2116                 err = f(&w, context);
2117                 kunmap(v.bv_page);
2118                 err;})
2119         )
2120         return err;
2121 }
2122 EXPORT_SYMBOL(iov_iter_for_each_range);