1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <crypto/hash.h>
3 #include <linux/export.h>
4 #include <linux/bvec.h>
6 #include <linux/pagemap.h>
7 #include <linux/slab.h>
8 #include <linux/vmalloc.h>
9 #include <linux/splice.h>
10 #include <net/checksum.h>
11 #include <linux/scatterlist.h>
12 #include <linux/instrumented.h>
14 #define PIPE_PARANOIA /* for now */
16 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
20 __v.iov_len = min(n, __p->iov_len - skip); \
21 if (likely(__v.iov_len)) { \
22 __v.iov_base = __p->iov_base + skip; \
24 __v.iov_len -= left; \
25 skip += __v.iov_len; \
30 while (unlikely(!left && n)) { \
32 __v.iov_len = min(n, __p->iov_len); \
33 if (unlikely(!__v.iov_len)) \
35 __v.iov_base = __p->iov_base; \
37 __v.iov_len -= left; \
44 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
47 __v.iov_len = min(n, __p->iov_len - skip); \
48 if (likely(__v.iov_len)) { \
49 __v.iov_base = __p->iov_base + skip; \
51 skip += __v.iov_len; \
54 while (unlikely(n)) { \
56 __v.iov_len = min(n, __p->iov_len); \
57 if (unlikely(!__v.iov_len)) \
59 __v.iov_base = __p->iov_base; \
67 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
68 struct bvec_iter __start; \
69 __start.bi_size = n; \
70 __start.bi_bvec_done = skip; \
72 for_each_bvec(__v, i->bvec, __bi, __start) { \
79 #define iterate_all_kinds(i, n, v, I, B, K) { \
81 size_t skip = i->iov_offset; \
82 if (unlikely(i->type & ITER_BVEC)) { \
84 struct bvec_iter __bi; \
85 iterate_bvec(i, n, v, __bi, skip, (B)) \
86 } else if (unlikely(i->type & ITER_KVEC)) { \
87 const struct kvec *kvec; \
89 iterate_kvec(i, n, v, kvec, skip, (K)) \
90 } else if (unlikely(i->type & ITER_DISCARD)) { \
92 const struct iovec *iov; \
94 iterate_iovec(i, n, v, iov, skip, (I)) \
99 #define iterate_and_advance(i, n, v, I, B, K) { \
100 if (unlikely(i->count < n)) \
103 size_t skip = i->iov_offset; \
104 if (unlikely(i->type & ITER_BVEC)) { \
105 const struct bio_vec *bvec = i->bvec; \
107 struct bvec_iter __bi; \
108 iterate_bvec(i, n, v, __bi, skip, (B)) \
109 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
110 i->nr_segs -= i->bvec - bvec; \
111 skip = __bi.bi_bvec_done; \
112 } else if (unlikely(i->type & ITER_KVEC)) { \
113 const struct kvec *kvec; \
115 iterate_kvec(i, n, v, kvec, skip, (K)) \
116 if (skip == kvec->iov_len) { \
120 i->nr_segs -= kvec - i->kvec; \
122 } else if (unlikely(i->type & ITER_DISCARD)) { \
125 const struct iovec *iov; \
127 iterate_iovec(i, n, v, iov, skip, (I)) \
128 if (skip == iov->iov_len) { \
132 i->nr_segs -= iov - i->iov; \
136 i->iov_offset = skip; \
140 static int copyout(void __user *to, const void *from, size_t n)
142 if (access_ok(to, n)) {
143 instrument_copy_to_user(to, from, n);
144 n = raw_copy_to_user(to, from, n);
149 static int copyin(void *to, const void __user *from, size_t n)
151 if (access_ok(from, n)) {
152 instrument_copy_from_user(to, from, n);
153 n = raw_copy_from_user(to, from, n);
158 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
161 size_t skip, copy, left, wanted;
162 const struct iovec *iov;
166 if (unlikely(bytes > i->count))
169 if (unlikely(!bytes))
175 skip = i->iov_offset;
176 buf = iov->iov_base + skip;
177 copy = min(bytes, iov->iov_len - skip);
179 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
180 kaddr = kmap_atomic(page);
181 from = kaddr + offset;
183 /* first chunk, usually the only one */
184 left = copyout(buf, from, copy);
190 while (unlikely(!left && bytes)) {
193 copy = min(bytes, iov->iov_len);
194 left = copyout(buf, from, copy);
200 if (likely(!bytes)) {
201 kunmap_atomic(kaddr);
204 offset = from - kaddr;
206 kunmap_atomic(kaddr);
207 copy = min(bytes, iov->iov_len - skip);
209 /* Too bad - revert to non-atomic kmap */
212 from = kaddr + offset;
213 left = copyout(buf, from, copy);
218 while (unlikely(!left && bytes)) {
221 copy = min(bytes, iov->iov_len);
222 left = copyout(buf, from, copy);
231 if (skip == iov->iov_len) {
235 i->count -= wanted - bytes;
236 i->nr_segs -= iov - i->iov;
238 i->iov_offset = skip;
239 return wanted - bytes;
242 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
245 size_t skip, copy, left, wanted;
246 const struct iovec *iov;
250 if (unlikely(bytes > i->count))
253 if (unlikely(!bytes))
259 skip = i->iov_offset;
260 buf = iov->iov_base + skip;
261 copy = min(bytes, iov->iov_len - skip);
263 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
264 kaddr = kmap_atomic(page);
267 /* first chunk, usually the only one */
268 left = copyin(to, buf, copy);
274 while (unlikely(!left && bytes)) {
277 copy = min(bytes, iov->iov_len);
278 left = copyin(to, buf, copy);
284 if (likely(!bytes)) {
285 kunmap_atomic(kaddr);
290 kunmap_atomic(kaddr);
291 copy = min(bytes, iov->iov_len - skip);
293 /* Too bad - revert to non-atomic kmap */
297 left = copyin(to, buf, copy);
302 while (unlikely(!left && bytes)) {
305 copy = min(bytes, iov->iov_len);
306 left = copyin(to, buf, copy);
315 if (skip == iov->iov_len) {
319 i->count -= wanted - bytes;
320 i->nr_segs -= iov - i->iov;
322 i->iov_offset = skip;
323 return wanted - bytes;
327 static bool sanity(const struct iov_iter *i)
329 struct pipe_inode_info *pipe = i->pipe;
330 unsigned int p_head = pipe->head;
331 unsigned int p_tail = pipe->tail;
332 unsigned int p_mask = pipe->ring_size - 1;
333 unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
334 unsigned int i_head = i->head;
338 struct pipe_buffer *p;
339 if (unlikely(p_occupancy == 0))
340 goto Bad; // pipe must be non-empty
341 if (unlikely(i_head != p_head - 1))
342 goto Bad; // must be at the last buffer...
344 p = &pipe->bufs[i_head & p_mask];
345 if (unlikely(p->offset + p->len != i->iov_offset))
346 goto Bad; // ... at the end of segment
348 if (i_head != p_head)
349 goto Bad; // must be right after the last buffer
353 printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
354 printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
355 p_head, p_tail, pipe->ring_size);
356 for (idx = 0; idx < pipe->ring_size; idx++)
357 printk(KERN_ERR "[%p %p %d %d]\n",
359 pipe->bufs[idx].page,
360 pipe->bufs[idx].offset,
361 pipe->bufs[idx].len);
366 #define sanity(i) true
369 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
372 struct pipe_inode_info *pipe = i->pipe;
373 struct pipe_buffer *buf;
374 unsigned int p_tail = pipe->tail;
375 unsigned int p_mask = pipe->ring_size - 1;
376 unsigned int i_head = i->head;
379 if (unlikely(bytes > i->count))
382 if (unlikely(!bytes))
389 buf = &pipe->bufs[i_head & p_mask];
391 if (offset == off && buf->page == page) {
392 /* merge with the last one */
394 i->iov_offset += bytes;
398 buf = &pipe->bufs[i_head & p_mask];
400 if (pipe_full(i_head, p_tail, pipe->max_usage))
403 buf->ops = &page_cache_pipe_buf_ops;
406 buf->offset = offset;
409 pipe->head = i_head + 1;
410 i->iov_offset = offset + bytes;
418 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
419 * bytes. For each iovec, fault in each page that constitutes the iovec.
421 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
422 * because it is an invalid address).
424 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
426 size_t skip = i->iov_offset;
427 const struct iovec *iov;
431 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
432 iterate_iovec(i, bytes, v, iov, skip, ({
433 err = fault_in_pages_readable(v.iov_base, v.iov_len);
440 EXPORT_SYMBOL(iov_iter_fault_in_readable);
442 void iov_iter_init(struct iov_iter *i, unsigned int direction,
443 const struct iovec *iov, unsigned long nr_segs,
446 WARN_ON(direction & ~(READ | WRITE));
447 direction &= READ | WRITE;
449 /* It will get better. Eventually... */
450 if (uaccess_kernel()) {
451 i->type = ITER_KVEC | direction;
452 i->kvec = (struct kvec *)iov;
454 i->type = ITER_IOVEC | direction;
457 i->nr_segs = nr_segs;
461 EXPORT_SYMBOL(iov_iter_init);
463 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
465 char *from = kmap_atomic(page);
466 memcpy(to, from + offset, len);
470 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
472 char *to = kmap_atomic(page);
473 memcpy(to + offset, from, len);
477 static void memzero_page(struct page *page, size_t offset, size_t len)
479 char *addr = kmap_atomic(page);
480 memset(addr + offset, 0, len);
484 static inline bool allocated(struct pipe_buffer *buf)
486 return buf->ops == &default_pipe_buf_ops;
489 static inline void data_start(const struct iov_iter *i,
490 unsigned int *iter_headp, size_t *offp)
492 unsigned int p_mask = i->pipe->ring_size - 1;
493 unsigned int iter_head = i->head;
494 size_t off = i->iov_offset;
496 if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
501 *iter_headp = iter_head;
505 static size_t push_pipe(struct iov_iter *i, size_t size,
506 int *iter_headp, size_t *offp)
508 struct pipe_inode_info *pipe = i->pipe;
509 unsigned int p_tail = pipe->tail;
510 unsigned int p_mask = pipe->ring_size - 1;
511 unsigned int iter_head;
515 if (unlikely(size > i->count))
521 data_start(i, &iter_head, &off);
522 *iter_headp = iter_head;
525 left -= PAGE_SIZE - off;
527 pipe->bufs[iter_head & p_mask].len += size;
530 pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
533 while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
534 struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
535 struct page *page = alloc_page(GFP_USER);
539 buf->ops = &default_pipe_buf_ops;
542 buf->len = min_t(ssize_t, left, PAGE_SIZE);
545 pipe->head = iter_head;
553 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
556 struct pipe_inode_info *pipe = i->pipe;
557 unsigned int p_mask = pipe->ring_size - 1;
564 bytes = n = push_pipe(i, bytes, &i_head, &off);
568 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
569 memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
571 i->iov_offset = off + chunk;
581 static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
582 __wsum sum, size_t off)
584 __wsum next = csum_partial_copy_nocheck(from, to, len, 0);
585 return csum_block_add(sum, next, off);
588 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
589 __wsum *csum, struct iov_iter *i)
591 struct pipe_inode_info *pipe = i->pipe;
592 unsigned int p_mask = pipe->ring_size - 1;
601 bytes = n = push_pipe(i, bytes, &i_head, &r);
605 size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
606 char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
607 sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
610 i->iov_offset = r + chunk;
622 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
624 const char *from = addr;
625 if (unlikely(iov_iter_is_pipe(i)))
626 return copy_pipe_to_iter(addr, bytes, i);
627 if (iter_is_iovec(i))
629 iterate_and_advance(i, bytes, v,
630 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
631 memcpy_to_page(v.bv_page, v.bv_offset,
632 (from += v.bv_len) - v.bv_len, v.bv_len),
633 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
638 EXPORT_SYMBOL(_copy_to_iter);
640 #ifdef CONFIG_ARCH_HAS_COPY_MC
641 static int copyout_mc(void __user *to, const void *from, size_t n)
643 if (access_ok(to, n)) {
644 instrument_copy_to_user(to, from, n);
645 n = copy_mc_to_user((__force void *) to, from, n);
650 static unsigned long copy_mc_to_page(struct page *page, size_t offset,
651 const char *from, size_t len)
656 to = kmap_atomic(page);
657 ret = copy_mc_to_kernel(to + offset, from, len);
663 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
666 struct pipe_inode_info *pipe = i->pipe;
667 unsigned int p_mask = pipe->ring_size - 1;
669 size_t n, off, xfer = 0;
674 bytes = n = push_pipe(i, bytes, &i_head, &off);
678 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
681 rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
684 i->iov_offset = off + chunk - rem;
698 * _copy_mc_to_iter - copy to iter with source memory error exception handling
699 * @addr: source kernel address
700 * @bytes: total transfer length
701 * @iter: destination iterator
703 * The pmem driver deploys this for the dax operation
704 * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
705 * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
706 * successfully copied.
708 * The main differences between this and typical _copy_to_iter().
710 * * Typical tail/residue handling after a fault retries the copy
711 * byte-by-byte until the fault happens again. Re-triggering machine
712 * checks is potentially fatal so the implementation uses source
713 * alignment and poison alignment assumptions to avoid re-triggering
714 * hardware exceptions.
716 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
717 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
720 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
722 const char *from = addr;
723 unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
725 if (unlikely(iov_iter_is_pipe(i)))
726 return copy_mc_pipe_to_iter(addr, bytes, i);
727 if (iter_is_iovec(i))
729 iterate_and_advance(i, bytes, v,
730 copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
733 rem = copy_mc_to_page(v.bv_page, v.bv_offset,
734 (from += v.bv_len) - v.bv_len, v.bv_len);
736 curr_addr = (unsigned long) from;
737 bytes = curr_addr - s_addr - rem;
742 rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
743 - v.iov_len, v.iov_len);
745 curr_addr = (unsigned long) from;
746 bytes = curr_addr - s_addr - rem;
754 EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
755 #endif /* CONFIG_ARCH_HAS_COPY_MC */
757 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
760 if (unlikely(iov_iter_is_pipe(i))) {
764 if (iter_is_iovec(i))
766 iterate_and_advance(i, bytes, v,
767 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
768 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
769 v.bv_offset, v.bv_len),
770 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
775 EXPORT_SYMBOL(_copy_from_iter);
777 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
780 if (unlikely(iov_iter_is_pipe(i))) {
784 if (unlikely(i->count < bytes))
787 if (iter_is_iovec(i))
789 iterate_all_kinds(i, bytes, v, ({
790 if (copyin((to += v.iov_len) - v.iov_len,
791 v.iov_base, v.iov_len))
794 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
795 v.bv_offset, v.bv_len),
796 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
799 iov_iter_advance(i, bytes);
802 EXPORT_SYMBOL(_copy_from_iter_full);
804 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
807 if (unlikely(iov_iter_is_pipe(i))) {
811 iterate_and_advance(i, bytes, v,
812 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
813 v.iov_base, v.iov_len),
814 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
815 v.bv_offset, v.bv_len),
816 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
821 EXPORT_SYMBOL(_copy_from_iter_nocache);
823 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
825 * _copy_from_iter_flushcache - write destination through cpu cache
826 * @addr: destination kernel address
827 * @bytes: total transfer length
828 * @iter: source iterator
830 * The pmem driver arranges for filesystem-dax to use this facility via
831 * dax_copy_from_iter() for ensuring that writes to persistent memory
832 * are flushed through the CPU cache. It is differentiated from
833 * _copy_from_iter_nocache() in that guarantees all data is flushed for
834 * all iterator types. The _copy_from_iter_nocache() only attempts to
835 * bypass the cache for the ITER_IOVEC case, and on some archs may use
836 * instructions that strand dirty-data in the cache.
838 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
841 if (unlikely(iov_iter_is_pipe(i))) {
845 iterate_and_advance(i, bytes, v,
846 __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
847 v.iov_base, v.iov_len),
848 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
849 v.bv_offset, v.bv_len),
850 memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
856 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
859 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
862 if (unlikely(iov_iter_is_pipe(i))) {
866 if (unlikely(i->count < bytes))
868 iterate_all_kinds(i, bytes, v, ({
869 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
870 v.iov_base, v.iov_len))
873 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
874 v.bv_offset, v.bv_len),
875 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
878 iov_iter_advance(i, bytes);
881 EXPORT_SYMBOL(_copy_from_iter_full_nocache);
883 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
886 size_t v = n + offset;
889 * The general case needs to access the page order in order
890 * to compute the page size.
891 * However, we mostly deal with order-0 pages and thus can
892 * avoid a possible cache line miss for requests that fit all
895 if (n <= v && v <= PAGE_SIZE)
898 head = compound_head(page);
899 v += (page - head) << PAGE_SHIFT;
901 if (likely(n <= v && v <= (page_size(head))))
907 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
910 if (unlikely(!page_copy_sane(page, offset, bytes)))
912 if (i->type & (ITER_BVEC|ITER_KVEC)) {
913 void *kaddr = kmap_atomic(page);
914 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
915 kunmap_atomic(kaddr);
917 } else if (unlikely(iov_iter_is_discard(i)))
919 else if (likely(!iov_iter_is_pipe(i)))
920 return copy_page_to_iter_iovec(page, offset, bytes, i);
922 return copy_page_to_iter_pipe(page, offset, bytes, i);
924 EXPORT_SYMBOL(copy_page_to_iter);
926 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
929 if (unlikely(!page_copy_sane(page, offset, bytes)))
931 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
935 if (i->type & (ITER_BVEC|ITER_KVEC)) {
936 void *kaddr = kmap_atomic(page);
937 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
938 kunmap_atomic(kaddr);
941 return copy_page_from_iter_iovec(page, offset, bytes, i);
943 EXPORT_SYMBOL(copy_page_from_iter);
945 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
947 struct pipe_inode_info *pipe = i->pipe;
948 unsigned int p_mask = pipe->ring_size - 1;
955 bytes = n = push_pipe(i, bytes, &i_head, &off);
960 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
961 memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
963 i->iov_offset = off + chunk;
972 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
974 if (unlikely(iov_iter_is_pipe(i)))
975 return pipe_zero(bytes, i);
976 iterate_and_advance(i, bytes, v,
977 clear_user(v.iov_base, v.iov_len),
978 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
979 memset(v.iov_base, 0, v.iov_len)
984 EXPORT_SYMBOL(iov_iter_zero);
986 size_t iov_iter_copy_from_user_atomic(struct page *page,
987 struct iov_iter *i, unsigned long offset, size_t bytes)
989 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
990 if (unlikely(!page_copy_sane(page, offset, bytes))) {
991 kunmap_atomic(kaddr);
994 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
995 kunmap_atomic(kaddr);
999 iterate_all_kinds(i, bytes, v,
1000 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
1001 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
1002 v.bv_offset, v.bv_len),
1003 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
1005 kunmap_atomic(kaddr);
1008 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
1010 static inline void pipe_truncate(struct iov_iter *i)
1012 struct pipe_inode_info *pipe = i->pipe;
1013 unsigned int p_tail = pipe->tail;
1014 unsigned int p_head = pipe->head;
1015 unsigned int p_mask = pipe->ring_size - 1;
1017 if (!pipe_empty(p_head, p_tail)) {
1018 struct pipe_buffer *buf;
1019 unsigned int i_head = i->head;
1020 size_t off = i->iov_offset;
1023 buf = &pipe->bufs[i_head & p_mask];
1024 buf->len = off - buf->offset;
1027 while (p_head != i_head) {
1029 pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
1032 pipe->head = p_head;
1036 static void pipe_advance(struct iov_iter *i, size_t size)
1038 struct pipe_inode_info *pipe = i->pipe;
1039 if (unlikely(i->count < size))
1042 struct pipe_buffer *buf;
1043 unsigned int p_mask = pipe->ring_size - 1;
1044 unsigned int i_head = i->head;
1045 size_t off = i->iov_offset, left = size;
1047 if (off) /* make it relative to the beginning of buffer */
1048 left += off - pipe->bufs[i_head & p_mask].offset;
1050 buf = &pipe->bufs[i_head & p_mask];
1051 if (left <= buf->len)
1057 i->iov_offset = buf->offset + left;
1060 /* ... and discard everything past that point */
1064 void iov_iter_advance(struct iov_iter *i, size_t size)
1066 if (unlikely(iov_iter_is_pipe(i))) {
1067 pipe_advance(i, size);
1070 if (unlikely(iov_iter_is_discard(i))) {
1074 iterate_and_advance(i, size, v, 0, 0, 0)
1076 EXPORT_SYMBOL(iov_iter_advance);
1078 void iov_iter_revert(struct iov_iter *i, size_t unroll)
1082 if (WARN_ON(unroll > MAX_RW_COUNT))
1085 if (unlikely(iov_iter_is_pipe(i))) {
1086 struct pipe_inode_info *pipe = i->pipe;
1087 unsigned int p_mask = pipe->ring_size - 1;
1088 unsigned int i_head = i->head;
1089 size_t off = i->iov_offset;
1091 struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1092 size_t n = off - b->offset;
1098 if (!unroll && i_head == i->start_head) {
1103 b = &pipe->bufs[i_head & p_mask];
1104 off = b->offset + b->len;
1106 i->iov_offset = off;
1111 if (unlikely(iov_iter_is_discard(i)))
1113 if (unroll <= i->iov_offset) {
1114 i->iov_offset -= unroll;
1117 unroll -= i->iov_offset;
1118 if (iov_iter_is_bvec(i)) {
1119 const struct bio_vec *bvec = i->bvec;
1121 size_t n = (--bvec)->bv_len;
1125 i->iov_offset = n - unroll;
1130 } else { /* same logics for iovec and kvec */
1131 const struct iovec *iov = i->iov;
1133 size_t n = (--iov)->iov_len;
1137 i->iov_offset = n - unroll;
1144 EXPORT_SYMBOL(iov_iter_revert);
1147 * Return the count of just the current iov_iter segment.
1149 size_t iov_iter_single_seg_count(const struct iov_iter *i)
1151 if (unlikely(iov_iter_is_pipe(i)))
1152 return i->count; // it is a silly place, anyway
1153 if (i->nr_segs == 1)
1155 if (unlikely(iov_iter_is_discard(i)))
1157 else if (iov_iter_is_bvec(i))
1158 return min(i->count, i->bvec->bv_len - i->iov_offset);
1160 return min(i->count, i->iov->iov_len - i->iov_offset);
1162 EXPORT_SYMBOL(iov_iter_single_seg_count);
1164 void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
1165 const struct kvec *kvec, unsigned long nr_segs,
1168 WARN_ON(direction & ~(READ | WRITE));
1169 i->type = ITER_KVEC | (direction & (READ | WRITE));
1171 i->nr_segs = nr_segs;
1175 EXPORT_SYMBOL(iov_iter_kvec);
1177 void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
1178 const struct bio_vec *bvec, unsigned long nr_segs,
1181 WARN_ON(direction & ~(READ | WRITE));
1182 i->type = ITER_BVEC | (direction & (READ | WRITE));
1184 i->nr_segs = nr_segs;
1188 EXPORT_SYMBOL(iov_iter_bvec);
1190 void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
1191 struct pipe_inode_info *pipe,
1194 BUG_ON(direction != READ);
1195 WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1196 i->type = ITER_PIPE | READ;
1198 i->head = pipe->head;
1201 i->start_head = i->head;
1203 EXPORT_SYMBOL(iov_iter_pipe);
1206 * iov_iter_discard - Initialise an I/O iterator that discards data
1207 * @i: The iterator to initialise.
1208 * @direction: The direction of the transfer.
1209 * @count: The size of the I/O buffer in bytes.
1211 * Set up an I/O iterator that just discards everything that's written to it.
1212 * It's only available as a READ iterator.
1214 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1216 BUG_ON(direction != READ);
1217 i->type = ITER_DISCARD | READ;
1221 EXPORT_SYMBOL(iov_iter_discard);
1223 unsigned long iov_iter_alignment(const struct iov_iter *i)
1225 unsigned long res = 0;
1226 size_t size = i->count;
1228 if (unlikely(iov_iter_is_pipe(i))) {
1229 unsigned int p_mask = i->pipe->ring_size - 1;
1231 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
1232 return size | i->iov_offset;
1235 iterate_all_kinds(i, size, v,
1236 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
1237 res |= v.bv_offset | v.bv_len,
1238 res |= (unsigned long)v.iov_base | v.iov_len
1242 EXPORT_SYMBOL(iov_iter_alignment);
1244 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1246 unsigned long res = 0;
1247 size_t size = i->count;
1249 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1254 iterate_all_kinds(i, size, v,
1255 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1256 (size != v.iov_len ? size : 0), 0),
1257 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1258 (size != v.bv_len ? size : 0)),
1259 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1260 (size != v.iov_len ? size : 0))
1264 EXPORT_SYMBOL(iov_iter_gap_alignment);
1266 static inline ssize_t __pipe_get_pages(struct iov_iter *i,
1268 struct page **pages,
1272 struct pipe_inode_info *pipe = i->pipe;
1273 unsigned int p_mask = pipe->ring_size - 1;
1274 ssize_t n = push_pipe(i, maxsize, &iter_head, start);
1281 get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1289 static ssize_t pipe_get_pages(struct iov_iter *i,
1290 struct page **pages, size_t maxsize, unsigned maxpages,
1293 unsigned int iter_head, npages;
1302 data_start(i, &iter_head, start);
1303 /* Amount of free space: some of this one + all after this one */
1304 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1305 capacity = min(npages, maxpages) * PAGE_SIZE - *start;
1307 return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
1310 ssize_t iov_iter_get_pages(struct iov_iter *i,
1311 struct page **pages, size_t maxsize, unsigned maxpages,
1314 if (maxsize > i->count)
1317 if (unlikely(iov_iter_is_pipe(i)))
1318 return pipe_get_pages(i, pages, maxsize, maxpages, start);
1319 if (unlikely(iov_iter_is_discard(i)))
1322 iterate_all_kinds(i, maxsize, v, ({
1323 unsigned long addr = (unsigned long)v.iov_base;
1324 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1328 if (len > maxpages * PAGE_SIZE)
1329 len = maxpages * PAGE_SIZE;
1330 addr &= ~(PAGE_SIZE - 1);
1331 n = DIV_ROUND_UP(len, PAGE_SIZE);
1332 res = get_user_pages_fast(addr, n,
1333 iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0,
1335 if (unlikely(res < 0))
1337 return (res == n ? len : res * PAGE_SIZE) - *start;
1339 /* can't be more than PAGE_SIZE */
1340 *start = v.bv_offset;
1341 get_page(*pages = v.bv_page);
1349 EXPORT_SYMBOL(iov_iter_get_pages);
1351 static struct page **get_pages_array(size_t n)
1353 return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1356 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1357 struct page ***pages, size_t maxsize,
1361 unsigned int iter_head, npages;
1370 data_start(i, &iter_head, start);
1371 /* Amount of free space: some of this one + all after this one */
1372 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1373 n = npages * PAGE_SIZE - *start;
1377 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1378 p = get_pages_array(npages);
1381 n = __pipe_get_pages(i, maxsize, p, iter_head, start);
1389 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1390 struct page ***pages, size_t maxsize,
1395 if (maxsize > i->count)
1398 if (unlikely(iov_iter_is_pipe(i)))
1399 return pipe_get_pages_alloc(i, pages, maxsize, start);
1400 if (unlikely(iov_iter_is_discard(i)))
1403 iterate_all_kinds(i, maxsize, v, ({
1404 unsigned long addr = (unsigned long)v.iov_base;
1405 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1409 addr &= ~(PAGE_SIZE - 1);
1410 n = DIV_ROUND_UP(len, PAGE_SIZE);
1411 p = get_pages_array(n);
1414 res = get_user_pages_fast(addr, n,
1415 iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p);
1416 if (unlikely(res < 0)) {
1421 return (res == n ? len : res * PAGE_SIZE) - *start;
1423 /* can't be more than PAGE_SIZE */
1424 *start = v.bv_offset;
1425 *pages = p = get_pages_array(1);
1428 get_page(*p = v.bv_page);
1436 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1438 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1445 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1449 iterate_and_advance(i, bytes, v, ({
1451 next = csum_and_copy_from_user(v.iov_base,
1452 (to += v.iov_len) - v.iov_len,
1453 v.iov_len, 0, &err);
1455 sum = csum_block_add(sum, next, off);
1458 err ? v.iov_len : 0;
1460 char *p = kmap_atomic(v.bv_page);
1461 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1462 p + v.bv_offset, v.bv_len,
1467 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1468 v.iov_base, v.iov_len,
1476 EXPORT_SYMBOL(csum_and_copy_from_iter);
1478 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1485 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1489 if (unlikely(i->count < bytes))
1491 iterate_all_kinds(i, bytes, v, ({
1493 next = csum_and_copy_from_user(v.iov_base,
1494 (to += v.iov_len) - v.iov_len,
1495 v.iov_len, 0, &err);
1498 sum = csum_block_add(sum, next, off);
1502 char *p = kmap_atomic(v.bv_page);
1503 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1504 p + v.bv_offset, v.bv_len,
1509 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1510 v.iov_base, v.iov_len,
1516 iov_iter_advance(i, bytes);
1519 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1521 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,
1524 const char *from = addr;
1525 __wsum *csum = csump;
1529 if (unlikely(iov_iter_is_pipe(i)))
1530 return csum_and_copy_to_pipe_iter(addr, bytes, csum, i);
1533 if (unlikely(iov_iter_is_discard(i))) {
1534 WARN_ON(1); /* for now */
1537 iterate_and_advance(i, bytes, v, ({
1539 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1541 v.iov_len, 0, &err);
1543 sum = csum_block_add(sum, next, off);
1546 err ? v.iov_len : 0;
1548 char *p = kmap_atomic(v.bv_page);
1549 sum = csum_and_memcpy(p + v.bv_offset,
1550 (from += v.bv_len) - v.bv_len,
1551 v.bv_len, sum, off);
1555 sum = csum_and_memcpy(v.iov_base,
1556 (from += v.iov_len) - v.iov_len,
1557 v.iov_len, sum, off);
1564 EXPORT_SYMBOL(csum_and_copy_to_iter);
1566 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1569 #ifdef CONFIG_CRYPTO_HASH
1570 struct ahash_request *hash = hashp;
1571 struct scatterlist sg;
1574 copied = copy_to_iter(addr, bytes, i);
1575 sg_init_one(&sg, addr, copied);
1576 ahash_request_set_crypt(hash, &sg, NULL, copied);
1577 crypto_ahash_update(hash);
1583 EXPORT_SYMBOL(hash_and_copy_to_iter);
1585 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1587 size_t size = i->count;
1592 if (unlikely(iov_iter_is_discard(i)))
1595 if (unlikely(iov_iter_is_pipe(i))) {
1596 struct pipe_inode_info *pipe = i->pipe;
1597 unsigned int iter_head;
1603 data_start(i, &iter_head, &off);
1604 /* some of this one + all after this one */
1605 npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
1606 if (npages >= maxpages)
1608 } else iterate_all_kinds(i, size, v, ({
1609 unsigned long p = (unsigned long)v.iov_base;
1610 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1612 if (npages >= maxpages)
1616 if (npages >= maxpages)
1619 unsigned long p = (unsigned long)v.iov_base;
1620 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1622 if (npages >= maxpages)
1628 EXPORT_SYMBOL(iov_iter_npages);
1630 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1633 if (unlikely(iov_iter_is_pipe(new))) {
1637 if (unlikely(iov_iter_is_discard(new)))
1639 if (iov_iter_is_bvec(new))
1640 return new->bvec = kmemdup(new->bvec,
1641 new->nr_segs * sizeof(struct bio_vec),
1644 /* iovec and kvec have identical layout */
1645 return new->iov = kmemdup(new->iov,
1646 new->nr_segs * sizeof(struct iovec),
1649 EXPORT_SYMBOL(dup_iter);
1652 * import_iovec() - Copy an array of &struct iovec from userspace
1653 * into the kernel, check that it is valid, and initialize a new
1654 * &struct iov_iter iterator to access it.
1656 * @type: One of %READ or %WRITE.
1657 * @uvector: Pointer to the userspace array.
1658 * @nr_segs: Number of elements in userspace array.
1659 * @fast_segs: Number of elements in @iov.
1660 * @iov: (input and output parameter) Pointer to pointer to (usually small
1661 * on-stack) kernel array.
1662 * @i: Pointer to iterator that will be initialized on success.
1664 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1665 * then this function places %NULL in *@iov on return. Otherwise, a new
1666 * array will be allocated and the result placed in *@iov. This means that
1667 * the caller may call kfree() on *@iov regardless of whether the small
1668 * on-stack array was used or not (and regardless of whether this function
1669 * returns an error or not).
1671 * Return: Negative error code on error, bytes imported on success
1673 ssize_t import_iovec(int type, const struct iovec __user * uvector,
1674 unsigned nr_segs, unsigned fast_segs,
1675 struct iovec **iov, struct iov_iter *i)
1679 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1687 iov_iter_init(i, type, p, nr_segs, n);
1688 *iov = p == *iov ? NULL : p;
1691 EXPORT_SYMBOL(import_iovec);
1693 #ifdef CONFIG_COMPAT
1694 #include <linux/compat.h>
1696 ssize_t compat_import_iovec(int type,
1697 const struct compat_iovec __user * uvector,
1698 unsigned nr_segs, unsigned fast_segs,
1699 struct iovec **iov, struct iov_iter *i)
1703 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1711 iov_iter_init(i, type, p, nr_segs, n);
1712 *iov = p == *iov ? NULL : p;
1715 EXPORT_SYMBOL(compat_import_iovec);
1718 int import_single_range(int rw, void __user *buf, size_t len,
1719 struct iovec *iov, struct iov_iter *i)
1721 if (len > MAX_RW_COUNT)
1723 if (unlikely(!access_ok(buf, len)))
1726 iov->iov_base = buf;
1728 iov_iter_init(i, rw, iov, 1, len);
1731 EXPORT_SYMBOL(import_single_range);
1733 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
1734 int (*f)(struct kvec *vec, void *context),
1742 iterate_all_kinds(i, bytes, v, -EINVAL, ({
1743 w.iov_base = kmap(v.bv_page) + v.bv_offset;
1744 w.iov_len = v.bv_len;
1745 err = f(&w, context);
1749 err = f(&w, context);})
1753 EXPORT_SYMBOL(iov_iter_for_each_range);