2 * linux/net/sunrpc/xdr.c
6 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/types.h>
12 #include <linux/string.h>
13 #include <linux/kernel.h>
14 #include <linux/pagemap.h>
15 #include <linux/errno.h>
16 #include <linux/sunrpc/xdr.h>
17 #include <linux/sunrpc/msg_prot.h>
18 #include <linux/bvec.h>
19 #include <trace/events/sunrpc.h>
22 * XDR functions for basic NFS types
25 xdr_encode_netobj(__be32 *p, const struct xdr_netobj *obj)
27 unsigned int quadlen = XDR_QUADLEN(obj->len);
29 p[quadlen] = 0; /* zero trailing bytes */
30 *p++ = cpu_to_be32(obj->len);
31 memcpy(p, obj->data, obj->len);
32 return p + XDR_QUADLEN(obj->len);
34 EXPORT_SYMBOL_GPL(xdr_encode_netobj);
37 xdr_decode_netobj(__be32 *p, struct xdr_netobj *obj)
41 if ((len = be32_to_cpu(*p++)) > XDR_MAX_NETOBJ)
45 return p + XDR_QUADLEN(len);
47 EXPORT_SYMBOL_GPL(xdr_decode_netobj);
50 * xdr_encode_opaque_fixed - Encode fixed length opaque data
51 * @p: pointer to current position in XDR buffer.
52 * @ptr: pointer to data to encode (or NULL)
53 * @nbytes: size of data.
55 * Copy the array of data of length nbytes at ptr to the XDR buffer
56 * at position p, then align to the next 32-bit boundary by padding
57 * with zero bytes (see RFC1832).
58 * Note: if ptr is NULL, only the padding is performed.
60 * Returns the updated current XDR buffer position
63 __be32 *xdr_encode_opaque_fixed(__be32 *p, const void *ptr, unsigned int nbytes)
65 if (likely(nbytes != 0)) {
66 unsigned int quadlen = XDR_QUADLEN(nbytes);
67 unsigned int padding = (quadlen << 2) - nbytes;
70 memcpy(p, ptr, nbytes);
72 memset((char *)p + nbytes, 0, padding);
77 EXPORT_SYMBOL_GPL(xdr_encode_opaque_fixed);
80 * xdr_encode_opaque - Encode variable length opaque data
81 * @p: pointer to current position in XDR buffer.
82 * @ptr: pointer to data to encode (or NULL)
83 * @nbytes: size of data.
85 * Returns the updated current XDR buffer position
87 __be32 *xdr_encode_opaque(__be32 *p, const void *ptr, unsigned int nbytes)
89 *p++ = cpu_to_be32(nbytes);
90 return xdr_encode_opaque_fixed(p, ptr, nbytes);
92 EXPORT_SYMBOL_GPL(xdr_encode_opaque);
95 xdr_encode_string(__be32 *p, const char *string)
97 return xdr_encode_array(p, string, strlen(string));
99 EXPORT_SYMBOL_GPL(xdr_encode_string);
102 xdr_decode_string_inplace(__be32 *p, char **sp,
103 unsigned int *lenp, unsigned int maxlen)
107 len = be32_to_cpu(*p++);
112 return p + XDR_QUADLEN(len);
114 EXPORT_SYMBOL_GPL(xdr_decode_string_inplace);
117 * xdr_terminate_string - '\0'-terminate a string residing in an xdr_buf
118 * @buf: XDR buffer where string resides
119 * @len: length of string, in bytes
123 xdr_terminate_string(struct xdr_buf *buf, const u32 len)
127 kaddr = kmap_atomic(buf->pages[0]);
128 kaddr[buf->page_base + len] = '\0';
129 kunmap_atomic(kaddr);
131 EXPORT_SYMBOL_GPL(xdr_terminate_string);
134 xdr_buf_pagecount(struct xdr_buf *buf)
138 return (buf->page_base + buf->page_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
142 xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp)
144 size_t i, n = xdr_buf_pagecount(buf);
146 if (n != 0 && buf->bvec == NULL) {
147 buf->bvec = kmalloc_array(n, sizeof(buf->bvec[0]), gfp);
150 for (i = 0; i < n; i++) {
151 buf->bvec[i].bv_page = buf->pages[i];
152 buf->bvec[i].bv_len = PAGE_SIZE;
153 buf->bvec[i].bv_offset = 0;
160 xdr_free_bvec(struct xdr_buf *buf)
167 xdr_inline_pages(struct xdr_buf *xdr, unsigned int offset,
168 struct page **pages, unsigned int base, unsigned int len)
170 struct kvec *head = xdr->head;
171 struct kvec *tail = xdr->tail;
172 char *buf = (char *)head->iov_base;
173 unsigned int buflen = head->iov_len;
175 head->iov_len = offset;
178 xdr->page_base = base;
181 tail->iov_base = buf + offset;
182 tail->iov_len = buflen - offset;
186 EXPORT_SYMBOL_GPL(xdr_inline_pages);
189 * Helper routines for doing 'memmove' like operations on a struct xdr_buf
193 * _shift_data_right_pages
194 * @pages: vector of pages containing both the source and dest memory area.
195 * @pgto_base: page vector address of destination
196 * @pgfrom_base: page vector address of source
197 * @len: number of bytes to copy
199 * Note: the addresses pgto_base and pgfrom_base are both calculated in
201 * if a memory area starts at byte 'base' in page 'pages[i]',
202 * then its address is given as (i << PAGE_SHIFT) + base
203 * Also note: pgfrom_base must be < pgto_base, but the memory areas
204 * they point to may overlap.
207 _shift_data_right_pages(struct page **pages, size_t pgto_base,
208 size_t pgfrom_base, size_t len)
210 struct page **pgfrom, **pgto;
214 BUG_ON(pgto_base <= pgfrom_base);
219 pgto = pages + (pgto_base >> PAGE_SHIFT);
220 pgfrom = pages + (pgfrom_base >> PAGE_SHIFT);
222 pgto_base &= ~PAGE_MASK;
223 pgfrom_base &= ~PAGE_MASK;
226 /* Are any pointers crossing a page boundary? */
227 if (pgto_base == 0) {
228 pgto_base = PAGE_SIZE;
231 if (pgfrom_base == 0) {
232 pgfrom_base = PAGE_SIZE;
237 if (copy > pgto_base)
239 if (copy > pgfrom_base)
244 vto = kmap_atomic(*pgto);
245 if (*pgto != *pgfrom) {
246 vfrom = kmap_atomic(*pgfrom);
247 memcpy(vto + pgto_base, vfrom + pgfrom_base, copy);
248 kunmap_atomic(vfrom);
250 memmove(vto + pgto_base, vto + pgfrom_base, copy);
251 flush_dcache_page(*pgto);
254 } while ((len -= copy) != 0);
259 * @pages: array of pages
260 * @pgbase: page vector address of destination
261 * @p: pointer to source data
264 * Copies data from an arbitrary memory location into an array of pages
265 * The copy is assumed to be non-overlapping.
268 _copy_to_pages(struct page **pages, size_t pgbase, const char *p, size_t len)
274 pgto = pages + (pgbase >> PAGE_SHIFT);
275 pgbase &= ~PAGE_MASK;
278 copy = PAGE_SIZE - pgbase;
282 vto = kmap_atomic(*pgto);
283 memcpy(vto + pgbase, p, copy);
291 if (pgbase == PAGE_SIZE) {
292 flush_dcache_page(*pgto);
298 flush_dcache_page(*pgto);
303 * @p: pointer to destination
304 * @pages: array of pages
305 * @pgbase: offset of source data
308 * Copies data into an arbitrary memory location from an array of pages
309 * The copy is assumed to be non-overlapping.
312 _copy_from_pages(char *p, struct page **pages, size_t pgbase, size_t len)
314 struct page **pgfrom;
318 pgfrom = pages + (pgbase >> PAGE_SHIFT);
319 pgbase &= ~PAGE_MASK;
322 copy = PAGE_SIZE - pgbase;
326 vfrom = kmap_atomic(*pgfrom);
327 memcpy(p, vfrom + pgbase, copy);
328 kunmap_atomic(vfrom);
331 if (pgbase == PAGE_SIZE) {
337 } while ((len -= copy) != 0);
339 EXPORT_SYMBOL_GPL(_copy_from_pages);
344 * @len: bytes to remove from buf->head[0]
346 * Shrinks XDR buffer's header kvec buf->head[0] by
347 * 'len' bytes. The extra data is not lost, but is instead
348 * moved into the inlined pages and/or the tail.
351 xdr_shrink_bufhead(struct xdr_buf *buf, size_t len)
353 struct kvec *head, *tail;
355 unsigned int pglen = buf->page_len;
362 WARN_ON_ONCE(len > head->iov_len);
363 if (len > head->iov_len)
366 /* Shift the tail first */
367 if (tail->iov_len != 0) {
368 if (tail->iov_len > len) {
369 copy = tail->iov_len - len;
370 memmove((char *)tail->iov_base + len,
371 tail->iov_base, copy);
374 /* Copy from the inlined pages into the tail */
379 if (offs >= tail->iov_len)
381 else if (copy > tail->iov_len - offs)
382 copy = tail->iov_len - offs;
384 _copy_from_pages((char *)tail->iov_base + offs,
386 buf->page_base + pglen + offs - len,
390 /* Do we also need to copy data from the head into the tail ? */
392 offs = copy = len - pglen;
393 if (copy > tail->iov_len)
394 copy = tail->iov_len;
395 memcpy(tail->iov_base,
396 (char *)head->iov_base +
397 head->iov_len - offs,
402 /* Now handle pages */
405 _shift_data_right_pages(buf->pages,
406 buf->page_base + len,
412 _copy_to_pages(buf->pages, buf->page_base,
413 (char *)head->iov_base + head->iov_len - len,
417 head->iov_len -= len;
419 /* Have we truncated the message? */
420 if (buf->len > buf->buflen)
421 buf->len = buf->buflen;
429 * @len: bytes to remove from buf->pages
431 * Shrinks XDR buffer's page array buf->pages by
432 * 'len' bytes. The extra data is not lost, but is instead
433 * moved into the tail.
436 xdr_shrink_pagelen(struct xdr_buf *buf, size_t len)
440 unsigned int pglen = buf->page_len;
441 unsigned int tailbuf_len;
446 BUG_ON (len > pglen);
448 tailbuf_len = buf->buflen - buf->head->iov_len - buf->page_len;
450 /* Shift the tail first */
451 if (tailbuf_len != 0) {
452 unsigned int free_space = tailbuf_len - tail->iov_len;
454 if (len < free_space)
456 tail->iov_len += free_space;
459 if (tail->iov_len > len) {
460 char *p = (char *)tail->iov_base + len;
461 memmove(p, tail->iov_base, tail->iov_len - len);
462 result += tail->iov_len - len;
464 copy = tail->iov_len;
465 /* Copy from the inlined pages into the tail */
466 _copy_from_pages((char *)tail->iov_base,
467 buf->pages, buf->page_base + pglen - len,
471 buf->page_len -= len;
473 /* Have we truncated the message? */
474 if (buf->len > buf->buflen)
475 buf->len = buf->buflen;
481 xdr_shift_buf(struct xdr_buf *buf, size_t len)
483 xdr_shrink_bufhead(buf, len);
485 EXPORT_SYMBOL_GPL(xdr_shift_buf);
488 * xdr_stream_pos - Return the current offset from the start of the xdr_stream
489 * @xdr: pointer to struct xdr_stream
491 unsigned int xdr_stream_pos(const struct xdr_stream *xdr)
493 return (unsigned int)(XDR_QUADLEN(xdr->buf->len) - xdr->nwords) << 2;
495 EXPORT_SYMBOL_GPL(xdr_stream_pos);
498 * xdr_init_encode - Initialize a struct xdr_stream for sending data.
499 * @xdr: pointer to xdr_stream struct
500 * @buf: pointer to XDR buffer in which to encode data
501 * @p: current pointer inside XDR buffer
502 * @rqst: pointer to controlling rpc_rqst, for debugging
504 * Note: at the moment the RPC client only passes the length of our
505 * scratch buffer in the xdr_buf's header kvec. Previously this
506 * meant we needed to call xdr_adjust_iovec() after encoding the
507 * data. With the new scheme, the xdr_stream manages the details
508 * of the buffer length, and takes care of adjusting the kvec
511 void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
512 struct rpc_rqst *rqst)
514 struct kvec *iov = buf->head;
515 int scratch_len = buf->buflen - buf->page_len - buf->tail[0].iov_len;
517 xdr_set_scratch_buffer(xdr, NULL, 0);
518 BUG_ON(scratch_len < 0);
521 xdr->p = (__be32 *)((char *)iov->iov_base + iov->iov_len);
522 xdr->end = (__be32 *)((char *)iov->iov_base + scratch_len);
523 BUG_ON(iov->iov_len > scratch_len);
525 if (p != xdr->p && p != NULL) {
528 BUG_ON(p < xdr->p || p > xdr->end);
529 len = (char *)p - (char *)xdr->p;
536 EXPORT_SYMBOL_GPL(xdr_init_encode);
539 * xdr_commit_encode - Ensure all data is written to buffer
540 * @xdr: pointer to xdr_stream
542 * We handle encoding across page boundaries by giving the caller a
543 * temporary location to write to, then later copying the data into
544 * place; xdr_commit_encode does that copying.
546 * Normally the caller doesn't need to call this directly, as the
547 * following xdr_reserve_space will do it. But an explicit call may be
548 * required at the end of encoding, or any other time when the xdr_buf
549 * data might be read.
551 void xdr_commit_encode(struct xdr_stream *xdr)
553 int shift = xdr->scratch.iov_len;
558 page = page_address(*xdr->page_ptr);
559 memcpy(xdr->scratch.iov_base, page, shift);
560 memmove(page, page + shift, (void *)xdr->p - page);
561 xdr->scratch.iov_len = 0;
563 EXPORT_SYMBOL_GPL(xdr_commit_encode);
565 static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr,
570 int frag1bytes, frag2bytes;
572 if (nbytes > PAGE_SIZE)
573 goto out_overflow; /* Bigger buffers require special handling */
574 if (xdr->buf->len + nbytes > xdr->buf->buflen)
575 goto out_overflow; /* Sorry, we're totally out of space */
576 frag1bytes = (xdr->end - xdr->p) << 2;
577 frag2bytes = nbytes - frag1bytes;
579 xdr->iov->iov_len += frag1bytes;
581 xdr->buf->page_len += frag1bytes;
585 * If the last encode didn't end exactly on a page boundary, the
586 * next one will straddle boundaries. Encode into the next
587 * page, then copy it back later in xdr_commit_encode. We use
588 * the "scratch" iov to track any temporarily unused fragment of
589 * space at the end of the previous buffer:
591 xdr->scratch.iov_base = xdr->p;
592 xdr->scratch.iov_len = frag1bytes;
593 p = page_address(*xdr->page_ptr);
595 * Note this is where the next encode will start after we've
596 * shifted this one back:
598 xdr->p = (void *)p + frag2bytes;
599 space_left = xdr->buf->buflen - xdr->buf->len;
600 xdr->end = (void *)p + min_t(int, space_left, PAGE_SIZE);
601 xdr->buf->page_len += frag2bytes;
602 xdr->buf->len += nbytes;
605 trace_rpc_xdr_overflow(xdr, nbytes);
610 * xdr_reserve_space - Reserve buffer space for sending
611 * @xdr: pointer to xdr_stream
612 * @nbytes: number of bytes to reserve
614 * Checks that we have enough buffer space to encode 'nbytes' more
615 * bytes of data. If so, update the total xdr_buf length, and
616 * adjust the length of the current kvec.
618 __be32 * xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes)
623 xdr_commit_encode(xdr);
624 /* align nbytes on the next 32-bit boundary */
627 q = p + (nbytes >> 2);
628 if (unlikely(q > xdr->end || q < p))
629 return xdr_get_next_encode_buffer(xdr, nbytes);
632 xdr->iov->iov_len += nbytes;
634 xdr->buf->page_len += nbytes;
635 xdr->buf->len += nbytes;
638 EXPORT_SYMBOL_GPL(xdr_reserve_space);
641 * xdr_truncate_encode - truncate an encode buffer
642 * @xdr: pointer to xdr_stream
643 * @len: new length of buffer
645 * Truncates the xdr stream, so that xdr->buf->len == len,
646 * and xdr->p points at offset len from the start of the buffer, and
647 * head, tail, and page lengths are adjusted to correspond.
649 * If this means moving xdr->p to a different buffer, we assume that
650 * that the end pointer should be set to the end of the current page,
651 * except in the case of the head buffer when we assume the head
652 * buffer's current length represents the end of the available buffer.
654 * This is *not* safe to use on a buffer that already has inlined page
655 * cache pages (as in a zero-copy server read reply), except for the
656 * simple case of truncating from one position in the tail to another.
659 void xdr_truncate_encode(struct xdr_stream *xdr, size_t len)
661 struct xdr_buf *buf = xdr->buf;
662 struct kvec *head = buf->head;
663 struct kvec *tail = buf->tail;
667 if (len > buf->len) {
671 xdr_commit_encode(xdr);
673 fraglen = min_t(int, buf->len - len, tail->iov_len);
674 tail->iov_len -= fraglen;
677 xdr->p = tail->iov_base + tail->iov_len;
678 WARN_ON_ONCE(!xdr->end);
679 WARN_ON_ONCE(!xdr->iov);
682 WARN_ON_ONCE(fraglen);
683 fraglen = min_t(int, buf->len - len, buf->page_len);
684 buf->page_len -= fraglen;
687 new = buf->page_base + buf->page_len;
689 xdr->page_ptr = buf->pages + (new >> PAGE_SHIFT);
692 xdr->p = page_address(*xdr->page_ptr);
693 xdr->end = (void *)xdr->p + PAGE_SIZE;
694 xdr->p = (void *)xdr->p + (new % PAGE_SIZE);
695 WARN_ON_ONCE(xdr->iov);
699 xdr->end = head->iov_base + head->iov_len;
700 /* (otherwise assume xdr->end is already set) */
704 xdr->p = head->iov_base + head->iov_len;
705 xdr->iov = buf->head;
707 EXPORT_SYMBOL(xdr_truncate_encode);
710 * xdr_restrict_buflen - decrease available buffer space
711 * @xdr: pointer to xdr_stream
712 * @newbuflen: new maximum number of bytes available
714 * Adjust our idea of how much space is available in the buffer.
715 * If we've already used too much space in the buffer, returns -1.
716 * If the available space is already smaller than newbuflen, returns 0
717 * and does nothing. Otherwise, adjusts xdr->buf->buflen to newbuflen
718 * and ensures xdr->end is set at most offset newbuflen from the start
721 int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen)
723 struct xdr_buf *buf = xdr->buf;
724 int left_in_this_buf = (void *)xdr->end - (void *)xdr->p;
725 int end_offset = buf->len + left_in_this_buf;
727 if (newbuflen < 0 || newbuflen < buf->len)
729 if (newbuflen > buf->buflen)
731 if (newbuflen < end_offset)
732 xdr->end = (void *)xdr->end + newbuflen - end_offset;
733 buf->buflen = newbuflen;
736 EXPORT_SYMBOL(xdr_restrict_buflen);
739 * xdr_write_pages - Insert a list of pages into an XDR buffer for sending
740 * @xdr: pointer to xdr_stream
741 * @pages: list of pages
742 * @base: offset of first byte
743 * @len: length of data in bytes
746 void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, unsigned int base,
749 struct xdr_buf *buf = xdr->buf;
750 struct kvec *iov = buf->tail;
752 buf->page_base = base;
755 iov->iov_base = (char *)xdr->p;
760 unsigned int pad = 4 - (len & 3);
762 BUG_ON(xdr->p >= xdr->end);
763 iov->iov_base = (char *)xdr->p + (len & 3);
771 EXPORT_SYMBOL_GPL(xdr_write_pages);
773 static void xdr_set_iov(struct xdr_stream *xdr, struct kvec *iov,
776 if (len > iov->iov_len)
778 xdr->p = (__be32*)iov->iov_base;
779 xdr->end = (__be32*)(iov->iov_base + len);
781 xdr->page_ptr = NULL;
784 static int xdr_set_page_base(struct xdr_stream *xdr,
785 unsigned int base, unsigned int len)
793 maxlen = xdr->buf->page_len;
800 base += xdr->buf->page_base;
802 pgnr = base >> PAGE_SHIFT;
803 xdr->page_ptr = &xdr->buf->pages[pgnr];
804 kaddr = page_address(*xdr->page_ptr);
806 pgoff = base & ~PAGE_MASK;
807 xdr->p = (__be32*)(kaddr + pgoff);
810 if (pgend > PAGE_SIZE)
812 xdr->end = (__be32*)(kaddr + pgend);
817 static void xdr_set_next_page(struct xdr_stream *xdr)
819 unsigned int newbase;
821 newbase = (1 + xdr->page_ptr - xdr->buf->pages) << PAGE_SHIFT;
822 newbase -= xdr->buf->page_base;
824 if (xdr_set_page_base(xdr, newbase, PAGE_SIZE) < 0)
825 xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);
828 static bool xdr_set_next_buffer(struct xdr_stream *xdr)
830 if (xdr->page_ptr != NULL)
831 xdr_set_next_page(xdr);
832 else if (xdr->iov == xdr->buf->head) {
833 if (xdr_set_page_base(xdr, 0, PAGE_SIZE) < 0)
834 xdr_set_iov(xdr, xdr->buf->tail, xdr->nwords << 2);
836 return xdr->p != xdr->end;
840 * xdr_init_decode - Initialize an xdr_stream for decoding data.
841 * @xdr: pointer to xdr_stream struct
842 * @buf: pointer to XDR buffer from which to decode data
843 * @p: current pointer inside XDR buffer
844 * @rqst: pointer to controlling rpc_rqst, for debugging
846 void xdr_init_decode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
847 struct rpc_rqst *rqst)
850 xdr->scratch.iov_base = NULL;
851 xdr->scratch.iov_len = 0;
852 xdr->nwords = XDR_QUADLEN(buf->len);
853 if (buf->head[0].iov_len != 0)
854 xdr_set_iov(xdr, buf->head, buf->len);
855 else if (buf->page_len != 0)
856 xdr_set_page_base(xdr, 0, buf->len);
858 xdr_set_iov(xdr, buf->head, buf->len);
859 if (p != NULL && p > xdr->p && xdr->end >= p) {
860 xdr->nwords -= p - xdr->p;
865 EXPORT_SYMBOL_GPL(xdr_init_decode);
868 * xdr_init_decode_pages - Initialize an xdr_stream for decoding into pages
869 * @xdr: pointer to xdr_stream struct
870 * @buf: pointer to XDR buffer from which to decode data
871 * @pages: list of pages to decode into
872 * @len: length in bytes of buffer in pages
874 void xdr_init_decode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
875 struct page **pages, unsigned int len)
877 memset(buf, 0, sizeof(*buf));
882 xdr_init_decode(xdr, buf, NULL, NULL);
884 EXPORT_SYMBOL_GPL(xdr_init_decode_pages);
886 static __be32 * __xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
888 unsigned int nwords = XDR_QUADLEN(nbytes);
890 __be32 *q = p + nwords;
892 if (unlikely(nwords > xdr->nwords || q > xdr->end || q < p))
895 xdr->nwords -= nwords;
900 * xdr_set_scratch_buffer - Attach a scratch buffer for decoding data.
901 * @xdr: pointer to xdr_stream struct
902 * @buf: pointer to an empty buffer
903 * @buflen: size of 'buf'
905 * The scratch buffer is used when decoding from an array of pages.
906 * If an xdr_inline_decode() call spans across page boundaries, then
907 * we copy the data into the scratch buffer in order to allow linear
910 void xdr_set_scratch_buffer(struct xdr_stream *xdr, void *buf, size_t buflen)
912 xdr->scratch.iov_base = buf;
913 xdr->scratch.iov_len = buflen;
915 EXPORT_SYMBOL_GPL(xdr_set_scratch_buffer);
917 static __be32 *xdr_copy_to_scratch(struct xdr_stream *xdr, size_t nbytes)
920 char *cpdest = xdr->scratch.iov_base;
921 size_t cplen = (char *)xdr->end - (char *)xdr->p;
923 if (nbytes > xdr->scratch.iov_len)
925 p = __xdr_inline_decode(xdr, cplen);
928 memcpy(cpdest, p, cplen);
929 if (!xdr_set_next_buffer(xdr))
933 p = __xdr_inline_decode(xdr, nbytes);
936 memcpy(cpdest, p, nbytes);
937 return xdr->scratch.iov_base;
939 trace_rpc_xdr_overflow(xdr, nbytes);
944 * xdr_inline_decode - Retrieve XDR data to decode
945 * @xdr: pointer to xdr_stream struct
946 * @nbytes: number of bytes of data to decode
948 * Check if the input buffer is long enough to enable us to decode
949 * 'nbytes' more bytes of data starting at the current position.
950 * If so return the current pointer, then update the current
953 __be32 * xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes)
957 if (unlikely(nbytes == 0))
959 if (xdr->p == xdr->end && !xdr_set_next_buffer(xdr))
961 p = __xdr_inline_decode(xdr, nbytes);
964 return xdr_copy_to_scratch(xdr, nbytes);
966 trace_rpc_xdr_overflow(xdr, nbytes);
969 EXPORT_SYMBOL_GPL(xdr_inline_decode);
971 static unsigned int xdr_align_pages(struct xdr_stream *xdr, unsigned int len)
973 struct xdr_buf *buf = xdr->buf;
975 unsigned int nwords = XDR_QUADLEN(len);
976 unsigned int cur = xdr_stream_pos(xdr);
977 unsigned int copied, offset;
979 if (xdr->nwords == 0)
982 /* Realign pages to current pointer position */
984 if (iov->iov_len > cur) {
985 offset = iov->iov_len - cur;
986 copied = xdr_shrink_bufhead(buf, offset);
987 trace_rpc_xdr_alignment(xdr, offset, copied);
988 xdr->nwords = XDR_QUADLEN(buf->len - cur);
991 if (nwords > xdr->nwords) {
992 nwords = xdr->nwords;
995 if (buf->page_len <= len)
997 else if (nwords < xdr->nwords) {
998 /* Truncate page data and move it into the tail */
999 offset = buf->page_len - len;
1000 copied = xdr_shrink_pagelen(buf, offset);
1001 trace_rpc_xdr_alignment(xdr, offset, copied);
1002 xdr->nwords = XDR_QUADLEN(buf->len - cur);
1008 * xdr_read_pages - Ensure page-based XDR data to decode is aligned at current pointer position
1009 * @xdr: pointer to xdr_stream struct
1010 * @len: number of bytes of page data
1012 * Moves data beyond the current pointer position from the XDR head[] buffer
1013 * into the page list. Any data that lies beyond current position + "len"
1014 * bytes is moved into the XDR tail[].
1016 * Returns the number of XDR encoded bytes now contained in the pages
1018 unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len)
1020 struct xdr_buf *buf = xdr->buf;
1022 unsigned int nwords;
1024 unsigned int padding;
1026 len = xdr_align_pages(xdr, len);
1029 nwords = XDR_QUADLEN(len);
1030 padding = (nwords << 2) - len;
1031 xdr->iov = iov = buf->tail;
1032 /* Compute remaining message length. */
1033 end = ((xdr->nwords - nwords) << 2) + padding;
1034 if (end > iov->iov_len)
1038 * Position current pointer at beginning of tail, and
1039 * set remaining message length.
1041 xdr->p = (__be32 *)((char *)iov->iov_base + padding);
1042 xdr->end = (__be32 *)((char *)iov->iov_base + end);
1043 xdr->page_ptr = NULL;
1044 xdr->nwords = XDR_QUADLEN(end - padding);
1047 EXPORT_SYMBOL_GPL(xdr_read_pages);
1050 * xdr_enter_page - decode data from the XDR page
1051 * @xdr: pointer to xdr_stream struct
1052 * @len: number of bytes of page data
1054 * Moves data beyond the current pointer position from the XDR head[] buffer
1055 * into the page list. Any data that lies beyond current position + "len"
1056 * bytes is moved into the XDR tail[]. The current pointer is then
1057 * repositioned at the beginning of the first XDR page.
1059 void xdr_enter_page(struct xdr_stream *xdr, unsigned int len)
1061 len = xdr_align_pages(xdr, len);
1063 * Position current pointer at beginning of tail, and
1064 * set remaining message length.
1067 xdr_set_page_base(xdr, 0, len);
1069 EXPORT_SYMBOL_GPL(xdr_enter_page);
1071 static struct kvec empty_iov = {.iov_base = NULL, .iov_len = 0};
1074 xdr_buf_from_iov(struct kvec *iov, struct xdr_buf *buf)
1076 buf->head[0] = *iov;
1077 buf->tail[0] = empty_iov;
1079 buf->buflen = buf->len = iov->iov_len;
1081 EXPORT_SYMBOL_GPL(xdr_buf_from_iov);
1084 * xdr_buf_subsegment - set subbuf to a portion of buf
1085 * @buf: an xdr buffer
1086 * @subbuf: the result buffer
1087 * @base: beginning of range in bytes
1088 * @len: length of range in bytes
1090 * sets @subbuf to an xdr buffer representing the portion of @buf of
1091 * length @len starting at offset @base.
1093 * @buf and @subbuf may be pointers to the same struct xdr_buf.
1095 * Returns -1 if base of length are out of bounds.
1098 xdr_buf_subsegment(struct xdr_buf *buf, struct xdr_buf *subbuf,
1099 unsigned int base, unsigned int len)
1101 subbuf->buflen = subbuf->len = len;
1102 if (base < buf->head[0].iov_len) {
1103 subbuf->head[0].iov_base = buf->head[0].iov_base + base;
1104 subbuf->head[0].iov_len = min_t(unsigned int, len,
1105 buf->head[0].iov_len - base);
1106 len -= subbuf->head[0].iov_len;
1109 base -= buf->head[0].iov_len;
1110 subbuf->head[0].iov_len = 0;
1113 if (base < buf->page_len) {
1114 subbuf->page_len = min(buf->page_len - base, len);
1115 base += buf->page_base;
1116 subbuf->page_base = base & ~PAGE_MASK;
1117 subbuf->pages = &buf->pages[base >> PAGE_SHIFT];
1118 len -= subbuf->page_len;
1121 base -= buf->page_len;
1122 subbuf->page_len = 0;
1125 if (base < buf->tail[0].iov_len) {
1126 subbuf->tail[0].iov_base = buf->tail[0].iov_base + base;
1127 subbuf->tail[0].iov_len = min_t(unsigned int, len,
1128 buf->tail[0].iov_len - base);
1129 len -= subbuf->tail[0].iov_len;
1132 base -= buf->tail[0].iov_len;
1133 subbuf->tail[0].iov_len = 0;
1140 EXPORT_SYMBOL_GPL(xdr_buf_subsegment);
1142 static void __read_bytes_from_xdr_buf(struct xdr_buf *subbuf, void *obj, unsigned int len)
1144 unsigned int this_len;
1146 this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);
1147 memcpy(obj, subbuf->head[0].iov_base, this_len);
1150 this_len = min_t(unsigned int, len, subbuf->page_len);
1152 _copy_from_pages(obj, subbuf->pages, subbuf->page_base, this_len);
1155 this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);
1156 memcpy(obj, subbuf->tail[0].iov_base, this_len);
1159 /* obj is assumed to point to allocated memory of size at least len: */
1160 int read_bytes_from_xdr_buf(struct xdr_buf *buf, unsigned int base, void *obj, unsigned int len)
1162 struct xdr_buf subbuf;
1165 status = xdr_buf_subsegment(buf, &subbuf, base, len);
1168 __read_bytes_from_xdr_buf(&subbuf, obj, len);
1171 EXPORT_SYMBOL_GPL(read_bytes_from_xdr_buf);
1173 static void __write_bytes_to_xdr_buf(struct xdr_buf *subbuf, void *obj, unsigned int len)
1175 unsigned int this_len;
1177 this_len = min_t(unsigned int, len, subbuf->head[0].iov_len);
1178 memcpy(subbuf->head[0].iov_base, obj, this_len);
1181 this_len = min_t(unsigned int, len, subbuf->page_len);
1183 _copy_to_pages(subbuf->pages, subbuf->page_base, obj, this_len);
1186 this_len = min_t(unsigned int, len, subbuf->tail[0].iov_len);
1187 memcpy(subbuf->tail[0].iov_base, obj, this_len);
1190 /* obj is assumed to point to allocated memory of size at least len: */
1191 int write_bytes_to_xdr_buf(struct xdr_buf *buf, unsigned int base, void *obj, unsigned int len)
1193 struct xdr_buf subbuf;
1196 status = xdr_buf_subsegment(buf, &subbuf, base, len);
1199 __write_bytes_to_xdr_buf(&subbuf, obj, len);
1202 EXPORT_SYMBOL_GPL(write_bytes_to_xdr_buf);
1205 xdr_decode_word(struct xdr_buf *buf, unsigned int base, u32 *obj)
1210 status = read_bytes_from_xdr_buf(buf, base, &raw, sizeof(*obj));
1213 *obj = be32_to_cpu(raw);
1216 EXPORT_SYMBOL_GPL(xdr_decode_word);
1219 xdr_encode_word(struct xdr_buf *buf, unsigned int base, u32 obj)
1221 __be32 raw = cpu_to_be32(obj);
1223 return write_bytes_to_xdr_buf(buf, base, &raw, sizeof(obj));
1225 EXPORT_SYMBOL_GPL(xdr_encode_word);
1227 /* If the netobj starting offset bytes from the start of xdr_buf is contained
1228 * entirely in the head or the tail, set object to point to it; otherwise
1229 * try to find space for it at the end of the tail, copy it there, and
1230 * set obj to point to it. */
1231 int xdr_buf_read_netobj(struct xdr_buf *buf, struct xdr_netobj *obj, unsigned int offset)
1233 struct xdr_buf subbuf;
1235 if (xdr_decode_word(buf, offset, &obj->len))
1237 if (xdr_buf_subsegment(buf, &subbuf, offset + 4, obj->len))
1240 /* Is the obj contained entirely in the head? */
1241 obj->data = subbuf.head[0].iov_base;
1242 if (subbuf.head[0].iov_len == obj->len)
1244 /* ..or is the obj contained entirely in the tail? */
1245 obj->data = subbuf.tail[0].iov_base;
1246 if (subbuf.tail[0].iov_len == obj->len)
1249 /* use end of tail as storage for obj:
1250 * (We don't copy to the beginning because then we'd have
1251 * to worry about doing a potentially overlapping copy.
1252 * This assumes the object is at most half the length of the
1254 if (obj->len > buf->buflen - buf->len)
1256 if (buf->tail[0].iov_len != 0)
1257 obj->data = buf->tail[0].iov_base + buf->tail[0].iov_len;
1259 obj->data = buf->head[0].iov_base + buf->head[0].iov_len;
1260 __read_bytes_from_xdr_buf(&subbuf, obj->data, obj->len);
1263 EXPORT_SYMBOL_GPL(xdr_buf_read_netobj);
1265 /* Returns 0 on success, or else a negative error code. */
1267 xdr_xcode_array2(struct xdr_buf *buf, unsigned int base,
1268 struct xdr_array2_desc *desc, int encode)
1270 char *elem = NULL, *c;
1271 unsigned int copied = 0, todo, avail_here;
1272 struct page **ppages = NULL;
1276 if (xdr_encode_word(buf, base, desc->array_len) != 0)
1279 if (xdr_decode_word(buf, base, &desc->array_len) != 0 ||
1280 desc->array_len > desc->array_maxlen ||
1281 (unsigned long) base + 4 + desc->array_len *
1282 desc->elem_size > buf->len)
1290 todo = desc->array_len * desc->elem_size;
1293 if (todo && base < buf->head->iov_len) {
1294 c = buf->head->iov_base + base;
1295 avail_here = min_t(unsigned int, todo,
1296 buf->head->iov_len - base);
1299 while (avail_here >= desc->elem_size) {
1300 err = desc->xcode(desc, c);
1303 c += desc->elem_size;
1304 avail_here -= desc->elem_size;
1308 elem = kmalloc(desc->elem_size, GFP_KERNEL);
1314 err = desc->xcode(desc, elem);
1317 memcpy(c, elem, avail_here);
1319 memcpy(elem, c, avail_here);
1320 copied = avail_here;
1322 base = buf->head->iov_len; /* align to start of pages */
1325 /* process pages array */
1326 base -= buf->head->iov_len;
1327 if (todo && base < buf->page_len) {
1328 unsigned int avail_page;
1330 avail_here = min(todo, buf->page_len - base);
1333 base += buf->page_base;
1334 ppages = buf->pages + (base >> PAGE_SHIFT);
1336 avail_page = min_t(unsigned int, PAGE_SIZE - base,
1338 c = kmap(*ppages) + base;
1340 while (avail_here) {
1341 avail_here -= avail_page;
1342 if (copied || avail_page < desc->elem_size) {
1343 unsigned int l = min(avail_page,
1344 desc->elem_size - copied);
1346 elem = kmalloc(desc->elem_size,
1354 err = desc->xcode(desc, elem);
1358 memcpy(c, elem + copied, l);
1360 if (copied == desc->elem_size)
1363 memcpy(elem + copied, c, l);
1365 if (copied == desc->elem_size) {
1366 err = desc->xcode(desc, elem);
1375 while (avail_page >= desc->elem_size) {
1376 err = desc->xcode(desc, c);
1379 c += desc->elem_size;
1380 avail_page -= desc->elem_size;
1383 unsigned int l = min(avail_page,
1384 desc->elem_size - copied);
1386 elem = kmalloc(desc->elem_size,
1394 err = desc->xcode(desc, elem);
1398 memcpy(c, elem + copied, l);
1400 if (copied == desc->elem_size)
1403 memcpy(elem + copied, c, l);
1405 if (copied == desc->elem_size) {
1406 err = desc->xcode(desc, elem);
1419 avail_page = min(avail_here,
1420 (unsigned int) PAGE_SIZE);
1422 base = buf->page_len; /* align to start of tail */
1426 base -= buf->page_len;
1428 c = buf->tail->iov_base + base;
1430 unsigned int l = desc->elem_size - copied;
1433 memcpy(c, elem + copied, l);
1435 memcpy(elem + copied, c, l);
1436 err = desc->xcode(desc, elem);
1444 err = desc->xcode(desc, c);
1447 c += desc->elem_size;
1448 todo -= desc->elem_size;
1461 xdr_decode_array2(struct xdr_buf *buf, unsigned int base,
1462 struct xdr_array2_desc *desc)
1464 if (base >= buf->len)
1467 return xdr_xcode_array2(buf, base, desc, 0);
1469 EXPORT_SYMBOL_GPL(xdr_decode_array2);
1472 xdr_encode_array2(struct xdr_buf *buf, unsigned int base,
1473 struct xdr_array2_desc *desc)
1475 if ((unsigned long) base + 4 + desc->array_len * desc->elem_size >
1476 buf->head->iov_len + buf->page_len + buf->tail->iov_len)
1479 return xdr_xcode_array2(buf, base, desc, 1);
1481 EXPORT_SYMBOL_GPL(xdr_encode_array2);
1484 xdr_process_buf(struct xdr_buf *buf, unsigned int offset, unsigned int len,
1485 int (*actor)(struct scatterlist *, void *), void *data)
1488 unsigned int page_len, thislen, page_offset;
1489 struct scatterlist sg[1];
1491 sg_init_table(sg, 1);
1493 if (offset >= buf->head[0].iov_len) {
1494 offset -= buf->head[0].iov_len;
1496 thislen = buf->head[0].iov_len - offset;
1499 sg_set_buf(sg, buf->head[0].iov_base + offset, thislen);
1500 ret = actor(sg, data);
1509 if (offset >= buf->page_len) {
1510 offset -= buf->page_len;
1512 page_len = buf->page_len - offset;
1516 page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1);
1517 i = (offset + buf->page_base) >> PAGE_SHIFT;
1518 thislen = PAGE_SIZE - page_offset;
1520 if (thislen > page_len)
1522 sg_set_page(sg, buf->pages[i], thislen, page_offset);
1523 ret = actor(sg, data);
1526 page_len -= thislen;
1529 thislen = PAGE_SIZE;
1530 } while (page_len != 0);
1535 if (offset < buf->tail[0].iov_len) {
1536 thislen = buf->tail[0].iov_len - offset;
1539 sg_set_buf(sg, buf->tail[0].iov_base + offset, thislen);
1540 ret = actor(sg, data);
1548 EXPORT_SYMBOL_GPL(xdr_process_buf);
1551 * xdr_stream_decode_opaque - Decode variable length opaque
1552 * @xdr: pointer to xdr_stream
1553 * @ptr: location to store opaque data
1554 * @size: size of storage buffer @ptr
1557 * On success, returns size of object stored in *@ptr
1558 * %-EBADMSG on XDR buffer overflow
1559 * %-EMSGSIZE on overflow of storage buffer @ptr
1561 ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size)
1566 ret = xdr_stream_decode_opaque_inline(xdr, &p, size);
1569 memcpy(ptr, p, ret);
1572 EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque);
1575 * xdr_stream_decode_opaque_dup - Decode and duplicate variable length opaque
1576 * @xdr: pointer to xdr_stream
1577 * @ptr: location to store pointer to opaque data
1578 * @maxlen: maximum acceptable object size
1579 * @gfp_flags: GFP mask to use
1582 * On success, returns size of object stored in *@ptr
1583 * %-EBADMSG on XDR buffer overflow
1584 * %-EMSGSIZE if the size of the object would exceed @maxlen
1585 * %-ENOMEM on memory allocation failure
1587 ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,
1588 size_t maxlen, gfp_t gfp_flags)
1593 ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
1595 *ptr = kmemdup(p, ret, gfp_flags);
1603 EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_dup);
1606 * xdr_stream_decode_string - Decode variable length string
1607 * @xdr: pointer to xdr_stream
1608 * @str: location to store string
1609 * @size: size of storage buffer @str
1612 * On success, returns length of NUL-terminated string stored in *@str
1613 * %-EBADMSG on XDR buffer overflow
1614 * %-EMSGSIZE on overflow of storage buffer @str
1616 ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size)
1621 ret = xdr_stream_decode_opaque_inline(xdr, &p, size);
1623 memcpy(str, p, ret);
1630 EXPORT_SYMBOL_GPL(xdr_stream_decode_string);
1633 * xdr_stream_decode_string_dup - Decode and duplicate variable length string
1634 * @xdr: pointer to xdr_stream
1635 * @str: location to store pointer to string
1636 * @maxlen: maximum acceptable string length
1637 * @gfp_flags: GFP mask to use
1640 * On success, returns length of NUL-terminated string stored in *@ptr
1641 * %-EBADMSG on XDR buffer overflow
1642 * %-EMSGSIZE if the size of the string would exceed @maxlen
1643 * %-ENOMEM on memory allocation failure
1645 ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,
1646 size_t maxlen, gfp_t gfp_flags)
1651 ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
1653 char *s = kmalloc(ret + 1, gfp_flags);
1665 EXPORT_SYMBOL_GPL(xdr_stream_decode_string_dup);