return ERR_PTR(-ENAMETOOLONG);
}
-static void prepend(struct prepend_buffer *p, const char *str, int namelen)
+static bool prepend_char(struct prepend_buffer *p, unsigned char c)
{
- p->len -= namelen;
- if (likely(p->len >= 0)) {
- p->buf -= namelen;
- memcpy(p->buf, str, namelen);
+ if (likely(p->len > 0)) {
+ p->len--;
+ *--p->buf = c;
+ return true;
+ }
+ p->len = -1;
+ return false;
+}
+
+/*
+ * The source of the prepend data can be an optimistoc load
+ * of a dentry name and length. And because we don't hold any
+ * locks, the length and the pointer to the name may not be
+ * in sync if a concurrent rename happens, and the kernel
+ * copy might fault as a result.
+ *
+ * The end result will correct itself when we check the
+ * rename sequence count, but we need to be able to handle
+ * the fault gracefully.
+ */
+static bool prepend_copy(void *dst, const void *src, int len)
+{
+ if (unlikely(copy_from_kernel_nofault(dst, src, len))) {
+ memset(dst, 'x', len);
+ return false;
}
+ return true;
+}
+
+static bool prepend(struct prepend_buffer *p, const char *str, int namelen)
+{
+ // Already overflowed?
+ if (p->len < 0)
+ return false;
+
+ // Will overflow?
+ if (p->len < namelen) {
+ // Fill as much as possible from the end of the name
+ str += namelen - p->len;
+ p->buf -= p->len;
+ prepend_copy(p->buf, str, p->len);
+ p->len = -1;
+ return false;
+ }
+
+ // Fits fully
+ p->len -= namelen;
+ p->buf -= namelen;
+ return prepend_copy(p->buf, str, namelen);
}
/**
* With RCU path tracing, it may race with d_move(). Use READ_ONCE() to
* make sure that either the old or the new name pointer and length are
* fetched. However, there may be mismatch between length and pointer.
- * The length cannot be trusted, we need to copy it byte-by-byte until
- * the length is reached or a null byte is found. It also prepends "/" at
+ * But since the length cannot be trusted, we need to copy the name very
+ * carefully when doing the prepend_copy(). It also prepends "/" at
* the beginning of the name. The sequence number check at the caller will
* retry it again when a d_move() does happen. So any garbage in the buffer
* due to mismatched pointer and length will be discarded.
*
- * Load acquire is needed to make sure that we see that terminating NUL.
+ * Load acquire is needed to make sure that we see the new name data even
+ * if we might get the length wrong.
*/
static bool prepend_name(struct prepend_buffer *p, const struct qstr *name)
{
const char *dname = smp_load_acquire(&name->name); /* ^^^ */
u32 dlen = READ_ONCE(name->len);
- char *s;
- p->len -= dlen + 1;
- if (unlikely(p->len < 0))
- return false;
- s = p->buf -= dlen + 1;
- *s++ = '/';
- while (dlen--) {
- char c = *dname++;
- if (!c)
- break;
- *s++ = c;
- }
- return true;
+ return prepend(p, dname, dlen) && prepend_char(p, '/');
}
static int __prepend_path(const struct dentry *dentry, const struct mount *mnt,
b = *p;
if (b.len == p->len)
- prepend(&b, "/", 1);
+ prepend_char(&b, '/');
*p = b;
return error;
{
DECLARE_BUFFER(b, buf, buflen);
- prepend(&b, "", 1);
+ prepend_char(&b, 0);
if (unlikely(prepend_path(path, root, &b) > 0))
return NULL;
return extract_string(&b);
struct path root = {};
DECLARE_BUFFER(b, buf, buflen);
- prepend(&b, "", 1);
+ prepend_char(&b, 0);
if (unlikely(prepend_path(path, &root, &b) > 1))
return ERR_PTR(-EINVAL);
return extract_string(&b);
if (unlikely(d_unlinked(path->dentry)))
prepend(&b, " (deleted)", 11);
else
- prepend(&b, "", 1);
+ prepend_char(&b, 0);
prepend_path(path, &root, &b);
rcu_read_unlock();
/* these dentries are never renamed, so d_lock is not needed */
prepend(&b, " (deleted)", 11);
prepend(&b, dentry->d_name.name, dentry->d_name.len);
- prepend(&b, "/", 1);
+ prepend_char(&b, '/');
return extract_string(&b);
}
}
done_seqretry(&rename_lock, seq);
if (b.len == p->len)
- prepend(&b, "/", 1);
+ prepend_char(&b, '/');
return extract_string(&b);
}
{
DECLARE_BUFFER(b, buf, buflen);
- prepend(&b, "", 1);
+ prepend_char(&b, 0);
return __dentry_path(dentry, &b);
}
EXPORT_SYMBOL(dentry_path_raw);
if (unlikely(d_unlinked(dentry)))
prepend(&b, "//deleted", 10);
else
- prepend(&b, "", 1);
+ prepend_char(&b, 0);
return __dentry_path(dentry, &b);
}
unsigned len;
DECLARE_BUFFER(b, page, PATH_MAX);
- prepend(&b, "", 1);
+ prepend_char(&b, 0);
if (unlikely(prepend_path(&pwd, &root, &b) > 0))
prepend(&b, "(unreachable)", 13);
rcu_read_unlock();