fs: fs_struct rwlock to spinlock

fs: fs_struct rwlock to spinlock

struct fs_struct.lock is an rwlock with the read-side used to protect root and
pwd members while taking references to them. Taking a reference to a path
typically requires just 2 atomic ops, so the critical section is very small.
Parallel read-side operations would have cacheline contention on the lock, the
dentry, and the vfsmount cachelines, so the rwlock is unlikely to ever give a
real parallelism increase.

Replace it with a spinlock to avoid one or two atomic operations in typical
path lookup fastpath.

Signed-off-by: Nick Piggin <npiggin@kernel.dk>
Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

diff --git a/drivers/staging/pohmelfs/path_entry.c b/drivers/staging/pohmelfs/path_entry.c
index cdc4dd5..8ec83d2 100644 (file)
--- a/drivers/staging/pohmelfs/path_entry.c
+++ b/drivers/staging/pohmelfs/path_entry.c
@@ -44,9 +44,9 @@ int pohmelfs_construct_path_string(struct pohmelfs_inode *pi, void *data, int le
                return -ENOENT;
        }
 
-       read_lock(&current->fs->lock);
+       spin_lock(&current->fs->lock);
        path.mnt = mntget(current->fs->root.mnt);
-       read_unlock(&current->fs->lock);
+       spin_unlock(&current->fs->lock);
 
        path.dentry = d;
 
@@ -91,9 +91,9 @@ int pohmelfs_path_length(struct pohmelfs_inode *pi)
                return -ENOENT;
        }
 
-       read_lock(&current->fs->lock);
+       spin_lock(&current->fs->lock);
        root = dget(current->fs->root.dentry);
-       read_unlock(&current->fs->lock);
+       spin_unlock(&current->fs->lock);
 
        spin_lock(&dcache_lock);
 

diff --git a/fs/exec.c b/fs/exec.c
index 7761837..5adab2c 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1117,7 +1117,7 @@ int check_unsafe_exec(struct linux_binprm *bprm)
        bprm->unsafe = tracehook_unsafe_exec(p);
 
        n_fs = 1;
-       write_lock(&p->fs->lock);
+       spin_lock(&p->fs->lock);
        rcu_read_lock();
        for (t = next_thread(p); t != p; t = next_thread(t)) {
                if (t->fs == p->fs)
@@ -1134,7 +1134,7 @@ int check_unsafe_exec(struct linux_binprm *bprm)
                        res = 1;
                }
        }
-       write_unlock(&p->fs->lock);
+       spin_unlock(&p->fs->lock);
 
        return res;
 }

diff --git a/fs/fs_struct.c b/fs/fs_struct.c
index 1ee40eb..ed45a9c 100644 (file)
--- a/fs/fs_struct.c
+++ b/fs/fs_struct.c
@@ -13,11 +13,11 @@ void set_fs_root(struct fs_struct *fs, struct path *path)
 {
        struct path old_root;
 
-       write_lock(&fs->lock);
+       spin_lock(&fs->lock);
        old_root = fs->root;
        fs->root = *path;
        path_get(path);
-       write_unlock(&fs->lock);
+       spin_unlock(&fs->lock);
        if (old_root.dentry)
                path_put(&old_root);
 }
@@ -30,11 +30,11 @@ void set_fs_pwd(struct fs_struct *fs, struct path *path)
 {
        struct path old_pwd;
 
-       write_lock(&fs->lock);
+       spin_lock(&fs->lock);
        old_pwd = fs->pwd;
        fs->pwd = *path;
        path_get(path);
-       write_unlock(&fs->lock);
+       spin_unlock(&fs->lock);
 
        if (old_pwd.dentry)
                path_put(&old_pwd);
@@ -51,7 +51,7 @@ void chroot_fs_refs(struct path *old_root, struct path *new_root)
                task_lock(p);
                fs = p->fs;
                if (fs) {
-                       write_lock(&fs->lock);
+                       spin_lock(&fs->lock);
                        if (fs->root.dentry == old_root->dentry
                            && fs->root.mnt == old_root->mnt) {
                                path_get(new_root);
@@ -64,7 +64,7 @@ void chroot_fs_refs(struct path *old_root, struct path *new_root)
                                fs->pwd = *new_root;
                                count++;
                        }
-                       write_unlock(&fs->lock);
+                       spin_unlock(&fs->lock);
                }
                task_unlock(p);
        } while_each_thread(g, p);
@@ -87,10 +87,10 @@ void exit_fs(struct task_struct *tsk)
        if (fs) {
                int kill;
                task_lock(tsk);
-               write_lock(&fs->lock);
+               spin_lock(&fs->lock);
                tsk->fs = NULL;
                kill = !--fs->users;
-               write_unlock(&fs->lock);
+               spin_unlock(&fs->lock);
                task_unlock(tsk);
                if (kill)
                        free_fs_struct(fs);
@@ -104,7 +104,7 @@ struct fs_struct *copy_fs_struct(struct fs_struct *old)
        if (fs) {
                fs->users = 1;
                fs->in_exec = 0;
-               rwlock_init(&fs->lock);
+               spin_lock_init(&fs->lock);
                fs->umask = old->umask;
                get_fs_root_and_pwd(old, &fs->root, &fs->pwd);
        }
@@ -121,10 +121,10 @@ int unshare_fs_struct(void)
                return -ENOMEM;
 
        task_lock(current);
-       write_lock(&fs->lock);
+       spin_lock(&fs->lock);
        kill = !--fs->users;
        current->fs = new_fs;
-       write_unlock(&fs->lock);
+       spin_unlock(&fs->lock);
        task_unlock(current);
 
        if (kill)
@@ -143,7 +143,7 @@ EXPORT_SYMBOL(current_umask);
 /* to be mentioned only in INIT_TASK */
 struct fs_struct init_fs = {
        .users          = 1,
-       .lock           = __RW_LOCK_UNLOCKED(init_fs.lock),
+       .lock           = __SPIN_LOCK_UNLOCKED(init_fs.lock),
        .umask          = 0022,
 };
 
@@ -156,14 +156,14 @@ void daemonize_fs_struct(void)
 
                task_lock(current);
 
-               write_lock(&init_fs.lock);
+               spin_lock(&init_fs.lock);
                init_fs.users++;
-               write_unlock(&init_fs.lock);
+               spin_unlock(&init_fs.lock);
 
-               write_lock(&fs->lock);
+               spin_lock(&fs->lock);
                current->fs = &init_fs;
                kill = !--fs->users;
-               write_unlock(&fs->lock);
+               spin_unlock(&fs->lock);
 
                task_unlock(current);
                if (kill)

diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h
index eca3d52..a42b5bf 100644 (file)
--- a/include/linux/fs_struct.h
+++ b/include/linux/fs_struct.h
@@ -5,7 +5,7 @@
 
 struct fs_struct {
        int users;
-       rwlock_t lock;
+       spinlock_t lock;
        int umask;
        int in_exec;
        struct path root, pwd;
@@ -23,29 +23,29 @@ extern int unshare_fs_struct(void);
 
 static inline void get_fs_root(struct fs_struct *fs, struct path *root)
 {
-       read_lock(&fs->lock);
+       spin_lock(&fs->lock);
        *root = fs->root;
        path_get(root);
-       read_unlock(&fs->lock);
+       spin_unlock(&fs->lock);
 }
 
 static inline void get_fs_pwd(struct fs_struct *fs, struct path *pwd)
 {
-       read_lock(&fs->lock);
+       spin_lock(&fs->lock);
        *pwd = fs->pwd;
        path_get(pwd);
-       read_unlock(&fs->lock);
+       spin_unlock(&fs->lock);
 }
 
 static inline void get_fs_root_and_pwd(struct fs_struct *fs, struct path *root,
                                       struct path *pwd)
 {
-       read_lock(&fs->lock);
+       spin_lock(&fs->lock);
        *root = fs->root;
        path_get(root);
        *pwd = fs->pwd;
        path_get(pwd);
-       read_unlock(&fs->lock);
+       spin_unlock(&fs->lock);
 }
 
 #endif /* _LINUX_FS_STRUCT_H */

diff --git a/kernel/fork.c b/kernel/fork.c
index 98b4508..856eac3 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -752,13 +752,13 @@ static int copy_fs(unsigned long clone_flags, struct task_struct *tsk)
        struct fs_struct *fs = current->fs;
        if (clone_flags & CLONE_FS) {
                /* tsk->fs is already what we want */
-               write_lock(&fs->lock);
+               spin_lock(&fs->lock);
                if (fs->in_exec) {
-                       write_unlock(&fs->lock);
+                       spin_unlock(&fs->lock);
                        return -EAGAIN;
                }
                fs->users++;
-               write_unlock(&fs->lock);
+               spin_unlock(&fs->lock);
                return 0;
        }
        tsk->fs = copy_fs_struct(fs);
@@ -1676,13 +1676,13 @@ SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags)
 
                if (new_fs) {
                        fs = current->fs;
-                       write_lock(&fs->lock);
+                       spin_lock(&fs->lock);
                        current->fs = new_fs;
                        if (--fs->users)
                                new_fs = NULL;
                        else
                                new_fs = fs;
-                       write_unlock(&fs->lock);
+                       spin_unlock(&fs->lock);
                }
 
                if (new_mm) {