[PATCH] shrink dentry struct

Some long time ago, dentry struct was carefully tuned so that on 32 bits
UP, sizeof(struct dentry) was exactly 128, ie a power of 2, and a multiple
of memory cache lines.

Then RCU was added and dentry struct enlarged by two pointers, with nice
results for SMP, but not so good on UP, because breaking the above tuning
(128 + 8 = 136 bytes)

This patch reverts this unwanted side effect, by using an union (d_u),
where d_rcu and d_child are placed so that these two fields can share their
memory needs.

At the time d_free() is called (and d_rcu is really used), d_child is known
to be empty and not touched by the dentry freeing.

Lockless lookups only access d_name, d_parent, d_lock, d_op, d_flags (so
the previous content of d_child is not needed if said dentry was unhashed
but still accessed by a CPU because of RCU constraints)

As dentry cache easily contains millions of entries, a size reduction is
worth the extra complexity of the ugly C union.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Cc: Dipankar Sarma <dipankar@in.ibm.com>
Cc: Maneesh Soni <maneesh@in.ibm.com>
Cc: Miklos Szeredi <miklos@szeredi.hu>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Cc: Ian Kent <raven@themaw.net>
Cc: Paul Jackson <pj@sgi.com>
Cc: Al Viro <viro@ftp.linux.org.uk>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Cc: Neil Brown <neilb@cse.unsw.edu.au>
Cc: James Morris <jmorris@namei.org>
Cc: Stephen Smalley <sds@epoch.ncsc.mil>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/drivers/usb/core/inode.c b/drivers/usb/core/inode.c
index c44bbedec817df655b7b61e12b0433818e7abdc4..4ddc453023a264362f8681d9ec050678da877dba 100644 (file)
--- a/drivers/usb/core/inode.c
+++ b/drivers/usb/core/inode.c
@@ -186,7 +186,7 @@ static void update_bus(struct dentry *bus)
 
        down(&bus->d_inode->i_sem);
 
-       list_for_each_entry(dev, &bus->d_subdirs, d_child)
+       list_for_each_entry(dev, &bus->d_subdirs, d_u.d_child)
                if (dev->d_inode)
                        update_dev(dev);
 
@@ -203,7 +203,7 @@ static void update_sb(struct super_block *sb)
 
        down(&root->d_inode->i_sem);
 
-       list_for_each_entry(bus, &root->d_subdirs, d_child) {
+       list_for_each_entry(bus, &root->d_subdirs, d_u.d_child) {
                if (bus->d_inode) {
                        switch (S_IFMT & bus->d_inode->i_mode) {
                        case S_IFDIR:
@@ -319,7 +319,7 @@ static int usbfs_empty (struct dentry *dentry)
        spin_lock(&dcache_lock);
 
        list_for_each(list, &dentry->d_subdirs) {
-               struct dentry *de = list_entry(list, struct dentry, d_child);
+               struct dentry *de = list_entry(list, struct dentry, d_u.d_child);
                if (usbfs_positive(de)) {
                        spin_unlock(&dcache_lock);
                        return 0;

diff --git a/fs/autofs4/autofs_i.h b/fs/autofs4/autofs_i.h
index fca83e28edcf678cb8de5bb6838c7ec654c2455f..385bed09b0d84005674acc1bba64ad816ced5e6c 100644 (file)
--- a/fs/autofs4/autofs_i.h
+++ b/fs/autofs4/autofs_i.h
@@ -209,7 +209,7 @@ static inline int simple_empty_nolock(struct dentry *dentry)
        struct dentry *child;
        int ret = 0;
 
-       list_for_each_entry(child, &dentry->d_subdirs, d_child)
+       list_for_each_entry(child, &dentry->d_subdirs, d_u.d_child)
                if (simple_positive(child))
                        goto out;
        ret = 1;

diff --git a/fs/autofs4/expire.c b/fs/autofs4/expire.c
index feb6ac427d058b8a1c02b420bbe6ad81ffd581aa..dc39589df165a049a465208949f9d22c760ce0c2 100644 (file)
--- a/fs/autofs4/expire.c
+++ b/fs/autofs4/expire.c
@@ -105,7 +105,7 @@ repeat:
        next = this_parent->d_subdirs.next;
 resume:
        while (next != &this_parent->d_subdirs) {
-               struct dentry *dentry = list_entry(next, struct dentry, d_child);
+               struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
 
                /* Negative dentry - give up */
                if (!simple_positive(dentry)) {
@@ -138,7 +138,7 @@ resume:
        }
 
        if (this_parent != top) {
-               next = this_parent->d_child.next;
+               next = this_parent->d_u.d_child.next;
                this_parent = this_parent->d_parent;
                goto resume;
        }
@@ -163,7 +163,7 @@ repeat:
        next = this_parent->d_subdirs.next;
 resume:
        while (next != &this_parent->d_subdirs) {
-               struct dentry *dentry = list_entry(next, struct dentry, d_child);
+               struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
 
                /* Negative dentry - give up */
                if (!simple_positive(dentry)) {
@@ -199,7 +199,7 @@ cont:
        }
 
        if (this_parent != parent) {
-               next = this_parent->d_child.next;
+               next = this_parent->d_u.d_child.next;
                this_parent = this_parent->d_parent;
                goto resume;
        }
@@ -238,7 +238,7 @@ static struct dentry *autofs4_expire(struct super_block *sb,
        /* On exit from the loop expire is set to a dgot dentry
         * to expire or it's NULL */
        while ( next != &root->d_subdirs ) {
-               struct dentry *dentry = list_entry(next, struct dentry, d_child);
+               struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
 
                /* Negative dentry - give up */
                if ( !simple_positive(dentry) ) {
@@ -302,7 +302,7 @@ next:
                        expired, (int)expired->d_name.len, expired->d_name.name);
                spin_lock(&dcache_lock);
                list_del(&expired->d_parent->d_subdirs);
-               list_add(&expired->d_parent->d_subdirs, &expired->d_child);
+               list_add(&expired->d_parent->d_subdirs, &expired->d_u.d_child);
                spin_unlock(&dcache_lock);
                return expired;
        }

diff --git a/fs/autofs4/inode.c b/fs/autofs4/inode.c
index 818b37be5153f3d7508ac00c215c2f523f01999f..2d3082854a292dec3fc1ad749b81b9c700a4addb 100644 (file)
--- a/fs/autofs4/inode.c
+++ b/fs/autofs4/inode.c
@@ -91,7 +91,7 @@ repeat:
        next = this_parent->d_subdirs.next;
 resume:
        while (next != &this_parent->d_subdirs) {
-               struct dentry *dentry = list_entry(next, struct dentry, d_child);
+               struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
 
                /* Negative dentry - don`t care */
                if (!simple_positive(dentry)) {
@@ -117,7 +117,7 @@ resume:
        if (this_parent != sbi->root) {
                struct dentry *dentry = this_parent;
 
-               next = this_parent->d_child.next;
+               next = this_parent->d_u.d_child.next;
                this_parent = this_parent->d_parent;
                spin_unlock(&dcache_lock);
                DPRINTK("parent dentry %p %.*s",

diff --git a/fs/autofs4/root.c b/fs/autofs4/root.c
index 2a771ec66956f92615de35694dc512f0144aedba..2241405ffc413a49035cad0085be9c481c325165 100644 (file)
--- a/fs/autofs4/root.c
+++ b/fs/autofs4/root.c
@@ -143,7 +143,8 @@ static int autofs4_dcache_readdir(struct file * filp, void * dirent, filldir_t f
                        }
 
                        while(1) {
-                               struct dentry *de = list_entry(list, struct dentry, d_child);
+                               struct dentry *de = list_entry(list,
+                                               struct dentry, d_u.d_child);
 
                                if (!d_unhashed(de) && de->d_inode) {
                                        spin_unlock(&dcache_lock);

diff --git a/fs/coda/cache.c b/fs/coda/cache.c
index 80072fd9b7faf6a6c21747d65ffc6bb2c65e8cbd..c607d923350a4ee1bb46259ddd39d9761301bb72 100644 (file)
--- a/fs/coda/cache.c
+++ b/fs/coda/cache.c
@@ -93,7 +93,7 @@ static void coda_flag_children(struct dentry *parent, int flag)
        spin_lock(&dcache_lock);
        list_for_each(child, &parent->d_subdirs)
        {
-               de = list_entry(child, struct dentry, d_child);
+               de = list_entry(child, struct dentry, d_u.d_child);
                /* don't know what to do with negative dentries */
                if ( ! de->d_inode ) 
                        continue;

diff --git a/fs/dcache.c b/fs/dcache.c
index 17e4391386818d405ca0203123b9738f8b886cc8..1536f15c4d4c785a1ad2d49a78d03c5c4bd0067c 100644 (file)
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -71,7 +71,7 @@ struct dentry_stat_t dentry_stat = {
 
 static void d_callback(struct rcu_head *head)
 {
-       struct dentry * dentry = container_of(head, struct dentry, d_rcu);
+       struct dentry * dentry = container_of(head, struct dentry, d_u.d_rcu);
 
        if (dname_external(dentry))
                kfree(dentry->d_name.name);
@@ -86,7 +86,7 @@ static void d_free(struct dentry *dentry)
 {
        if (dentry->d_op && dentry->d_op->d_release)
                dentry->d_op->d_release(dentry);
-       call_rcu(&dentry->d_rcu, d_callback);
+       call_rcu(&dentry->d_u.d_rcu, d_callback);
 }
 
 /*
@@ -193,7 +193,7 @@ kill_it: {
                        list_del(&dentry->d_lru);
                        dentry_stat.nr_unused--;
                }
-               list_del(&dentry->d_child);
+               list_del(&dentry->d_u.d_child);
                dentry_stat.nr_dentry--;        /* For d_free, below */
                /*drops the locks, at that point nobody can reach this dentry */
                dentry_iput(dentry);
@@ -367,7 +367,7 @@ static inline void prune_one_dentry(struct dentry * dentry)
        struct dentry * parent;
 
        __d_drop(dentry);
-       list_del(&dentry->d_child);
+       list_del(&dentry->d_u.d_child);
        dentry_stat.nr_dentry--;        /* For d_free, below */
        dentry_iput(dentry);
        parent = dentry->d_parent;
@@ -518,7 +518,7 @@ repeat:
 resume:
        while (next != &this_parent->d_subdirs) {
                struct list_head *tmp = next;
-               struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
+               struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
                next = tmp->next;
                /* Have we found a mount point ? */
                if (d_mountpoint(dentry))
@@ -532,7 +532,7 @@ resume:
         * All done at this level ... ascend and resume the search.
         */
        if (this_parent != parent) {
-               next = this_parent->d_child.next; 
+               next = this_parent->d_u.d_child.next;
                this_parent = this_parent->d_parent;
                goto resume;
        }
@@ -569,7 +569,7 @@ repeat:
 resume:
        while (next != &this_parent->d_subdirs) {
                struct list_head *tmp = next;
-               struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
+               struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
                next = tmp->next;
 
                if (!list_empty(&dentry->d_lru)) {
@@ -610,7 +610,7 @@ dentry->d_parent->d_name.name, dentry->d_name.name, found);
         * All done at this level ... ascend and resume the search.
         */
        if (this_parent != parent) {
-               next = this_parent->d_child.next; 
+               next = this_parent->d_u.d_child.next;
                this_parent = this_parent->d_parent;
 #ifdef DCACHE_DEBUG
 printk(KERN_DEBUG "select_parent: ascending to %s/%s, found=%d\n",
@@ -753,12 +753,12 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
                dentry->d_parent = dget(parent);
                dentry->d_sb = parent->d_sb;
        } else {
-               INIT_LIST_HEAD(&dentry->d_child);
+               INIT_LIST_HEAD(&dentry->d_u.d_child);
        }
 
        spin_lock(&dcache_lock);
        if (parent)
-               list_add(&dentry->d_child, &parent->d_subdirs);
+               list_add(&dentry->d_u.d_child, &parent->d_subdirs);
        dentry_stat.nr_dentry++;
        spin_unlock(&dcache_lock);
 
@@ -1310,8 +1310,8 @@ already_unhashed:
        /* Unhash the target: dput() will then get rid of it */
        __d_drop(target);
 
-       list_del(&dentry->d_child);
-       list_del(&target->d_child);
+       list_del(&dentry->d_u.d_child);
+       list_del(&target->d_u.d_child);
 
        /* Switch the names.. */
        switch_names(dentry, target);
@@ -1322,15 +1322,15 @@ already_unhashed:
        if (IS_ROOT(dentry)) {
                dentry->d_parent = target->d_parent;
                target->d_parent = target;
-               INIT_LIST_HEAD(&target->d_child);
+               INIT_LIST_HEAD(&target->d_u.d_child);
        } else {
                do_switch(dentry->d_parent, target->d_parent);
 
                /* And add them back to the (new) parent lists */
-               list_add(&target->d_child, &target->d_parent->d_subdirs);
+               list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
        }
 
-       list_add(&dentry->d_child, &dentry->d_parent->d_subdirs);
+       list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
        spin_unlock(&target->d_lock);
        spin_unlock(&dentry->d_lock);
        write_sequnlock(&rename_lock);
@@ -1568,7 +1568,7 @@ repeat:
 resume:
        while (next != &this_parent->d_subdirs) {
                struct list_head *tmp = next;
-               struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
+               struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
                next = tmp->next;
                if (d_unhashed(dentry)||!dentry->d_inode)
                        continue;
@@ -1579,7 +1579,7 @@ resume:
                atomic_dec(&dentry->d_count);
        }
        if (this_parent != root) {
-               next = this_parent->d_child.next; 
+               next = this_parent->d_u.d_child.next;
                atomic_dec(&this_parent->d_count);
                this_parent = this_parent->d_parent;
                goto resume;

diff --git a/fs/libfs.c b/fs/libfs.c
index 58101dff2c66de94fc4fa82bf13696e9d9b8d665..9c50523382e7eced4a45a44ce64e07ebaca5d9c5 100644 (file)
--- a/fs/libfs.c
+++ b/fs/libfs.c
@@ -93,16 +93,16 @@ loff_t dcache_dir_lseek(struct file *file, loff_t offset, int origin)
                        loff_t n = file->f_pos - 2;
 
                        spin_lock(&dcache_lock);
-                       list_del(&cursor->d_child);
+                       list_del(&cursor->d_u.d_child);
                        p = file->f_dentry->d_subdirs.next;
                        while (n && p != &file->f_dentry->d_subdirs) {
                                struct dentry *next;
-                               next = list_entry(p, struct dentry, d_child);
+                               next = list_entry(p, struct dentry, d_u.d_child);
                                if (!d_unhashed(next) && next->d_inode)
                                        n--;
                                p = p->next;
                        }
-                       list_add_tail(&cursor->d_child, p);
+                       list_add_tail(&cursor->d_u.d_child, p);
                        spin_unlock(&dcache_lock);
                }
        }
@@ -126,7 +126,7 @@ int dcache_readdir(struct file * filp, void * dirent, filldir_t filldir)
 {
        struct dentry *dentry = filp->f_dentry;
        struct dentry *cursor = filp->private_data;
-       struct list_head *p, *q = &cursor->d_child;
+       struct list_head *p, *q = &cursor->d_u.d_child;
        ino_t ino;
        int i = filp->f_pos;
 
@@ -153,7 +153,7 @@ int dcache_readdir(struct file * filp, void * dirent, filldir_t filldir)
                        }
                        for (p=q->next; p != &dentry->d_subdirs; p=p->next) {
                                struct dentry *next;
-                               next = list_entry(p, struct dentry, d_child);
+                               next = list_entry(p, struct dentry, d_u.d_child);
                                if (d_unhashed(next) || !next->d_inode)
                                        continue;
 
@@ -261,7 +261,7 @@ int simple_empty(struct dentry *dentry)
        int ret = 0;
 
        spin_lock(&dcache_lock);
-       list_for_each_entry(child, &dentry->d_subdirs, d_child)
+       list_for_each_entry(child, &dentry->d_subdirs, d_u.d_child)
                if (simple_positive(child))
                        goto out;
        ret = 1;

diff --git a/fs/ncpfs/dir.c b/fs/ncpfs/dir.c
index a9f7a8ab1d595d30fcb656eea4a1349a7f324d43..cfd76f431dc0ecba78d8e8bb96c668c402979cc5 100644 (file)
--- a/fs/ncpfs/dir.c
+++ b/fs/ncpfs/dir.c
@@ -365,7 +365,7 @@ ncp_dget_fpos(struct dentry *dentry, struct dentry *parent, unsigned long fpos)
        spin_lock(&dcache_lock);
        next = parent->d_subdirs.next;
        while (next != &parent->d_subdirs) {
-               dent = list_entry(next, struct dentry, d_child);
+               dent = list_entry(next, struct dentry, d_u.d_child);
                if ((unsigned long)dent->d_fsdata == fpos) {
                        if (dent->d_inode)
                                dget_locked(dent);

diff --git a/fs/ncpfs/ncplib_kernel.h b/fs/ncpfs/ncplib_kernel.h
index 9e4dc30c2435c207ddce4d2c67311db24429d457..799e5c2bec55bf7d1f401e2b207d2c1e96e5d312 100644 (file)
--- a/fs/ncpfs/ncplib_kernel.h
+++ b/fs/ncpfs/ncplib_kernel.h
@@ -196,7 +196,7 @@ ncp_renew_dentries(struct dentry *parent)
        spin_lock(&dcache_lock);
        next = parent->d_subdirs.next;
        while (next != &parent->d_subdirs) {
-               dentry = list_entry(next, struct dentry, d_child);
+               dentry = list_entry(next, struct dentry, d_u.d_child);
 
                if (dentry->d_fsdata == NULL)
                        ncp_age_dentry(server, dentry);
@@ -218,7 +218,7 @@ ncp_invalidate_dircache_entries(struct dentry *parent)
        spin_lock(&dcache_lock);
        next = parent->d_subdirs.next;
        while (next != &parent->d_subdirs) {
-               dentry = list_entry(next, struct dentry, d_child);
+               dentry = list_entry(next, struct dentry, d_u.d_child);
                dentry->d_fsdata = NULL;
                ncp_age_dentry(server, dentry);
                next = next->next;

diff --git a/fs/smbfs/cache.c b/fs/smbfs/cache.c
index f3e6b81288abf4b407daa02debcb45b10f35be2a..74b86d9725a63cce6a65ad53514cecc3d9437a82 100644 (file)
--- a/fs/smbfs/cache.c
+++ b/fs/smbfs/cache.c
@@ -66,7 +66,7 @@ smb_invalidate_dircache_entries(struct dentry *parent)
        spin_lock(&dcache_lock);
        next = parent->d_subdirs.next;
        while (next != &parent->d_subdirs) {
-               dentry = list_entry(next, struct dentry, d_child);
+               dentry = list_entry(next, struct dentry, d_u.d_child);
                dentry->d_fsdata = NULL;
                smb_age_dentry(server, dentry);
                next = next->next;
@@ -100,7 +100,7 @@ smb_dget_fpos(struct dentry *dentry, struct dentry *parent, unsigned long fpos)
        spin_lock(&dcache_lock);
        next = parent->d_subdirs.next;
        while (next != &parent->d_subdirs) {
-               dent = list_entry(next, struct dentry, d_child);
+               dent = list_entry(next, struct dentry, d_u.d_child);
                if ((unsigned long)dent->d_fsdata == fpos) {
                        if (dent->d_inode)
                                dget_locked(dent);

diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 46a2ba6175954880fcdfa89bc9a9a063f4e76443..a3ed5e059d479eb2ff7ee712f6adff5170c2a4be 100644 (file)
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -95,14 +95,19 @@ struct dentry {
        struct qstr d_name;
 
        struct list_head d_lru;         /* LRU list */
-       struct list_head d_child;       /* child of parent list */
+       /*
+        * d_child and d_rcu can share memory
+        */
+       union {
+               struct list_head d_child;       /* child of parent list */
+               struct rcu_head d_rcu;
+       } d_u;
        struct list_head d_subdirs;     /* our children */
        struct list_head d_alias;       /* inode alias list */
        unsigned long d_time;           /* used by d_revalidate */
        struct dentry_operations *d_op;
        struct super_block *d_sb;       /* The root of the dentry tree */
        void *d_fsdata;                 /* fs-specific data */
-       struct rcu_head d_rcu;
        struct dcookie_struct *d_cookie; /* cookie, if any */
        int d_mounted;
        unsigned char d_iname[DNAME_INLINE_LEN_MIN];    /* small names */

diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index e04c2da9dadbdd24b636ebd8f91884d03669d374..eab64e23bcae82a2e729e135d73c140ee21e8ef0 100644 (file)
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -331,7 +331,7 @@ static void cpuset_d_remove_dir(struct dentry *dentry)
        spin_lock(&dcache_lock);
        node = dentry->d_subdirs.next;
        while (node != &dentry->d_subdirs) {
-               struct dentry *d = list_entry(node, struct dentry, d_child);
+               struct dentry *d = list_entry(node, struct dentry, d_u.d_child);
                list_del_init(node);
                if (d->d_inode) {
                        d = dget_locked(d);
@@ -343,7 +343,7 @@ static void cpuset_d_remove_dir(struct dentry *dentry)
                }
                node = dentry->d_subdirs.next;
        }
-       list_del_init(&dentry->d_child);
+       list_del_init(&dentry->d_u.d_child);
        spin_unlock(&dcache_lock);
        remove_dir(dentry);
 }

diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c
index 24cc23af9b95d7d0ca1950ce7bcaf5ac098bdc6f..e14c1cae74600d5035b571a50414a65f7429ac8a 100644 (file)
--- a/net/sunrpc/rpc_pipe.c
+++ b/net/sunrpc/rpc_pipe.c
@@ -495,7 +495,7 @@ rpc_depopulate(struct dentry *parent)
 repeat:
        spin_lock(&dcache_lock);
        list_for_each_safe(pos, next, &parent->d_subdirs) {
-               dentry = list_entry(pos, struct dentry, d_child);
+               dentry = list_entry(pos, struct dentry, d_u.d_child);
                spin_lock(&dentry->d_lock);
                if (!d_unhashed(dentry)) {
                        dget_locked(dentry);

diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index e59da6398d44508ca902c7dad417ddb0cdba888a..b5fa02d17b1eae8ff3a863ec8520fae951b1de12 100644 (file)
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -889,7 +889,7 @@ static void sel_remove_bools(struct dentry *de)
        spin_lock(&dcache_lock);
        node = de->d_subdirs.next;
        while (node != &de->d_subdirs) {
-               struct dentry *d = list_entry(node, struct dentry, d_child);
+               struct dentry *d = list_entry(node, struct dentry, d_u.d_child);
                list_del_init(node);
 
                if (d->d_inode) {