do_wait() optimization: do not place sub-threads on task_struct->children list

Thanks to Roland who pointed out de_thread() issues.

Currently we add sub-threads to ->real_parent->children list.  This buys
nothing but slows down do_wait().

With this patch ->children contains only main threads (group leaders).
The only complication is that forget_original_parent() should iterate over
sub-threads by hand, and de_thread() needs another list_replace() when it
changes ->group_leader.

Henceforth do_wait_thread() can never see task_detached() && !EXIT_DEAD
tasks, we can remove this check (and we can unify do_wait_thread() and
ptrace_do_wait()).

This change can confuse the optimistic search in mm_update_next_owner(),
but this is fixable and minor.

Perhaps badness() and oom_kill_process() should be updated, but they
should be fixed in any case.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Cc: Roland McGrath <roland@redhat.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Ratan Nalumasu <rnalumasu@gmail.com>
Cc: Vitaly Mayatskikh <vmayatsk@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/fs/exec.c b/fs/exec.c
index 623a5cc3076a114af8808ca030f04bdd82638829..77db9a97a773fff7ee88e1be4c65bb18347ae15c 100644 (file)
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -826,7 +826,9 @@ static int de_thread(struct task_struct *tsk)
                attach_pid(tsk, PIDTYPE_PID,  task_pid(leader));
                transfer_pid(leader, tsk, PIDTYPE_PGID);
                transfer_pid(leader, tsk, PIDTYPE_SID);
+
                list_replace_rcu(&leader->tasks, &tsk->tasks);
+               list_replace_init(&leader->sibling, &tsk->sibling);
 
                tsk->group_leader = tsk;
                leader->group_leader = tsk;

diff --git a/kernel/exit.c b/kernel/exit.c
index 5962d7ccf24371ed7aedf4832cac0cbd5e9c3820..546774a31a66752a698faa02c4e54f7c39fbef9e 100644 (file)
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -68,10 +68,10 @@ static void __unhash_process(struct task_struct *p)
                detach_pid(p, PIDTYPE_SID);
 
                list_del_rcu(&p->tasks);
+               list_del_init(&p->sibling);
                __get_cpu_var(process_counts)--;
        }
        list_del_rcu(&p->thread_group);
-       list_del_init(&p->sibling);
 }
 
 /*
@@ -736,12 +736,9 @@ static struct task_struct *find_new_reaper(struct task_struct *father)
 /*
 * Any that need to be release_task'd are put on the @dead list.
  */
-static void reparent_thread(struct task_struct *father, struct task_struct *p,
+static void reparent_leader(struct task_struct *father, struct task_struct *p,
                                struct list_head *dead)
 {
-       if (p->pdeath_signal)
-               group_send_sig_info(p->pdeath_signal, SEND_SIG_NOINFO, p);
-
        list_move_tail(&p->sibling, &p->real_parent->children);
 
        if (task_detached(p))
@@ -780,12 +777,18 @@ static void forget_original_parent(struct task_struct *father)
        reaper = find_new_reaper(father);
 
        list_for_each_entry_safe(p, n, &father->children, sibling) {
-               p->real_parent = reaper;
-               if (p->parent == father) {
-                       BUG_ON(task_ptrace(p));
-                       p->parent = p->real_parent;
-               }
-               reparent_thread(father, p, &dead_children);
+               struct task_struct *t = p;
+               do {
+                       t->real_parent = reaper;
+                       if (t->parent == father) {
+                               BUG_ON(task_ptrace(t));
+                               t->parent = t->real_parent;
+                       }
+                       if (t->pdeath_signal)
+                               group_send_sig_info(t->pdeath_signal,
+                                                   SEND_SIG_NOINFO, t);
+               } while_each_thread(p, t);
+               reparent_leader(father, p, &dead_children);
        }
        write_unlock_irq(&tasklist_lock);
 
@@ -1551,14 +1554,9 @@ static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
        struct task_struct *p;
 
        list_for_each_entry(p, &tsk->children, sibling) {
-               /*
-                * Do not consider detached threads.
-                */
-               if (!task_detached(p)) {
-                       int ret = wait_consider_task(wo, 0, p);
-                       if (ret)
-                               return ret;
-               }
+               int ret = wait_consider_task(wo, 0, p);
+               if (ret)
+                       return ret;
        }
 
        return 0;

diff --git a/kernel/fork.c b/kernel/fork.c
index 202a0ba63d3c198a2c891536fc93b749abb955b2..5b2959b3ffc2c239244c25548f3d5c5506baf674 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1291,7 +1291,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
        }
 
        if (likely(p->pid)) {
-               list_add_tail(&p->sibling, &p->real_parent->children);
                tracehook_finish_clone(p, clone_flags, trace);
 
                if (thread_group_leader(p)) {
@@ -1303,6 +1302,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
                        p->signal->tty = tty_kref_get(current->signal->tty);
                        attach_pid(p, PIDTYPE_PGID, task_pgrp(current));
                        attach_pid(p, PIDTYPE_SID, task_session(current));
+                       list_add_tail(&p->sibling, &p->real_parent->children);
                        list_add_tail_rcu(&p->tasks, &init_task.tasks);
                        __get_cpu_var(process_counts)++;
                }