IPC reduce lock contention in semctl

Signed-off-by: Chris Mason <chris.mason@oracle.com>

diff --git a/ipc/sem.c b/ipc/sem.c
index 3cf16b408f6d85ff8374681fefffa447203067f5..1dada2a9de7222eb71c3554b5dde15f73f04879e 100644 (file)
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -196,6 +196,12 @@ static inline void sem_lock_and_putref(struct sem_array *sma)
        ipc_rcu_putref(sma);
 }
 
+static inline void sem_read_lock_and_putref(struct sem_array *sma)
+{
+       ipc_read_lock_by_ptr(&sma->sem_perm);
+       ipc_rcu_putref(sma);
+}
+
 static inline void sem_getref_and_unlock(struct sem_array *sma)
 {
        ipc_rcu_getref(sma);
@@ -1017,8 +1023,9 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
        ushort* sem_io = fast_sem_io;
        int nsems;
        struct list_head tasks;
+       int write_locked = 0;
 
-       sma = sem_lock_check(ns, semid);
+       sma = sem_read_lock_check(ns, semid);
        if (IS_ERR(sma))
                return PTR_ERR(sma);
 
@@ -1042,7 +1049,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
                int i;
 
                if(nsems > SEMMSL_FAST) {
-                       sem_getref_and_unlock(sma);
+                       sem_getref_and_read_unlock(sma);
 
                        sem_io = ipc_alloc(sizeof(ushort)*nsems);
                        if(sem_io == NULL) {
@@ -1050,17 +1057,16 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
                                return -ENOMEM;
                        }
 
-                       sem_lock_and_putref(sma);
+                       sem_read_lock_and_putref(sma);
                        if (sma->sem_perm.deleted) {
-                               sem_unlock(sma);
                                err = -EIDRM;
-                               goto out_free;
+                               goto out_unlock;
                        }
                }
 
                for (i = 0; i < sma->sem_nsems; i++)
                        sem_io[i] = sma->sem_base[i].semval;
-               sem_unlock(sma);
+               sem_read_unlock(sma);
                err = 0;
                if(copy_to_user(array, sem_io, nsems*sizeof(ushort)))
                        err = -EFAULT;
@@ -1072,7 +1078,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
                struct sem_undo *un;
                LIST_HEAD(pending);
 
-               sem_getref_and_unlock(sma);
+               sem_getref_and_read_unlock(sma);
 
                if(nsems > SEMMSL_FAST) {
                        sem_io = ipc_alloc(sizeof(ushort)*nsems);
@@ -1095,11 +1101,19 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
                                goto out_free;
                        }
                }
-               sem_lock_and_putref(sma);
+               sem_read_lock_and_putref(sma);
+
+               /* no new undos can come in while we have the read lock
+                * but we have to take the write lock if there are any
+                */
+               if (!list_empty(&sma->list_id)) {
+                       sem_getref_and_read_unlock(sma);
+                       sem_lock_and_putref(sma);
+                       write_locked = 1;
+               }
                if (sma->sem_perm.deleted) {
-                       sem_unlock(sma);
                        err = -EIDRM;
-                       goto out_free;
+                       goto out_unlock;
                }
 
                for (i = 0; i < nsems; i++) {
@@ -1154,7 +1168,16 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
                if (val > SEMVMX || val < 0)
                        goto out_unlock;
 
-               //assert_spin_locked(&sma->sem_perm.lock);
+               if (!list_empty(&sma->list_id)) {
+                       sem_getref_and_read_unlock(sma);
+                       sem_lock_and_putref(sma);
+                       write_locked = 1;
+               }
+               if (sma->sem_perm.deleted) {
+                       err = -EIDRM;
+                       goto out_unlock;
+               }
+
                list_for_each_entry(un, &sma->list_id, list_id)
                        un->semadj[semnum] = 0;
 
@@ -1173,7 +1196,10 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
        }
        }
 out_unlock:
-       sem_unlock(sma);
+       if (write_locked)
+               sem_unlock(sma);
+       else
+               sem_read_unlock(sma);
        wake_up_sem_queue_do(&tasks);
 
 out_free:

diff --git a/ipc/util.h b/ipc/util.h
index 25e9808bac683134a93ff8aca6cf90e92d642b26..3fe3e6a01ac0f2dc0dbd8bab6721ca138b74e4c3 100644 (file)
--- a/ipc/util.h
+++ b/ipc/util.h
@@ -164,6 +164,12 @@ static inline void ipc_lock_by_ptr(struct kern_ipc_perm *perm)
        write_lock(&perm->lock);
 }
 
+static inline void ipc_read_lock_by_ptr(struct kern_ipc_perm *perm)
+{
+       rcu_read_lock();
+       read_lock(&perm->lock);
+}
+
 static inline void ipc_unlock(struct kern_ipc_perm *perm)
 {
        write_unlock(&perm->lock);