Convert rw_semaphore to use waitlists

Saves one pointer per rw_semaphore, which shrinks struct inode by 16 bytes
(as it contains two rw_semaphores), anon_vma by 8 bytes, mm_struct by 8
bytes and the superblock by 8 bytes.  Individual filesystem per-inode
data structures also often contain one or more rw_semaphore, so the
savings may be even larger.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index a66038d88878f32df1e4ba2a64036dee648815a7..d89f6f460b26bf97fbc91813a0ed93db25d2e744 100644 (file)
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -44,7 +44,7 @@ struct rw_semaphore {
        struct optimistic_spin_queue osq; /* spinner MCS lock */
 #endif
        raw_spinlock_t wait_lock;
-       struct list_head wait_list;
+       struct wlist_head waiters;
 #ifdef CONFIG_DEBUG_RWSEMS
        void *magic;
 #endif
@@ -91,7 +91,7 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem)
          .owner = ATOMIC_LONG_INIT(0),                         \
          __RWSEM_OPT_INIT(name)                                \
          .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\
-         .wait_list = LIST_HEAD_INIT((name).wait_list),        \
+         .waiters = WLIST_HEAD_INIT,                           \
          __RWSEM_DEBUG_INIT(name)                              \
          __RWSEM_DEP_MAP_INIT(name) }
 
@@ -114,9 +114,9 @@ do {                                                                \
  * rwsem to see if somebody from an incompatible type is wanting access to the
  * lock.
  */
-static inline int rwsem_is_contended(struct rw_semaphore *sem)
+static inline bool rwsem_is_contended(struct rw_semaphore *sem)
 {
-       return !list_empty(&sem->wait_list);
+       return !wlist_empty(&sem->waiters);
 }
 
 /*

diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index 809b0016d3445f039ec1e369bf028b2fefdfbb48..d54878222f4a3139ea90e5fce88826aec181e84c 100644 (file)
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -70,7 +70,7 @@
                #c, atomic_long_read(&(sem)->count),            \
                (unsigned long) sem->magic,                     \
                atomic_long_read(&(sem)->owner), (long)current, \
-               list_empty(&(sem)->wait_list) ? "" : "not "))   \
+               wlist_empty(&(sem)->waiters) ? "" : "not "))    \
                        debug_locks_off();                      \
        } while (0)
 #else
@@ -110,7 +110,7 @@
  * 3) Error path of rwsem_down_write_slowpath().
  *
  * For all the above cases, wait_lock will be held. A writer must also
- * be the first one in the wait_list to be eligible for setting the handoff
+ * be the first one in the waiters list to be eligible for setting the handoff
  * bit. So concurrent setting/clearing of handoff bit is not possible.
  */
 #define RWSEM_WRITER_LOCKED    (1UL << 0)
@@ -316,7 +316,7 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
 #endif
        atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
        raw_spin_lock_init(&sem->wait_lock);
-       INIT_LIST_HEAD(&sem->wait_list);
+       INIT_WLIST_HEAD(&sem->waiters);
        atomic_long_set(&sem->owner, 0L);
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
        osq_lock_init(&sem->osq);
@@ -330,13 +330,13 @@ enum rwsem_waiter_type {
 };
 
 struct rwsem_waiter {
-       struct list_head list;
+       struct wlist_node list;
        struct task_struct *task;
        enum rwsem_waiter_type type;
        unsigned long timeout;
 };
 #define rwsem_first_waiter(sem) \
-       list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
+       wlist_first_entry(&sem->waiters, struct rwsem_waiter, list)
 
 enum rwsem_wake_type {
        RWSEM_WAKE_ANY,         /* Wake whatever's at head of wait list */
@@ -381,9 +381,10 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
                            enum rwsem_wake_type wake_type,
                            struct wake_q_head *wake_q)
 {
-       struct rwsem_waiter *waiter, *tmp;
+       struct rwsem_waiter *waiter;
        long oldcount, woken = 0, adjustment = 0;
-       struct list_head wlist;
+       struct wlist_node *next;
+       struct wlist_head wlist;
 
        lockdep_assert_held(&sem->wait_lock);
 
@@ -473,13 +474,13 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
         * 2) For each waiters in the new list, clear waiter->task and
         *    put them into wake_q to be woken up later.
         */
-       INIT_LIST_HEAD(&wlist);
-       list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {
+       INIT_WLIST_HEAD(&wlist);
+       wlist_for_each_entry_safe(waiter, next, &sem->waiters, list) {
                if (waiter->type == RWSEM_WAITING_FOR_WRITE)
                        continue;
 
                woken++;
-               list_move_tail(&waiter->list, &wlist);
+               wlist_move(&sem->waiters, &waiter->list, &wlist);
 
                /*
                 * Limit # of readers that can be woken up per wakeup call.
@@ -490,7 +491,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
 
        adjustment = woken * RWSEM_READER_BIAS - adjustment;
        lockevent_cond_inc(rwsem_wake_reader, woken);
-       if (list_empty(&sem->wait_list)) {
+       if (wlist_empty(&sem->waiters)) {
                /* hit end of list above */
                adjustment -= RWSEM_FLAG_WAITERS;
        }
@@ -506,7 +507,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem,
                atomic_long_add(adjustment, &sem->count);
 
        /* 2nd pass */
-       list_for_each_entry_safe(waiter, tmp, &wlist, list) {
+       wlist_for_each_entry_safe(waiter, next, &wlist, list) {
                struct task_struct *tsk;
 
                tsk = waiter->task;
@@ -560,7 +561,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
                        new |= RWSEM_WRITER_LOCKED;
                        new &= ~RWSEM_FLAG_HANDOFF;
 
-                       if (list_is_singular(&sem->wait_list))
+                       if (wlist_is_singular(&sem->waiters))
                                new &= ~RWSEM_FLAG_WAITERS;
                }
        } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new));
@@ -919,7 +920,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, int state)
                 */
                if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) {
                        raw_spin_lock_irq(&sem->wait_lock);
-                       if (!list_empty(&sem->wait_list))
+                       if (!wlist_empty(&sem->waiters))
                                rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED,
                                                &wake_q);
                        raw_spin_unlock_irq(&sem->wait_lock);
@@ -934,7 +935,7 @@ queue:
        waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;
 
        raw_spin_lock_irq(&sem->wait_lock);
-       if (list_empty(&sem->wait_list)) {
+       if (wlist_empty(&sem->waiters)) {
                /*
                 * In case the wait queue is empty and the lock isn't owned
                 * by a writer or has the handoff bit set, this reader can
@@ -952,7 +953,7 @@ queue:
                }
                adjustment += RWSEM_FLAG_WAITERS;
        }
-       list_add_tail(&waiter.list, &sem->wait_list);
+       wlist_add(&sem->waiters, &waiter.list);
 
        /* we're now waiting on the lock, but no longer actively locking */
        count = atomic_long_add_return(adjustment, &sem->count);
@@ -998,8 +999,7 @@ queue:
        return sem;
 
 out_nolock:
-       list_del(&waiter.list);
-       if (list_empty(&sem->wait_list)) {
+       if (wlist_del(&sem->waiters, &waiter.list)) {
                atomic_long_andnot(RWSEM_FLAG_WAITERS|RWSEM_FLAG_HANDOFF,
                                   &sem->count);
        }
@@ -1037,10 +1037,10 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
 
        raw_spin_lock_irq(&sem->wait_lock);
 
-       /* account for this before adding a new element to the list */
-       wstate = list_empty(&sem->wait_list) ? WRITER_FIRST : WRITER_NOT_FIRST;
-
-       list_add_tail(&waiter.list, &sem->wait_list);
+       if (wlist_add(&sem->waiters, &waiter.list))
+               wstate = WRITER_FIRST;
+       else
+               wstate = WRITER_NOT_FIRST;
 
        /* we're now waiting on the lock */
        if (wstate == WRITER_NOT_FIRST) {
@@ -1136,7 +1136,7 @@ trylock_again:
                raw_spin_lock_irq(&sem->wait_lock);
        }
        __set_current_state(TASK_RUNNING);
-       list_del(&waiter.list);
+       wlist_del(&sem->waiters, &waiter.list);
        raw_spin_unlock_irq(&sem->wait_lock);
        lockevent_inc(rwsem_wlock);
 
@@ -1145,12 +1145,11 @@ trylock_again:
 out_nolock:
        __set_current_state(TASK_RUNNING);
        raw_spin_lock_irq(&sem->wait_lock);
-       list_del(&waiter.list);
 
        if (unlikely(wstate == WRITER_HANDOFF))
                atomic_long_add(-RWSEM_FLAG_HANDOFF,  &sem->count);
 
-       if (list_empty(&sem->wait_list))
+       if (wlist_del(&sem->waiters, &waiter.list))
                atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count);
        else
                rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
@@ -1172,7 +1171,7 @@ static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem, long count)
 
        raw_spin_lock_irqsave(&sem->wait_lock, flags);
 
-       if (!list_empty(&sem->wait_list))
+       if (!wlist_empty(&sem->waiters))
                rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);
 
        raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
@@ -1193,7 +1192,7 @@ static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)
 
        raw_spin_lock_irqsave(&sem->wait_lock, flags);
 
-       if (!list_empty(&sem->wait_list))
+       if (!wlist_empty(&sem->waiters))
                rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q);
 
        raw_spin_unlock_irqrestore(&sem->wait_lock, flags);