From: Matthew Wilcox (Oracle) Date: Fri, 11 Jun 2021 14:36:16 +0000 (-0400) Subject: Convert rw_semaphore to use waitlists X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=71af1fae7e5839144ce9cad8d01cbf4d83d2ebbe;p=users%2Fwilly%2Flinux.git Convert rw_semaphore to use waitlists Saves one pointer per rw_semaphore, which shrinks struct inode by 16 bytes (as it contains two rw_semaphores), anon_vma by 8 bytes, mm_struct by 8 bytes and the superblock by 8 bytes. Individual filesystem per-inode data structures also often contain one or more rw_semaphore, so the savings may be even larger. Signed-off-by: Matthew Wilcox (Oracle) --- diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index a66038d88878..d89f6f460b26 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -44,7 +44,7 @@ struct rw_semaphore { struct optimistic_spin_queue osq; /* spinner MCS lock */ #endif raw_spinlock_t wait_lock; - struct list_head wait_list; + struct wlist_head waiters; #ifdef CONFIG_DEBUG_RWSEMS void *magic; #endif @@ -91,7 +91,7 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) .owner = ATOMIC_LONG_INIT(0), \ __RWSEM_OPT_INIT(name) \ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\ - .wait_list = LIST_HEAD_INIT((name).wait_list), \ + .waiters = WLIST_HEAD_INIT, \ __RWSEM_DEBUG_INIT(name) \ __RWSEM_DEP_MAP_INIT(name) } @@ -114,9 +114,9 @@ do { \ * rwsem to see if somebody from an incompatible type is wanting access to the * lock. */ -static inline int rwsem_is_contended(struct rw_semaphore *sem) +static inline bool rwsem_is_contended(struct rw_semaphore *sem) { - return !list_empty(&sem->wait_list); + return !wlist_empty(&sem->waiters); } /* diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 809b0016d344..d54878222f4a 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -70,7 +70,7 @@ #c, atomic_long_read(&(sem)->count), \ (unsigned long) sem->magic, \ atomic_long_read(&(sem)->owner), (long)current, \ - list_empty(&(sem)->wait_list) ? "" : "not ")) \ + wlist_empty(&(sem)->waiters) ? "" : "not ")) \ debug_locks_off(); \ } while (0) #else @@ -110,7 +110,7 @@ * 3) Error path of rwsem_down_write_slowpath(). * * For all the above cases, wait_lock will be held. A writer must also - * be the first one in the wait_list to be eligible for setting the handoff + * be the first one in the waiters list to be eligible for setting the handoff * bit. So concurrent setting/clearing of handoff bit is not possible. */ #define RWSEM_WRITER_LOCKED (1UL << 0) @@ -316,7 +316,7 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name, #endif atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); raw_spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); + INIT_WLIST_HEAD(&sem->waiters); atomic_long_set(&sem->owner, 0L); #ifdef CONFIG_RWSEM_SPIN_ON_OWNER osq_lock_init(&sem->osq); @@ -330,13 +330,13 @@ enum rwsem_waiter_type { }; struct rwsem_waiter { - struct list_head list; + struct wlist_node list; struct task_struct *task; enum rwsem_waiter_type type; unsigned long timeout; }; #define rwsem_first_waiter(sem) \ - list_first_entry(&sem->wait_list, struct rwsem_waiter, list) + wlist_first_entry(&sem->waiters, struct rwsem_waiter, list) enum rwsem_wake_type { RWSEM_WAKE_ANY, /* Wake whatever's at head of wait list */ @@ -381,9 +381,10 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, enum rwsem_wake_type wake_type, struct wake_q_head *wake_q) { - struct rwsem_waiter *waiter, *tmp; + struct rwsem_waiter *waiter; long oldcount, woken = 0, adjustment = 0; - struct list_head wlist; + struct wlist_node *next; + struct wlist_head wlist; lockdep_assert_held(&sem->wait_lock); @@ -473,13 +474,13 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, * 2) For each waiters in the new list, clear waiter->task and * put them into wake_q to be woken up later. */ - INIT_LIST_HEAD(&wlist); - list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { + INIT_WLIST_HEAD(&wlist); + wlist_for_each_entry_safe(waiter, next, &sem->waiters, list) { if (waiter->type == RWSEM_WAITING_FOR_WRITE) continue; woken++; - list_move_tail(&waiter->list, &wlist); + wlist_move(&sem->waiters, &waiter->list, &wlist); /* * Limit # of readers that can be woken up per wakeup call. @@ -490,7 +491,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, adjustment = woken * RWSEM_READER_BIAS - adjustment; lockevent_cond_inc(rwsem_wake_reader, woken); - if (list_empty(&sem->wait_list)) { + if (wlist_empty(&sem->waiters)) { /* hit end of list above */ adjustment -= RWSEM_FLAG_WAITERS; } @@ -506,7 +507,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, atomic_long_add(adjustment, &sem->count); /* 2nd pass */ - list_for_each_entry_safe(waiter, tmp, &wlist, list) { + wlist_for_each_entry_safe(waiter, next, &wlist, list) { struct task_struct *tsk; tsk = waiter->task; @@ -560,7 +561,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, new |= RWSEM_WRITER_LOCKED; new &= ~RWSEM_FLAG_HANDOFF; - if (list_is_singular(&sem->wait_list)) + if (wlist_is_singular(&sem->waiters)) new &= ~RWSEM_FLAG_WAITERS; } } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); @@ -919,7 +920,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, int state) */ if ((rcnt == 1) && (count & RWSEM_FLAG_WAITERS)) { raw_spin_lock_irq(&sem->wait_lock); - if (!list_empty(&sem->wait_list)) + if (!wlist_empty(&sem->waiters)) rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); raw_spin_unlock_irq(&sem->wait_lock); @@ -934,7 +935,7 @@ queue: waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT; raw_spin_lock_irq(&sem->wait_lock); - if (list_empty(&sem->wait_list)) { + if (wlist_empty(&sem->waiters)) { /* * In case the wait queue is empty and the lock isn't owned * by a writer or has the handoff bit set, this reader can @@ -952,7 +953,7 @@ queue: } adjustment += RWSEM_FLAG_WAITERS; } - list_add_tail(&waiter.list, &sem->wait_list); + wlist_add(&sem->waiters, &waiter.list); /* we're now waiting on the lock, but no longer actively locking */ count = atomic_long_add_return(adjustment, &sem->count); @@ -998,8 +999,7 @@ queue: return sem; out_nolock: - list_del(&waiter.list); - if (list_empty(&sem->wait_list)) { + if (wlist_del(&sem->waiters, &waiter.list)) { atomic_long_andnot(RWSEM_FLAG_WAITERS|RWSEM_FLAG_HANDOFF, &sem->count); } @@ -1037,10 +1037,10 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) raw_spin_lock_irq(&sem->wait_lock); - /* account for this before adding a new element to the list */ - wstate = list_empty(&sem->wait_list) ? WRITER_FIRST : WRITER_NOT_FIRST; - - list_add_tail(&waiter.list, &sem->wait_list); + if (wlist_add(&sem->waiters, &waiter.list)) + wstate = WRITER_FIRST; + else + wstate = WRITER_NOT_FIRST; /* we're now waiting on the lock */ if (wstate == WRITER_NOT_FIRST) { @@ -1136,7 +1136,7 @@ trylock_again: raw_spin_lock_irq(&sem->wait_lock); } __set_current_state(TASK_RUNNING); - list_del(&waiter.list); + wlist_del(&sem->waiters, &waiter.list); raw_spin_unlock_irq(&sem->wait_lock); lockevent_inc(rwsem_wlock); @@ -1145,12 +1145,11 @@ trylock_again: out_nolock: __set_current_state(TASK_RUNNING); raw_spin_lock_irq(&sem->wait_lock); - list_del(&waiter.list); if (unlikely(wstate == WRITER_HANDOFF)) atomic_long_add(-RWSEM_FLAG_HANDOFF, &sem->count); - if (list_empty(&sem->wait_list)) + if (wlist_del(&sem->waiters, &waiter.list)) atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count); else rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); @@ -1172,7 +1171,7 @@ static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem, long count) raw_spin_lock_irqsave(&sem->wait_lock, flags); - if (!list_empty(&sem->wait_list)) + if (!wlist_empty(&sem->waiters)) rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); raw_spin_unlock_irqrestore(&sem->wait_lock, flags); @@ -1193,7 +1192,7 @@ static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) raw_spin_lock_irqsave(&sem->wait_lock, flags); - if (!list_empty(&sem->wait_list)) + if (!wlist_empty(&sem->waiters)) rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); raw_spin_unlock_irqrestore(&sem->wait_lock, flags);