From ddbd556497e2eab43d30eb3d5957e1f43262c849 Mon Sep 17 00:00:00 2001 From: Bang Nguyen Date: Tue, 11 Feb 2014 19:34:25 -0800 Subject: [PATCH] RDS: bind hash table size increase, add per-bucket rw lock The single global rw lock in bind hash table is replaced with a per-bucket lock. The size of the hash table is increased from 1024 to 8192. Orabug: 18071861 Tested-by: Michael Nowak Signed-off-by: Bang Nguyen Signed-off-by: Mukesh Kacker Acked-by: Venkat Venkatsubra (cherry picked from commit 4fc6d8f3870b5ac39415f475b2a5817a5a89d48e) --- net/rds/af_rds.c | 3 +++ net/rds/bind.c | 49 ++++++++++++++++++++++++++++++++---------------- net/rds/rds.h | 1 + 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/net/rds/af_rds.c b/net/rds/af_rds.c index ac41334f57c8b..7f7b2baa8d3ef 100644 --- a/net/rds/af_rds.c +++ b/net/rds/af_rds.c @@ -754,6 +754,7 @@ static void rds_exit(void) rds_page_exit(); rds_info_deregister_func(RDS_INFO_SOCKETS, rds_sock_info); rds_info_deregister_func(RDS_INFO_RECV_MESSAGES, rds_sock_inc_info); + } module_exit(rds_exit); @@ -761,6 +762,8 @@ static int rds_init(void) { int ret; + rds_bind_lock_init(); + ret = rds_conn_init(); if (ret) goto out; diff --git a/net/rds/bind.c b/net/rds/bind.c index 02975163bd065..cbbabb3a82577 100644 --- a/net/rds/bind.c +++ b/net/rds/bind.c @@ -37,11 +37,15 @@ #include #include "rds.h" -#define BIND_HASH_SIZE 1024 -static struct hlist_head bind_hash_table[BIND_HASH_SIZE]; -static DEFINE_RWLOCK(rds_bind_lock); +struct bind_bucket { + rwlock_t lock; + struct hlist_head head; +}; -static struct hlist_head *hash_to_bucket(__be32 addr, __be16 port) +#define BIND_HASH_SIZE 8192 +static struct bind_bucket bind_hash_table[BIND_HASH_SIZE]; + +static struct bind_bucket *hash_to_bucket(__be32 addr, __be16 port) { return bind_hash_table + (jhash_2words((u32)addr, (u32)port, 0) & (BIND_HASH_SIZE - 1)); @@ -50,11 +54,12 @@ static struct hlist_head *hash_to_bucket(__be32 addr, __be16 port) /* * must hold either read or write lock (write lock for insert != NULL) */ -static struct rds_sock *rds_bind_lookup(__be32 addr, __be16 port, +static struct rds_sock *rds_bind_lookup(struct bind_bucket *bucket, + __be32 addr, __be16 port, struct rds_sock *insert) { struct rds_sock *rs; - struct hlist_head *head = hash_to_bucket(addr, port); + struct hlist_head *head = &bucket->head; u64 cmp; u64 needle = ((u64)be32_to_cpu(addr) << 32) | be16_to_cpu(port); @@ -92,10 +97,11 @@ struct rds_sock *rds_find_bound(__be32 addr, __be16 port) { struct rds_sock *rs; unsigned long flags; + struct bind_bucket *bucket = hash_to_bucket(addr, port); - read_lock_irqsave(&rds_bind_lock, flags); - rs = rds_bind_lookup(addr, port, NULL); - read_unlock_irqrestore(&rds_bind_lock, flags); + read_lock_irqsave(&bucket->lock, flags); + rs = rds_bind_lookup(bucket, addr, port, NULL); + read_unlock_irqrestore(&bucket->lock, flags); if (rs && sock_flag(rds_rs_to_sk(rs), SOCK_DEAD)) { rds_sock_put(rs); @@ -114,6 +120,7 @@ static int rds_add_bound(struct rds_sock *rs, __be32 addr, __be16 *port) unsigned long flags; int ret = -EADDRINUSE; u16 rover, last; + struct bind_bucket *bucket; if (*port != 0) { rover = be16_to_cpu(*port); @@ -123,13 +130,16 @@ static int rds_add_bound(struct rds_sock *rs, __be32 addr, __be16 *port) last = rover - 1; } - write_lock_irqsave(&rds_bind_lock, flags); - do { struct rds_sock *rrs; if (rover == 0) rover++; - rrs = rds_bind_lookup(addr, cpu_to_be16(rover), rs); + + bucket = hash_to_bucket(addr, cpu_to_be16(rover)); + + write_lock_irqsave(&bucket->lock, flags); + rrs = rds_bind_lookup(bucket, addr, cpu_to_be16(rover), rs); + write_unlock_irqrestore(&bucket->lock, flags); if (!rrs) { *port = rs->rs_bound_port; @@ -141,16 +151,16 @@ static int rds_add_bound(struct rds_sock *rs, __be32 addr, __be16 *port) rds_sock_put(rrs); } while (rover++ != last); - write_unlock_irqrestore(&rds_bind_lock, flags); - return ret; } void rds_remove_bound(struct rds_sock *rs) { unsigned long flags; + struct bind_bucket *bucket = + hash_to_bucket(rs->rs_bound_addr, rs->rs_bound_port); - write_lock_irqsave(&rds_bind_lock, flags); + write_lock_irqsave(&bucket->lock, flags); if (rs->rs_bound_addr) { rdsdebug("rs %p unbinding from %pI4:%d\n", @@ -162,7 +172,7 @@ void rds_remove_bound(struct rds_sock *rs) rs->rs_bound_addr = 0; } - write_unlock_irqrestore(&rds_bind_lock, flags); + write_unlock_irqrestore(&bucket->lock, flags); } int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) @@ -204,3 +214,10 @@ out: release_sock(sk); return ret; } + +void rds_bind_lock_init(void) +{ + int i; + for (i = 0; i < BIND_HASH_SIZE; i++) + rwlock_init(&bind_hash_table[i].lock); +} diff --git a/net/rds/rds.h b/net/rds/rds.h index 4e747773f4495..4c764a75a742a 100644 --- a/net/rds/rds.h +++ b/net/rds/rds.h @@ -645,6 +645,7 @@ void debug_sock_put(struct sock *sock); int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len); void rds_remove_bound(struct rds_sock *rs); struct rds_sock *rds_find_bound(__be32 addr, __be16 port); +void rds_bind_lock_init(void); /* cong.c */ int rds_cong_get_maps(struct rds_connection *conn); -- 2.50.1