net: make SK_MEMORY_PCPU_RESERV tunable

This patch adds /proc/sys/net/core/mem_pcpu_rsv sysctl file,
to make SK_MEMORY_PCPU_RESERV tunable.

Commit 3cd3399dd7a8 ("net: implement per-cpu reserves for
memory_allocated") introduced per-cpu forward alloc cache:

"Implement a per-cpu cache of +1/-1 MB, to reduce number
of changes to sk->sk_prot->memory_allocated, which
would otherwise be cause of false sharing."

sk_prot->memory_allocated points to global atomic variable:
atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;

If increasing the per-cpu cache size from 1MB to e.g. 16MB,
changes to sk->sk_prot->memory_allocated can be further reduced.
Performance may be improved on system with many cores.

Signed-off-by: Adam Li <adamli@os.amperecomputing.com>
Reviewed-by: Christoph Lameter (Ampere) <cl@linux.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst
index 3960916519557f60cb91a7477015a5aca285e268..7250c0542828b4990832b7249d7efaa2a2cd83cd 100644 (file)
--- a/Documentation/admin-guide/sysctl/net.rst
+++ b/Documentation/admin-guide/sysctl/net.rst
@@ -206,6 +206,11 @@ Will increase power usage.
 
 Default: 0 (off)
 
+mem_pcpu_rsv
+------------
+
+Per-cpu reserved forward alloc cache size in page units. Default 1MB per CPU.
+
 rmem_default
 ------------
 

diff --git a/include/net/sock.h b/include/net/sock.h
index 796a902cf4c19374ff2a44aa7fbce962e20a9e2e..09a0cde8bf52286167f3b9e9b32f632f3d1b5487 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1443,6 +1443,7 @@ sk_memory_allocated(const struct sock *sk)
 
 /* 1 MB per cpu, in page units */
 #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
+extern int sysctl_mem_pcpu_rsv;
 
 static inline void
 sk_memory_allocated_add(struct sock *sk, int amt)
@@ -1451,7 +1452,7 @@ sk_memory_allocated_add(struct sock *sk, int amt)
 
        preempt_disable();
        local_reserve = __this_cpu_add_return(*sk->sk_prot->per_cpu_fw_alloc, amt);
-       if (local_reserve >= SK_MEMORY_PCPU_RESERVE) {
+       if (local_reserve >= READ_ONCE(sysctl_mem_pcpu_rsv)) {
                __this_cpu_sub(*sk->sk_prot->per_cpu_fw_alloc, local_reserve);
                atomic_long_add(local_reserve, sk->sk_prot->memory_allocated);
        }
@@ -1465,7 +1466,7 @@ sk_memory_allocated_sub(struct sock *sk, int amt)
 
        preempt_disable();
        local_reserve = __this_cpu_sub_return(*sk->sk_prot->per_cpu_fw_alloc, amt);
-       if (local_reserve <= -SK_MEMORY_PCPU_RESERVE) {
+       if (local_reserve <= -READ_ONCE(sysctl_mem_pcpu_rsv)) {
                __this_cpu_sub(*sk->sk_prot->per_cpu_fw_alloc, local_reserve);
                atomic_long_add(local_reserve, sk->sk_prot->memory_allocated);
        }

diff --git a/net/core/sock.c b/net/core/sock.c
index 8d86886e39fa69bb18ded7781ef51f83851916b1..df2ac54a8f74e75881eb3dcf59e150a55b36eb99 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -283,6 +283,7 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
 EXPORT_SYMBOL(sysctl_rmem_max);
 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
+int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE;
 
 int sysctl_tstamp_allow_data __read_mostly = 1;
 

diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index 0f0cb1465e089245d0ddc0a2c6499bc54f1e9860..986f15e5d6c41250c8b9099fc1d2883112e77ffb 100644 (file)
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -30,6 +30,7 @@ static int int_3600 = 3600;
 static int min_sndbuf = SOCK_MIN_SNDBUF;
 static int min_rcvbuf = SOCK_MIN_RCVBUF;
 static int max_skb_frags = MAX_SKB_FRAGS;
+static int min_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE;
 
 static int net_msg_warn;       /* Unused, but still a sysctl */
 
@@ -407,6 +408,14 @@ static struct ctl_table net_core_table[] = {
                .proc_handler   = proc_dointvec_minmax,
                .extra1         = &min_rcvbuf,
        },
+       {
+               .procname       = "mem_pcpu_rsv",
+               .data           = &sysctl_mem_pcpu_rsv,
+               .maxlen         = sizeof(int),
+               .mode           = 0644,
+               .proc_handler   = proc_dointvec_minmax,
+               .extra1         = &min_mem_pcpu_rsv,
+       },
        {
                .procname       = "dev_weight",
                .data           = &weight_p,