net: fix races in page->_count manipulation

This is illegal to use atomic_set(&page->_count, ...) even if we 'own'
the page. Other entities in the kernel need to use get_page_unless_zero()
to get a reference to the page before testing page properties, so we could
loose a refcount increment.

The only case it is valid is when page->_count is 0

Fixes: 540eb7bf0bbed ("net: Update alloc frag to reduce get/put page usage and recycle pages")
Signed-off-by: Eric Dumaze <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index a30d750647e7c63277113472fdeb3fd6c48d9c18..829d013745abf2940d923d192470a40fc19c5fd1 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -360,18 +360,29 @@ refill:
                                goto end;
                }
                nc->frag.size = PAGE_SIZE << order;
-recycle:
-               atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
+               /* Even if we own the page, we do not use atomic_set().
+                * This would break get_page_unless_zero() users.
+                */
+               atomic_add(NETDEV_PAGECNT_MAX_BIAS - 1,
+                          &nc->frag.page->_count);
                nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
                nc->frag.offset = 0;
        }
 
        if (nc->frag.offset + fragsz > nc->frag.size) {
-               /* avoid unnecessary locked operations if possible */
-               if ((atomic_read(&nc->frag.page->_count) == nc->pagecnt_bias) ||
-                   atomic_sub_and_test(nc->pagecnt_bias, &nc->frag.page->_count))
-                       goto recycle;
-               goto refill;
+               if (atomic_read(&nc->frag.page->_count) != nc->pagecnt_bias) {
+                       if (!atomic_sub_and_test(nc->pagecnt_bias,
+                                                &nc->frag.page->_count))
+                               goto refill;
+                       /* OK, page count is 0, we can safely set it */
+                       atomic_set(&nc->frag.page->_count,
+                                  NETDEV_PAGECNT_MAX_BIAS);
+               } else {
+                       atomic_add(NETDEV_PAGECNT_MAX_BIAS - nc->pagecnt_bias,
+                                  &nc->frag.page->_count);
+               }
+               nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
+               nc->frag.offset = 0;
        }
 
        data = page_address(nc->frag.page) + nc->frag.offset;