mm: fix struct page layout on 32-bit systems

commit 9ddb3c14afba8bc5950ed297f02d4ae05ff35cd1 upstream.

32-bit architectures which expect 8-byte alignment for 8-byte integers and
need 64-bit DMA addresses (arm, mips, ppc) had their struct page
inadvertently expanded in 2019.  When the dma_addr_t was added, it forced
the alignment of the union to 8 bytes, which inserted a 4 byte gap between
'flags' and the union.

Fix this by storing the dma_addr_t in one or two adjacent unsigned longs.
This restores the alignment to that of an unsigned long.  We always
store the low bits in the first word to prevent the PageTail bit from
being inadvertently set on a big endian platform.  If that happened,
get_user_pages_fast() racing against a page which was freed and
reallocated to the page_pool could dereference a bogus compound_head(),
which would be hard to trace back to this cause.

Link: https://lkml.kernel.org/r/20210510153211.1504886-1-willy@infradead.org
Fixes: c25fff7171be ("mm: add dma_addr_t to struct page")
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Acked-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Tested-by: Matteo Croce <mcroce@linux.microsoft.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 270aa8fd2800b4234297fa7c27577705f8c24395..2b3b2fc1cb33f23252b524ccbd0cc475df79cbf7 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -95,10 +95,10 @@ struct page {
                };
                struct {        /* page_pool used by netstack */
                        /**
-                        * @dma_addr: might require a 64-bit value even on
+                        * @dma_addr: might require a 64-bit value on
                         * 32-bit architectures.
                         */
-                       dma_addr_t dma_addr;
+                       unsigned long dma_addr[2];
                };
                struct {        /* slab, slob and slub */
                        union {

diff --git a/include/net/page_pool.h b/include/net/page_pool.h
index 1121faa99c1220235af460d001a005c9a081eabd..cf086e13bd253e43c9bed7b5f57c0cbfde745d63 100644 (file)
--- a/include/net/page_pool.h
+++ b/include/net/page_pool.h
@@ -185,7 +185,17 @@ static inline void page_pool_release_page(struct page_pool *pool,
 
 static inline dma_addr_t page_pool_get_dma_addr(struct page *page)
 {
-       return page->dma_addr;
+       dma_addr_t ret = page->dma_addr[0];
+       if (sizeof(dma_addr_t) > sizeof(unsigned long))
+               ret |= (dma_addr_t)page->dma_addr[1] << 16 << 16;
+       return ret;
+}
+
+static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
+{
+       page->dma_addr[0] = addr;
+       if (sizeof(dma_addr_t) > sizeof(unsigned long))
+               page->dma_addr[1] = upper_32_bits(addr);
 }
 
 static inline bool is_page_pool_compiled_in(void)

diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index dfc2501c35d9c8e69f97cb82e7db6764398f9bcd..335f68eaaa05c9f14d8377cc940f38f7b48b14ff 100644 (file)
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -157,7 +157,7 @@ static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
                put_page(page);
                return NULL;
        }
-       page->dma_addr = dma;
+       page_pool_set_dma_addr(page, dma);
 
 skip_dma_map:
        /* Track how many pages are held 'in-flight' */
@@ -216,12 +216,12 @@ static void __page_pool_clean_page(struct page_pool *pool,
        if (!(pool->p.flags & PP_FLAG_DMA_MAP))
                goto skip_dma_unmap;
 
-       dma = page->dma_addr;
+       dma = page_pool_get_dma_addr(page);
        /* DMA unmap */
        dma_unmap_page_attrs(pool->p.dev, dma,
                             PAGE_SIZE << pool->p.order, pool->p.dma_dir,
                             DMA_ATTR_SKIP_CPU_SYNC);
-       page->dma_addr = 0;
+       page_pool_set_dma_addr(page, 0);
 skip_dma_unmap:
        /* This may be the last page returned, releasing the pool, so
         * it is not safe to reference pool afterwards.