xfs: add kmem_alloc_io()

Memory we use to submit for IO needs strict alignment to the
underlying driver contraints. Worst case, this is 512 bytes. Given
that all allocations for IO are always a power of 2 multiple of 512
bytes, the kernel heap provides natural alignment for objects of
these sizes and that suffices.

Until, of course, memory debugging of some kind is turned on (e.g.
red zones, poisoning, KASAN) and then the alignment of the heap
objects is thrown out the window. Then we get weird IO errors and
data corruption problems because drivers don't validate alignment
and do the wrong thing when passed unaligned memory buffers in bios.

TO fix this, introduce kmem_alloc_io(), which will guaranteeat least
512 byte alignment of buffers for IO, even if memory debugging
options are turned on. It is assumed that the minimum allocation
size will be 512 bytes, and that sizes will be power of 2 mulitples
of 512 bytes.

Use this everywhere we allocate buffers for IO.

This no longer fails with log recovery errors when KASAN is enabled
due to the brd driver not handling unaligned memory buffers:

# mkfs.xfs -f /dev/ram0 ; mount /dev/ram0 /mnt/test

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>

diff --git a/fs/xfs/kmem.c b/fs/xfs/kmem.c
index 9f32af534ce4f05c57d05a9d7d3a599aababa7f4..da031b93e1828dc83084ab29ef108eb203c42089 100644 (file)
--- a/fs/xfs/kmem.c
+++ b/fs/xfs/kmem.c
@@ -30,30 +30,24 @@ kmem_alloc(size_t size, xfs_km_flags_t flags)
        } while (1);
 }
 
-void *
-kmem_alloc_large(size_t size, xfs_km_flags_t flags)
+
+/*
+ * __vmalloc() will allocate data pages and auxillary structures (e.g.
+ * pagetables) with GFP_KERNEL, yet we may be under GFP_NOFS context here. Hence
+ * we need to tell memory reclaim that we are in such a context via
+ * PF_MEMALLOC_NOFS to prevent memory reclaim re-entering the filesystem here
+ * and potentially deadlocking.
+ */
+static void *
+__kmem_vmalloc(size_t size, xfs_km_flags_t flags)
 {
        unsigned nofs_flag = 0;
        void    *ptr;
-       gfp_t   lflags;
-
-       trace_kmem_alloc_large(size, flags, _RET_IP_);
-
-       ptr = kmem_alloc(size, flags | KM_MAYFAIL);
-       if (ptr)
-               return ptr;
+       gfp_t   lflags = kmem_flags_convert(flags);
 
-       /*
-        * __vmalloc() will allocate data pages and auxillary structures (e.g.
-        * pagetables) with GFP_KERNEL, yet we may be under GFP_NOFS context
-        * here. Hence we need to tell memory reclaim that we are in such a
-        * context via PF_MEMALLOC_NOFS to prevent memory reclaim re-entering
-        * the filesystem here and potentially deadlocking.
-        */
        if (flags & KM_NOFS)
                nofs_flag = memalloc_nofs_save();
 
-       lflags = kmem_flags_convert(flags);
        ptr = __vmalloc(size, lflags, PAGE_KERNEL);
 
        if (flags & KM_NOFS)
@@ -62,6 +56,44 @@ kmem_alloc_large(size_t size, xfs_km_flags_t flags)
        return ptr;
 }
 
+/*
+ * Same as kmem_alloc_large, except we guarantee the buffer returned is aligned
+ * to the @align_mask. We only guarantee alignment up to page size, we'll clamp
+ * alignment at page size if it is larger. vmalloc always returns a PAGE_SIZE
+ * aligned region.
+ */
+void *
+kmem_alloc_io(size_t size, int align_mask, xfs_km_flags_t flags)
+{
+       void    *ptr;
+
+       trace_kmem_alloc_io(size, flags, _RET_IP_);
+
+       if (WARN_ON_ONCE(align_mask >= PAGE_SIZE))
+               align_mask = PAGE_SIZE - 1;
+
+       ptr = kmem_alloc(size, flags | KM_MAYFAIL);
+       if (ptr) {
+               if (!((uintptr_t)ptr & align_mask))
+                       return ptr;
+               kfree(ptr);
+       }
+       return __kmem_vmalloc(size, flags);
+}
+
+void *
+kmem_alloc_large(size_t size, xfs_km_flags_t flags)
+{
+       void    *ptr;
+
+       trace_kmem_alloc_large(size, flags, _RET_IP_);
+
+       ptr = kmem_alloc(size, flags | KM_MAYFAIL);
+       if (ptr)
+               return ptr;
+       return __kmem_vmalloc(size, flags);
+}
+
 void *
 kmem_realloc(const void *old, size_t newsize, xfs_km_flags_t flags)
 {

diff --git a/fs/xfs/kmem.h b/fs/xfs/kmem.h
index cb6fa7984ffa088e9ceded5c6458bf56556446ec..8170d95cf930460c1ef0cfc944f06c634b4a43d5 100644 (file)
--- a/fs/xfs/kmem.h
+++ b/fs/xfs/kmem.h
@@ -53,6 +53,7 @@ kmem_flags_convert(xfs_km_flags_t flags)
 }
 
 extern void *kmem_alloc(size_t, xfs_km_flags_t);
+extern void *kmem_alloc_io(size_t size, int align_mask, xfs_km_flags_t flags);
 extern void *kmem_alloc_large(size_t size, xfs_km_flags_t);
 extern void *kmem_realloc(const void *, size_t, xfs_km_flags_t);
 static inline void  kmem_free(const void *ptr)

diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
index d3be9ab0359bcef06751bd24199eaa6bfd3eecb3..120ef99d09e833099985c87618da5eb49c2cc599 100644 (file)
--- a/fs/xfs/xfs_buf.c
+++ b/fs/xfs/xfs_buf.c
@@ -353,7 +353,8 @@ xfs_buf_allocate_memory(
         */
        size = BBTOB(bp->b_length);
        if (size < PAGE_SIZE) {
-               bp->b_addr = kmem_alloc(size, KM_NOFS);
+               int align_mask = xfs_buftarg_dma_alignment(bp->b_target);
+               bp->b_addr = kmem_alloc_io(size, align_mask, KM_NOFS);
                if (!bp->b_addr) {
                        /* low memory - use alloc_page loop instead */
                        goto use_alloc_page;
@@ -368,7 +369,7 @@ xfs_buf_allocate_memory(
                }
                bp->b_offset = offset_in_page(bp->b_addr);
                bp->b_pages = bp->b_page_array;
-               bp->b_pages[0] = virt_to_page(bp->b_addr);
+               bp->b_pages[0] = kmem_to_page(bp->b_addr);
                bp->b_page_count = 1;
                bp->b_flags |= _XBF_KMEM;
                return 0;

diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c
index 50d854bfc45cad784e0e9a543abaacb9b258aea9..3e0a105b6cc989c40a979bd3764787b837203230 100644 (file)
--- a/fs/xfs/xfs_log.c
+++ b/fs/xfs/xfs_log.c
@@ -1403,6 +1403,7 @@ xlog_alloc_log(
         */
        ASSERT(log->l_iclog_size >= 4096);
        for (i = 0; i < log->l_iclog_bufs; i++) {
+               int align_mask = xfs_buftarg_dma_alignment(mp->m_logdev_targp);
                size_t bvec_size = howmany(log->l_iclog_size, PAGE_SIZE) *
                                sizeof(struct bio_vec);
 
@@ -1414,8 +1415,8 @@ xlog_alloc_log(
                iclog->ic_prev = prev_iclog;
                prev_iclog = iclog;
 
-               iclog->ic_data = kmem_alloc_large(log->l_iclog_size,
-                               KM_MAYFAIL);
+               iclog->ic_data = kmem_alloc_io(log->l_iclog_size, align_mask,
+                                               KM_MAYFAIL);
                if (!iclog->ic_data)
                        goto out_free_iclog;
 #ifdef DEBUG

diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c
index eafb36cb4c66bb12f74585bf3200dceb6b7bed14..f05c6c99c4f32c89d66df427d90a7e1c8868ae1f 100644 (file)
--- a/fs/xfs/xfs_log_recover.c
+++ b/fs/xfs/xfs_log_recover.c
@@ -97,6 +97,8 @@ xlog_alloc_buffer(
        struct xlog     *log,
        int             nbblks)
 {
+       int align_mask = xfs_buftarg_dma_alignment(log->l_targ);
+
        /*
         * Pass log block 0 since we don't have an addr yet, buffer will be
         * verified on read.
@@ -125,7 +127,7 @@ xlog_alloc_buffer(
        if (nbblks > 1 && log->l_sectBBsize > 1)
                nbblks += log->l_sectBBsize;
        nbblks = round_up(nbblks, log->l_sectBBsize);
-       return kmem_alloc_large(BBTOB(nbblks), KM_MAYFAIL);
+       return kmem_alloc_io(BBTOB(nbblks), align_mask, KM_MAYFAIL);
 }
 
 /*

diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 8bb8b4704a00de6ac0bff80587f3eaa36d3fa51c..eaae275ed43084dc6ba14715cd854871341698b4 100644 (file)
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -3604,6 +3604,7 @@ DEFINE_EVENT(xfs_kmem_class, name, \
        TP_PROTO(ssize_t size, int flags, unsigned long caller_ip), \
        TP_ARGS(size, flags, caller_ip))
 DEFINE_KMEM_EVENT(kmem_alloc);
+DEFINE_KMEM_EVENT(kmem_alloc_io);
 DEFINE_KMEM_EVENT(kmem_alloc_large);
 DEFINE_KMEM_EVENT(kmem_realloc);
 DEFINE_KMEM_EVENT(kmem_zone_alloc);