diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index e4399983c50c..25e6d34d3427 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -1237,6 +1237,7 @@ void do_user_addr_fault(struct pt_regs *regs,
struct mm_struct *mm;
vm_fault_t fault;
unsigned int flags = FAULT_FLAG_DEFAULT;
+ int err;
tsk = current;
mm = tsk->mm;
@@ -1405,11 +1406,16 @@ void do_user_addr_fault(struct pt_regs *regs,
}
if (likely(vma->vm_start <= address))
goto good_area;
- if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
- bad_area(regs, error_code, address);
- return;
+ err = expand_stack(vma, address, false);
+ if (err == -EAGAIN) {
+ mmap_read_unlock(mm);
+ mmap_write_lock(mm);
+ vma = find_vma(mm, address);
+ if (vma)
+ err = expand_stack(vma, address, true);
+ mmap_write_downgrade(mm);
}
- if (unlikely(expand_stack(vma, address))) {
+ if (unlikely(err)) {
bad_area(regs, error_code, address);
return;
}
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 44b4c42ab8e8..8acf480595b4 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -320,10 +320,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
* Grow the stack manually; some architectures have a limit on how
* far ahead a user-space access may be in order to grow the stack.
*/
- if (mmap_read_lock_killable(mm))
+ if (mmap_write_lock_killable(mm))
return -EINTR;
- vma = find_extend_vma(mm, bprm->p);
- mmap_read_unlock(mm);
+ vma = find_extend_vma_locked(mm, bprm->p, true);
+ mmap_write_unlock(mm);
if (!vma)
return -EFAULT;
diff --git a/fs/exec.c b/fs/exec.c
index a8863195df85..9c887d444d17 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -203,9 +203,11 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
#ifdef CONFIG_STACK_GROWSUP
if (write) {
- ret = expand_downwards(bprm->vma, pos);
+ mmap_write_lock(bprm->mm);
+ ret = expand_downwards(bprm->vma, pos, true);
if (ret < 0)
return NULL;
+ mmap_write_unlock(bprm->mm);
}
#endif
@@ -852,7 +854,7 @@ int setup_arg_pages(struct linux_binprm *bprm,
stack_base = vma->vm_end - stack_expand;
#endif
current->mm->start_stack = bprm->p;
- ret = expand_stack(vma, stack_base);
+ ret = expand_stack(vma, stack_base, true);
if (ret)
ret = -EFAULT;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 17317b1673b0..7a7ea1212a65 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3228,11 +3228,12 @@ extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
extern unsigned long stack_guard_gap;
/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
-extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked);
/* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */
-extern int expand_downwards(struct vm_area_struct *vma,
- unsigned long address);
+int expand_downwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked);
#if VM_GROWSUP
extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
#else
@@ -3333,6 +3334,8 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
#endif
struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *,
+ unsigned long addr, bool write_locked);
int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t);
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
diff --git a/mm/mmap.c b/mm/mmap.c
index 474a0d856622..0a1ac1fb121e 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1942,7 +1942,8 @@ static int acct_stack_growth(struct vm_area_struct *vma,
* PA-RISC uses this for its stack; IA64 for its Register Backing Store.
* vma is the last one with address > vma->vm_end. Have to extend vma.
*/
-int expand_upwards(struct vm_area_struct *vma, unsigned long address)
+int expand_upwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *next;
@@ -2038,13 +2039,17 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
/*
* vma is the first one with address < vma->vm_start. Have to extend vma.
*/
-int expand_downwards(struct vm_area_struct *vma, unsigned long address)
+int expand_downwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
struct mm_struct *mm = vma->vm_mm;
MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
struct vm_area_struct *prev;
int error = 0;
+ if (!(vma->vm_flags & VM_GROWSDOWN))
+ return -EFAULT;
+
address &= PAGE_MASK;
if (address < mmap_min_addr)
return -EPERM;
@@ -2137,34 +2142,37 @@ static int __init cmdline_parse_stack_guard_gap(char *p)
__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
#ifdef CONFIG_STACK_GROWSUP
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
- return expand_upwards(vma, address);
+ return expand_upwards(vma, address, write_locked);
}
-struct vm_area_struct *
-find_extend_vma(struct mm_struct *mm, unsigned long addr)
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm,
+ unsigned long addr, bool write_locked)
{
struct vm_area_struct *vma, *prev;
+ int err = 0;
addr &= PAGE_MASK;
vma = find_vma_prev(mm, addr, &prev);
if (vma && (vma->vm_start <= addr))
return vma;
- if (!prev || expand_stack(prev, addr))
+ if (!prev || expand_stack(prev, addr, write_locked))
return NULL;
if (prev->vm_flags & VM_LOCKED)
populate_vma_page_range(prev, addr, prev->vm_end, NULL);
return prev;
}
#else
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
- return expand_downwards(vma, address);
+ return expand_downwards(vma, address, write_locked);
}
-struct vm_area_struct *
-find_extend_vma(struct mm_struct *mm, unsigned long addr)
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm,
+ unsigned long addr, bool write_locked)
{
struct vm_area_struct *vma;
unsigned long start;
@@ -2178,7 +2186,7 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
if (!(vma->vm_flags & VM_GROWSDOWN))
return NULL;
start = vma->vm_start;
- if (expand_stack(vma, addr))
+ if (expand_stack(vma, addr, write_locked))
return NULL;
if (vma->vm_flags & VM_LOCKED)
populate_vma_page_range(vma, addr, start, NULL);
@@ -2186,6 +2194,11 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
}
#endif
+struct vm_area_struct *find_extend_vma(struct mm_struct *mm,
+ unsigned long addr)
+{
+ return find_extend_vma_locked(mm, addr, false);
+}
EXPORT_SYMBOL_GPL(find_extend_vma);
/*
diff --git a/mm/nommu.c b/mm/nommu.c
index a96b889cc17e..64573cf995a0 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -644,7 +644,8 @@ struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
* expand a stack to a given address
* - not supported under NOMMU conditions
*/
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
return -ENOMEM;
}