extern unsigned long stack_guard_gap;
/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
-extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked);
/* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */
-extern int expand_downwards(struct vm_area_struct *vma,
- unsigned long address);
+int expand_downwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked);
#if VM_GROWSUP
extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
#else
#endif
struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *,
+ unsigned long addr, bool write_locked);
int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t);
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
* PA-RISC uses this for its stack; IA64 for its Register Backing Store.
* vma is the last one with address > vma->vm_end. Have to extend vma.
*/
-int expand_upwards(struct vm_area_struct *vma, unsigned long address)
+int expand_upwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *next;
if (gap_addr < address || gap_addr > TASK_SIZE)
gap_addr = TASK_SIZE;
+ if (!write_locked)
+ return -EAGAIN;
next = find_vma_intersection(mm, vma->vm_end, gap_addr);
if (next && vma_is_accessible(next)) {
if (!(next->vm_flags & VM_GROWSUP))
/*
* vma is the first one with address < vma->vm_start. Have to extend vma.
*/
-int expand_downwards(struct vm_area_struct *vma, unsigned long address)
+int expand_downwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
struct mm_struct *mm = vma->vm_mm;
MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
/* Enforce stack_guard_gap */
prev = mas_prev(&mas, 0);
/* Check that both stack segments have the same anon_vma? */
- if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
- vma_is_accessible(prev)) {
- if (address - prev->vm_end < stack_guard_gap)
+ if (prev) {
+ if (!(prev->vm_flags & VM_GROWSDOWN) &&
+ vma_is_accessible(prev) &&
+ (address - prev->vm_end < stack_guard_gap))
return -ENOMEM;
+ if (!write_locked && (prev->vm_end == address))
+ return -EAGAIN;
}
if (mas_preallocate(&mas, GFP_KERNEL))
__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
#ifdef CONFIG_STACK_GROWSUP
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
- return expand_upwards(vma, address);
+ return expand_upwards(vma, address, write_locked);
}
-struct vm_area_struct *
-find_extend_vma(struct mm_struct *mm, unsigned long addr)
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm,
+ unsigned long addr, bool write_locked)
{
struct vm_area_struct *vma, *prev;
+ int err;
addr &= PAGE_MASK;
vma = find_vma_prev(mm, addr, &prev);
if (vma && (vma->vm_start <= addr))
return vma;
- if (!prev || expand_stack(prev, addr))
+ if (!prev)
+ return NULL;
+ if (write_locked) {
+ err = expand_stack(prev, addr, true);
+ } else {
+ mmap_read_unlock(mm);
+ mmap_write_lock(mm);
+ vma = find_vma_prev(mm, addr, &prev);
+ if (vma && (vma->vm_start <= addr)) {
+ mmap_write_downgrade(mm);
+ return vma;
+ }
+ if (prev)
+ err = expand_stack(prev, addr, true);
+ mmap_write_downgrade(mm);
+ }
+ if (err)
return NULL;
if (prev->vm_flags & VM_LOCKED)
populate_vma_page_range(prev, addr, prev->vm_end, NULL);
return prev;
}
#else
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
- return expand_downwards(vma, address);
+ if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
+ return -EINVAL;
+ return expand_downwards(vma, address, write_locked);
}
-struct vm_area_struct *
-find_extend_vma(struct mm_struct *mm, unsigned long addr)
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm,
+ unsigned long addr, bool write_locked)
{
struct vm_area_struct *vma;
unsigned long start;
+ int err;
addr &= PAGE_MASK;
vma = find_vma(mm, addr);
return NULL;
if (vma->vm_start <= addr)
return vma;
- if (!(vma->vm_flags & VM_GROWSDOWN))
- return NULL;
start = vma->vm_start;
- if (expand_stack(vma, addr))
+ err = expand_stack(vma, addr, write_locked);
+ if (unlikely(err == -EAGAIN)) {
+ mmap_read_unlock(mm);
+ mmap_write_lock(mm);
+ vma = find_vma(mm, addr);
+ if (vma)
+ err = expand_stack(vma, addr, true);
+ mmap_write_downgrade(mm);
+ }
+ if (err)
return NULL;
if (vma->vm_flags & VM_LOCKED)
populate_vma_page_range(vma, addr, start, NULL);
}
#endif
+struct vm_area_struct *find_extend_vma(struct mm_struct *mm,
+ unsigned long addr)
+{
+ return find_extend_vma_locked(mm, addr, false);
+}
EXPORT_SYMBOL_GPL(find_extend_vma);
/*