diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbddb3..79408a17a1b0da 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3695,14 +3695,7 @@ static inline void __pagetable_free(struct ptdesc *pt) __free_pages(page, compound_order(page)); } -#ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE void pagetable_free_kernel(struct ptdesc *pt); -#else -static inline void pagetable_free_kernel(struct ptdesc *pt) -{ - __pagetable_free(pt); -} -#endif /** * pagetable_free - Free pagetables * @pt: The page table descriptor diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 3ae2586ff45b44..ee7250ad057197 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -620,7 +620,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, * Note: Be careful to walk the kernel pages tables, the caller may be need to * take other effective approaches (mmap lock may be insufficient) to prevent * the intermediate kernel page tables belonging to the specified address range - * from being freed (e.g. memory hot-remove). + * from being freed (e.g. memory hot-remove, vmap huge page promotion). */ int walk_kernel_page_table_range(unsigned long start, unsigned long end, const struct mm_walk_ops *ops, pgd_t *pgd, void *private) @@ -643,7 +643,7 @@ int walk_kernel_page_table_range(unsigned long start, unsigned long end, * Use this function to walk the kernel page tables locklessly. It should be * guaranteed that the caller has exclusive access over the range they are * operating on - that there should be no concurrent access, for example, - * changing permissions for vmalloc objects. + * changing permissions for vmalloc objects, or vmap huge page promotion. */ int walk_kernel_page_table_range_lockless(unsigned long start, unsigned long end, const struct mm_walk_ops *ops, pgd_t *pgd, void *private) @@ -692,9 +692,34 @@ int walk_page_range_debug(struct mm_struct *mm, unsigned long start, }; /* For convenience, we allow traversal of kernel mappings. */ - if (mm == &init_mm) - return walk_kernel_page_table_range(start, end, ops, - pgd, private); + if (mm == &init_mm) { + unsigned long addr = start; + + /* + * Walk in bounded chunks so the RCU read lock is never held + * across the whole kernel address space. A kernel page table + * freed via pagetable_free_kernel() stays valid until the walk + * that may have observed it drops the lock; releasing the lock + * between chunks is safe as no page table pointer is held + * across the gap. + */ + while (addr < end) { + unsigned long next = min(end, ALIGN(addr + 1, PGDIR_SIZE)); + int err; + + rcu_read_lock(); + err = walk_kernel_page_table_range(addr, next, ops, + pgd, private); + rcu_read_unlock(); + if (err) + return err; + + addr = next; + cond_resched(); + } + return 0; + } + if (start >= end || !walk.mm) return -EINVAL; if (!check_ops_safe(ops)) diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a98029c7f..7a32e4821957cc 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -410,6 +410,13 @@ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, goto again; } +static void kernel_pgtable_free_rcu(struct rcu_head *head) +{ + struct ptdesc *pt = container_of(head, struct ptdesc, pt_rcu_head); + + __pagetable_free(pt); +} + #ifdef CONFIG_ASYNC_KERNEL_PGTABLE_FREE static void kernel_pgtable_work_func(struct work_struct *work); @@ -434,8 +441,15 @@ static void kernel_pgtable_work_func(struct work_struct *work) spin_unlock(&kernel_pgtable_work.lock); iommu_sva_invalidate_kva_range(PAGE_OFFSET, TLB_FLUSH_ALL); + + /* + * Debug walkers (ptdump) may walk ranges they do not own and race this + * free, so they walk under rcu_read_lock(). Free after a grace period: + * a walker either already saw the cleared PMD, or keeps the page alive + * until it drops the RCU lock. + */ list_for_each_entry_safe(pt, next, &page_list, pt_list) - __pagetable_free(pt); + call_rcu(&pt->pt_rcu_head, kernel_pgtable_free_rcu); } void pagetable_free_kernel(struct ptdesc *pt) @@ -446,4 +460,10 @@ void pagetable_free_kernel(struct ptdesc *pt) schedule_work(&kernel_pgtable_work.work); } +#else +void pagetable_free_kernel(struct ptdesc *pt) +{ + /* Defer the free by a grace period; see kernel_pgtable_work_func(). */ + call_rcu(&pt->pt_rcu_head, kernel_pgtable_free_rcu); +} #endif diff --git a/mm/ptdump.c b/mm/ptdump.c index 973020000096cf..4fc6313f290cab 100644 --- a/mm/ptdump.c +++ b/mm/ptdump.c @@ -177,13 +177,11 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) const struct ptdump_range *range = st->range; get_online_mems(); - mmap_write_lock(mm); while (range->start != range->end) { walk_page_range_debug(mm, range->start, range->end, &ptdump_ops, pgd, st); range++; } - mmap_write_unlock(mm); put_online_mems(); /* Flush out the last page */