dlan17/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mm/huge_memory: move unrelated code out of __split_unmapped_folio()

Browse Source 
Patch series "__folio_split() clean up", v5.

This patchset refactors __folio_split() and __split_unmapped_folio() to:
1. make __split_unmapped_folio() reusable for splitting unmapped
   folios.  It avoids the need for a new boolean unmapped parameter to
   guard mapping-related code when __split_unmapped_folio() is reused to
   split unmapped folios.
2. improve code readability and prevent smatch/coverity checkers from
   complaining about NULL mapping referencing.

An additional benefit for __split_unmapped_folio() refactoring is that
__split_unmapped_folio() could be called on after-split folios by
__folio_split().  It can enable new split methods.  For example, at
deferred split time, unmapped subpages can scatter arbitrarily within a
large folio, neither uniform nor non-uniform split can maximize
after-split folio orders for mapped subpages.  The hope is that by calling
__split_unmapped_folio() multiple times, a better split result can be
achieved.


This patch (of 6):

remap(), folio_ref_unfreeze(), lru_add_split_folio() are not relevant to
splitting unmapped folio operations.  Move them out to __folio_split() so
that __split_unmapped_folio() only handles unmapped folio splits.  This
makes __split_unmapped_folio() reusable.

Remove the swapcache folio split check code before
__split_unmapped_folio() call, since it is already checked at the
beginning of __folio_split() in uniform_split_supported() and
non_uniform_split_supported().

Along with the code move, there are some variable renames:

1. release is renamed to new_folio,
2. origin_folio is now folio, since __folio_split() has folio pointing to
   the original folio already.

Link: https://lkml.kernel.org/r/20250718023000.4044406-1-ziy@nvidia.com
Link: https://lkml.kernel.org/r/20250718023000.4044406-2-ziy@nvidia.com
Link: https://lkml.kernel.org/r/20250718183720.4054515-1-ziy@nvidia.com
Link: https://lkml.kernel.org/r/20250718183720.4054515-2-ziy@nvidia.com
Signed-off-by: Zi Yan <ziy@nvidia.com>
Acked-by: David Hildenbrand <david@redhat.com>
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Antonio Quartulli <antonio@mandelbit.com>
Cc: Balbir Singh <balbirs@nvidia.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Dan Carpenter <dan.carpenter@linaro.org>
Cc: Dev Jain <dev.jain@arm.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Kirill A. Shutemov <k.shutemov@gmail.com>
Cc: Liam Howlett <liam.howlett@oracle.com>
Cc: Mariano Pache <npache@redhat.com>
Cc: Mathew Brost <matthew.brost@intel.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Zi Yan
2025-07-24 19:12:38 -07:00
committed by Andrew Morton
parent 6470fb2bb1
commit 6c7de9c83b
1 changed files with 133 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mm/huge_memory.c
+133 -137
View File
@@ -3385,10 +3385,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
* order - 1 to new_order).
* @split_at: in buddy allocator like split, the folio containing @split_at
* will be split until its order becomes @new_order.
* @lock_at: the folio containing @lock_at is left locked for caller.
* @list: the after split folios will be added to @list if it is not NULL,
* otherwise to LRU lists.
* @end: the end of the file @folio maps to. -1 if @folio is anonymous memory.
* @xas: xa_state pointing to folio->mapping->i_pages and locked by caller
* @mapping: @folio->mapping
* @uniform_split: if the split is uniform or not (buddy allocator like split)
@@ -3414,52 +3410,26 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
* @page, which is split in next for loop.
*
* After splitting, the caller's folio reference will be transferred to the
* folio containing @page. The other folios may be freed if they are not mapped.
*
* In terms of locking, after splitting,
* 1. uniform split leaves @page (or the folio contains it) locked;
* 2. buddy allocator like (non-uniform) split leaves @folio locked.
*
* folio containing @page. The caller needs to unlock and/or free after-split
* folios if necessary.
*
* For !uniform_split, when -ENOMEM is returned, the original folio might be
* split. The caller needs to check the input folio.
*/
static int __split_unmapped_folio(struct folio *folio, int new_order,
struct page *split_at, struct page *lock_at,
struct list_head *list, pgoff_t end,
struct xa_state *xas, struct address_space *mapping,
bool uniform_split)
struct page *split_at, struct xa_state *xas,
struct address_space *mapping, bool uniform_split)
{
struct lruvec *lruvec;
struct address_space *swap_cache = NULL;
struct folio *origin_folio = folio;
struct folio *next_folio = folio_next(folio);
struct folio *new_folio;
struct folio *next;
int order = folio_order(folio);
int split_order;
int start_order = uniform_split ? new_order : order - 1;
int nr_dropped = 0;
int ret = 0;
bool stop_split = false;
if (folio_test_swapcache(folio)) {
VM_BUG_ON(mapping);
/* a swapcache folio can only be uniformly split to order-0 */
if (!uniform_split || new_order != 0)
return -EINVAL;
swap_cache = swap_address_space(folio->swap);
xa_lock(&swap_cache->i_pages);
}
struct folio *next;
int split_order;
int ret = 0;
if (folio_test_anon(folio))
mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
/* lock lru list/PageCompound, ref frozen by page_ref_freeze */
lruvec = folio_lruvec_lock(folio);
folio_clear_has_hwpoisoned(folio);
/*
@@ -3469,9 +3439,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
for (split_order = start_order;
split_order >= new_order && !stop_split;
split_order--) {
int old_order = folio_order(folio);
struct folio *release;
struct folio *end_folio = folio_next(folio);
int old_order = folio_order(folio);
struct folio *new_folio;
/* order-1 anonymous folio is not supported */
if (folio_test_anon(folio) && split_order == 1)
@@ -3506,113 +3476,32 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
after_split:
/*
* Iterate through after-split folios and perform related
* operations. But in buddy allocator like split, the folio
* Iterate through after-split folios and update folio stats.
* But in buddy allocator like split, the folio
* containing the specified page is skipped until its order
* is new_order, since the folio will be worked on in next
* iteration.
*/
for (release = folio; release != end_folio; release = next) {
next = folio_next(release);
for (new_folio = folio; new_folio != end_folio; new_folio = next) {
next = folio_next(new_folio);
/*
* for buddy allocator like split, the folio containing
* page will be split next and should not be released,
* until the folio's order is new_order or stop_split
* is set to true by the above xas_split() failure.
* for buddy allocator like split, new_folio containing
* @split_at page could be split again, thus do not
* change stats yet. Wait until new_folio's order is
* @new_order or stop_split is set to true by the above
* xas_split() failure.
*/
if (release == page_folio(split_at)) {
folio = release;
if (new_folio == page_folio(split_at)) {
folio = new_folio;
if (split_order != new_order && !stop_split)
continue;
}
if (folio_test_anon(release)) {
mod_mthp_stat(folio_order(release),
MTHP_STAT_NR_ANON, 1);
}
/*
* origin_folio should be kept frozon until page cache
* entries are updated with all the other after-split
* folios to prevent others seeing stale page cache
* entries.
*/
if (release == origin_folio)
continue;
folio_ref_unfreeze(release, 1 +
((mapping || swap_cache) ?
folio_nr_pages(release) : 0));
lru_add_split_folio(origin_folio, release, lruvec,
list);
/* Some pages can be beyond EOF: drop them from cache */
if (release->index >= end) {
if (shmem_mapping(mapping))
nr_dropped += folio_nr_pages(release);
else if (folio_test_clear_dirty(release))
folio_account_cleaned(release,
inode_to_wb(mapping->host));
__filemap_remove_folio(release, NULL);
folio_put_refs(release, folio_nr_pages(release));
} else if (mapping) {
__xa_store(&mapping->i_pages,
release->index, release, 0);
} else if (swap_cache) {
__xa_store(&swap_cache->i_pages,
swap_cache_index(release->swap),
release, 0);
}
if (folio_test_anon(new_folio))
mod_mthp_stat(folio_order(new_folio),
MTHP_STAT_NR_ANON, 1);
}
}
/*
* Unfreeze origin_folio only after all page cache entries, which used
* to point to it, have been updated with new folios. Otherwise,
* a parallel folio_try_get() can grab origin_folio and its caller can
* see stale page cache entries.
*/
folio_ref_unfreeze(origin_folio, 1 +
((mapping || swap_cache) ? folio_nr_pages(origin_folio) : 0));
unlock_page_lruvec(lruvec);
if (swap_cache)
xa_unlock(&swap_cache->i_pages);
if (mapping)
xa_unlock(&mapping->i_pages);
/* Caller disabled irqs, so they are still disabled here */
local_irq_enable();
if (nr_dropped)
shmem_uncharge(mapping->host, nr_dropped);
remap_page(origin_folio, 1 << order,
folio_test_anon(origin_folio) ?
RMP_USE_SHARED_ZEROPAGE : 0);
/*
* At this point, folio should contain the specified page.
* For uniform split, it is left for caller to unlock.
* For buddy allocator like split, the first after-split folio is left
* for caller to unlock.
*/
for (new_folio = origin_folio; new_folio != next_folio; new_folio = next) {
next = folio_next(new_folio);
if (new_folio == page_folio(lock_at))
continue;
folio_unlock(new_folio);
/*
* Subpages may be freed if there wasn't any mapping
* like if add_to_swap() is running on a lru page that
* had its mapping zapped. And freeing these pages
* requires taking the lru_lock so we do the put_page
* of the tail pages after the split is complete.
*/
free_folio_and_swap_cache(new_folio);
}
return ret;
}
@@ -3686,6 +3575,11 @@ bool uniform_split_supported(struct folio *folio, unsigned int new_order,
* It is in charge of checking whether the split is supported or not and
* preparing @folio for __split_unmapped_folio().
*
* After splitting, the after-split folio containing @lock_at remains locked
* and others are unlocked:
* 1. for uniform split, @lock_at points to one of @folio's subpages;
* 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio.
*
* return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be
* split but not to @new_order, the caller needs to check)
*/
@@ -3695,10 +3589,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
{
struct deferred_split *ds_queue = get_deferred_split_queue(folio);
XA_STATE(xas, &folio->mapping->i_pages, folio->index);
struct folio *end_folio = folio_next(folio);
bool is_anon = folio_test_anon(folio);
struct address_space *mapping = NULL;
struct anon_vma *anon_vma = NULL;
int order = folio_order(folio);
struct folio *new_folio, *next;
int extra_pins, ret;
pgoff_t end;
bool is_hzp;
@@ -3829,6 +3725,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
/* Prevent deferred_split_scan() touching ->_refcount */
spin_lock(&ds_queue->split_queue_lock);
if (folio_ref_freeze(folio, 1 + extra_pins)) {
struct address_space *swap_cache = NULL;
int nr_dropped = 0;
struct lruvec *lruvec;
if (folio_order(folio) > 1 &&
!list_empty(&folio->_deferred_list)) {
ds_queue->split_queue_len--;
@@ -3862,9 +3762,105 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
}
}
ret = __split_unmapped_folio(folio, new_order,
split_at, lock_at, list, end, &xas, mapping,
uniform_split);
if (folio_test_swapcache(folio)) {
VM_BUG_ON(mapping);
swap_cache = swap_address_space(folio->swap);
xa_lock(&swap_cache->i_pages);
}
/* lock lru list/PageCompound, ref frozen by page_ref_freeze */
lruvec = folio_lruvec_lock(folio);
ret = __split_unmapped_folio(folio, new_order, split_at, &xas,
mapping, uniform_split);
/*
* Unfreeze after-split folios and put them back to the right
* list. @folio should be kept frozon until page cache
* entries are updated with all the other after-split folios
* to prevent others seeing stale page cache entries.
* As a result, new_folio starts from the next folio of
* @folio.
*/
for (new_folio = folio_next(folio); new_folio != end_folio;
new_folio = next) {
next = folio_next(new_folio);
folio_ref_unfreeze(
new_folio,
1 + ((mapping || swap_cache) ?
folio_nr_pages(new_folio) :
0));
lru_add_split_folio(folio, new_folio, lruvec, list);
/* Some pages can be beyond EOF: drop them from cache */
if (new_folio->index >= end) {
if (shmem_mapping(mapping))
nr_dropped += folio_nr_pages(new_folio);
else if (folio_test_clear_dirty(new_folio))
folio_account_cleaned(
new_folio,
inode_to_wb(mapping->host));
__filemap_remove_folio(new_folio, NULL);
folio_put_refs(new_folio,
folio_nr_pages(new_folio));
} else if (mapping) {
__xa_store(&mapping->i_pages, new_folio->index,
new_folio, 0);
} else if (swap_cache) {
__xa_store(&swap_cache->i_pages,
swap_cache_index(new_folio->swap),
new_folio, 0);
}
}
/*
* Unfreeze @folio only after all page cache entries, which
* used to point to it, have been updated with new folios.
* Otherwise, a parallel folio_try_get() can grab @folio
* and its caller can see stale page cache entries.
*/
folio_ref_unfreeze(folio, 1 +
((mapping || swap_cache) ? folio_nr_pages(folio) : 0));
unlock_page_lruvec(lruvec);
if (swap_cache)
xa_unlock(&swap_cache->i_pages);
if (mapping)
xas_unlock(&xas);
local_irq_enable();
if (nr_dropped)
shmem_uncharge(mapping->host, nr_dropped);
remap_page(folio, 1 << order,
!ret && folio_test_anon(folio) ?
RMP_USE_SHARED_ZEROPAGE :
0);
/*
* Unlock all after-split folios except the one containing
* @lock_at page. If @folio is not split, it will be kept locked.
*/
for (new_folio = folio; new_folio != end_folio;
new_folio = next) {
next = folio_next(new_folio);
if (new_folio == page_folio(lock_at))
continue;
folio_unlock(new_folio);
/*
* Subpages may be freed if there wasn't any mapping
* like if add_to_swap() is running on a lru page that
* had its mapping zapped. And freeing these pages
* requires taking the lru_lock so we do the put_page
* of the tail pages after the split is complete.
*/
free_folio_and_swap_cache(new_folio);
}
} else {
spin_unlock(&ds_queue->split_queue_lock);
fail: