1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/swap_state.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 * Swap reorganised 29.12.95, Stephen Tweedie
7 *
8 * Rewritten to use page cache, (C) 1998 Stephen Tweedie
9 */
10#include <linux/mm.h>
11#include <linux/gfp.h>
12#include <linux/kernel_stat.h>
13#include <linux/mempolicy.h>
14#include <linux/swap.h>
15#include <linux/swapops.h>
16#include <linux/init.h>
17#include <linux/pagemap.h>
18#include <linux/pagevec.h>
19#include <linux/backing-dev.h>
20#include <linux/blkdev.h>
21#include <linux/migrate.h>
22#include <linux/vmalloc.h>
23#include <linux/huge_mm.h>
24#include <linux/shmem_fs.h>
25#include "internal.h"
26#include "swap_table.h"
27#include "swap.h"
28
29/*
30 * swapper_space is a fiction, retained to simplify the path through
31 * vmscan's shrink_folio_list.
32 */
33static const struct address_space_operations swap_aops = {
34 .dirty_folio = noop_dirty_folio,
35#ifdef CONFIG_MIGRATION
36 .migrate_folio = migrate_folio,
37#endif
38};
39
40/* Set swap_space as read only as swap cache is handled by swap table */
41struct address_space swap_space __ro_after_init = {
42 .a_ops = &swap_aops,
43};
44
45static bool enable_vma_readahead __read_mostly = true;
46
47#define SWAP_RA_ORDER_CEILING 5
48
49#define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2)
50#define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1)
51#define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK
52#define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK)
53
54#define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK)
55#define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT)
56#define SWAP_RA_ADDR(v) ((v) & PAGE_MASK)
57
58#define SWAP_RA_VAL(addr, win, hits) \
59 (((addr) & PAGE_MASK) | \
60 (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \
61 ((hits) & SWAP_RA_HITS_MASK))
62
63/* Initial readahead hits is 4 to start up with a small window */
64#define GET_SWAP_RA_VAL(vma) \
65 (atomic_long_read(&(vma)->swap_readahead_info) ? : 4)
66
67static atomic_t swapin_readahead_hits = ATOMIC_INIT(4);
68
69void show_swap_cache_info(void)
70{
71 printk("%lu pages in swap cache\n", total_swapcache_pages());
72 printk("Free swap = %ldkB\n", K(get_nr_swap_pages()));
73 printk("Total swap = %lukB\n", K(total_swap_pages));
74}
75
76/**
77 * swap_cache_get_folio - Looks up a folio in the swap cache.
78 * @entry: swap entry used for the lookup.
79 *
80 * A found folio will be returned unlocked and with its refcount increased.
81 *
82 * Context: Caller must ensure @entry is valid and protect the swap device
83 * with reference count or locks.
84 * Return: Returns the found folio on success, NULL otherwise. The caller
85 * must lock nd check if the folio still matches the swap entry before
86 * use (e.g., folio_matches_swap_entry).
87 */
88struct folio *swap_cache_get_folio(swp_entry_t entry)
89{
90 unsigned long swp_tb;
91 struct folio *folio;
92
93 for (;;) {
94 swp_tb = swap_table_get(ci: __swap_entry_to_cluster(entry),
95 off: swp_cluster_offset(entry));
96 if (!swp_tb_is_folio(swp_tb))
97 return NULL;
98 folio = swp_tb_to_folio(swp_tb);
99 if (likely(folio_try_get(folio)))
100 return folio;
101 }
102
103 return NULL;
104}
105
106/**
107 * swap_cache_get_shadow - Looks up a shadow in the swap cache.
108 * @entry: swap entry used for the lookup.
109 *
110 * Context: Caller must ensure @entry is valid and protect the swap device
111 * with reference count or locks.
112 * Return: Returns either NULL or an XA_VALUE (shadow).
113 */
114void *swap_cache_get_shadow(swp_entry_t entry)
115{
116 unsigned long swp_tb;
117
118 swp_tb = swap_table_get(ci: __swap_entry_to_cluster(entry),
119 off: swp_cluster_offset(entry));
120 if (swp_tb_is_shadow(swp_tb))
121 return swp_tb_to_shadow(swp_tb);
122 return NULL;
123}
124
125/**
126 * swap_cache_add_folio - Add a folio into the swap cache.
127 * @folio: The folio to be added.
128 * @entry: The swap entry corresponding to the folio.
129 * @gfp: gfp_mask for XArray node allocation.
130 * @shadowp: If a shadow is found, return the shadow.
131 *
132 * Context: Caller must ensure @entry is valid and protect the swap device
133 * with reference count or locks.
134 * The caller also needs to update the corresponding swap_map slots with
135 * SWAP_HAS_CACHE bit to avoid race or conflict.
136 */
137void swap_cache_add_folio(struct folio *folio, swp_entry_t entry, void **shadowp)
138{
139 void *shadow = NULL;
140 unsigned long old_tb, new_tb;
141 struct swap_cluster_info *ci;
142 unsigned int ci_start, ci_off, ci_end;
143 unsigned long nr_pages = folio_nr_pages(folio);
144
145 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
146 VM_WARN_ON_ONCE_FOLIO(folio_test_swapcache(folio), folio);
147 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapbacked(folio), folio);
148
149 new_tb = folio_to_swp_tb(folio);
150 ci_start = swp_cluster_offset(entry);
151 ci_end = ci_start + nr_pages;
152 ci_off = ci_start;
153 ci = swap_cluster_lock(si: __swap_entry_to_info(entry), offset: swp_offset(entry));
154 do {
155 old_tb = __swap_table_xchg(ci, off: ci_off, swp_tb: new_tb);
156 WARN_ON_ONCE(swp_tb_is_folio(old_tb));
157 if (swp_tb_is_shadow(swp_tb: old_tb))
158 shadow = swp_tb_to_shadow(swp_tb: old_tb);
159 } while (++ci_off < ci_end);
160
161 folio_ref_add(folio, nr: nr_pages);
162 folio_set_swapcache(folio);
163 folio->swap = entry;
164 swap_cluster_unlock(ci);
165
166 node_stat_mod_folio(folio, item: NR_FILE_PAGES, nr: nr_pages);
167 lruvec_stat_mod_folio(folio, idx: NR_SWAPCACHE, val: nr_pages);
168
169 if (shadowp)
170 *shadowp = shadow;
171}
172
173/**
174 * __swap_cache_del_folio - Removes a folio from the swap cache.
175 * @ci: The locked swap cluster.
176 * @folio: The folio.
177 * @entry: The first swap entry that the folio corresponds to.
178 * @shadow: shadow value to be filled in the swap cache.
179 *
180 * Removes a folio from the swap cache and fills a shadow in place.
181 * This won't put the folio's refcount. The caller has to do that.
182 *
183 * Context: Caller must ensure the folio is locked and in the swap cache
184 * using the index of @entry, and lock the cluster that holds the entries.
185 */
186void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio,
187 swp_entry_t entry, void *shadow)
188{
189 unsigned long old_tb, new_tb;
190 unsigned int ci_start, ci_off, ci_end;
191 unsigned long nr_pages = folio_nr_pages(folio);
192
193 VM_WARN_ON_ONCE(__swap_entry_to_cluster(entry) != ci);
194 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
195 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio);
196 VM_WARN_ON_ONCE_FOLIO(folio_test_writeback(folio), folio);
197
198 new_tb = shadow_swp_to_tb(shadow);
199 ci_start = swp_cluster_offset(entry);
200 ci_end = ci_start + nr_pages;
201 ci_off = ci_start;
202 do {
203 /* If shadow is NULL, we sets an empty shadow */
204 old_tb = __swap_table_xchg(ci, off: ci_off, swp_tb: new_tb);
205 WARN_ON_ONCE(!swp_tb_is_folio(old_tb) ||
206 swp_tb_to_folio(old_tb) != folio);
207 } while (++ci_off < ci_end);
208
209 folio->swap.val = 0;
210 folio_clear_swapcache(folio);
211 node_stat_mod_folio(folio, item: NR_FILE_PAGES, nr: -nr_pages);
212 lruvec_stat_mod_folio(folio, idx: NR_SWAPCACHE, val: -nr_pages);
213}
214
215/**
216 * swap_cache_del_folio - Removes a folio from the swap cache.
217 * @folio: The folio.
218 *
219 * Same as __swap_cache_del_folio, but handles lock and refcount. The
220 * caller must ensure the folio is either clean or has a swap count
221 * equal to zero, or it may cause data loss.
222 *
223 * Context: Caller must ensure the folio is locked and in the swap cache.
224 */
225void swap_cache_del_folio(struct folio *folio)
226{
227 struct swap_cluster_info *ci;
228 swp_entry_t entry = folio->swap;
229
230 ci = swap_cluster_lock(si: __swap_entry_to_info(entry), offset: swp_offset(entry));
231 __swap_cache_del_folio(ci, folio, entry, NULL);
232 swap_cluster_unlock(ci);
233
234 put_swap_folio(folio, entry);
235 folio_ref_sub(folio, nr: folio_nr_pages(folio));
236}
237
238/**
239 * __swap_cache_replace_folio - Replace a folio in the swap cache.
240 * @ci: The locked swap cluster.
241 * @old: The old folio to be replaced.
242 * @new: The new folio.
243 *
244 * Replace an existing folio in the swap cache with a new folio. The
245 * caller is responsible for setting up the new folio's flag and swap
246 * entries. Replacement will take the new folio's swap entry value as
247 * the starting offset to override all slots covered by the new folio.
248 *
249 * Context: Caller must ensure both folios are locked, and lock the
250 * cluster that holds the old folio to be replaced.
251 */
252void __swap_cache_replace_folio(struct swap_cluster_info *ci,
253 struct folio *old, struct folio *new)
254{
255 swp_entry_t entry = new->swap;
256 unsigned long nr_pages = folio_nr_pages(folio: new);
257 unsigned int ci_off = swp_cluster_offset(entry);
258 unsigned int ci_end = ci_off + nr_pages;
259 unsigned long old_tb, new_tb;
260
261 VM_WARN_ON_ONCE(!folio_test_swapcache(old) || !folio_test_swapcache(new));
262 VM_WARN_ON_ONCE(!folio_test_locked(old) || !folio_test_locked(new));
263 VM_WARN_ON_ONCE(!entry.val);
264
265 /* Swap cache still stores N entries instead of a high-order entry */
266 new_tb = folio_to_swp_tb(folio: new);
267 do {
268 old_tb = __swap_table_xchg(ci, off: ci_off, swp_tb: new_tb);
269 WARN_ON_ONCE(!swp_tb_is_folio(old_tb) || swp_tb_to_folio(old_tb) != old);
270 } while (++ci_off < ci_end);
271
272 /*
273 * If the old folio is partially replaced (e.g., splitting a large
274 * folio, the old folio is shrunk, and new split sub folios replace
275 * the shrunk part), ensure the new folio doesn't overlap it.
276 */
277 if (IS_ENABLED(CONFIG_DEBUG_VM) &&
278 folio_order(folio: old) != folio_order(folio: new)) {
279 ci_off = swp_cluster_offset(entry: old->swap);
280 ci_end = ci_off + folio_nr_pages(folio: old);
281 while (ci_off++ < ci_end)
282 WARN_ON_ONCE(swp_tb_to_folio(__swap_table_get(ci, ci_off)) != old);
283 }
284}
285
286/**
287 * swap_cache_clear_shadow - Clears a set of shadows in the swap cache.
288 * @entry: The starting index entry.
289 * @nr_ents: How many slots need to be cleared.
290 *
291 * Context: Caller must ensure the range is valid, all in one single cluster,
292 * not occupied by any folio, and lock the cluster.
293 */
294void __swap_cache_clear_shadow(swp_entry_t entry, int nr_ents)
295{
296 struct swap_cluster_info *ci = __swap_entry_to_cluster(entry);
297 unsigned int ci_off = swp_cluster_offset(entry), ci_end;
298 unsigned long old;
299
300 ci_end = ci_off + nr_ents;
301 do {
302 old = __swap_table_xchg(ci, off: ci_off, swp_tb: null_to_swp_tb());
303 WARN_ON_ONCE(swp_tb_is_folio(old));
304 } while (++ci_off < ci_end);
305}
306
307/*
308 * If we are the only user, then try to free up the swap cache.
309 *
310 * Its ok to check the swapcache flag without the folio lock
311 * here because we are going to recheck again inside
312 * folio_free_swap() _with_ the lock.
313 * - Marcelo
314 */
315void free_swap_cache(struct folio *folio)
316{
317 if (folio_test_swapcache(folio) && !folio_mapped(folio) &&
318 folio_trylock(folio)) {
319 folio_free_swap(folio);
320 folio_unlock(folio);
321 }
322}
323
324/*
325 * Freeing a folio and also freeing any swap cache associated with
326 * this folio if it is the last user.
327 */
328void free_folio_and_swap_cache(struct folio *folio)
329{
330 free_swap_cache(folio);
331 if (!is_huge_zero_folio(folio))
332 folio_put(folio);
333}
334
335/*
336 * Passed an array of pages, drop them all from swapcache and then release
337 * them. They are removed from the LRU and freed if this is their last use.
338 */
339void free_pages_and_swap_cache(struct encoded_page **pages, int nr)
340{
341 struct folio_batch folios;
342 unsigned int refs[PAGEVEC_SIZE];
343
344 folio_batch_init(fbatch: &folios);
345 for (int i = 0; i < nr; i++) {
346 struct folio *folio = page_folio(encoded_page_ptr(pages[i]));
347
348 free_swap_cache(folio);
349 refs[folios.nr] = 1;
350 if (unlikely(encoded_page_flags(pages[i]) &
351 ENCODED_PAGE_BIT_NR_PAGES_NEXT))
352 refs[folios.nr] = encoded_nr_pages(page: pages[++i]);
353
354 if (folio_batch_add(fbatch: &folios, folio) == 0)
355 folios_put_refs(folios: &folios, refs);
356 }
357 if (folios.nr)
358 folios_put_refs(folios: &folios, refs);
359}
360
361static inline bool swap_use_vma_readahead(void)
362{
363 return READ_ONCE(enable_vma_readahead) && !atomic_read(v: &nr_rotate_swap);
364}
365
366/**
367 * swap_update_readahead - Update the readahead statistics of VMA or globally.
368 * @folio: the swap cache folio that just got hit.
369 * @vma: the VMA that should be updated, could be NULL for global update.
370 * @addr: the addr that triggered the swapin, ignored if @vma is NULL.
371 */
372void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma,
373 unsigned long addr)
374{
375 bool readahead, vma_ra = swap_use_vma_readahead();
376
377 /*
378 * At the moment, we don't support PG_readahead for anon THP
379 * so let's bail out rather than confusing the readahead stat.
380 */
381 if (unlikely(folio_test_large(folio)))
382 return;
383
384 readahead = folio_test_clear_readahead(folio);
385 if (vma && vma_ra) {
386 unsigned long ra_val;
387 int win, hits;
388
389 ra_val = GET_SWAP_RA_VAL(vma);
390 win = SWAP_RA_WIN(ra_val);
391 hits = SWAP_RA_HITS(ra_val);
392 if (readahead)
393 hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX);
394 atomic_long_set(v: &vma->swap_readahead_info,
395 SWAP_RA_VAL(addr, win, hits));
396 }
397
398 if (readahead) {
399 count_vm_event(item: SWAP_RA_HIT);
400 if (!vma || !vma_ra)
401 atomic_inc(v: &swapin_readahead_hits);
402 }
403}
404
405struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
406 struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated,
407 bool skip_if_exists)
408{
409 struct swap_info_struct *si = __swap_entry_to_info(entry);
410 struct folio *folio;
411 struct folio *new_folio = NULL;
412 struct folio *result = NULL;
413 void *shadow = NULL;
414
415 *new_page_allocated = false;
416 for (;;) {
417 int err;
418
419 /*
420 * Check the swap cache first, if a cached folio is found,
421 * return it unlocked. The caller will lock and check it.
422 */
423 folio = swap_cache_get_folio(entry);
424 if (folio)
425 goto got_folio;
426
427 /*
428 * Just skip read ahead for unused swap slot.
429 */
430 if (!swap_entry_swapped(si, entry))
431 goto put_and_return;
432
433 /*
434 * Get a new folio to read into from swap. Allocate it now if
435 * new_folio not exist, before marking swap_map SWAP_HAS_CACHE,
436 * when -EEXIST will cause any racers to loop around until we
437 * add it to cache.
438 */
439 if (!new_folio) {
440 new_folio = folio_alloc_mpol(gfp_mask, 0, mpol, ilx, numa_node_id());
441 if (!new_folio)
442 goto put_and_return;
443 }
444
445 /*
446 * Swap entry may have been freed since our caller observed it.
447 */
448 err = swapcache_prepare(entry, nr: 1);
449 if (!err)
450 break;
451 else if (err != -EEXIST)
452 goto put_and_return;
453
454 /*
455 * Protect against a recursive call to __read_swap_cache_async()
456 * on the same entry waiting forever here because SWAP_HAS_CACHE
457 * is set but the folio is not the swap cache yet. This can
458 * happen today if mem_cgroup_swapin_charge_folio() below
459 * triggers reclaim through zswap, which may call
460 * __read_swap_cache_async() in the writeback path.
461 */
462 if (skip_if_exists)
463 goto put_and_return;
464
465 /*
466 * We might race against __swap_cache_del_folio(), and
467 * stumble across a swap_map entry whose SWAP_HAS_CACHE
468 * has not yet been cleared. Or race against another
469 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE
470 * in swap_map, but not yet added its folio to swap cache.
471 */
472 schedule_timeout_uninterruptible(timeout: 1);
473 }
474
475 /*
476 * The swap entry is ours to swap in. Prepare the new folio.
477 */
478 __folio_set_locked(folio: new_folio);
479 __folio_set_swapbacked(folio: new_folio);
480
481 if (mem_cgroup_swapin_charge_folio(folio: new_folio, NULL, gfp: gfp_mask, entry))
482 goto fail_unlock;
483
484 swap_cache_add_folio(folio: new_folio, entry, shadowp: &shadow);
485 memcg1_swapin(entry, nr_pages: 1);
486
487 if (shadow)
488 workingset_refault(folio: new_folio, shadow);
489
490 /* Caller will initiate read into locked new_folio */
491 folio_add_lru(new_folio);
492 *new_page_allocated = true;
493 folio = new_folio;
494got_folio:
495 result = folio;
496 goto put_and_return;
497
498fail_unlock:
499 put_swap_folio(folio: new_folio, entry);
500 folio_unlock(folio: new_folio);
501put_and_return:
502 if (!(*new_page_allocated) && new_folio)
503 folio_put(folio: new_folio);
504 return result;
505}
506
507/*
508 * Locate a page of swap in physical memory, reserving swap cache space
509 * and reading the disk if it is not already cached.
510 * A failure return means that either the page allocation failed or that
511 * the swap entry is no longer in use.
512 *
513 * get/put_swap_device() aren't needed to call this function, because
514 * __read_swap_cache_async() call them and swap_read_folio() holds the
515 * swap cache folio lock.
516 */
517struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
518 struct vm_area_struct *vma, unsigned long addr,
519 struct swap_iocb **plug)
520{
521 struct swap_info_struct *si;
522 bool page_allocated;
523 struct mempolicy *mpol;
524 pgoff_t ilx;
525 struct folio *folio;
526
527 si = get_swap_device(entry);
528 if (!si)
529 return NULL;
530
531 mpol = get_vma_policy(vma, addr, order: 0, ilx: &ilx);
532 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx,
533 new_page_allocated: &page_allocated, skip_if_exists: false);
534 mpol_cond_put(pol: mpol);
535
536 if (page_allocated)
537 swap_read_folio(folio, plug);
538
539 put_swap_device(si);
540 return folio;
541}
542
543static unsigned int __swapin_nr_pages(unsigned long prev_offset,
544 unsigned long offset,
545 int hits,
546 int max_pages,
547 int prev_win)
548{
549 unsigned int pages, last_ra;
550
551 /*
552 * This heuristic has been found to work well on both sequential and
553 * random loads, swapping to hard disk or to SSD: please don't ask
554 * what the "+ 2" means, it just happens to work well, that's all.
555 */
556 pages = hits + 2;
557 if (pages == 2) {
558 /*
559 * We can have no readahead hits to judge by: but must not get
560 * stuck here forever, so check for an adjacent offset instead
561 * (and don't even bother to check whether swap type is same).
562 */
563 if (offset != prev_offset + 1 && offset != prev_offset - 1)
564 pages = 1;
565 } else {
566 unsigned int roundup = 4;
567 while (roundup < pages)
568 roundup <<= 1;
569 pages = roundup;
570 }
571
572 if (pages > max_pages)
573 pages = max_pages;
574
575 /* Don't shrink readahead too fast */
576 last_ra = prev_win / 2;
577 if (pages < last_ra)
578 pages = last_ra;
579
580 return pages;
581}
582
583static unsigned long swapin_nr_pages(unsigned long offset)
584{
585 static unsigned long prev_offset;
586 unsigned int hits, pages, max_pages;
587 static atomic_t last_readahead_pages;
588
589 max_pages = 1 << READ_ONCE(page_cluster);
590 if (max_pages <= 1)
591 return 1;
592
593 hits = atomic_xchg(v: &swapin_readahead_hits, new: 0);
594 pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits,
595 max_pages,
596 prev_win: atomic_read(v: &last_readahead_pages));
597 if (!hits)
598 WRITE_ONCE(prev_offset, offset);
599 atomic_set(v: &last_readahead_pages, i: pages);
600
601 return pages;
602}
603
604/**
605 * swap_cluster_readahead - swap in pages in hope we need them soon
606 * @entry: swap entry of this memory
607 * @gfp_mask: memory allocation flags
608 * @mpol: NUMA memory allocation policy to be applied
609 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE
610 *
611 * Returns the struct folio for entry and addr, after queueing swapin.
612 *
613 * Primitive swap readahead code. We simply read an aligned block of
614 * (1 << page_cluster) entries in the swap area. This method is chosen
615 * because it doesn't cost us any seek time. We also make sure to queue
616 * the 'original' request together with the readahead ones...
617 *
618 * Note: it is intentional that the same NUMA policy and interleave index
619 * are used for every page of the readahead: neighbouring pages on swap
620 * are fairly likely to have been swapped out from the same node.
621 */
622struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
623 struct mempolicy *mpol, pgoff_t ilx)
624{
625 struct folio *folio;
626 unsigned long entry_offset = swp_offset(entry);
627 unsigned long offset = entry_offset;
628 unsigned long start_offset, end_offset;
629 unsigned long mask;
630 struct swap_info_struct *si = __swap_entry_to_info(entry);
631 struct blk_plug plug;
632 struct swap_iocb *splug = NULL;
633 bool page_allocated;
634
635 mask = swapin_nr_pages(offset) - 1;
636 if (!mask)
637 goto skip;
638
639 /* Read a page_cluster sized and aligned cluster around offset. */
640 start_offset = offset & ~mask;
641 end_offset = offset | mask;
642 if (!start_offset) /* First page is swap header. */
643 start_offset++;
644 if (end_offset >= si->max)
645 end_offset = si->max - 1;
646
647 blk_start_plug(&plug);
648 for (offset = start_offset; offset <= end_offset ; offset++) {
649 /* Ok, do the async read-ahead now */
650 folio = __read_swap_cache_async(
651 entry: swp_entry(type: swp_type(entry), offset),
652 gfp_mask, mpol, ilx, new_page_allocated: &page_allocated, skip_if_exists: false);
653 if (!folio)
654 continue;
655 if (page_allocated) {
656 swap_read_folio(folio, plug: &splug);
657 if (offset != entry_offset) {
658 folio_set_readahead(folio);
659 count_vm_event(item: SWAP_RA);
660 }
661 }
662 folio_put(folio);
663 }
664 blk_finish_plug(&plug);
665 swap_read_unplug(plug: splug);
666 lru_add_drain(); /* Push any new pages onto the LRU now */
667skip:
668 /* The page was likely read above, so no need for plugging here */
669 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx,
670 new_page_allocated: &page_allocated, skip_if_exists: false);
671 if (unlikely(page_allocated))
672 swap_read_folio(folio, NULL);
673 return folio;
674}
675
676static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
677 unsigned long *end)
678{
679 struct vm_area_struct *vma = vmf->vma;
680 unsigned long ra_val;
681 unsigned long faddr, prev_faddr, left, right;
682 unsigned int max_win, hits, prev_win, win;
683
684 max_win = 1 << min(READ_ONCE(page_cluster), SWAP_RA_ORDER_CEILING);
685 if (max_win == 1)
686 return 1;
687
688 faddr = vmf->address;
689 ra_val = GET_SWAP_RA_VAL(vma);
690 prev_faddr = SWAP_RA_ADDR(ra_val);
691 prev_win = SWAP_RA_WIN(ra_val);
692 hits = SWAP_RA_HITS(ra_val);
693 win = __swapin_nr_pages(PFN_DOWN(prev_faddr), PFN_DOWN(faddr), hits,
694 max_pages: max_win, prev_win);
695 atomic_long_set(v: &vma->swap_readahead_info, SWAP_RA_VAL(faddr, win, 0));
696 if (win == 1)
697 return 1;
698
699 if (faddr == prev_faddr + PAGE_SIZE)
700 left = faddr;
701 else if (prev_faddr == faddr + PAGE_SIZE)
702 left = faddr - (win << PAGE_SHIFT) + PAGE_SIZE;
703 else
704 left = faddr - (((win - 1) / 2) << PAGE_SHIFT);
705 right = left + (win << PAGE_SHIFT);
706 if ((long)left < 0)
707 left = 0;
708 *start = max3(left, vma->vm_start, faddr & PMD_MASK);
709 *end = min3(right, vma->vm_end, (faddr & PMD_MASK) + PMD_SIZE);
710
711 return win;
712}
713
714/**
715 * swap_vma_readahead - swap in pages in hope we need them soon
716 * @targ_entry: swap entry of the targeted memory
717 * @gfp_mask: memory allocation flags
718 * @mpol: NUMA memory allocation policy to be applied
719 * @targ_ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE
720 * @vmf: fault information
721 *
722 * Returns the struct folio for entry and addr, after queueing swapin.
723 *
724 * Primitive swap readahead code. We simply read in a few pages whose
725 * virtual addresses are around the fault address in the same vma.
726 *
727 * Caller must hold read mmap_lock if vmf->vma is not NULL.
728 *
729 */
730static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
731 struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf)
732{
733 struct blk_plug plug;
734 struct swap_iocb *splug = NULL;
735 struct folio *folio;
736 pte_t *pte = NULL, pentry;
737 int win;
738 unsigned long start, end, addr;
739 swp_entry_t entry;
740 pgoff_t ilx;
741 bool page_allocated;
742
743 win = swap_vma_ra_win(vmf, start: &start, end: &end);
744 if (win == 1)
745 goto skip;
746
747 ilx = targ_ilx - PFN_DOWN(vmf->address - start);
748
749 blk_start_plug(&plug);
750 for (addr = start; addr < end; ilx++, addr += PAGE_SIZE) {
751 if (!pte++) {
752 pte = pte_offset_map(pmd: vmf->pmd, addr);
753 if (!pte)
754 break;
755 }
756 pentry = ptep_get_lockless(ptep: pte);
757 if (!is_swap_pte(pte: pentry))
758 continue;
759 entry = pte_to_swp_entry(pte: pentry);
760 if (unlikely(non_swap_entry(entry)))
761 continue;
762 pte_unmap(pte);
763 pte = NULL;
764 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx,
765 new_page_allocated: &page_allocated, skip_if_exists: false);
766 if (!folio)
767 continue;
768 if (page_allocated) {
769 swap_read_folio(folio, plug: &splug);
770 if (addr != vmf->address) {
771 folio_set_readahead(folio);
772 count_vm_event(item: SWAP_RA);
773 }
774 }
775 folio_put(folio);
776 }
777 if (pte)
778 pte_unmap(pte);
779 blk_finish_plug(&plug);
780 swap_read_unplug(plug: splug);
781 lru_add_drain();
782skip:
783 /* The folio was likely read above, so no need for plugging here */
784 folio = __read_swap_cache_async(entry: targ_entry, gfp_mask, mpol, ilx: targ_ilx,
785 new_page_allocated: &page_allocated, skip_if_exists: false);
786 if (unlikely(page_allocated))
787 swap_read_folio(folio, NULL);
788 return folio;
789}
790
791/**
792 * swapin_readahead - swap in pages in hope we need them soon
793 * @entry: swap entry of this memory
794 * @gfp_mask: memory allocation flags
795 * @vmf: fault information
796 *
797 * Returns the struct folio for entry and addr, after queueing swapin.
798 *
799 * It's a main entry function for swap readahead. By the configuration,
800 * it will read ahead blocks by cluster-based(ie, physical disk based)
801 * or vma-based(ie, virtual address based on faulty address) readahead.
802 */
803struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
804 struct vm_fault *vmf)
805{
806 struct mempolicy *mpol;
807 pgoff_t ilx;
808 struct folio *folio;
809
810 mpol = get_vma_policy(vma: vmf->vma, addr: vmf->address, order: 0, ilx: &ilx);
811 folio = swap_use_vma_readahead() ?
812 swap_vma_readahead(targ_entry: entry, gfp_mask, mpol, targ_ilx: ilx, vmf) :
813 swap_cluster_readahead(entry, gfp_mask, mpol, ilx);
814 mpol_cond_put(pol: mpol);
815
816 return folio;
817}
818
819#ifdef CONFIG_SYSFS
820static ssize_t vma_ra_enabled_show(struct kobject *kobj,
821 struct kobj_attribute *attr, char *buf)
822{
823 return sysfs_emit(buf, fmt: "%s\n", str_true_false(v: enable_vma_readahead));
824}
825static ssize_t vma_ra_enabled_store(struct kobject *kobj,
826 struct kobj_attribute *attr,
827 const char *buf, size_t count)
828{
829 ssize_t ret;
830
831 ret = kstrtobool(s: buf, res: &enable_vma_readahead);
832 if (ret)
833 return ret;
834
835 return count;
836}
837static struct kobj_attribute vma_ra_enabled_attr = __ATTR_RW(vma_ra_enabled);
838
839static struct attribute *swap_attrs[] = {
840 &vma_ra_enabled_attr.attr,
841 NULL,
842};
843
844static const struct attribute_group swap_attr_group = {
845 .attrs = swap_attrs,
846};
847
848static int __init swap_init(void)
849{
850 int err;
851 struct kobject *swap_kobj;
852
853 swap_kobj = kobject_create_and_add(name: "swap", parent: mm_kobj);
854 if (!swap_kobj) {
855 pr_err("failed to create swap kobject\n");
856 return -ENOMEM;
857 }
858 err = sysfs_create_group(kobj: swap_kobj, grp: &swap_attr_group);
859 if (err) {
860 pr_err("failed to register swap group\n");
861 goto delete_obj;
862 }
863 /* Swap cache writeback is LRU based, no tags for it */
864 mapping_set_no_writeback_tags(mapping: &swap_space);
865 return 0;
866
867delete_obj:
868 kobject_put(kobj: swap_kobj);
869 return err;
870}
871subsys_initcall(swap_init);
872#endif
873