1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/swapfile.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 * Swap reorganised 29.12.95, Stephen Tweedie
7 */
8
9#include <linux/blkdev.h>
10#include <linux/mm.h>
11#include <linux/sched/mm.h>
12#include <linux/sched/task.h>
13#include <linux/hugetlb.h>
14#include <linux/mman.h>
15#include <linux/slab.h>
16#include <linux/kernel_stat.h>
17#include <linux/swap.h>
18#include <linux/vmalloc.h>
19#include <linux/pagemap.h>
20#include <linux/namei.h>
21#include <linux/shmem_fs.h>
22#include <linux/blk-cgroup.h>
23#include <linux/random.h>
24#include <linux/writeback.h>
25#include <linux/proc_fs.h>
26#include <linux/seq_file.h>
27#include <linux/init.h>
28#include <linux/ksm.h>
29#include <linux/rmap.h>
30#include <linux/security.h>
31#include <linux/backing-dev.h>
32#include <linux/mutex.h>
33#include <linux/capability.h>
34#include <linux/syscalls.h>
35#include <linux/memcontrol.h>
36#include <linux/poll.h>
37#include <linux/oom.h>
38#include <linux/swapfile.h>
39#include <linux/export.h>
40#include <linux/sort.h>
41#include <linux/completion.h>
42#include <linux/suspend.h>
43#include <linux/zswap.h>
44#include <linux/plist.h>
45
46#include <asm/tlbflush.h>
47#include <linux/swapops.h>
48#include <linux/swap_cgroup.h>
49#include "swap_table.h"
50#include "internal.h"
51#include "swap.h"
52
53static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
54 unsigned char);
55static void free_swap_count_continuations(struct swap_info_struct *);
56static void swap_entries_free(struct swap_info_struct *si,
57 struct swap_cluster_info *ci,
58 swp_entry_t entry, unsigned int nr_pages);
59static void swap_range_alloc(struct swap_info_struct *si,
60 unsigned int nr_entries);
61static bool folio_swapcache_freeable(struct folio *folio);
62static void move_cluster(struct swap_info_struct *si,
63 struct swap_cluster_info *ci, struct list_head *list,
64 enum swap_cluster_flags new_flags);
65
66static DEFINE_SPINLOCK(swap_lock);
67static unsigned int nr_swapfiles;
68atomic_long_t nr_swap_pages;
69/*
70 * Some modules use swappable objects and may try to swap them out under
71 * memory pressure (via the shrinker). Before doing so, they may wish to
72 * check to see if any swap space is available.
73 */
74EXPORT_SYMBOL_GPL(nr_swap_pages);
75/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
76long total_swap_pages;
77static int least_priority = -1;
78unsigned long swapfile_maximum_size;
79#ifdef CONFIG_MIGRATION
80bool swap_migration_ad_supported;
81#endif /* CONFIG_MIGRATION */
82
83static const char Bad_file[] = "Bad swap file entry ";
84static const char Unused_file[] = "Unused swap file entry ";
85static const char Bad_offset[] = "Bad swap offset entry ";
86static const char Unused_offset[] = "Unused swap offset entry ";
87
88/*
89 * all active swap_info_structs
90 * protected with swap_lock, and ordered by priority.
91 */
92static PLIST_HEAD(swap_active_head);
93
94/*
95 * all available (active, not full) swap_info_structs
96 * protected with swap_avail_lock, ordered by priority.
97 * This is used by folio_alloc_swap() instead of swap_active_head
98 * because swap_active_head includes all swap_info_structs,
99 * but folio_alloc_swap() doesn't need to look at full ones.
100 * This uses its own lock instead of swap_lock because when a
101 * swap_info_struct changes between not-full/full, it needs to
102 * add/remove itself to/from this list, but the swap_info_struct->lock
103 * is held and the locking order requires swap_lock to be taken
104 * before any swap_info_struct->lock.
105 */
106static struct plist_head *swap_avail_heads;
107static DEFINE_SPINLOCK(swap_avail_lock);
108
109struct swap_info_struct *swap_info[MAX_SWAPFILES];
110
111static struct kmem_cache *swap_table_cachep;
112
113static DEFINE_MUTEX(swapon_mutex);
114
115static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
116/* Activity counter to indicate that a swapon or swapoff has occurred */
117static atomic_t proc_poll_event = ATOMIC_INIT(0);
118
119atomic_t nr_rotate_swap = ATOMIC_INIT(0);
120
121struct percpu_swap_cluster {
122 struct swap_info_struct *si[SWAP_NR_ORDERS];
123 unsigned long offset[SWAP_NR_ORDERS];
124 local_lock_t lock;
125};
126
127static DEFINE_PER_CPU(struct percpu_swap_cluster, percpu_swap_cluster) = {
128 .si = { NULL },
129 .offset = { SWAP_ENTRY_INVALID },
130 .lock = INIT_LOCAL_LOCK(),
131};
132
133/* May return NULL on invalid type, caller must check for NULL return */
134static struct swap_info_struct *swap_type_to_info(int type)
135{
136 if (type >= MAX_SWAPFILES)
137 return NULL;
138 return READ_ONCE(swap_info[type]); /* rcu_dereference() */
139}
140
141/* May return NULL on invalid entry, caller must check for NULL return */
142static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry)
143{
144 return swap_type_to_info(type: swp_type(entry));
145}
146
147static inline unsigned char swap_count(unsigned char ent)
148{
149 return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */
150}
151
152/*
153 * Use the second highest bit of inuse_pages counter as the indicator
154 * if one swap device is on the available plist, so the atomic can
155 * still be updated arithmetically while having special data embedded.
156 *
157 * inuse_pages counter is the only thing indicating if a device should
158 * be on avail_lists or not (except swapon / swapoff). By embedding the
159 * off-list bit in the atomic counter, updates no longer need any lock
160 * to check the list status.
161 *
162 * This bit will be set if the device is not on the plist and not
163 * usable, will be cleared if the device is on the plist.
164 */
165#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2))
166#define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT)
167static long swap_usage_in_pages(struct swap_info_struct *si)
168{
169 return atomic_long_read(v: &si->inuse_pages) & SWAP_USAGE_COUNTER_MASK;
170}
171
172/* Reclaim the swap entry anyway if possible */
173#define TTRS_ANYWAY 0x1
174/*
175 * Reclaim the swap entry if there are no more mappings of the
176 * corresponding page
177 */
178#define TTRS_UNMAPPED 0x2
179/* Reclaim the swap entry if swap is getting full */
180#define TTRS_FULL 0x4
181
182static bool swap_only_has_cache(struct swap_info_struct *si,
183 unsigned long offset, int nr_pages)
184{
185 unsigned char *map = si->swap_map + offset;
186 unsigned char *map_end = map + nr_pages;
187
188 do {
189 VM_BUG_ON(!(*map & SWAP_HAS_CACHE));
190 if (*map != SWAP_HAS_CACHE)
191 return false;
192 } while (++map < map_end);
193
194 return true;
195}
196
197static bool swap_is_last_map(struct swap_info_struct *si,
198 unsigned long offset, int nr_pages, bool *has_cache)
199{
200 unsigned char *map = si->swap_map + offset;
201 unsigned char *map_end = map + nr_pages;
202 unsigned char count = *map;
203
204 if (swap_count(ent: count) != 1 && swap_count(ent: count) != SWAP_MAP_SHMEM)
205 return false;
206
207 while (++map < map_end) {
208 if (*map != count)
209 return false;
210 }
211
212 *has_cache = !!(count & SWAP_HAS_CACHE);
213 return true;
214}
215
216/*
217 * returns number of pages in the folio that backs the swap entry. If positive,
218 * the folio was reclaimed. If negative, the folio was not reclaimed. If 0, no
219 * folio was associated with the swap entry.
220 */
221static int __try_to_reclaim_swap(struct swap_info_struct *si,
222 unsigned long offset, unsigned long flags)
223{
224 const swp_entry_t entry = swp_entry(type: si->type, offset);
225 struct swap_cluster_info *ci;
226 struct folio *folio;
227 int ret, nr_pages;
228 bool need_reclaim;
229
230again:
231 folio = swap_cache_get_folio(entry);
232 if (!folio)
233 return 0;
234
235 nr_pages = folio_nr_pages(folio);
236 ret = -nr_pages;
237
238 /*
239 * When this function is called from scan_swap_map_slots() and it's
240 * called by vmscan.c at reclaiming folios. So we hold a folio lock
241 * here. We have to use trylock for avoiding deadlock. This is a special
242 * case and you should use folio_free_swap() with explicit folio_lock()
243 * in usual operations.
244 */
245 if (!folio_trylock(folio))
246 goto out;
247
248 /*
249 * Offset could point to the middle of a large folio, or folio
250 * may no longer point to the expected offset before it's locked.
251 */
252 if (!folio_matches_swap_entry(folio, entry)) {
253 folio_unlock(folio);
254 folio_put(folio);
255 goto again;
256 }
257 offset = swp_offset(entry: folio->swap);
258
259 need_reclaim = ((flags & TTRS_ANYWAY) ||
260 ((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) ||
261 ((flags & TTRS_FULL) && mem_cgroup_swap_full(folio)));
262 if (!need_reclaim || !folio_swapcache_freeable(folio))
263 goto out_unlock;
264
265 /*
266 * It's safe to delete the folio from swap cache only if the folio's
267 * swap_map is HAS_CACHE only, which means the slots have no page table
268 * reference or pending writeback, and can't be allocated to others.
269 */
270 ci = swap_cluster_lock(si, offset);
271 need_reclaim = swap_only_has_cache(si, offset, nr_pages);
272 swap_cluster_unlock(ci);
273 if (!need_reclaim)
274 goto out_unlock;
275
276 swap_cache_del_folio(folio);
277 folio_set_dirty(folio);
278 ret = nr_pages;
279out_unlock:
280 folio_unlock(folio);
281out:
282 folio_put(folio);
283 return ret;
284}
285
286static inline struct swap_extent *first_se(struct swap_info_struct *sis)
287{
288 struct rb_node *rb = rb_first(&sis->swap_extent_root);
289 return rb_entry(rb, struct swap_extent, rb_node);
290}
291
292static inline struct swap_extent *next_se(struct swap_extent *se)
293{
294 struct rb_node *rb = rb_next(&se->rb_node);
295 return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL;
296}
297
298/*
299 * swapon tell device that all the old swap contents can be discarded,
300 * to allow the swap device to optimize its wear-levelling.
301 */
302static int discard_swap(struct swap_info_struct *si)
303{
304 struct swap_extent *se;
305 sector_t start_block;
306 sector_t nr_blocks;
307 int err = 0;
308
309 /* Do not discard the swap header page! */
310 se = first_se(sis: si);
311 start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
312 nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
313 if (nr_blocks) {
314 err = blkdev_issue_discard(bdev: si->bdev, sector: start_block,
315 nr_sects: nr_blocks, GFP_KERNEL);
316 if (err)
317 return err;
318 cond_resched();
319 }
320
321 for (se = next_se(se); se; se = next_se(se)) {
322 start_block = se->start_block << (PAGE_SHIFT - 9);
323 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
324
325 err = blkdev_issue_discard(bdev: si->bdev, sector: start_block,
326 nr_sects: nr_blocks, GFP_KERNEL);
327 if (err)
328 break;
329
330 cond_resched();
331 }
332 return err; /* That will often be -EOPNOTSUPP */
333}
334
335static struct swap_extent *
336offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)
337{
338 struct swap_extent *se;
339 struct rb_node *rb;
340
341 rb = sis->swap_extent_root.rb_node;
342 while (rb) {
343 se = rb_entry(rb, struct swap_extent, rb_node);
344 if (offset < se->start_page)
345 rb = rb->rb_left;
346 else if (offset >= se->start_page + se->nr_pages)
347 rb = rb->rb_right;
348 else
349 return se;
350 }
351 /* It *must* be present */
352 BUG();
353}
354
355sector_t swap_folio_sector(struct folio *folio)
356{
357 struct swap_info_struct *sis = __swap_entry_to_info(entry: folio->swap);
358 struct swap_extent *se;
359 sector_t sector;
360 pgoff_t offset;
361
362 offset = swp_offset(entry: folio->swap);
363 se = offset_to_swap_extent(sis, offset);
364 sector = se->start_block + (offset - se->start_page);
365 return sector << (PAGE_SHIFT - 9);
366}
367
368/*
369 * swap allocation tell device that a cluster of swap can now be discarded,
370 * to allow the swap device to optimize its wear-levelling.
371 */
372static void discard_swap_cluster(struct swap_info_struct *si,
373 pgoff_t start_page, pgoff_t nr_pages)
374{
375 struct swap_extent *se = offset_to_swap_extent(sis: si, offset: start_page);
376
377 while (nr_pages) {
378 pgoff_t offset = start_page - se->start_page;
379 sector_t start_block = se->start_block + offset;
380 sector_t nr_blocks = se->nr_pages - offset;
381
382 if (nr_blocks > nr_pages)
383 nr_blocks = nr_pages;
384 start_page += nr_blocks;
385 nr_pages -= nr_blocks;
386
387 start_block <<= PAGE_SHIFT - 9;
388 nr_blocks <<= PAGE_SHIFT - 9;
389 if (blkdev_issue_discard(bdev: si->bdev, sector: start_block,
390 nr_sects: nr_blocks, GFP_NOIO))
391 break;
392
393 se = next_se(se);
394 }
395}
396
397#define LATENCY_LIMIT 256
398
399static inline bool cluster_is_empty(struct swap_cluster_info *info)
400{
401 return info->count == 0;
402}
403
404static inline bool cluster_is_discard(struct swap_cluster_info *info)
405{
406 return info->flags == CLUSTER_FLAG_DISCARD;
407}
408
409static inline bool cluster_table_is_alloced(struct swap_cluster_info *ci)
410{
411 return rcu_dereference_protected(ci->table, lockdep_is_held(&ci->lock));
412}
413
414static inline bool cluster_is_usable(struct swap_cluster_info *ci, int order)
415{
416 if (unlikely(ci->flags > CLUSTER_FLAG_USABLE))
417 return false;
418 if (!cluster_table_is_alloced(ci))
419 return false;
420 if (!order)
421 return true;
422 return cluster_is_empty(info: ci) || order == ci->order;
423}
424
425static inline unsigned int cluster_index(struct swap_info_struct *si,
426 struct swap_cluster_info *ci)
427{
428 return ci - si->cluster_info;
429}
430
431static inline unsigned int cluster_offset(struct swap_info_struct *si,
432 struct swap_cluster_info *ci)
433{
434 return cluster_index(si, ci) * SWAPFILE_CLUSTER;
435}
436
437static struct swap_table *swap_table_alloc(gfp_t gfp)
438{
439 struct folio *folio;
440
441 if (!SWP_TABLE_USE_PAGE)
442 return kmem_cache_zalloc(swap_table_cachep, gfp);
443
444 folio = folio_alloc(gfp | __GFP_ZERO, 0);
445 if (folio)
446 return folio_address(folio);
447 return NULL;
448}
449
450static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
451{
452 struct folio *folio;
453
454 folio = page_folio(container_of(head, struct page, rcu_head));
455 folio_put(folio);
456}
457
458static void swap_table_free(struct swap_table *table)
459{
460 if (!SWP_TABLE_USE_PAGE) {
461 kmem_cache_free(s: swap_table_cachep, objp: table);
462 return;
463 }
464
465 call_rcu(head: &(folio_page(virt_to_folio(table), 0)->rcu_head),
466 func: swap_table_free_folio_rcu_cb);
467}
468
469static void swap_cluster_free_table(struct swap_cluster_info *ci)
470{
471 unsigned int ci_off;
472 struct swap_table *table;
473
474 /* Only empty cluster's table is allow to be freed */
475 lockdep_assert_held(&ci->lock);
476 VM_WARN_ON_ONCE(!cluster_is_empty(ci));
477 for (ci_off = 0; ci_off < SWAPFILE_CLUSTER; ci_off++)
478 VM_WARN_ON_ONCE(!swp_tb_is_null(__swap_table_get(ci, ci_off)));
479 table = (void *)rcu_dereference_protected(ci->table, true);
480 rcu_assign_pointer(ci->table, NULL);
481
482 swap_table_free(table);
483}
484
485/*
486 * Allocate swap table for one cluster. Attempt an atomic allocation first,
487 * then fallback to sleeping allocation.
488 */
489static struct swap_cluster_info *
490swap_cluster_alloc_table(struct swap_info_struct *si,
491 struct swap_cluster_info *ci)
492{
493 struct swap_table *table;
494
495 /*
496 * Only cluster isolation from the allocator does table allocation.
497 * Swap allocator uses percpu clusters and holds the local lock.
498 */
499 lockdep_assert_held(&ci->lock);
500 lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock);
501
502 /* The cluster must be free and was just isolated from the free list. */
503 VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
504
505 table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
506 if (table) {
507 rcu_assign_pointer(ci->table, table);
508 return ci;
509 }
510
511 /*
512 * Try a sleep allocation. Each isolated free cluster may cause
513 * a sleep allocation, but there is a limited number of them, so
514 * the potential recursive allocation is limited.
515 */
516 spin_unlock(lock: &ci->lock);
517 if (!(si->flags & SWP_SOLIDSTATE))
518 spin_unlock(lock: &si->global_cluster_lock);
519 local_unlock(&percpu_swap_cluster.lock);
520
521 table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | GFP_KERNEL);
522
523 /*
524 * Back to atomic context. We might have migrated to a new CPU with a
525 * usable percpu cluster. But just keep using the isolated cluster to
526 * make things easier. Migration indicates a slight change of workload
527 * so using a new free cluster might not be a bad idea, and the worst
528 * could happen with ignoring the percpu cluster is fragmentation,
529 * which is acceptable since this fallback and race is rare.
530 */
531 local_lock(&percpu_swap_cluster.lock);
532 if (!(si->flags & SWP_SOLIDSTATE))
533 spin_lock(lock: &si->global_cluster_lock);
534 spin_lock(lock: &ci->lock);
535
536 /* Nothing except this helper should touch a dangling empty cluster. */
537 if (WARN_ON_ONCE(cluster_table_is_alloced(ci))) {
538 if (table)
539 swap_table_free(table);
540 return ci;
541 }
542
543 if (!table) {
544 move_cluster(si, ci, list: &si->free_clusters, new_flags: CLUSTER_FLAG_FREE);
545 spin_unlock(lock: &ci->lock);
546 return NULL;
547 }
548
549 rcu_assign_pointer(ci->table, table);
550 return ci;
551}
552
553static void move_cluster(struct swap_info_struct *si,
554 struct swap_cluster_info *ci, struct list_head *list,
555 enum swap_cluster_flags new_flags)
556{
557 VM_WARN_ON(ci->flags == new_flags);
558
559 BUILD_BUG_ON(1 << sizeof(ci->flags) * BITS_PER_BYTE < CLUSTER_FLAG_MAX);
560 lockdep_assert_held(&ci->lock);
561
562 spin_lock(lock: &si->lock);
563 if (ci->flags == CLUSTER_FLAG_NONE)
564 list_add_tail(new: &ci->list, head: list);
565 else
566 list_move_tail(list: &ci->list, head: list);
567 spin_unlock(lock: &si->lock);
568 ci->flags = new_flags;
569}
570
571/* Add a cluster to discard list and schedule it to do discard */
572static void swap_cluster_schedule_discard(struct swap_info_struct *si,
573 struct swap_cluster_info *ci)
574{
575 VM_BUG_ON(ci->flags == CLUSTER_FLAG_FREE);
576 move_cluster(si, ci, list: &si->discard_clusters, new_flags: CLUSTER_FLAG_DISCARD);
577 schedule_work(work: &si->discard_work);
578}
579
580static void __free_cluster(struct swap_info_struct *si, struct swap_cluster_info *ci)
581{
582 swap_cluster_free_table(ci);
583 move_cluster(si, ci, list: &si->free_clusters, new_flags: CLUSTER_FLAG_FREE);
584 ci->order = 0;
585}
586
587/*
588 * Isolate and lock the first cluster that is not contented on a list,
589 * clean its flag before taken off-list. Cluster flag must be in sync
590 * with list status, so cluster updaters can always know the cluster
591 * list status without touching si lock.
592 *
593 * Note it's possible that all clusters on a list are contented so
594 * this returns NULL for an non-empty list.
595 */
596static struct swap_cluster_info *isolate_lock_cluster(
597 struct swap_info_struct *si, struct list_head *list, int order)
598{
599 struct swap_cluster_info *ci, *found = NULL;
600
601 spin_lock(lock: &si->lock);
602 list_for_each_entry(ci, list, list) {
603 if (!spin_trylock(lock: &ci->lock))
604 continue;
605
606 /* We may only isolate and clear flags of following lists */
607 VM_BUG_ON(!ci->flags);
608 VM_BUG_ON(ci->flags > CLUSTER_FLAG_USABLE &&
609 ci->flags != CLUSTER_FLAG_FULL);
610
611 list_del(entry: &ci->list);
612 ci->flags = CLUSTER_FLAG_NONE;
613 found = ci;
614 break;
615 }
616 spin_unlock(lock: &si->lock);
617
618 if (found && !cluster_table_is_alloced(ci: found)) {
619 /* Only an empty free cluster's swap table can be freed. */
620 VM_WARN_ON_ONCE(list != &si->free_clusters);
621 VM_WARN_ON_ONCE(!cluster_is_empty(found));
622 return swap_cluster_alloc_table(si, ci: found);
623 }
624
625 return found;
626}
627
628/*
629 * Doing discard actually. After a cluster discard is finished, the cluster
630 * will be added to free cluster list. Discard cluster is a bit special as
631 * they don't participate in allocation or reclaim, so clusters marked as
632 * CLUSTER_FLAG_DISCARD must remain off-list or on discard list.
633 */
634static bool swap_do_scheduled_discard(struct swap_info_struct *si)
635{
636 struct swap_cluster_info *ci;
637 bool ret = false;
638 unsigned int idx;
639
640 spin_lock(lock: &si->lock);
641 while (!list_empty(head: &si->discard_clusters)) {
642 ci = list_first_entry(&si->discard_clusters, struct swap_cluster_info, list);
643 /*
644 * Delete the cluster from list to prepare for discard, but keep
645 * the CLUSTER_FLAG_DISCARD flag, percpu_swap_cluster could be
646 * pointing to it, or ran into by relocate_cluster.
647 */
648 list_del(entry: &ci->list);
649 idx = cluster_index(si, ci);
650 spin_unlock(lock: &si->lock);
651 discard_swap_cluster(si, start_page: idx * SWAPFILE_CLUSTER,
652 SWAPFILE_CLUSTER);
653
654 spin_lock(lock: &ci->lock);
655 /*
656 * Discard is done, clear its flags as it's off-list, then
657 * return the cluster to allocation list.
658 */
659 ci->flags = CLUSTER_FLAG_NONE;
660 __free_cluster(si, ci);
661 spin_unlock(lock: &ci->lock);
662 ret = true;
663 spin_lock(lock: &si->lock);
664 }
665 spin_unlock(lock: &si->lock);
666 return ret;
667}
668
669static void swap_discard_work(struct work_struct *work)
670{
671 struct swap_info_struct *si;
672
673 si = container_of(work, struct swap_info_struct, discard_work);
674
675 swap_do_scheduled_discard(si);
676}
677
678static void swap_users_ref_free(struct percpu_ref *ref)
679{
680 struct swap_info_struct *si;
681
682 si = container_of(ref, struct swap_info_struct, users);
683 complete(&si->comp);
684}
685
686/*
687 * Must be called after freeing if ci->count == 0, moves the cluster to free
688 * or discard list.
689 */
690static void free_cluster(struct swap_info_struct *si, struct swap_cluster_info *ci)
691{
692 VM_BUG_ON(ci->count != 0);
693 VM_BUG_ON(ci->flags == CLUSTER_FLAG_FREE);
694 lockdep_assert_held(&ci->lock);
695
696 /*
697 * If the swap is discardable, prepare discard the cluster
698 * instead of free it immediately. The cluster will be freed
699 * after discard.
700 */
701 if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
702 (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
703 swap_cluster_schedule_discard(si, ci);
704 return;
705 }
706
707 __free_cluster(si, ci);
708}
709
710/*
711 * Must be called after freeing if ci->count != 0, moves the cluster to
712 * nonfull list.
713 */
714static void partial_free_cluster(struct swap_info_struct *si,
715 struct swap_cluster_info *ci)
716{
717 VM_BUG_ON(!ci->count || ci->count == SWAPFILE_CLUSTER);
718 lockdep_assert_held(&ci->lock);
719
720 if (ci->flags != CLUSTER_FLAG_NONFULL)
721 move_cluster(si, ci, list: &si->nonfull_clusters[ci->order],
722 new_flags: CLUSTER_FLAG_NONFULL);
723}
724
725/*
726 * Must be called after allocation, moves the cluster to full or frag list.
727 * Note: allocation doesn't acquire si lock, and may drop the ci lock for
728 * reclaim, so the cluster could be any where when called.
729 */
730static void relocate_cluster(struct swap_info_struct *si,
731 struct swap_cluster_info *ci)
732{
733 lockdep_assert_held(&ci->lock);
734
735 /* Discard cluster must remain off-list or on discard list */
736 if (cluster_is_discard(info: ci))
737 return;
738
739 if (!ci->count) {
740 if (ci->flags != CLUSTER_FLAG_FREE)
741 free_cluster(si, ci);
742 } else if (ci->count != SWAPFILE_CLUSTER) {
743 if (ci->flags != CLUSTER_FLAG_FRAG)
744 move_cluster(si, ci, list: &si->frag_clusters[ci->order],
745 new_flags: CLUSTER_FLAG_FRAG);
746 } else {
747 if (ci->flags != CLUSTER_FLAG_FULL)
748 move_cluster(si, ci, list: &si->full_clusters,
749 new_flags: CLUSTER_FLAG_FULL);
750 }
751}
752
753/*
754 * The cluster corresponding to page_nr will be used. The cluster will not be
755 * added to free cluster list and its usage counter will be increased by 1.
756 * Only used for initialization.
757 */
758static int inc_cluster_info_page(struct swap_info_struct *si,
759 struct swap_cluster_info *cluster_info, unsigned long page_nr)
760{
761 unsigned long idx = page_nr / SWAPFILE_CLUSTER;
762 struct swap_table *table;
763 struct swap_cluster_info *ci;
764
765 ci = cluster_info + idx;
766 if (!ci->table) {
767 table = swap_table_alloc(GFP_KERNEL);
768 if (!table)
769 return -ENOMEM;
770 rcu_assign_pointer(ci->table, table);
771 }
772
773 ci->count++;
774
775 VM_BUG_ON(ci->count > SWAPFILE_CLUSTER);
776 VM_BUG_ON(ci->flags);
777
778 return 0;
779}
780
781static bool cluster_reclaim_range(struct swap_info_struct *si,
782 struct swap_cluster_info *ci,
783 unsigned long start, unsigned long end)
784{
785 unsigned char *map = si->swap_map;
786 unsigned long offset = start;
787 int nr_reclaim;
788
789 spin_unlock(lock: &ci->lock);
790 do {
791 switch (READ_ONCE(map[offset])) {
792 case 0:
793 offset++;
794 break;
795 case SWAP_HAS_CACHE:
796 nr_reclaim = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
797 if (nr_reclaim > 0)
798 offset += nr_reclaim;
799 else
800 goto out;
801 break;
802 default:
803 goto out;
804 }
805 } while (offset < end);
806out:
807 spin_lock(lock: &ci->lock);
808 /*
809 * Recheck the range no matter reclaim succeeded or not, the slot
810 * could have been be freed while we are not holding the lock.
811 */
812 for (offset = start; offset < end; offset++)
813 if (READ_ONCE(map[offset]))
814 return false;
815
816 return true;
817}
818
819static bool cluster_scan_range(struct swap_info_struct *si,
820 struct swap_cluster_info *ci,
821 unsigned long start, unsigned int nr_pages,
822 bool *need_reclaim)
823{
824 unsigned long offset, end = start + nr_pages;
825 unsigned char *map = si->swap_map;
826
827 if (cluster_is_empty(info: ci))
828 return true;
829
830 for (offset = start; offset < end; offset++) {
831 switch (READ_ONCE(map[offset])) {
832 case 0:
833 continue;
834 case SWAP_HAS_CACHE:
835 if (!vm_swap_full())
836 return false;
837 *need_reclaim = true;
838 continue;
839 default:
840 return false;
841 }
842 }
843
844 return true;
845}
846
847/*
848 * Currently, the swap table is not used for count tracking, just
849 * do a sanity check here to ensure nothing leaked, so the swap
850 * table should be empty upon freeing.
851 */
852static void swap_cluster_assert_table_empty(struct swap_cluster_info *ci,
853 unsigned int start, unsigned int nr)
854{
855 unsigned int ci_off = start % SWAPFILE_CLUSTER;
856 unsigned int ci_end = ci_off + nr;
857 unsigned long swp_tb;
858
859 if (IS_ENABLED(CONFIG_DEBUG_VM)) {
860 do {
861 swp_tb = __swap_table_get(ci, off: ci_off);
862 VM_WARN_ON_ONCE(!swp_tb_is_null(swp_tb));
863 } while (++ci_off < ci_end);
864 }
865}
866
867static bool cluster_alloc_range(struct swap_info_struct *si, struct swap_cluster_info *ci,
868 unsigned int start, unsigned char usage,
869 unsigned int order)
870{
871 unsigned int nr_pages = 1 << order;
872
873 lockdep_assert_held(&ci->lock);
874
875 if (!(si->flags & SWP_WRITEOK))
876 return false;
877
878 /*
879 * The first allocation in a cluster makes the
880 * cluster exclusive to this order
881 */
882 if (cluster_is_empty(info: ci))
883 ci->order = order;
884
885 memset(s: si->swap_map + start, c: usage, n: nr_pages);
886 swap_cluster_assert_table_empty(ci, start, nr: nr_pages);
887 swap_range_alloc(si, nr_entries: nr_pages);
888 ci->count += nr_pages;
889
890 return true;
891}
892
893/* Try use a new cluster for current CPU and allocate from it. */
894static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
895 struct swap_cluster_info *ci,
896 unsigned long offset,
897 unsigned int order,
898 unsigned char usage)
899{
900 unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
901 unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER);
902 unsigned long end = min(start + SWAPFILE_CLUSTER, si->max);
903 unsigned int nr_pages = 1 << order;
904 bool need_reclaim, ret;
905
906 lockdep_assert_held(&ci->lock);
907
908 if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER)
909 goto out;
910
911 for (end -= nr_pages; offset <= end; offset += nr_pages) {
912 need_reclaim = false;
913 if (!cluster_scan_range(si, ci, start: offset, nr_pages, need_reclaim: &need_reclaim))
914 continue;
915 if (need_reclaim) {
916 ret = cluster_reclaim_range(si, ci, start: offset, end: offset + nr_pages);
917 /*
918 * Reclaim drops ci->lock and cluster could be used
919 * by another order. Not checking flag as off-list
920 * cluster has no flag set, and change of list
921 * won't cause fragmentation.
922 */
923 if (!cluster_is_usable(ci, order))
924 goto out;
925 if (cluster_is_empty(info: ci))
926 offset = start;
927 /* Reclaim failed but cluster is usable, try next */
928 if (!ret)
929 continue;
930 }
931 if (!cluster_alloc_range(si, ci, start: offset, usage, order))
932 break;
933 found = offset;
934 offset += nr_pages;
935 if (ci->count < SWAPFILE_CLUSTER && offset <= end)
936 next = offset;
937 break;
938 }
939out:
940 relocate_cluster(si, ci);
941 swap_cluster_unlock(ci);
942 if (si->flags & SWP_SOLIDSTATE) {
943 this_cpu_write(percpu_swap_cluster.offset[order], next);
944 this_cpu_write(percpu_swap_cluster.si[order], si);
945 } else {
946 si->global_cluster->next[order] = next;
947 }
948 return found;
949}
950
951static unsigned int alloc_swap_scan_list(struct swap_info_struct *si,
952 struct list_head *list,
953 unsigned int order,
954 unsigned char usage,
955 bool scan_all)
956{
957 unsigned int found = SWAP_ENTRY_INVALID;
958
959 do {
960 struct swap_cluster_info *ci = isolate_lock_cluster(si, list, order);
961 unsigned long offset;
962
963 if (!ci)
964 break;
965 offset = cluster_offset(si, ci);
966 found = alloc_swap_scan_cluster(si, ci, offset, order, usage);
967 if (found)
968 break;
969 } while (scan_all);
970
971 return found;
972}
973
974static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force)
975{
976 long to_scan = 1;
977 unsigned long offset, end;
978 struct swap_cluster_info *ci;
979 unsigned char *map = si->swap_map;
980 int nr_reclaim;
981
982 if (force)
983 to_scan = swap_usage_in_pages(si) / SWAPFILE_CLUSTER;
984
985 while ((ci = isolate_lock_cluster(si, list: &si->full_clusters, order: 0))) {
986 offset = cluster_offset(si, ci);
987 end = min(si->max, offset + SWAPFILE_CLUSTER);
988 to_scan--;
989
990 while (offset < end) {
991 if (READ_ONCE(map[offset]) == SWAP_HAS_CACHE) {
992 spin_unlock(lock: &ci->lock);
993 nr_reclaim = __try_to_reclaim_swap(si, offset,
994 TTRS_ANYWAY);
995 spin_lock(lock: &ci->lock);
996 if (nr_reclaim) {
997 offset += abs(nr_reclaim);
998 continue;
999 }
1000 }
1001 offset++;
1002 }
1003
1004 /* in case no swap cache is reclaimed */
1005 if (ci->flags == CLUSTER_FLAG_NONE)
1006 relocate_cluster(si, ci);
1007
1008 swap_cluster_unlock(ci);
1009 if (to_scan <= 0)
1010 break;
1011 }
1012}
1013
1014static void swap_reclaim_work(struct work_struct *work)
1015{
1016 struct swap_info_struct *si;
1017
1018 si = container_of(work, struct swap_info_struct, reclaim_work);
1019
1020 swap_reclaim_full_clusters(si, force: true);
1021}
1022
1023/*
1024 * Try to allocate swap entries with specified order and try set a new
1025 * cluster for current CPU too.
1026 */
1027static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int order,
1028 unsigned char usage)
1029{
1030 struct swap_cluster_info *ci;
1031 unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
1032
1033 /*
1034 * Swapfile is not block device so unable
1035 * to allocate large entries.
1036 */
1037 if (order && !(si->flags & SWP_BLKDEV))
1038 return 0;
1039
1040 if (!(si->flags & SWP_SOLIDSTATE)) {
1041 /* Serialize HDD SWAP allocation for each device. */
1042 spin_lock(lock: &si->global_cluster_lock);
1043 offset = si->global_cluster->next[order];
1044 if (offset == SWAP_ENTRY_INVALID)
1045 goto new_cluster;
1046
1047 ci = swap_cluster_lock(si, offset);
1048 /* Cluster could have been used by another order */
1049 if (cluster_is_usable(ci, order)) {
1050 if (cluster_is_empty(info: ci))
1051 offset = cluster_offset(si, ci);
1052 found = alloc_swap_scan_cluster(si, ci, offset,
1053 order, usage);
1054 } else {
1055 swap_cluster_unlock(ci);
1056 }
1057 if (found)
1058 goto done;
1059 }
1060
1061new_cluster:
1062 /*
1063 * If the device need discard, prefer new cluster over nonfull
1064 * to spread out the writes.
1065 */
1066 if (si->flags & SWP_PAGE_DISCARD) {
1067 found = alloc_swap_scan_list(si, list: &si->free_clusters, order, usage,
1068 scan_all: false);
1069 if (found)
1070 goto done;
1071 }
1072
1073 if (order < PMD_ORDER) {
1074 found = alloc_swap_scan_list(si, list: &si->nonfull_clusters[order],
1075 order, usage, scan_all: true);
1076 if (found)
1077 goto done;
1078 }
1079
1080 if (!(si->flags & SWP_PAGE_DISCARD)) {
1081 found = alloc_swap_scan_list(si, list: &si->free_clusters, order, usage,
1082 scan_all: false);
1083 if (found)
1084 goto done;
1085 }
1086
1087 /* Try reclaim full clusters if free and nonfull lists are drained */
1088 if (vm_swap_full())
1089 swap_reclaim_full_clusters(si, force: false);
1090
1091 if (order < PMD_ORDER) {
1092 /*
1093 * Scan only one fragment cluster is good enough. Order 0
1094 * allocation will surely success, and large allocation
1095 * failure is not critical. Scanning one cluster still
1096 * keeps the list rotated and reclaimed (for HAS_CACHE).
1097 */
1098 found = alloc_swap_scan_list(si, list: &si->frag_clusters[order], order,
1099 usage, scan_all: false);
1100 if (found)
1101 goto done;
1102 }
1103
1104 /*
1105 * We don't have free cluster but have some clusters in discarding,
1106 * do discard now and reclaim them.
1107 */
1108 if ((si->flags & SWP_PAGE_DISCARD) && swap_do_scheduled_discard(si))
1109 goto new_cluster;
1110
1111 if (order)
1112 goto done;
1113
1114 /* Order 0 stealing from higher order */
1115 for (int o = 1; o < SWAP_NR_ORDERS; o++) {
1116 /*
1117 * Clusters here have at least one usable slots and can't fail order 0
1118 * allocation, but reclaim may drop si->lock and race with another user.
1119 */
1120 found = alloc_swap_scan_list(si, list: &si->frag_clusters[o],
1121 order: 0, usage, scan_all: true);
1122 if (found)
1123 goto done;
1124
1125 found = alloc_swap_scan_list(si, list: &si->nonfull_clusters[o],
1126 order: 0, usage, scan_all: true);
1127 if (found)
1128 goto done;
1129 }
1130done:
1131 if (!(si->flags & SWP_SOLIDSTATE))
1132 spin_unlock(lock: &si->global_cluster_lock);
1133
1134 return found;
1135}
1136
1137/* SWAP_USAGE_OFFLIST_BIT can only be set by this helper. */
1138static void del_from_avail_list(struct swap_info_struct *si, bool swapoff)
1139{
1140 int nid;
1141 unsigned long pages;
1142
1143 spin_lock(lock: &swap_avail_lock);
1144
1145 if (swapoff) {
1146 /*
1147 * Forcefully remove it. Clear the SWP_WRITEOK flags for
1148 * swapoff here so it's synchronized by both si->lock and
1149 * swap_avail_lock, to ensure the result can be seen by
1150 * add_to_avail_list.
1151 */
1152 lockdep_assert_held(&si->lock);
1153 si->flags &= ~SWP_WRITEOK;
1154 atomic_long_or(SWAP_USAGE_OFFLIST_BIT, v: &si->inuse_pages);
1155 } else {
1156 /*
1157 * If not called by swapoff, take it off-list only if it's
1158 * full and SWAP_USAGE_OFFLIST_BIT is not set (strictly
1159 * si->inuse_pages == pages), any concurrent slot freeing,
1160 * or device already removed from plist by someone else
1161 * will make this return false.
1162 */
1163 pages = si->pages;
1164 if (!atomic_long_try_cmpxchg(v: &si->inuse_pages, old: &pages,
1165 new: pages | SWAP_USAGE_OFFLIST_BIT))
1166 goto skip;
1167 }
1168
1169 for_each_node(nid)
1170 plist_del(node: &si->avail_lists[nid], head: &swap_avail_heads[nid]);
1171
1172skip:
1173 spin_unlock(lock: &swap_avail_lock);
1174}
1175
1176/* SWAP_USAGE_OFFLIST_BIT can only be cleared by this helper. */
1177static void add_to_avail_list(struct swap_info_struct *si, bool swapon)
1178{
1179 int nid;
1180 long val;
1181 unsigned long pages;
1182
1183 spin_lock(lock: &swap_avail_lock);
1184
1185 /* Corresponding to SWP_WRITEOK clearing in del_from_avail_list */
1186 if (swapon) {
1187 lockdep_assert_held(&si->lock);
1188 si->flags |= SWP_WRITEOK;
1189 } else {
1190 if (!(READ_ONCE(si->flags) & SWP_WRITEOK))
1191 goto skip;
1192 }
1193
1194 if (!(atomic_long_read(v: &si->inuse_pages) & SWAP_USAGE_OFFLIST_BIT))
1195 goto skip;
1196
1197 val = atomic_long_fetch_and_relaxed(i: ~SWAP_USAGE_OFFLIST_BIT, v: &si->inuse_pages);
1198
1199 /*
1200 * When device is full and device is on the plist, only one updater will
1201 * see (inuse_pages == si->pages) and will call del_from_avail_list. If
1202 * that updater happen to be here, just skip adding.
1203 */
1204 pages = si->pages;
1205 if (val == pages) {
1206 /* Just like the cmpxchg in del_from_avail_list */
1207 if (atomic_long_try_cmpxchg(v: &si->inuse_pages, old: &pages,
1208 new: pages | SWAP_USAGE_OFFLIST_BIT))
1209 goto skip;
1210 }
1211
1212 for_each_node(nid)
1213 plist_add(node: &si->avail_lists[nid], head: &swap_avail_heads[nid]);
1214
1215skip:
1216 spin_unlock(lock: &swap_avail_lock);
1217}
1218
1219/*
1220 * swap_usage_add / swap_usage_sub of each slot are serialized by ci->lock
1221 * within each cluster, so the total contribution to the global counter should
1222 * always be positive and cannot exceed the total number of usable slots.
1223 */
1224static bool swap_usage_add(struct swap_info_struct *si, unsigned int nr_entries)
1225{
1226 long val = atomic_long_add_return_relaxed(i: nr_entries, v: &si->inuse_pages);
1227
1228 /*
1229 * If device is full, and SWAP_USAGE_OFFLIST_BIT is not set,
1230 * remove it from the plist.
1231 */
1232 if (unlikely(val == si->pages)) {
1233 del_from_avail_list(si, swapoff: false);
1234 return true;
1235 }
1236
1237 return false;
1238}
1239
1240static void swap_usage_sub(struct swap_info_struct *si, unsigned int nr_entries)
1241{
1242 long val = atomic_long_sub_return_relaxed(i: nr_entries, v: &si->inuse_pages);
1243
1244 /*
1245 * If device is not full, and SWAP_USAGE_OFFLIST_BIT is set,
1246 * add it to the plist.
1247 */
1248 if (unlikely(val & SWAP_USAGE_OFFLIST_BIT))
1249 add_to_avail_list(si, swapon: false);
1250}
1251
1252static void swap_range_alloc(struct swap_info_struct *si,
1253 unsigned int nr_entries)
1254{
1255 if (swap_usage_add(si, nr_entries)) {
1256 if (vm_swap_full())
1257 schedule_work(work: &si->reclaim_work);
1258 }
1259 atomic_long_sub(i: nr_entries, v: &nr_swap_pages);
1260}
1261
1262static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
1263 unsigned int nr_entries)
1264{
1265 unsigned long begin = offset;
1266 unsigned long end = offset + nr_entries - 1;
1267 void (*swap_slot_free_notify)(struct block_device *, unsigned long);
1268 unsigned int i;
1269
1270 /*
1271 * Use atomic clear_bit operations only on zeromap instead of non-atomic
1272 * bitmap_clear to prevent adjacent bits corruption due to simultaneous writes.
1273 */
1274 for (i = 0; i < nr_entries; i++) {
1275 clear_bit(nr: offset + i, addr: si->zeromap);
1276 zswap_invalidate(swp: swp_entry(type: si->type, offset: offset + i));
1277 }
1278
1279 if (si->flags & SWP_BLKDEV)
1280 swap_slot_free_notify =
1281 si->bdev->bd_disk->fops->swap_slot_free_notify;
1282 else
1283 swap_slot_free_notify = NULL;
1284 while (offset <= end) {
1285 arch_swap_invalidate_page(type: si->type, offset);
1286 if (swap_slot_free_notify)
1287 swap_slot_free_notify(si->bdev, offset);
1288 offset++;
1289 }
1290 __swap_cache_clear_shadow(entry: swp_entry(type: si->type, offset: begin), nr_ents: nr_entries);
1291
1292 /*
1293 * Make sure that try_to_unuse() observes si->inuse_pages reaching 0
1294 * only after the above cleanups are done.
1295 */
1296 smp_wmb();
1297 atomic_long_add(i: nr_entries, v: &nr_swap_pages);
1298 swap_usage_sub(si, nr_entries);
1299}
1300
1301static bool get_swap_device_info(struct swap_info_struct *si)
1302{
1303 if (!percpu_ref_tryget_live(ref: &si->users))
1304 return false;
1305 /*
1306 * Guarantee the si->users are checked before accessing other
1307 * fields of swap_info_struct, and si->flags (SWP_WRITEOK) is
1308 * up to dated.
1309 *
1310 * Paired with the spin_unlock() after setup_swap_info() in
1311 * enable_swap_info(), and smp_wmb() in swapoff.
1312 */
1313 smp_rmb();
1314 return true;
1315}
1316
1317/*
1318 * Fast path try to get swap entries with specified order from current
1319 * CPU's swap entry pool (a cluster).
1320 */
1321static bool swap_alloc_fast(swp_entry_t *entry,
1322 int order)
1323{
1324 struct swap_cluster_info *ci;
1325 struct swap_info_struct *si;
1326 unsigned int offset, found = SWAP_ENTRY_INVALID;
1327
1328 /*
1329 * Once allocated, swap_info_struct will never be completely freed,
1330 * so checking it's liveness by get_swap_device_info is enough.
1331 */
1332 si = this_cpu_read(percpu_swap_cluster.si[order]);
1333 offset = this_cpu_read(percpu_swap_cluster.offset[order]);
1334 if (!si || !offset || !get_swap_device_info(si))
1335 return false;
1336
1337 ci = swap_cluster_lock(si, offset);
1338 if (cluster_is_usable(ci, order)) {
1339 if (cluster_is_empty(info: ci))
1340 offset = cluster_offset(si, ci);
1341 found = alloc_swap_scan_cluster(si, ci, offset, order, SWAP_HAS_CACHE);
1342 if (found)
1343 *entry = swp_entry(type: si->type, offset: found);
1344 } else {
1345 swap_cluster_unlock(ci);
1346 }
1347
1348 put_swap_device(si);
1349 return !!found;
1350}
1351
1352/* Rotate the device and switch to a new cluster */
1353static bool swap_alloc_slow(swp_entry_t *entry,
1354 int order)
1355{
1356 int node;
1357 unsigned long offset;
1358 struct swap_info_struct *si, *next;
1359
1360 node = numa_node_id();
1361 spin_lock(lock: &swap_avail_lock);
1362start_over:
1363 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) {
1364 /* Rotate the device and switch to a new cluster */
1365 plist_requeue(node: &si->avail_lists[node], head: &swap_avail_heads[node]);
1366 spin_unlock(lock: &swap_avail_lock);
1367 if (get_swap_device_info(si)) {
1368 offset = cluster_alloc_swap_entry(si, order, SWAP_HAS_CACHE);
1369 put_swap_device(si);
1370 if (offset) {
1371 *entry = swp_entry(type: si->type, offset);
1372 return true;
1373 }
1374 if (order)
1375 return false;
1376 }
1377
1378 spin_lock(lock: &swap_avail_lock);
1379 /*
1380 * if we got here, it's likely that si was almost full before,
1381 * and since scan_swap_map_slots() can drop the si->lock,
1382 * multiple callers probably all tried to get a page from the
1383 * same si and it filled up before we could get one; or, the si
1384 * filled up between us dropping swap_avail_lock and taking
1385 * si->lock. Since we dropped the swap_avail_lock, the
1386 * swap_avail_head list may have been modified; so if next is
1387 * still in the swap_avail_head list then try it, otherwise
1388 * start over if we have not gotten any slots.
1389 */
1390 if (plist_node_empty(node: &next->avail_lists[node]))
1391 goto start_over;
1392 }
1393 spin_unlock(lock: &swap_avail_lock);
1394 return false;
1395}
1396
1397/**
1398 * folio_alloc_swap - allocate swap space for a folio
1399 * @folio: folio we want to move to swap
1400 * @gfp: gfp mask for shadow nodes
1401 *
1402 * Allocate swap space for the folio and add the folio to the
1403 * swap cache.
1404 *
1405 * Context: Caller needs to hold the folio lock.
1406 * Return: Whether the folio was added to the swap cache.
1407 */
1408int folio_alloc_swap(struct folio *folio, gfp_t gfp)
1409{
1410 unsigned int order = folio_order(folio);
1411 unsigned int size = 1 << order;
1412 swp_entry_t entry = {};
1413
1414 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1415 VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio);
1416
1417 if (order) {
1418 /*
1419 * Reject large allocation when THP_SWAP is disabled,
1420 * the caller should split the folio and try again.
1421 */
1422 if (!IS_ENABLED(CONFIG_THP_SWAP))
1423 return -EAGAIN;
1424
1425 /*
1426 * Allocation size should never exceed cluster size
1427 * (HPAGE_PMD_SIZE).
1428 */
1429 if (size > SWAPFILE_CLUSTER) {
1430 VM_WARN_ON_ONCE(1);
1431 return -EINVAL;
1432 }
1433 }
1434
1435 local_lock(&percpu_swap_cluster.lock);
1436 if (!swap_alloc_fast(entry: &entry, order))
1437 swap_alloc_slow(entry: &entry, order);
1438 local_unlock(&percpu_swap_cluster.lock);
1439
1440 /* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */
1441 if (mem_cgroup_try_charge_swap(folio, entry))
1442 goto out_free;
1443
1444 if (!entry.val)
1445 return -ENOMEM;
1446
1447 swap_cache_add_folio(folio, entry, NULL);
1448
1449 return 0;
1450
1451out_free:
1452 put_swap_folio(folio, entry);
1453 return -ENOMEM;
1454}
1455
1456static struct swap_info_struct *_swap_info_get(swp_entry_t entry)
1457{
1458 struct swap_info_struct *si;
1459 unsigned long offset;
1460
1461 if (!entry.val)
1462 goto out;
1463 si = swap_entry_to_info(entry);
1464 if (!si)
1465 goto bad_nofile;
1466 if (data_race(!(si->flags & SWP_USED)))
1467 goto bad_device;
1468 offset = swp_offset(entry);
1469 if (offset >= si->max)
1470 goto bad_offset;
1471 if (data_race(!si->swap_map[swp_offset(entry)]))
1472 goto bad_free;
1473 return si;
1474
1475bad_free:
1476 pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val);
1477 goto out;
1478bad_offset:
1479 pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1480 goto out;
1481bad_device:
1482 pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val);
1483 goto out;
1484bad_nofile:
1485 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1486out:
1487 return NULL;
1488}
1489
1490static unsigned char swap_entry_put_locked(struct swap_info_struct *si,
1491 struct swap_cluster_info *ci,
1492 swp_entry_t entry,
1493 unsigned char usage)
1494{
1495 unsigned long offset = swp_offset(entry);
1496 unsigned char count;
1497 unsigned char has_cache;
1498
1499 count = si->swap_map[offset];
1500
1501 has_cache = count & SWAP_HAS_CACHE;
1502 count &= ~SWAP_HAS_CACHE;
1503
1504 if (usage == SWAP_HAS_CACHE) {
1505 VM_BUG_ON(!has_cache);
1506 has_cache = 0;
1507 } else if (count == SWAP_MAP_SHMEM) {
1508 /*
1509 * Or we could insist on shmem.c using a special
1510 * swap_shmem_free() and free_shmem_swap_and_cache()...
1511 */
1512 count = 0;
1513 } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
1514 if (count == COUNT_CONTINUED) {
1515 if (swap_count_continued(si, offset, count))
1516 count = SWAP_MAP_MAX | COUNT_CONTINUED;
1517 else
1518 count = SWAP_MAP_MAX;
1519 } else
1520 count--;
1521 }
1522
1523 usage = count | has_cache;
1524 if (usage)
1525 WRITE_ONCE(si->swap_map[offset], usage);
1526 else
1527 swap_entries_free(si, ci, entry, nr_pages: 1);
1528
1529 return usage;
1530}
1531
1532/*
1533 * When we get a swap entry, if there aren't some other ways to
1534 * prevent swapoff, such as the folio in swap cache is locked, RCU
1535 * reader side is locked, etc., the swap entry may become invalid
1536 * because of swapoff. Then, we need to enclose all swap related
1537 * functions with get_swap_device() and put_swap_device(), unless the
1538 * swap functions call get/put_swap_device() by themselves.
1539 *
1540 * RCU reader side lock (including any spinlock) is sufficient to
1541 * prevent swapoff, because synchronize_rcu() is called in swapoff()
1542 * before freeing data structures.
1543 *
1544 * Check whether swap entry is valid in the swap device. If so,
1545 * return pointer to swap_info_struct, and keep the swap entry valid
1546 * via preventing the swap device from being swapoff, until
1547 * put_swap_device() is called. Otherwise return NULL.
1548 *
1549 * Notice that swapoff or swapoff+swapon can still happen before the
1550 * percpu_ref_tryget_live() in get_swap_device() or after the
1551 * percpu_ref_put() in put_swap_device() if there isn't any other way
1552 * to prevent swapoff. The caller must be prepared for that. For
1553 * example, the following situation is possible.
1554 *
1555 * CPU1 CPU2
1556 * do_swap_page()
1557 * ... swapoff+swapon
1558 * __read_swap_cache_async()
1559 * swapcache_prepare()
1560 * __swap_duplicate()
1561 * // check swap_map
1562 * // verify PTE not changed
1563 *
1564 * In __swap_duplicate(), the swap_map need to be checked before
1565 * changing partly because the specified swap entry may be for another
1566 * swap device which has been swapoff. And in do_swap_page(), after
1567 * the page is read from the swap device, the PTE is verified not
1568 * changed with the page table locked to check whether the swap device
1569 * has been swapoff or swapoff+swapon.
1570 */
1571struct swap_info_struct *get_swap_device(swp_entry_t entry)
1572{
1573 struct swap_info_struct *si;
1574 unsigned long offset;
1575
1576 if (!entry.val)
1577 goto out;
1578 si = swap_entry_to_info(entry);
1579 if (!si)
1580 goto bad_nofile;
1581 if (!get_swap_device_info(si))
1582 goto out;
1583 offset = swp_offset(entry);
1584 if (offset >= si->max)
1585 goto put_out;
1586
1587 return si;
1588bad_nofile:
1589 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1590out:
1591 return NULL;
1592put_out:
1593 pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1594 percpu_ref_put(ref: &si->users);
1595 return NULL;
1596}
1597
1598static void swap_entries_put_cache(struct swap_info_struct *si,
1599 swp_entry_t entry, int nr)
1600{
1601 unsigned long offset = swp_offset(entry);
1602 struct swap_cluster_info *ci;
1603
1604 ci = swap_cluster_lock(si, offset);
1605 if (swap_only_has_cache(si, offset, nr_pages: nr)) {
1606 swap_entries_free(si, ci, entry, nr_pages: nr);
1607 } else {
1608 for (int i = 0; i < nr; i++, entry.val++)
1609 swap_entry_put_locked(si, ci, entry, SWAP_HAS_CACHE);
1610 }
1611 swap_cluster_unlock(ci);
1612}
1613
1614static bool swap_entries_put_map(struct swap_info_struct *si,
1615 swp_entry_t entry, int nr)
1616{
1617 unsigned long offset = swp_offset(entry);
1618 struct swap_cluster_info *ci;
1619 bool has_cache = false;
1620 unsigned char count;
1621 int i;
1622
1623 if (nr <= 1)
1624 goto fallback;
1625 count = swap_count(data_race(si->swap_map[offset]));
1626 if (count != 1 && count != SWAP_MAP_SHMEM)
1627 goto fallback;
1628
1629 ci = swap_cluster_lock(si, offset);
1630 if (!swap_is_last_map(si, offset, nr_pages: nr, has_cache: &has_cache)) {
1631 goto locked_fallback;
1632 }
1633 if (!has_cache)
1634 swap_entries_free(si, ci, entry, nr_pages: nr);
1635 else
1636 for (i = 0; i < nr; i++)
1637 WRITE_ONCE(si->swap_map[offset + i], SWAP_HAS_CACHE);
1638 swap_cluster_unlock(ci);
1639
1640 return has_cache;
1641
1642fallback:
1643 ci = swap_cluster_lock(si, offset);
1644locked_fallback:
1645 for (i = 0; i < nr; i++, entry.val++) {
1646 count = swap_entry_put_locked(si, ci, entry, usage: 1);
1647 if (count == SWAP_HAS_CACHE)
1648 has_cache = true;
1649 }
1650 swap_cluster_unlock(ci);
1651 return has_cache;
1652}
1653
1654/*
1655 * Only functions with "_nr" suffix are able to free entries spanning
1656 * cross multi clusters, so ensure the range is within a single cluster
1657 * when freeing entries with functions without "_nr" suffix.
1658 */
1659static bool swap_entries_put_map_nr(struct swap_info_struct *si,
1660 swp_entry_t entry, int nr)
1661{
1662 int cluster_nr, cluster_rest;
1663 unsigned long offset = swp_offset(entry);
1664 bool has_cache = false;
1665
1666 cluster_rest = SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER;
1667 while (nr) {
1668 cluster_nr = min(nr, cluster_rest);
1669 has_cache |= swap_entries_put_map(si, entry, nr: cluster_nr);
1670 cluster_rest = SWAPFILE_CLUSTER;
1671 nr -= cluster_nr;
1672 entry.val += cluster_nr;
1673 }
1674
1675 return has_cache;
1676}
1677
1678/*
1679 * Check if it's the last ref of swap entry in the freeing path.
1680 * Qualified vlaue includes 1, SWAP_HAS_CACHE or SWAP_MAP_SHMEM.
1681 */
1682static inline bool __maybe_unused swap_is_last_ref(unsigned char count)
1683{
1684 return (count == SWAP_HAS_CACHE) || (count == 1) ||
1685 (count == SWAP_MAP_SHMEM);
1686}
1687
1688/*
1689 * Drop the last ref of swap entries, caller have to ensure all entries
1690 * belong to the same cgroup and cluster.
1691 */
1692static void swap_entries_free(struct swap_info_struct *si,
1693 struct swap_cluster_info *ci,
1694 swp_entry_t entry, unsigned int nr_pages)
1695{
1696 unsigned long offset = swp_offset(entry);
1697 unsigned char *map = si->swap_map + offset;
1698 unsigned char *map_end = map + nr_pages;
1699
1700 /* It should never free entries across different clusters */
1701 VM_BUG_ON(ci != __swap_offset_to_cluster(si, offset + nr_pages - 1));
1702 VM_BUG_ON(cluster_is_empty(ci));
1703 VM_BUG_ON(ci->count < nr_pages);
1704
1705 ci->count -= nr_pages;
1706 do {
1707 VM_BUG_ON(!swap_is_last_ref(*map));
1708 *map = 0;
1709 } while (++map < map_end);
1710
1711 mem_cgroup_uncharge_swap(entry, nr_pages);
1712 swap_range_free(si, offset, nr_entries: nr_pages);
1713 swap_cluster_assert_table_empty(ci, start: offset, nr: nr_pages);
1714
1715 if (!ci->count)
1716 free_cluster(si, ci);
1717 else
1718 partial_free_cluster(si, ci);
1719}
1720
1721/*
1722 * Caller has made sure that the swap device corresponding to entry
1723 * is still around or has not been recycled.
1724 */
1725void swap_free_nr(swp_entry_t entry, int nr_pages)
1726{
1727 int nr;
1728 struct swap_info_struct *sis;
1729 unsigned long offset = swp_offset(entry);
1730
1731 sis = _swap_info_get(entry);
1732 if (!sis)
1733 return;
1734
1735 while (nr_pages) {
1736 nr = min_t(int, nr_pages, SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER);
1737 swap_entries_put_map(si: sis, entry: swp_entry(type: sis->type, offset), nr);
1738 offset += nr;
1739 nr_pages -= nr;
1740 }
1741}
1742
1743/*
1744 * Called after dropping swapcache to decrease refcnt to swap entries.
1745 */
1746void put_swap_folio(struct folio *folio, swp_entry_t entry)
1747{
1748 struct swap_info_struct *si;
1749 int size = 1 << swap_entry_order(folio_order(folio));
1750
1751 si = _swap_info_get(entry);
1752 if (!si)
1753 return;
1754
1755 swap_entries_put_cache(si, entry, nr: size);
1756}
1757
1758int __swap_count(swp_entry_t entry)
1759{
1760 struct swap_info_struct *si = __swap_entry_to_info(entry);
1761 pgoff_t offset = swp_offset(entry);
1762
1763 return swap_count(ent: si->swap_map[offset]);
1764}
1765
1766/*
1767 * How many references to @entry are currently swapped out?
1768 * This does not give an exact answer when swap count is continued,
1769 * but does include the high COUNT_CONTINUED flag to allow for that.
1770 */
1771bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry)
1772{
1773 pgoff_t offset = swp_offset(entry);
1774 struct swap_cluster_info *ci;
1775 int count;
1776
1777 ci = swap_cluster_lock(si, offset);
1778 count = swap_count(ent: si->swap_map[offset]);
1779 swap_cluster_unlock(ci);
1780 return !!count;
1781}
1782
1783/*
1784 * How many references to @entry are currently swapped out?
1785 * This considers COUNT_CONTINUED so it returns exact answer.
1786 */
1787int swp_swapcount(swp_entry_t entry)
1788{
1789 int count, tmp_count, n;
1790 struct swap_info_struct *si;
1791 struct swap_cluster_info *ci;
1792 struct page *page;
1793 pgoff_t offset;
1794 unsigned char *map;
1795
1796 si = _swap_info_get(entry);
1797 if (!si)
1798 return 0;
1799
1800 offset = swp_offset(entry);
1801
1802 ci = swap_cluster_lock(si, offset);
1803
1804 count = swap_count(ent: si->swap_map[offset]);
1805 if (!(count & COUNT_CONTINUED))
1806 goto out;
1807
1808 count &= ~COUNT_CONTINUED;
1809 n = SWAP_MAP_MAX + 1;
1810
1811 page = vmalloc_to_page(addr: si->swap_map + offset);
1812 offset &= ~PAGE_MASK;
1813 VM_BUG_ON(page_private(page) != SWP_CONTINUED);
1814
1815 do {
1816 page = list_next_entry(page, lru);
1817 map = kmap_local_page(page);
1818 tmp_count = map[offset];
1819 kunmap_local(map);
1820
1821 count += (tmp_count & ~COUNT_CONTINUED) * n;
1822 n *= (SWAP_CONT_MAX + 1);
1823 } while (tmp_count & COUNT_CONTINUED);
1824out:
1825 swap_cluster_unlock(ci);
1826 return count;
1827}
1828
1829static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
1830 swp_entry_t entry, int order)
1831{
1832 struct swap_cluster_info *ci;
1833 unsigned char *map = si->swap_map;
1834 unsigned int nr_pages = 1 << order;
1835 unsigned long roffset = swp_offset(entry);
1836 unsigned long offset = round_down(roffset, nr_pages);
1837 int i;
1838 bool ret = false;
1839
1840 ci = swap_cluster_lock(si, offset);
1841 if (nr_pages == 1) {
1842 if (swap_count(ent: map[roffset]))
1843 ret = true;
1844 goto unlock_out;
1845 }
1846 for (i = 0; i < nr_pages; i++) {
1847 if (swap_count(ent: map[offset + i])) {
1848 ret = true;
1849 break;
1850 }
1851 }
1852unlock_out:
1853 swap_cluster_unlock(ci);
1854 return ret;
1855}
1856
1857static bool folio_swapped(struct folio *folio)
1858{
1859 swp_entry_t entry = folio->swap;
1860 struct swap_info_struct *si = _swap_info_get(entry);
1861
1862 if (!si)
1863 return false;
1864
1865 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio)))
1866 return swap_entry_swapped(si, entry);
1867
1868 return swap_page_trans_huge_swapped(si, entry, order: folio_order(folio));
1869}
1870
1871static bool folio_swapcache_freeable(struct folio *folio)
1872{
1873 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1874
1875 if (!folio_test_swapcache(folio))
1876 return false;
1877 if (folio_test_writeback(folio))
1878 return false;
1879
1880 /*
1881 * Once hibernation has begun to create its image of memory,
1882 * there's a danger that one of the calls to folio_free_swap()
1883 * - most probably a call from __try_to_reclaim_swap() while
1884 * hibernation is allocating its own swap pages for the image,
1885 * but conceivably even a call from memory reclaim - will free
1886 * the swap from a folio which has already been recorded in the
1887 * image as a clean swapcache folio, and then reuse its swap for
1888 * another page of the image. On waking from hibernation, the
1889 * original folio might be freed under memory pressure, then
1890 * later read back in from swap, now with the wrong data.
1891 *
1892 * Hibernation suspends storage while it is writing the image
1893 * to disk so check that here.
1894 */
1895 if (pm_suspended_storage())
1896 return false;
1897
1898 return true;
1899}
1900
1901/**
1902 * folio_free_swap() - Free the swap space used for this folio.
1903 * @folio: The folio to remove.
1904 *
1905 * If swap is getting full, or if there are no more mappings of this folio,
1906 * then call folio_free_swap to free its swap space.
1907 *
1908 * Return: true if we were able to release the swap space.
1909 */
1910bool folio_free_swap(struct folio *folio)
1911{
1912 if (!folio_swapcache_freeable(folio))
1913 return false;
1914 if (folio_swapped(folio))
1915 return false;
1916
1917 swap_cache_del_folio(folio);
1918 folio_set_dirty(folio);
1919 return true;
1920}
1921
1922/**
1923 * free_swap_and_cache_nr() - Release reference on range of swap entries and
1924 * reclaim their cache if no more references remain.
1925 * @entry: First entry of range.
1926 * @nr: Number of entries in range.
1927 *
1928 * For each swap entry in the contiguous range, release a reference. If any swap
1929 * entries become free, try to reclaim their underlying folios, if present. The
1930 * offset range is defined by [entry.offset, entry.offset + nr).
1931 */
1932void free_swap_and_cache_nr(swp_entry_t entry, int nr)
1933{
1934 const unsigned long start_offset = swp_offset(entry);
1935 const unsigned long end_offset = start_offset + nr;
1936 struct swap_info_struct *si;
1937 bool any_only_cache = false;
1938 unsigned long offset;
1939
1940 si = get_swap_device(entry);
1941 if (!si)
1942 return;
1943
1944 if (WARN_ON(end_offset > si->max))
1945 goto out;
1946
1947 /*
1948 * First free all entries in the range.
1949 */
1950 any_only_cache = swap_entries_put_map_nr(si, entry, nr);
1951
1952 /*
1953 * Short-circuit the below loop if none of the entries had their
1954 * reference drop to zero.
1955 */
1956 if (!any_only_cache)
1957 goto out;
1958
1959 /*
1960 * Now go back over the range trying to reclaim the swap cache.
1961 */
1962 for (offset = start_offset; offset < end_offset; offset += nr) {
1963 nr = 1;
1964 if (READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) {
1965 /*
1966 * Folios are always naturally aligned in swap so
1967 * advance forward to the next boundary. Zero means no
1968 * folio was found for the swap entry, so advance by 1
1969 * in this case. Negative value means folio was found
1970 * but could not be reclaimed. Here we can still advance
1971 * to the next boundary.
1972 */
1973 nr = __try_to_reclaim_swap(si, offset,
1974 TTRS_UNMAPPED | TTRS_FULL);
1975 if (nr == 0)
1976 nr = 1;
1977 else if (nr < 0)
1978 nr = -nr;
1979 nr = ALIGN(offset + 1, nr) - offset;
1980 }
1981 }
1982
1983out:
1984 put_swap_device(si);
1985}
1986
1987#ifdef CONFIG_HIBERNATION
1988
1989swp_entry_t get_swap_page_of_type(int type)
1990{
1991 struct swap_info_struct *si = swap_type_to_info(type);
1992 unsigned long offset;
1993 swp_entry_t entry = {0};
1994
1995 if (!si)
1996 goto fail;
1997
1998 /* This is called for allocating swap entry, not cache */
1999 if (get_swap_device_info(si)) {
2000 if (si->flags & SWP_WRITEOK) {
2001 /*
2002 * Grab the local lock to be complaint
2003 * with swap table allocation.
2004 */
2005 local_lock(&percpu_swap_cluster.lock);
2006 offset = cluster_alloc_swap_entry(si, order: 0, usage: 1);
2007 local_unlock(&percpu_swap_cluster.lock);
2008 if (offset) {
2009 entry = swp_entry(type: si->type, offset);
2010 atomic_long_dec(v: &nr_swap_pages);
2011 }
2012 }
2013 put_swap_device(si);
2014 }
2015fail:
2016 return entry;
2017}
2018
2019/*
2020 * Find the swap type that corresponds to given device (if any).
2021 *
2022 * @offset - number of the PAGE_SIZE-sized block of the device, starting
2023 * from 0, in which the swap header is expected to be located.
2024 *
2025 * This is needed for the suspend to disk (aka swsusp).
2026 */
2027int swap_type_of(dev_t device, sector_t offset)
2028{
2029 int type;
2030
2031 if (!device)
2032 return -1;
2033
2034 spin_lock(lock: &swap_lock);
2035 for (type = 0; type < nr_swapfiles; type++) {
2036 struct swap_info_struct *sis = swap_info[type];
2037
2038 if (!(sis->flags & SWP_WRITEOK))
2039 continue;
2040
2041 if (device == sis->bdev->bd_dev) {
2042 struct swap_extent *se = first_se(sis);
2043
2044 if (se->start_block == offset) {
2045 spin_unlock(lock: &swap_lock);
2046 return type;
2047 }
2048 }
2049 }
2050 spin_unlock(lock: &swap_lock);
2051 return -ENODEV;
2052}
2053
2054int find_first_swap(dev_t *device)
2055{
2056 int type;
2057
2058 spin_lock(lock: &swap_lock);
2059 for (type = 0; type < nr_swapfiles; type++) {
2060 struct swap_info_struct *sis = swap_info[type];
2061
2062 if (!(sis->flags & SWP_WRITEOK))
2063 continue;
2064 *device = sis->bdev->bd_dev;
2065 spin_unlock(lock: &swap_lock);
2066 return type;
2067 }
2068 spin_unlock(lock: &swap_lock);
2069 return -ENODEV;
2070}
2071
2072/*
2073 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
2074 * corresponding to given index in swap_info (swap type).
2075 */
2076sector_t swapdev_block(int type, pgoff_t offset)
2077{
2078 struct swap_info_struct *si = swap_type_to_info(type);
2079 struct swap_extent *se;
2080
2081 if (!si || !(si->flags & SWP_WRITEOK))
2082 return 0;
2083 se = offset_to_swap_extent(sis: si, offset);
2084 return se->start_block + (offset - se->start_page);
2085}
2086
2087/*
2088 * Return either the total number of swap pages of given type, or the number
2089 * of free pages of that type (depending on @free)
2090 *
2091 * This is needed for software suspend
2092 */
2093unsigned int count_swap_pages(int type, int free)
2094{
2095 unsigned int n = 0;
2096
2097 spin_lock(lock: &swap_lock);
2098 if ((unsigned int)type < nr_swapfiles) {
2099 struct swap_info_struct *sis = swap_info[type];
2100
2101 spin_lock(lock: &sis->lock);
2102 if (sis->flags & SWP_WRITEOK) {
2103 n = sis->pages;
2104 if (free)
2105 n -= swap_usage_in_pages(si: sis);
2106 }
2107 spin_unlock(lock: &sis->lock);
2108 }
2109 spin_unlock(lock: &swap_lock);
2110 return n;
2111}
2112#endif /* CONFIG_HIBERNATION */
2113
2114static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
2115{
2116 return pte_same(a: pte_swp_clear_flags(pte), b: swp_pte);
2117}
2118
2119/*
2120 * No need to decide whether this PTE shares the swap entry with others,
2121 * just let do_wp_page work it out if a write is requested later - to
2122 * force COW, vm_page_prot omits write permission from any private vma.
2123 */
2124static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
2125 unsigned long addr, swp_entry_t entry, struct folio *folio)
2126{
2127 struct page *page;
2128 struct folio *swapcache;
2129 spinlock_t *ptl;
2130 pte_t *pte, new_pte, old_pte;
2131 bool hwpoisoned = false;
2132 int ret = 1;
2133
2134 /*
2135 * If the folio is removed from swap cache by others, continue to
2136 * unuse other PTEs. try_to_unuse may try again if we missed this one.
2137 */
2138 if (!folio_matches_swap_entry(folio, entry))
2139 return 0;
2140
2141 swapcache = folio;
2142 folio = ksm_might_need_to_copy(folio, vma, addr);
2143 if (unlikely(!folio))
2144 return -ENOMEM;
2145 else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
2146 hwpoisoned = true;
2147 folio = swapcache;
2148 }
2149
2150 page = folio_file_page(folio, index: swp_offset(entry));
2151 if (PageHWPoison(page))
2152 hwpoisoned = true;
2153
2154 pte = pte_offset_map_lock(mm: vma->vm_mm, pmd, addr, ptlp: &ptl);
2155 if (unlikely(!pte || !pte_same_as_swp(ptep_get(pte),
2156 swp_entry_to_pte(entry)))) {
2157 ret = 0;
2158 goto out;
2159 }
2160
2161 old_pte = ptep_get(ptep: pte);
2162
2163 if (unlikely(hwpoisoned || !folio_test_uptodate(folio))) {
2164 swp_entry_t swp_entry;
2165
2166 dec_mm_counter(mm: vma->vm_mm, member: MM_SWAPENTS);
2167 if (hwpoisoned) {
2168 swp_entry = make_hwpoison_entry(page);
2169 } else {
2170 swp_entry = make_poisoned_swp_entry();
2171 }
2172 new_pte = swp_entry_to_pte(entry: swp_entry);
2173 ret = 0;
2174 goto setpte;
2175 }
2176
2177 /*
2178 * Some architectures may have to restore extra metadata to the page
2179 * when reading from swap. This metadata may be indexed by swap entry
2180 * so this must be called before swap_free().
2181 */
2182 arch_swap_restore(entry: folio_swap(entry, folio), folio);
2183
2184 dec_mm_counter(mm: vma->vm_mm, member: MM_SWAPENTS);
2185 inc_mm_counter(mm: vma->vm_mm, member: MM_ANONPAGES);
2186 folio_get(folio);
2187 if (folio == swapcache) {
2188 rmap_t rmap_flags = RMAP_NONE;
2189
2190 /*
2191 * See do_swap_page(): writeback would be problematic.
2192 * However, we do a folio_wait_writeback() just before this
2193 * call and have the folio locked.
2194 */
2195 VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
2196 if (pte_swp_exclusive(pte: old_pte))
2197 rmap_flags |= RMAP_EXCLUSIVE;
2198 /*
2199 * We currently only expect small !anon folios, which are either
2200 * fully exclusive or fully shared. If we ever get large folios
2201 * here, we have to be careful.
2202 */
2203 if (!folio_test_anon(folio)) {
2204 VM_WARN_ON_ONCE(folio_test_large(folio));
2205 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
2206 folio_add_new_anon_rmap(folio, vma, address: addr, flags: rmap_flags);
2207 } else {
2208 folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags);
2209 }
2210 } else { /* ksm created a completely new copy */
2211 folio_add_new_anon_rmap(folio, vma, address: addr, RMAP_EXCLUSIVE);
2212 folio_add_lru_vma(folio, vma);
2213 }
2214 new_pte = pte_mkold(pte: mk_pte(page, pgprot: vma->vm_page_prot));
2215 if (pte_swp_soft_dirty(pte: old_pte))
2216 new_pte = pte_mksoft_dirty(pte: new_pte);
2217 if (pte_swp_uffd_wp(pte: old_pte))
2218 new_pte = pte_mkuffd_wp(pte: new_pte);
2219setpte:
2220 set_pte_at(vma->vm_mm, addr, pte, new_pte);
2221 swap_free(entry);
2222out:
2223 if (pte)
2224 pte_unmap_unlock(pte, ptl);
2225 if (folio != swapcache) {
2226 folio_unlock(folio);
2227 folio_put(folio);
2228 }
2229 return ret;
2230}
2231
2232static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
2233 unsigned long addr, unsigned long end,
2234 unsigned int type)
2235{
2236 pte_t *pte = NULL;
2237 struct swap_info_struct *si;
2238
2239 si = swap_info[type];
2240 do {
2241 struct folio *folio;
2242 unsigned long offset;
2243 unsigned char swp_count;
2244 swp_entry_t entry;
2245 int ret;
2246 pte_t ptent;
2247
2248 if (!pte++) {
2249 pte = pte_offset_map(pmd, addr);
2250 if (!pte)
2251 break;
2252 }
2253
2254 ptent = ptep_get_lockless(ptep: pte);
2255
2256 if (!is_swap_pte(pte: ptent))
2257 continue;
2258
2259 entry = pte_to_swp_entry(pte: ptent);
2260 if (swp_type(entry) != type)
2261 continue;
2262
2263 offset = swp_offset(entry);
2264 pte_unmap(pte);
2265 pte = NULL;
2266
2267 folio = swap_cache_get_folio(entry);
2268 if (!folio) {
2269 struct vm_fault vmf = {
2270 .vma = vma,
2271 .address = addr,
2272 .real_address = addr,
2273 .pmd = pmd,
2274 };
2275
2276 folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
2277 vmf: &vmf);
2278 }
2279 if (!folio) {
2280 swp_count = READ_ONCE(si->swap_map[offset]);
2281 if (swp_count == 0 || swp_count == SWAP_MAP_BAD)
2282 continue;
2283 return -ENOMEM;
2284 }
2285
2286 folio_lock(folio);
2287 folio_wait_writeback(folio);
2288 ret = unuse_pte(vma, pmd, addr, entry, folio);
2289 if (ret < 0) {
2290 folio_unlock(folio);
2291 folio_put(folio);
2292 return ret;
2293 }
2294
2295 folio_free_swap(folio);
2296 folio_unlock(folio);
2297 folio_put(folio);
2298 } while (addr += PAGE_SIZE, addr != end);
2299
2300 if (pte)
2301 pte_unmap(pte);
2302 return 0;
2303}
2304
2305static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
2306 unsigned long addr, unsigned long end,
2307 unsigned int type)
2308{
2309 pmd_t *pmd;
2310 unsigned long next;
2311 int ret;
2312
2313 pmd = pmd_offset(pud, address: addr);
2314 do {
2315 cond_resched();
2316 next = pmd_addr_end(addr, end);
2317 ret = unuse_pte_range(vma, pmd, addr, end: next, type);
2318 if (ret)
2319 return ret;
2320 } while (pmd++, addr = next, addr != end);
2321 return 0;
2322}
2323
2324static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
2325 unsigned long addr, unsigned long end,
2326 unsigned int type)
2327{
2328 pud_t *pud;
2329 unsigned long next;
2330 int ret;
2331
2332 pud = pud_offset(p4d, address: addr);
2333 do {
2334 next = pud_addr_end(addr, end);
2335 if (pud_none_or_clear_bad(pud))
2336 continue;
2337 ret = unuse_pmd_range(vma, pud, addr, end: next, type);
2338 if (ret)
2339 return ret;
2340 } while (pud++, addr = next, addr != end);
2341 return 0;
2342}
2343
2344static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
2345 unsigned long addr, unsigned long end,
2346 unsigned int type)
2347{
2348 p4d_t *p4d;
2349 unsigned long next;
2350 int ret;
2351
2352 p4d = p4d_offset(pgd, address: addr);
2353 do {
2354 next = p4d_addr_end(addr, end);
2355 if (p4d_none_or_clear_bad(p4d))
2356 continue;
2357 ret = unuse_pud_range(vma, p4d, addr, end: next, type);
2358 if (ret)
2359 return ret;
2360 } while (p4d++, addr = next, addr != end);
2361 return 0;
2362}
2363
2364static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
2365{
2366 pgd_t *pgd;
2367 unsigned long addr, end, next;
2368 int ret;
2369
2370 addr = vma->vm_start;
2371 end = vma->vm_end;
2372
2373 pgd = pgd_offset(vma->vm_mm, addr);
2374 do {
2375 next = pgd_addr_end(addr, end);
2376 if (pgd_none_or_clear_bad(pgd))
2377 continue;
2378 ret = unuse_p4d_range(vma, pgd, addr, end: next, type);
2379 if (ret)
2380 return ret;
2381 } while (pgd++, addr = next, addr != end);
2382 return 0;
2383}
2384
2385static int unuse_mm(struct mm_struct *mm, unsigned int type)
2386{
2387 struct vm_area_struct *vma;
2388 int ret = 0;
2389 VMA_ITERATOR(vmi, mm, 0);
2390
2391 mmap_read_lock(mm);
2392 if (check_stable_address_space(mm))
2393 goto unlock;
2394 for_each_vma(vmi, vma) {
2395 if (vma->anon_vma && !is_vm_hugetlb_page(vma)) {
2396 ret = unuse_vma(vma, type);
2397 if (ret)
2398 break;
2399 }
2400
2401 cond_resched();
2402 }
2403unlock:
2404 mmap_read_unlock(mm);
2405 return ret;
2406}
2407
2408/*
2409 * Scan swap_map from current position to next entry still in use.
2410 * Return 0 if there are no inuse entries after prev till end of
2411 * the map.
2412 */
2413static unsigned int find_next_to_unuse(struct swap_info_struct *si,
2414 unsigned int prev)
2415{
2416 unsigned int i;
2417 unsigned char count;
2418
2419 /*
2420 * No need for swap_lock here: we're just looking
2421 * for whether an entry is in use, not modifying it; false
2422 * hits are okay, and sys_swapoff() has already prevented new
2423 * allocations from this area (while holding swap_lock).
2424 */
2425 for (i = prev + 1; i < si->max; i++) {
2426 count = READ_ONCE(si->swap_map[i]);
2427 if (count && swap_count(ent: count) != SWAP_MAP_BAD)
2428 break;
2429 if ((i % LATENCY_LIMIT) == 0)
2430 cond_resched();
2431 }
2432
2433 if (i == si->max)
2434 i = 0;
2435
2436 return i;
2437}
2438
2439static int try_to_unuse(unsigned int type)
2440{
2441 struct mm_struct *prev_mm;
2442 struct mm_struct *mm;
2443 struct list_head *p;
2444 int retval = 0;
2445 struct swap_info_struct *si = swap_info[type];
2446 struct folio *folio;
2447 swp_entry_t entry;
2448 unsigned int i;
2449
2450 if (!swap_usage_in_pages(si))
2451 goto success;
2452
2453retry:
2454 retval = shmem_unuse(type);
2455 if (retval)
2456 return retval;
2457
2458 prev_mm = &init_mm;
2459 mmget(mm: prev_mm);
2460
2461 spin_lock(lock: &mmlist_lock);
2462 p = &init_mm.mmlist;
2463 while (swap_usage_in_pages(si) &&
2464 !signal_pending(current) &&
2465 (p = p->next) != &init_mm.mmlist) {
2466
2467 mm = list_entry(p, struct mm_struct, mmlist);
2468 if (!mmget_not_zero(mm))
2469 continue;
2470 spin_unlock(lock: &mmlist_lock);
2471 mmput(prev_mm);
2472 prev_mm = mm;
2473 retval = unuse_mm(mm, type);
2474 if (retval) {
2475 mmput(prev_mm);
2476 return retval;
2477 }
2478
2479 /*
2480 * Make sure that we aren't completely killing
2481 * interactive performance.
2482 */
2483 cond_resched();
2484 spin_lock(lock: &mmlist_lock);
2485 }
2486 spin_unlock(lock: &mmlist_lock);
2487
2488 mmput(prev_mm);
2489
2490 i = 0;
2491 while (swap_usage_in_pages(si) &&
2492 !signal_pending(current) &&
2493 (i = find_next_to_unuse(si, prev: i)) != 0) {
2494
2495 entry = swp_entry(type, offset: i);
2496 folio = swap_cache_get_folio(entry);
2497 if (!folio)
2498 continue;
2499
2500 /*
2501 * It is conceivable that a racing task removed this folio from
2502 * swap cache just before we acquired the page lock. The folio
2503 * might even be back in swap cache on another swap area. But
2504 * that is okay, folio_free_swap() only removes stale folios.
2505 */
2506 folio_lock(folio);
2507 folio_wait_writeback(folio);
2508 folio_free_swap(folio);
2509 folio_unlock(folio);
2510 folio_put(folio);
2511 }
2512
2513 /*
2514 * Lets check again to see if there are still swap entries in the map.
2515 * If yes, we would need to do retry the unuse logic again.
2516 * Under global memory pressure, swap entries can be reinserted back
2517 * into process space after the mmlist loop above passes over them.
2518 *
2519 * Limit the number of retries? No: when mmget_not_zero()
2520 * above fails, that mm is likely to be freeing swap from
2521 * exit_mmap(), which proceeds at its own independent pace;
2522 * and even shmem_writeout() could have been preempted after
2523 * folio_alloc_swap(), temporarily hiding that swap. It's easy
2524 * and robust (though cpu-intensive) just to keep retrying.
2525 */
2526 if (swap_usage_in_pages(si)) {
2527 if (!signal_pending(current))
2528 goto retry;
2529 return -EINTR;
2530 }
2531
2532success:
2533 /*
2534 * Make sure that further cleanups after try_to_unuse() returns happen
2535 * after swap_range_free() reduces si->inuse_pages to 0.
2536 */
2537 smp_mb();
2538 return 0;
2539}
2540
2541/*
2542 * After a successful try_to_unuse, if no swap is now in use, we know
2543 * we can empty the mmlist. swap_lock must be held on entry and exit.
2544 * Note that mmlist_lock nests inside swap_lock, and an mm must be
2545 * added to the mmlist just after page_duplicate - before would be racy.
2546 */
2547static void drain_mmlist(void)
2548{
2549 struct list_head *p, *next;
2550 unsigned int type;
2551
2552 for (type = 0; type < nr_swapfiles; type++)
2553 if (swap_usage_in_pages(si: swap_info[type]))
2554 return;
2555 spin_lock(lock: &mmlist_lock);
2556 list_for_each_safe(p, next, &init_mm.mmlist)
2557 list_del_init(entry: p);
2558 spin_unlock(lock: &mmlist_lock);
2559}
2560
2561/*
2562 * Free all of a swapdev's extent information
2563 */
2564static void destroy_swap_extents(struct swap_info_struct *sis)
2565{
2566 while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) {
2567 struct rb_node *rb = sis->swap_extent_root.rb_node;
2568 struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node);
2569
2570 rb_erase(rb, &sis->swap_extent_root);
2571 kfree(objp: se);
2572 }
2573
2574 if (sis->flags & SWP_ACTIVATED) {
2575 struct file *swap_file = sis->swap_file;
2576 struct address_space *mapping = swap_file->f_mapping;
2577
2578 sis->flags &= ~SWP_ACTIVATED;
2579 if (mapping->a_ops->swap_deactivate)
2580 mapping->a_ops->swap_deactivate(swap_file);
2581 }
2582}
2583
2584/*
2585 * Add a block range (and the corresponding page range) into this swapdev's
2586 * extent tree.
2587 *
2588 * This function rather assumes that it is called in ascending page order.
2589 */
2590int
2591add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
2592 unsigned long nr_pages, sector_t start_block)
2593{
2594 struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL;
2595 struct swap_extent *se;
2596 struct swap_extent *new_se;
2597
2598 /*
2599 * place the new node at the right most since the
2600 * function is called in ascending page order.
2601 */
2602 while (*link) {
2603 parent = *link;
2604 link = &parent->rb_right;
2605 }
2606
2607 if (parent) {
2608 se = rb_entry(parent, struct swap_extent, rb_node);
2609 BUG_ON(se->start_page + se->nr_pages != start_page);
2610 if (se->start_block + se->nr_pages == start_block) {
2611 /* Merge it */
2612 se->nr_pages += nr_pages;
2613 return 0;
2614 }
2615 }
2616
2617 /* No merge, insert a new extent. */
2618 new_se = kmalloc(sizeof(*se), GFP_KERNEL);
2619 if (new_se == NULL)
2620 return -ENOMEM;
2621 new_se->start_page = start_page;
2622 new_se->nr_pages = nr_pages;
2623 new_se->start_block = start_block;
2624
2625 rb_link_node(node: &new_se->rb_node, parent, rb_link: link);
2626 rb_insert_color(&new_se->rb_node, &sis->swap_extent_root);
2627 return 1;
2628}
2629EXPORT_SYMBOL_GPL(add_swap_extent);
2630
2631/*
2632 * A `swap extent' is a simple thing which maps a contiguous range of pages
2633 * onto a contiguous range of disk blocks. A rbtree of swap extents is
2634 * built at swapon time and is then used at swap_writepage/swap_read_folio
2635 * time for locating where on disk a page belongs.
2636 *
2637 * If the swapfile is an S_ISBLK block device, a single extent is installed.
2638 * This is done so that the main operating code can treat S_ISBLK and S_ISREG
2639 * swap files identically.
2640 *
2641 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2642 * extent rbtree operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
2643 * swapfiles are handled *identically* after swapon time.
2644 *
2645 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
2646 * and will parse them into a rbtree, in PAGE_SIZE chunks. If some stray
2647 * blocks are found which do not fall within the PAGE_SIZE alignment
2648 * requirements, they are simply tossed out - we will never use those blocks
2649 * for swapping.
2650 *
2651 * For all swap devices we set S_SWAPFILE across the life of the swapon. This
2652 * prevents users from writing to the swap device, which will corrupt memory.
2653 *
2654 * The amount of disk space which a single swap extent represents varies.
2655 * Typically it is in the 1-4 megabyte range. So we can have hundreds of
2656 * extents in the rbtree. - akpm.
2657 */
2658static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
2659{
2660 struct file *swap_file = sis->swap_file;
2661 struct address_space *mapping = swap_file->f_mapping;
2662 struct inode *inode = mapping->host;
2663 int ret;
2664
2665 if (S_ISBLK(inode->i_mode)) {
2666 ret = add_swap_extent(sis, 0, sis->max, 0);
2667 *span = sis->pages;
2668 return ret;
2669 }
2670
2671 if (mapping->a_ops->swap_activate) {
2672 ret = mapping->a_ops->swap_activate(sis, swap_file, span);
2673 if (ret < 0)
2674 return ret;
2675 sis->flags |= SWP_ACTIVATED;
2676 if ((sis->flags & SWP_FS_OPS) &&
2677 sio_pool_init() != 0) {
2678 destroy_swap_extents(sis);
2679 return -ENOMEM;
2680 }
2681 return ret;
2682 }
2683
2684 return generic_swapfile_activate(sis, swap_file, span);
2685}
2686
2687static int swap_node(struct swap_info_struct *si)
2688{
2689 struct block_device *bdev;
2690
2691 if (si->bdev)
2692 bdev = si->bdev;
2693 else
2694 bdev = si->swap_file->f_inode->i_sb->s_bdev;
2695
2696 return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE;
2697}
2698
2699static void setup_swap_info(struct swap_info_struct *si, int prio,
2700 unsigned char *swap_map,
2701 struct swap_cluster_info *cluster_info,
2702 unsigned long *zeromap)
2703{
2704 int i;
2705
2706 if (prio >= 0)
2707 si->prio = prio;
2708 else
2709 si->prio = --least_priority;
2710 /*
2711 * the plist prio is negated because plist ordering is
2712 * low-to-high, while swap ordering is high-to-low
2713 */
2714 si->list.prio = -si->prio;
2715 for_each_node(i) {
2716 if (si->prio >= 0)
2717 si->avail_lists[i].prio = -si->prio;
2718 else {
2719 if (swap_node(si) == i)
2720 si->avail_lists[i].prio = 1;
2721 else
2722 si->avail_lists[i].prio = -si->prio;
2723 }
2724 }
2725 si->swap_map = swap_map;
2726 si->cluster_info = cluster_info;
2727 si->zeromap = zeromap;
2728}
2729
2730static void _enable_swap_info(struct swap_info_struct *si)
2731{
2732 atomic_long_add(i: si->pages, v: &nr_swap_pages);
2733 total_swap_pages += si->pages;
2734
2735 assert_spin_locked(&swap_lock);
2736 /*
2737 * both lists are plists, and thus priority ordered.
2738 * swap_active_head needs to be priority ordered for swapoff(),
2739 * which on removal of any swap_info_struct with an auto-assigned
2740 * (i.e. negative) priority increments the auto-assigned priority
2741 * of any lower-priority swap_info_structs.
2742 * swap_avail_head needs to be priority ordered for folio_alloc_swap(),
2743 * which allocates swap pages from the highest available priority
2744 * swap_info_struct.
2745 */
2746 plist_add(node: &si->list, head: &swap_active_head);
2747
2748 /* Add back to available list */
2749 add_to_avail_list(si, swapon: true);
2750}
2751
2752static void enable_swap_info(struct swap_info_struct *si, int prio,
2753 unsigned char *swap_map,
2754 struct swap_cluster_info *cluster_info,
2755 unsigned long *zeromap)
2756{
2757 spin_lock(lock: &swap_lock);
2758 spin_lock(lock: &si->lock);
2759 setup_swap_info(si, prio, swap_map, cluster_info, zeromap);
2760 spin_unlock(lock: &si->lock);
2761 spin_unlock(lock: &swap_lock);
2762 /*
2763 * Finished initializing swap device, now it's safe to reference it.
2764 */
2765 percpu_ref_resurrect(ref: &si->users);
2766 spin_lock(lock: &swap_lock);
2767 spin_lock(lock: &si->lock);
2768 _enable_swap_info(si);
2769 spin_unlock(lock: &si->lock);
2770 spin_unlock(lock: &swap_lock);
2771}
2772
2773static void reinsert_swap_info(struct swap_info_struct *si)
2774{
2775 spin_lock(lock: &swap_lock);
2776 spin_lock(lock: &si->lock);
2777 setup_swap_info(si, prio: si->prio, swap_map: si->swap_map, cluster_info: si->cluster_info, zeromap: si->zeromap);
2778 _enable_swap_info(si);
2779 spin_unlock(lock: &si->lock);
2780 spin_unlock(lock: &swap_lock);
2781}
2782
2783/*
2784 * Called after clearing SWP_WRITEOK, ensures cluster_alloc_range
2785 * see the updated flags, so there will be no more allocations.
2786 */
2787static void wait_for_allocation(struct swap_info_struct *si)
2788{
2789 unsigned long offset;
2790 unsigned long end = ALIGN(si->max, SWAPFILE_CLUSTER);
2791 struct swap_cluster_info *ci;
2792
2793 BUG_ON(si->flags & SWP_WRITEOK);
2794
2795 for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
2796 ci = swap_cluster_lock(si, offset);
2797 swap_cluster_unlock(ci);
2798 }
2799}
2800
2801static void free_cluster_info(struct swap_cluster_info *cluster_info,
2802 unsigned long maxpages)
2803{
2804 struct swap_cluster_info *ci;
2805 int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2806
2807 if (!cluster_info)
2808 return;
2809 for (i = 0; i < nr_clusters; i++) {
2810 ci = cluster_info + i;
2811 /* Cluster with bad marks count will have a remaining table */
2812 spin_lock(lock: &ci->lock);
2813 if (rcu_dereference_protected(ci->table, true)) {
2814 ci->count = 0;
2815 swap_cluster_free_table(ci);
2816 }
2817 spin_unlock(lock: &ci->lock);
2818 }
2819 kvfree(addr: cluster_info);
2820}
2821
2822/*
2823 * Called after swap device's reference count is dead, so
2824 * neither scan nor allocation will use it.
2825 */
2826static void flush_percpu_swap_cluster(struct swap_info_struct *si)
2827{
2828 int cpu, i;
2829 struct swap_info_struct **pcp_si;
2830
2831 for_each_possible_cpu(cpu) {
2832 pcp_si = per_cpu_ptr(percpu_swap_cluster.si, cpu);
2833 /*
2834 * Invalidate the percpu swap cluster cache, si->users
2835 * is dead, so no new user will point to it, just flush
2836 * any existing user.
2837 */
2838 for (i = 0; i < SWAP_NR_ORDERS; i++)
2839 cmpxchg(&pcp_si[i], si, NULL);
2840 }
2841}
2842
2843
2844SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
2845{
2846 struct swap_info_struct *p = NULL;
2847 unsigned char *swap_map;
2848 unsigned long *zeromap;
2849 struct swap_cluster_info *cluster_info;
2850 struct file *swap_file, *victim;
2851 struct address_space *mapping;
2852 struct inode *inode;
2853 struct filename *pathname;
2854 unsigned int maxpages;
2855 int err, found = 0;
2856
2857 if (!capable(CAP_SYS_ADMIN))
2858 return -EPERM;
2859
2860 BUG_ON(!current->mm);
2861
2862 pathname = getname(name: specialfile);
2863 if (IS_ERR(ptr: pathname))
2864 return PTR_ERR(ptr: pathname);
2865
2866 victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
2867 err = PTR_ERR(ptr: victim);
2868 if (IS_ERR(ptr: victim))
2869 goto out;
2870
2871 mapping = victim->f_mapping;
2872 spin_lock(lock: &swap_lock);
2873 plist_for_each_entry(p, &swap_active_head, list) {
2874 if (p->flags & SWP_WRITEOK) {
2875 if (p->swap_file->f_mapping == mapping) {
2876 found = 1;
2877 break;
2878 }
2879 }
2880 }
2881 if (!found) {
2882 err = -EINVAL;
2883 spin_unlock(lock: &swap_lock);
2884 goto out_dput;
2885 }
2886 if (!security_vm_enough_memory_mm(current->mm, pages: p->pages))
2887 vm_unacct_memory(pages: p->pages);
2888 else {
2889 err = -ENOMEM;
2890 spin_unlock(lock: &swap_lock);
2891 goto out_dput;
2892 }
2893 spin_lock(lock: &p->lock);
2894 del_from_avail_list(si: p, swapoff: true);
2895 if (p->prio < 0) {
2896 struct swap_info_struct *si = p;
2897 int nid;
2898
2899 plist_for_each_entry_continue(si, &swap_active_head, list) {
2900 si->prio++;
2901 si->list.prio--;
2902 for_each_node(nid) {
2903 if (si->avail_lists[nid].prio != 1)
2904 si->avail_lists[nid].prio--;
2905 }
2906 }
2907 least_priority++;
2908 }
2909 plist_del(node: &p->list, head: &swap_active_head);
2910 atomic_long_sub(i: p->pages, v: &nr_swap_pages);
2911 total_swap_pages -= p->pages;
2912 spin_unlock(lock: &p->lock);
2913 spin_unlock(lock: &swap_lock);
2914
2915 wait_for_allocation(si: p);
2916
2917 set_current_oom_origin();
2918 err = try_to_unuse(type: p->type);
2919 clear_current_oom_origin();
2920
2921 if (err) {
2922 /* re-insert swap space back into swap_list */
2923 reinsert_swap_info(si: p);
2924 goto out_dput;
2925 }
2926
2927 /*
2928 * Wait for swap operations protected by get/put_swap_device()
2929 * to complete. Because of synchronize_rcu() here, all swap
2930 * operations protected by RCU reader side lock (including any
2931 * spinlock) will be waited too. This makes it easy to
2932 * prevent folio_test_swapcache() and the following swap cache
2933 * operations from racing with swapoff.
2934 */
2935 percpu_ref_kill(ref: &p->users);
2936 synchronize_rcu();
2937 wait_for_completion(&p->comp);
2938
2939 flush_work(work: &p->discard_work);
2940 flush_work(work: &p->reclaim_work);
2941 flush_percpu_swap_cluster(si: p);
2942
2943 destroy_swap_extents(sis: p);
2944 if (p->flags & SWP_CONTINUED)
2945 free_swap_count_continuations(p);
2946
2947 if (!p->bdev || !bdev_nonrot(bdev: p->bdev))
2948 atomic_dec(v: &nr_rotate_swap);
2949
2950 mutex_lock(lock: &swapon_mutex);
2951 spin_lock(lock: &swap_lock);
2952 spin_lock(lock: &p->lock);
2953 drain_mmlist();
2954
2955 swap_file = p->swap_file;
2956 p->swap_file = NULL;
2957 swap_map = p->swap_map;
2958 p->swap_map = NULL;
2959 zeromap = p->zeromap;
2960 p->zeromap = NULL;
2961 maxpages = p->max;
2962 cluster_info = p->cluster_info;
2963 p->max = 0;
2964 p->cluster_info = NULL;
2965 spin_unlock(lock: &p->lock);
2966 spin_unlock(lock: &swap_lock);
2967 arch_swap_invalidate_area(type: p->type);
2968 zswap_swapoff(type: p->type);
2969 mutex_unlock(lock: &swapon_mutex);
2970 kfree(objp: p->global_cluster);
2971 p->global_cluster = NULL;
2972 vfree(addr: swap_map);
2973 kvfree(addr: zeromap);
2974 free_cluster_info(cluster_info, maxpages);
2975 /* Destroy swap account information */
2976 swap_cgroup_swapoff(type: p->type);
2977
2978 inode = mapping->host;
2979
2980 inode_lock(inode);
2981 inode->i_flags &= ~S_SWAPFILE;
2982 inode_unlock(inode);
2983 filp_close(swap_file, NULL);
2984
2985 /*
2986 * Clear the SWP_USED flag after all resources are freed so that swapon
2987 * can reuse this swap_info in alloc_swap_info() safely. It is ok to
2988 * not hold p->lock after we cleared its SWP_WRITEOK.
2989 */
2990 spin_lock(lock: &swap_lock);
2991 p->flags = 0;
2992 spin_unlock(lock: &swap_lock);
2993
2994 err = 0;
2995 atomic_inc(v: &proc_poll_event);
2996 wake_up_interruptible(&proc_poll_wait);
2997
2998out_dput:
2999 filp_close(victim, NULL);
3000out:
3001 putname(name: pathname);
3002 return err;
3003}
3004
3005#ifdef CONFIG_PROC_FS
3006static __poll_t swaps_poll(struct file *file, poll_table *wait)
3007{
3008 struct seq_file *seq = file->private_data;
3009
3010 poll_wait(filp: file, wait_address: &proc_poll_wait, p: wait);
3011
3012 if (seq->poll_event != atomic_read(v: &proc_poll_event)) {
3013 seq->poll_event = atomic_read(v: &proc_poll_event);
3014 return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI;
3015 }
3016
3017 return EPOLLIN | EPOLLRDNORM;
3018}
3019
3020/* iterator */
3021static void *swap_start(struct seq_file *swap, loff_t *pos)
3022{
3023 struct swap_info_struct *si;
3024 int type;
3025 loff_t l = *pos;
3026
3027 mutex_lock(lock: &swapon_mutex);
3028
3029 if (!l)
3030 return SEQ_START_TOKEN;
3031
3032 for (type = 0; (si = swap_type_to_info(type)); type++) {
3033 if (!(si->flags & SWP_USED) || !si->swap_map)
3034 continue;
3035 if (!--l)
3036 return si;
3037 }
3038
3039 return NULL;
3040}
3041
3042static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
3043{
3044 struct swap_info_struct *si = v;
3045 int type;
3046
3047 if (v == SEQ_START_TOKEN)
3048 type = 0;
3049 else
3050 type = si->type + 1;
3051
3052 ++(*pos);
3053 for (; (si = swap_type_to_info(type)); type++) {
3054 if (!(si->flags & SWP_USED) || !si->swap_map)
3055 continue;
3056 return si;
3057 }
3058
3059 return NULL;
3060}
3061
3062static void swap_stop(struct seq_file *swap, void *v)
3063{
3064 mutex_unlock(lock: &swapon_mutex);
3065}
3066
3067static int swap_show(struct seq_file *swap, void *v)
3068{
3069 struct swap_info_struct *si = v;
3070 struct file *file;
3071 int len;
3072 unsigned long bytes, inuse;
3073
3074 if (si == SEQ_START_TOKEN) {
3075 seq_puts(m: swap, s: "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");
3076 return 0;
3077 }
3078
3079 bytes = K(si->pages);
3080 inuse = K(swap_usage_in_pages(si));
3081
3082 file = si->swap_file;
3083 len = seq_file_path(swap, file, " \t\n\\");
3084 seq_printf(m: swap, fmt: "%*s%s\t%lu\t%s%lu\t%s%d\n",
3085 len < 40 ? 40 - len : 1, " ",
3086 S_ISBLK(file_inode(file)->i_mode) ?
3087 "partition" : "file\t",
3088 bytes, bytes < 10000000 ? "\t" : "",
3089 inuse, inuse < 10000000 ? "\t" : "",
3090 si->prio);
3091 return 0;
3092}
3093
3094static const struct seq_operations swaps_op = {
3095 .start = swap_start,
3096 .next = swap_next,
3097 .stop = swap_stop,
3098 .show = swap_show
3099};
3100
3101static int swaps_open(struct inode *inode, struct file *file)
3102{
3103 struct seq_file *seq;
3104 int ret;
3105
3106 ret = seq_open(file, &swaps_op);
3107 if (ret)
3108 return ret;
3109
3110 seq = file->private_data;
3111 seq->poll_event = atomic_read(v: &proc_poll_event);
3112 return 0;
3113}
3114
3115static const struct proc_ops swaps_proc_ops = {
3116 .proc_flags = PROC_ENTRY_PERMANENT,
3117 .proc_open = swaps_open,
3118 .proc_read = seq_read,
3119 .proc_lseek = seq_lseek,
3120 .proc_release = seq_release,
3121 .proc_poll = swaps_poll,
3122};
3123
3124static int __init procswaps_init(void)
3125{
3126 proc_create(name: "swaps", mode: 0, NULL, proc_ops: &swaps_proc_ops);
3127 return 0;
3128}
3129__initcall(procswaps_init);
3130#endif /* CONFIG_PROC_FS */
3131
3132#ifdef MAX_SWAPFILES_CHECK
3133static int __init max_swapfiles_check(void)
3134{
3135 MAX_SWAPFILES_CHECK();
3136 return 0;
3137}
3138late_initcall(max_swapfiles_check);
3139#endif
3140
3141static struct swap_info_struct *alloc_swap_info(void)
3142{
3143 struct swap_info_struct *p;
3144 struct swap_info_struct *defer = NULL;
3145 unsigned int type;
3146 int i;
3147
3148 p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL);
3149 if (!p)
3150 return ERR_PTR(error: -ENOMEM);
3151
3152 if (percpu_ref_init(ref: &p->users, release: swap_users_ref_free,
3153 flags: PERCPU_REF_INIT_DEAD, GFP_KERNEL)) {
3154 kvfree(addr: p);
3155 return ERR_PTR(error: -ENOMEM);
3156 }
3157
3158 spin_lock(lock: &swap_lock);
3159 for (type = 0; type < nr_swapfiles; type++) {
3160 if (!(swap_info[type]->flags & SWP_USED))
3161 break;
3162 }
3163 if (type >= MAX_SWAPFILES) {
3164 spin_unlock(lock: &swap_lock);
3165 percpu_ref_exit(ref: &p->users);
3166 kvfree(addr: p);
3167 return ERR_PTR(error: -EPERM);
3168 }
3169 if (type >= nr_swapfiles) {
3170 p->type = type;
3171 /*
3172 * Publish the swap_info_struct after initializing it.
3173 * Note that kvzalloc() above zeroes all its fields.
3174 */
3175 smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */
3176 nr_swapfiles++;
3177 } else {
3178 defer = p;
3179 p = swap_info[type];
3180 /*
3181 * Do not memset this entry: a racing procfs swap_next()
3182 * would be relying on p->type to remain valid.
3183 */
3184 }
3185 p->swap_extent_root = RB_ROOT;
3186 plist_node_init(node: &p->list, prio: 0);
3187 for_each_node(i)
3188 plist_node_init(node: &p->avail_lists[i], prio: 0);
3189 p->flags = SWP_USED;
3190 spin_unlock(lock: &swap_lock);
3191 if (defer) {
3192 percpu_ref_exit(ref: &defer->users);
3193 kvfree(addr: defer);
3194 }
3195 spin_lock_init(&p->lock);
3196 spin_lock_init(&p->cont_lock);
3197 atomic_long_set(v: &p->inuse_pages, SWAP_USAGE_OFFLIST_BIT);
3198 init_completion(x: &p->comp);
3199
3200 return p;
3201}
3202
3203static int claim_swapfile(struct swap_info_struct *si, struct inode *inode)
3204{
3205 if (S_ISBLK(inode->i_mode)) {
3206 si->bdev = I_BDEV(inode);
3207 /*
3208 * Zoned block devices contain zones that have a sequential
3209 * write only restriction. Hence zoned block devices are not
3210 * suitable for swapping. Disallow them here.
3211 */
3212 if (bdev_is_zoned(bdev: si->bdev))
3213 return -EINVAL;
3214 si->flags |= SWP_BLKDEV;
3215 } else if (S_ISREG(inode->i_mode)) {
3216 si->bdev = inode->i_sb->s_bdev;
3217 }
3218
3219 return 0;
3220}
3221
3222
3223/*
3224 * Find out how many pages are allowed for a single swap device. There
3225 * are two limiting factors:
3226 * 1) the number of bits for the swap offset in the swp_entry_t type, and
3227 * 2) the number of bits in the swap pte, as defined by the different
3228 * architectures.
3229 *
3230 * In order to find the largest possible bit mask, a swap entry with
3231 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
3232 * decoded to a swp_entry_t again, and finally the swap offset is
3233 * extracted.
3234 *
3235 * This will mask all the bits from the initial ~0UL mask that can't
3236 * be encoded in either the swp_entry_t or the architecture definition
3237 * of a swap pte.
3238 */
3239unsigned long generic_max_swapfile_size(void)
3240{
3241 return swp_offset(entry: pte_to_swp_entry(
3242 pte: swp_entry_to_pte(entry: swp_entry(type: 0, offset: ~0UL)))) + 1;
3243}
3244
3245/* Can be overridden by an architecture for additional checks. */
3246__weak unsigned long arch_max_swapfile_size(void)
3247{
3248 return generic_max_swapfile_size();
3249}
3250
3251static unsigned long read_swap_header(struct swap_info_struct *si,
3252 union swap_header *swap_header,
3253 struct inode *inode)
3254{
3255 int i;
3256 unsigned long maxpages;
3257 unsigned long swapfilepages;
3258 unsigned long last_page;
3259
3260 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
3261 pr_err("Unable to find swap-space signature\n");
3262 return 0;
3263 }
3264
3265 /* swap partition endianness hack... */
3266 if (swab32(swap_header->info.version) == 1) {
3267 swab32s(p: &swap_header->info.version);
3268 swab32s(p: &swap_header->info.last_page);
3269 swab32s(p: &swap_header->info.nr_badpages);
3270 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3271 return 0;
3272 for (i = 0; i < swap_header->info.nr_badpages; i++)
3273 swab32s(p: &swap_header->info.badpages[i]);
3274 }
3275 /* Check the swap header's sub-version */
3276 if (swap_header->info.version != 1) {
3277 pr_warn("Unable to handle swap header version %d\n",
3278 swap_header->info.version);
3279 return 0;
3280 }
3281
3282 maxpages = swapfile_maximum_size;
3283 last_page = swap_header->info.last_page;
3284 if (!last_page) {
3285 pr_warn("Empty swap-file\n");
3286 return 0;
3287 }
3288 if (last_page > maxpages) {
3289 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
3290 K(maxpages), K(last_page));
3291 }
3292 if (maxpages > last_page) {
3293 maxpages = last_page + 1;
3294 /* p->max is an unsigned int: don't overflow it */
3295 if ((unsigned int)maxpages == 0)
3296 maxpages = UINT_MAX;
3297 }
3298
3299 if (!maxpages)
3300 return 0;
3301 swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
3302 if (swapfilepages && maxpages > swapfilepages) {
3303 pr_warn("Swap area shorter than signature indicates\n");
3304 return 0;
3305 }
3306 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
3307 return 0;
3308 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3309 return 0;
3310
3311 return maxpages;
3312}
3313
3314static int setup_swap_map(struct swap_info_struct *si,
3315 union swap_header *swap_header,
3316 unsigned char *swap_map,
3317 unsigned long maxpages)
3318{
3319 unsigned long i;
3320
3321 swap_map[0] = SWAP_MAP_BAD; /* omit header page */
3322 for (i = 0; i < swap_header->info.nr_badpages; i++) {
3323 unsigned int page_nr = swap_header->info.badpages[i];
3324 if (page_nr == 0 || page_nr > swap_header->info.last_page)
3325 return -EINVAL;
3326 if (page_nr < maxpages) {
3327 swap_map[page_nr] = SWAP_MAP_BAD;
3328 si->pages--;
3329 }
3330 }
3331
3332 if (!si->pages) {
3333 pr_warn("Empty swap-file\n");
3334 return -EINVAL;
3335 }
3336
3337 return 0;
3338}
3339
3340static struct swap_cluster_info *setup_clusters(struct swap_info_struct *si,
3341 union swap_header *swap_header,
3342 unsigned long maxpages)
3343{
3344 unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3345 struct swap_cluster_info *cluster_info;
3346 int err = -ENOMEM;
3347 unsigned long i;
3348
3349 cluster_info = kvcalloc(nr_clusters, sizeof(*cluster_info), GFP_KERNEL);
3350 if (!cluster_info)
3351 goto err;
3352
3353 for (i = 0; i < nr_clusters; i++)
3354 spin_lock_init(&cluster_info[i].lock);
3355
3356 if (!(si->flags & SWP_SOLIDSTATE)) {
3357 si->global_cluster = kmalloc(sizeof(*si->global_cluster),
3358 GFP_KERNEL);
3359 if (!si->global_cluster)
3360 goto err_free;
3361 for (i = 0; i < SWAP_NR_ORDERS; i++)
3362 si->global_cluster->next[i] = SWAP_ENTRY_INVALID;
3363 spin_lock_init(&si->global_cluster_lock);
3364 }
3365
3366 /*
3367 * Mark unusable pages as unavailable. The clusters aren't
3368 * marked free yet, so no list operations are involved yet.
3369 *
3370 * See setup_swap_map(): header page, bad pages,
3371 * and the EOF part of the last cluster.
3372 */
3373 err = inc_cluster_info_page(si, cluster_info, page_nr: 0);
3374 if (err)
3375 goto err;
3376 for (i = 0; i < swap_header->info.nr_badpages; i++) {
3377 unsigned int page_nr = swap_header->info.badpages[i];
3378
3379 if (page_nr >= maxpages)
3380 continue;
3381 err = inc_cluster_info_page(si, cluster_info, page_nr);
3382 if (err)
3383 goto err;
3384 }
3385 for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) {
3386 err = inc_cluster_info_page(si, cluster_info, page_nr: i);
3387 if (err)
3388 goto err;
3389 }
3390
3391 INIT_LIST_HEAD(list: &si->free_clusters);
3392 INIT_LIST_HEAD(list: &si->full_clusters);
3393 INIT_LIST_HEAD(list: &si->discard_clusters);
3394
3395 for (i = 0; i < SWAP_NR_ORDERS; i++) {
3396 INIT_LIST_HEAD(list: &si->nonfull_clusters[i]);
3397 INIT_LIST_HEAD(list: &si->frag_clusters[i]);
3398 }
3399
3400 for (i = 0; i < nr_clusters; i++) {
3401 struct swap_cluster_info *ci = &cluster_info[i];
3402
3403 if (ci->count) {
3404 ci->flags = CLUSTER_FLAG_NONFULL;
3405 list_add_tail(new: &ci->list, head: &si->nonfull_clusters[0]);
3406 } else {
3407 ci->flags = CLUSTER_FLAG_FREE;
3408 list_add_tail(new: &ci->list, head: &si->free_clusters);
3409 }
3410 }
3411
3412 return cluster_info;
3413err_free:
3414 free_cluster_info(cluster_info, maxpages);
3415err:
3416 return ERR_PTR(error: err);
3417}
3418
3419SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
3420{
3421 struct swap_info_struct *si;
3422 struct filename *name;
3423 struct file *swap_file = NULL;
3424 struct address_space *mapping;
3425 struct dentry *dentry;
3426 int prio;
3427 int error;
3428 union swap_header *swap_header;
3429 int nr_extents;
3430 sector_t span;
3431 unsigned long maxpages;
3432 unsigned char *swap_map = NULL;
3433 unsigned long *zeromap = NULL;
3434 struct swap_cluster_info *cluster_info = NULL;
3435 struct folio *folio = NULL;
3436 struct inode *inode = NULL;
3437 bool inced_nr_rotate_swap = false;
3438
3439 if (swap_flags & ~SWAP_FLAGS_VALID)
3440 return -EINVAL;
3441
3442 if (!capable(CAP_SYS_ADMIN))
3443 return -EPERM;
3444
3445 if (!swap_avail_heads)
3446 return -ENOMEM;
3447
3448 si = alloc_swap_info();
3449 if (IS_ERR(ptr: si))
3450 return PTR_ERR(ptr: si);
3451
3452 INIT_WORK(&si->discard_work, swap_discard_work);
3453 INIT_WORK(&si->reclaim_work, swap_reclaim_work);
3454
3455 name = getname(name: specialfile);
3456 if (IS_ERR(ptr: name)) {
3457 error = PTR_ERR(ptr: name);
3458 name = NULL;
3459 goto bad_swap;
3460 }
3461 swap_file = file_open_name(name, O_RDWR | O_LARGEFILE | O_EXCL, 0);
3462 if (IS_ERR(ptr: swap_file)) {
3463 error = PTR_ERR(ptr: swap_file);
3464 swap_file = NULL;
3465 goto bad_swap;
3466 }
3467
3468 si->swap_file = swap_file;
3469 mapping = swap_file->f_mapping;
3470 dentry = swap_file->f_path.dentry;
3471 inode = mapping->host;
3472
3473 error = claim_swapfile(si, inode);
3474 if (unlikely(error))
3475 goto bad_swap;
3476
3477 inode_lock(inode);
3478 if (d_unlinked(dentry) || cant_mount(dentry)) {
3479 error = -ENOENT;
3480 goto bad_swap_unlock_inode;
3481 }
3482 if (IS_SWAPFILE(inode)) {
3483 error = -EBUSY;
3484 goto bad_swap_unlock_inode;
3485 }
3486
3487 /*
3488 * The swap subsystem needs a major overhaul to support this.
3489 * It doesn't work yet so just disable it for now.
3490 */
3491 if (mapping_min_folio_order(mapping) > 0) {
3492 error = -EINVAL;
3493 goto bad_swap_unlock_inode;
3494 }
3495
3496 /*
3497 * Read the swap header.
3498 */
3499 if (!mapping->a_ops->read_folio) {
3500 error = -EINVAL;
3501 goto bad_swap_unlock_inode;
3502 }
3503 folio = read_mapping_folio(mapping, index: 0, file: swap_file);
3504 if (IS_ERR(ptr: folio)) {
3505 error = PTR_ERR(ptr: folio);
3506 goto bad_swap_unlock_inode;
3507 }
3508 swap_header = kmap_local_folio(folio, offset: 0);
3509
3510 maxpages = read_swap_header(si, swap_header, inode);
3511 if (unlikely(!maxpages)) {
3512 error = -EINVAL;
3513 goto bad_swap_unlock_inode;
3514 }
3515
3516 si->max = maxpages;
3517 si->pages = maxpages - 1;
3518 nr_extents = setup_swap_extents(sis: si, span: &span);
3519 if (nr_extents < 0) {
3520 error = nr_extents;
3521 goto bad_swap_unlock_inode;
3522 }
3523 if (si->pages != si->max - 1) {
3524 pr_err("swap:%u != (max:%u - 1)\n", si->pages, si->max);
3525 error = -EINVAL;
3526 goto bad_swap_unlock_inode;
3527 }
3528
3529 maxpages = si->max;
3530
3531 /* OK, set up the swap map and apply the bad block list */
3532 swap_map = vzalloc(maxpages);
3533 if (!swap_map) {
3534 error = -ENOMEM;
3535 goto bad_swap_unlock_inode;
3536 }
3537
3538 error = swap_cgroup_swapon(type: si->type, max_pages: maxpages);
3539 if (error)
3540 goto bad_swap_unlock_inode;
3541
3542 error = setup_swap_map(si, swap_header, swap_map, maxpages);
3543 if (error)
3544 goto bad_swap_unlock_inode;
3545
3546 /*
3547 * Use kvmalloc_array instead of bitmap_zalloc as the allocation order might
3548 * be above MAX_PAGE_ORDER incase of a large swap file.
3549 */
3550 zeromap = kvmalloc_array(BITS_TO_LONGS(maxpages), sizeof(long),
3551 GFP_KERNEL | __GFP_ZERO);
3552 if (!zeromap) {
3553 error = -ENOMEM;
3554 goto bad_swap_unlock_inode;
3555 }
3556
3557 if (si->bdev && bdev_stable_writes(bdev: si->bdev))
3558 si->flags |= SWP_STABLE_WRITES;
3559
3560 if (si->bdev && bdev_synchronous(bdev: si->bdev))
3561 si->flags |= SWP_SYNCHRONOUS_IO;
3562
3563 if (si->bdev && bdev_nonrot(bdev: si->bdev)) {
3564 si->flags |= SWP_SOLIDSTATE;
3565 } else {
3566 atomic_inc(v: &nr_rotate_swap);
3567 inced_nr_rotate_swap = true;
3568 }
3569
3570 cluster_info = setup_clusters(si, swap_header, maxpages);
3571 if (IS_ERR(ptr: cluster_info)) {
3572 error = PTR_ERR(ptr: cluster_info);
3573 cluster_info = NULL;
3574 goto bad_swap_unlock_inode;
3575 }
3576
3577 if ((swap_flags & SWAP_FLAG_DISCARD) &&
3578 si->bdev && bdev_max_discard_sectors(bdev: si->bdev)) {
3579 /*
3580 * When discard is enabled for swap with no particular
3581 * policy flagged, we set all swap discard flags here in
3582 * order to sustain backward compatibility with older
3583 * swapon(8) releases.
3584 */
3585 si->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
3586 SWP_PAGE_DISCARD);
3587
3588 /*
3589 * By flagging sys_swapon, a sysadmin can tell us to
3590 * either do single-time area discards only, or to just
3591 * perform discards for released swap page-clusters.
3592 * Now it's time to adjust the p->flags accordingly.
3593 */
3594 if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
3595 si->flags &= ~SWP_PAGE_DISCARD;
3596 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
3597 si->flags &= ~SWP_AREA_DISCARD;
3598
3599 /* issue a swapon-time discard if it's still required */
3600 if (si->flags & SWP_AREA_DISCARD) {
3601 int err = discard_swap(si);
3602 if (unlikely(err))
3603 pr_err("swapon: discard_swap(%p): %d\n",
3604 si, err);
3605 }
3606 }
3607
3608 error = zswap_swapon(type: si->type, nr_pages: maxpages);
3609 if (error)
3610 goto bad_swap_unlock_inode;
3611
3612 /*
3613 * Flush any pending IO and dirty mappings before we start using this
3614 * swap device.
3615 */
3616 inode->i_flags |= S_SWAPFILE;
3617 error = inode_drain_writes(inode);
3618 if (error) {
3619 inode->i_flags &= ~S_SWAPFILE;
3620 goto free_swap_zswap;
3621 }
3622
3623 mutex_lock(lock: &swapon_mutex);
3624 prio = -1;
3625 if (swap_flags & SWAP_FLAG_PREFER)
3626 prio = swap_flags & SWAP_FLAG_PRIO_MASK;
3627 enable_swap_info(si, prio, swap_map, cluster_info, zeromap);
3628
3629 pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s\n",
3630 K(si->pages), name->name, si->prio, nr_extents,
3631 K((unsigned long long)span),
3632 (si->flags & SWP_SOLIDSTATE) ? "SS" : "",
3633 (si->flags & SWP_DISCARDABLE) ? "D" : "",
3634 (si->flags & SWP_AREA_DISCARD) ? "s" : "",
3635 (si->flags & SWP_PAGE_DISCARD) ? "c" : "");
3636
3637 mutex_unlock(lock: &swapon_mutex);
3638 atomic_inc(v: &proc_poll_event);
3639 wake_up_interruptible(&proc_poll_wait);
3640
3641 error = 0;
3642 goto out;
3643free_swap_zswap:
3644 zswap_swapoff(type: si->type);
3645bad_swap_unlock_inode:
3646 inode_unlock(inode);
3647bad_swap:
3648 kfree(objp: si->global_cluster);
3649 si->global_cluster = NULL;
3650 inode = NULL;
3651 destroy_swap_extents(sis: si);
3652 swap_cgroup_swapoff(type: si->type);
3653 spin_lock(lock: &swap_lock);
3654 si->swap_file = NULL;
3655 si->flags = 0;
3656 spin_unlock(lock: &swap_lock);
3657 vfree(addr: swap_map);
3658 kvfree(addr: zeromap);
3659 if (cluster_info)
3660 free_cluster_info(cluster_info, maxpages);
3661 if (inced_nr_rotate_swap)
3662 atomic_dec(v: &nr_rotate_swap);
3663 if (swap_file)
3664 filp_close(swap_file, NULL);
3665out:
3666 if (!IS_ERR_OR_NULL(ptr: folio))
3667 folio_release_kmap(folio, addr: swap_header);
3668 if (name)
3669 putname(name);
3670 if (inode)
3671 inode_unlock(inode);
3672 return error;
3673}
3674
3675void si_swapinfo(struct sysinfo *val)
3676{
3677 unsigned int type;
3678 unsigned long nr_to_be_unused = 0;
3679
3680 spin_lock(lock: &swap_lock);
3681 for (type = 0; type < nr_swapfiles; type++) {
3682 struct swap_info_struct *si = swap_info[type];
3683
3684 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
3685 nr_to_be_unused += swap_usage_in_pages(si);
3686 }
3687 val->freeswap = atomic_long_read(v: &nr_swap_pages) + nr_to_be_unused;
3688 val->totalswap = total_swap_pages + nr_to_be_unused;
3689 spin_unlock(lock: &swap_lock);
3690}
3691
3692/*
3693 * Verify that nr swap entries are valid and increment their swap map counts.
3694 *
3695 * Returns error code in following case.
3696 * - success -> 0
3697 * - swp_entry is invalid -> EINVAL
3698 * - swap-cache reference is requested but there is already one. -> EEXIST
3699 * - swap-cache reference is requested but the entry is not used. -> ENOENT
3700 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
3701 */
3702static int __swap_duplicate(swp_entry_t entry, unsigned char usage, int nr)
3703{
3704 struct swap_info_struct *si;
3705 struct swap_cluster_info *ci;
3706 unsigned long offset;
3707 unsigned char count;
3708 unsigned char has_cache;
3709 int err, i;
3710
3711 si = swap_entry_to_info(entry);
3712 if (WARN_ON_ONCE(!si)) {
3713 pr_err("%s%08lx\n", Bad_file, entry.val);
3714 return -EINVAL;
3715 }
3716
3717 offset = swp_offset(entry);
3718 VM_WARN_ON(nr > SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER);
3719 VM_WARN_ON(usage == 1 && nr > 1);
3720 ci = swap_cluster_lock(si, offset);
3721
3722 err = 0;
3723 for (i = 0; i < nr; i++) {
3724 count = si->swap_map[offset + i];
3725
3726 /*
3727 * swapin_readahead() doesn't check if a swap entry is valid, so the
3728 * swap entry could be SWAP_MAP_BAD. Check here with lock held.
3729 */
3730 if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
3731 err = -ENOENT;
3732 goto unlock_out;
3733 }
3734
3735 has_cache = count & SWAP_HAS_CACHE;
3736 count &= ~SWAP_HAS_CACHE;
3737
3738 if (!count && !has_cache) {
3739 err = -ENOENT;
3740 } else if (usage == SWAP_HAS_CACHE) {
3741 if (has_cache)
3742 err = -EEXIST;
3743 } else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) {
3744 err = -EINVAL;
3745 }
3746
3747 if (err)
3748 goto unlock_out;
3749 }
3750
3751 for (i = 0; i < nr; i++) {
3752 count = si->swap_map[offset + i];
3753 has_cache = count & SWAP_HAS_CACHE;
3754 count &= ~SWAP_HAS_CACHE;
3755
3756 if (usage == SWAP_HAS_CACHE)
3757 has_cache = SWAP_HAS_CACHE;
3758 else if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
3759 count += usage;
3760 else if (swap_count_continued(si, offset + i, count))
3761 count = COUNT_CONTINUED;
3762 else {
3763 /*
3764 * Don't need to rollback changes, because if
3765 * usage == 1, there must be nr == 1.
3766 */
3767 err = -ENOMEM;
3768 goto unlock_out;
3769 }
3770
3771 WRITE_ONCE(si->swap_map[offset + i], count | has_cache);
3772 }
3773
3774unlock_out:
3775 swap_cluster_unlock(ci);
3776 return err;
3777}
3778
3779/*
3780 * Help swapoff by noting that swap entry belongs to shmem/tmpfs
3781 * (in which case its reference count is never incremented).
3782 */
3783void swap_shmem_alloc(swp_entry_t entry, int nr)
3784{
3785 __swap_duplicate(entry, SWAP_MAP_SHMEM, nr);
3786}
3787
3788/*
3789 * Increase reference count of swap entry by 1.
3790 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
3791 * but could not be atomically allocated. Returns 0, just as if it succeeded,
3792 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
3793 * might occur if a page table entry has got corrupted.
3794 */
3795int swap_duplicate(swp_entry_t entry)
3796{
3797 int err = 0;
3798
3799 while (!err && __swap_duplicate(entry, usage: 1, nr: 1) == -ENOMEM)
3800 err = add_swap_count_continuation(entry, GFP_ATOMIC);
3801 return err;
3802}
3803
3804/*
3805 * @entry: first swap entry from which we allocate nr swap cache.
3806 *
3807 * Called when allocating swap cache for existing swap entries,
3808 * This can return error codes. Returns 0 at success.
3809 * -EEXIST means there is a swap cache.
3810 * Note: return code is different from swap_duplicate().
3811 */
3812int swapcache_prepare(swp_entry_t entry, int nr)
3813{
3814 return __swap_duplicate(entry, SWAP_HAS_CACHE, nr);
3815}
3816
3817/*
3818 * Caller should ensure entries belong to the same folio so
3819 * the entries won't span cross cluster boundary.
3820 */
3821void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr)
3822{
3823 swap_entries_put_cache(si, entry, nr);
3824}
3825
3826/*
3827 * add_swap_count_continuation - called when a swap count is duplicated
3828 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
3829 * page of the original vmalloc'ed swap_map, to hold the continuation count
3830 * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called
3831 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
3832 *
3833 * These continuation pages are seldom referenced: the common paths all work
3834 * on the original swap_map, only referring to a continuation page when the
3835 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
3836 *
3837 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
3838 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
3839 * can be called after dropping locks.
3840 */
3841int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
3842{
3843 struct swap_info_struct *si;
3844 struct swap_cluster_info *ci;
3845 struct page *head;
3846 struct page *page;
3847 struct page *list_page;
3848 pgoff_t offset;
3849 unsigned char count;
3850 int ret = 0;
3851
3852 /*
3853 * When debugging, it's easier to use __GFP_ZERO here; but it's better
3854 * for latency not to zero a page while GFP_ATOMIC and holding locks.
3855 */
3856 page = alloc_page(gfp_mask | __GFP_HIGHMEM);
3857
3858 si = get_swap_device(entry);
3859 if (!si) {
3860 /*
3861 * An acceptable race has occurred since the failing
3862 * __swap_duplicate(): the swap device may be swapoff
3863 */
3864 goto outer;
3865 }
3866
3867 offset = swp_offset(entry);
3868
3869 ci = swap_cluster_lock(si, offset);
3870
3871 count = swap_count(ent: si->swap_map[offset]);
3872
3873 if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
3874 /*
3875 * The higher the swap count, the more likely it is that tasks
3876 * will race to add swap count continuation: we need to avoid
3877 * over-provisioning.
3878 */
3879 goto out;
3880 }
3881
3882 if (!page) {
3883 ret = -ENOMEM;
3884 goto out;
3885 }
3886
3887 head = vmalloc_to_page(addr: si->swap_map + offset);
3888 offset &= ~PAGE_MASK;
3889
3890 spin_lock(lock: &si->cont_lock);
3891 /*
3892 * Page allocation does not initialize the page's lru field,
3893 * but it does always reset its private field.
3894 */
3895 if (!page_private(head)) {
3896 BUG_ON(count & COUNT_CONTINUED);
3897 INIT_LIST_HEAD(list: &head->lru);
3898 set_page_private(page: head, private: SWP_CONTINUED);
3899 si->flags |= SWP_CONTINUED;
3900 }
3901
3902 list_for_each_entry(list_page, &head->lru, lru) {
3903 unsigned char *map;
3904
3905 /*
3906 * If the previous map said no continuation, but we've found
3907 * a continuation page, free our allocation and use this one.
3908 */
3909 if (!(count & COUNT_CONTINUED))
3910 goto out_unlock_cont;
3911
3912 map = kmap_local_page(page: list_page) + offset;
3913 count = *map;
3914 kunmap_local(map);
3915
3916 /*
3917 * If this continuation count now has some space in it,
3918 * free our allocation and use this one.
3919 */
3920 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
3921 goto out_unlock_cont;
3922 }
3923
3924 list_add_tail(new: &page->lru, head: &head->lru);
3925 page = NULL; /* now it's attached, don't free it */
3926out_unlock_cont:
3927 spin_unlock(lock: &si->cont_lock);
3928out:
3929 swap_cluster_unlock(ci);
3930 put_swap_device(si);
3931outer:
3932 if (page)
3933 __free_page(page);
3934 return ret;
3935}
3936
3937/*
3938 * swap_count_continued - when the original swap_map count is incremented
3939 * from SWAP_MAP_MAX, check if there is already a continuation page to carry
3940 * into, carry if so, or else fail until a new continuation page is allocated;
3941 * when the original swap_map count is decremented from 0 with continuation,
3942 * borrow from the continuation and report whether it still holds more.
3943 * Called while __swap_duplicate() or caller of swap_entry_put_locked()
3944 * holds cluster lock.
3945 */
3946static bool swap_count_continued(struct swap_info_struct *si,
3947 pgoff_t offset, unsigned char count)
3948{
3949 struct page *head;
3950 struct page *page;
3951 unsigned char *map;
3952 bool ret;
3953
3954 head = vmalloc_to_page(addr: si->swap_map + offset);
3955 if (page_private(head) != SWP_CONTINUED) {
3956 BUG_ON(count & COUNT_CONTINUED);
3957 return false; /* need to add count continuation */
3958 }
3959
3960 spin_lock(lock: &si->cont_lock);
3961 offset &= ~PAGE_MASK;
3962 page = list_next_entry(head, lru);
3963 map = kmap_local_page(page) + offset;
3964
3965 if (count == SWAP_MAP_MAX) /* initial increment from swap_map */
3966 goto init_map; /* jump over SWAP_CONT_MAX checks */
3967
3968 if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
3969 /*
3970 * Think of how you add 1 to 999
3971 */
3972 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
3973 kunmap_local(map);
3974 page = list_next_entry(page, lru);
3975 BUG_ON(page == head);
3976 map = kmap_local_page(page) + offset;
3977 }
3978 if (*map == SWAP_CONT_MAX) {
3979 kunmap_local(map);
3980 page = list_next_entry(page, lru);
3981 if (page == head) {
3982 ret = false; /* add count continuation */
3983 goto out;
3984 }
3985 map = kmap_local_page(page) + offset;
3986init_map: *map = 0; /* we didn't zero the page */
3987 }
3988 *map += 1;
3989 kunmap_local(map);
3990 while ((page = list_prev_entry(page, lru)) != head) {
3991 map = kmap_local_page(page) + offset;
3992 *map = COUNT_CONTINUED;
3993 kunmap_local(map);
3994 }
3995 ret = true; /* incremented */
3996
3997 } else { /* decrementing */
3998 /*
3999 * Think of how you subtract 1 from 1000
4000 */
4001 BUG_ON(count != COUNT_CONTINUED);
4002 while (*map == COUNT_CONTINUED) {
4003 kunmap_local(map);
4004 page = list_next_entry(page, lru);
4005 BUG_ON(page == head);
4006 map = kmap_local_page(page) + offset;
4007 }
4008 BUG_ON(*map == 0);
4009 *map -= 1;
4010 if (*map == 0)
4011 count = 0;
4012 kunmap_local(map);
4013 while ((page = list_prev_entry(page, lru)) != head) {
4014 map = kmap_local_page(page) + offset;
4015 *map = SWAP_CONT_MAX | count;
4016 count = COUNT_CONTINUED;
4017 kunmap_local(map);
4018 }
4019 ret = count == COUNT_CONTINUED;
4020 }
4021out:
4022 spin_unlock(lock: &si->cont_lock);
4023 return ret;
4024}
4025
4026/*
4027 * free_swap_count_continuations - swapoff free all the continuation pages
4028 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
4029 */
4030static void free_swap_count_continuations(struct swap_info_struct *si)
4031{
4032 pgoff_t offset;
4033
4034 for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
4035 struct page *head;
4036 head = vmalloc_to_page(addr: si->swap_map + offset);
4037 if (page_private(head)) {
4038 struct page *page, *next;
4039
4040 list_for_each_entry_safe(page, next, &head->lru, lru) {
4041 list_del(entry: &page->lru);
4042 __free_page(page);
4043 }
4044 }
4045 }
4046}
4047
4048#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
4049static bool __has_usable_swap(void)
4050{
4051 return !plist_head_empty(&swap_active_head);
4052}
4053
4054void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
4055{
4056 struct swap_info_struct *si, *next;
4057 int nid = folio_nid(folio);
4058
4059 if (!(gfp & __GFP_IO))
4060 return;
4061
4062 if (!__has_usable_swap())
4063 return;
4064
4065 if (!blk_cgroup_congested())
4066 return;
4067
4068 /*
4069 * We've already scheduled a throttle, avoid taking the global swap
4070 * lock.
4071 */
4072 if (current->throttle_disk)
4073 return;
4074
4075 spin_lock(&swap_avail_lock);
4076 plist_for_each_entry_safe(si, next, &swap_avail_heads[nid],
4077 avail_lists[nid]) {
4078 if (si->bdev) {
4079 blkcg_schedule_throttle(si->bdev->bd_disk, true);
4080 break;
4081 }
4082 }
4083 spin_unlock(&swap_avail_lock);
4084}
4085#endif
4086
4087static int __init swapfile_init(void)
4088{
4089 int nid;
4090
4091 swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head),
4092 GFP_KERNEL);
4093 if (!swap_avail_heads) {
4094 pr_emerg("Not enough memory for swap heads, swap is disabled\n");
4095 return -ENOMEM;
4096 }
4097
4098 for_each_node(nid)
4099 plist_head_init(head: &swap_avail_heads[nid]);
4100
4101 swapfile_maximum_size = arch_max_swapfile_size();
4102
4103 /*
4104 * Once a cluster is freed, it's swap table content is read
4105 * only, and all swap cache readers (swap_cache_*) verifies
4106 * the content before use. So it's safe to use RCU slab here.
4107 */
4108 if (!SWP_TABLE_USE_PAGE)
4109 swap_table_cachep = kmem_cache_create("swap_table",
4110 sizeof(struct swap_table),
4111 0, SLAB_PANIC | SLAB_TYPESAFE_BY_RCU, NULL);
4112
4113#ifdef CONFIG_MIGRATION
4114 if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
4115 swap_migration_ad_supported = true;
4116#endif /* CONFIG_MIGRATION */
4117
4118 return 0;
4119}
4120subsys_initcall(swapfile_init);
4121