1// SPDX-License-Identifier: GPL-2.0
2/*
3 * linux/mm/madvise.c
4 *
5 * Copyright (C) 1999 Linus Torvalds
6 * Copyright (C) 2002 Christoph Hellwig
7 */
8
9#include <linux/mman.h>
10#include <linux/pagemap.h>
11#include <linux/syscalls.h>
12#include <linux/mempolicy.h>
13#include <linux/page-isolation.h>
14#include <linux/page_idle.h>
15#include <linux/userfaultfd_k.h>
16#include <linux/hugetlb.h>
17#include <linux/falloc.h>
18#include <linux/fadvise.h>
19#include <linux/sched.h>
20#include <linux/sched/mm.h>
21#include <linux/mm_inline.h>
22#include <linux/mmu_context.h>
23#include <linux/string.h>
24#include <linux/uio.h>
25#include <linux/ksm.h>
26#include <linux/fs.h>
27#include <linux/file.h>
28#include <linux/blkdev.h>
29#include <linux/backing-dev.h>
30#include <linux/pagewalk.h>
31#include <linux/swap.h>
32#include <linux/swapops.h>
33#include <linux/shmem_fs.h>
34#include <linux/mmu_notifier.h>
35
36#include <asm/tlb.h>
37
38#include "internal.h"
39#include "swap.h"
40
41#define __MADV_SET_ANON_VMA_NAME (-1)
42
43/*
44 * Maximum number of attempts we make to install guard pages before we give up
45 * and return -ERESTARTNOINTR to have userspace try again.
46 */
47#define MAX_MADVISE_GUARD_RETRIES 3
48
49struct madvise_walk_private {
50 struct mmu_gather *tlb;
51 bool pageout;
52};
53
54enum madvise_lock_mode {
55 MADVISE_NO_LOCK,
56 MADVISE_MMAP_READ_LOCK,
57 MADVISE_MMAP_WRITE_LOCK,
58 MADVISE_VMA_READ_LOCK,
59};
60
61struct madvise_behavior_range {
62 unsigned long start;
63 unsigned long end;
64};
65
66struct madvise_behavior {
67 struct mm_struct *mm;
68 int behavior;
69 struct mmu_gather *tlb;
70 enum madvise_lock_mode lock_mode;
71 struct anon_vma_name *anon_name;
72
73 /*
74 * The range over which the behaviour is currently being applied. If
75 * traversing multiple VMAs, this is updated for each.
76 */
77 struct madvise_behavior_range range;
78 /* The VMA and VMA preceding it (if applicable) currently targeted. */
79 struct vm_area_struct *prev;
80 struct vm_area_struct *vma;
81 bool lock_dropped;
82};
83
84#ifdef CONFIG_ANON_VMA_NAME
85static int madvise_walk_vmas(struct madvise_behavior *madv_behavior);
86
87struct anon_vma_name *anon_vma_name_alloc(const char *name)
88{
89 struct anon_vma_name *anon_name;
90 size_t count;
91
92 /* Add 1 for NUL terminator at the end of the anon_name->name */
93 count = strlen(name) + 1;
94 anon_name = kmalloc(struct_size(anon_name, name, count), GFP_KERNEL);
95 if (anon_name) {
96 kref_init(&anon_name->kref);
97 memcpy(anon_name->name, name, count);
98 }
99
100 return anon_name;
101}
102
103void anon_vma_name_free(struct kref *kref)
104{
105 struct anon_vma_name *anon_name =
106 container_of(kref, struct anon_vma_name, kref);
107 kfree(anon_name);
108}
109
110struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
111{
112 if (!rwsem_is_locked(&vma->vm_mm->mmap_lock))
113 vma_assert_locked(vma);
114
115 return vma->anon_name;
116}
117
118/* mmap_lock should be write-locked */
119static int replace_anon_vma_name(struct vm_area_struct *vma,
120 struct anon_vma_name *anon_name)
121{
122 struct anon_vma_name *orig_name = anon_vma_name(vma);
123
124 if (!anon_name) {
125 vma->anon_name = NULL;
126 anon_vma_name_put(orig_name);
127 return 0;
128 }
129
130 if (anon_vma_name_eq(orig_name, anon_name))
131 return 0;
132
133 vma->anon_name = anon_vma_name_reuse(anon_name);
134 anon_vma_name_put(orig_name);
135
136 return 0;
137}
138#else /* CONFIG_ANON_VMA_NAME */
139static int replace_anon_vma_name(struct vm_area_struct *vma,
140 struct anon_vma_name *anon_name)
141{
142 if (anon_name)
143 return -EINVAL;
144
145 return 0;
146}
147#endif /* CONFIG_ANON_VMA_NAME */
148/*
149 * Update the vm_flags or anon_name on region of a vma, splitting it or merging
150 * it as necessary. Must be called with mmap_lock held for writing.
151 */
152static int madvise_update_vma(vm_flags_t new_flags,
153 struct madvise_behavior *madv_behavior)
154{
155 struct vm_area_struct *vma = madv_behavior->vma;
156 struct madvise_behavior_range *range = &madv_behavior->range;
157 struct anon_vma_name *anon_name = madv_behavior->anon_name;
158 bool set_new_anon_name = madv_behavior->behavior == __MADV_SET_ANON_VMA_NAME;
159 VMA_ITERATOR(vmi, madv_behavior->mm, range->start);
160
161 if (new_flags == vma->vm_flags && (!set_new_anon_name ||
162 anon_vma_name_eq(anon_name1: anon_vma_name(vma), anon_name2: anon_name)))
163 return 0;
164
165 if (set_new_anon_name)
166 vma = vma_modify_name(vmi: &vmi, prev: madv_behavior->prev, vma,
167 start: range->start, end: range->end, new_name: anon_name);
168 else
169 vma = vma_modify_flags(vmi: &vmi, prev: madv_behavior->prev, vma,
170 start: range->start, end: range->end, vm_flags: new_flags);
171
172 if (IS_ERR(ptr: vma))
173 return PTR_ERR(ptr: vma);
174
175 madv_behavior->vma = vma;
176
177 /* vm_flags is protected by the mmap_lock held in write mode. */
178 vma_start_write(vma);
179 vm_flags_reset(vma, flags: new_flags);
180 if (set_new_anon_name)
181 return replace_anon_vma_name(vma, anon_name);
182
183 return 0;
184}
185
186#ifdef CONFIG_SWAP
187static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
188 unsigned long end, struct mm_walk *walk)
189{
190 struct vm_area_struct *vma = walk->private;
191 struct swap_iocb *splug = NULL;
192 pte_t *ptep = NULL;
193 spinlock_t *ptl;
194 unsigned long addr;
195
196 for (addr = start; addr < end; addr += PAGE_SIZE) {
197 pte_t pte;
198 swp_entry_t entry;
199 struct folio *folio;
200
201 if (!ptep++) {
202 ptep = pte_offset_map_lock(mm: vma->vm_mm, pmd, addr, ptlp: &ptl);
203 if (!ptep)
204 break;
205 }
206
207 pte = ptep_get(ptep);
208 if (!is_swap_pte(pte))
209 continue;
210 entry = pte_to_swp_entry(pte);
211 if (unlikely(non_swap_entry(entry)))
212 continue;
213
214 pte_unmap_unlock(ptep, ptl);
215 ptep = NULL;
216
217 folio = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
218 vma, addr, plug: &splug);
219 if (folio)
220 folio_put(folio);
221 }
222
223 if (ptep)
224 pte_unmap_unlock(ptep, ptl);
225 swap_read_unplug(plug: splug);
226 cond_resched();
227
228 return 0;
229}
230
231static const struct mm_walk_ops swapin_walk_ops = {
232 .pmd_entry = swapin_walk_pmd_entry,
233 .walk_lock = PGWALK_RDLOCK,
234};
235
236static void shmem_swapin_range(struct vm_area_struct *vma,
237 unsigned long start, unsigned long end,
238 struct address_space *mapping)
239{
240 XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start));
241 pgoff_t end_index = linear_page_index(vma, address: end) - 1;
242 struct folio *folio;
243 struct swap_iocb *splug = NULL;
244
245 rcu_read_lock();
246 xas_for_each(&xas, folio, end_index) {
247 unsigned long addr;
248 swp_entry_t entry;
249
250 if (!xa_is_value(entry: folio))
251 continue;
252 entry = radix_to_swp_entry(arg: folio);
253 /* There might be swapin error entries in shmem mapping. */
254 if (non_swap_entry(entry))
255 continue;
256
257 addr = vma->vm_start +
258 ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT);
259 xas_pause(&xas);
260 rcu_read_unlock();
261
262 folio = read_swap_cache_async(entry, gfp_mask: mapping_gfp_mask(mapping),
263 vma, addr, plug: &splug);
264 if (folio)
265 folio_put(folio);
266
267 rcu_read_lock();
268 }
269 rcu_read_unlock();
270 swap_read_unplug(plug: splug);
271}
272#endif /* CONFIG_SWAP */
273
274static void mark_mmap_lock_dropped(struct madvise_behavior *madv_behavior)
275{
276 VM_WARN_ON_ONCE(madv_behavior->lock_mode == MADVISE_VMA_READ_LOCK);
277 madv_behavior->lock_dropped = true;
278}
279
280/*
281 * Schedule all required I/O operations. Do not wait for completion.
282 */
283static long madvise_willneed(struct madvise_behavior *madv_behavior)
284{
285 struct vm_area_struct *vma = madv_behavior->vma;
286 struct mm_struct *mm = madv_behavior->mm;
287 struct file *file = vma->vm_file;
288 unsigned long start = madv_behavior->range.start;
289 unsigned long end = madv_behavior->range.end;
290 loff_t offset;
291
292#ifdef CONFIG_SWAP
293 if (!file) {
294 walk_page_range_vma(vma, start, end, ops: &swapin_walk_ops, private: vma);
295 lru_add_drain(); /* Push any new pages onto the LRU now */
296 return 0;
297 }
298
299 if (shmem_mapping(mapping: file->f_mapping)) {
300 shmem_swapin_range(vma, start, end, mapping: file->f_mapping);
301 lru_add_drain(); /* Push any new pages onto the LRU now */
302 return 0;
303 }
304#else
305 if (!file)
306 return -EBADF;
307#endif
308
309 if (IS_DAX(file_inode(file))) {
310 /* no bad return value, but ignore advice */
311 return 0;
312 }
313
314 /*
315 * Filesystem's fadvise may need to take various locks. We need to
316 * explicitly grab a reference because the vma (and hence the
317 * vma's reference to the file) can go away as soon as we drop
318 * mmap_lock.
319 */
320 mark_mmap_lock_dropped(madv_behavior);
321 get_file(f: file);
322 offset = (loff_t)(start - vma->vm_start)
323 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
324 mmap_read_unlock(mm);
325 vfs_fadvise(file, offset, len: end - start, POSIX_FADV_WILLNEED);
326 fput(file);
327 mmap_read_lock(mm);
328 return 0;
329}
330
331static inline bool can_do_file_pageout(struct vm_area_struct *vma)
332{
333 if (!vma->vm_file)
334 return false;
335 /*
336 * paging out pagecache only for non-anonymous mappings that correspond
337 * to the files the calling process could (if tried) open for writing;
338 * otherwise we'd be including shared non-exclusive mappings, which
339 * opens a side channel.
340 */
341 return inode_owner_or_capable(idmap: &nop_mnt_idmap,
342 inode: file_inode(f: vma->vm_file)) ||
343 file_permission(file: vma->vm_file, MAY_WRITE) == 0;
344}
345
346static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end,
347 struct folio *folio, pte_t *ptep,
348 pte_t *ptentp)
349{
350 int max_nr = (end - addr) / PAGE_SIZE;
351
352 return folio_pte_batch_flags(folio, NULL, ptep, ptentp, max_nr,
353 FPB_MERGE_YOUNG_DIRTY);
354}
355
356static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
357 unsigned long addr, unsigned long end,
358 struct mm_walk *walk)
359{
360 struct madvise_walk_private *private = walk->private;
361 struct mmu_gather *tlb = private->tlb;
362 bool pageout = private->pageout;
363 struct mm_struct *mm = tlb->mm;
364 struct vm_area_struct *vma = walk->vma;
365 pte_t *start_pte, *pte, ptent;
366 spinlock_t *ptl;
367 struct folio *folio = NULL;
368 LIST_HEAD(folio_list);
369 bool pageout_anon_only_filter;
370 unsigned int batch_count = 0;
371 int nr;
372
373 if (fatal_signal_pending(current))
374 return -EINTR;
375
376 pageout_anon_only_filter = pageout && !vma_is_anonymous(vma) &&
377 !can_do_file_pageout(vma);
378
379#ifdef CONFIG_TRANSPARENT_HUGEPAGE
380 if (pmd_trans_huge(*pmd)) {
381 pmd_t orig_pmd;
382 unsigned long next = pmd_addr_end(addr, end);
383
384 tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
385 ptl = pmd_trans_huge_lock(pmd, vma);
386 if (!ptl)
387 return 0;
388
389 orig_pmd = *pmd;
390 if (is_huge_zero_pmd(orig_pmd))
391 goto huge_unlock;
392
393 if (unlikely(!pmd_present(orig_pmd))) {
394 VM_BUG_ON(thp_migration_supported() &&
395 !is_pmd_migration_entry(orig_pmd));
396 goto huge_unlock;
397 }
398
399 folio = pmd_folio(orig_pmd);
400
401 /* Do not interfere with other mappings of this folio */
402 if (folio_maybe_mapped_shared(folio))
403 goto huge_unlock;
404
405 if (pageout_anon_only_filter && !folio_test_anon(folio))
406 goto huge_unlock;
407
408 if (next - addr != HPAGE_PMD_SIZE) {
409 int err;
410
411 folio_get(folio);
412 spin_unlock(ptl);
413 folio_lock(folio);
414 err = split_folio(folio);
415 folio_unlock(folio);
416 folio_put(folio);
417 if (!err)
418 goto regular_folio;
419 return 0;
420 }
421
422 if (!pageout && pmd_young(orig_pmd)) {
423 pmdp_invalidate(vma, addr, pmd);
424 orig_pmd = pmd_mkold(orig_pmd);
425
426 set_pmd_at(mm, addr, pmd, orig_pmd);
427 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
428 }
429
430 folio_clear_referenced(folio);
431 folio_test_clear_young(folio);
432 if (folio_test_active(folio))
433 folio_set_workingset(folio);
434 if (pageout) {
435 if (folio_isolate_lru(folio)) {
436 if (folio_test_unevictable(folio))
437 folio_putback_lru(folio);
438 else
439 list_add(&folio->lru, &folio_list);
440 }
441 } else
442 folio_deactivate(folio);
443huge_unlock:
444 spin_unlock(ptl);
445 if (pageout)
446 reclaim_pages(&folio_list);
447 return 0;
448 }
449
450regular_folio:
451#endif
452 tlb_change_page_size(tlb, PAGE_SIZE);
453restart:
454 start_pte = pte = pte_offset_map_lock(mm: vma->vm_mm, pmd, addr, ptlp: &ptl);
455 if (!start_pte)
456 return 0;
457 flush_tlb_batched_pending(mm);
458 arch_enter_lazy_mmu_mode();
459 for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) {
460 nr = 1;
461 ptent = ptep_get(ptep: pte);
462
463 if (++batch_count == SWAP_CLUSTER_MAX) {
464 batch_count = 0;
465 if (need_resched()) {
466 arch_leave_lazy_mmu_mode();
467 pte_unmap_unlock(start_pte, ptl);
468 cond_resched();
469 goto restart;
470 }
471 }
472
473 if (pte_none(pte: ptent))
474 continue;
475
476 if (!pte_present(a: ptent))
477 continue;
478
479 folio = vm_normal_folio(vma, addr, pte: ptent);
480 if (!folio || folio_is_zone_device(folio))
481 continue;
482
483 /*
484 * If we encounter a large folio, only split it if it is not
485 * fully mapped within the range we are operating on. Otherwise
486 * leave it as is so that it can be swapped out whole. If we
487 * fail to split a folio, leave it in place and advance to the
488 * next pte in the range.
489 */
490 if (folio_test_large(folio)) {
491 nr = madvise_folio_pte_batch(addr, end, folio, ptep: pte, ptentp: &ptent);
492 if (nr < folio_nr_pages(folio)) {
493 int err;
494
495 if (folio_maybe_mapped_shared(folio))
496 continue;
497 if (pageout_anon_only_filter && !folio_test_anon(folio))
498 continue;
499 if (!folio_trylock(folio))
500 continue;
501 folio_get(folio);
502 arch_leave_lazy_mmu_mode();
503 pte_unmap_unlock(start_pte, ptl);
504 start_pte = NULL;
505 err = split_folio(folio);
506 folio_unlock(folio);
507 folio_put(folio);
508 start_pte = pte =
509 pte_offset_map_lock(mm, pmd, addr, ptlp: &ptl);
510 if (!start_pte)
511 break;
512 flush_tlb_batched_pending(mm);
513 arch_enter_lazy_mmu_mode();
514 if (!err)
515 nr = 0;
516 continue;
517 }
518 }
519
520 /*
521 * Do not interfere with other mappings of this folio and
522 * non-LRU folio. If we have a large folio at this point, we
523 * know it is fully mapped so if its mapcount is the same as its
524 * number of pages, it must be exclusive.
525 */
526 if (!folio_test_lru(folio) ||
527 folio_mapcount(folio) != folio_nr_pages(folio))
528 continue;
529
530 if (pageout_anon_only_filter && !folio_test_anon(folio))
531 continue;
532
533 if (!pageout && pte_young(pte: ptent)) {
534 clear_young_dirty_ptes(vma, addr, ptep: pte, nr,
535 CYDP_CLEAR_YOUNG);
536 tlb_remove_tlb_entries(tlb, ptep: pte, nr, address: addr);
537 }
538
539 /*
540 * We are deactivating a folio for accelerating reclaiming.
541 * VM couldn't reclaim the folio unless we clear PG_young.
542 * As a side effect, it makes confuse idle-page tracking
543 * because they will miss recent referenced history.
544 */
545 folio_clear_referenced(folio);
546 folio_test_clear_young(folio);
547 if (folio_test_active(folio))
548 folio_set_workingset(folio);
549 if (pageout) {
550 if (folio_isolate_lru(folio)) {
551 if (folio_test_unevictable(folio))
552 folio_putback_lru(folio);
553 else
554 list_add(new: &folio->lru, head: &folio_list);
555 }
556 } else
557 folio_deactivate(folio);
558 }
559
560 if (start_pte) {
561 arch_leave_lazy_mmu_mode();
562 pte_unmap_unlock(start_pte, ptl);
563 }
564 if (pageout)
565 reclaim_pages(folio_list: &folio_list);
566 cond_resched();
567
568 return 0;
569}
570
571static const struct mm_walk_ops cold_walk_ops = {
572 .pmd_entry = madvise_cold_or_pageout_pte_range,
573 .walk_lock = PGWALK_RDLOCK,
574};
575
576static void madvise_cold_page_range(struct mmu_gather *tlb,
577 struct madvise_behavior *madv_behavior)
578
579{
580 struct vm_area_struct *vma = madv_behavior->vma;
581 struct madvise_behavior_range *range = &madv_behavior->range;
582 struct madvise_walk_private walk_private = {
583 .pageout = false,
584 .tlb = tlb,
585 };
586
587 tlb_start_vma(tlb, vma);
588 walk_page_range_vma(vma, start: range->start, end: range->end, ops: &cold_walk_ops,
589 private: &walk_private);
590 tlb_end_vma(tlb, vma);
591}
592
593static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
594{
595 return !(vma->vm_flags & (VM_LOCKED|VM_PFNMAP|VM_HUGETLB));
596}
597
598static long madvise_cold(struct madvise_behavior *madv_behavior)
599{
600 struct vm_area_struct *vma = madv_behavior->vma;
601 struct mmu_gather tlb;
602
603 if (!can_madv_lru_vma(vma))
604 return -EINVAL;
605
606 lru_add_drain();
607 tlb_gather_mmu(tlb: &tlb, mm: madv_behavior->mm);
608 madvise_cold_page_range(tlb: &tlb, madv_behavior);
609 tlb_finish_mmu(tlb: &tlb);
610
611 return 0;
612}
613
614static void madvise_pageout_page_range(struct mmu_gather *tlb,
615 struct vm_area_struct *vma,
616 struct madvise_behavior_range *range)
617{
618 struct madvise_walk_private walk_private = {
619 .pageout = true,
620 .tlb = tlb,
621 };
622
623 tlb_start_vma(tlb, vma);
624 walk_page_range_vma(vma, start: range->start, end: range->end, ops: &cold_walk_ops,
625 private: &walk_private);
626 tlb_end_vma(tlb, vma);
627}
628
629static long madvise_pageout(struct madvise_behavior *madv_behavior)
630{
631 struct mmu_gather tlb;
632 struct vm_area_struct *vma = madv_behavior->vma;
633
634 if (!can_madv_lru_vma(vma))
635 return -EINVAL;
636
637 /*
638 * If the VMA belongs to a private file mapping, there can be private
639 * dirty pages which can be paged out if even this process is neither
640 * owner nor write capable of the file. We allow private file mappings
641 * further to pageout dirty anon pages.
642 */
643 if (!vma_is_anonymous(vma) && (!can_do_file_pageout(vma) &&
644 (vma->vm_flags & VM_MAYSHARE)))
645 return 0;
646
647 lru_add_drain();
648 tlb_gather_mmu(tlb: &tlb, mm: madv_behavior->mm);
649 madvise_pageout_page_range(tlb: &tlb, vma, range: &madv_behavior->range);
650 tlb_finish_mmu(tlb: &tlb);
651
652 return 0;
653}
654
655static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
656 unsigned long end, struct mm_walk *walk)
657
658{
659 const cydp_t cydp_flags = CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY;
660 struct mmu_gather *tlb = walk->private;
661 struct mm_struct *mm = tlb->mm;
662 struct vm_area_struct *vma = walk->vma;
663 spinlock_t *ptl;
664 pte_t *start_pte, *pte, ptent;
665 struct folio *folio;
666 int nr_swap = 0;
667 unsigned long next;
668 int nr, max_nr;
669
670 next = pmd_addr_end(addr, end);
671 if (pmd_trans_huge(pmd: *pmd))
672 if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next))
673 return 0;
674
675 tlb_change_page_size(tlb, PAGE_SIZE);
676 start_pte = pte = pte_offset_map_lock(mm, pmd, addr, ptlp: &ptl);
677 if (!start_pte)
678 return 0;
679 flush_tlb_batched_pending(mm);
680 arch_enter_lazy_mmu_mode();
681 for (; addr != end; pte += nr, addr += PAGE_SIZE * nr) {
682 nr = 1;
683 ptent = ptep_get(ptep: pte);
684
685 if (pte_none(pte: ptent))
686 continue;
687 /*
688 * If the pte has swp_entry, just clear page table to
689 * prevent swap-in which is more expensive rather than
690 * (page allocation + zeroing).
691 */
692 if (!pte_present(a: ptent)) {
693 swp_entry_t entry;
694
695 entry = pte_to_swp_entry(pte: ptent);
696 if (!non_swap_entry(entry)) {
697 max_nr = (end - addr) / PAGE_SIZE;
698 nr = swap_pte_batch(start_ptep: pte, max_nr, pte: ptent);
699 nr_swap -= nr;
700 free_swap_and_cache_nr(entry, nr);
701 clear_not_present_full_ptes(mm, addr, ptep: pte, nr, full: tlb->fullmm);
702 } else if (is_hwpoison_entry(swp: entry) ||
703 is_poisoned_swp_entry(entry)) {
704 pte_clear_not_present_full(mm, address: addr, ptep: pte, full: tlb->fullmm);
705 }
706 continue;
707 }
708
709 folio = vm_normal_folio(vma, addr, pte: ptent);
710 if (!folio || folio_is_zone_device(folio))
711 continue;
712
713 /*
714 * If we encounter a large folio, only split it if it is not
715 * fully mapped within the range we are operating on. Otherwise
716 * leave it as is so that it can be marked as lazyfree. If we
717 * fail to split a folio, leave it in place and advance to the
718 * next pte in the range.
719 */
720 if (folio_test_large(folio)) {
721 nr = madvise_folio_pte_batch(addr, end, folio, ptep: pte, ptentp: &ptent);
722 if (nr < folio_nr_pages(folio)) {
723 int err;
724
725 if (folio_maybe_mapped_shared(folio))
726 continue;
727 if (!folio_trylock(folio))
728 continue;
729 folio_get(folio);
730 arch_leave_lazy_mmu_mode();
731 pte_unmap_unlock(start_pte, ptl);
732 start_pte = NULL;
733 err = split_folio(folio);
734 folio_unlock(folio);
735 folio_put(folio);
736 pte = pte_offset_map_lock(mm, pmd, addr, ptlp: &ptl);
737 start_pte = pte;
738 if (!start_pte)
739 break;
740 flush_tlb_batched_pending(mm);
741 arch_enter_lazy_mmu_mode();
742 if (!err)
743 nr = 0;
744 continue;
745 }
746 }
747
748 if (folio_test_swapcache(folio) || folio_test_dirty(folio)) {
749 if (!folio_trylock(folio))
750 continue;
751 /*
752 * If we have a large folio at this point, we know it is
753 * fully mapped so if its mapcount is the same as its
754 * number of pages, it must be exclusive.
755 */
756 if (folio_mapcount(folio) != folio_nr_pages(folio)) {
757 folio_unlock(folio);
758 continue;
759 }
760
761 if (folio_test_swapcache(folio) &&
762 !folio_free_swap(folio)) {
763 folio_unlock(folio);
764 continue;
765 }
766
767 folio_clear_dirty(folio);
768 folio_unlock(folio);
769 }
770
771 if (pte_young(pte: ptent) || pte_dirty(pte: ptent)) {
772 clear_young_dirty_ptes(vma, addr, ptep: pte, nr, flags: cydp_flags);
773 tlb_remove_tlb_entries(tlb, ptep: pte, nr, address: addr);
774 }
775 folio_mark_lazyfree(folio);
776 }
777
778 if (nr_swap)
779 add_mm_counter(mm, member: MM_SWAPENTS, value: nr_swap);
780 if (start_pte) {
781 arch_leave_lazy_mmu_mode();
782 pte_unmap_unlock(start_pte, ptl);
783 }
784 cond_resched();
785
786 return 0;
787}
788
789static inline enum page_walk_lock get_walk_lock(enum madvise_lock_mode mode)
790{
791 switch (mode) {
792 case MADVISE_VMA_READ_LOCK:
793 return PGWALK_VMA_RDLOCK_VERIFY;
794 case MADVISE_MMAP_READ_LOCK:
795 return PGWALK_RDLOCK;
796 default:
797 /* Other modes don't require fixing up the walk_lock */
798 WARN_ON_ONCE(1);
799 return PGWALK_RDLOCK;
800 }
801}
802
803static int madvise_free_single_vma(struct madvise_behavior *madv_behavior)
804{
805 struct mm_struct *mm = madv_behavior->mm;
806 struct vm_area_struct *vma = madv_behavior->vma;
807 unsigned long start_addr = madv_behavior->range.start;
808 unsigned long end_addr = madv_behavior->range.end;
809 struct mmu_notifier_range range;
810 struct mmu_gather *tlb = madv_behavior->tlb;
811 struct mm_walk_ops walk_ops = {
812 .pmd_entry = madvise_free_pte_range,
813 };
814
815 /* MADV_FREE works for only anon vma at the moment */
816 if (!vma_is_anonymous(vma))
817 return -EINVAL;
818
819 range.start = max(vma->vm_start, start_addr);
820 if (range.start >= vma->vm_end)
821 return -EINVAL;
822 range.end = min(vma->vm_end, end_addr);
823 if (range.end <= vma->vm_start)
824 return -EINVAL;
825 mmu_notifier_range_init(range: &range, event: MMU_NOTIFY_CLEAR, flags: 0, mm,
826 start: range.start, end: range.end);
827
828 lru_add_drain();
829 update_hiwater_rss(mm);
830
831 mmu_notifier_invalidate_range_start(range: &range);
832 tlb_start_vma(tlb, vma);
833 walk_ops.walk_lock = get_walk_lock(mode: madv_behavior->lock_mode);
834 walk_page_range_vma(vma, start: range.start, end: range.end,
835 ops: &walk_ops, private: tlb);
836 tlb_end_vma(tlb, vma);
837 mmu_notifier_invalidate_range_end(range: &range);
838 return 0;
839}
840
841/*
842 * Application no longer needs these pages. If the pages are dirty,
843 * it's OK to just throw them away. The app will be more careful about
844 * data it wants to keep. Be sure to free swap resources too. The
845 * zap_page_range_single call sets things up for shrink_active_list to actually
846 * free these pages later if no one else has touched them in the meantime,
847 * although we could add these pages to a global reuse list for
848 * shrink_active_list to pick up before reclaiming other pages.
849 *
850 * NB: This interface discards data rather than pushes it out to swap,
851 * as some implementations do. This has performance implications for
852 * applications like large transactional databases which want to discard
853 * pages in anonymous maps after committing to backing store the data
854 * that was kept in them. There is no reason to write this data out to
855 * the swap area if the application is discarding it.
856 *
857 * An interface that causes the system to free clean pages and flush
858 * dirty pages is already available as msync(MS_INVALIDATE).
859 */
860static long madvise_dontneed_single_vma(struct madvise_behavior *madv_behavior)
861
862{
863 struct madvise_behavior_range *range = &madv_behavior->range;
864 struct zap_details details = {
865 .reclaim_pt = true,
866 .even_cows = true,
867 };
868
869 zap_page_range_single_batched(
870 tlb: madv_behavior->tlb, vma: madv_behavior->vma, addr: range->start,
871 size: range->end - range->start, details: &details);
872 return 0;
873}
874
875static
876bool madvise_dontneed_free_valid_vma(struct madvise_behavior *madv_behavior)
877{
878 struct vm_area_struct *vma = madv_behavior->vma;
879 int behavior = madv_behavior->behavior;
880 struct madvise_behavior_range *range = &madv_behavior->range;
881
882 if (!is_vm_hugetlb_page(vma)) {
883 unsigned int forbidden = VM_PFNMAP;
884
885 if (behavior != MADV_DONTNEED_LOCKED)
886 forbidden |= VM_LOCKED;
887
888 return !(vma->vm_flags & forbidden);
889 }
890
891 if (behavior != MADV_DONTNEED && behavior != MADV_DONTNEED_LOCKED)
892 return false;
893 if (range->start & ~huge_page_mask(h: hstate_vma(vma)))
894 return false;
895
896 /*
897 * Madvise callers expect the length to be rounded up to PAGE_SIZE
898 * boundaries, and may be unaware that this VMA uses huge pages.
899 * Avoid unexpected data loss by rounding down the number of
900 * huge pages freed.
901 */
902 range->end = ALIGN_DOWN(range->end, huge_page_size(hstate_vma(vma)));
903
904 return true;
905}
906
907static long madvise_dontneed_free(struct madvise_behavior *madv_behavior)
908{
909 struct mm_struct *mm = madv_behavior->mm;
910 struct madvise_behavior_range *range = &madv_behavior->range;
911 int behavior = madv_behavior->behavior;
912
913 if (!madvise_dontneed_free_valid_vma(madv_behavior))
914 return -EINVAL;
915
916 if (range->start == range->end)
917 return 0;
918
919 if (!userfaultfd_remove(vma: madv_behavior->vma, start: range->start, end: range->end)) {
920 struct vm_area_struct *vma;
921
922 mark_mmap_lock_dropped(madv_behavior);
923 mmap_read_lock(mm);
924 madv_behavior->vma = vma = vma_lookup(mm, addr: range->start);
925 if (!vma)
926 return -ENOMEM;
927 /*
928 * Potential end adjustment for hugetlb vma is OK as
929 * the check below keeps end within vma.
930 */
931 if (!madvise_dontneed_free_valid_vma(madv_behavior))
932 return -EINVAL;
933 if (range->end > vma->vm_end) {
934 /*
935 * Don't fail if end > vma->vm_end. If the old
936 * vma was split while the mmap_lock was
937 * released the effect of the concurrent
938 * operation may not cause madvise() to
939 * have an undefined result. There may be an
940 * adjacent next vma that we'll walk
941 * next. userfaultfd_remove() will generate an
942 * UFFD_EVENT_REMOVE repetition on the
943 * end-vma->vm_end range, but the manager can
944 * handle a repetition fine.
945 */
946 range->end = vma->vm_end;
947 }
948 /*
949 * If the memory region between start and end was
950 * originally backed by 4kB pages and then remapped to
951 * be backed by hugepages while mmap_lock was dropped,
952 * the adjustment for hugetlb vma above may have rounded
953 * end down to the start address.
954 */
955 if (range->start == range->end)
956 return 0;
957 VM_WARN_ON(range->start > range->end);
958 }
959
960 if (behavior == MADV_DONTNEED || behavior == MADV_DONTNEED_LOCKED)
961 return madvise_dontneed_single_vma(madv_behavior);
962 else if (behavior == MADV_FREE)
963 return madvise_free_single_vma(madv_behavior);
964 else
965 return -EINVAL;
966}
967
968static long madvise_populate(struct madvise_behavior *madv_behavior)
969{
970 struct mm_struct *mm = madv_behavior->mm;
971 const bool write = madv_behavior->behavior == MADV_POPULATE_WRITE;
972 int locked = 1;
973 unsigned long start = madv_behavior->range.start;
974 unsigned long end = madv_behavior->range.end;
975 long pages;
976
977 while (start < end) {
978 /* Populate (prefault) page tables readable/writable. */
979 pages = faultin_page_range(mm, start, end, write, locked: &locked);
980 if (!locked) {
981 mmap_read_lock(mm);
982 locked = 1;
983 }
984 if (pages < 0) {
985 switch (pages) {
986 case -EINTR:
987 return -EINTR;
988 case -EINVAL: /* Incompatible mappings / permissions. */
989 return -EINVAL;
990 case -EHWPOISON:
991 return -EHWPOISON;
992 case -EFAULT: /* VM_FAULT_SIGBUS or VM_FAULT_SIGSEGV */
993 return -EFAULT;
994 default:
995 pr_warn_once("%s: unhandled return value: %ld\n",
996 __func__, pages);
997 fallthrough;
998 case -ENOMEM: /* No VMA or out of memory. */
999 return -ENOMEM;
1000 }
1001 }
1002 start += pages * PAGE_SIZE;
1003 }
1004 return 0;
1005}
1006
1007/*
1008 * Application wants to free up the pages and associated backing store.
1009 * This is effectively punching a hole into the middle of a file.
1010 */
1011static long madvise_remove(struct madvise_behavior *madv_behavior)
1012{
1013 loff_t offset;
1014 int error;
1015 struct file *f;
1016 struct mm_struct *mm = madv_behavior->mm;
1017 struct vm_area_struct *vma = madv_behavior->vma;
1018 unsigned long start = madv_behavior->range.start;
1019 unsigned long end = madv_behavior->range.end;
1020
1021 mark_mmap_lock_dropped(madv_behavior);
1022
1023 if (vma->vm_flags & VM_LOCKED)
1024 return -EINVAL;
1025
1026 f = vma->vm_file;
1027
1028 if (!f || !f->f_mapping || !f->f_mapping->host) {
1029 return -EINVAL;
1030 }
1031
1032 if (!vma_is_shared_maywrite(vma))
1033 return -EACCES;
1034
1035 offset = (loff_t)(start - vma->vm_start)
1036 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
1037
1038 /*
1039 * Filesystem's fallocate may need to take i_rwsem. We need to
1040 * explicitly grab a reference because the vma (and hence the
1041 * vma's reference to the file) can go away as soon as we drop
1042 * mmap_lock.
1043 */
1044 get_file(f);
1045 if (userfaultfd_remove(vma, start, end)) {
1046 /* mmap_lock was not released by userfaultfd_remove() */
1047 mmap_read_unlock(mm);
1048 }
1049 error = vfs_fallocate(file: f,
1050 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
1051 offset, len: end - start);
1052 fput(f);
1053 mmap_read_lock(mm);
1054 return error;
1055}
1056
1057static bool is_valid_guard_vma(struct vm_area_struct *vma, bool allow_locked)
1058{
1059 vm_flags_t disallowed = VM_SPECIAL | VM_HUGETLB;
1060
1061 /*
1062 * A user could lock after setting a guard range but that's fine, as
1063 * they'd not be able to fault in. The issue arises when we try to zap
1064 * existing locked VMAs. We don't want to do that.
1065 */
1066 if (!allow_locked)
1067 disallowed |= VM_LOCKED;
1068
1069 return !(vma->vm_flags & disallowed);
1070}
1071
1072static bool is_guard_pte_marker(pte_t ptent)
1073{
1074 return is_swap_pte(pte: ptent) &&
1075 is_guard_swp_entry(entry: pte_to_swp_entry(pte: ptent));
1076}
1077
1078static int guard_install_pud_entry(pud_t *pud, unsigned long addr,
1079 unsigned long next, struct mm_walk *walk)
1080{
1081 pud_t pudval = pudp_get(pudp: pud);
1082
1083 /* If huge return >0 so we abort the operation + zap. */
1084 return pud_trans_huge(pud: pudval);
1085}
1086
1087static int guard_install_pmd_entry(pmd_t *pmd, unsigned long addr,
1088 unsigned long next, struct mm_walk *walk)
1089{
1090 pmd_t pmdval = pmdp_get(pmdp: pmd);
1091
1092 /* If huge return >0 so we abort the operation + zap. */
1093 return pmd_trans_huge(pmd: pmdval);
1094}
1095
1096static int guard_install_pte_entry(pte_t *pte, unsigned long addr,
1097 unsigned long next, struct mm_walk *walk)
1098{
1099 pte_t pteval = ptep_get(ptep: pte);
1100 unsigned long *nr_pages = (unsigned long *)walk->private;
1101
1102 /* If there is already a guard page marker, we have nothing to do. */
1103 if (is_guard_pte_marker(ptent: pteval)) {
1104 (*nr_pages)++;
1105
1106 return 0;
1107 }
1108
1109 /* If populated return >0 so we abort the operation + zap. */
1110 return 1;
1111}
1112
1113static int guard_install_set_pte(unsigned long addr, unsigned long next,
1114 pte_t *ptep, struct mm_walk *walk)
1115{
1116 unsigned long *nr_pages = (unsigned long *)walk->private;
1117
1118 /* Simply install a PTE marker, this causes segfault on access. */
1119 *ptep = make_pte_marker(PTE_MARKER_GUARD);
1120 (*nr_pages)++;
1121
1122 return 0;
1123}
1124
1125static const struct mm_walk_ops guard_install_walk_ops = {
1126 .pud_entry = guard_install_pud_entry,
1127 .pmd_entry = guard_install_pmd_entry,
1128 .pte_entry = guard_install_pte_entry,
1129 .install_pte = guard_install_set_pte,
1130 .walk_lock = PGWALK_RDLOCK,
1131};
1132
1133static long madvise_guard_install(struct madvise_behavior *madv_behavior)
1134{
1135 struct vm_area_struct *vma = madv_behavior->vma;
1136 struct madvise_behavior_range *range = &madv_behavior->range;
1137 long err;
1138 int i;
1139
1140 if (!is_valid_guard_vma(vma, /* allow_locked = */false))
1141 return -EINVAL;
1142
1143 /*
1144 * If we install guard markers, then the range is no longer
1145 * empty from a page table perspective and therefore it's
1146 * appropriate to have an anon_vma.
1147 *
1148 * This ensures that on fork, we copy page tables correctly.
1149 */
1150 err = anon_vma_prepare(vma);
1151 if (err)
1152 return err;
1153
1154 /*
1155 * Optimistically try to install the guard marker pages first. If any
1156 * non-guard pages are encountered, give up and zap the range before
1157 * trying again.
1158 *
1159 * We try a few times before giving up and releasing back to userland to
1160 * loop around, releasing locks in the process to avoid contention. This
1161 * would only happen if there was a great many racing page faults.
1162 *
1163 * In most cases we should simply install the guard markers immediately
1164 * with no zap or looping.
1165 */
1166 for (i = 0; i < MAX_MADVISE_GUARD_RETRIES; i++) {
1167 unsigned long nr_pages = 0;
1168
1169 /* Returns < 0 on error, == 0 if success, > 0 if zap needed. */
1170 err = walk_page_range_mm(mm: vma->vm_mm, start: range->start, end: range->end,
1171 ops: &guard_install_walk_ops, private: &nr_pages);
1172 if (err < 0)
1173 return err;
1174
1175 if (err == 0) {
1176 unsigned long nr_expected_pages =
1177 PHYS_PFN(range->end - range->start);
1178
1179 VM_WARN_ON(nr_pages != nr_expected_pages);
1180 return 0;
1181 }
1182
1183 /*
1184 * OK some of the range have non-guard pages mapped, zap
1185 * them. This leaves existing guard pages in place.
1186 */
1187 zap_page_range_single(vma, address: range->start,
1188 size: range->end - range->start, NULL);
1189 }
1190
1191 /*
1192 * We were unable to install the guard pages due to being raced by page
1193 * faults. This should not happen ordinarily. We return to userspace and
1194 * immediately retry, relieving lock contention.
1195 */
1196 return restart_syscall();
1197}
1198
1199static int guard_remove_pud_entry(pud_t *pud, unsigned long addr,
1200 unsigned long next, struct mm_walk *walk)
1201{
1202 pud_t pudval = pudp_get(pudp: pud);
1203
1204 /* If huge, cannot have guard pages present, so no-op - skip. */
1205 if (pud_trans_huge(pud: pudval))
1206 walk->action = ACTION_CONTINUE;
1207
1208 return 0;
1209}
1210
1211static int guard_remove_pmd_entry(pmd_t *pmd, unsigned long addr,
1212 unsigned long next, struct mm_walk *walk)
1213{
1214 pmd_t pmdval = pmdp_get(pmdp: pmd);
1215
1216 /* If huge, cannot have guard pages present, so no-op - skip. */
1217 if (pmd_trans_huge(pmd: pmdval))
1218 walk->action = ACTION_CONTINUE;
1219
1220 return 0;
1221}
1222
1223static int guard_remove_pte_entry(pte_t *pte, unsigned long addr,
1224 unsigned long next, struct mm_walk *walk)
1225{
1226 pte_t ptent = ptep_get(ptep: pte);
1227
1228 if (is_guard_pte_marker(ptent)) {
1229 /* Simply clear the PTE marker. */
1230 pte_clear_not_present_full(mm: walk->mm, address: addr, ptep: pte, full: false);
1231 update_mmu_cache(vma: walk->vma, addr, ptep: pte);
1232 }
1233
1234 return 0;
1235}
1236
1237static const struct mm_walk_ops guard_remove_walk_ops = {
1238 .pud_entry = guard_remove_pud_entry,
1239 .pmd_entry = guard_remove_pmd_entry,
1240 .pte_entry = guard_remove_pte_entry,
1241 .walk_lock = PGWALK_RDLOCK,
1242};
1243
1244static long madvise_guard_remove(struct madvise_behavior *madv_behavior)
1245{
1246 struct vm_area_struct *vma = madv_behavior->vma;
1247 struct madvise_behavior_range *range = &madv_behavior->range;
1248
1249 /*
1250 * We're ok with removing guards in mlock()'d ranges, as this is a
1251 * non-destructive action.
1252 */
1253 if (!is_valid_guard_vma(vma, /* allow_locked = */true))
1254 return -EINVAL;
1255
1256 return walk_page_range_vma(vma, start: range->start, end: range->end,
1257 ops: &guard_remove_walk_ops, NULL);
1258}
1259
1260#ifdef CONFIG_64BIT
1261/* Does the madvise operation result in discarding of mapped data? */
1262static bool is_discard(int behavior)
1263{
1264 switch (behavior) {
1265 case MADV_FREE:
1266 case MADV_DONTNEED:
1267 case MADV_DONTNEED_LOCKED:
1268 case MADV_REMOVE:
1269 case MADV_DONTFORK:
1270 case MADV_WIPEONFORK:
1271 case MADV_GUARD_INSTALL:
1272 return true;
1273 }
1274
1275 return false;
1276}
1277
1278/*
1279 * We are restricted from madvise()'ing mseal()'d VMAs only in very particular
1280 * circumstances - discarding of data from read-only anonymous SEALED mappings.
1281 *
1282 * This is because users cannot trivally discard data from these VMAs, and may
1283 * only do so via an appropriate madvise() call.
1284 */
1285static bool can_madvise_modify(struct madvise_behavior *madv_behavior)
1286{
1287 struct vm_area_struct *vma = madv_behavior->vma;
1288
1289 /* If the VMA isn't sealed we're good. */
1290 if (!vma_is_sealed(vma))
1291 return true;
1292
1293 /* For a sealed VMA, we only care about discard operations. */
1294 if (!is_discard(behavior: madv_behavior->behavior))
1295 return true;
1296
1297 /*
1298 * We explicitly permit all file-backed mappings, whether MAP_SHARED or
1299 * MAP_PRIVATE.
1300 *
1301 * The latter causes some complications. Because now, one can mmap()
1302 * read/write a MAP_PRIVATE mapping, write to it, then mprotect()
1303 * read-only, mseal() and a discard will be permitted.
1304 *
1305 * However, in order to avoid issues with potential use of madvise(...,
1306 * MADV_DONTNEED) of mseal()'d .text mappings we, for the time being,
1307 * permit this.
1308 */
1309 if (!vma_is_anonymous(vma))
1310 return true;
1311
1312 /* If the user could write to the mapping anyway, then this is fine. */
1313 if ((vma->vm_flags & VM_WRITE) &&
1314 arch_vma_access_permitted(vma, /* write= */ true,
1315 /* execute= */ false, /* foreign= */ false))
1316 return true;
1317
1318 /* Otherwise, we are not permitted to perform this operation. */
1319 return false;
1320}
1321#else
1322static bool can_madvise_modify(struct madvise_behavior *madv_behavior)
1323{
1324 return true;
1325}
1326#endif
1327
1328/*
1329 * Apply an madvise behavior to a region of a vma. madvise_update_vma
1330 * will handle splitting a vm area into separate areas, each area with its own
1331 * behavior.
1332 */
1333static int madvise_vma_behavior(struct madvise_behavior *madv_behavior)
1334{
1335 int behavior = madv_behavior->behavior;
1336 struct vm_area_struct *vma = madv_behavior->vma;
1337 vm_flags_t new_flags = vma->vm_flags;
1338 struct madvise_behavior_range *range = &madv_behavior->range;
1339 int error;
1340
1341 if (unlikely(!can_madvise_modify(madv_behavior)))
1342 return -EPERM;
1343
1344 switch (behavior) {
1345 case MADV_REMOVE:
1346 return madvise_remove(madv_behavior);
1347 case MADV_WILLNEED:
1348 return madvise_willneed(madv_behavior);
1349 case MADV_COLD:
1350 return madvise_cold(madv_behavior);
1351 case MADV_PAGEOUT:
1352 return madvise_pageout(madv_behavior);
1353 case MADV_FREE:
1354 case MADV_DONTNEED:
1355 case MADV_DONTNEED_LOCKED:
1356 return madvise_dontneed_free(madv_behavior);
1357 case MADV_COLLAPSE:
1358 return madvise_collapse(vma, start: range->start, end: range->end,
1359 lock_dropped: &madv_behavior->lock_dropped);
1360 case MADV_GUARD_INSTALL:
1361 return madvise_guard_install(madv_behavior);
1362 case MADV_GUARD_REMOVE:
1363 return madvise_guard_remove(madv_behavior);
1364
1365 /* The below behaviours update VMAs via madvise_update_vma(). */
1366
1367 case MADV_NORMAL:
1368 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
1369 break;
1370 case MADV_SEQUENTIAL:
1371 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
1372 break;
1373 case MADV_RANDOM:
1374 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
1375 break;
1376 case MADV_DONTFORK:
1377 new_flags |= VM_DONTCOPY;
1378 break;
1379 case MADV_DOFORK:
1380 if (new_flags & VM_IO)
1381 return -EINVAL;
1382 new_flags &= ~VM_DONTCOPY;
1383 break;
1384 case MADV_WIPEONFORK:
1385 /* MADV_WIPEONFORK is only supported on anonymous memory. */
1386 if (vma->vm_file || new_flags & VM_SHARED)
1387 return -EINVAL;
1388 new_flags |= VM_WIPEONFORK;
1389 break;
1390 case MADV_KEEPONFORK:
1391 if (new_flags & VM_DROPPABLE)
1392 return -EINVAL;
1393 new_flags &= ~VM_WIPEONFORK;
1394 break;
1395 case MADV_DONTDUMP:
1396 new_flags |= VM_DONTDUMP;
1397 break;
1398 case MADV_DODUMP:
1399 if ((!is_vm_hugetlb_page(vma) && (new_flags & VM_SPECIAL)) ||
1400 (new_flags & VM_DROPPABLE))
1401 return -EINVAL;
1402 new_flags &= ~VM_DONTDUMP;
1403 break;
1404 case MADV_MERGEABLE:
1405 case MADV_UNMERGEABLE:
1406 error = ksm_madvise(vma, start: range->start, end: range->end,
1407 advice: behavior, vm_flags: &new_flags);
1408 if (error)
1409 goto out;
1410 break;
1411 case MADV_HUGEPAGE:
1412 case MADV_NOHUGEPAGE:
1413 error = hugepage_madvise(vma, vm_flags: &new_flags, advice: behavior);
1414 if (error)
1415 goto out;
1416 break;
1417 case __MADV_SET_ANON_VMA_NAME:
1418 /* Only anonymous mappings can be named */
1419 if (vma->vm_file && !vma_is_anon_shmem(vma))
1420 return -EBADF;
1421 break;
1422 }
1423
1424 /* This is a write operation.*/
1425 VM_WARN_ON_ONCE(madv_behavior->lock_mode != MADVISE_MMAP_WRITE_LOCK);
1426
1427 error = madvise_update_vma(new_flags, madv_behavior);
1428out:
1429 /*
1430 * madvise() returns EAGAIN if kernel resources, such as
1431 * slab, are temporarily unavailable.
1432 */
1433 if (error == -ENOMEM)
1434 error = -EAGAIN;
1435 return error;
1436}
1437
1438#ifdef CONFIG_MEMORY_FAILURE
1439/*
1440 * Error injection support for memory error handling.
1441 */
1442static int madvise_inject_error(struct madvise_behavior *madv_behavior)
1443{
1444 unsigned long size;
1445 unsigned long start = madv_behavior->range.start;
1446 unsigned long end = madv_behavior->range.end;
1447
1448 if (!capable(CAP_SYS_ADMIN))
1449 return -EPERM;
1450
1451 for (; start < end; start += size) {
1452 unsigned long pfn;
1453 struct page *page;
1454 int ret;
1455
1456 ret = get_user_pages_fast(start, 1, 0, &page);
1457 if (ret != 1)
1458 return ret;
1459 pfn = page_to_pfn(page);
1460
1461 /*
1462 * When soft offlining hugepages, after migrating the page
1463 * we dissolve it, therefore in the second loop "page" will
1464 * no longer be a compound page.
1465 */
1466 size = page_size(compound_head(page));
1467
1468 if (madv_behavior->behavior == MADV_SOFT_OFFLINE) {
1469 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
1470 pfn, start);
1471 ret = soft_offline_page(pfn, MF_COUNT_INCREASED);
1472 } else {
1473 pr_info("Injecting memory failure for pfn %#lx at process virtual address %#lx\n",
1474 pfn, start);
1475 ret = memory_failure(pfn, MF_ACTION_REQUIRED | MF_COUNT_INCREASED | MF_SW_SIMULATED);
1476 if (ret == -EOPNOTSUPP)
1477 ret = 0;
1478 }
1479
1480 if (ret)
1481 return ret;
1482 }
1483
1484 return 0;
1485}
1486
1487static bool is_memory_failure(struct madvise_behavior *madv_behavior)
1488{
1489 switch (madv_behavior->behavior) {
1490 case MADV_HWPOISON:
1491 case MADV_SOFT_OFFLINE:
1492 return true;
1493 default:
1494 return false;
1495 }
1496}
1497
1498#else
1499
1500static int madvise_inject_error(struct madvise_behavior *madv_behavior)
1501{
1502 return 0;
1503}
1504
1505static bool is_memory_failure(struct madvise_behavior *madv_behavior)
1506{
1507 return false;
1508}
1509
1510#endif /* CONFIG_MEMORY_FAILURE */
1511
1512static bool
1513madvise_behavior_valid(int behavior)
1514{
1515 switch (behavior) {
1516 case MADV_DOFORK:
1517 case MADV_DONTFORK:
1518 case MADV_NORMAL:
1519 case MADV_SEQUENTIAL:
1520 case MADV_RANDOM:
1521 case MADV_REMOVE:
1522 case MADV_WILLNEED:
1523 case MADV_DONTNEED:
1524 case MADV_DONTNEED_LOCKED:
1525 case MADV_FREE:
1526 case MADV_COLD:
1527 case MADV_PAGEOUT:
1528 case MADV_POPULATE_READ:
1529 case MADV_POPULATE_WRITE:
1530#ifdef CONFIG_KSM
1531 case MADV_MERGEABLE:
1532 case MADV_UNMERGEABLE:
1533#endif
1534#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1535 case MADV_HUGEPAGE:
1536 case MADV_NOHUGEPAGE:
1537 case MADV_COLLAPSE:
1538#endif
1539 case MADV_DONTDUMP:
1540 case MADV_DODUMP:
1541 case MADV_WIPEONFORK:
1542 case MADV_KEEPONFORK:
1543 case MADV_GUARD_INSTALL:
1544 case MADV_GUARD_REMOVE:
1545#ifdef CONFIG_MEMORY_FAILURE
1546 case MADV_SOFT_OFFLINE:
1547 case MADV_HWPOISON:
1548#endif
1549 return true;
1550
1551 default:
1552 return false;
1553 }
1554}
1555
1556/* Can we invoke process_madvise() on a remote mm for the specified behavior? */
1557static bool process_madvise_remote_valid(int behavior)
1558{
1559 switch (behavior) {
1560 case MADV_COLD:
1561 case MADV_PAGEOUT:
1562 case MADV_WILLNEED:
1563 case MADV_COLLAPSE:
1564 return true;
1565 default:
1566 return false;
1567 }
1568}
1569
1570/*
1571 * Try to acquire a VMA read lock if possible.
1572 *
1573 * We only support this lock over a single VMA, which the input range must
1574 * span either partially or fully.
1575 *
1576 * This function always returns with an appropriate lock held. If a VMA read
1577 * lock could be acquired, we return true and set madv_behavior state
1578 * accordingly.
1579 *
1580 * If a VMA read lock could not be acquired, we return false and expect caller to
1581 * fallback to mmap lock behaviour.
1582 */
1583static bool try_vma_read_lock(struct madvise_behavior *madv_behavior)
1584{
1585 struct mm_struct *mm = madv_behavior->mm;
1586 struct vm_area_struct *vma;
1587
1588 vma = lock_vma_under_rcu(mm, address: madv_behavior->range.start);
1589 if (!vma)
1590 goto take_mmap_read_lock;
1591 /*
1592 * Must span only a single VMA; uffd and remote processes are
1593 * unsupported.
1594 */
1595 if (madv_behavior->range.end > vma->vm_end || current->mm != mm ||
1596 userfaultfd_armed(vma)) {
1597 vma_end_read(vma);
1598 goto take_mmap_read_lock;
1599 }
1600 madv_behavior->vma = vma;
1601 return true;
1602
1603take_mmap_read_lock:
1604 mmap_read_lock(mm);
1605 madv_behavior->lock_mode = MADVISE_MMAP_READ_LOCK;
1606 return false;
1607}
1608
1609/*
1610 * Walk the vmas in range [start,end), and call the madvise_vma_behavior
1611 * function on each one. The function will get start and end parameters that
1612 * cover the overlap between the current vma and the original range. Any
1613 * unmapped regions in the original range will result in this function returning
1614 * -ENOMEM while still calling the madvise_vma_behavior function on all of the
1615 * existing vmas in the range. Must be called with the mmap_lock held for
1616 * reading or writing.
1617 */
1618static
1619int madvise_walk_vmas(struct madvise_behavior *madv_behavior)
1620{
1621 struct mm_struct *mm = madv_behavior->mm;
1622 struct madvise_behavior_range *range = &madv_behavior->range;
1623 /* range is updated to span each VMA, so store end of entire range. */
1624 unsigned long last_end = range->end;
1625 int unmapped_error = 0;
1626 int error;
1627 struct vm_area_struct *prev, *vma;
1628
1629 /*
1630 * If VMA read lock is supported, apply madvise to a single VMA
1631 * tentatively, avoiding walking VMAs.
1632 */
1633 if (madv_behavior->lock_mode == MADVISE_VMA_READ_LOCK &&
1634 try_vma_read_lock(madv_behavior)) {
1635 error = madvise_vma_behavior(madv_behavior);
1636 vma_end_read(vma: madv_behavior->vma);
1637 return error;
1638 }
1639
1640 vma = find_vma_prev(mm, addr: range->start, pprev: &prev);
1641 if (vma && range->start > vma->vm_start)
1642 prev = vma;
1643
1644 for (;;) {
1645 /* Still start < end. */
1646 if (!vma)
1647 return -ENOMEM;
1648
1649 /* Here start < (last_end|vma->vm_end). */
1650 if (range->start < vma->vm_start) {
1651 /*
1652 * This indicates a gap between VMAs in the input
1653 * range. This does not cause the operation to abort,
1654 * rather we simply return -ENOMEM to indicate that this
1655 * has happened, but carry on.
1656 */
1657 unmapped_error = -ENOMEM;
1658 range->start = vma->vm_start;
1659 if (range->start >= last_end)
1660 break;
1661 }
1662
1663 /* Here vma->vm_start <= range->start < (last_end|vma->vm_end) */
1664 range->end = min(vma->vm_end, last_end);
1665
1666 /* Here vma->vm_start <= range->start < range->end <= (last_end|vma->vm_end). */
1667 madv_behavior->prev = prev;
1668 madv_behavior->vma = vma;
1669 error = madvise_vma_behavior(madv_behavior);
1670 if (error)
1671 return error;
1672 if (madv_behavior->lock_dropped) {
1673 /* We dropped the mmap lock, we can't ref the VMA. */
1674 prev = NULL;
1675 vma = NULL;
1676 madv_behavior->lock_dropped = false;
1677 } else {
1678 vma = madv_behavior->vma;
1679 prev = vma;
1680 }
1681
1682 if (vma && range->end < vma->vm_end)
1683 range->end = vma->vm_end;
1684 if (range->end >= last_end)
1685 break;
1686
1687 vma = find_vma(mm, addr: vma ? vma->vm_end : range->end);
1688 range->start = range->end;
1689 }
1690
1691 return unmapped_error;
1692}
1693
1694/*
1695 * Any behaviour which results in changes to the vma->vm_flags needs to
1696 * take mmap_lock for writing. Others, which simply traverse vmas, need
1697 * to only take it for reading.
1698 */
1699static enum madvise_lock_mode get_lock_mode(struct madvise_behavior *madv_behavior)
1700{
1701 if (is_memory_failure(madv_behavior))
1702 return MADVISE_NO_LOCK;
1703
1704 switch (madv_behavior->behavior) {
1705 case MADV_REMOVE:
1706 case MADV_WILLNEED:
1707 case MADV_COLD:
1708 case MADV_PAGEOUT:
1709 case MADV_POPULATE_READ:
1710 case MADV_POPULATE_WRITE:
1711 case MADV_COLLAPSE:
1712 case MADV_GUARD_INSTALL:
1713 case MADV_GUARD_REMOVE:
1714 return MADVISE_MMAP_READ_LOCK;
1715 case MADV_DONTNEED:
1716 case MADV_DONTNEED_LOCKED:
1717 case MADV_FREE:
1718 return MADVISE_VMA_READ_LOCK;
1719 default:
1720 return MADVISE_MMAP_WRITE_LOCK;
1721 }
1722}
1723
1724static int madvise_lock(struct madvise_behavior *madv_behavior)
1725{
1726 struct mm_struct *mm = madv_behavior->mm;
1727 enum madvise_lock_mode lock_mode = get_lock_mode(madv_behavior);
1728
1729 switch (lock_mode) {
1730 case MADVISE_NO_LOCK:
1731 break;
1732 case MADVISE_MMAP_WRITE_LOCK:
1733 if (mmap_write_lock_killable(mm))
1734 return -EINTR;
1735 break;
1736 case MADVISE_MMAP_READ_LOCK:
1737 mmap_read_lock(mm);
1738 break;
1739 case MADVISE_VMA_READ_LOCK:
1740 /* We will acquire the lock per-VMA in madvise_walk_vmas(). */
1741 break;
1742 }
1743
1744 madv_behavior->lock_mode = lock_mode;
1745 return 0;
1746}
1747
1748static void madvise_unlock(struct madvise_behavior *madv_behavior)
1749{
1750 struct mm_struct *mm = madv_behavior->mm;
1751
1752 switch (madv_behavior->lock_mode) {
1753 case MADVISE_NO_LOCK:
1754 return;
1755 case MADVISE_MMAP_WRITE_LOCK:
1756 mmap_write_unlock(mm);
1757 break;
1758 case MADVISE_MMAP_READ_LOCK:
1759 mmap_read_unlock(mm);
1760 break;
1761 case MADVISE_VMA_READ_LOCK:
1762 /* We will drop the lock per-VMA in madvise_walk_vmas(). */
1763 break;
1764 }
1765
1766 madv_behavior->lock_mode = MADVISE_NO_LOCK;
1767}
1768
1769static bool madvise_batch_tlb_flush(int behavior)
1770{
1771 switch (behavior) {
1772 case MADV_DONTNEED:
1773 case MADV_DONTNEED_LOCKED:
1774 case MADV_FREE:
1775 return true;
1776 default:
1777 return false;
1778 }
1779}
1780
1781static void madvise_init_tlb(struct madvise_behavior *madv_behavior)
1782{
1783 if (madvise_batch_tlb_flush(behavior: madv_behavior->behavior))
1784 tlb_gather_mmu(tlb: madv_behavior->tlb, mm: madv_behavior->mm);
1785}
1786
1787static void madvise_finish_tlb(struct madvise_behavior *madv_behavior)
1788{
1789 if (madvise_batch_tlb_flush(behavior: madv_behavior->behavior))
1790 tlb_finish_mmu(tlb: madv_behavior->tlb);
1791}
1792
1793static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior)
1794{
1795 size_t len;
1796
1797 if (!madvise_behavior_valid(behavior))
1798 return false;
1799
1800 if (!PAGE_ALIGNED(start))
1801 return false;
1802 len = PAGE_ALIGN(len_in);
1803
1804 /* Check to see whether len was rounded up from small -ve to zero */
1805 if (len_in && !len)
1806 return false;
1807
1808 if (start + len < start)
1809 return false;
1810
1811 return true;
1812}
1813
1814/*
1815 * madvise_should_skip() - Return if the request is invalid or nothing.
1816 * @start: Start address of madvise-requested address range.
1817 * @len_in: Length of madvise-requested address range.
1818 * @behavior: Requested madvise behavor.
1819 * @err: Pointer to store an error code from the check.
1820 *
1821 * If the specified behaviour is invalid or nothing would occur, we skip the
1822 * operation. This function returns true in the cases, otherwise false. In
1823 * the former case we store an error on @err.
1824 */
1825static bool madvise_should_skip(unsigned long start, size_t len_in,
1826 int behavior, int *err)
1827{
1828 if (!is_valid_madvise(start, len_in, behavior)) {
1829 *err = -EINVAL;
1830 return true;
1831 }
1832 if (start + PAGE_ALIGN(len_in) == start) {
1833 *err = 0;
1834 return true;
1835 }
1836 return false;
1837}
1838
1839static bool is_madvise_populate(struct madvise_behavior *madv_behavior)
1840{
1841 switch (madv_behavior->behavior) {
1842 case MADV_POPULATE_READ:
1843 case MADV_POPULATE_WRITE:
1844 return true;
1845 default:
1846 return false;
1847 }
1848}
1849
1850/*
1851 * untagged_addr_remote() assumes mmap_lock is already held. On
1852 * architectures like x86 and RISC-V, tagging is tricky because each
1853 * mm may have a different tagging mask. However, we might only hold
1854 * the per-VMA lock (currently only local processes are supported),
1855 * so untagged_addr is used to avoid the mmap_lock assertion for
1856 * local processes.
1857 */
1858static inline unsigned long get_untagged_addr(struct mm_struct *mm,
1859 unsigned long start)
1860{
1861 return current->mm == mm ? untagged_addr(start) :
1862 untagged_addr_remote(mm, start);
1863}
1864
1865static int madvise_do_behavior(unsigned long start, size_t len_in,
1866 struct madvise_behavior *madv_behavior)
1867{
1868 struct blk_plug plug;
1869 int error;
1870 struct madvise_behavior_range *range = &madv_behavior->range;
1871
1872 if (is_memory_failure(madv_behavior)) {
1873 range->start = start;
1874 range->end = start + len_in;
1875 return madvise_inject_error(madv_behavior);
1876 }
1877
1878 range->start = get_untagged_addr(mm: madv_behavior->mm, start);
1879 range->end = range->start + PAGE_ALIGN(len_in);
1880
1881 blk_start_plug(&plug);
1882 if (is_madvise_populate(madv_behavior))
1883 error = madvise_populate(madv_behavior);
1884 else
1885 error = madvise_walk_vmas(madv_behavior);
1886 blk_finish_plug(&plug);
1887 return error;
1888}
1889
1890/*
1891 * The madvise(2) system call.
1892 *
1893 * Applications can use madvise() to advise the kernel how it should
1894 * handle paging I/O in this VM area. The idea is to help the kernel
1895 * use appropriate read-ahead and caching techniques. The information
1896 * provided is advisory only, and can be safely disregarded by the
1897 * kernel without affecting the correct operation of the application.
1898 *
1899 * behavior values:
1900 * MADV_NORMAL - the default behavior is to read clusters. This
1901 * results in some read-ahead and read-behind.
1902 * MADV_RANDOM - the system should read the minimum amount of data
1903 * on any access, since it is unlikely that the appli-
1904 * cation will need more than what it asks for.
1905 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
1906 * once, so they can be aggressively read ahead, and
1907 * can be freed soon after they are accessed.
1908 * MADV_WILLNEED - the application is notifying the system to read
1909 * some pages ahead.
1910 * MADV_DONTNEED - the application is finished with the given range,
1911 * so the kernel can free resources associated with it.
1912 * MADV_FREE - the application marks pages in the given range as lazy free,
1913 * where actual purges are postponed until memory pressure happens.
1914 * MADV_REMOVE - the application wants to free up the given range of
1915 * pages and associated backing store.
1916 * MADV_DONTFORK - omit this area from child's address space when forking:
1917 * typically, to avoid COWing pages pinned by get_user_pages().
1918 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
1919 * MADV_WIPEONFORK - present the child process with zero-filled memory in this
1920 * range after a fork.
1921 * MADV_KEEPONFORK - undo the effect of MADV_WIPEONFORK
1922 * MADV_HWPOISON - trigger memory error handler as if the given memory range
1923 * were corrupted by unrecoverable hardware memory failure.
1924 * MADV_SOFT_OFFLINE - try to soft-offline the given range of memory.
1925 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
1926 * this area with pages of identical content from other such areas.
1927 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
1928 * MADV_HUGEPAGE - the application wants to back the given range by transparent
1929 * huge pages in the future. Existing pages might be coalesced and
1930 * new pages might be allocated as THP.
1931 * MADV_NOHUGEPAGE - mark the given range as not worth being backed by
1932 * transparent huge pages so the existing pages will not be
1933 * coalesced into THP and new pages will not be allocated as THP.
1934 * MADV_COLLAPSE - synchronously coalesce pages into new THP.
1935 * MADV_DONTDUMP - the application wants to prevent pages in the given range
1936 * from being included in its core dump.
1937 * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump.
1938 * MADV_COLD - the application is not expected to use this memory soon,
1939 * deactivate pages in this range so that they can be reclaimed
1940 * easily if memory pressure happens.
1941 * MADV_PAGEOUT - the application is not expected to use this memory soon,
1942 * page out the pages in this range immediately.
1943 * MADV_POPULATE_READ - populate (prefault) page tables readable by
1944 * triggering read faults if required
1945 * MADV_POPULATE_WRITE - populate (prefault) page tables writable by
1946 * triggering write faults if required
1947 *
1948 * return values:
1949 * zero - success
1950 * -EINVAL - start + len < 0, start is not page-aligned,
1951 * "behavior" is not a valid value, or application
1952 * is attempting to release locked or shared pages,
1953 * or the specified address range includes file, Huge TLB,
1954 * MAP_SHARED or VMPFNMAP range.
1955 * -ENOMEM - addresses in the specified range are not currently
1956 * mapped, or are outside the AS of the process.
1957 * -EIO - an I/O error occurred while paging in data.
1958 * -EBADF - map exists, but area maps something that isn't a file.
1959 * -EAGAIN - a kernel resource was temporarily unavailable.
1960 * -EPERM - memory is sealed.
1961 */
1962int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior)
1963{
1964 int error;
1965 struct mmu_gather tlb;
1966 struct madvise_behavior madv_behavior = {
1967 .mm = mm,
1968 .behavior = behavior,
1969 .tlb = &tlb,
1970 };
1971
1972 if (madvise_should_skip(start, len_in, behavior, err: &error))
1973 return error;
1974 error = madvise_lock(madv_behavior: &madv_behavior);
1975 if (error)
1976 return error;
1977 madvise_init_tlb(madv_behavior: &madv_behavior);
1978 error = madvise_do_behavior(start, len_in, madv_behavior: &madv_behavior);
1979 madvise_finish_tlb(madv_behavior: &madv_behavior);
1980 madvise_unlock(madv_behavior: &madv_behavior);
1981
1982 return error;
1983}
1984
1985SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
1986{
1987 return do_madvise(current->mm, start, len_in, behavior);
1988}
1989
1990/* Perform an madvise operation over a vector of addresses and lengths. */
1991static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
1992 int behavior)
1993{
1994 ssize_t ret = 0;
1995 size_t total_len;
1996 struct mmu_gather tlb;
1997 struct madvise_behavior madv_behavior = {
1998 .mm = mm,
1999 .behavior = behavior,
2000 .tlb = &tlb,
2001 };
2002
2003 total_len = iov_iter_count(i: iter);
2004
2005 ret = madvise_lock(madv_behavior: &madv_behavior);
2006 if (ret)
2007 return ret;
2008 madvise_init_tlb(madv_behavior: &madv_behavior);
2009
2010 while (iov_iter_count(i: iter)) {
2011 unsigned long start = (unsigned long)iter_iov_addr(iter);
2012 size_t len_in = iter_iov_len(i: iter);
2013 int error;
2014
2015 if (madvise_should_skip(start, len_in, behavior, err: &error))
2016 ret = error;
2017 else
2018 ret = madvise_do_behavior(start, len_in, madv_behavior: &madv_behavior);
2019 /*
2020 * An madvise operation is attempting to restart the syscall,
2021 * but we cannot proceed as it would not be correct to repeat
2022 * the operation in aggregate, and would be surprising to the
2023 * user.
2024 *
2025 * We drop and reacquire locks so it is safe to just loop and
2026 * try again. We check for fatal signals in case we need exit
2027 * early anyway.
2028 */
2029 if (ret == -ERESTARTNOINTR) {
2030 if (fatal_signal_pending(current)) {
2031 ret = -EINTR;
2032 break;
2033 }
2034
2035 /* Drop and reacquire lock to unwind race. */
2036 madvise_finish_tlb(madv_behavior: &madv_behavior);
2037 madvise_unlock(madv_behavior: &madv_behavior);
2038 ret = madvise_lock(madv_behavior: &madv_behavior);
2039 if (ret)
2040 goto out;
2041 madvise_init_tlb(madv_behavior: &madv_behavior);
2042 continue;
2043 }
2044 if (ret < 0)
2045 break;
2046 iov_iter_advance(i: iter, bytes: iter_iov_len(i: iter));
2047 }
2048 madvise_finish_tlb(madv_behavior: &madv_behavior);
2049 madvise_unlock(madv_behavior: &madv_behavior);
2050
2051out:
2052 ret = (total_len - iov_iter_count(i: iter)) ? : ret;
2053
2054 return ret;
2055}
2056
2057SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
2058 size_t, vlen, int, behavior, unsigned int, flags)
2059{
2060 ssize_t ret;
2061 struct iovec iovstack[UIO_FASTIOV];
2062 struct iovec *iov = iovstack;
2063 struct iov_iter iter;
2064 struct task_struct *task;
2065 struct mm_struct *mm;
2066 unsigned int f_flags;
2067
2068 if (flags != 0) {
2069 ret = -EINVAL;
2070 goto out;
2071 }
2072
2073 ret = import_iovec(ITER_DEST, uvec: vec, nr_segs: vlen, ARRAY_SIZE(iovstack), iovp: &iov, i: &iter);
2074 if (ret < 0)
2075 goto out;
2076
2077 task = pidfd_get_task(pidfd, flags: &f_flags);
2078 if (IS_ERR(ptr: task)) {
2079 ret = PTR_ERR(ptr: task);
2080 goto free_iov;
2081 }
2082
2083 /* Require PTRACE_MODE_READ to avoid leaking ASLR metadata. */
2084 mm = mm_access(task, PTRACE_MODE_READ_FSCREDS);
2085 if (IS_ERR(ptr: mm)) {
2086 ret = PTR_ERR(ptr: mm);
2087 goto release_task;
2088 }
2089
2090 /*
2091 * We need only perform this check if we are attempting to manipulate a
2092 * remote process's address space.
2093 */
2094 if (mm != current->mm && !process_madvise_remote_valid(behavior)) {
2095 ret = -EINVAL;
2096 goto release_mm;
2097 }
2098
2099 /*
2100 * Require CAP_SYS_NICE for influencing process performance. Note that
2101 * only non-destructive hints are currently supported for remote
2102 * processes.
2103 */
2104 if (mm != current->mm && !capable(CAP_SYS_NICE)) {
2105 ret = -EPERM;
2106 goto release_mm;
2107 }
2108
2109 ret = vector_madvise(mm, iter: &iter, behavior);
2110
2111release_mm:
2112 mmput(mm);
2113release_task:
2114 put_task_struct(t: task);
2115free_iov:
2116 kfree(objp: iov);
2117out:
2118 return ret;
2119}
2120
2121#ifdef CONFIG_ANON_VMA_NAME
2122
2123#define ANON_VMA_NAME_MAX_LEN 80
2124#define ANON_VMA_NAME_INVALID_CHARS "\\`$[]"
2125
2126static inline bool is_valid_name_char(char ch)
2127{
2128 /* printable ascii characters, excluding ANON_VMA_NAME_INVALID_CHARS */
2129 return ch > 0x1f && ch < 0x7f &&
2130 !strchr(ANON_VMA_NAME_INVALID_CHARS, ch);
2131}
2132
2133static int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
2134 unsigned long len_in, struct anon_vma_name *anon_name)
2135{
2136 unsigned long end;
2137 unsigned long len;
2138 int error;
2139 struct madvise_behavior madv_behavior = {
2140 .mm = mm,
2141 .behavior = __MADV_SET_ANON_VMA_NAME,
2142 .anon_name = anon_name,
2143 };
2144
2145 if (start & ~PAGE_MASK)
2146 return -EINVAL;
2147 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
2148
2149 /* Check to see whether len was rounded up from small -ve to zero */
2150 if (len_in && !len)
2151 return -EINVAL;
2152
2153 end = start + len;
2154 if (end < start)
2155 return -EINVAL;
2156
2157 if (end == start)
2158 return 0;
2159
2160 madv_behavior.range.start = start;
2161 madv_behavior.range.end = end;
2162
2163 error = madvise_lock(&madv_behavior);
2164 if (error)
2165 return error;
2166 error = madvise_walk_vmas(&madv_behavior);
2167 madvise_unlock(&madv_behavior);
2168
2169 return error;
2170}
2171
2172int set_anon_vma_name(unsigned long addr, unsigned long size,
2173 const char __user *uname)
2174{
2175 struct anon_vma_name *anon_name = NULL;
2176 struct mm_struct *mm = current->mm;
2177 int error;
2178
2179 if (uname) {
2180 char *name, *pch;
2181
2182 name = strndup_user(uname, ANON_VMA_NAME_MAX_LEN);
2183 if (IS_ERR(name))
2184 return PTR_ERR(name);
2185
2186 for (pch = name; *pch != '\0'; pch++) {
2187 if (!is_valid_name_char(*pch)) {
2188 kfree(name);
2189 return -EINVAL;
2190 }
2191 }
2192 /* anon_vma has its own copy */
2193 anon_name = anon_vma_name_alloc(name);
2194 kfree(name);
2195 if (!anon_name)
2196 return -ENOMEM;
2197 }
2198
2199 error = madvise_set_anon_name(mm, addr, size, anon_name);
2200 anon_vma_name_put(anon_name);
2201
2202 return error;
2203}
2204#endif
2205