1// SPDX-License-Identifier: GPL-2.0-or-later
2
3/*
4 * Functions for initializing, allocating, freeing and duplicating VMAs. Shared
5 * between CONFIG_MMU and non-CONFIG_MMU kernel configurations.
6 */
7
8#include "vma_internal.h"
9#include "vma.h"
10
11/* SLAB cache for vm_area_struct structures */
12static struct kmem_cache *vm_area_cachep;
13
14void __init vma_state_init(void)
15{
16 struct kmem_cache_args args = {
17 .use_freeptr_offset = true,
18 .freeptr_offset = offsetof(struct vm_area_struct, vm_freeptr),
19 .sheaf_capacity = 32,
20 };
21
22 vm_area_cachep = kmem_cache_create("vm_area_struct",
23 sizeof(struct vm_area_struct), &args,
24 SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU|
25 SLAB_ACCOUNT);
26}
27
28struct vm_area_struct *vm_area_alloc(struct mm_struct *mm)
29{
30 struct vm_area_struct *vma;
31
32 vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
33 if (!vma)
34 return NULL;
35
36 vma_init(vma, mm);
37
38 return vma;
39}
40
41static void vm_area_init_from(const struct vm_area_struct *src,
42 struct vm_area_struct *dest)
43{
44 dest->vm_mm = src->vm_mm;
45 dest->vm_ops = src->vm_ops;
46 dest->vm_start = src->vm_start;
47 dest->vm_end = src->vm_end;
48 dest->anon_vma = src->anon_vma;
49 dest->vm_pgoff = src->vm_pgoff;
50 dest->vm_file = src->vm_file;
51 dest->vm_private_data = src->vm_private_data;
52 vm_flags_init(vma: dest, flags: src->vm_flags);
53 memcpy(to: &dest->vm_page_prot, from: &src->vm_page_prot,
54 len: sizeof(dest->vm_page_prot));
55 /*
56 * src->shared.rb may be modified concurrently when called from
57 * dup_mmap(), but the clone will reinitialize it.
58 */
59 data_race(memcpy(&dest->shared, &src->shared, sizeof(dest->shared)));
60 memcpy(to: &dest->vm_userfaultfd_ctx, from: &src->vm_userfaultfd_ctx,
61 len: sizeof(dest->vm_userfaultfd_ctx));
62#ifdef CONFIG_ANON_VMA_NAME
63 dest->anon_name = src->anon_name;
64#endif
65#ifdef CONFIG_SWAP
66 memcpy(to: &dest->swap_readahead_info, from: &src->swap_readahead_info,
67 len: sizeof(dest->swap_readahead_info));
68#endif
69#ifndef CONFIG_MMU
70 dest->vm_region = src->vm_region;
71#endif
72#ifdef CONFIG_NUMA
73 dest->vm_policy = src->vm_policy;
74#endif
75#ifdef __HAVE_PFNMAP_TRACKING
76 dest->pfnmap_track_ctx = NULL;
77#endif
78}
79
80#ifdef __HAVE_PFNMAP_TRACKING
81static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
82 struct vm_area_struct *new)
83{
84 struct pfnmap_track_ctx *ctx = orig->pfnmap_track_ctx;
85
86 if (likely(!ctx))
87 return 0;
88
89 /*
90 * We don't expect to ever hit this. If ever required, we would have
91 * to duplicate the tracking.
92 */
93 if (unlikely(kref_read(&ctx->kref) >= REFCOUNT_MAX))
94 return -ENOMEM;
95 kref_get(kref: &ctx->kref);
96 new->pfnmap_track_ctx = ctx;
97 return 0;
98}
99
100static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
101{
102 struct pfnmap_track_ctx *ctx = vma->pfnmap_track_ctx;
103
104 if (likely(!ctx))
105 return;
106
107 kref_put(kref: &ctx->kref, release: pfnmap_track_ctx_release);
108 vma->pfnmap_track_ctx = NULL;
109}
110#else
111static inline int vma_pfnmap_track_ctx_dup(struct vm_area_struct *orig,
112 struct vm_area_struct *new)
113{
114 return 0;
115}
116static inline void vma_pfnmap_track_ctx_release(struct vm_area_struct *vma)
117{
118}
119#endif
120
121struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
122{
123 struct vm_area_struct *new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
124
125 if (!new)
126 return NULL;
127
128 ASSERT_EXCLUSIVE_WRITER(orig->vm_flags);
129 ASSERT_EXCLUSIVE_WRITER(orig->vm_file);
130 vm_area_init_from(src: orig, dest: new);
131
132 if (vma_pfnmap_track_ctx_dup(orig, new)) {
133 kmem_cache_free(s: vm_area_cachep, objp: new);
134 return NULL;
135 }
136 vma_lock_init(vma: new, reset_refcnt: true);
137 INIT_LIST_HEAD(list: &new->anon_vma_chain);
138 vma_numab_state_init(vma: new);
139 dup_anon_vma_name(orig_vma: orig, new_vma: new);
140
141 return new;
142}
143
144void vm_area_free(struct vm_area_struct *vma)
145{
146 /* The vma should be detached while being destroyed. */
147 vma_assert_detached(vma);
148 vma_numab_state_free(vma);
149 free_anon_vma_name(vma);
150 vma_pfnmap_track_ctx_release(vma);
151 kmem_cache_free(s: vm_area_cachep, objp: vma);
152}
153