1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Macros for manipulating and testing page->flags
4 */
5
6#ifndef PAGE_FLAGS_H
7#define PAGE_FLAGS_H
8
9#include <linux/types.h>
10#include <linux/bug.h>
11#include <linux/mmdebug.h>
12#ifndef __GENERATING_BOUNDS_H
13#include <linux/mm_types.h>
14#include <generated/bounds.h>
15#endif /* !__GENERATING_BOUNDS_H */
16
17/*
18 * Various page->flags bits:
19 *
20 * PG_reserved is set for special pages. The "struct page" of such a page
21 * should in general not be touched (e.g. set dirty) except by its owner.
22 * Pages marked as PG_reserved include:
23 * - Pages part of the kernel image (including vDSO) and similar (e.g. BIOS,
24 * initrd, HW tables)
25 * - Pages reserved or allocated early during boot (before the page allocator
26 * was initialized). This includes (depending on the architecture) the
27 * initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
28 * much more. Once (if ever) freed, PG_reserved is cleared and they will
29 * be given to the page allocator.
30 * - Pages falling into physical memory gaps - not IORESOURCE_SYSRAM. Trying
31 * to read/write these pages might end badly. Don't touch!
32 * - The zero page(s)
33 * - Pages allocated in the context of kexec/kdump (loaded kernel image,
34 * control pages, vmcoreinfo)
35 * - MMIO/DMA pages. Some architectures don't allow to ioremap pages that are
36 * not marked PG_reserved (as they might be in use by somebody else who does
37 * not respect the caching strategy).
38 * - MCA pages on ia64
39 * - Pages holding CPU notes for POWER Firmware Assisted Dump
40 * - Device memory (e.g. PMEM, DAX, HMM)
41 * Some PG_reserved pages will be excluded from the hibernation image.
42 * PG_reserved does in general not hinder anybody from dumping or swapping
43 * and is no longer required for remap_pfn_range(). ioremap might require it.
44 * Consequently, PG_reserved for a page mapped into user space can indicate
45 * the zero page, the vDSO, MMIO pages or device memory.
46 *
47 * The PG_private bitflag is set on pagecache pages if they contain filesystem
48 * specific data (which is normally at page->private). It can be used by
49 * private allocations for its own usage.
50 *
51 * During initiation of disk I/O, PG_locked is set. This bit is set before I/O
52 * and cleared when writeback _starts_ or when read _completes_. PG_writeback
53 * is set before writeback starts and cleared when it finishes.
54 *
55 * PG_locked also pins a page in pagecache, and blocks truncation of the file
56 * while it is held.
57 *
58 * page_waitqueue(page) is a wait queue of all tasks waiting for the page
59 * to become unlocked.
60 *
61 * PG_swapbacked is set when a page uses swap as a backing storage. This are
62 * usually PageAnon or shmem pages but please note that even anonymous pages
63 * might lose their PG_swapbacked flag when they simply can be dropped (e.g. as
64 * a result of MADV_FREE).
65 *
66 * PG_referenced, PG_reclaim are used for page reclaim for anonymous and
67 * file-backed pagecache (see mm/vmscan.c).
68 *
69 * PG_arch_1 is an architecture specific page state bit. The generic code
70 * guarantees that this bit is cleared for a page when it first is entered into
71 * the page cache.
72 *
73 * PG_hwpoison indicates that a page got corrupted in hardware and contains
74 * data with incorrect ECC bits that triggered a machine check. Accessing is
75 * not safe since it may cause another machine check. Don't touch!
76 */
77
78/*
79 * Don't use the pageflags directly. Use the PageFoo macros.
80 *
81 * The page flags field is split into two parts, the main flags area
82 * which extends from the low bits upwards, and the fields area which
83 * extends from the high bits downwards.
84 *
85 * | FIELD | ... | FLAGS |
86 * N-1 ^ 0
87 * (NR_PAGEFLAGS)
88 *
89 * The fields area is reserved for fields mapping zone, node (for NUMA) and
90 * SPARSEMEM section (for variants of SPARSEMEM that require section ids like
91 * SPARSEMEM_EXTREME with !SPARSEMEM_VMEMMAP).
92 */
93enum pageflags {
94 PG_locked, /* Page is locked. Don't touch. */
95 PG_writeback, /* Page is under writeback */
96 PG_referenced,
97 PG_uptodate,
98 PG_dirty,
99 PG_lru,
100 PG_head, /* Must be in bit 6 */
101 PG_waiters, /* Page has waiters, check its waitqueue. Must be bit #7 and in the same byte as "PG_locked" */
102 PG_active,
103 PG_workingset,
104 PG_owner_priv_1, /* Owner use. If pagecache, fs may use */
105 PG_owner_2, /* Owner use. If pagecache, fs may use */
106 PG_arch_1,
107 PG_reserved,
108 PG_private, /* If pagecache, has fs-private data */
109 PG_private_2, /* If pagecache, has fs aux data */
110 PG_reclaim, /* To be reclaimed asap */
111 PG_swapbacked, /* Page is backed by RAM/swap */
112 PG_unevictable, /* Page is "unevictable" */
113 PG_dropbehind, /* drop pages on IO completion */
114#ifdef CONFIG_MMU
115 PG_mlocked, /* Page is vma mlocked */
116#endif
117#ifdef CONFIG_MEMORY_FAILURE
118 PG_hwpoison, /* hardware poisoned page. Don't touch */
119#endif
120#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT)
121 PG_young,
122 PG_idle,
123#endif
124#ifdef CONFIG_ARCH_USES_PG_ARCH_2
125 PG_arch_2,
126#endif
127#ifdef CONFIG_ARCH_USES_PG_ARCH_3
128 PG_arch_3,
129#endif
130 __NR_PAGEFLAGS,
131
132 PG_readahead = PG_reclaim,
133
134 /* Anonymous memory (and shmem) */
135 PG_swapcache = PG_owner_priv_1, /* Swap page: swp_entry_t in private */
136 /* Some filesystems */
137 PG_checked = PG_owner_priv_1,
138
139 /*
140 * Depending on the way an anonymous folio can be mapped into a page
141 * table (e.g., single PMD/PUD/CONT of the head page vs. PTE-mapped
142 * THP), PG_anon_exclusive may be set only for the head page or for
143 * tail pages of an anonymous folio. For now, we only expect it to be
144 * set on tail pages for PTE-mapped THP.
145 */
146 PG_anon_exclusive = PG_owner_2,
147
148 /*
149 * Set if all buffer heads in the folio are mapped.
150 * Filesystems which do not use BHs can use it for their own purpose.
151 */
152 PG_mappedtodisk = PG_owner_2,
153
154 /* Two page bits are conscripted by FS-Cache to maintain local caching
155 * state. These bits are set on pages belonging to the netfs's inodes
156 * when those inodes are being locally cached.
157 */
158 PG_fscache = PG_private_2, /* page backed by cache */
159
160 /* XEN */
161 /* Pinned in Xen as a read-only pagetable page. */
162 PG_pinned = PG_owner_priv_1,
163 /* Pinned as part of domain save (see xen_mm_pin_all()). */
164 PG_savepinned = PG_dirty,
165 /* Has a grant mapping of another (foreign) domain's page. */
166 PG_foreign = PG_owner_priv_1,
167 /* Remapped by swiotlb-xen. */
168 PG_xen_remapped = PG_owner_priv_1,
169
170#ifdef CONFIG_MIGRATION
171 /* movable_ops page that is isolated for migration */
172 PG_movable_ops_isolated = PG_reclaim,
173 /* this is a movable_ops page (for selected typed pages only) */
174 PG_movable_ops = PG_uptodate,
175#endif
176
177 /* Only valid for buddy pages. Used to track pages that are reported */
178 PG_reported = PG_uptodate,
179
180#ifdef CONFIG_MEMORY_HOTPLUG
181 /* For self-hosted memmap pages */
182 PG_vmemmap_self_hosted = PG_owner_priv_1,
183#endif
184
185 /*
186 * Flags only valid for compound pages. Stored in first tail page's
187 * flags word. Cannot use the first 8 flags or any flag marked as
188 * PF_ANY.
189 */
190
191 /* At least one page in this folio has the hwpoison flag set */
192 PG_has_hwpoisoned = PG_active,
193 PG_large_rmappable = PG_workingset, /* anon or file-backed */
194 PG_partially_mapped = PG_reclaim, /* was identified to be partially mapped */
195};
196
197#define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1)
198
199#ifndef __GENERATING_BOUNDS_H
200
201#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
202DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
203
204/*
205 * Return the real head page struct iff the @page is a fake head page, otherwise
206 * return the @page itself. See Documentation/mm/vmemmap_dedup.rst.
207 */
208static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
209{
210 if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
211 return page;
212
213 /*
214 * Only addresses aligned with PAGE_SIZE of struct page may be fake head
215 * struct page. The alignment check aims to avoid access the fields (
216 * e.g. compound_head) of the @page[1]. It can avoid touch a (possibly)
217 * cold cacheline in some cases.
218 */
219 if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) &&
220 test_bit(PG_head, &page->flags.f)) {
221 /*
222 * We can safely access the field of the @page[1] with PG_head
223 * because the @page is a compound page composed with at least
224 * two contiguous pages.
225 */
226 unsigned long head = READ_ONCE(page[1].compound_head);
227
228 if (likely(head & 1))
229 return (const struct page *)(head - 1);
230 }
231 return page;
232}
233
234static __always_inline bool page_count_writable(const struct page *page, int u)
235{
236 if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key))
237 return true;
238
239 /*
240 * The refcount check is ordered before the fake-head check to prevent
241 * the following race:
242 * CPU 1 (HVO) CPU 2 (speculative PFN walker)
243 *
244 * page_ref_freeze()
245 * synchronize_rcu()
246 * rcu_read_lock()
247 * page_is_fake_head() is false
248 * vmemmap_remap_pte()
249 * XXX: struct page[] becomes r/o
250 *
251 * page_ref_unfreeze()
252 * page_ref_count() is not zero
253 *
254 * atomic_add_unless(&page->_refcount)
255 * XXX: try to modify r/o struct page[]
256 *
257 * The refcount check also prevents modification attempts to other (r/o)
258 * tail pages that are not fake heads.
259 */
260 if (atomic_read_acquire(v: &page->_refcount) == u)
261 return false;
262
263 return page_fixed_fake_head(page) == page;
264}
265#else
266static inline const struct page *page_fixed_fake_head(const struct page *page)
267{
268 return page;
269}
270
271static inline bool page_count_writable(const struct page *page, int u)
272{
273 return true;
274}
275#endif
276
277static __always_inline int page_is_fake_head(const struct page *page)
278{
279 return page_fixed_fake_head(page) != page;
280}
281
282static __always_inline unsigned long _compound_head(const struct page *page)
283{
284 unsigned long head = READ_ONCE(page->compound_head);
285
286 if (unlikely(head & 1))
287 return head - 1;
288 return (unsigned long)page_fixed_fake_head(page);
289}
290
291#define compound_head(page) ((typeof(page))_compound_head(page))
292
293/**
294 * page_folio - Converts from page to folio.
295 * @p: The page.
296 *
297 * Every page is part of a folio. This function cannot be called on a
298 * NULL pointer.
299 *
300 * Context: No reference, nor lock is required on @page. If the caller
301 * does not hold a reference, this call may race with a folio split, so
302 * it should re-check the folio still contains this page after gaining
303 * a reference on the folio.
304 * Return: The folio which contains this page.
305 */
306#define page_folio(p) (_Generic((p), \
307 const struct page *: (const struct folio *)_compound_head(p), \
308 struct page *: (struct folio *)_compound_head(p)))
309
310/**
311 * folio_page - Return a page from a folio.
312 * @folio: The folio.
313 * @n: The page number to return.
314 *
315 * @n is relative to the start of the folio. This function does not
316 * check that the page number lies within @folio; the caller is presumed
317 * to have a reference to the page.
318 */
319#define folio_page(folio, n) (&(folio)->page + (n))
320
321static __always_inline int PageTail(const struct page *page)
322{
323 return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page);
324}
325
326static __always_inline int PageCompound(const struct page *page)
327{
328 return test_bit(PG_head, &page->flags.f) ||
329 READ_ONCE(page->compound_head) & 1;
330}
331
332#define PAGE_POISON_PATTERN -1l
333static inline int PagePoisoned(const struct page *page)
334{
335 return READ_ONCE(page->flags.f) == PAGE_POISON_PATTERN;
336}
337
338#ifdef CONFIG_DEBUG_VM
339void page_init_poison(struct page *page, size_t size);
340#else
341static inline void page_init_poison(struct page *page, size_t size)
342{
343}
344#endif
345
346static const unsigned long *const_folio_flags(const struct folio *folio,
347 unsigned n)
348{
349 const struct page *page = &folio->page;
350
351 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
352 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
353 return &page[n].flags.f;
354}
355
356static unsigned long *folio_flags(struct folio *folio, unsigned n)
357{
358 struct page *page = &folio->page;
359
360 VM_BUG_ON_PGFLAGS(page->compound_head & 1, page);
361 VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page);
362 return &page[n].flags.f;
363}
364
365/*
366 * Page flags policies wrt compound pages
367 *
368 * PF_POISONED_CHECK
369 * check if this struct page poisoned/uninitialized
370 *
371 * PF_ANY:
372 * the page flag is relevant for small, head and tail pages.
373 *
374 * PF_HEAD:
375 * for compound page all operations related to the page flag applied to
376 * head page.
377 *
378 * PF_NO_TAIL:
379 * modifications of the page flag must be done on small or head pages,
380 * checks can be done on tail pages too.
381 *
382 * PF_NO_COMPOUND:
383 * the page flag is not relevant for compound pages.
384 *
385 * PF_SECOND:
386 * the page flag is stored in the first tail page.
387 */
388#define PF_POISONED_CHECK(page) ({ \
389 VM_BUG_ON_PGFLAGS(PagePoisoned(page), page); \
390 page; })
391#define PF_ANY(page, enforce) PF_POISONED_CHECK(page)
392#define PF_HEAD(page, enforce) PF_POISONED_CHECK(compound_head(page))
393#define PF_NO_TAIL(page, enforce) ({ \
394 VM_BUG_ON_PGFLAGS(enforce && PageTail(page), page); \
395 PF_POISONED_CHECK(compound_head(page)); })
396#define PF_NO_COMPOUND(page, enforce) ({ \
397 VM_BUG_ON_PGFLAGS(enforce && PageCompound(page), page); \
398 PF_POISONED_CHECK(page); })
399#define PF_SECOND(page, enforce) ({ \
400 VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
401 PF_POISONED_CHECK(&page[1]); })
402
403/* Which page is the flag stored in */
404#define FOLIO_PF_ANY 0
405#define FOLIO_PF_HEAD 0
406#define FOLIO_PF_NO_TAIL 0
407#define FOLIO_PF_NO_COMPOUND 0
408#define FOLIO_PF_SECOND 1
409
410#define FOLIO_HEAD_PAGE 0
411#define FOLIO_SECOND_PAGE 1
412
413/*
414 * Macros to create function definitions for page flags
415 */
416#define FOLIO_TEST_FLAG(name, page) \
417static __always_inline bool folio_test_##name(const struct folio *folio) \
418{ return test_bit(PG_##name, const_folio_flags(folio, page)); }
419
420#define FOLIO_SET_FLAG(name, page) \
421static __always_inline void folio_set_##name(struct folio *folio) \
422{ set_bit(PG_##name, folio_flags(folio, page)); }
423
424#define FOLIO_CLEAR_FLAG(name, page) \
425static __always_inline void folio_clear_##name(struct folio *folio) \
426{ clear_bit(PG_##name, folio_flags(folio, page)); }
427
428#define __FOLIO_SET_FLAG(name, page) \
429static __always_inline void __folio_set_##name(struct folio *folio) \
430{ __set_bit(PG_##name, folio_flags(folio, page)); }
431
432#define __FOLIO_CLEAR_FLAG(name, page) \
433static __always_inline void __folio_clear_##name(struct folio *folio) \
434{ __clear_bit(PG_##name, folio_flags(folio, page)); }
435
436#define FOLIO_TEST_SET_FLAG(name, page) \
437static __always_inline bool folio_test_set_##name(struct folio *folio) \
438{ return test_and_set_bit(PG_##name, folio_flags(folio, page)); }
439
440#define FOLIO_TEST_CLEAR_FLAG(name, page) \
441static __always_inline bool folio_test_clear_##name(struct folio *folio) \
442{ return test_and_clear_bit(PG_##name, folio_flags(folio, page)); }
443
444#define FOLIO_FLAG(name, page) \
445FOLIO_TEST_FLAG(name, page) \
446FOLIO_SET_FLAG(name, page) \
447FOLIO_CLEAR_FLAG(name, page)
448
449#define TESTPAGEFLAG(uname, lname, policy) \
450FOLIO_TEST_FLAG(lname, FOLIO_##policy) \
451static __always_inline int Page##uname(const struct page *page) \
452{ return test_bit(PG_##lname, &policy(page, 0)->flags.f); }
453
454#define SETPAGEFLAG(uname, lname, policy) \
455FOLIO_SET_FLAG(lname, FOLIO_##policy) \
456static __always_inline void SetPage##uname(struct page *page) \
457{ set_bit(PG_##lname, &policy(page, 1)->flags.f); }
458
459#define CLEARPAGEFLAG(uname, lname, policy) \
460FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \
461static __always_inline void ClearPage##uname(struct page *page) \
462{ clear_bit(PG_##lname, &policy(page, 1)->flags.f); }
463
464#define __SETPAGEFLAG(uname, lname, policy) \
465__FOLIO_SET_FLAG(lname, FOLIO_##policy) \
466static __always_inline void __SetPage##uname(struct page *page) \
467{ __set_bit(PG_##lname, &policy(page, 1)->flags.f); }
468
469#define __CLEARPAGEFLAG(uname, lname, policy) \
470__FOLIO_CLEAR_FLAG(lname, FOLIO_##policy) \
471static __always_inline void __ClearPage##uname(struct page *page) \
472{ __clear_bit(PG_##lname, &policy(page, 1)->flags.f); }
473
474#define TESTSETFLAG(uname, lname, policy) \
475FOLIO_TEST_SET_FLAG(lname, FOLIO_##policy) \
476static __always_inline int TestSetPage##uname(struct page *page) \
477{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags.f); }
478
479#define TESTCLEARFLAG(uname, lname, policy) \
480FOLIO_TEST_CLEAR_FLAG(lname, FOLIO_##policy) \
481static __always_inline int TestClearPage##uname(struct page *page) \
482{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags.f); }
483
484#define PAGEFLAG(uname, lname, policy) \
485 TESTPAGEFLAG(uname, lname, policy) \
486 SETPAGEFLAG(uname, lname, policy) \
487 CLEARPAGEFLAG(uname, lname, policy)
488
489#define __PAGEFLAG(uname, lname, policy) \
490 TESTPAGEFLAG(uname, lname, policy) \
491 __SETPAGEFLAG(uname, lname, policy) \
492 __CLEARPAGEFLAG(uname, lname, policy)
493
494#define TESTSCFLAG(uname, lname, policy) \
495 TESTSETFLAG(uname, lname, policy) \
496 TESTCLEARFLAG(uname, lname, policy)
497
498#define FOLIO_TEST_FLAG_FALSE(name) \
499static inline bool folio_test_##name(const struct folio *folio) \
500{ return false; }
501#define FOLIO_SET_FLAG_NOOP(name) \
502static inline void folio_set_##name(struct folio *folio) { }
503#define FOLIO_CLEAR_FLAG_NOOP(name) \
504static inline void folio_clear_##name(struct folio *folio) { }
505#define __FOLIO_SET_FLAG_NOOP(name) \
506static inline void __folio_set_##name(struct folio *folio) { }
507#define __FOLIO_CLEAR_FLAG_NOOP(name) \
508static inline void __folio_clear_##name(struct folio *folio) { }
509#define FOLIO_TEST_SET_FLAG_FALSE(name) \
510static inline bool folio_test_set_##name(struct folio *folio) \
511{ return false; }
512#define FOLIO_TEST_CLEAR_FLAG_FALSE(name) \
513static inline bool folio_test_clear_##name(struct folio *folio) \
514{ return false; }
515
516#define FOLIO_FLAG_FALSE(name) \
517FOLIO_TEST_FLAG_FALSE(name) \
518FOLIO_SET_FLAG_NOOP(name) \
519FOLIO_CLEAR_FLAG_NOOP(name)
520
521#define TESTPAGEFLAG_FALSE(uname, lname) \
522FOLIO_TEST_FLAG_FALSE(lname) \
523static inline int Page##uname(const struct page *page) { return 0; }
524
525#define SETPAGEFLAG_NOOP(uname, lname) \
526FOLIO_SET_FLAG_NOOP(lname) \
527static inline void SetPage##uname(struct page *page) { }
528
529#define CLEARPAGEFLAG_NOOP(uname, lname) \
530FOLIO_CLEAR_FLAG_NOOP(lname) \
531static inline void ClearPage##uname(struct page *page) { }
532
533#define __CLEARPAGEFLAG_NOOP(uname, lname) \
534__FOLIO_CLEAR_FLAG_NOOP(lname) \
535static inline void __ClearPage##uname(struct page *page) { }
536
537#define TESTSETFLAG_FALSE(uname, lname) \
538FOLIO_TEST_SET_FLAG_FALSE(lname) \
539static inline int TestSetPage##uname(struct page *page) { return 0; }
540
541#define TESTCLEARFLAG_FALSE(uname, lname) \
542FOLIO_TEST_CLEAR_FLAG_FALSE(lname) \
543static inline int TestClearPage##uname(struct page *page) { return 0; }
544
545#define PAGEFLAG_FALSE(uname, lname) TESTPAGEFLAG_FALSE(uname, lname) \
546 SETPAGEFLAG_NOOP(uname, lname) CLEARPAGEFLAG_NOOP(uname, lname)
547
548#define TESTSCFLAG_FALSE(uname, lname) \
549 TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname)
550
551__PAGEFLAG(Locked, locked, PF_NO_TAIL)
552FOLIO_FLAG(waiters, FOLIO_HEAD_PAGE)
553FOLIO_FLAG(referenced, FOLIO_HEAD_PAGE)
554 FOLIO_TEST_CLEAR_FLAG(referenced, FOLIO_HEAD_PAGE)
555 __FOLIO_SET_FLAG(referenced, FOLIO_HEAD_PAGE)
556PAGEFLAG(Dirty, dirty, PF_HEAD) TESTSCFLAG(Dirty, dirty, PF_HEAD)
557 __CLEARPAGEFLAG(Dirty, dirty, PF_HEAD)
558PAGEFLAG(LRU, lru, PF_HEAD) __CLEARPAGEFLAG(LRU, lru, PF_HEAD)
559 TESTCLEARFLAG(LRU, lru, PF_HEAD)
560FOLIO_FLAG(active, FOLIO_HEAD_PAGE)
561 __FOLIO_CLEAR_FLAG(active, FOLIO_HEAD_PAGE)
562 FOLIO_TEST_CLEAR_FLAG(active, FOLIO_HEAD_PAGE)
563PAGEFLAG(Workingset, workingset, PF_HEAD)
564 TESTCLEARFLAG(Workingset, workingset, PF_HEAD)
565PAGEFLAG(Checked, checked, PF_NO_COMPOUND) /* Used by some filesystems */
566
567/* Xen */
568PAGEFLAG(Pinned, pinned, PF_NO_COMPOUND)
569 TESTSCFLAG(Pinned, pinned, PF_NO_COMPOUND)
570PAGEFLAG(SavePinned, savepinned, PF_NO_COMPOUND);
571PAGEFLAG(Foreign, foreign, PF_NO_COMPOUND);
572PAGEFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
573 TESTCLEARFLAG(XenRemapped, xen_remapped, PF_NO_COMPOUND)
574
575PAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
576 __CLEARPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
577 __SETPAGEFLAG(Reserved, reserved, PF_NO_COMPOUND)
578FOLIO_FLAG(swapbacked, FOLIO_HEAD_PAGE)
579 __FOLIO_CLEAR_FLAG(swapbacked, FOLIO_HEAD_PAGE)
580 __FOLIO_SET_FLAG(swapbacked, FOLIO_HEAD_PAGE)
581
582/*
583 * Private page markings that may be used by the filesystem that owns the page
584 * for its own purposes.
585 * - PG_private and PG_private_2 cause release_folio() and co to be invoked
586 */
587PAGEFLAG(Private, private, PF_ANY)
588FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE)
589
590/* owner_2 can be set on tail pages for anon memory */
591FOLIO_FLAG(owner_2, FOLIO_HEAD_PAGE)
592
593/*
594 * Only test-and-set exist for PG_writeback. The unconditional operators are
595 * risky: they bypass page accounting.
596 */
597TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL)
598 TESTSCFLAG(Writeback, writeback, PF_NO_TAIL)
599FOLIO_FLAG(mappedtodisk, FOLIO_HEAD_PAGE)
600
601/* PG_readahead is only used for reads; PG_reclaim is only for writes */
602PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
603 TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
604FOLIO_FLAG(readahead, FOLIO_HEAD_PAGE)
605 FOLIO_TEST_CLEAR_FLAG(readahead, FOLIO_HEAD_PAGE)
606
607FOLIO_FLAG(dropbehind, FOLIO_HEAD_PAGE)
608 FOLIO_TEST_CLEAR_FLAG(dropbehind, FOLIO_HEAD_PAGE)
609 __FOLIO_SET_FLAG(dropbehind, FOLIO_HEAD_PAGE)
610
611#ifdef CONFIG_HIGHMEM
612/*
613 * Must use a macro here due to header dependency issues. page_zone() is not
614 * available at this point.
615 */
616#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
617#define folio_test_highmem(__f) is_highmem_idx(folio_zonenum(__f))
618#else
619PAGEFLAG_FALSE(HighMem, highmem)
620#endif
621#define PhysHighMem(__p) (PageHighMem(phys_to_page(__p)))
622
623/* Does kmap_local_folio() only allow access to one page of the folio? */
624#ifdef CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP
625#define folio_test_partial_kmap(f) true
626#else
627#define folio_test_partial_kmap(f) folio_test_highmem(f)
628#endif
629
630#ifdef CONFIG_SWAP
631static __always_inline bool folio_test_swapcache(const struct folio *folio)
632{
633 return folio_test_swapbacked(folio) &&
634 test_bit(PG_swapcache, const_folio_flags(folio, 0));
635}
636
637FOLIO_SET_FLAG(swapcache, FOLIO_HEAD_PAGE)
638FOLIO_CLEAR_FLAG(swapcache, FOLIO_HEAD_PAGE)
639#else
640FOLIO_FLAG_FALSE(swapcache)
641#endif
642
643FOLIO_FLAG(unevictable, FOLIO_HEAD_PAGE)
644 __FOLIO_CLEAR_FLAG(unevictable, FOLIO_HEAD_PAGE)
645 FOLIO_TEST_CLEAR_FLAG(unevictable, FOLIO_HEAD_PAGE)
646
647#ifdef CONFIG_MMU
648FOLIO_FLAG(mlocked, FOLIO_HEAD_PAGE)
649 __FOLIO_CLEAR_FLAG(mlocked, FOLIO_HEAD_PAGE)
650 FOLIO_TEST_CLEAR_FLAG(mlocked, FOLIO_HEAD_PAGE)
651 FOLIO_TEST_SET_FLAG(mlocked, FOLIO_HEAD_PAGE)
652#else
653FOLIO_FLAG_FALSE(mlocked)
654 __FOLIO_CLEAR_FLAG_NOOP(mlocked)
655 FOLIO_TEST_CLEAR_FLAG_FALSE(mlocked)
656 FOLIO_TEST_SET_FLAG_FALSE(mlocked)
657#endif
658
659#ifdef CONFIG_MEMORY_FAILURE
660PAGEFLAG(HWPoison, hwpoison, PF_ANY)
661TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
662#define __PG_HWPOISON (1UL << PG_hwpoison)
663#else
664PAGEFLAG_FALSE(HWPoison, hwpoison)
665#define __PG_HWPOISON 0
666#endif
667
668#ifdef CONFIG_PAGE_IDLE_FLAG
669#ifdef CONFIG_64BIT
670FOLIO_TEST_FLAG(young, FOLIO_HEAD_PAGE)
671FOLIO_SET_FLAG(young, FOLIO_HEAD_PAGE)
672FOLIO_TEST_CLEAR_FLAG(young, FOLIO_HEAD_PAGE)
673FOLIO_FLAG(idle, FOLIO_HEAD_PAGE)
674#endif
675/* See page_idle.h for !64BIT workaround */
676#else /* !CONFIG_PAGE_IDLE_FLAG */
677FOLIO_FLAG_FALSE(young)
678FOLIO_TEST_CLEAR_FLAG_FALSE(young)
679FOLIO_FLAG_FALSE(idle)
680#endif
681
682/*
683 * PageReported() is used to track reported free pages within the Buddy
684 * allocator. We can use the non-atomic version of the test and set
685 * operations as both should be shielded with the zone lock to prevent
686 * any possible races on the setting or clearing of the bit.
687 */
688__PAGEFLAG(Reported, reported, PF_NO_COMPOUND)
689
690#ifdef CONFIG_MEMORY_HOTPLUG
691PAGEFLAG(VmemmapSelfHosted, vmemmap_self_hosted, PF_ANY)
692#else
693PAGEFLAG_FALSE(VmemmapSelfHosted, vmemmap_self_hosted)
694#endif
695
696/*
697 * On an anonymous folio mapped into a user virtual memory area,
698 * folio->mapping points to its anon_vma, not to a struct address_space;
699 * with the FOLIO_MAPPING_ANON bit set to distinguish it. See rmap.h.
700 *
701 * On an anonymous folio in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
702 * the FOLIO_MAPPING_ANON_KSM bit may be set along with the FOLIO_MAPPING_ANON
703 * bit; and then folio->mapping points, not to an anon_vma, but to a private
704 * structure which KSM associates with that merged folio. See ksm.h.
705 *
706 * Please note that, confusingly, "folio_mapping" refers to the inode
707 * address_space which maps the folio from disk; whereas "folio_mapped"
708 * refers to user virtual address space into which the folio is mapped.
709 *
710 * For slab pages, since slab reuses the bits in struct page to store its
711 * internal states, the folio->mapping does not exist as such, nor do
712 * these flags below. So in order to avoid testing non-existent bits,
713 * please make sure that folio_test_slab(folio) actually evaluates to
714 * false before calling the following functions (e.g., folio_test_anon).
715 * See mm/slab.h.
716 */
717#define FOLIO_MAPPING_ANON 0x1
718#define FOLIO_MAPPING_ANON_KSM 0x2
719#define FOLIO_MAPPING_KSM (FOLIO_MAPPING_ANON | FOLIO_MAPPING_ANON_KSM)
720#define FOLIO_MAPPING_FLAGS (FOLIO_MAPPING_ANON | FOLIO_MAPPING_ANON_KSM)
721
722static __always_inline bool folio_test_anon(const struct folio *folio)
723{
724 return ((unsigned long)folio->mapping & FOLIO_MAPPING_ANON) != 0;
725}
726
727static __always_inline bool PageAnonNotKsm(const struct page *page)
728{
729 unsigned long flags = (unsigned long)page_folio(page)->mapping;
730
731 return (flags & FOLIO_MAPPING_FLAGS) == FOLIO_MAPPING_ANON;
732}
733
734static __always_inline bool PageAnon(const struct page *page)
735{
736 return folio_test_anon(page_folio(page));
737}
738#ifdef CONFIG_KSM
739/*
740 * A KSM page is one of those write-protected "shared pages" or "merged pages"
741 * which KSM maps into multiple mms, wherever identical anonymous page content
742 * is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
743 * anon_vma, but to that page's node of the stable tree.
744 */
745static __always_inline bool folio_test_ksm(const struct folio *folio)
746{
747 return ((unsigned long)folio->mapping & FOLIO_MAPPING_FLAGS) ==
748 FOLIO_MAPPING_KSM;
749}
750#else
751FOLIO_TEST_FLAG_FALSE(ksm)
752#endif
753
754u64 stable_page_flags(const struct page *page);
755
756/**
757 * folio_xor_flags_has_waiters - Change some folio flags.
758 * @folio: The folio.
759 * @mask: Bits set in this word will be changed.
760 *
761 * This must only be used for flags which are changed with the folio
762 * lock held. For example, it is unsafe to use for PG_dirty as that
763 * can be set without the folio lock held. It can also only be used
764 * on flags which are in the range 0-6 as some of the implementations
765 * only affect those bits.
766 *
767 * Return: Whether there are tasks waiting on the folio.
768 */
769static inline bool folio_xor_flags_has_waiters(struct folio *folio,
770 unsigned long mask)
771{
772 return xor_unlock_is_negative_byte(mask, addr: folio_flags(folio, n: 0));
773}
774
775/**
776 * folio_test_uptodate - Is this folio up to date?
777 * @folio: The folio.
778 *
779 * The uptodate flag is set on a folio when every byte in the folio is
780 * at least as new as the corresponding bytes on storage. Anonymous
781 * and CoW folios are always uptodate. If the folio is not uptodate,
782 * some of the bytes in it may be; see the is_partially_uptodate()
783 * address_space operation.
784 */
785static inline bool folio_test_uptodate(const struct folio *folio)
786{
787 bool ret = test_bit(PG_uptodate, const_folio_flags(folio, 0));
788 /*
789 * Must ensure that the data we read out of the folio is loaded
790 * _after_ we've loaded folio->flags to check the uptodate bit.
791 * We can skip the barrier if the folio is not uptodate, because
792 * we wouldn't be reading anything from it.
793 *
794 * See folio_mark_uptodate() for the other side of the story.
795 */
796 if (ret)
797 smp_rmb();
798
799 return ret;
800}
801
802static inline bool PageUptodate(const struct page *page)
803{
804 return folio_test_uptodate(page_folio(page));
805}
806
807static __always_inline void __folio_mark_uptodate(struct folio *folio)
808{
809 smp_wmb();
810 __set_bit(PG_uptodate, folio_flags(folio, 0));
811}
812
813static __always_inline void folio_mark_uptodate(struct folio *folio)
814{
815 /*
816 * Memory barrier must be issued before setting the PG_uptodate bit,
817 * so that all previous stores issued in order to bring the folio
818 * uptodate are actually visible before folio_test_uptodate becomes true.
819 */
820 smp_wmb();
821 set_bit(nr: PG_uptodate, addr: folio_flags(folio, n: 0));
822}
823
824static __always_inline void __SetPageUptodate(struct page *page)
825{
826 __folio_mark_uptodate(folio: (struct folio *)page);
827}
828
829static __always_inline void SetPageUptodate(struct page *page)
830{
831 folio_mark_uptodate(folio: (struct folio *)page);
832}
833
834CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
835
836void __folio_start_writeback(struct folio *folio, bool keep_write);
837void set_page_writeback(struct page *page);
838
839#define folio_start_writeback(folio) \
840 __folio_start_writeback(folio, false)
841
842static __always_inline bool folio_test_head(const struct folio *folio)
843{
844 return test_bit(PG_head, const_folio_flags(folio, FOLIO_PF_ANY));
845}
846
847static __always_inline int PageHead(const struct page *page)
848{
849 PF_POISONED_CHECK(page);
850 return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page);
851}
852
853__SETPAGEFLAG(Head, head, PF_ANY)
854__CLEARPAGEFLAG(Head, head, PF_ANY)
855CLEARPAGEFLAG(Head, head, PF_ANY)
856
857/**
858 * folio_test_large() - Does this folio contain more than one page?
859 * @folio: The folio to test.
860 *
861 * Return: True if the folio is larger than one page.
862 */
863static inline bool folio_test_large(const struct folio *folio)
864{
865 return folio_test_head(folio);
866}
867
868static __always_inline void set_compound_head(struct page *page, struct page *head)
869{
870 WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
871}
872
873static __always_inline void clear_compound_head(struct page *page)
874{
875 WRITE_ONCE(page->compound_head, 0);
876}
877
878#ifdef CONFIG_TRANSPARENT_HUGEPAGE
879static inline void ClearPageCompound(struct page *page)
880{
881 BUG_ON(!PageHead(page));
882 ClearPageHead(page);
883}
884FOLIO_FLAG(large_rmappable, FOLIO_SECOND_PAGE)
885FOLIO_FLAG(partially_mapped, FOLIO_SECOND_PAGE)
886#else
887FOLIO_FLAG_FALSE(large_rmappable)
888FOLIO_FLAG_FALSE(partially_mapped)
889#endif
890
891#define PG_head_mask ((1UL << PG_head))
892
893#ifdef CONFIG_TRANSPARENT_HUGEPAGE
894/*
895 * PageTransCompound returns true for both transparent huge pages
896 * and hugetlbfs pages, so it should only be called when it's known
897 * that hugetlbfs pages aren't involved.
898 */
899static inline int PageTransCompound(const struct page *page)
900{
901 return PageCompound(page);
902}
903#else
904TESTPAGEFLAG_FALSE(TransCompound, transcompound)
905#endif
906
907#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
908/*
909 * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the
910 * compound page.
911 *
912 * This flag is set by hwpoison handler. Cleared by THP split or free page.
913 */
914FOLIO_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE)
915#else
916FOLIO_FLAG_FALSE(has_hwpoisoned)
917#endif
918
919/*
920 * For pages that do not use mapcount, page_type may be used.
921 * The low 24 bits of pagetype may be used for your own purposes, as long
922 * as you are careful to not affect the top 8 bits. The low bits of
923 * pagetype will be overwritten when you clear the page_type from the page.
924 */
925enum pagetype {
926 /* 0x00-0x7f are positive numbers, ie mapcount */
927 /* Reserve 0x80-0xef for mapcount overflow. */
928 PGTY_buddy = 0xf0,
929 PGTY_offline = 0xf1,
930 PGTY_table = 0xf2,
931 PGTY_guard = 0xf3,
932 PGTY_hugetlb = 0xf4,
933 PGTY_slab = 0xf5,
934 PGTY_zsmalloc = 0xf6,
935 PGTY_unaccepted = 0xf7,
936 PGTY_large_kmalloc = 0xf8,
937
938 PGTY_mapcount_underflow = 0xff
939};
940
941static inline bool page_type_has_type(int page_type)
942{
943 return page_type < (PGTY_mapcount_underflow << 24);
944}
945
946/* This takes a mapcount which is one more than page->_mapcount */
947static inline bool page_mapcount_is_type(unsigned int mapcount)
948{
949 return page_type_has_type(page_type: mapcount - 1);
950}
951
952static inline bool page_has_type(const struct page *page)
953{
954 return page_type_has_type(data_race(page->page_type));
955}
956
957#define FOLIO_TYPE_OPS(lname, fname) \
958static __always_inline bool folio_test_##fname(const struct folio *folio) \
959{ \
960 return data_race(folio->page.page_type >> 24) == PGTY_##lname; \
961} \
962static __always_inline void __folio_set_##fname(struct folio *folio) \
963{ \
964 if (folio_test_##fname(folio)) \
965 return; \
966 VM_BUG_ON_FOLIO(data_race(folio->page.page_type) != UINT_MAX, \
967 folio); \
968 folio->page.page_type = (unsigned int)PGTY_##lname << 24; \
969} \
970static __always_inline void __folio_clear_##fname(struct folio *folio) \
971{ \
972 if (folio->page.page_type == UINT_MAX) \
973 return; \
974 VM_BUG_ON_FOLIO(!folio_test_##fname(folio), folio); \
975 folio->page.page_type = UINT_MAX; \
976}
977
978#define PAGE_TYPE_OPS(uname, lname, fname) \
979FOLIO_TYPE_OPS(lname, fname) \
980static __always_inline int Page##uname(const struct page *page) \
981{ \
982 return data_race(page->page_type >> 24) == PGTY_##lname; \
983} \
984static __always_inline void __SetPage##uname(struct page *page) \
985{ \
986 if (Page##uname(page)) \
987 return; \
988 VM_BUG_ON_PAGE(data_race(page->page_type) != UINT_MAX, page); \
989 page->page_type = (unsigned int)PGTY_##lname << 24; \
990} \
991static __always_inline void __ClearPage##uname(struct page *page) \
992{ \
993 if (page->page_type == UINT_MAX) \
994 return; \
995 VM_BUG_ON_PAGE(!Page##uname(page), page); \
996 page->page_type = UINT_MAX; \
997}
998
999/*
1000 * PageBuddy() indicates that the page is free and in the buddy system
1001 * (see mm/page_alloc.c).
1002 */
1003PAGE_TYPE_OPS(Buddy, buddy, buddy)
1004
1005/*
1006 * PageOffline() indicates that the page is logically offline although the
1007 * containing section is online. (e.g. inflated in a balloon driver or
1008 * not onlined when onlining the section).
1009 * The content of these pages is effectively stale. Such pages should not
1010 * be touched (read/write/dump/save) except by their owner.
1011 *
1012 * When a memory block gets onlined, all pages are initialized with a
1013 * refcount of 1 and PageOffline(). generic_online_page() will
1014 * take care of clearing PageOffline().
1015 *
1016 * If a driver wants to allow to offline unmovable PageOffline() pages without
1017 * putting them back to the buddy, it can do so via the memory notifier by
1018 * decrementing the reference count in MEM_GOING_OFFLINE and incrementing the
1019 * reference count in MEM_CANCEL_OFFLINE. When offlining, the PageOffline()
1020 * pages (now with a reference count of zero) are treated like free (unmanaged)
1021 * pages, allowing the containing memory block to get offlined. A driver that
1022 * relies on this feature is aware that re-onlining the memory block will
1023 * require not giving them to the buddy via generic_online_page().
1024 *
1025 * Memory offlining code will not adjust the managed page count for any
1026 * PageOffline() pages, treating them like they were never exposed to the
1027 * buddy using generic_online_page().
1028 *
1029 * There are drivers that mark a page PageOffline() and expect there won't be
1030 * any further access to page content. PFN walkers that read content of random
1031 * pages should check PageOffline() and synchronize with such drivers using
1032 * page_offline_freeze()/page_offline_thaw().
1033 */
1034PAGE_TYPE_OPS(Offline, offline, offline)
1035
1036extern void page_offline_freeze(void);
1037extern void page_offline_thaw(void);
1038extern void page_offline_begin(void);
1039extern void page_offline_end(void);
1040
1041/*
1042 * Marks pages in use as page tables.
1043 */
1044PAGE_TYPE_OPS(Table, table, pgtable)
1045
1046/*
1047 * Marks guardpages used with debug_pagealloc.
1048 */
1049PAGE_TYPE_OPS(Guard, guard, guard)
1050
1051FOLIO_TYPE_OPS(slab, slab)
1052
1053/**
1054 * PageSlab - Determine if the page belongs to the slab allocator
1055 * @page: The page to test.
1056 *
1057 * Context: Any context.
1058 * Return: True for slab pages, false for any other kind of page.
1059 */
1060static inline bool PageSlab(const struct page *page)
1061{
1062 return folio_test_slab(page_folio(page));
1063}
1064
1065#ifdef CONFIG_HUGETLB_PAGE
1066FOLIO_TYPE_OPS(hugetlb, hugetlb)
1067#else
1068FOLIO_TEST_FLAG_FALSE(hugetlb)
1069#endif
1070
1071PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
1072
1073/*
1074 * Mark pages that has to be accepted before touched for the first time.
1075 *
1076 * Serialized with zone lock.
1077 */
1078PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
1079FOLIO_TYPE_OPS(large_kmalloc, large_kmalloc)
1080
1081/**
1082 * PageHuge - Determine if the page belongs to hugetlbfs
1083 * @page: The page to test.
1084 *
1085 * Context: Any context.
1086 * Return: True for hugetlbfs pages, false for anon pages or pages
1087 * belonging to other filesystems.
1088 */
1089static inline bool PageHuge(const struct page *page)
1090{
1091 return folio_test_hugetlb(page_folio(page));
1092}
1093
1094/*
1095 * Check if a page is currently marked HWPoisoned. Note that this check is
1096 * best effort only and inherently racy: there is no way to synchronize with
1097 * failing hardware.
1098 */
1099static inline bool is_page_hwpoison(const struct page *page)
1100{
1101 const struct folio *folio;
1102
1103 if (PageHWPoison(page))
1104 return true;
1105 folio = page_folio(page);
1106 return folio_test_hugetlb(folio) && PageHWPoison(page: &folio->page);
1107}
1108
1109static inline bool folio_contain_hwpoisoned_page(struct folio *folio)
1110{
1111 return folio_test_hwpoison(folio) ||
1112 (folio_test_large(folio) && folio_test_has_hwpoisoned(folio));
1113}
1114
1115bool is_free_buddy_page(const struct page *page);
1116
1117#ifdef CONFIG_MIGRATION
1118/*
1119 * This page is migratable through movable_ops (for selected typed pages
1120 * only).
1121 *
1122 * Page migration of such pages might fail, for example, if the page is
1123 * already isolated by somebody else, or if the page is about to get freed.
1124 *
1125 * While a subsystem might set selected typed pages that support page migration
1126 * as being movable through movable_ops, it must never clear this flag.
1127 *
1128 * This flag is only cleared when the page is freed back to the buddy.
1129 *
1130 * Only selected page types support this flag (see page_movable_ops()) and
1131 * the flag might be used in other context for other pages. Always use
1132 * page_has_movable_ops() instead.
1133 */
1134TESTPAGEFLAG(MovableOps, movable_ops, PF_NO_TAIL);
1135SETPAGEFLAG(MovableOps, movable_ops, PF_NO_TAIL);
1136/*
1137 * A movable_ops page has this flag set while it is isolated for migration.
1138 * This flag primarily protects against concurrent migration attempts.
1139 *
1140 * Once migration ended (success or failure), the flag is cleared. The
1141 * flag is managed by the migration core.
1142 */
1143PAGEFLAG(MovableOpsIsolated, movable_ops_isolated, PF_NO_TAIL);
1144#else /* !CONFIG_MIGRATION */
1145TESTPAGEFLAG_FALSE(MovableOps, movable_ops);
1146SETPAGEFLAG_NOOP(MovableOps, movable_ops);
1147PAGEFLAG_FALSE(MovableOpsIsolated, movable_ops_isolated);
1148#endif /* CONFIG_MIGRATION */
1149
1150/**
1151 * page_has_movable_ops - test for a movable_ops page
1152 * @page: The page to test.
1153 *
1154 * Test whether this is a movable_ops page. Such pages will stay that
1155 * way until freed.
1156 *
1157 * Returns true if this is a movable_ops page, otherwise false.
1158 */
1159static inline bool page_has_movable_ops(const struct page *page)
1160{
1161 return PageMovableOps(page) &&
1162 (PageOffline(page) || PageZsmalloc(page));
1163}
1164
1165static __always_inline int PageAnonExclusive(const struct page *page)
1166{
1167 VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1168 /*
1169 * HugeTLB stores this information on the head page; THP keeps it per
1170 * page
1171 */
1172 if (PageHuge(page))
1173 page = compound_head(page);
1174 return test_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags.f);
1175}
1176
1177static __always_inline void SetPageAnonExclusive(struct page *page)
1178{
1179 VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);
1180 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1181 set_bit(nr: PG_anon_exclusive, addr: &PF_ANY(page, 1)->flags.f);
1182}
1183
1184static __always_inline void ClearPageAnonExclusive(struct page *page)
1185{
1186 VM_BUG_ON_PGFLAGS(!PageAnonNotKsm(page), page);
1187 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1188 clear_bit(nr: PG_anon_exclusive, addr: &PF_ANY(page, 1)->flags.f);
1189}
1190
1191static __always_inline void __ClearPageAnonExclusive(struct page *page)
1192{
1193 VM_BUG_ON_PGFLAGS(!PageAnon(page), page);
1194 VM_BUG_ON_PGFLAGS(PageHuge(page) && !PageHead(page), page);
1195 __clear_bit(PG_anon_exclusive, &PF_ANY(page, 1)->flags.f);
1196}
1197
1198#ifdef CONFIG_MMU
1199#define __PG_MLOCKED (1UL << PG_mlocked)
1200#else
1201#define __PG_MLOCKED 0
1202#endif
1203
1204/*
1205 * Flags checked when a page is freed. Pages being freed should not have
1206 * these flags set. If they are, there is a problem.
1207 */
1208#define PAGE_FLAGS_CHECK_AT_FREE \
1209 (1UL << PG_lru | 1UL << PG_locked | \
1210 1UL << PG_private | 1UL << PG_private_2 | \
1211 1UL << PG_writeback | 1UL << PG_reserved | \
1212 1UL << PG_active | \
1213 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK)
1214
1215/*
1216 * Flags checked when a page is prepped for return by the page allocator.
1217 * Pages being prepped should not have these flags set. If they are set,
1218 * there has been a kernel bug or struct page corruption.
1219 *
1220 * __PG_HWPOISON is exceptional because it needs to be kept beyond page's
1221 * alloc-free cycle to prevent from reusing the page.
1222 */
1223#define PAGE_FLAGS_CHECK_AT_PREP \
1224 ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK)
1225
1226/*
1227 * Flags stored in the second page of a compound page. They may overlap
1228 * the CHECK_AT_FREE flags above, so need to be cleared.
1229 */
1230#define PAGE_FLAGS_SECOND \
1231 (0xffUL /* order */ | 1UL << PG_has_hwpoisoned | \
1232 1UL << PG_large_rmappable | 1UL << PG_partially_mapped)
1233
1234#define PAGE_FLAGS_PRIVATE \
1235 (1UL << PG_private | 1UL << PG_private_2)
1236/**
1237 * folio_has_private - Determine if folio has private stuff
1238 * @folio: The folio to be checked
1239 *
1240 * Determine if a folio has private stuff, indicating that release routines
1241 * should be invoked upon it.
1242 */
1243static inline int folio_has_private(const struct folio *folio)
1244{
1245 return !!(folio->flags.f & PAGE_FLAGS_PRIVATE);
1246}
1247
1248#undef PF_ANY
1249#undef PF_HEAD
1250#undef PF_NO_TAIL
1251#undef PF_NO_COMPOUND
1252#undef PF_SECOND
1253#endif /* !__GENERATING_BOUNDS_H */
1254
1255#endif /* PAGE_FLAGS_H */
1256