1// SPDX-License-Identifier: GPL-2.0
2/*
3 * sparse memory mappings.
4 */
5#include <linux/mm.h>
6#include <linux/slab.h>
7#include <linux/mmzone.h>
8#include <linux/memblock.h>
9#include <linux/compiler.h>
10#include <linux/highmem.h>
11#include <linux/export.h>
12#include <linux/spinlock.h>
13#include <linux/vmalloc.h>
14#include <linux/swap.h>
15#include <linux/swapops.h>
16#include <linux/bootmem_info.h>
17#include <linux/vmstat.h>
18#include "internal.h"
19#include <asm/dma.h>
20
21/*
22 * Permanent SPARSEMEM data:
23 *
24 * 1) mem_section - memory sections, mem_map's for valid memory
25 */
26#ifdef CONFIG_SPARSEMEM_EXTREME
27struct mem_section **mem_section;
28#else
29struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
30 ____cacheline_internodealigned_in_smp;
31#endif
32EXPORT_SYMBOL(mem_section);
33
34#ifdef NODE_NOT_IN_PAGE_FLAGS
35/*
36 * If we did not store the node number in the page then we have to
37 * do a lookup in the section_to_node_table in order to find which
38 * node the page belongs to.
39 */
40#if MAX_NUMNODES <= 256
41static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
42#else
43static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
44#endif
45
46int memdesc_nid(memdesc_flags_t mdf)
47{
48 return section_to_node_table[memdesc_section(mdf)];
49}
50EXPORT_SYMBOL(memdesc_nid);
51
52static void set_section_nid(unsigned long section_nr, int nid)
53{
54 section_to_node_table[section_nr] = nid;
55}
56#else /* !NODE_NOT_IN_PAGE_FLAGS */
57static inline void set_section_nid(unsigned long section_nr, int nid)
58{
59}
60#endif
61
62#ifdef CONFIG_SPARSEMEM_EXTREME
63static noinline struct mem_section __ref *sparse_index_alloc(int nid)
64{
65 struct mem_section *section = NULL;
66 unsigned long array_size = SECTIONS_PER_ROOT *
67 sizeof(struct mem_section);
68
69 if (slab_is_available()) {
70 section = kzalloc_node(array_size, GFP_KERNEL, nid);
71 } else {
72 section = memblock_alloc_node(size: array_size, SMP_CACHE_BYTES,
73 nid);
74 if (!section)
75 panic(fmt: "%s: Failed to allocate %lu bytes nid=%d\n",
76 __func__, array_size, nid);
77 }
78
79 return section;
80}
81
82static int __meminit sparse_index_init(unsigned long section_nr, int nid)
83{
84 unsigned long root = SECTION_NR_TO_ROOT(section_nr);
85 struct mem_section *section;
86
87 /*
88 * An existing section is possible in the sub-section hotplug
89 * case. First hot-add instantiates, follow-on hot-add reuses
90 * the existing section.
91 *
92 * The mem_hotplug_lock resolves the apparent race below.
93 */
94 if (mem_section[root])
95 return 0;
96
97 section = sparse_index_alloc(nid);
98 if (!section)
99 return -ENOMEM;
100
101 mem_section[root] = section;
102
103 return 0;
104}
105#else /* !SPARSEMEM_EXTREME */
106static inline int sparse_index_init(unsigned long section_nr, int nid)
107{
108 return 0;
109}
110#endif
111
112/*
113 * During early boot, before section_mem_map is used for an actual
114 * mem_map, we use section_mem_map to store the section's NUMA
115 * node. This keeps us from having to use another data structure. The
116 * node information is cleared just before we store the real mem_map.
117 */
118static inline unsigned long sparse_encode_early_nid(int nid)
119{
120 return ((unsigned long)nid << SECTION_NID_SHIFT);
121}
122
123static inline int sparse_early_nid(struct mem_section *section)
124{
125 return (section->section_mem_map >> SECTION_NID_SHIFT);
126}
127
128/* Validate the physical addressing limitations of the model */
129static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
130 unsigned long *end_pfn)
131{
132 unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT;
133
134 /*
135 * Sanity checks - do not allow an architecture to pass
136 * in larger pfns than the maximum scope of sparsemem:
137 */
138 if (*start_pfn > max_sparsemem_pfn) {
139 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
141 *start_pfn, *end_pfn, max_sparsemem_pfn);
142 WARN_ON_ONCE(1);
143 *start_pfn = max_sparsemem_pfn;
144 *end_pfn = max_sparsemem_pfn;
145 } else if (*end_pfn > max_sparsemem_pfn) {
146 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
148 *start_pfn, *end_pfn, max_sparsemem_pfn);
149 WARN_ON_ONCE(1);
150 *end_pfn = max_sparsemem_pfn;
151 }
152}
153
154/*
155 * There are a number of times that we loop over NR_MEM_SECTIONS,
156 * looking for section_present() on each. But, when we have very
157 * large physical address spaces, NR_MEM_SECTIONS can also be
158 * very large which makes the loops quite long.
159 *
160 * Keeping track of this gives us an easy way to break out of
161 * those loops early.
162 */
163unsigned long __highest_present_section_nr;
164static void __section_mark_present(struct mem_section *ms,
165 unsigned long section_nr)
166{
167 if (section_nr > __highest_present_section_nr)
168 __highest_present_section_nr = section_nr;
169
170 ms->section_mem_map |= SECTION_MARKED_PRESENT;
171}
172
173static inline unsigned long first_present_section_nr(void)
174{
175 return next_present_section_nr(section_nr: -1);
176}
177
178#ifdef CONFIG_SPARSEMEM_VMEMMAP
179static void subsection_mask_set(unsigned long *map, unsigned long pfn,
180 unsigned long nr_pages)
181{
182 int idx = subsection_map_index(pfn);
183 int end = subsection_map_index(pfn: pfn + nr_pages - 1);
184
185 bitmap_set(map, start: idx, nbits: end - idx + 1);
186}
187
188void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages)
189{
190 int end_sec_nr = pfn_to_section_nr(pfn: pfn + nr_pages - 1);
191 unsigned long nr, start_sec_nr = pfn_to_section_nr(pfn);
192
193 for (nr = start_sec_nr; nr <= end_sec_nr; nr++) {
194 struct mem_section *ms;
195 unsigned long pfns;
196
197 pfns = min(nr_pages, PAGES_PER_SECTION
198 - (pfn & ~PAGE_SECTION_MASK));
199 ms = __nr_to_section(nr);
200 subsection_mask_set(map: ms->usage->subsection_map, pfn, nr_pages: pfns);
201
202 pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr,
203 pfns, subsection_map_index(pfn),
204 subsection_map_index(pfn + pfns - 1));
205
206 pfn += pfns;
207 nr_pages -= pfns;
208 }
209}
210#else
211void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages)
212{
213}
214#endif
215
216/* Record a memory area against a node. */
217static void __init memory_present(int nid, unsigned long start, unsigned long end)
218{
219 unsigned long pfn;
220
221 start &= PAGE_SECTION_MASK;
222 mminit_validate_memmodel_limits(start_pfn: &start, end_pfn: &end);
223 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
224 unsigned long section_nr = pfn_to_section_nr(pfn);
225 struct mem_section *ms;
226
227 sparse_index_init(section_nr, nid);
228 set_section_nid(section_nr, nid);
229
230 ms = __nr_to_section(nr: section_nr);
231 if (!ms->section_mem_map) {
232 ms->section_mem_map = sparse_encode_early_nid(nid) |
233 SECTION_IS_ONLINE;
234 __section_mark_present(ms, section_nr);
235 }
236 }
237}
238
239/*
240 * Mark all memblocks as present using memory_present().
241 * This is a convenience function that is useful to mark all of the systems
242 * memory as present during initialization.
243 */
244static void __init memblocks_present(void)
245{
246 unsigned long start, end;
247 int i, nid;
248
249#ifdef CONFIG_SPARSEMEM_EXTREME
250 if (unlikely(!mem_section)) {
251 unsigned long size, align;
252
253 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
254 align = 1 << (INTERNODE_CACHE_SHIFT);
255 mem_section = memblock_alloc_or_panic(size, align);
256 }
257#endif
258
259 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
260 memory_present(nid, start, end);
261}
262
263/*
264 * Subtle, we encode the real pfn into the mem_map such that
265 * the identity pfn - section_mem_map will return the actual
266 * physical page frame number.
267 */
268static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
269{
270 unsigned long coded_mem_map =
271 (unsigned long)(mem_map - (section_nr_to_pfn(sec: pnum)));
272 BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT);
273 BUG_ON(coded_mem_map & ~SECTION_MAP_MASK);
274 return coded_mem_map;
275}
276
277#ifdef CONFIG_MEMORY_HOTPLUG
278/*
279 * Decode mem_map from the coded memmap
280 */
281struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
282{
283 /* mask off the extra low bits of information */
284 coded_mem_map &= SECTION_MAP_MASK;
285 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
286}
287#endif /* CONFIG_MEMORY_HOTPLUG */
288
289static void __meminit sparse_init_one_section(struct mem_section *ms,
290 unsigned long pnum, struct page *mem_map,
291 struct mem_section_usage *usage, unsigned long flags)
292{
293 ms->section_mem_map &= ~SECTION_MAP_MASK;
294 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum)
295 | SECTION_HAS_MEM_MAP | flags;
296 ms->usage = usage;
297}
298
299static unsigned long usemap_size(void)
300{
301 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);
302}
303
304size_t mem_section_usage_size(void)
305{
306 return sizeof(struct mem_section_usage) + usemap_size();
307}
308
309#ifdef CONFIG_MEMORY_HOTREMOVE
310static inline phys_addr_t pgdat_to_phys(struct pglist_data *pgdat)
311{
312#ifndef CONFIG_NUMA
313 VM_BUG_ON(pgdat != &contig_page_data);
314 return __pa_symbol(&contig_page_data);
315#else
316 return __pa(pgdat);
317#endif
318}
319
320static struct mem_section_usage * __init
321sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
322 unsigned long size)
323{
324 struct mem_section_usage *usage;
325 unsigned long goal, limit;
326 int nid;
327 /*
328 * A page may contain usemaps for other sections preventing the
329 * page being freed and making a section unremovable while
330 * other sections referencing the usemap remain active. Similarly,
331 * a pgdat can prevent a section being removed. If section A
332 * contains a pgdat and section B contains the usemap, both
333 * sections become inter-dependent. This allocates usemaps
334 * from the same section as the pgdat where possible to avoid
335 * this problem.
336 */
337 goal = pgdat_to_phys(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT);
338 limit = goal + (1UL << PA_SECTION_SHIFT);
339 nid = early_pfn_to_nid(goal >> PAGE_SHIFT);
340again:
341 usage = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid);
342 if (!usage && limit) {
343 limit = MEMBLOCK_ALLOC_ACCESSIBLE;
344 goto again;
345 }
346 return usage;
347}
348
349static void __init check_usemap_section_nr(int nid,
350 struct mem_section_usage *usage)
351{
352 unsigned long usemap_snr, pgdat_snr;
353 static unsigned long old_usemap_snr;
354 static unsigned long old_pgdat_snr;
355 struct pglist_data *pgdat = NODE_DATA(nid);
356 int usemap_nid;
357
358 /* First call */
359 if (!old_usemap_snr) {
360 old_usemap_snr = NR_MEM_SECTIONS;
361 old_pgdat_snr = NR_MEM_SECTIONS;
362 }
363
364 usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT);
365 pgdat_snr = pfn_to_section_nr(pgdat_to_phys(pgdat) >> PAGE_SHIFT);
366 if (usemap_snr == pgdat_snr)
367 return;
368
369 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)
370 /* skip redundant message */
371 return;
372
373 old_usemap_snr = usemap_snr;
374 old_pgdat_snr = pgdat_snr;
375
376 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
377 if (usemap_nid != nid) {
378 pr_info("node %d must be removed before remove section %ld\n",
379 nid, usemap_snr);
380 return;
381 }
382 /*
383 * There is a circular dependency.
384 * Some platforms allow un-removable section because they will just
385 * gather other removable sections for dynamic partitioning.
386 * Just notify un-removable section's number here.
387 */
388 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n",
389 usemap_snr, pgdat_snr, nid);
390}
391#else
392static struct mem_section_usage * __init
393sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
394 unsigned long size)
395{
396 return memblock_alloc_node(size, SMP_CACHE_BYTES, nid: pgdat->node_id);
397}
398
399static void __init check_usemap_section_nr(int nid,
400 struct mem_section_usage *usage)
401{
402}
403#endif /* CONFIG_MEMORY_HOTREMOVE */
404
405#ifdef CONFIG_SPARSEMEM_VMEMMAP
406unsigned long __init section_map_size(void)
407{
408 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
409}
410
411#else
412unsigned long __init section_map_size(void)
413{
414 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
415}
416
417struct page __init *__populate_section_memmap(unsigned long pfn,
418 unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
419 struct dev_pagemap *pgmap)
420{
421 unsigned long size = section_map_size();
422 struct page *map = sparse_buffer_alloc(size);
423 phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
424
425 if (map)
426 return map;
427
428 map = memmap_alloc(size, size, addr, nid, false);
429 if (!map)
430 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
431 __func__, size, PAGE_SIZE, nid, &addr);
432
433 return map;
434}
435#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
436
437static void *sparsemap_buf __meminitdata;
438static void *sparsemap_buf_end __meminitdata;
439
440static inline void __meminit sparse_buffer_free(unsigned long size)
441{
442 WARN_ON(!sparsemap_buf || size == 0);
443 memblock_free(ptr: sparsemap_buf, size);
444}
445
446static void __init sparse_buffer_init(unsigned long size, int nid)
447{
448 phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
449 WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */
450 /*
451 * Pre-allocated buffer is mainly used by __populate_section_memmap
452 * and we want it to be properly aligned to the section size - this is
453 * especially the case for VMEMMAP which maps memmap to PMDs
454 */
455 sparsemap_buf = memmap_alloc(size, align: section_map_size(), min_addr: addr, nid, exact_nid: true);
456 sparsemap_buf_end = sparsemap_buf + size;
457}
458
459static void __init sparse_buffer_fini(void)
460{
461 unsigned long size = sparsemap_buf_end - sparsemap_buf;
462
463 if (sparsemap_buf && size > 0)
464 sparse_buffer_free(size);
465 sparsemap_buf = NULL;
466}
467
468void * __meminit sparse_buffer_alloc(unsigned long size)
469{
470 void *ptr = NULL;
471
472 if (sparsemap_buf) {
473 ptr = (void *) roundup((unsigned long)sparsemap_buf, size);
474 if (ptr + size > sparsemap_buf_end)
475 ptr = NULL;
476 else {
477 /* Free redundant aligned space */
478 if ((unsigned long)(ptr - sparsemap_buf) > 0)
479 sparse_buffer_free(size: (unsigned long)(ptr - sparsemap_buf));
480 sparsemap_buf = ptr + size;
481 }
482 }
483 return ptr;
484}
485
486void __weak __meminit vmemmap_populate_print_last(void)
487{
488}
489
490static void *sparse_usagebuf __meminitdata;
491static void *sparse_usagebuf_end __meminitdata;
492
493/*
494 * Helper function that is used for generic section initialization, and
495 * can also be used by any hooks added above.
496 */
497void __init sparse_init_early_section(int nid, struct page *map,
498 unsigned long pnum, unsigned long flags)
499{
500 BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);
501 check_usemap_section_nr(nid, usage: sparse_usagebuf);
502 sparse_init_one_section(ms: __nr_to_section(nr: pnum), pnum, mem_map: map,
503 usage: sparse_usagebuf, SECTION_IS_EARLY | flags);
504 sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();
505}
506
507static int __init sparse_usage_init(int nid, unsigned long map_count)
508{
509 unsigned long size;
510
511 size = mem_section_usage_size() * map_count;
512 sparse_usagebuf = sparse_early_usemaps_alloc_pgdat_section(
513 NODE_DATA(nid), size);
514 if (!sparse_usagebuf) {
515 sparse_usagebuf_end = NULL;
516 return -ENOMEM;
517 }
518
519 sparse_usagebuf_end = sparse_usagebuf + size;
520 return 0;
521}
522
523static void __init sparse_usage_fini(void)
524{
525 sparse_usagebuf = sparse_usagebuf_end = NULL;
526}
527
528/*
529 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
530 * And number of present sections in this node is map_count.
531 */
532static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
533 unsigned long pnum_end,
534 unsigned long map_count)
535{
536 unsigned long pnum;
537 struct page *map;
538 struct mem_section *ms;
539
540 if (sparse_usage_init(nid, map_count)) {
541 pr_err("%s: node[%d] usemap allocation failed", __func__, nid);
542 goto failed;
543 }
544
545 sparse_buffer_init(size: map_count * section_map_size(), nid);
546
547 sparse_vmemmap_init_nid_early(nid);
548
549 for_each_present_section_nr(pnum_begin, pnum) {
550 unsigned long pfn = section_nr_to_pfn(sec: pnum);
551
552 if (pnum >= pnum_end)
553 break;
554
555 ms = __nr_to_section(nr: pnum);
556 if (!preinited_vmemmap_section(section: ms)) {
557 map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
558 nid, NULL, NULL);
559 if (!map) {
560 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
561 __func__, nid);
562 pnum_begin = pnum;
563 sparse_usage_fini();
564 sparse_buffer_fini();
565 goto failed;
566 }
567 memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
568 PAGE_SIZE));
569 sparse_init_early_section(nid, map, pnum, flags: 0);
570 }
571 }
572 sparse_usage_fini();
573 sparse_buffer_fini();
574 return;
575failed:
576 /*
577 * We failed to allocate, mark all the following pnums as not present,
578 * except the ones already initialized earlier.
579 */
580 for_each_present_section_nr(pnum_begin, pnum) {
581 if (pnum >= pnum_end)
582 break;
583 ms = __nr_to_section(nr: pnum);
584 if (!preinited_vmemmap_section(section: ms))
585 ms->section_mem_map = 0;
586 ms->section_mem_map = 0;
587 }
588}
589
590/*
591 * Allocate the accumulated non-linear sections, allocate a mem_map
592 * for each and record the physical to section mapping.
593 */
594void __init sparse_init(void)
595{
596 unsigned long pnum_end, pnum_begin, map_count = 1;
597 int nid_begin;
598
599 /* see include/linux/mmzone.h 'struct mem_section' definition */
600 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
601 memblocks_present();
602
603 pnum_begin = first_present_section_nr();
604 nid_begin = sparse_early_nid(section: __nr_to_section(nr: pnum_begin));
605
606 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
607 set_pageblock_order();
608
609 for_each_present_section_nr(pnum_begin + 1, pnum_end) {
610 int nid = sparse_early_nid(section: __nr_to_section(nr: pnum_end));
611
612 if (nid == nid_begin) {
613 map_count++;
614 continue;
615 }
616 /* Init node with sections in range [pnum_begin, pnum_end) */
617 sparse_init_nid(nid: nid_begin, pnum_begin, pnum_end, map_count);
618 nid_begin = nid;
619 pnum_begin = pnum_end;
620 map_count = 1;
621 }
622 /* cover the last node */
623 sparse_init_nid(nid: nid_begin, pnum_begin, pnum_end, map_count);
624 vmemmap_populate_print_last();
625}
626
627#ifdef CONFIG_MEMORY_HOTPLUG
628
629/* Mark all memory sections within the pfn range as online */
630void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
631{
632 unsigned long pfn;
633
634 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
635 unsigned long section_nr = pfn_to_section_nr(pfn);
636 struct mem_section *ms;
637
638 /* onlining code should never touch invalid ranges */
639 if (WARN_ON(!valid_section_nr(section_nr)))
640 continue;
641
642 ms = __nr_to_section(section_nr);
643 ms->section_mem_map |= SECTION_IS_ONLINE;
644 }
645}
646
647/* Mark all memory sections within the pfn range as offline */
648void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
649{
650 unsigned long pfn;
651
652 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
653 unsigned long section_nr = pfn_to_section_nr(pfn);
654 struct mem_section *ms;
655
656 /*
657 * TODO this needs some double checking. Offlining code makes
658 * sure to check pfn_valid but those checks might be just bogus
659 */
660 if (WARN_ON(!valid_section_nr(section_nr)))
661 continue;
662
663 ms = __nr_to_section(section_nr);
664 ms->section_mem_map &= ~SECTION_IS_ONLINE;
665 }
666}
667
668#ifdef CONFIG_SPARSEMEM_VMEMMAP
669static struct page * __meminit populate_section_memmap(unsigned long pfn,
670 unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
671 struct dev_pagemap *pgmap)
672{
673 return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
674}
675
676static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
677 struct vmem_altmap *altmap)
678{
679 unsigned long start = (unsigned long) pfn_to_page(pfn);
680 unsigned long end = start + nr_pages * sizeof(struct page);
681
682 vmemmap_free(start, end, altmap);
683}
684static void free_map_bootmem(struct page *memmap)
685{
686 unsigned long start = (unsigned long)memmap;
687 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
688
689 vmemmap_free(start, end, NULL);
690}
691
692static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages)
693{
694 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
695 DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 };
696 struct mem_section *ms = __pfn_to_section(pfn);
697 unsigned long *subsection_map = ms->usage
698 ? &ms->usage->subsection_map[0] : NULL;
699
700 subsection_mask_set(map, pfn, nr_pages);
701 if (subsection_map)
702 bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION);
703
704 if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION),
705 "section already deactivated (%#lx + %ld)\n",
706 pfn, nr_pages))
707 return -EINVAL;
708
709 bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION);
710 return 0;
711}
712
713static bool is_subsection_map_empty(struct mem_section *ms)
714{
715 return bitmap_empty(&ms->usage->subsection_map[0],
716 SUBSECTIONS_PER_SECTION);
717}
718
719static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
720{
721 struct mem_section *ms = __pfn_to_section(pfn);
722 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 };
723 unsigned long *subsection_map;
724 int rc = 0;
725
726 subsection_mask_set(map, pfn, nr_pages);
727
728 subsection_map = &ms->usage->subsection_map[0];
729
730 if (bitmap_empty(map, SUBSECTIONS_PER_SECTION))
731 rc = -EINVAL;
732 else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION))
733 rc = -EEXIST;
734 else
735 bitmap_or(subsection_map, map, subsection_map,
736 SUBSECTIONS_PER_SECTION);
737
738 return rc;
739}
740#else
741static struct page * __meminit populate_section_memmap(unsigned long pfn,
742 unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
743 struct dev_pagemap *pgmap)
744{
745 return kvmalloc_node(array_size(sizeof(struct page),
746 PAGES_PER_SECTION), GFP_KERNEL, nid);
747}
748
749static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
750 struct vmem_altmap *altmap)
751{
752 kvfree(pfn_to_page(pfn));
753}
754
755static void free_map_bootmem(struct page *memmap)
756{
757 unsigned long maps_section_nr, removing_section_nr, i;
758 unsigned long type, nr_pages;
759 struct page *page = virt_to_page(memmap);
760
761 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page))
762 >> PAGE_SHIFT;
763
764 for (i = 0; i < nr_pages; i++, page++) {
765 type = bootmem_type(page);
766
767 BUG_ON(type == NODE_INFO);
768
769 maps_section_nr = pfn_to_section_nr(page_to_pfn(page));
770 removing_section_nr = bootmem_info(page);
771
772 /*
773 * When this function is called, the removing section is
774 * logical offlined state. This means all pages are isolated
775 * from page allocator. If removing section's memmap is placed
776 * on the same section, it must not be freed.
777 * If it is freed, page allocator may allocate it which will
778 * be removed physically soon.
779 */
780 if (maps_section_nr != removing_section_nr)
781 put_page_bootmem(page);
782 }
783}
784
785static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages)
786{
787 return 0;
788}
789
790static bool is_subsection_map_empty(struct mem_section *ms)
791{
792 return true;
793}
794
795static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
796{
797 return 0;
798}
799#endif /* CONFIG_SPARSEMEM_VMEMMAP */
800
801/*
802 * To deactivate a memory region, there are 3 cases to handle across
803 * two configurations (SPARSEMEM_VMEMMAP={y,n}):
804 *
805 * 1. deactivation of a partial hot-added section (only possible in
806 * the SPARSEMEM_VMEMMAP=y case).
807 * a) section was present at memory init.
808 * b) section was hot-added post memory init.
809 * 2. deactivation of a complete hot-added section.
810 * 3. deactivation of a complete section from memory init.
811 *
812 * For 1, when subsection_map does not empty we will not be freeing the
813 * usage map, but still need to free the vmemmap range.
814 *
815 * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified
816 */
817static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
818 struct vmem_altmap *altmap)
819{
820 struct mem_section *ms = __pfn_to_section(pfn);
821 bool section_is_early = early_section(ms);
822 struct page *memmap = NULL;
823 bool empty;
824
825 if (clear_subsection_map(pfn, nr_pages))
826 return;
827
828 empty = is_subsection_map_empty(ms);
829 if (empty) {
830 unsigned long section_nr = pfn_to_section_nr(pfn);
831
832 /*
833 * Mark the section invalid so that valid_section()
834 * return false. This prevents code from dereferencing
835 * ms->usage array.
836 */
837 ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;
838
839 /*
840 * When removing an early section, the usage map is kept (as the
841 * usage maps of other sections fall into the same page). It
842 * will be re-used when re-adding the section - which is then no
843 * longer an early section. If the usage map is PageReserved, it
844 * was allocated during boot.
845 */
846 if (!PageReserved(virt_to_page(ms->usage))) {
847 kfree_rcu(ms->usage, rcu);
848 WRITE_ONCE(ms->usage, NULL);
849 }
850 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
851 }
852
853 /*
854 * The memmap of early sections is always fully populated. See
855 * section_activate() and pfn_valid() .
856 */
857 if (!section_is_early) {
858 memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)));
859 depopulate_section_memmap(pfn, nr_pages, altmap);
860 } else if (memmap) {
861 memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page),
862 PAGE_SIZE)));
863 free_map_bootmem(memmap);
864 }
865
866 if (empty)
867 ms->section_mem_map = (unsigned long)NULL;
868}
869
870static struct page * __meminit section_activate(int nid, unsigned long pfn,
871 unsigned long nr_pages, struct vmem_altmap *altmap,
872 struct dev_pagemap *pgmap)
873{
874 struct mem_section *ms = __pfn_to_section(pfn);
875 struct mem_section_usage *usage = NULL;
876 struct page *memmap;
877 int rc;
878
879 if (!ms->usage) {
880 usage = kzalloc(mem_section_usage_size(), GFP_KERNEL);
881 if (!usage)
882 return ERR_PTR(-ENOMEM);
883 ms->usage = usage;
884 }
885
886 rc = fill_subsection_map(pfn, nr_pages);
887 if (rc) {
888 if (usage)
889 ms->usage = NULL;
890 kfree(usage);
891 return ERR_PTR(rc);
892 }
893
894 /*
895 * The early init code does not consider partially populated
896 * initial sections, it simply assumes that memory will never be
897 * referenced. If we hot-add memory into such a section then we
898 * do not need to populate the memmap and can simply reuse what
899 * is already there.
900 */
901 if (nr_pages < PAGES_PER_SECTION && early_section(ms))
902 return pfn_to_page(pfn);
903
904 memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
905 if (!memmap) {
906 section_deactivate(pfn, nr_pages, altmap);
907 return ERR_PTR(-ENOMEM);
908 }
909 memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE));
910
911 return memmap;
912}
913
914/**
915 * sparse_add_section - add a memory section, or populate an existing one
916 * @nid: The node to add section on
917 * @start_pfn: start pfn of the memory range
918 * @nr_pages: number of pfns to add in the section
919 * @altmap: alternate pfns to allocate the memmap backing store
920 * @pgmap: alternate compound page geometry for devmap mappings
921 *
922 * This is only intended for hotplug.
923 *
924 * Note that only VMEMMAP supports sub-section aligned hotplug,
925 * the proper alignment and size are gated by check_pfn_span().
926 *
927 *
928 * Return:
929 * * 0 - On success.
930 * * -EEXIST - Section has been present.
931 * * -ENOMEM - Out of memory.
932 */
933int __meminit sparse_add_section(int nid, unsigned long start_pfn,
934 unsigned long nr_pages, struct vmem_altmap *altmap,
935 struct dev_pagemap *pgmap)
936{
937 unsigned long section_nr = pfn_to_section_nr(start_pfn);
938 struct mem_section *ms;
939 struct page *memmap;
940 int ret;
941
942 ret = sparse_index_init(section_nr, nid);
943 if (ret < 0)
944 return ret;
945
946 memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap);
947 if (IS_ERR(memmap))
948 return PTR_ERR(memmap);
949
950 /*
951 * Poison uninitialized struct pages in order to catch invalid flags
952 * combinations.
953 */
954 if (!altmap || !altmap->inaccessible)
955 page_init_poison(memmap, sizeof(struct page) * nr_pages);
956
957 ms = __nr_to_section(section_nr);
958 set_section_nid(section_nr, nid);
959 __section_mark_present(ms, section_nr);
960
961 /* Align memmap to section boundary in the subsection case */
962 if (section_nr_to_pfn(section_nr) != start_pfn)
963 memmap = pfn_to_page(section_nr_to_pfn(section_nr));
964 sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0);
965
966 return 0;
967}
968
969void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
970 struct vmem_altmap *altmap)
971{
972 struct mem_section *ms = __pfn_to_section(pfn);
973
974 if (WARN_ON_ONCE(!valid_section(ms)))
975 return;
976
977 section_deactivate(pfn, nr_pages, altmap);
978}
979#endif /* CONFIG_MEMORY_HOTPLUG */
980