1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * kexec.c - kexec system call core code.
4 * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com>
5 */
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/btf.h>
10#include <linux/capability.h>
11#include <linux/mm.h>
12#include <linux/file.h>
13#include <linux/slab.h>
14#include <linux/fs.h>
15#include <linux/kexec.h>
16#include <linux/mutex.h>
17#include <linux/list.h>
18#include <linux/highmem.h>
19#include <linux/syscalls.h>
20#include <linux/reboot.h>
21#include <linux/ioport.h>
22#include <linux/hardirq.h>
23#include <linux/elf.h>
24#include <linux/elfcore.h>
25#include <linux/utsname.h>
26#include <linux/numa.h>
27#include <linux/suspend.h>
28#include <linux/device.h>
29#include <linux/freezer.h>
30#include <linux/panic_notifier.h>
31#include <linux/pm.h>
32#include <linux/cpu.h>
33#include <linux/uaccess.h>
34#include <linux/io.h>
35#include <linux/console.h>
36#include <linux/vmalloc.h>
37#include <linux/swap.h>
38#include <linux/syscore_ops.h>
39#include <linux/compiler.h>
40#include <linux/hugetlb.h>
41#include <linux/objtool.h>
42#include <linux/kmsg_dump.h>
43#include <linux/dma-map-ops.h>
44
45#include <asm/page.h>
46#include <asm/sections.h>
47
48#include <crypto/hash.h>
49#include "kexec_internal.h"
50
51atomic_t __kexec_lock = ATOMIC_INIT(0);
52
53/* Flag to indicate we are going to kexec a new kernel */
54bool kexec_in_progress = false;
55
56bool kexec_file_dbg_print;
57
58/*
59 * When kexec transitions to the new kernel there is a one-to-one
60 * mapping between physical and virtual addresses. On processors
61 * where you can disable the MMU this is trivial, and easy. For
62 * others it is still a simple predictable page table to setup.
63 *
64 * In that environment kexec copies the new kernel to its final
65 * resting place. This means I can only support memory whose
66 * physical address can fit in an unsigned long. In particular
67 * addresses where (pfn << PAGE_SHIFT) > ULONG_MAX cannot be handled.
68 * If the assembly stub has more restrictive requirements
69 * KEXEC_SOURCE_MEMORY_LIMIT and KEXEC_DEST_MEMORY_LIMIT can be
70 * defined more restrictively in <asm/kexec.h>.
71 *
72 * The code for the transition from the current kernel to the
73 * new kernel is placed in the control_code_buffer, whose size
74 * is given by KEXEC_CONTROL_PAGE_SIZE. In the best case only a single
75 * page of memory is necessary, but some architectures require more.
76 * Because this memory must be identity mapped in the transition from
77 * virtual to physical addresses it must live in the range
78 * 0 - TASK_SIZE, as only the user space mappings are arbitrarily
79 * modifiable.
80 *
81 * The assembly stub in the control code buffer is passed a linked list
82 * of descriptor pages detailing the source pages of the new kernel,
83 * and the destination addresses of those source pages. As this data
84 * structure is not used in the context of the current OS, it must
85 * be self-contained.
86 *
87 * The code has been made to work with highmem pages and will use a
88 * destination page in its final resting place (if it happens
89 * to allocate it). The end product of this is that most of the
90 * physical address space, and most of RAM can be used.
91 *
92 * Future directions include:
93 * - allocating a page table with the control code buffer identity
94 * mapped, to simplify machine_kexec and make kexec_on_panic more
95 * reliable.
96 */
97
98/*
99 * KIMAGE_NO_DEST is an impossible destination address..., for
100 * allocating pages whose destination address we do not care about.
101 */
102#define KIMAGE_NO_DEST (-1UL)
103#define PAGE_COUNT(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT)
104
105static struct page *kimage_alloc_page(struct kimage *image,
106 gfp_t gfp_mask,
107 unsigned long dest);
108
109int sanity_check_segment_list(struct kimage *image)
110{
111 int i;
112 unsigned long nr_segments = image->nr_segments;
113 unsigned long total_pages = 0;
114 unsigned long nr_pages = totalram_pages();
115
116 /*
117 * Verify we have good destination addresses. The caller is
118 * responsible for making certain we don't attempt to load
119 * the new image into invalid or reserved areas of RAM. This
120 * just verifies it is an address we can use.
121 *
122 * Since the kernel does everything in page size chunks ensure
123 * the destination addresses are page aligned. Too many
124 * special cases crop of when we don't do this. The most
125 * insidious is getting overlapping destination addresses
126 * simply because addresses are changed to page size
127 * granularity.
128 */
129 for (i = 0; i < nr_segments; i++) {
130 unsigned long mstart, mend;
131
132 mstart = image->segment[i].mem;
133 mend = mstart + image->segment[i].memsz;
134 if (mstart > mend)
135 return -EADDRNOTAVAIL;
136 if ((mstart & ~PAGE_MASK) || (mend & ~PAGE_MASK))
137 return -EADDRNOTAVAIL;
138 if (mend >= KEXEC_DESTINATION_MEMORY_LIMIT)
139 return -EADDRNOTAVAIL;
140 }
141
142 /* Verify our destination addresses do not overlap.
143 * If we alloed overlapping destination addresses
144 * through very weird things can happen with no
145 * easy explanation as one segment stops on another.
146 */
147 for (i = 0; i < nr_segments; i++) {
148 unsigned long mstart, mend;
149 unsigned long j;
150
151 mstart = image->segment[i].mem;
152 mend = mstart + image->segment[i].memsz;
153 for (j = 0; j < i; j++) {
154 unsigned long pstart, pend;
155
156 pstart = image->segment[j].mem;
157 pend = pstart + image->segment[j].memsz;
158 /* Do the segments overlap ? */
159 if ((mend > pstart) && (mstart < pend))
160 return -EINVAL;
161 }
162 }
163
164 /* Ensure our buffer sizes are strictly less than
165 * our memory sizes. This should always be the case,
166 * and it is easier to check up front than to be surprised
167 * later on.
168 */
169 for (i = 0; i < nr_segments; i++) {
170 if (image->segment[i].bufsz > image->segment[i].memsz)
171 return -EINVAL;
172 }
173
174 /*
175 * Verify that no more than half of memory will be consumed. If the
176 * request from userspace is too large, a large amount of time will be
177 * wasted allocating pages, which can cause a soft lockup.
178 */
179 for (i = 0; i < nr_segments; i++) {
180 if (PAGE_COUNT(image->segment[i].memsz) > nr_pages / 2)
181 return -EINVAL;
182
183 total_pages += PAGE_COUNT(image->segment[i].memsz);
184 }
185
186 if (total_pages > nr_pages / 2)
187 return -EINVAL;
188
189#ifdef CONFIG_CRASH_DUMP
190 /*
191 * Verify we have good destination addresses. Normally
192 * the caller is responsible for making certain we don't
193 * attempt to load the new image into invalid or reserved
194 * areas of RAM. But crash kernels are preloaded into a
195 * reserved area of ram. We must ensure the addresses
196 * are in the reserved area otherwise preloading the
197 * kernel could corrupt things.
198 */
199
200 if (image->type == KEXEC_TYPE_CRASH) {
201 for (i = 0; i < nr_segments; i++) {
202 unsigned long mstart, mend;
203
204 mstart = image->segment[i].mem;
205 mend = mstart + image->segment[i].memsz - 1;
206 /* Ensure we are within the crash kernel limits */
207 if ((mstart < phys_to_boot_phys(phys: crashk_res.start)) ||
208 (mend > phys_to_boot_phys(phys: crashk_res.end)))
209 return -EADDRNOTAVAIL;
210 }
211 }
212#endif
213
214 /*
215 * The destination addresses are searched from system RAM rather than
216 * being allocated from the buddy allocator, so they are not guaranteed
217 * to be accepted by the current kernel. Accept the destination
218 * addresses before kexec swaps their content with the segments' source
219 * pages to avoid accessing memory before it is accepted.
220 */
221 for (i = 0; i < nr_segments; i++)
222 accept_memory(start: image->segment[i].mem, size: image->segment[i].memsz);
223
224 return 0;
225}
226
227struct kimage *do_kimage_alloc_init(void)
228{
229 struct kimage *image;
230
231 /* Allocate a controlling structure */
232 image = kzalloc(sizeof(*image), GFP_KERNEL);
233 if (!image)
234 return NULL;
235
236 image->entry = &image->head;
237 image->last_entry = &image->head;
238 image->control_page = ~0; /* By default this does not apply */
239 image->type = KEXEC_TYPE_DEFAULT;
240
241 /* Initialize the list of control pages */
242 INIT_LIST_HEAD(list: &image->control_pages);
243
244 /* Initialize the list of destination pages */
245 INIT_LIST_HEAD(list: &image->dest_pages);
246
247 /* Initialize the list of unusable pages */
248 INIT_LIST_HEAD(list: &image->unusable_pages);
249
250#ifdef CONFIG_CRASH_HOTPLUG
251 image->hp_action = KEXEC_CRASH_HP_NONE;
252 image->elfcorehdr_index = -1;
253 image->elfcorehdr_updated = false;
254#endif
255
256 return image;
257}
258
259int kimage_is_destination_range(struct kimage *image,
260 unsigned long start,
261 unsigned long end)
262{
263 unsigned long i;
264
265 for (i = 0; i < image->nr_segments; i++) {
266 unsigned long mstart, mend;
267
268 mstart = image->segment[i].mem;
269 mend = mstart + image->segment[i].memsz - 1;
270 if ((end >= mstart) && (start <= mend))
271 return 1;
272 }
273
274 return 0;
275}
276
277static struct page *kimage_alloc_pages(gfp_t gfp_mask, unsigned int order)
278{
279 struct page *pages;
280
281 if (fatal_signal_pending(current))
282 return NULL;
283 pages = alloc_pages(gfp_mask & ~__GFP_ZERO, order);
284 if (pages) {
285 unsigned int count, i;
286
287 pages->mapping = NULL;
288 set_page_private(page: pages, private: order);
289 count = 1 << order;
290 for (i = 0; i < count; i++)
291 SetPageReserved(pages + i);
292
293 arch_kexec_post_alloc_pages(page_address(pages), pages: count,
294 gfp: gfp_mask);
295
296 if (gfp_mask & __GFP_ZERO)
297 for (i = 0; i < count; i++)
298 clear_highpage(page: pages + i);
299 }
300
301 return pages;
302}
303
304static void kimage_free_pages(struct page *page)
305{
306 unsigned int order, count, i;
307
308 order = page_private(page);
309 count = 1 << order;
310
311 arch_kexec_pre_free_pages(page_address(page), pages: count);
312
313 for (i = 0; i < count; i++)
314 ClearPageReserved(page: page + i);
315 __free_pages(page, order);
316}
317
318void kimage_free_page_list(struct list_head *list)
319{
320 struct page *page, *next;
321
322 list_for_each_entry_safe(page, next, list, lru) {
323 list_del(entry: &page->lru);
324 kimage_free_pages(page);
325 }
326}
327
328static struct page *kimage_alloc_normal_control_pages(struct kimage *image,
329 unsigned int order)
330{
331 /* Control pages are special, they are the intermediaries
332 * that are needed while we copy the rest of the pages
333 * to their final resting place. As such they must
334 * not conflict with either the destination addresses
335 * or memory the kernel is already using.
336 *
337 * The only case where we really need more than one of
338 * these are for architectures where we cannot disable
339 * the MMU and must instead generate an identity mapped
340 * page table for all of the memory.
341 *
342 * At worst this runs in O(N) of the image size.
343 */
344 struct list_head extra_pages;
345 struct page *pages;
346 unsigned int count;
347
348 count = 1 << order;
349 INIT_LIST_HEAD(list: &extra_pages);
350
351 /* Loop while I can allocate a page and the page allocated
352 * is a destination page.
353 */
354 do {
355 unsigned long pfn, epfn, addr, eaddr;
356
357 pages = kimage_alloc_pages(KEXEC_CONTROL_MEMORY_GFP, order);
358 if (!pages)
359 break;
360 pfn = page_to_boot_pfn(page: pages);
361 epfn = pfn + count;
362 addr = pfn << PAGE_SHIFT;
363 eaddr = (epfn << PAGE_SHIFT) - 1;
364 if ((epfn >= (KEXEC_CONTROL_MEMORY_LIMIT >> PAGE_SHIFT)) ||
365 kimage_is_destination_range(image, start: addr, end: eaddr)) {
366 list_add(new: &pages->lru, head: &extra_pages);
367 pages = NULL;
368 }
369 } while (!pages);
370
371 if (pages) {
372 /* Remember the allocated page... */
373 list_add(new: &pages->lru, head: &image->control_pages);
374
375 /* Because the page is already in it's destination
376 * location we will never allocate another page at
377 * that address. Therefore kimage_alloc_pages
378 * will not return it (again) and we don't need
379 * to give it an entry in image->segment[].
380 */
381 }
382 /* Deal with the destination pages I have inadvertently allocated.
383 *
384 * Ideally I would convert multi-page allocations into single
385 * page allocations, and add everything to image->dest_pages.
386 *
387 * For now it is simpler to just free the pages.
388 */
389 kimage_free_page_list(list: &extra_pages);
390
391 return pages;
392}
393
394#ifdef CONFIG_CRASH_DUMP
395static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
396 unsigned int order)
397{
398 /* Control pages are special, they are the intermediaries
399 * that are needed while we copy the rest of the pages
400 * to their final resting place. As such they must
401 * not conflict with either the destination addresses
402 * or memory the kernel is already using.
403 *
404 * Control pages are also the only pags we must allocate
405 * when loading a crash kernel. All of the other pages
406 * are specified by the segments and we just memcpy
407 * into them directly.
408 *
409 * The only case where we really need more than one of
410 * these are for architectures where we cannot disable
411 * the MMU and must instead generate an identity mapped
412 * page table for all of the memory.
413 *
414 * Given the low demand this implements a very simple
415 * allocator that finds the first hole of the appropriate
416 * size in the reserved memory region, and allocates all
417 * of the memory up to and including the hole.
418 */
419 unsigned long hole_start, hole_end, size;
420 struct page *pages;
421
422 pages = NULL;
423 size = (1 << order) << PAGE_SHIFT;
424 hole_start = ALIGN(image->control_page, size);
425 hole_end = hole_start + size - 1;
426 while (hole_end <= crashk_res.end) {
427 unsigned long i;
428
429 cond_resched();
430
431 if (hole_end > KEXEC_CRASH_CONTROL_MEMORY_LIMIT)
432 break;
433 /* See if I overlap any of the segments */
434 for (i = 0; i < image->nr_segments; i++) {
435 unsigned long mstart, mend;
436
437 mstart = image->segment[i].mem;
438 mend = mstart + image->segment[i].memsz - 1;
439 if ((hole_end >= mstart) && (hole_start <= mend)) {
440 /* Advance the hole to the end of the segment */
441 hole_start = ALIGN(mend, size);
442 hole_end = hole_start + size - 1;
443 break;
444 }
445 }
446 /* If I don't overlap any segments I have found my hole! */
447 if (i == image->nr_segments) {
448 pages = pfn_to_page(hole_start >> PAGE_SHIFT);
449 image->control_page = hole_end + 1;
450 break;
451 }
452 }
453
454 /* Ensure that these pages are decrypted if SME is enabled. */
455 if (pages)
456 arch_kexec_post_alloc_pages(page_address(pages), pages: 1 << order, gfp: 0);
457
458 return pages;
459}
460#endif
461
462
463struct page *kimage_alloc_control_pages(struct kimage *image,
464 unsigned int order)
465{
466 struct page *pages = NULL;
467
468 switch (image->type) {
469 case KEXEC_TYPE_DEFAULT:
470 pages = kimage_alloc_normal_control_pages(image, order);
471 break;
472#ifdef CONFIG_CRASH_DUMP
473 case KEXEC_TYPE_CRASH:
474 pages = kimage_alloc_crash_control_pages(image, order);
475 break;
476#endif
477 }
478
479 return pages;
480}
481
482static int kimage_add_entry(struct kimage *image, kimage_entry_t entry)
483{
484 if (*image->entry != 0)
485 image->entry++;
486
487 if (image->entry == image->last_entry) {
488 kimage_entry_t *ind_page;
489 struct page *page;
490
491 page = kimage_alloc_page(image, GFP_KERNEL, KIMAGE_NO_DEST);
492 if (!page)
493 return -ENOMEM;
494
495 ind_page = page_address(page);
496 *image->entry = virt_to_boot_phys(addr: ind_page) | IND_INDIRECTION;
497 image->entry = ind_page;
498 image->last_entry = ind_page +
499 ((PAGE_SIZE/sizeof(kimage_entry_t)) - 1);
500 }
501 *image->entry = entry;
502 image->entry++;
503 *image->entry = 0;
504
505 return 0;
506}
507
508static int kimage_set_destination(struct kimage *image,
509 unsigned long destination)
510{
511 destination &= PAGE_MASK;
512
513 return kimage_add_entry(image, entry: destination | IND_DESTINATION);
514}
515
516
517static int kimage_add_page(struct kimage *image, unsigned long page)
518{
519 page &= PAGE_MASK;
520
521 return kimage_add_entry(image, entry: page | IND_SOURCE);
522}
523
524
525static void kimage_free_extra_pages(struct kimage *image)
526{
527 /* Walk through and free any extra destination pages I may have */
528 kimage_free_page_list(list: &image->dest_pages);
529
530 /* Walk through and free any unusable pages I have cached */
531 kimage_free_page_list(list: &image->unusable_pages);
532
533}
534
535void kimage_terminate(struct kimage *image)
536{
537 if (*image->entry != 0)
538 image->entry++;
539
540 *image->entry = IND_DONE;
541}
542
543#define for_each_kimage_entry(image, ptr, entry) \
544 for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); \
545 ptr = (entry & IND_INDIRECTION) ? \
546 boot_phys_to_virt((entry & PAGE_MASK)) : ptr + 1)
547
548static void kimage_free_entry(kimage_entry_t entry)
549{
550 struct page *page;
551
552 page = boot_pfn_to_page(boot_pfn: entry >> PAGE_SHIFT);
553 kimage_free_pages(page);
554}
555
556static void kimage_free_cma(struct kimage *image)
557{
558 unsigned long i;
559
560 for (i = 0; i < image->nr_segments; i++) {
561 struct page *cma = image->segment_cma[i];
562 u32 nr_pages = image->segment[i].memsz >> PAGE_SHIFT;
563
564 if (!cma)
565 continue;
566
567 arch_kexec_pre_free_pages(page_address(cma), pages: nr_pages);
568 dma_release_from_contiguous(NULL, pages: cma, count: nr_pages);
569 image->segment_cma[i] = NULL;
570 }
571
572}
573
574void kimage_free(struct kimage *image)
575{
576 kimage_entry_t *ptr, entry;
577 kimage_entry_t ind = 0;
578
579 if (!image)
580 return;
581
582#ifdef CONFIG_CRASH_DUMP
583 if (image->vmcoreinfo_data_copy) {
584 crash_update_vmcoreinfo_safecopy(NULL);
585 vunmap(addr: image->vmcoreinfo_data_copy);
586 }
587#endif
588
589 kimage_free_extra_pages(image);
590 for_each_kimage_entry(image, ptr, entry) {
591 if (entry & IND_INDIRECTION) {
592 /* Free the previous indirection page */
593 if (ind & IND_INDIRECTION)
594 kimage_free_entry(entry: ind);
595 /* Save this indirection page until we are
596 * done with it.
597 */
598 ind = entry;
599 } else if (entry & IND_SOURCE)
600 kimage_free_entry(entry);
601 }
602 /* Free the final indirection page */
603 if (ind & IND_INDIRECTION)
604 kimage_free_entry(entry: ind);
605
606 /* Handle any machine specific cleanup */
607 machine_kexec_cleanup(image);
608
609 /* Free the kexec control pages... */
610 kimage_free_page_list(list: &image->control_pages);
611
612 /* Free CMA allocations */
613 kimage_free_cma(image);
614
615 /*
616 * Free up any temporary buffers allocated. This might hit if
617 * error occurred much later after buffer allocation.
618 */
619 if (image->file_mode)
620 kimage_file_post_load_cleanup(image);
621
622 kfree(objp: image);
623}
624
625static kimage_entry_t *kimage_dst_used(struct kimage *image,
626 unsigned long page)
627{
628 kimage_entry_t *ptr, entry;
629 unsigned long destination = 0;
630
631 for_each_kimage_entry(image, ptr, entry) {
632 if (entry & IND_DESTINATION)
633 destination = entry & PAGE_MASK;
634 else if (entry & IND_SOURCE) {
635 if (page == destination)
636 return ptr;
637 destination += PAGE_SIZE;
638 }
639 }
640
641 return NULL;
642}
643
644static struct page *kimage_alloc_page(struct kimage *image,
645 gfp_t gfp_mask,
646 unsigned long destination)
647{
648 /*
649 * Here we implement safeguards to ensure that a source page
650 * is not copied to its destination page before the data on
651 * the destination page is no longer useful.
652 *
653 * To do this we maintain the invariant that a source page is
654 * either its own destination page, or it is not a
655 * destination page at all.
656 *
657 * That is slightly stronger than required, but the proof
658 * that no problems will not occur is trivial, and the
659 * implementation is simply to verify.
660 *
661 * When allocating all pages normally this algorithm will run
662 * in O(N) time, but in the worst case it will run in O(N^2)
663 * time. If the runtime is a problem the data structures can
664 * be fixed.
665 */
666 struct page *page;
667 unsigned long addr;
668
669 /*
670 * Walk through the list of destination pages, and see if I
671 * have a match.
672 */
673 list_for_each_entry(page, &image->dest_pages, lru) {
674 addr = page_to_boot_pfn(page) << PAGE_SHIFT;
675 if (addr == destination) {
676 list_del(entry: &page->lru);
677 return page;
678 }
679 }
680 page = NULL;
681 while (1) {
682 kimage_entry_t *old;
683
684 /* Allocate a page, if we run out of memory give up */
685 page = kimage_alloc_pages(gfp_mask, order: 0);
686 if (!page)
687 return NULL;
688 /* If the page cannot be used file it away */
689 if (page_to_boot_pfn(page) >
690 (KEXEC_SOURCE_MEMORY_LIMIT >> PAGE_SHIFT)) {
691 list_add(new: &page->lru, head: &image->unusable_pages);
692 continue;
693 }
694 addr = page_to_boot_pfn(page) << PAGE_SHIFT;
695
696 /* If it is the destination page we want use it */
697 if (addr == destination)
698 break;
699
700 /* If the page is not a destination page use it */
701 if (!kimage_is_destination_range(image, start: addr,
702 end: addr + PAGE_SIZE - 1))
703 break;
704
705 /*
706 * I know that the page is someones destination page.
707 * See if there is already a source page for this
708 * destination page. And if so swap the source pages.
709 */
710 old = kimage_dst_used(image, page: addr);
711 if (old) {
712 /* If so move it */
713 unsigned long old_addr;
714 struct page *old_page;
715
716 old_addr = *old & PAGE_MASK;
717 old_page = boot_pfn_to_page(boot_pfn: old_addr >> PAGE_SHIFT);
718 copy_highpage(to: page, from: old_page);
719 *old = addr | (*old & ~PAGE_MASK);
720
721 /* The old page I have found cannot be a
722 * destination page, so return it if it's
723 * gfp_flags honor the ones passed in.
724 */
725 if (!(gfp_mask & __GFP_HIGHMEM) &&
726 PageHighMem(page: old_page)) {
727 kimage_free_pages(page: old_page);
728 continue;
729 }
730 page = old_page;
731 break;
732 }
733 /* Place the page on the destination list, to be used later */
734 list_add(new: &page->lru, head: &image->dest_pages);
735 }
736
737 return page;
738}
739
740static int kimage_load_cma_segment(struct kimage *image, int idx)
741{
742 struct kexec_segment *segment = &image->segment[idx];
743 struct page *cma = image->segment_cma[idx];
744 char *ptr = page_address(cma);
745 unsigned long maddr;
746 size_t ubytes, mbytes;
747 int result = 0;
748 unsigned char __user *buf = NULL;
749 unsigned char *kbuf = NULL;
750
751 if (image->file_mode)
752 kbuf = segment->kbuf;
753 else
754 buf = segment->buf;
755 ubytes = segment->bufsz;
756 mbytes = segment->memsz;
757 maddr = segment->mem;
758
759 /* Then copy from source buffer to the CMA one */
760 while (mbytes) {
761 size_t uchunk, mchunk;
762
763 ptr += maddr & ~PAGE_MASK;
764 mchunk = min_t(size_t, mbytes,
765 PAGE_SIZE - (maddr & ~PAGE_MASK));
766 uchunk = min(ubytes, mchunk);
767
768 if (uchunk) {
769 /* For file based kexec, source pages are in kernel memory */
770 if (image->file_mode)
771 memcpy(to: ptr, from: kbuf, len: uchunk);
772 else
773 result = copy_from_user(to: ptr, from: buf, n: uchunk);
774 ubytes -= uchunk;
775 if (image->file_mode)
776 kbuf += uchunk;
777 else
778 buf += uchunk;
779 }
780
781 if (result) {
782 result = -EFAULT;
783 goto out;
784 }
785
786 ptr += mchunk;
787 maddr += mchunk;
788 mbytes -= mchunk;
789
790 cond_resched();
791 }
792
793 /* Clear any remainder */
794 memset(s: ptr, c: 0, n: mbytes);
795
796out:
797 return result;
798}
799
800static int kimage_load_normal_segment(struct kimage *image, int idx)
801{
802 struct kexec_segment *segment = &image->segment[idx];
803 unsigned long maddr;
804 size_t ubytes, mbytes;
805 int result;
806 unsigned char __user *buf = NULL;
807 unsigned char *kbuf = NULL;
808
809 if (image->file_mode)
810 kbuf = segment->kbuf;
811 else
812 buf = segment->buf;
813 ubytes = segment->bufsz;
814 mbytes = segment->memsz;
815 maddr = segment->mem;
816
817 if (image->segment_cma[idx])
818 return kimage_load_cma_segment(image, idx);
819
820 result = kimage_set_destination(image, destination: maddr);
821 if (result < 0)
822 goto out;
823
824 while (mbytes) {
825 struct page *page;
826 char *ptr;
827 size_t uchunk, mchunk;
828
829 page = kimage_alloc_page(image, GFP_HIGHUSER, destination: maddr);
830 if (!page) {
831 result = -ENOMEM;
832 goto out;
833 }
834 result = kimage_add_page(image, page: page_to_boot_pfn(page)
835 << PAGE_SHIFT);
836 if (result < 0)
837 goto out;
838
839 ptr = kmap_local_page(page);
840 /* Start with a clear page */
841 clear_page(page: ptr);
842 ptr += maddr & ~PAGE_MASK;
843 mchunk = min_t(size_t, mbytes,
844 PAGE_SIZE - (maddr & ~PAGE_MASK));
845 uchunk = min(ubytes, mchunk);
846
847 if (uchunk) {
848 /* For file based kexec, source pages are in kernel memory */
849 if (image->file_mode)
850 memcpy(to: ptr, from: kbuf, len: uchunk);
851 else
852 result = copy_from_user(to: ptr, from: buf, n: uchunk);
853 ubytes -= uchunk;
854 if (image->file_mode)
855 kbuf += uchunk;
856 else
857 buf += uchunk;
858 }
859 kunmap_local(ptr);
860 if (result) {
861 result = -EFAULT;
862 goto out;
863 }
864 maddr += mchunk;
865 mbytes -= mchunk;
866
867 cond_resched();
868 }
869out:
870 return result;
871}
872
873#ifdef CONFIG_CRASH_DUMP
874static int kimage_load_crash_segment(struct kimage *image, int idx)
875{
876 /* For crash dumps kernels we simply copy the data from
877 * user space to it's destination.
878 * We do things a page at a time for the sake of kmap.
879 */
880 struct kexec_segment *segment = &image->segment[idx];
881 unsigned long maddr;
882 size_t ubytes, mbytes;
883 int result;
884 unsigned char __user *buf = NULL;
885 unsigned char *kbuf = NULL;
886
887 result = 0;
888 if (image->file_mode)
889 kbuf = segment->kbuf;
890 else
891 buf = segment->buf;
892 ubytes = segment->bufsz;
893 mbytes = segment->memsz;
894 maddr = segment->mem;
895 while (mbytes) {
896 struct page *page;
897 char *ptr;
898 size_t uchunk, mchunk;
899
900 page = boot_pfn_to_page(boot_pfn: maddr >> PAGE_SHIFT);
901 if (!page) {
902 result = -ENOMEM;
903 goto out;
904 }
905 arch_kexec_post_alloc_pages(page_address(page), pages: 1, gfp: 0);
906 ptr = kmap_local_page(page);
907 ptr += maddr & ~PAGE_MASK;
908 mchunk = min_t(size_t, mbytes,
909 PAGE_SIZE - (maddr & ~PAGE_MASK));
910 uchunk = min(ubytes, mchunk);
911 if (mchunk > uchunk) {
912 /* Zero the trailing part of the page */
913 memset(s: ptr + uchunk, c: 0, n: mchunk - uchunk);
914 }
915
916 if (uchunk) {
917 /* For file based kexec, source pages are in kernel memory */
918 if (image->file_mode)
919 memcpy(to: ptr, from: kbuf, len: uchunk);
920 else
921 result = copy_from_user(to: ptr, from: buf, n: uchunk);
922 ubytes -= uchunk;
923 if (image->file_mode)
924 kbuf += uchunk;
925 else
926 buf += uchunk;
927 }
928 kexec_flush_icache_page(page);
929 kunmap_local(ptr);
930 arch_kexec_pre_free_pages(page_address(page), pages: 1);
931 if (result) {
932 result = -EFAULT;
933 goto out;
934 }
935 maddr += mchunk;
936 mbytes -= mchunk;
937
938 cond_resched();
939 }
940out:
941 return result;
942}
943#endif
944
945int kimage_load_segment(struct kimage *image, int idx)
946{
947 int result = -ENOMEM;
948
949 switch (image->type) {
950 case KEXEC_TYPE_DEFAULT:
951 result = kimage_load_normal_segment(image, idx);
952 break;
953#ifdef CONFIG_CRASH_DUMP
954 case KEXEC_TYPE_CRASH:
955 result = kimage_load_crash_segment(image, idx);
956 break;
957#endif
958 }
959
960 return result;
961}
962
963void *kimage_map_segment(struct kimage *image,
964 unsigned long addr, unsigned long size)
965{
966 unsigned long src_page_addr, dest_page_addr = 0;
967 unsigned long eaddr = addr + size;
968 kimage_entry_t *ptr, entry;
969 struct page **src_pages;
970 unsigned int npages;
971 void *vaddr = NULL;
972 int i;
973
974 /*
975 * Collect the source pages and map them in a contiguous VA range.
976 */
977 npages = PFN_UP(eaddr) - PFN_DOWN(addr);
978 src_pages = kmalloc_array(npages, sizeof(*src_pages), GFP_KERNEL);
979 if (!src_pages) {
980 pr_err("Could not allocate ima pages array.\n");
981 return NULL;
982 }
983
984 i = 0;
985 for_each_kimage_entry(image, ptr, entry) {
986 if (entry & IND_DESTINATION) {
987 dest_page_addr = entry & PAGE_MASK;
988 } else if (entry & IND_SOURCE) {
989 if (dest_page_addr >= addr && dest_page_addr < eaddr) {
990 src_page_addr = entry & PAGE_MASK;
991 src_pages[i++] =
992 virt_to_page(__va(src_page_addr));
993 if (i == npages)
994 break;
995 dest_page_addr += PAGE_SIZE;
996 }
997 }
998 }
999
1000 /* Sanity check. */
1001 WARN_ON(i < npages);
1002
1003 vaddr = vmap(pages: src_pages, count: npages, VM_MAP, PAGE_KERNEL);
1004 kfree(objp: src_pages);
1005
1006 if (!vaddr)
1007 pr_err("Could not map ima buffer.\n");
1008
1009 return vaddr;
1010}
1011
1012void kimage_unmap_segment(void *segment_buffer)
1013{
1014 vunmap(addr: segment_buffer);
1015}
1016
1017struct kexec_load_limit {
1018 /* Mutex protects the limit count. */
1019 struct mutex mutex;
1020 int limit;
1021};
1022
1023static struct kexec_load_limit load_limit_reboot = {
1024 .mutex = __MUTEX_INITIALIZER(load_limit_reboot.mutex),
1025 .limit = -1,
1026};
1027
1028static struct kexec_load_limit load_limit_panic = {
1029 .mutex = __MUTEX_INITIALIZER(load_limit_panic.mutex),
1030 .limit = -1,
1031};
1032
1033struct kimage *kexec_image;
1034struct kimage *kexec_crash_image;
1035static int kexec_load_disabled;
1036
1037#ifdef CONFIG_SYSCTL
1038static int kexec_limit_handler(const struct ctl_table *table, int write,
1039 void *buffer, size_t *lenp, loff_t *ppos)
1040{
1041 struct kexec_load_limit *limit = table->data;
1042 int val;
1043 struct ctl_table tmp = {
1044 .data = &val,
1045 .maxlen = sizeof(val),
1046 .mode = table->mode,
1047 };
1048 int ret;
1049
1050 if (write) {
1051 ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
1052 if (ret)
1053 return ret;
1054
1055 if (val < 0)
1056 return -EINVAL;
1057
1058 mutex_lock(lock: &limit->mutex);
1059 if (limit->limit != -1 && val >= limit->limit)
1060 ret = -EINVAL;
1061 else
1062 limit->limit = val;
1063 mutex_unlock(lock: &limit->mutex);
1064
1065 return ret;
1066 }
1067
1068 mutex_lock(lock: &limit->mutex);
1069 val = limit->limit;
1070 mutex_unlock(lock: &limit->mutex);
1071
1072 return proc_dointvec(&tmp, write, buffer, lenp, ppos);
1073}
1074
1075static const struct ctl_table kexec_core_sysctls[] = {
1076 {
1077 .procname = "kexec_load_disabled",
1078 .data = &kexec_load_disabled,
1079 .maxlen = sizeof(int),
1080 .mode = 0644,
1081 /* only handle a transition from default "0" to "1" */
1082 .proc_handler = proc_dointvec_minmax,
1083 .extra1 = SYSCTL_ONE,
1084 .extra2 = SYSCTL_ONE,
1085 },
1086 {
1087 .procname = "kexec_load_limit_panic",
1088 .data = &load_limit_panic,
1089 .mode = 0644,
1090 .proc_handler = kexec_limit_handler,
1091 },
1092 {
1093 .procname = "kexec_load_limit_reboot",
1094 .data = &load_limit_reboot,
1095 .mode = 0644,
1096 .proc_handler = kexec_limit_handler,
1097 },
1098};
1099
1100static int __init kexec_core_sysctl_init(void)
1101{
1102 register_sysctl_init("kernel", kexec_core_sysctls);
1103 return 0;
1104}
1105late_initcall(kexec_core_sysctl_init);
1106#endif
1107
1108bool kexec_load_permitted(int kexec_image_type)
1109{
1110 struct kexec_load_limit *limit;
1111
1112 /*
1113 * Only the superuser can use the kexec syscall and if it has not
1114 * been disabled.
1115 */
1116 if (!capable(CAP_SYS_BOOT) || kexec_load_disabled)
1117 return false;
1118
1119 /* Check limit counter and decrease it.*/
1120 limit = (kexec_image_type == KEXEC_TYPE_CRASH) ?
1121 &load_limit_panic : &load_limit_reboot;
1122 mutex_lock(lock: &limit->mutex);
1123 if (!limit->limit) {
1124 mutex_unlock(lock: &limit->mutex);
1125 return false;
1126 }
1127 if (limit->limit != -1)
1128 limit->limit--;
1129 mutex_unlock(lock: &limit->mutex);
1130
1131 return true;
1132}
1133
1134/*
1135 * Move into place and start executing a preloaded standalone
1136 * executable. If nothing was preloaded return an error.
1137 */
1138int kernel_kexec(void)
1139{
1140 int error = 0;
1141
1142 if (!kexec_trylock())
1143 return -EBUSY;
1144 if (!kexec_image) {
1145 error = -EINVAL;
1146 goto Unlock;
1147 }
1148
1149#ifdef CONFIG_KEXEC_JUMP
1150 if (kexec_image->preserve_context) {
1151 /*
1152 * This flow is analogous to hibernation flows that occur
1153 * before creating an image and before jumping from the
1154 * restore kernel to the image one, so it uses the same
1155 * device callbacks as those two flows.
1156 */
1157 pm_prepare_console();
1158 error = freeze_processes();
1159 if (error) {
1160 error = -EBUSY;
1161 goto Restore_console;
1162 }
1163 console_suspend_all();
1164 error = dpm_suspend_start(PMSG_FREEZE);
1165 if (error)
1166 goto Resume_devices;
1167 /*
1168 * dpm_suspend_end() must be called after dpm_suspend_start()
1169 * to complete the transition, like in the hibernation flows
1170 * mentioned above.
1171 */
1172 error = dpm_suspend_end(PMSG_FREEZE);
1173 if (error)
1174 goto Resume_devices;
1175 error = suspend_disable_secondary_cpus();
1176 if (error)
1177 goto Enable_cpus;
1178 local_irq_disable();
1179 error = syscore_suspend();
1180 if (error)
1181 goto Enable_irqs;
1182 } else
1183#endif
1184 {
1185 kexec_in_progress = true;
1186 kernel_restart_prepare(cmd: "kexec reboot");
1187 migrate_to_reboot_cpu();
1188 syscore_shutdown();
1189
1190 /*
1191 * migrate_to_reboot_cpu() disables CPU hotplug assuming that
1192 * no further code needs to use CPU hotplug (which is true in
1193 * the reboot case). However, the kexec path depends on using
1194 * CPU hotplug again; so re-enable it here.
1195 */
1196 cpu_hotplug_enable();
1197 pr_notice("Starting new kernel\n");
1198 machine_shutdown();
1199 }
1200
1201 kmsg_dump(reason: KMSG_DUMP_SHUTDOWN);
1202 machine_kexec(image: kexec_image);
1203
1204#ifdef CONFIG_KEXEC_JUMP
1205 if (kexec_image->preserve_context) {
1206 /*
1207 * This flow is analogous to hibernation flows that occur after
1208 * creating an image and after the image kernel has got control
1209 * back, and in case the devices have been reset or otherwise
1210 * manipulated in the meantime, it uses the device callbacks
1211 * used by the latter.
1212 */
1213 syscore_resume();
1214 Enable_irqs:
1215 local_irq_enable();
1216 Enable_cpus:
1217 suspend_enable_secondary_cpus();
1218 dpm_resume_start(PMSG_RESTORE);
1219 Resume_devices:
1220 dpm_resume_end(PMSG_RESTORE);
1221 console_resume_all();
1222 thaw_processes();
1223 Restore_console:
1224 pm_restore_console();
1225 }
1226#endif
1227
1228 Unlock:
1229 kexec_unlock();
1230 return error;
1231}
1232