1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Architecture specific (i386/x86_64) functions for kexec based crash dumps.
4 *
5 * Created by: Hariprasad Nellitheertha (hari@in.ibm.com)
6 *
7 * Copyright (C) IBM Corporation, 2004. All rights reserved.
8 * Copyright (C) Red Hat Inc., 2014. All rights reserved.
9 * Authors:
10 * Vivek Goyal <vgoyal@redhat.com>
11 *
12 */
13
14#define pr_fmt(fmt) "kexec: " fmt
15
16#include <linux/types.h>
17#include <linux/kernel.h>
18#include <linux/smp.h>
19#include <linux/reboot.h>
20#include <linux/kexec.h>
21#include <linux/delay.h>
22#include <linux/elf.h>
23#include <linux/elfcore.h>
24#include <linux/export.h>
25#include <linux/slab.h>
26#include <linux/vmalloc.h>
27#include <linux/memblock.h>
28
29#include <asm/bootparam.h>
30#include <asm/processor.h>
31#include <asm/hardirq.h>
32#include <asm/nmi.h>
33#include <asm/hw_irq.h>
34#include <asm/apic.h>
35#include <asm/e820/types.h>
36#include <asm/io_apic.h>
37#include <asm/hpet.h>
38#include <linux/kdebug.h>
39#include <asm/cpu.h>
40#include <asm/reboot.h>
41#include <asm/intel_pt.h>
42#include <asm/crash.h>
43#include <asm/cmdline.h>
44#include <asm/sev.h>
45
46/* Used while preparing memory map entries for second kernel */
47struct crash_memmap_data {
48 struct boot_params *params;
49 /* Type of memory */
50 unsigned int type;
51};
52
53#if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC)
54
55static void kdump_nmi_callback(int cpu, struct pt_regs *regs)
56{
57 crash_save_cpu(regs, cpu);
58
59 /*
60 * Disable Intel PT to stop its logging
61 */
62 cpu_emergency_stop_pt();
63
64 kdump_sev_callback();
65
66 disable_local_APIC();
67}
68
69void kdump_nmi_shootdown_cpus(void)
70{
71 nmi_shootdown_cpus(callback: kdump_nmi_callback);
72
73 disable_local_APIC();
74}
75
76/* Override the weak function in kernel/panic.c */
77void crash_smp_send_stop(void)
78{
79 static int cpus_stopped;
80
81 if (cpus_stopped)
82 return;
83
84 if (smp_ops.crash_stop_other_cpus)
85 smp_ops.crash_stop_other_cpus();
86 else
87 smp_send_stop();
88
89 cpus_stopped = 1;
90}
91
92#else
93void crash_smp_send_stop(void)
94{
95 /* There are no cpus to shootdown */
96}
97#endif
98
99void native_machine_crash_shutdown(struct pt_regs *regs)
100{
101 /* This function is only called after the system
102 * has panicked or is otherwise in a critical state.
103 * The minimum amount of code to allow a kexec'd kernel
104 * to run successfully needs to happen here.
105 *
106 * In practice this means shooting down the other cpus in
107 * an SMP system.
108 */
109 /* The kernel is broken so disable interrupts */
110 local_irq_disable();
111
112 crash_smp_send_stop();
113
114 cpu_emergency_disable_virtualization();
115
116 /*
117 * Disable Intel PT to stop its logging
118 */
119 cpu_emergency_stop_pt();
120
121#ifdef CONFIG_X86_IO_APIC
122 /* Prevent crash_kexec() from deadlocking on ioapic_lock. */
123 ioapic_zap_locks();
124 clear_IO_APIC();
125#endif
126 lapic_shutdown();
127 restore_boot_irq_mode();
128#ifdef CONFIG_HPET_TIMER
129 hpet_disable();
130#endif
131
132 /*
133 * Non-crash kexec calls enc_kexec_begin() while scheduling is still
134 * active. This allows the callback to wait until all in-flight
135 * shared<->private conversions are complete. In a crash scenario,
136 * enc_kexec_begin() gets called after all but one CPU have been shut
137 * down and interrupts have been disabled. This allows the callback to
138 * detect a race with the conversion and report it.
139 */
140 x86_platform.guest.enc_kexec_begin();
141 x86_platform.guest.enc_kexec_finish();
142
143 crash_save_cpu(regs, smp_processor_id());
144}
145
146#if defined(CONFIG_KEXEC_FILE) || defined(CONFIG_CRASH_HOTPLUG)
147static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
148{
149 unsigned int *nr_ranges = arg;
150
151 (*nr_ranges)++;
152 return 0;
153}
154
155/* Gather all the required information to prepare elf headers for ram regions */
156static struct crash_mem *fill_up_crash_elf_data(void)
157{
158 unsigned int nr_ranges = 0;
159 struct crash_mem *cmem;
160
161 walk_system_ram_res(start: 0, end: -1, arg: &nr_ranges, func: get_nr_ram_ranges_callback);
162 if (!nr_ranges)
163 return NULL;
164
165 /*
166 * Exclusion of crash region, crashk_low_res and/or crashk_cma_ranges
167 * may cause range splits. So add extra slots here.
168 *
169 * Exclusion of low 1M may not cause another range split, because the
170 * range of exclude is [0, 1M] and the condition for splitting a new
171 * region is that the start, end parameters are both in a certain
172 * existing region in cmem and cannot be equal to existing region's
173 * start or end. Obviously, the start of [0, 1M] cannot meet this
174 * condition.
175 *
176 * But in order to lest the low 1M could be changed in the future,
177 * (e.g. [start, 1M]), add a extra slot.
178 */
179 nr_ranges += 3 + crashk_cma_cnt;
180 cmem = vzalloc(struct_size(cmem, ranges, nr_ranges));
181 if (!cmem)
182 return NULL;
183
184 cmem->max_nr_ranges = nr_ranges;
185
186 return cmem;
187}
188
189/*
190 * Look for any unwanted ranges between mstart, mend and remove them. This
191 * might lead to split and split ranges are put in cmem->ranges[] array
192 */
193static int elf_header_exclude_ranges(struct crash_mem *cmem)
194{
195 int ret = 0;
196 int i;
197
198 /* Exclude the low 1M because it is always reserved */
199 ret = crash_exclude_mem_range(mem: cmem, mstart: 0, SZ_1M - 1);
200 if (ret)
201 return ret;
202
203 /* Exclude crashkernel region */
204 ret = crash_exclude_mem_range(mem: cmem, mstart: crashk_res.start, mend: crashk_res.end);
205 if (ret)
206 return ret;
207
208 if (crashk_low_res.end)
209 ret = crash_exclude_mem_range(mem: cmem, mstart: crashk_low_res.start,
210 mend: crashk_low_res.end);
211 if (ret)
212 return ret;
213
214 for (i = 0; i < crashk_cma_cnt; ++i) {
215 ret = crash_exclude_mem_range(mem: cmem, mstart: crashk_cma_ranges[i].start,
216 mend: crashk_cma_ranges[i].end);
217 if (ret)
218 return ret;
219 }
220
221 return 0;
222}
223
224static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
225{
226 struct crash_mem *cmem = arg;
227
228 cmem->ranges[cmem->nr_ranges].start = res->start;
229 cmem->ranges[cmem->nr_ranges].end = res->end;
230 cmem->nr_ranges++;
231
232 return 0;
233}
234
235/* Prepare elf headers. Return addr and size */
236static int prepare_elf_headers(void **addr, unsigned long *sz,
237 unsigned long *nr_mem_ranges)
238{
239 struct crash_mem *cmem;
240 int ret;
241
242 cmem = fill_up_crash_elf_data();
243 if (!cmem)
244 return -ENOMEM;
245
246 ret = walk_system_ram_res(start: 0, end: -1, arg: cmem, func: prepare_elf64_ram_headers_callback);
247 if (ret)
248 goto out;
249
250 /* Exclude unwanted mem ranges */
251 ret = elf_header_exclude_ranges(cmem);
252 if (ret)
253 goto out;
254
255 /* Return the computed number of memory ranges, for hotplug usage */
256 *nr_mem_ranges = cmem->nr_ranges;
257
258 /* By default prepare 64bit headers */
259 ret = crash_prepare_elf64_headers(mem: cmem, IS_ENABLED(CONFIG_X86_64), addr, sz);
260
261out:
262 vfree(addr: cmem);
263 return ret;
264}
265#endif
266
267#ifdef CONFIG_KEXEC_FILE
268static int add_e820_entry(struct boot_params *params, struct e820_entry *entry)
269{
270 unsigned int nr_e820_entries;
271
272 nr_e820_entries = params->e820_entries;
273 if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE)
274 return 1;
275
276 memcpy(&params->e820_table[nr_e820_entries], entry, sizeof(struct e820_entry));
277 params->e820_entries++;
278 return 0;
279}
280
281static int memmap_entry_callback(struct resource *res, void *arg)
282{
283 struct crash_memmap_data *cmd = arg;
284 struct boot_params *params = cmd->params;
285 struct e820_entry ei;
286
287 ei.addr = res->start;
288 ei.size = resource_size(res);
289 ei.type = cmd->type;
290 add_e820_entry(params, &ei);
291
292 return 0;
293}
294
295static int memmap_exclude_ranges(struct kimage *image, struct crash_mem *cmem,
296 unsigned long long mstart,
297 unsigned long long mend)
298{
299 unsigned long start, end;
300 int ret;
301
302 cmem->ranges[0].start = mstart;
303 cmem->ranges[0].end = mend;
304 cmem->nr_ranges = 1;
305
306 /* Exclude elf header region */
307 start = image->elf_load_addr;
308 end = start + image->elf_headers_sz - 1;
309 ret = crash_exclude_mem_range(cmem, start, end);
310
311 if (ret)
312 return ret;
313
314 /* Exclude dm crypt keys region */
315 if (image->dm_crypt_keys_addr) {
316 start = image->dm_crypt_keys_addr;
317 end = start + image->dm_crypt_keys_sz - 1;
318 return crash_exclude_mem_range(cmem, start, end);
319 }
320
321 return ret;
322}
323
324/* Prepare memory map for crash dump kernel */
325int crash_setup_memmap_entries(struct kimage *image, struct boot_params *params)
326{
327 unsigned int nr_ranges = 0;
328 int i, ret = 0;
329 unsigned long flags;
330 struct e820_entry ei;
331 struct crash_memmap_data cmd;
332 struct crash_mem *cmem;
333
334 /*
335 * In the current x86 architecture code, the elfheader is always
336 * allocated at crashk_res.start. But it depends on the allocation
337 * position of elfheader in crashk_res. To avoid potential out of
338 * bounds in future, add an extra slot.
339 *
340 * And using random kexec_buf for passing dm crypt keys may cause a
341 * range split too, add another extra slot here.
342 */
343 nr_ranges = 3;
344 cmem = vzalloc(struct_size(cmem, ranges, nr_ranges));
345 if (!cmem)
346 return -ENOMEM;
347
348 cmem->max_nr_ranges = nr_ranges;
349
350 memset(&cmd, 0, sizeof(struct crash_memmap_data));
351 cmd.params = params;
352
353 /* Add the low 1M */
354 cmd.type = E820_TYPE_RAM;
355 flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
356 walk_iomem_res_desc(IORES_DESC_NONE, flags, 0, (1<<20)-1, &cmd,
357 memmap_entry_callback);
358
359 /* Add ACPI tables */
360 cmd.type = E820_TYPE_ACPI;
361 flags = IORESOURCE_MEM | IORESOURCE_BUSY;
362 walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, &cmd,
363 memmap_entry_callback);
364
365 /* Add ACPI Non-volatile Storage */
366 cmd.type = E820_TYPE_NVS;
367 walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, &cmd,
368 memmap_entry_callback);
369
370 /* Add e820 reserved ranges */
371 cmd.type = E820_TYPE_RESERVED;
372 flags = IORESOURCE_MEM;
373 walk_iomem_res_desc(IORES_DESC_RESERVED, flags, 0, -1, &cmd,
374 memmap_entry_callback);
375
376 /* Add crashk_low_res region */
377 if (crashk_low_res.end) {
378 ei.addr = crashk_low_res.start;
379 ei.size = resource_size(&crashk_low_res);
380 ei.type = E820_TYPE_RAM;
381 add_e820_entry(params, &ei);
382 }
383
384 /* Exclude some ranges from crashk_res and add rest to memmap */
385 ret = memmap_exclude_ranges(image, cmem, crashk_res.start, crashk_res.end);
386 if (ret)
387 goto out;
388
389 for (i = 0; i < cmem->nr_ranges; i++) {
390 ei.size = cmem->ranges[i].end - cmem->ranges[i].start + 1;
391
392 /* If entry is less than a page, skip it */
393 if (ei.size < PAGE_SIZE)
394 continue;
395 ei.addr = cmem->ranges[i].start;
396 ei.type = E820_TYPE_RAM;
397 add_e820_entry(params, &ei);
398 }
399
400 for (i = 0; i < crashk_cma_cnt; ++i) {
401 ei.addr = crashk_cma_ranges[i].start;
402 ei.size = crashk_cma_ranges[i].end -
403 crashk_cma_ranges[i].start + 1;
404 ei.type = E820_TYPE_RAM;
405 add_e820_entry(params, &ei);
406 }
407
408out:
409 vfree(cmem);
410 return ret;
411}
412
413int crash_load_segments(struct kimage *image)
414{
415 int ret;
416 unsigned long pnum = 0;
417 struct kexec_buf kbuf = { .image = image, .buf_min = 0,
418 .buf_max = ULONG_MAX, .top_down = false };
419
420 /* Prepare elf headers and add a segment */
421 ret = prepare_elf_headers(&kbuf.buffer, &kbuf.bufsz, &pnum);
422 if (ret)
423 return ret;
424
425 image->elf_headers = kbuf.buffer;
426 image->elf_headers_sz = kbuf.bufsz;
427 kbuf.memsz = kbuf.bufsz;
428
429#ifdef CONFIG_CRASH_HOTPLUG
430 /*
431 * The elfcorehdr segment size accounts for VMCOREINFO, kernel_map,
432 * maximum CPUs and maximum memory ranges.
433 */
434 if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
435 pnum = 2 + CONFIG_NR_CPUS_DEFAULT + CONFIG_CRASH_MAX_MEMORY_RANGES;
436 else
437 pnum += 2 + CONFIG_NR_CPUS_DEFAULT;
438
439 if (pnum < (unsigned long)PN_XNUM) {
440 kbuf.memsz = pnum * sizeof(Elf64_Phdr);
441 kbuf.memsz += sizeof(Elf64_Ehdr);
442
443 image->elfcorehdr_index = image->nr_segments;
444
445 /* Mark as usable to crash kernel, else crash kernel fails on boot */
446 image->elf_headers_sz = kbuf.memsz;
447 } else {
448 pr_err("number of Phdrs %lu exceeds max\n", pnum);
449 }
450#endif
451
452 kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
453 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
454 ret = kexec_add_buffer(&kbuf);
455 if (ret)
456 return ret;
457 image->elf_load_addr = kbuf.mem;
458 kexec_dprintk("Loaded ELF headers at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
459 image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
460
461 return ret;
462}
463#endif /* CONFIG_KEXEC_FILE */
464
465#ifdef CONFIG_CRASH_HOTPLUG
466
467#undef pr_fmt
468#define pr_fmt(fmt) "crash hp: " fmt
469
470int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags)
471{
472
473#ifdef CONFIG_KEXEC_FILE
474 if (image->file_mode)
475 return 1;
476#endif
477 /*
478 * Initially, crash hotplug support for kexec_load was added
479 * with the KEXEC_UPDATE_ELFCOREHDR flag. Later, this
480 * functionality was expanded to accommodate multiple kexec
481 * segment updates, leading to the introduction of the
482 * KEXEC_CRASH_HOTPLUG_SUPPORT kexec flag bit. Consequently,
483 * when the kexec tool sends either of these flags, it indicates
484 * that the required kexec segment (elfcorehdr) is excluded from
485 * the SHA calculation.
486 */
487 return (kexec_flags & KEXEC_UPDATE_ELFCOREHDR ||
488 kexec_flags & KEXEC_CRASH_HOTPLUG_SUPPORT);
489}
490
491unsigned int arch_crash_get_elfcorehdr_size(void)
492{
493 unsigned int sz;
494
495 /* kernel_map, VMCOREINFO and maximum CPUs */
496 sz = 2 + CONFIG_NR_CPUS_DEFAULT;
497 if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
498 sz += CONFIG_CRASH_MAX_MEMORY_RANGES;
499 sz *= sizeof(Elf64_Phdr);
500 return sz;
501}
502
503/**
504 * arch_crash_handle_hotplug_event() - Handle hotplug elfcorehdr changes
505 * @image: a pointer to kexec_crash_image
506 * @arg: struct memory_notify handler for memory hotplug case and
507 * NULL for CPU hotplug case.
508 *
509 * Prepare the new elfcorehdr and replace the existing elfcorehdr.
510 */
511void arch_crash_handle_hotplug_event(struct kimage *image, void *arg)
512{
513 void *elfbuf = NULL, *old_elfcorehdr;
514 unsigned long nr_mem_ranges;
515 unsigned long mem, memsz;
516 unsigned long elfsz = 0;
517
518 /*
519 * As crash_prepare_elf64_headers() has already described all
520 * possible CPUs, there is no need to update the elfcorehdr
521 * for additional CPU changes.
522 */
523 if ((image->file_mode || image->elfcorehdr_updated) &&
524 ((image->hp_action == KEXEC_CRASH_HP_ADD_CPU) ||
525 (image->hp_action == KEXEC_CRASH_HP_REMOVE_CPU)))
526 return;
527
528 /*
529 * Create the new elfcorehdr reflecting the changes to CPU and/or
530 * memory resources.
531 */
532 if (prepare_elf_headers(addr: &elfbuf, sz: &elfsz, nr_mem_ranges: &nr_mem_ranges)) {
533 pr_err("unable to create new elfcorehdr");
534 goto out;
535 }
536
537 /*
538 * Obtain address and size of the elfcorehdr segment, and
539 * check it against the new elfcorehdr buffer.
540 */
541 mem = image->segment[image->elfcorehdr_index].mem;
542 memsz = image->segment[image->elfcorehdr_index].memsz;
543 if (elfsz > memsz) {
544 pr_err("update elfcorehdr elfsz %lu > memsz %lu",
545 elfsz, memsz);
546 goto out;
547 }
548
549 /*
550 * Copy new elfcorehdr over the old elfcorehdr at destination.
551 */
552 old_elfcorehdr = kmap_local_page(pfn_to_page(mem >> PAGE_SHIFT));
553 if (!old_elfcorehdr) {
554 pr_err("mapping elfcorehdr segment failed\n");
555 goto out;
556 }
557
558 /*
559 * Temporarily invalidate the crash image while the
560 * elfcorehdr is updated.
561 */
562 xchg(&kexec_crash_image, NULL);
563 memcpy_flushcache(dst: old_elfcorehdr, src: elfbuf, cnt: elfsz);
564 xchg(&kexec_crash_image, image);
565 kunmap_local(old_elfcorehdr);
566 pr_debug("updated elfcorehdr\n");
567
568out:
569 vfree(addr: elfbuf);
570}
571#endif
572