1// SPDX-License-Identifier: GPL-2.0-only
2// Copyright (C) 2022 Linutronix GmbH, John Ogness
3// Copyright (C) 2022 Intel, Thomas Gleixner
4
5#include <linux/atomic.h>
6#include <linux/bug.h>
7#include <linux/console.h>
8#include <linux/delay.h>
9#include <linux/errno.h>
10#include <linux/export.h>
11#include <linux/init.h>
12#include <linux/irqflags.h>
13#include <linux/kthread.h>
14#include <linux/minmax.h>
15#include <linux/panic.h>
16#include <linux/percpu.h>
17#include <linux/preempt.h>
18#include <linux/slab.h>
19#include <linux/smp.h>
20#include <linux/stddef.h>
21#include <linux/string.h>
22#include <linux/types.h>
23#include "internal.h"
24#include "printk_ringbuffer.h"
25/*
26 * Printk console printing implementation for consoles which does not depend
27 * on the legacy style console_lock mechanism.
28 *
29 * The state of the console is maintained in the "nbcon_state" atomic
30 * variable.
31 *
32 * The console is locked when:
33 *
34 * - The 'prio' field contains the priority of the context that owns the
35 * console. Only higher priority contexts are allowed to take over the
36 * lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked.
37 *
38 * - The 'cpu' field denotes on which CPU the console is locked. It is used
39 * to prevent busy waiting on the same CPU. Also it informs the lock owner
40 * that it has lost the lock in a more complex scenario when the lock was
41 * taken over by a higher priority context, released, and taken on another
42 * CPU with the same priority as the interrupted owner.
43 *
44 * The acquire mechanism uses a few more fields:
45 *
46 * - The 'req_prio' field is used by the handover approach to make the
47 * current owner aware that there is a context with a higher priority
48 * waiting for the friendly handover.
49 *
50 * - The 'unsafe' field allows to take over the console in a safe way in the
51 * middle of emitting a message. The field is set only when accessing some
52 * shared resources or when the console device is manipulated. It can be
53 * cleared, for example, after emitting one character when the console
54 * device is in a consistent state.
55 *
56 * - The 'unsafe_takeover' field is set when a hostile takeover took the
57 * console in an unsafe state. The console will stay in the unsafe state
58 * until re-initialized.
59 *
60 * The acquire mechanism uses three approaches:
61 *
62 * 1) Direct acquire when the console is not owned or is owned by a lower
63 * priority context and is in a safe state.
64 *
65 * 2) Friendly handover mechanism uses a request/grant handshake. It is used
66 * when the current owner has lower priority and the console is in an
67 * unsafe state.
68 *
69 * The requesting context:
70 *
71 * a) Sets its priority into the 'req_prio' field.
72 *
73 * b) Waits (with a timeout) for the owning context to unlock the
74 * console.
75 *
76 * c) Takes the lock and clears the 'req_prio' field.
77 *
78 * The owning context:
79 *
80 * a) Observes the 'req_prio' field set on exit from the unsafe
81 * console state.
82 *
83 * b) Gives up console ownership by clearing the 'prio' field.
84 *
85 * 3) Unsafe hostile takeover allows to take over the lock even when the
86 * console is an unsafe state. It is used only in panic() by the final
87 * attempt to flush consoles in a try and hope mode.
88 *
89 * Note that separate record buffers are used in panic(). As a result,
90 * the messages can be read and formatted without any risk even after
91 * using the hostile takeover in unsafe state.
92 *
93 * The release function simply clears the 'prio' field.
94 *
95 * All operations on @console::nbcon_state are atomic cmpxchg based to
96 * handle concurrency.
97 *
98 * The acquire/release functions implement only minimal policies:
99 *
100 * - Preference for higher priority contexts.
101 * - Protection of the panic CPU.
102 *
103 * All other policy decisions must be made at the call sites:
104 *
105 * - What is marked as an unsafe section.
106 * - Whether to spin-wait if there is already an owner and the console is
107 * in an unsafe state.
108 * - Whether to attempt an unsafe hostile takeover.
109 *
110 * The design allows to implement the well known:
111 *
112 * acquire()
113 * output_one_printk_record()
114 * release()
115 *
116 * The output of one printk record might be interrupted with a higher priority
117 * context. The new owner is supposed to reprint the entire interrupted record
118 * from scratch.
119 */
120
121/**
122 * nbcon_state_set - Helper function to set the console state
123 * @con: Console to update
124 * @new: The new state to write
125 *
126 * Only to be used when the console is not yet or no longer visible in the
127 * system. Otherwise use nbcon_state_try_cmpxchg().
128 */
129static inline void nbcon_state_set(struct console *con, struct nbcon_state *new)
130{
131 atomic_set(v: &ACCESS_PRIVATE(con, nbcon_state), i: new->atom);
132}
133
134/**
135 * nbcon_state_read - Helper function to read the console state
136 * @con: Console to read
137 * @state: The state to store the result
138 */
139static inline void nbcon_state_read(struct console *con, struct nbcon_state *state)
140{
141 state->atom = atomic_read(v: &ACCESS_PRIVATE(con, nbcon_state));
142}
143
144/**
145 * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
146 * @con: Console to update
147 * @cur: Old/expected state
148 * @new: New state
149 *
150 * Return: True on success. False on fail and @cur is updated.
151 */
152static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur,
153 struct nbcon_state *new)
154{
155 return atomic_try_cmpxchg(v: &ACCESS_PRIVATE(con, nbcon_state), old: &cur->atom, new: new->atom);
156}
157
158/**
159 * nbcon_seq_read - Read the current console sequence
160 * @con: Console to read the sequence of
161 *
162 * Return: Sequence number of the next record to print on @con.
163 */
164u64 nbcon_seq_read(struct console *con)
165{
166 unsigned long nbcon_seq = atomic_long_read(v: &ACCESS_PRIVATE(con, nbcon_seq));
167
168 return __ulseq_to_u64seq(prb, nbcon_seq);
169}
170
171/**
172 * nbcon_seq_force - Force console sequence to a specific value
173 * @con: Console to work on
174 * @seq: Sequence number value to set
175 *
176 * Only to be used during init (before registration) or in extreme situations
177 * (such as panic with CONSOLE_REPLAY_ALL).
178 */
179void nbcon_seq_force(struct console *con, u64 seq)
180{
181 /*
182 * If the specified record no longer exists, the oldest available record
183 * is chosen. This is especially important on 32bit systems because only
184 * the lower 32 bits of the sequence number are stored. The upper 32 bits
185 * are derived from the sequence numbers available in the ringbuffer.
186 */
187 u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));
188
189 atomic_long_set(v: &ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));
190}
191
192/**
193 * nbcon_seq_try_update - Try to update the console sequence number
194 * @ctxt: Pointer to an acquire context that contains
195 * all information about the acquire mode
196 * @new_seq: The new sequence number to set
197 *
198 * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
199 * the 64bit value). This could be a different value than @new_seq if
200 * nbcon_seq_force() was used or the current context no longer owns the
201 * console. In the later case, it will stop printing anyway.
202 */
203static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq)
204{
205 unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq);
206 struct console *con = ctxt->console;
207
208 if (atomic_long_try_cmpxchg(v: &ACCESS_PRIVATE(con, nbcon_seq), old: &nbcon_seq,
209 __u64seq_to_ulseq(new_seq))) {
210 ctxt->seq = new_seq;
211 } else {
212 ctxt->seq = nbcon_seq_read(con);
213 }
214}
215
216/**
217 * nbcon_context_try_acquire_direct - Try to acquire directly
218 * @ctxt: The context of the caller
219 * @cur: The current console state
220 * @is_reacquire: This acquire is a reacquire
221 *
222 * Acquire the console when it is released. Also acquire the console when
223 * the current owner has a lower priority and the console is in a safe state.
224 *
225 * Return: 0 on success. Otherwise, an error code on failure. Also @cur
226 * is updated to the latest state when failed to modify it.
227 *
228 * Errors:
229 *
230 * -EPERM: A panic is in progress and this is neither the panic
231 * CPU nor is this a reacquire. Or the current owner or
232 * waiter has the same or higher priority. No acquire
233 * method can be successful in these cases.
234 *
235 * -EBUSY: The current owner has a lower priority but the console
236 * in an unsafe state. The caller should try using
237 * the handover acquire method.
238 */
239static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt,
240 struct nbcon_state *cur, bool is_reacquire)
241{
242 unsigned int cpu = smp_processor_id();
243 struct console *con = ctxt->console;
244 struct nbcon_state new;
245
246 do {
247 /*
248 * Panic does not imply that the console is owned. However,
249 * since all non-panic CPUs are stopped during panic(), it
250 * is safer to have them avoid gaining console ownership.
251 *
252 * If this acquire is a reacquire (and an unsafe takeover
253 * has not previously occurred) then it is allowed to attempt
254 * a direct acquire in panic. This gives console drivers an
255 * opportunity to perform any necessary cleanup if they were
256 * interrupted by the panic CPU while printing.
257 */
258 if (panic_on_other_cpu() &&
259 (!is_reacquire || cur->unsafe_takeover)) {
260 return -EPERM;
261 }
262
263 if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio)
264 return -EPERM;
265
266 if (cur->unsafe)
267 return -EBUSY;
268
269 /*
270 * The console should never be safe for a direct acquire
271 * if an unsafe hostile takeover has ever happened.
272 */
273 WARN_ON_ONCE(cur->unsafe_takeover);
274
275 new.atom = cur->atom;
276 new.prio = ctxt->prio;
277 new.req_prio = NBCON_PRIO_NONE;
278 new.unsafe = cur->unsafe_takeover;
279 new.cpu = cpu;
280
281 } while (!nbcon_state_try_cmpxchg(con, cur, new: &new));
282
283 return 0;
284}
285
286static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio)
287{
288 /*
289 * The request context is well defined by the @req_prio because:
290 *
291 * - Only a context with a priority higher than the owner can become
292 * a waiter.
293 * - Only a context with a priority higher than the waiter can
294 * directly take over the request.
295 * - There are only three priorities.
296 * - Only one CPU is allowed to request PANIC priority.
297 * - Lower priorities are ignored during panic() until reboot.
298 *
299 * As a result, the following scenario is *not* possible:
300 *
301 * 1. This context is currently a waiter.
302 * 2. Another context with a higher priority than this context
303 * directly takes ownership.
304 * 3. The higher priority context releases the ownership.
305 * 4. Another lower priority context takes the ownership.
306 * 5. Another context with the same priority as this context
307 * creates a request and starts waiting.
308 *
309 * Event #1 implies this context is EMERGENCY.
310 * Event #2 implies the new context is PANIC.
311 * Event #3 occurs when panic() has flushed the console.
312 * Event #4 occurs when a non-panic CPU reacquires.
313 * Event #5 is not possible due to the panic_on_other_cpu() check
314 * in nbcon_context_try_acquire_handover().
315 */
316
317 return (cur->req_prio == expected_prio);
318}
319
320/**
321 * nbcon_context_try_acquire_requested - Try to acquire after having
322 * requested a handover
323 * @ctxt: The context of the caller
324 * @cur: The current console state
325 *
326 * This is a helper function for nbcon_context_try_acquire_handover().
327 * It is called when the console is in an unsafe state. The current
328 * owner will release the console on exit from the unsafe region.
329 *
330 * Return: 0 on success and @cur is updated to the new console state.
331 * Otherwise an error code on failure.
332 *
333 * Errors:
334 *
335 * -EPERM: A panic is in progress and this is not the panic CPU
336 * or this context is no longer the waiter.
337 *
338 * -EBUSY: The console is still locked. The caller should
339 * continue waiting.
340 *
341 * Note: The caller must still remove the request when an error has occurred
342 * except when this context is no longer the waiter.
343 */
344static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt,
345 struct nbcon_state *cur)
346{
347 unsigned int cpu = smp_processor_id();
348 struct console *con = ctxt->console;
349 struct nbcon_state new;
350
351 /* Note that the caller must still remove the request! */
352 if (panic_on_other_cpu())
353 return -EPERM;
354
355 /*
356 * Note that the waiter will also change if there was an unsafe
357 * hostile takeover.
358 */
359 if (!nbcon_waiter_matches(cur, expected_prio: ctxt->prio))
360 return -EPERM;
361
362 /* If still locked, caller should continue waiting. */
363 if (cur->prio != NBCON_PRIO_NONE)
364 return -EBUSY;
365
366 /*
367 * The previous owner should have never released ownership
368 * in an unsafe region.
369 */
370 WARN_ON_ONCE(cur->unsafe);
371
372 new.atom = cur->atom;
373 new.prio = ctxt->prio;
374 new.req_prio = NBCON_PRIO_NONE;
375 new.unsafe = cur->unsafe_takeover;
376 new.cpu = cpu;
377
378 if (!nbcon_state_try_cmpxchg(con, cur, new: &new)) {
379 /*
380 * The acquire could fail only when it has been taken
381 * over by a higher priority context.
382 */
383 WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio));
384 return -EPERM;
385 }
386
387 /* Handover success. This context now owns the console. */
388 return 0;
389}
390
391/**
392 * nbcon_context_try_acquire_handover - Try to acquire via handover
393 * @ctxt: The context of the caller
394 * @cur: The current console state
395 *
396 * The function must be called only when the context has higher priority
397 * than the current owner and the console is in an unsafe state.
398 * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
399 *
400 * The function sets "req_prio" field to make the current owner aware of
401 * the request. Then it waits until the current owner releases the console,
402 * or an even higher context takes over the request, or timeout expires.
403 *
404 * The current owner checks the "req_prio" field on exit from the unsafe
405 * region and releases the console. It does not touch the "req_prio" field
406 * so that the console stays reserved for the waiter.
407 *
408 * Return: 0 on success. Otherwise, an error code on failure. Also @cur
409 * is updated to the latest state when failed to modify it.
410 *
411 * Errors:
412 *
413 * -EPERM: A panic is in progress and this is not the panic CPU.
414 * Or a higher priority context has taken over the
415 * console or the handover request.
416 *
417 * -EBUSY: The current owner is on the same CPU so that the hand
418 * shake could not work. Or the current owner is not
419 * willing to wait (zero timeout). Or the console does
420 * not enter the safe state before timeout passed. The
421 * caller might still use the unsafe hostile takeover
422 * when allowed.
423 *
424 * -EAGAIN: @cur has changed when creating the handover request.
425 * The caller should retry with direct acquire.
426 */
427static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt,
428 struct nbcon_state *cur)
429{
430 unsigned int cpu = smp_processor_id();
431 struct console *con = ctxt->console;
432 struct nbcon_state new;
433 int timeout;
434 int request_err = -EBUSY;
435
436 /*
437 * Check that the handover is called when the direct acquire failed
438 * with -EBUSY.
439 */
440 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
441 WARN_ON_ONCE(!cur->unsafe);
442
443 /*
444 * Panic does not imply that the console is owned. However, it
445 * is critical that non-panic CPUs during panic are unable to
446 * wait for a handover in order to satisfy the assumptions of
447 * nbcon_waiter_matches(). In particular, the assumption that
448 * lower priorities are ignored during panic.
449 */
450 if (panic_on_other_cpu())
451 return -EPERM;
452
453 /* Handover is not possible on the same CPU. */
454 if (cur->cpu == cpu)
455 return -EBUSY;
456
457 /*
458 * Console stays unsafe after an unsafe takeover until re-initialized.
459 * Waiting is not going to help in this case.
460 */
461 if (cur->unsafe_takeover)
462 return -EBUSY;
463
464 /* Is the caller willing to wait? */
465 if (ctxt->spinwait_max_us == 0)
466 return -EBUSY;
467
468 /*
469 * Setup a request for the handover. The caller should try to acquire
470 * the console directly when the current state has been modified.
471 */
472 new.atom = cur->atom;
473 new.req_prio = ctxt->prio;
474 if (!nbcon_state_try_cmpxchg(con, cur, new: &new))
475 return -EAGAIN;
476
477 cur->atom = new.atom;
478
479 /* Wait until there is no owner and then acquire the console. */
480 for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) {
481 /* On successful acquire, this request is cleared. */
482 request_err = nbcon_context_try_acquire_requested(ctxt, cur);
483 if (!request_err)
484 return 0;
485
486 /*
487 * If the acquire should be aborted, it must be ensured
488 * that the request is removed before returning to caller.
489 */
490 if (request_err == -EPERM)
491 break;
492
493 udelay(usec: 1);
494
495 /* Re-read the state because some time has passed. */
496 nbcon_state_read(con, state: cur);
497 }
498
499 /* Timed out or aborted. Carefully remove handover request. */
500 do {
501 /*
502 * No need to remove request if there is a new waiter. This
503 * can only happen if a higher priority context has taken over
504 * the console or the handover request.
505 */
506 if (!nbcon_waiter_matches(cur, expected_prio: ctxt->prio))
507 return -EPERM;
508
509 /* Unset request for handover. */
510 new.atom = cur->atom;
511 new.req_prio = NBCON_PRIO_NONE;
512 if (nbcon_state_try_cmpxchg(con, cur, new: &new)) {
513 /*
514 * Request successfully unset. Report failure of
515 * acquiring via handover.
516 */
517 cur->atom = new.atom;
518 return request_err;
519 }
520
521 /*
522 * Unable to remove request. Try to acquire in case
523 * the owner has released the lock.
524 */
525 } while (nbcon_context_try_acquire_requested(ctxt, cur));
526
527 /* Lucky timing. The acquire succeeded while removing the request. */
528 return 0;
529}
530
531/**
532 * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
533 * @ctxt: The context of the caller
534 * @cur: The current console state
535 *
536 * Acquire the console even in the unsafe state.
537 *
538 * It can be permitted by setting the 'allow_unsafe_takeover' field only
539 * by the final attempt to flush messages in panic().
540 *
541 * Return: 0 on success. -EPERM when not allowed by the context.
542 */
543static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt,
544 struct nbcon_state *cur)
545{
546 unsigned int cpu = smp_processor_id();
547 struct console *con = ctxt->console;
548 struct nbcon_state new;
549
550 if (!ctxt->allow_unsafe_takeover)
551 return -EPERM;
552
553 /* Ensure caller is allowed to perform unsafe hostile takeovers. */
554 if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC))
555 return -EPERM;
556
557 /*
558 * Check that try_acquire_direct() and try_acquire_handover() returned
559 * -EBUSY in the right situation.
560 */
561 WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
562 WARN_ON_ONCE(cur->unsafe != true);
563
564 do {
565 new.atom = cur->atom;
566 new.cpu = cpu;
567 new.prio = ctxt->prio;
568 new.unsafe |= cur->unsafe_takeover;
569 new.unsafe_takeover |= cur->unsafe;
570
571 } while (!nbcon_state_try_cmpxchg(con, cur, new: &new));
572
573 return 0;
574}
575
576static struct printk_buffers panic_nbcon_pbufs;
577
578/**
579 * nbcon_context_try_acquire - Try to acquire nbcon console
580 * @ctxt: The context of the caller
581 * @is_reacquire: This acquire is a reacquire
582 *
583 * Context: Under @ctxt->con->device_lock() or local_irq_save().
584 * Return: True if the console was acquired. False otherwise.
585 *
586 * If the caller allowed an unsafe hostile takeover, on success the
587 * caller should check the current console state to see if it is
588 * in an unsafe state. Otherwise, on success the caller may assume
589 * the console is not in an unsafe state.
590 */
591static bool nbcon_context_try_acquire(struct nbcon_context *ctxt, bool is_reacquire)
592{
593 struct console *con = ctxt->console;
594 struct nbcon_state cur;
595 int err;
596
597 nbcon_state_read(con, state: &cur);
598try_again:
599 err = nbcon_context_try_acquire_direct(ctxt, cur: &cur, is_reacquire);
600 if (err != -EBUSY)
601 goto out;
602
603 err = nbcon_context_try_acquire_handover(ctxt, cur: &cur);
604 if (err == -EAGAIN)
605 goto try_again;
606 if (err != -EBUSY)
607 goto out;
608
609 err = nbcon_context_try_acquire_hostile(ctxt, cur: &cur);
610out:
611 if (err)
612 return false;
613
614 /* Acquire succeeded. */
615
616 /* Assign the appropriate buffer for this context. */
617 if (panic_on_this_cpu())
618 ctxt->pbufs = &panic_nbcon_pbufs;
619 else
620 ctxt->pbufs = con->pbufs;
621
622 /* Set the record sequence for this context to print. */
623 ctxt->seq = nbcon_seq_read(con: ctxt->console);
624
625 return true;
626}
627
628static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu,
629 int expected_prio)
630{
631 /*
632 * A similar function, nbcon_waiter_matches(), only deals with
633 * EMERGENCY and PANIC priorities. However, this function must also
634 * deal with the NORMAL priority, which requires additional checks
635 * and constraints.
636 *
637 * For the case where preemption and interrupts are disabled, it is
638 * enough to also verify that the owning CPU has not changed.
639 *
640 * For the case where preemption or interrupts are enabled, an
641 * external synchronization method *must* be used. In particular,
642 * the driver-specific locking mechanism used in device_lock()
643 * (including disabling migration) should be used. It prevents
644 * scenarios such as:
645 *
646 * 1. [Task A] owns a context with NBCON_PRIO_NORMAL on [CPU X] and
647 * is scheduled out.
648 * 2. Another context takes over the lock with NBCON_PRIO_EMERGENCY
649 * and releases it.
650 * 3. [Task B] acquires a context with NBCON_PRIO_NORMAL on [CPU X]
651 * and is scheduled out.
652 * 4. [Task A] gets running on [CPU X] and sees that the console is
653 * still owned by a task on [CPU X] with NBON_PRIO_NORMAL. Thus
654 * [Task A] thinks it is the owner when it is not.
655 */
656
657 if (cur->prio != expected_prio)
658 return false;
659
660 if (cur->cpu != expected_cpu)
661 return false;
662
663 return true;
664}
665
666/**
667 * nbcon_context_release - Release the console
668 * @ctxt: The nbcon context from nbcon_context_try_acquire()
669 */
670static void nbcon_context_release(struct nbcon_context *ctxt)
671{
672 unsigned int cpu = smp_processor_id();
673 struct console *con = ctxt->console;
674 struct nbcon_state cur;
675 struct nbcon_state new;
676
677 nbcon_state_read(con, state: &cur);
678
679 do {
680 if (!nbcon_owner_matches(cur: &cur, expected_cpu: cpu, expected_prio: ctxt->prio))
681 break;
682
683 new.atom = cur.atom;
684 new.prio = NBCON_PRIO_NONE;
685
686 /*
687 * If @unsafe_takeover is set, it is kept set so that
688 * the state remains permanently unsafe.
689 */
690 new.unsafe |= cur.unsafe_takeover;
691
692 } while (!nbcon_state_try_cmpxchg(con, cur: &cur, new: &new));
693
694 ctxt->pbufs = NULL;
695}
696
697/**
698 * nbcon_context_can_proceed - Check whether ownership can proceed
699 * @ctxt: The nbcon context from nbcon_context_try_acquire()
700 * @cur: The current console state
701 *
702 * Return: True if this context still owns the console. False if
703 * ownership was handed over or taken.
704 *
705 * Must be invoked when entering the unsafe state to make sure that it still
706 * owns the lock. Also must be invoked when exiting the unsafe context
707 * to eventually free the lock for a higher priority context which asked
708 * for the friendly handover.
709 *
710 * It can be called inside an unsafe section when the console is just
711 * temporary in safe state instead of exiting and entering the unsafe
712 * state.
713 *
714 * Also it can be called in the safe context before doing an expensive
715 * safe operation. It does not make sense to do the operation when
716 * a higher priority context took the lock.
717 *
718 * When this function returns false then the calling context no longer owns
719 * the console and is no longer allowed to go forward. In this case it must
720 * back out immediately and carefully. The buffer content is also no longer
721 * trusted since it no longer belongs to the calling context.
722 */
723static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur)
724{
725 unsigned int cpu = smp_processor_id();
726
727 /* Make sure this context still owns the console. */
728 if (!nbcon_owner_matches(cur, expected_cpu: cpu, expected_prio: ctxt->prio))
729 return false;
730
731 /* The console owner can proceed if there is no waiter. */
732 if (cur->req_prio == NBCON_PRIO_NONE)
733 return true;
734
735 /*
736 * A console owner within an unsafe region is always allowed to
737 * proceed, even if there are waiters. It can perform a handover
738 * when exiting the unsafe region. Otherwise the waiter will
739 * need to perform an unsafe hostile takeover.
740 */
741 if (cur->unsafe)
742 return true;
743
744 /* Waiters always have higher priorities than owners. */
745 WARN_ON_ONCE(cur->req_prio <= cur->prio);
746
747 /*
748 * Having a safe point for take over and eventually a few
749 * duplicated characters or a full line is way better than a
750 * hostile takeover. Post processing can take care of the garbage.
751 * Release and hand over.
752 */
753 nbcon_context_release(ctxt);
754
755 /*
756 * It is not clear whether the waiter really took over ownership. The
757 * outermost callsite must make the final decision whether console
758 * ownership is needed for it to proceed. If yes, it must reacquire
759 * ownership (possibly hostile) before carefully proceeding.
760 *
761 * The calling context no longer owns the console so go back all the
762 * way instead of trying to implement reacquire heuristics in tons of
763 * places.
764 */
765 return false;
766}
767
768/**
769 * nbcon_can_proceed - Check whether ownership can proceed
770 * @wctxt: The write context that was handed to the write function
771 *
772 * Return: True if this context still owns the console. False if
773 * ownership was handed over or taken.
774 *
775 * It is used in nbcon_enter_unsafe() to make sure that it still owns the
776 * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock
777 * for a higher priority context which asked for the friendly handover.
778 *
779 * It can be called inside an unsafe section when the console is just
780 * temporary in safe state instead of exiting and entering the unsafe state.
781 *
782 * Also it can be called in the safe context before doing an expensive safe
783 * operation. It does not make sense to do the operation when a higher
784 * priority context took the lock.
785 *
786 * When this function returns false then the calling context no longer owns
787 * the console and is no longer allowed to go forward. In this case it must
788 * back out immediately and carefully. The buffer content is also no longer
789 * trusted since it no longer belongs to the calling context.
790 */
791bool nbcon_can_proceed(struct nbcon_write_context *wctxt)
792{
793 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
794 struct console *con = ctxt->console;
795 struct nbcon_state cur;
796
797 nbcon_state_read(con, state: &cur);
798
799 return nbcon_context_can_proceed(ctxt, cur: &cur);
800}
801EXPORT_SYMBOL_GPL(nbcon_can_proceed);
802
803#define nbcon_context_enter_unsafe(c) __nbcon_context_update_unsafe(c, true)
804#define nbcon_context_exit_unsafe(c) __nbcon_context_update_unsafe(c, false)
805
806/**
807 * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
808 * @ctxt: The nbcon context from nbcon_context_try_acquire()
809 * @unsafe: The new value for the unsafe bit
810 *
811 * Return: True if the unsafe state was updated and this context still
812 * owns the console. Otherwise false if ownership was handed
813 * over or taken.
814 *
815 * This function allows console owners to modify the unsafe status of the
816 * console.
817 *
818 * When this function returns false then the calling context no longer owns
819 * the console and is no longer allowed to go forward. In this case it must
820 * back out immediately and carefully. The buffer content is also no longer
821 * trusted since it no longer belongs to the calling context.
822 *
823 * Internal helper to avoid duplicated code.
824 */
825static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe)
826{
827 struct console *con = ctxt->console;
828 struct nbcon_state cur;
829 struct nbcon_state new;
830
831 nbcon_state_read(con, state: &cur);
832
833 do {
834 /*
835 * The unsafe bit must not be cleared if an
836 * unsafe hostile takeover has occurred.
837 */
838 if (!unsafe && cur.unsafe_takeover)
839 goto out;
840
841 if (!nbcon_context_can_proceed(ctxt, cur: &cur))
842 return false;
843
844 new.atom = cur.atom;
845 new.unsafe = unsafe;
846 } while (!nbcon_state_try_cmpxchg(con, cur: &cur, new: &new));
847
848 cur.atom = new.atom;
849out:
850 return nbcon_context_can_proceed(ctxt, cur: &cur);
851}
852
853static void nbcon_write_context_set_buf(struct nbcon_write_context *wctxt,
854 char *buf, unsigned int len)
855{
856 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
857 struct console *con = ctxt->console;
858 struct nbcon_state cur;
859
860 wctxt->outbuf = buf;
861 wctxt->len = len;
862 nbcon_state_read(con, state: &cur);
863 wctxt->unsafe_takeover = cur.unsafe_takeover;
864}
865
866/**
867 * nbcon_enter_unsafe - Enter an unsafe region in the driver
868 * @wctxt: The write context that was handed to the write function
869 *
870 * Return: True if this context still owns the console. False if
871 * ownership was handed over or taken.
872 *
873 * When this function returns false then the calling context no longer owns
874 * the console and is no longer allowed to go forward. In this case it must
875 * back out immediately and carefully. The buffer content is also no longer
876 * trusted since it no longer belongs to the calling context.
877 */
878bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt)
879{
880 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
881 bool is_owner;
882
883 is_owner = nbcon_context_enter_unsafe(ctxt);
884 if (!is_owner)
885 nbcon_write_context_set_buf(wctxt, NULL, len: 0);
886 return is_owner;
887}
888EXPORT_SYMBOL_GPL(nbcon_enter_unsafe);
889
890/**
891 * nbcon_exit_unsafe - Exit an unsafe region in the driver
892 * @wctxt: The write context that was handed to the write function
893 *
894 * Return: True if this context still owns the console. False if
895 * ownership was handed over or taken.
896 *
897 * When this function returns false then the calling context no longer owns
898 * the console and is no longer allowed to go forward. In this case it must
899 * back out immediately and carefully. The buffer content is also no longer
900 * trusted since it no longer belongs to the calling context.
901 */
902bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt)
903{
904 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
905 bool ret;
906
907 ret = nbcon_context_exit_unsafe(ctxt);
908 if (!ret)
909 nbcon_write_context_set_buf(wctxt, NULL, len: 0);
910 return ret;
911}
912EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);
913
914/**
915 * nbcon_reacquire_nobuf - Reacquire a console after losing ownership
916 * while printing
917 * @wctxt: The write context that was handed to the write callback
918 *
919 * Since ownership can be lost at any time due to handover or takeover, a
920 * printing context _must_ be prepared to back out immediately and
921 * carefully. However, there are scenarios where the printing context must
922 * reacquire ownership in order to finalize or revert hardware changes.
923 *
924 * This function allows a printing context to reacquire ownership using the
925 * same priority as its previous ownership.
926 *
927 * Note that after a successful reacquire the printing context will have no
928 * output buffer because that has been lost. This function cannot be used to
929 * resume printing.
930 */
931void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt)
932{
933 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
934
935 while (!nbcon_context_try_acquire(ctxt, is_reacquire: true))
936 cpu_relax();
937
938 nbcon_write_context_set_buf(wctxt, NULL, len: 0);
939}
940EXPORT_SYMBOL_GPL(nbcon_reacquire_nobuf);
941
942/**
943 * nbcon_emit_next_record - Emit a record in the acquired context
944 * @wctxt: The write context that will be handed to the write function
945 * @use_atomic: True if the write_atomic() callback is to be used
946 *
947 * Return: True if this context still owns the console. False if
948 * ownership was handed over or taken.
949 *
950 * When this function returns false then the calling context no longer owns
951 * the console and is no longer allowed to go forward. In this case it must
952 * back out immediately and carefully. The buffer content is also no longer
953 * trusted since it no longer belongs to the calling context. If the caller
954 * wants to do more it must reacquire the console first.
955 *
956 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
957 * still records pending in the ringbuffer,
958 */
959static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic)
960{
961 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
962 struct console *con = ctxt->console;
963 bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
964 struct printk_message pmsg = {
965 .pbufs = ctxt->pbufs,
966 };
967 unsigned long con_dropped;
968 struct nbcon_state cur;
969 unsigned long dropped;
970 unsigned long ulseq;
971
972 /*
973 * This function should never be called for consoles that have not
974 * implemented the necessary callback for writing: i.e. legacy
975 * consoles and, when atomic, nbcon consoles with no write_atomic().
976 * Handle it as if ownership was lost and try to continue.
977 *
978 * Note that for nbcon consoles the write_thread() callback is
979 * mandatory and was already checked in nbcon_alloc().
980 */
981 if (WARN_ON_ONCE((use_atomic && !con->write_atomic) ||
982 !(console_srcu_read_flags(con) & CON_NBCON))) {
983 nbcon_context_release(ctxt);
984 return false;
985 }
986
987 /*
988 * The printk buffers are filled within an unsafe section. This
989 * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from
990 * clobbering each other.
991 */
992
993 if (!nbcon_context_enter_unsafe(ctxt))
994 return false;
995
996 ctxt->backlog = printk_get_next_message(pmsg: &pmsg, seq: ctxt->seq, is_extended, may_supress: true);
997 if (!ctxt->backlog)
998 return nbcon_context_exit_unsafe(ctxt);
999
1000 /*
1001 * @con->dropped is not protected in case of an unsafe hostile
1002 * takeover. In that situation the update can be racy so
1003 * annotate it accordingly.
1004 */
1005 con_dropped = data_race(READ_ONCE(con->dropped));
1006
1007 dropped = con_dropped + pmsg.dropped;
1008 if (dropped && !is_extended)
1009 console_prepend_dropped(pmsg: &pmsg, dropped);
1010
1011 /*
1012 * If the previous owner was assigned the same record, this context
1013 * has taken over ownership and is replaying the record. Prepend a
1014 * message to let the user know the record is replayed.
1015 */
1016 ulseq = atomic_long_read(v: &ACCESS_PRIVATE(con, nbcon_prev_seq));
1017 if (__ulseq_to_u64seq(prb, ulseq) == pmsg.seq) {
1018 console_prepend_replay(pmsg: &pmsg);
1019 } else {
1020 /*
1021 * Ensure this context is still the owner before trying to
1022 * update @nbcon_prev_seq. Otherwise the value in @ulseq may
1023 * not be from the previous owner and instead be some later
1024 * value from the context that took over ownership.
1025 */
1026 nbcon_state_read(con, state: &cur);
1027 if (!nbcon_context_can_proceed(ctxt, cur: &cur))
1028 return false;
1029
1030 atomic_long_try_cmpxchg(v: &ACCESS_PRIVATE(con, nbcon_prev_seq), old: &ulseq,
1031 __u64seq_to_ulseq(pmsg.seq));
1032 }
1033
1034 if (!nbcon_context_exit_unsafe(ctxt))
1035 return false;
1036
1037 /* For skipped records just update seq/dropped in @con. */
1038 if (pmsg.outbuf_len == 0)
1039 goto update_con;
1040
1041 /* Initialize the write context for driver callbacks. */
1042 nbcon_write_context_set_buf(wctxt, buf: &pmsg.pbufs->outbuf[0], len: pmsg.outbuf_len);
1043
1044 if (use_atomic)
1045 con->write_atomic(con, wctxt);
1046 else
1047 con->write_thread(con, wctxt);
1048
1049 if (!wctxt->outbuf) {
1050 /*
1051 * Ownership was lost and reacquired by the driver. Handle it
1052 * as if ownership was lost.
1053 */
1054 nbcon_context_release(ctxt);
1055 return false;
1056 }
1057
1058 /*
1059 * Ownership may have been lost but _not_ reacquired by the driver.
1060 * This case is detected and handled when entering unsafe to update
1061 * dropped/seq values.
1062 */
1063
1064 /*
1065 * Since any dropped message was successfully output, reset the
1066 * dropped count for the console.
1067 */
1068 dropped = 0;
1069update_con:
1070 /*
1071 * The dropped count and the sequence number are updated within an
1072 * unsafe section. This limits update races to the panic context and
1073 * allows the panic context to win.
1074 */
1075
1076 if (!nbcon_context_enter_unsafe(ctxt))
1077 return false;
1078
1079 if (dropped != con_dropped) {
1080 /* Counterpart to the READ_ONCE() above. */
1081 WRITE_ONCE(con->dropped, dropped);
1082 }
1083
1084 nbcon_seq_try_update(ctxt, new_seq: pmsg.seq + 1);
1085
1086 return nbcon_context_exit_unsafe(ctxt);
1087}
1088
1089/*
1090 * nbcon_emit_one - Print one record for an nbcon console using the
1091 * specified callback
1092 * @wctxt: An initialized write context struct to use for this context
1093 * @use_atomic: True if the write_atomic() callback is to be used
1094 *
1095 * Return: True, when a record has been printed and there are still
1096 * pending records. The caller might want to continue flushing.
1097 *
1098 * False, when there is no pending record, or when the console
1099 * context cannot be acquired, or the ownership has been lost.
1100 * The caller should give up. Either the job is done, cannot be
1101 * done, or will be handled by the owning context.
1102 *
1103 * This is an internal helper to handle the locking of the console before
1104 * calling nbcon_emit_next_record().
1105 */
1106static bool nbcon_emit_one(struct nbcon_write_context *wctxt, bool use_atomic)
1107{
1108 struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
1109 struct console *con = ctxt->console;
1110 unsigned long flags;
1111 bool ret = false;
1112
1113 if (!use_atomic) {
1114 con->device_lock(con, &flags);
1115
1116 /*
1117 * Ensure this stays on the CPU to make handover and
1118 * takeover possible.
1119 */
1120 cant_migrate();
1121 }
1122
1123 if (!nbcon_context_try_acquire(ctxt, is_reacquire: false))
1124 goto out;
1125
1126 /*
1127 * nbcon_emit_next_record() returns false when the console was
1128 * handed over or taken over. In both cases the context is no
1129 * longer valid.
1130 *
1131 * The higher priority printing context takes over responsibility
1132 * to print the pending records.
1133 */
1134 if (!nbcon_emit_next_record(wctxt, use_atomic))
1135 goto out;
1136
1137 nbcon_context_release(ctxt);
1138
1139 ret = ctxt->backlog;
1140out:
1141 if (!use_atomic)
1142 con->device_unlock(con, flags);
1143 return ret;
1144}
1145
1146/**
1147 * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup
1148 * @con: Console to operate on
1149 * @ctxt: The nbcon context from nbcon_context_try_acquire()
1150 *
1151 * Return: True if the thread should shutdown or if the console is
1152 * allowed to print and a record is available. False otherwise.
1153 *
1154 * After the thread wakes up, it must first check if it should shutdown before
1155 * attempting any printing.
1156 */
1157static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt)
1158{
1159 bool ret = false;
1160 short flags;
1161 int cookie;
1162
1163 if (kthread_should_stop())
1164 return true;
1165
1166 cookie = console_srcu_read_lock();
1167
1168 flags = console_srcu_read_flags(con);
1169 if (console_is_usable(con, flags, use_atomic: false)) {
1170 /* Bring the sequence in @ctxt up to date */
1171 ctxt->seq = nbcon_seq_read(con);
1172
1173 ret = prb_read_valid(rb: prb, seq: ctxt->seq, NULL);
1174 }
1175
1176 console_srcu_read_unlock(cookie);
1177 return ret;
1178}
1179
1180/**
1181 * nbcon_kthread_func - The printer thread function
1182 * @__console: Console to operate on
1183 *
1184 * Return: 0
1185 */
1186static int nbcon_kthread_func(void *__console)
1187{
1188 struct console *con = __console;
1189 struct nbcon_write_context wctxt = {
1190 .ctxt.console = con,
1191 .ctxt.prio = NBCON_PRIO_NORMAL,
1192 };
1193 struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
1194 short con_flags;
1195 bool backlog;
1196 int cookie;
1197
1198wait_for_event:
1199 /*
1200 * Guarantee this task is visible on the rcuwait before
1201 * checking the wake condition.
1202 *
1203 * The full memory barrier within set_current_state() of
1204 * ___rcuwait_wait_event() pairs with the full memory
1205 * barrier within rcuwait_has_sleeper().
1206 *
1207 * This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A.
1208 */
1209 rcuwait_wait_event(&con->rcuwait,
1210 nbcon_kthread_should_wakeup(con, ctxt),
1211 TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */
1212
1213 do {
1214 if (kthread_should_stop())
1215 return 0;
1216
1217 backlog = false;
1218
1219 /*
1220 * Keep the srcu read lock around the entire operation so that
1221 * synchronize_srcu() can guarantee that the kthread stopped
1222 * or suspended printing.
1223 */
1224 cookie = console_srcu_read_lock();
1225
1226 con_flags = console_srcu_read_flags(con);
1227
1228 if (console_is_usable(con, flags: con_flags, use_atomic: false))
1229 backlog = nbcon_emit_one(wctxt: &wctxt, use_atomic: false);
1230
1231 console_srcu_read_unlock(cookie);
1232
1233 cond_resched();
1234
1235 } while (backlog);
1236
1237 goto wait_for_event;
1238}
1239
1240/**
1241 * nbcon_irq_work - irq work to wake console printer thread
1242 * @irq_work: The irq work to operate on
1243 */
1244static void nbcon_irq_work(struct irq_work *irq_work)
1245{
1246 struct console *con = container_of(irq_work, struct console, irq_work);
1247
1248 nbcon_kthread_wake(con);
1249}
1250
1251static inline bool rcuwait_has_sleeper(struct rcuwait *w)
1252{
1253 /*
1254 * Guarantee any new records can be seen by tasks preparing to wait
1255 * before this context checks if the rcuwait is empty.
1256 *
1257 * This full memory barrier pairs with the full memory barrier within
1258 * set_current_state() of ___rcuwait_wait_event(), which is called
1259 * after prepare_to_rcuwait() adds the waiter but before it has
1260 * checked the wait condition.
1261 *
1262 * This pairs with nbcon_kthread_func:A.
1263 */
1264 smp_mb(); /* LMM(rcuwait_has_sleeper:A) */
1265 return rcuwait_active(w);
1266}
1267
1268/**
1269 * nbcon_kthreads_wake - Wake up printing threads using irq_work
1270 */
1271void nbcon_kthreads_wake(void)
1272{
1273 struct console *con;
1274 int cookie;
1275
1276 if (!printk_kthreads_running)
1277 return;
1278
1279 cookie = console_srcu_read_lock();
1280 for_each_console_srcu(con) {
1281 if (!(console_srcu_read_flags(con) & CON_NBCON))
1282 continue;
1283
1284 /*
1285 * Only schedule irq_work if the printing thread is
1286 * actively waiting. If not waiting, the thread will
1287 * notice by itself that it has work to do.
1288 */
1289 if (rcuwait_has_sleeper(w: &con->rcuwait))
1290 irq_work_queue(work: &con->irq_work);
1291 }
1292 console_srcu_read_unlock(cookie);
1293}
1294
1295/*
1296 * nbcon_kthread_stop - Stop a console printer thread
1297 * @con: Console to operate on
1298 */
1299void nbcon_kthread_stop(struct console *con)
1300{
1301 lockdep_assert_console_list_lock_held();
1302
1303 if (!con->kthread)
1304 return;
1305
1306 kthread_stop(k: con->kthread);
1307 con->kthread = NULL;
1308}
1309
1310/**
1311 * nbcon_kthread_create - Create a console printer thread
1312 * @con: Console to operate on
1313 *
1314 * Return: True if the kthread was started or already exists.
1315 * Otherwise false and @con must not be registered.
1316 *
1317 * This function is called when it will be expected that nbcon consoles are
1318 * flushed using the kthread. The messages printed with NBCON_PRIO_NORMAL
1319 * will be no longer flushed by the legacy loop. This is why failure must
1320 * be fatal for console registration.
1321 *
1322 * If @con was already registered and this function fails, @con must be
1323 * unregistered before the global state variable @printk_kthreads_running
1324 * can be set.
1325 */
1326bool nbcon_kthread_create(struct console *con)
1327{
1328 struct task_struct *kt;
1329
1330 lockdep_assert_console_list_lock_held();
1331
1332 if (con->kthread)
1333 return true;
1334
1335 kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index);
1336 if (WARN_ON(IS_ERR(kt))) {
1337 con_printk(KERN_ERR, con, "failed to start printing thread\n");
1338 return false;
1339 }
1340
1341 con->kthread = kt;
1342
1343 /*
1344 * It is important that console printing threads are scheduled
1345 * shortly after a printk call and with generous runtime budgets.
1346 */
1347 sched_set_normal(p: con->kthread, nice: -20);
1348
1349 return true;
1350}
1351
1352/* Track the nbcon emergency nesting per CPU. */
1353static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting);
1354static unsigned int early_nbcon_pcpu_emergency_nesting __initdata;
1355
1356/**
1357 * nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer
1358 *
1359 * Context: For reading, any context. For writing, any context which could
1360 * not be migrated to another CPU.
1361 * Return: Either a pointer to the per CPU emergency nesting counter of
1362 * the current CPU or to the init data during early boot.
1363 *
1364 * The function is safe for reading per-CPU variables in any context because
1365 * preemption is disabled if the current CPU is in the emergency state. See
1366 * also nbcon_cpu_emergency_enter().
1367 */
1368static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void)
1369{
1370 /*
1371 * The value of __printk_percpu_data_ready gets set in normal
1372 * context and before SMP initialization. As a result it could
1373 * never change while inside an nbcon emergency section.
1374 */
1375 if (!printk_percpu_data_ready())
1376 return &early_nbcon_pcpu_emergency_nesting;
1377
1378 return raw_cpu_ptr(&nbcon_pcpu_emergency_nesting);
1379}
1380
1381/**
1382 * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon
1383 * printing on the current CPU
1384 *
1385 * Context: Any context.
1386 * Return: The nbcon_prio to use for acquiring an nbcon console in this
1387 * context for printing.
1388 *
1389 * The function is safe for reading per-CPU data in any context because
1390 * preemption is disabled if the current CPU is in the emergency or panic
1391 * state.
1392 */
1393enum nbcon_prio nbcon_get_default_prio(void)
1394{
1395 unsigned int *cpu_emergency_nesting;
1396
1397 if (panic_on_this_cpu())
1398 return NBCON_PRIO_PANIC;
1399
1400 cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
1401 if (*cpu_emergency_nesting)
1402 return NBCON_PRIO_EMERGENCY;
1403
1404 return NBCON_PRIO_NORMAL;
1405}
1406
1407/**
1408 * nbcon_legacy_emit_next_record - Print one record for an nbcon console
1409 * in legacy contexts
1410 * @con: The console to print on
1411 * @handover: Will be set to true if a printk waiter has taken over the
1412 * console_lock, in which case the caller is no longer holding
1413 * both the console_lock and the SRCU read lock. Otherwise it
1414 * is set to false.
1415 * @cookie: The cookie from the SRCU read lock.
1416 * @use_atomic: Set true when called in an atomic or unknown context.
1417 * It affects which nbcon callback will be used: write_atomic()
1418 * or write_thread().
1419 *
1420 * When false, the write_thread() callback is used and would be
1421 * called in a preemtible context unless disabled by the
1422 * device_lock. The legacy handover is not allowed in this mode.
1423 *
1424 * Context: Any context except NMI.
1425 * Return: True, when a record has been printed and there are still
1426 * pending records. The caller might want to continue flushing.
1427 *
1428 * False, when there is no pending record, or when the console
1429 * context cannot be acquired, or the ownership has been lost.
1430 * The caller should give up. Either the job is done, cannot be
1431 * done, or will be handled by the owning context.
1432 *
1433 * This function is meant to be called by console_flush_all() to print records
1434 * on nbcon consoles from legacy context (printing via console unlocking).
1435 * Essentially it is the nbcon version of console_emit_next_record().
1436 */
1437bool nbcon_legacy_emit_next_record(struct console *con, bool *handover,
1438 int cookie, bool use_atomic)
1439{
1440 struct nbcon_write_context wctxt = { };
1441 struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
1442 unsigned long flags;
1443 bool progress;
1444
1445 ctxt->console = con;
1446 ctxt->prio = nbcon_get_default_prio();
1447
1448 if (use_atomic) {
1449 /*
1450 * In an atomic or unknown context, use the same procedure as
1451 * in console_emit_next_record(). It allows to handover.
1452 */
1453 printk_safe_enter_irqsave(flags);
1454 console_lock_spinning_enable();
1455 stop_critical_timings();
1456 }
1457
1458 progress = nbcon_emit_one(wctxt: &wctxt, use_atomic);
1459
1460 if (use_atomic) {
1461 start_critical_timings();
1462 *handover = console_lock_spinning_disable_and_check(cookie);
1463 printk_safe_exit_irqrestore(flags);
1464 } else {
1465 /* Non-atomic does not perform legacy spinning handovers. */
1466 *handover = false;
1467 }
1468
1469 return progress;
1470}
1471
1472/**
1473 * __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
1474 * write_atomic() callback
1475 * @con: The nbcon console to flush
1476 * @stop_seq: Flush up until this record
1477 * @allow_unsafe_takeover: True, to allow unsafe hostile takeovers
1478 *
1479 * Return: 0 if @con was flushed up to @stop_seq Otherwise, error code on
1480 * failure.
1481 *
1482 * Errors:
1483 *
1484 * -EPERM: Unable to acquire console ownership.
1485 *
1486 * -EAGAIN: Another context took over ownership while printing.
1487 *
1488 * -ENOENT: A record before @stop_seq is not available.
1489 *
1490 * If flushing up to @stop_seq was not successful, it only makes sense for the
1491 * caller to try again when -EAGAIN was returned. When -EPERM is returned,
1492 * this context is not allowed to acquire the console. When -ENOENT is
1493 * returned, it cannot be expected that the unfinalized record will become
1494 * available.
1495 */
1496static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,
1497 bool allow_unsafe_takeover)
1498{
1499 struct nbcon_write_context wctxt = { };
1500 struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
1501 int err = 0;
1502
1503 ctxt->console = con;
1504 ctxt->spinwait_max_us = 2000;
1505 ctxt->prio = nbcon_get_default_prio();
1506 ctxt->allow_unsafe_takeover = allow_unsafe_takeover;
1507
1508 if (!nbcon_context_try_acquire(ctxt, is_reacquire: false))
1509 return -EPERM;
1510
1511 while (nbcon_seq_read(con) < stop_seq) {
1512 /*
1513 * nbcon_emit_next_record() returns false when the console was
1514 * handed over or taken over. In both cases the context is no
1515 * longer valid.
1516 */
1517 if (!nbcon_emit_next_record(wctxt: &wctxt, use_atomic: true))
1518 return -EAGAIN;
1519
1520 if (!ctxt->backlog) {
1521 /* Are there reserved but not yet finalized records? */
1522 if (nbcon_seq_read(con) < stop_seq)
1523 err = -ENOENT;
1524 break;
1525 }
1526 }
1527
1528 nbcon_context_release(ctxt);
1529 return err;
1530}
1531
1532/**
1533 * nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
1534 * write_atomic() callback
1535 * @con: The nbcon console to flush
1536 * @stop_seq: Flush up until this record
1537 * @allow_unsafe_takeover: True, to allow unsafe hostile takeovers
1538 *
1539 * This will stop flushing before @stop_seq if another context has ownership.
1540 * That context is then responsible for the flushing. Likewise, if new records
1541 * are added while this context was flushing and there is no other context
1542 * to handle the printing, this context must also flush those records.
1543 */
1544static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,
1545 bool allow_unsafe_takeover)
1546{
1547 struct console_flush_type ft;
1548 unsigned long flags;
1549 int err;
1550
1551again:
1552 /*
1553 * Atomic flushing does not use console driver synchronization (i.e.
1554 * it does not hold the port lock for uart consoles). Therefore IRQs
1555 * must be disabled to avoid being interrupted and then calling into
1556 * a driver that will deadlock trying to acquire console ownership.
1557 */
1558 local_irq_save(flags);
1559
1560 err = __nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover);
1561
1562 local_irq_restore(flags);
1563
1564 /*
1565 * If there was a new owner (-EPERM, -EAGAIN), that context is
1566 * responsible for completing.
1567 *
1568 * Do not wait for records not yet finalized (-ENOENT) to avoid a
1569 * possible deadlock. They will either get flushed by the writer or
1570 * eventually skipped on panic CPU.
1571 */
1572 if (err)
1573 return;
1574
1575 /*
1576 * If flushing was successful but more records are available, this
1577 * context must flush those remaining records if the printer thread
1578 * is not available do it.
1579 */
1580 printk_get_console_flush_type(ft: &ft);
1581 if (!ft.nbcon_offload &&
1582 prb_read_valid(rb: prb, seq: nbcon_seq_read(con), NULL)) {
1583 stop_seq = prb_next_reserve_seq(rb: prb);
1584 goto again;
1585 }
1586}
1587
1588/**
1589 * __nbcon_atomic_flush_pending - Flush all nbcon consoles using their
1590 * write_atomic() callback
1591 * @stop_seq: Flush up until this record
1592 * @allow_unsafe_takeover: True, to allow unsafe hostile takeovers
1593 */
1594static void __nbcon_atomic_flush_pending(u64 stop_seq, bool allow_unsafe_takeover)
1595{
1596 struct console *con;
1597 int cookie;
1598
1599 cookie = console_srcu_read_lock();
1600 for_each_console_srcu(con) {
1601 short flags = console_srcu_read_flags(con);
1602
1603 if (!(flags & CON_NBCON))
1604 continue;
1605
1606 if (!console_is_usable(con, flags, use_atomic: true))
1607 continue;
1608
1609 if (nbcon_seq_read(con) >= stop_seq)
1610 continue;
1611
1612 nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover);
1613 }
1614 console_srcu_read_unlock(cookie);
1615}
1616
1617/**
1618 * nbcon_atomic_flush_pending - Flush all nbcon consoles using their
1619 * write_atomic() callback
1620 *
1621 * Flush the backlog up through the currently newest record. Any new
1622 * records added while flushing will not be flushed if there is another
1623 * context available to handle the flushing. This is to avoid one CPU
1624 * printing unbounded because other CPUs continue to add records.
1625 */
1626void nbcon_atomic_flush_pending(void)
1627{
1628 __nbcon_atomic_flush_pending(stop_seq: prb_next_reserve_seq(rb: prb), allow_unsafe_takeover: false);
1629}
1630
1631/**
1632 * nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their
1633 * write_atomic() callback and allowing unsafe hostile takeovers
1634 *
1635 * Flush the backlog up through the currently newest record. Unsafe hostile
1636 * takeovers will be performed, if necessary.
1637 */
1638void nbcon_atomic_flush_unsafe(void)
1639{
1640 __nbcon_atomic_flush_pending(stop_seq: prb_next_reserve_seq(rb: prb), allow_unsafe_takeover: true);
1641}
1642
1643/**
1644 * nbcon_cpu_emergency_enter - Enter an emergency section where printk()
1645 * messages for that CPU are flushed directly
1646 *
1647 * Context: Any context. Disables preemption.
1648 *
1649 * When within an emergency section, printk() calls will attempt to flush any
1650 * pending messages in the ringbuffer.
1651 */
1652void nbcon_cpu_emergency_enter(void)
1653{
1654 unsigned int *cpu_emergency_nesting;
1655
1656 preempt_disable();
1657
1658 cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
1659 (*cpu_emergency_nesting)++;
1660}
1661
1662/**
1663 * nbcon_cpu_emergency_exit - Exit an emergency section
1664 *
1665 * Context: Within an emergency section. Enables preemption.
1666 */
1667void nbcon_cpu_emergency_exit(void)
1668{
1669 unsigned int *cpu_emergency_nesting;
1670
1671 cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
1672
1673 if (!WARN_ON_ONCE(*cpu_emergency_nesting == 0))
1674 (*cpu_emergency_nesting)--;
1675
1676 preempt_enable();
1677}
1678
1679/**
1680 * nbcon_alloc - Allocate and init the nbcon console specific data
1681 * @con: Console to initialize
1682 *
1683 * Return: True if the console was fully allocated and initialized.
1684 * Otherwise @con must not be registered.
1685 *
1686 * When allocation and init was successful, the console must be properly
1687 * freed using nbcon_free() once it is no longer needed.
1688 */
1689bool nbcon_alloc(struct console *con)
1690{
1691 struct nbcon_state state = { };
1692
1693 /* Synchronize the kthread start. */
1694 lockdep_assert_console_list_lock_held();
1695
1696 /* The write_thread() callback is mandatory. */
1697 if (WARN_ON(!con->write_thread))
1698 return false;
1699
1700 rcuwait_init(w: &con->rcuwait);
1701 init_irq_work(work: &con->irq_work, func: nbcon_irq_work);
1702 atomic_long_set(v: &ACCESS_PRIVATE(con, nbcon_prev_seq), i: -1UL);
1703 nbcon_state_set(con, new: &state);
1704
1705 /*
1706 * Initialize @nbcon_seq to the highest possible sequence number so
1707 * that practically speaking it will have nothing to print until a
1708 * desired initial sequence number has been set via nbcon_seq_force().
1709 */
1710 atomic_long_set(v: &ACCESS_PRIVATE(con, nbcon_seq), ULSEQ_MAX(prb));
1711
1712 if (con->flags & CON_BOOT) {
1713 /*
1714 * Boot console printing is synchronized with legacy console
1715 * printing, so boot consoles can share the same global printk
1716 * buffers.
1717 */
1718 con->pbufs = &printk_shared_pbufs;
1719 } else {
1720 con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL);
1721 if (!con->pbufs) {
1722 con_printk(KERN_ERR, con, "failed to allocate printing buffer\n");
1723 return false;
1724 }
1725
1726 if (printk_kthreads_ready && !have_boot_console) {
1727 if (!nbcon_kthread_create(con)) {
1728 kfree(objp: con->pbufs);
1729 con->pbufs = NULL;
1730 return false;
1731 }
1732
1733 /* Might be the first kthread. */
1734 printk_kthreads_running = true;
1735 }
1736 }
1737
1738 return true;
1739}
1740
1741/**
1742 * nbcon_free - Free and cleanup the nbcon console specific data
1743 * @con: Console to free/cleanup nbcon data
1744 *
1745 * Important: @have_nbcon_console must be updated before calling
1746 * this function. In particular, it can be set only when there
1747 * is still another nbcon console registered.
1748 */
1749void nbcon_free(struct console *con)
1750{
1751 struct nbcon_state state = { };
1752
1753 /* Synchronize the kthread stop. */
1754 lockdep_assert_console_list_lock_held();
1755
1756 if (printk_kthreads_running) {
1757 nbcon_kthread_stop(con);
1758
1759 /* Might be the last nbcon console.
1760 *
1761 * Do not rely on printk_kthreads_check_locked(). It is not
1762 * called in some code paths, see nbcon_free() callers.
1763 */
1764 if (!have_nbcon_console)
1765 printk_kthreads_running = false;
1766 }
1767
1768 nbcon_state_set(con, new: &state);
1769
1770 /* Boot consoles share global printk buffers. */
1771 if (!(con->flags & CON_BOOT))
1772 kfree(objp: con->pbufs);
1773
1774 con->pbufs = NULL;
1775}
1776
1777/**
1778 * nbcon_device_try_acquire - Try to acquire nbcon console and enter unsafe
1779 * section
1780 * @con: The nbcon console to acquire
1781 *
1782 * Context: Under the locking mechanism implemented in
1783 * @con->device_lock() including disabling migration.
1784 * Return: True if the console was acquired. False otherwise.
1785 *
1786 * Console drivers will usually use their own internal synchronization
1787 * mechasism to synchronize between console printing and non-printing
1788 * activities (such as setting baud rates). However, nbcon console drivers
1789 * supporting atomic consoles may also want to mark unsafe sections when
1790 * performing non-printing activities in order to synchronize against their
1791 * atomic_write() callback.
1792 *
1793 * This function acquires the nbcon console using priority NBCON_PRIO_NORMAL
1794 * and marks it unsafe for handover/takeover.
1795 */
1796bool nbcon_device_try_acquire(struct console *con)
1797{
1798 struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);
1799
1800 cant_migrate();
1801
1802 memset(s: ctxt, c: 0, n: sizeof(*ctxt));
1803 ctxt->console = con;
1804 ctxt->prio = NBCON_PRIO_NORMAL;
1805
1806 if (!nbcon_context_try_acquire(ctxt, is_reacquire: false))
1807 return false;
1808
1809 if (!nbcon_context_enter_unsafe(ctxt))
1810 return false;
1811
1812 return true;
1813}
1814EXPORT_SYMBOL_GPL(nbcon_device_try_acquire);
1815
1816/**
1817 * nbcon_device_release - Exit unsafe section and release the nbcon console
1818 * @con: The nbcon console acquired in nbcon_device_try_acquire()
1819 */
1820void nbcon_device_release(struct console *con)
1821{
1822 struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);
1823 struct console_flush_type ft;
1824 int cookie;
1825
1826 if (!nbcon_context_exit_unsafe(ctxt))
1827 return;
1828
1829 nbcon_context_release(ctxt);
1830
1831 /*
1832 * This context must flush any new records added while the console
1833 * was locked if the printer thread is not available to do it. The
1834 * console_srcu_read_lock must be taken to ensure the console is
1835 * usable throughout flushing.
1836 */
1837 cookie = console_srcu_read_lock();
1838 printk_get_console_flush_type(ft: &ft);
1839 if (console_is_usable(con, flags: console_srcu_read_flags(con), use_atomic: true) &&
1840 !ft.nbcon_offload &&
1841 prb_read_valid(rb: prb, seq: nbcon_seq_read(con), NULL)) {
1842 /*
1843 * If nbcon_atomic flushing is not available, fallback to
1844 * using the legacy loop.
1845 */
1846 if (ft.nbcon_atomic) {
1847 __nbcon_atomic_flush_pending_con(con, stop_seq: prb_next_reserve_seq(rb: prb), allow_unsafe_takeover: false);
1848 } else if (ft.legacy_direct) {
1849 if (console_trylock())
1850 console_unlock();
1851 } else if (ft.legacy_offload) {
1852 printk_trigger_flush();
1853 }
1854 }
1855 console_srcu_read_unlock(cookie);
1856}
1857EXPORT_SYMBOL_GPL(nbcon_device_release);
1858