1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _KERNEL_PRINTK_RINGBUFFER_H
4#define _KERNEL_PRINTK_RINGBUFFER_H
5
6#include <linux/atomic.h>
7#include <linux/bits.h>
8#include <linux/dev_printk.h>
9#include <linux/stddef.h>
10#include <linux/types.h>
11
12/*
13 * Meta information about each stored message.
14 *
15 * All fields are set by the printk code except for @seq, which is
16 * set by the ringbuffer code.
17 */
18struct printk_info {
19 u64 seq; /* sequence number */
20 u64 ts_nsec; /* timestamp in nanoseconds */
21 u16 text_len; /* length of text message */
22 u8 facility; /* syslog facility */
23 u8 flags:5; /* internal record flags */
24 u8 level:3; /* syslog level */
25 u32 caller_id; /* thread id or processor id */
26
27 struct dev_printk_info dev_info;
28};
29
30/*
31 * A structure providing the buffers, used by writers and readers.
32 *
33 * Writers:
34 * Using prb_rec_init_wr(), a writer sets @text_buf_size before calling
35 * prb_reserve(). On success, prb_reserve() sets @info and @text_buf to
36 * buffers reserved for that writer.
37 *
38 * Readers:
39 * Using prb_rec_init_rd(), a reader sets all fields before calling
40 * prb_read_valid(). Note that the reader provides the @info and @text_buf,
41 * buffers. On success, the struct pointed to by @info will be filled and
42 * the char array pointed to by @text_buf will be filled with text data.
43 */
44struct printk_record {
45 struct printk_info *info;
46 char *text_buf;
47 unsigned int text_buf_size;
48};
49
50/* Specifies the logical position and span of a data block. */
51struct prb_data_blk_lpos {
52 unsigned long begin;
53 unsigned long next;
54};
55
56/*
57 * A descriptor: the complete meta-data for a record.
58 *
59 * @state_var: A bitwise combination of descriptor ID and descriptor state.
60 */
61struct prb_desc {
62 atomic_long_t state_var;
63 struct prb_data_blk_lpos text_blk_lpos;
64};
65
66/* A ringbuffer of "ID + data" elements. */
67struct prb_data_ring {
68 unsigned int size_bits;
69 char *data;
70 atomic_long_t head_lpos;
71 atomic_long_t tail_lpos;
72};
73
74/* A ringbuffer of "struct prb_desc" elements. */
75struct prb_desc_ring {
76 unsigned int count_bits;
77 struct prb_desc *descs;
78 struct printk_info *infos;
79 atomic_long_t head_id;
80 atomic_long_t tail_id;
81 atomic_long_t last_finalized_seq;
82};
83
84/*
85 * The high level structure representing the printk ringbuffer.
86 *
87 * @fail: Count of failed prb_reserve() calls where not even a data-less
88 * record was created.
89 */
90struct printk_ringbuffer {
91 struct prb_desc_ring desc_ring;
92 struct prb_data_ring text_data_ring;
93 atomic_long_t fail;
94};
95
96/*
97 * Used by writers as a reserve/commit handle.
98 *
99 * @rb: Ringbuffer where the entry is reserved.
100 * @irqflags: Saved irq flags to restore on entry commit.
101 * @id: ID of the reserved descriptor.
102 * @text_space: Total occupied buffer space in the text data ring, including
103 * ID, alignment padding, and wrapping data blocks.
104 *
105 * This structure is an opaque handle for writers. Its contents are only
106 * to be used by the ringbuffer implementation.
107 */
108struct prb_reserved_entry {
109 struct printk_ringbuffer *rb;
110 unsigned long irqflags;
111 unsigned long id;
112 unsigned int text_space;
113};
114
115/* The possible responses of a descriptor state-query. */
116enum desc_state {
117 desc_miss = -1, /* ID mismatch (pseudo state) */
118 desc_reserved = 0x0, /* reserved, in use by writer */
119 desc_committed = 0x1, /* committed by writer, could get reopened */
120 desc_finalized = 0x2, /* committed, no further modification allowed */
121 desc_reusable = 0x3, /* free, not yet used by any writer */
122};
123
124#define _DATA_SIZE(sz_bits) (1UL << (sz_bits))
125#define _DESCS_COUNT(ct_bits) (1U << (ct_bits))
126#define DESC_SV_BITS BITS_PER_LONG
127#define DESC_FLAGS_SHIFT (DESC_SV_BITS - 2)
128#define DESC_FLAGS_MASK (3UL << DESC_FLAGS_SHIFT)
129#define DESC_STATE(sv) (3UL & (sv >> DESC_FLAGS_SHIFT))
130#define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id)
131#define DESC_ID_MASK (~DESC_FLAGS_MASK)
132#define DESC_ID(sv) ((sv) & DESC_ID_MASK)
133
134/*
135 * Special data block logical position values (for fields of
136 * @prb_desc.text_blk_lpos).
137 *
138 * - Bit0 is used to identify if the record has no data block. (Implemented in
139 * the LPOS_DATALESS() macro.)
140 *
141 * - Bit1 specifies the reason for not having a data block.
142 *
143 * These special values could never be real lpos values because of the
144 * meta data and alignment padding of data blocks. (See to_blk_size() for
145 * details.)
146 */
147#define FAILED_LPOS 0x1
148#define EMPTY_LINE_LPOS 0x3
149
150#define FAILED_BLK_LPOS \
151{ \
152 .begin = FAILED_LPOS, \
153 .next = FAILED_LPOS, \
154}
155
156/*
157 * Descriptor Bootstrap
158 *
159 * The descriptor array is minimally initialized to allow immediate usage
160 * by readers and writers. The requirements that the descriptor array
161 * initialization must satisfy:
162 *
163 * Req1
164 * The tail must point to an existing (committed or reusable) descriptor.
165 * This is required by the implementation of prb_first_seq().
166 *
167 * Req2
168 * Readers must see that the ringbuffer is initially empty.
169 *
170 * Req3
171 * The first record reserved by a writer is assigned sequence number 0.
172 *
173 * To satisfy Req1, the tail initially points to a descriptor that is
174 * minimally initialized (having no data block, i.e. data-less with the
175 * data block's lpos @begin and @next values set to FAILED_LPOS).
176 *
177 * To satisfy Req2, the initial tail descriptor is initialized to the
178 * reusable state. Readers recognize reusable descriptors as existing
179 * records, but skip over them.
180 *
181 * To satisfy Req3, the last descriptor in the array is used as the initial
182 * head (and tail) descriptor. This allows the first record reserved by a
183 * writer (head + 1) to be the first descriptor in the array. (Only the first
184 * descriptor in the array could have a valid sequence number of 0.)
185 *
186 * The first time a descriptor is reserved, it is assigned a sequence number
187 * with the value of the array index. A "first time reserved" descriptor can
188 * be recognized because it has a sequence number of 0 but does not have an
189 * index of 0. (Only the first descriptor in the array could have a valid
190 * sequence number of 0.) After the first reservation, all future reservations
191 * (recycling) simply involve incrementing the sequence number by the array
192 * count.
193 *
194 * Hack #1
195 * Only the first descriptor in the array is allowed to have the sequence
196 * number 0. In this case it is not possible to recognize if it is being
197 * reserved the first time (set to index value) or has been reserved
198 * previously (increment by the array count). This is handled by _always_
199 * incrementing the sequence number by the array count when reserving the
200 * first descriptor in the array. In order to satisfy Req3, the sequence
201 * number of the first descriptor in the array is initialized to minus
202 * the array count. Then, upon the first reservation, it is incremented
203 * to 0, thus satisfying Req3.
204 *
205 * Hack #2
206 * prb_first_seq() can be called at any time by readers to retrieve the
207 * sequence number of the tail descriptor. However, due to Req2 and Req3,
208 * initially there are no records to report the sequence number of
209 * (sequence numbers are u64 and there is nothing less than 0). To handle
210 * this, the sequence number of the initial tail descriptor is initialized
211 * to 0. Technically this is incorrect, because there is no record with
212 * sequence number 0 (yet) and the tail descriptor is not the first
213 * descriptor in the array. But it allows prb_read_valid() to correctly
214 * report the existence of a record for _any_ given sequence number at all
215 * times. Bootstrapping is complete when the tail is pushed the first
216 * time, thus finally pointing to the first descriptor reserved by a
217 * writer, which has the assigned sequence number 0.
218 */
219
220/*
221 * Initiating Logical Value Overflows
222 *
223 * Both logical position (lpos) and ID values can be mapped to array indexes
224 * but may experience overflows during the lifetime of the system. To ensure
225 * that printk_ringbuffer can handle the overflows for these types, initial
226 * values are chosen that map to the correct initial array indexes, but will
227 * result in overflows soon.
228 *
229 * BLK0_LPOS
230 * The initial @head_lpos and @tail_lpos for data rings. It is at index
231 * 0 and the lpos value is such that it will overflow on the first wrap.
232 *
233 * DESC0_ID
234 * The initial @head_id and @tail_id for the desc ring. It is at the last
235 * index of the descriptor array (see Req3 above) and the ID value is such
236 * that it will overflow on the second wrap.
237 */
238#define BLK0_LPOS(sz_bits) (-(_DATA_SIZE(sz_bits)))
239#define DESC0_ID(ct_bits) DESC_ID(-(_DESCS_COUNT(ct_bits) + 1))
240#define DESC0_SV(ct_bits) DESC_SV(DESC0_ID(ct_bits), desc_reusable)
241
242/*
243 * Define a ringbuffer with an external text data buffer. The same as
244 * DEFINE_PRINTKRB() but requires specifying an external buffer for the
245 * text data.
246 *
247 * Note: The specified external buffer must be of the size:
248 * 2 ^ (descbits + avgtextbits)
249 */
250#define _DEFINE_PRINTKRB(name, descbits, avgtextbits, text_buf) \
251static struct prb_desc _##name##_descs[_DESCS_COUNT(descbits)] = { \
252 /* the initial head and tail */ \
253 [_DESCS_COUNT(descbits) - 1] = { \
254 /* reusable */ \
255 .state_var = ATOMIC_INIT(DESC0_SV(descbits)), \
256 /* no associated data block */ \
257 .text_blk_lpos = FAILED_BLK_LPOS, \
258 }, \
259}; \
260static struct printk_info _##name##_infos[_DESCS_COUNT(descbits)] = { \
261 /* this will be the first record reserved by a writer */ \
262 [0] = { \
263 /* will be incremented to 0 on the first reservation */ \
264 .seq = -(u64)_DESCS_COUNT(descbits), \
265 }, \
266 /* the initial head and tail */ \
267 [_DESCS_COUNT(descbits) - 1] = { \
268 /* reports the first seq value during the bootstrap phase */ \
269 .seq = 0, \
270 }, \
271}; \
272static struct printk_ringbuffer name = { \
273 .desc_ring = { \
274 .count_bits = descbits, \
275 .descs = &_##name##_descs[0], \
276 .infos = &_##name##_infos[0], \
277 .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \
278 .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \
279 .last_finalized_seq = ATOMIC_INIT(0), \
280 }, \
281 .text_data_ring = { \
282 .size_bits = (avgtextbits) + (descbits), \
283 .data = text_buf, \
284 .head_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \
285 .tail_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \
286 }, \
287 .fail = ATOMIC_LONG_INIT(0), \
288}
289
290/**
291 * DEFINE_PRINTKRB() - Define a ringbuffer.
292 *
293 * @name: The name of the ringbuffer variable.
294 * @descbits: The number of descriptors as a power-of-2 value.
295 * @avgtextbits: The average text data size per record as a power-of-2 value.
296 *
297 * This is a macro for defining a ringbuffer and all internal structures
298 * such that it is ready for immediate use. See _DEFINE_PRINTKRB() for a
299 * variant where the text data buffer can be specified externally.
300 */
301#define DEFINE_PRINTKRB(name, descbits, avgtextbits) \
302static char _##name##_text[1U << ((avgtextbits) + (descbits))] \
303 __aligned(__alignof__(unsigned long)); \
304_DEFINE_PRINTKRB(name, descbits, avgtextbits, &_##name##_text[0])
305
306/* Writer Interface */
307
308/**
309 * prb_rec_init_wr() - Initialize a buffer for writing records.
310 *
311 * @r: The record to initialize.
312 * @text_buf_size: The needed text buffer size.
313 */
314static inline void prb_rec_init_wr(struct printk_record *r,
315 unsigned int text_buf_size)
316{
317 r->info = NULL;
318 r->text_buf = NULL;
319 r->text_buf_size = text_buf_size;
320}
321
322bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
323 struct printk_record *r);
324bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb,
325 struct printk_record *r, u32 caller_id, unsigned int max_size);
326void prb_commit(struct prb_reserved_entry *e);
327void prb_final_commit(struct prb_reserved_entry *e);
328
329void prb_init(struct printk_ringbuffer *rb,
330 char *text_buf, unsigned int text_buf_size,
331 struct prb_desc *descs, unsigned int descs_count_bits,
332 struct printk_info *infos);
333unsigned int prb_record_text_space(struct prb_reserved_entry *e);
334
335/* Reader Interface */
336
337/**
338 * prb_rec_init_rd() - Initialize a buffer for reading records.
339 *
340 * @r: The record to initialize.
341 * @info: A buffer to store record meta-data.
342 * @text_buf: A buffer to store text data.
343 * @text_buf_size: The size of @text_buf.
344 *
345 * Initialize all the fields that a reader is interested in. All arguments
346 * (except @r) are optional. Only record data for arguments that are
347 * non-NULL or non-zero will be read.
348 */
349static inline void prb_rec_init_rd(struct printk_record *r,
350 struct printk_info *info,
351 char *text_buf, unsigned int text_buf_size)
352{
353 r->info = info;
354 r->text_buf = text_buf;
355 r->text_buf_size = text_buf_size;
356}
357
358/**
359 * prb_for_each_record() - Iterate over the records of a ringbuffer.
360 *
361 * @from: The sequence number to begin with.
362 * @rb: The ringbuffer to iterate over.
363 * @s: A u64 to store the sequence number on each iteration.
364 * @r: A printk_record to store the record on each iteration.
365 *
366 * This is a macro for conveniently iterating over a ringbuffer.
367 * Note that @s may not be the sequence number of the record on each
368 * iteration. For the sequence number, @r->info->seq should be checked.
369 *
370 * Context: Any context.
371 */
372#define prb_for_each_record(from, rb, s, r) \
373for ((s) = from; prb_read_valid(rb, s, r); (s) = (r)->info->seq + 1)
374
375/**
376 * prb_for_each_info() - Iterate over the meta data of a ringbuffer.
377 *
378 * @from: The sequence number to begin with.
379 * @rb: The ringbuffer to iterate over.
380 * @s: A u64 to store the sequence number on each iteration.
381 * @i: A printk_info to store the record meta data on each iteration.
382 * @lc: An unsigned int to store the text line count of each record.
383 *
384 * This is a macro for conveniently iterating over a ringbuffer.
385 * Note that @s may not be the sequence number of the record on each
386 * iteration. For the sequence number, @r->info->seq should be checked.
387 *
388 * Context: Any context.
389 */
390#define prb_for_each_info(from, rb, s, i, lc) \
391for ((s) = from; prb_read_valid_info(rb, s, i, lc); (s) = (i)->seq + 1)
392
393bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq,
394 struct printk_record *r);
395bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq,
396 struct printk_info *info, unsigned int *line_count);
397
398u64 prb_first_seq(struct printk_ringbuffer *rb);
399u64 prb_first_valid_seq(struct printk_ringbuffer *rb);
400u64 prb_next_seq(struct printk_ringbuffer *rb);
401u64 prb_next_reserve_seq(struct printk_ringbuffer *rb);
402
403#ifdef CONFIG_64BIT
404
405#define __u64seq_to_ulseq(u64seq) (u64seq)
406#define __ulseq_to_u64seq(rb, ulseq) (ulseq)
407#define ULSEQ_MAX(rb) (-1)
408
409#else /* CONFIG_64BIT */
410
411#define __u64seq_to_ulseq(u64seq) ((u32)u64seq)
412#define ULSEQ_MAX(rb) __u64seq_to_ulseq(prb_first_seq(rb) + 0x80000000UL)
413
414static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq)
415{
416 u64 rb_first_seq = prb_first_seq(rb);
417 u64 seq;
418
419 /*
420 * The provided sequence is only the lower 32 bits of the ringbuffer
421 * sequence. It needs to be expanded to 64bit. Get the first sequence
422 * number from the ringbuffer and fold it.
423 *
424 * Having a 32bit representation in the console is sufficient.
425 * If a console ever gets more than 2^31 records behind
426 * the ringbuffer then this is the least of the problems.
427 *
428 * Also the access to the ring buffer is always safe.
429 */
430 seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq);
431
432 return seq;
433}
434
435#endif /* CONFIG_64BIT */
436
437#endif /* _KERNEL_PRINTK_RINGBUFFER_H */
438