1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * A generic kernel FIFO implementation
4 *
5 * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net>
6 */
7
8#ifndef _LINUX_KFIFO_H
9#define _LINUX_KFIFO_H
10
11/*
12 * How to porting drivers to the new generic FIFO API:
13 *
14 * - Modify the declaration of the "struct kfifo *" object into a
15 * in-place "struct kfifo" object
16 * - Init the in-place object with kfifo_alloc() or kfifo_init()
17 * Note: The address of the in-place "struct kfifo" object must be
18 * passed as the first argument to this functions
19 * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
20 * into kfifo_out
21 * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
22 * into kfifo_out_spinlocked
23 * Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
24 * must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
25 * as the last parameter
26 * - The formerly __kfifo_* functions are renamed into kfifo_*
27 */
28
29/*
30 * Note about locking: There is no locking required until only one reader
31 * and one writer is using the fifo and no kfifo_reset() will be called.
32 * kfifo_reset_out() can be safely used, until it will be only called
33 * in the reader thread.
34 * For multiple writer and one reader there is only a need to lock the writer.
35 * And vice versa for only one writer and multiple reader there is only a need
36 * to lock the reader.
37 */
38
39#include <linux/array_size.h>
40#include <linux/spinlock.h>
41#include <linux/stddef.h>
42#include <linux/types.h>
43
44#include <asm/barrier.h>
45#include <asm/errno.h>
46
47struct scatterlist;
48
49struct __kfifo {
50 unsigned int in;
51 unsigned int out;
52 unsigned int mask;
53 unsigned int esize;
54 void *data;
55};
56
57#define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
58 union { \
59 struct __kfifo kfifo; \
60 datatype *type; \
61 const datatype *const_type; \
62 char (*rectype)[recsize]; \
63 ptrtype *ptr; \
64 ptrtype const *ptr_const; \
65 }
66
67#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
68{ \
69 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
70 type buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
71}
72
73#define STRUCT_KFIFO(type, size) \
74 struct __STRUCT_KFIFO(type, size, 0, type)
75
76#define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
77{ \
78 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
79 type buf[0]; \
80}
81
82#define STRUCT_KFIFO_PTR(type) \
83 struct __STRUCT_KFIFO_PTR(type, 0, type)
84
85/*
86 * define compatibility "struct kfifo" for dynamic allocated fifos
87 */
88struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
89
90#define STRUCT_KFIFO_REC_1(size) \
91 struct __STRUCT_KFIFO(unsigned char, size, 1, void)
92
93#define STRUCT_KFIFO_REC_2(size) \
94 struct __STRUCT_KFIFO(unsigned char, size, 2, void)
95
96/*
97 * define kfifo_rec types
98 */
99struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
100struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
101
102/*
103 * helper macro to distinguish between real in place fifo where the fifo
104 * array is a part of the structure and the fifo type where the array is
105 * outside of the fifo structure.
106 */
107#define __is_kfifo_ptr(fifo) \
108 (sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
109
110/**
111 * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
112 * @fifo: name of the declared fifo
113 * @type: type of the fifo elements
114 */
115#define DECLARE_KFIFO_PTR(fifo, type) STRUCT_KFIFO_PTR(type) fifo
116
117/**
118 * DECLARE_KFIFO - macro to declare a fifo object
119 * @fifo: name of the declared fifo
120 * @type: type of the fifo elements
121 * @size: the number of elements in the fifo, this must be a power of 2
122 */
123#define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo
124
125/**
126 * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
127 * @fifo: name of the declared fifo datatype
128 */
129#define INIT_KFIFO(fifo) \
130(void)({ \
131 typeof(&(fifo)) __tmp = &(fifo); \
132 struct __kfifo *__kfifo = &__tmp->kfifo; \
133 __kfifo->in = 0; \
134 __kfifo->out = 0; \
135 __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
136 __kfifo->esize = sizeof(*__tmp->buf); \
137 __kfifo->data = __is_kfifo_ptr(__tmp) ? NULL : __tmp->buf; \
138})
139
140/**
141 * DEFINE_KFIFO - macro to define and initialize a fifo
142 * @fifo: name of the declared fifo datatype
143 * @type: type of the fifo elements
144 * @size: the number of elements in the fifo, this must be a power of 2
145 *
146 * Note: the macro can be used for global and local fifo data type variables.
147 */
148#define DEFINE_KFIFO(fifo, type, size) \
149 DECLARE_KFIFO(fifo, type, size) = \
150 (typeof(fifo)) { \
151 { \
152 { \
153 .in = 0, \
154 .out = 0, \
155 .mask = __is_kfifo_ptr(&(fifo)) ? \
156 0 : \
157 ARRAY_SIZE((fifo).buf) - 1, \
158 .esize = sizeof(*(fifo).buf), \
159 .data = __is_kfifo_ptr(&(fifo)) ? \
160 NULL : \
161 (fifo).buf, \
162 } \
163 } \
164 }
165
166
167static inline unsigned int __must_check
168__kfifo_uint_must_check_helper(unsigned int val)
169{
170 return val;
171}
172
173static inline int __must_check
174__kfifo_int_must_check_helper(int val)
175{
176 return val;
177}
178
179/**
180 * kfifo_initialized - Check if the fifo is initialized
181 * @fifo: address of the fifo to check
182 *
183 * Return %true if fifo is initialized, otherwise %false.
184 * Assumes the fifo was 0 before.
185 */
186#define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
187
188/**
189 * kfifo_esize - returns the size of the element managed by the fifo
190 * @fifo: address of the fifo to be used
191 */
192#define kfifo_esize(fifo) ((fifo)->kfifo.esize)
193
194/**
195 * kfifo_recsize - returns the size of the record length field
196 * @fifo: address of the fifo to be used
197 */
198#define kfifo_recsize(fifo) (sizeof(*(fifo)->rectype))
199
200/**
201 * kfifo_size - returns the size of the fifo in elements
202 * @fifo: address of the fifo to be used
203 */
204#define kfifo_size(fifo) ((fifo)->kfifo.mask + 1)
205
206/**
207 * kfifo_reset - removes the entire fifo content
208 * @fifo: address of the fifo to be used
209 *
210 * Note: usage of kfifo_reset() is dangerous. It should be only called when the
211 * fifo is exclusived locked or when it is secured that no other thread is
212 * accessing the fifo.
213 */
214#define kfifo_reset(fifo) \
215(void)({ \
216 typeof((fifo) + 1) __tmp = (fifo); \
217 __tmp->kfifo.in = __tmp->kfifo.out = 0; \
218})
219
220/**
221 * kfifo_reset_out - skip fifo content
222 * @fifo: address of the fifo to be used
223 *
224 * Note: The usage of kfifo_reset_out() is safe until it will be only called
225 * from the reader thread and there is only one concurrent reader. Otherwise
226 * it is dangerous and must be handled in the same way as kfifo_reset().
227 */
228#define kfifo_reset_out(fifo) \
229(void)({ \
230 typeof((fifo) + 1) __tmp = (fifo); \
231 __tmp->kfifo.out = __tmp->kfifo.in; \
232})
233
234/**
235 * kfifo_len - returns the number of used elements in the fifo
236 * @fifo: address of the fifo to be used
237 */
238#define kfifo_len(fifo) \
239({ \
240 typeof((fifo) + 1) __tmpl = (fifo); \
241 __tmpl->kfifo.in - __tmpl->kfifo.out; \
242})
243
244/**
245 * kfifo_is_empty - returns true if the fifo is empty
246 * @fifo: address of the fifo to be used
247 */
248#define kfifo_is_empty(fifo) \
249({ \
250 typeof((fifo) + 1) __tmpq = (fifo); \
251 __tmpq->kfifo.in == __tmpq->kfifo.out; \
252})
253
254/**
255 * kfifo_is_empty_spinlocked - returns true if the fifo is empty using
256 * a spinlock for locking
257 * @fifo: address of the fifo to be used
258 * @lock: spinlock to be used for locking
259 */
260#define kfifo_is_empty_spinlocked(fifo, lock) \
261({ \
262 unsigned long __flags; \
263 bool __ret; \
264 spin_lock_irqsave(lock, __flags); \
265 __ret = kfifo_is_empty(fifo); \
266 spin_unlock_irqrestore(lock, __flags); \
267 __ret; \
268})
269
270/**
271 * kfifo_is_empty_spinlocked_noirqsave - returns true if the fifo is empty
272 * using a spinlock for locking, doesn't disable interrupts
273 * @fifo: address of the fifo to be used
274 * @lock: spinlock to be used for locking
275 */
276#define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \
277({ \
278 bool __ret; \
279 spin_lock(lock); \
280 __ret = kfifo_is_empty(fifo); \
281 spin_unlock(lock); \
282 __ret; \
283})
284
285/**
286 * kfifo_is_full - returns true if the fifo is full
287 * @fifo: address of the fifo to be used
288 */
289#define kfifo_is_full(fifo) \
290({ \
291 typeof((fifo) + 1) __tmpq = (fifo); \
292 kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
293})
294
295/**
296 * kfifo_avail - returns the number of unused elements in the fifo
297 * @fifo: address of the fifo to be used
298 */
299#define kfifo_avail(fifo) \
300__kfifo_uint_must_check_helper( \
301({ \
302 typeof((fifo) + 1) __tmpq = (fifo); \
303 const size_t __recsize = sizeof(*__tmpq->rectype); \
304 unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
305 (__recsize) ? ((__avail <= __recsize) ? 0 : \
306 __kfifo_max_r(__avail - __recsize, __recsize)) : \
307 __avail; \
308}) \
309)
310
311/**
312 * kfifo_skip_count - skip output data
313 * @fifo: address of the fifo to be used
314 * @count: count of data to skip
315 */
316#define kfifo_skip_count(fifo, count) do { \
317 typeof((fifo) + 1) __tmp = (fifo); \
318 const size_t __recsize = sizeof(*__tmp->rectype); \
319 struct __kfifo *__kfifo = &__tmp->kfifo; \
320 if (__recsize) \
321 __kfifo_skip_r(__kfifo, __recsize); \
322 else \
323 __kfifo->out += (count); \
324} while(0)
325
326/**
327 * kfifo_skip - skip output data
328 * @fifo: address of the fifo to be used
329 */
330#define kfifo_skip(fifo) kfifo_skip_count(fifo, 1)
331
332/**
333 * kfifo_peek_len - gets the size of the next fifo record
334 * @fifo: address of the fifo to be used
335 *
336 * This function returns the size of the next fifo record in number of bytes.
337 */
338#define kfifo_peek_len(fifo) \
339__kfifo_uint_must_check_helper( \
340({ \
341 typeof((fifo) + 1) __tmp = (fifo); \
342 const size_t __recsize = sizeof(*__tmp->rectype); \
343 struct __kfifo *__kfifo = &__tmp->kfifo; \
344 (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
345 __kfifo_len_r(__kfifo, __recsize); \
346}) \
347)
348
349/**
350 * kfifo_alloc - dynamically allocates a new fifo buffer
351 * @fifo: pointer to the fifo
352 * @size: the number of elements in the fifo, this must be a power of 2
353 * @gfp_mask: get_free_pages mask, passed to kmalloc()
354 *
355 * This macro dynamically allocates a new fifo buffer.
356 *
357 * The number of elements will be rounded-up to a power of 2.
358 * The fifo will be release with kfifo_free().
359 * Return 0 if no error, otherwise an error code.
360 */
361#define kfifo_alloc(fifo, size, gfp_mask) \
362__kfifo_int_must_check_helper( \
363({ \
364 typeof((fifo) + 1) __tmp = (fifo); \
365 struct __kfifo *__kfifo = &__tmp->kfifo; \
366 __is_kfifo_ptr(__tmp) ? \
367 __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
368 -EINVAL; \
369}) \
370)
371
372/**
373 * kfifo_free - frees the fifo
374 * @fifo: the fifo to be freed
375 */
376#define kfifo_free(fifo) \
377({ \
378 typeof((fifo) + 1) __tmp = (fifo); \
379 struct __kfifo *__kfifo = &__tmp->kfifo; \
380 if (__is_kfifo_ptr(__tmp)) \
381 __kfifo_free(__kfifo); \
382})
383
384/**
385 * kfifo_init - initialize a fifo using a preallocated buffer
386 * @fifo: the fifo to assign the buffer
387 * @buffer: the preallocated buffer to be used
388 * @size: the size of the internal buffer, this have to be a power of 2
389 *
390 * This macro initializes a fifo using a preallocated buffer.
391 *
392 * The number of elements will be rounded-up to a power of 2.
393 * Return 0 if no error, otherwise an error code.
394 */
395#define kfifo_init(fifo, buffer, size) \
396({ \
397 typeof((fifo) + 1) __tmp = (fifo); \
398 struct __kfifo *__kfifo = &__tmp->kfifo; \
399 __is_kfifo_ptr(__tmp) ? \
400 __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
401 -EINVAL; \
402})
403
404/**
405 * kfifo_put - put data into the fifo
406 * @fifo: address of the fifo to be used
407 * @val: the data to be added
408 *
409 * This macro copies the given value into the fifo.
410 * It returns 0 if the fifo was full. Otherwise it returns the number
411 * processed elements.
412 *
413 * Note that with only one concurrent reader and one concurrent
414 * writer, you don't need extra locking to use these macro.
415 */
416#define kfifo_put(fifo, val) \
417({ \
418 typeof((fifo) + 1) __tmp = (fifo); \
419 typeof(*__tmp->const_type) __val = (val); \
420 unsigned int __ret; \
421 size_t __recsize = sizeof(*__tmp->rectype); \
422 struct __kfifo *__kfifo = &__tmp->kfifo; \
423 if (__recsize) \
424 __ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
425 __recsize); \
426 else { \
427 __ret = !kfifo_is_full(__tmp); \
428 if (__ret) { \
429 (__is_kfifo_ptr(__tmp) ? \
430 ((typeof(__tmp->type))__kfifo->data) : \
431 (__tmp->buf) \
432 )[__kfifo->in & __tmp->kfifo.mask] = \
433 *(typeof(__tmp->type))&__val; \
434 smp_wmb(); \
435 __kfifo->in++; \
436 } \
437 } \
438 __ret; \
439})
440
441/**
442 * kfifo_get - get data from the fifo
443 * @fifo: address of the fifo to be used
444 * @val: address where to store the data
445 *
446 * This macro reads the data from the fifo.
447 * It returns 0 if the fifo was empty. Otherwise it returns the number
448 * processed elements.
449 *
450 * Note that with only one concurrent reader and one concurrent
451 * writer, you don't need extra locking to use these macro.
452 */
453#define kfifo_get(fifo, val) \
454__kfifo_uint_must_check_helper( \
455({ \
456 typeof((fifo) + 1) __tmp = (fifo); \
457 typeof(__tmp->ptr) __val = (val); \
458 unsigned int __ret; \
459 const size_t __recsize = sizeof(*__tmp->rectype); \
460 struct __kfifo *__kfifo = &__tmp->kfifo; \
461 if (__recsize) \
462 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
463 __recsize); \
464 else { \
465 __ret = !kfifo_is_empty(__tmp); \
466 if (__ret) { \
467 *(typeof(__tmp->type))__val = \
468 (__is_kfifo_ptr(__tmp) ? \
469 ((typeof(__tmp->type))__kfifo->data) : \
470 (__tmp->buf) \
471 )[__kfifo->out & __tmp->kfifo.mask]; \
472 smp_wmb(); \
473 __kfifo->out++; \
474 } \
475 } \
476 __ret; \
477}) \
478)
479
480/**
481 * kfifo_peek - get data from the fifo without removing
482 * @fifo: address of the fifo to be used
483 * @val: address where to store the data
484 *
485 * This reads the data from the fifo without removing it from the fifo.
486 * It returns 0 if the fifo was empty. Otherwise it returns the number
487 * processed elements.
488 *
489 * Note that with only one concurrent reader and one concurrent
490 * writer, you don't need extra locking to use these macro.
491 */
492#define kfifo_peek(fifo, val) \
493__kfifo_uint_must_check_helper( \
494({ \
495 typeof((fifo) + 1) __tmp = (fifo); \
496 typeof(__tmp->ptr) __val = (val); \
497 unsigned int __ret; \
498 const size_t __recsize = sizeof(*__tmp->rectype); \
499 struct __kfifo *__kfifo = &__tmp->kfifo; \
500 if (__recsize) \
501 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
502 __recsize); \
503 else { \
504 __ret = !kfifo_is_empty(__tmp); \
505 if (__ret) { \
506 *(typeof(__tmp->type))__val = \
507 (__is_kfifo_ptr(__tmp) ? \
508 ((typeof(__tmp->type))__kfifo->data) : \
509 (__tmp->buf) \
510 )[__kfifo->out & __tmp->kfifo.mask]; \
511 smp_wmb(); \
512 } \
513 } \
514 __ret; \
515}) \
516)
517
518/**
519 * kfifo_in - put data into the fifo
520 * @fifo: address of the fifo to be used
521 * @buf: the data to be added
522 * @n: number of elements to be added
523 *
524 * This macro copies the given buffer into the fifo and returns the
525 * number of copied elements.
526 *
527 * Note that with only one concurrent reader and one concurrent
528 * writer, you don't need extra locking to use these macro.
529 */
530#define kfifo_in(fifo, buf, n) \
531({ \
532 typeof((fifo) + 1) __tmp = (fifo); \
533 typeof(__tmp->ptr_const) __buf = (buf); \
534 unsigned long __n = (n); \
535 const size_t __recsize = sizeof(*__tmp->rectype); \
536 struct __kfifo *__kfifo = &__tmp->kfifo; \
537 (__recsize) ?\
538 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
539 __kfifo_in(__kfifo, __buf, __n); \
540})
541
542/**
543 * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
544 * @fifo: address of the fifo to be used
545 * @buf: the data to be added
546 * @n: number of elements to be added
547 * @lock: pointer to the spinlock to use for locking
548 *
549 * This macro copies the given values buffer into the fifo and returns the
550 * number of copied elements.
551 */
552#define kfifo_in_spinlocked(fifo, buf, n, lock) \
553({ \
554 unsigned long __flags; \
555 unsigned int __ret; \
556 spin_lock_irqsave(lock, __flags); \
557 __ret = kfifo_in(fifo, buf, n); \
558 spin_unlock_irqrestore(lock, __flags); \
559 __ret; \
560})
561
562/**
563 * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for
564 * locking, don't disable interrupts
565 * @fifo: address of the fifo to be used
566 * @buf: the data to be added
567 * @n: number of elements to be added
568 * @lock: pointer to the spinlock to use for locking
569 *
570 * This is a variant of kfifo_in_spinlocked() but uses spin_lock/unlock()
571 * for locking and doesn't disable interrupts.
572 */
573#define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \
574({ \
575 unsigned int __ret; \
576 spin_lock(lock); \
577 __ret = kfifo_in(fifo, buf, n); \
578 spin_unlock(lock); \
579 __ret; \
580})
581
582/* alias for kfifo_in_spinlocked, will be removed in a future release */
583#define kfifo_in_locked(fifo, buf, n, lock) \
584 kfifo_in_spinlocked(fifo, buf, n, lock)
585
586/**
587 * kfifo_out - get data from the fifo
588 * @fifo: address of the fifo to be used
589 * @buf: pointer to the storage buffer
590 * @n: max. number of elements to get
591 *
592 * This macro gets some data from the fifo and returns the numbers of elements
593 * copied.
594 *
595 * Note that with only one concurrent reader and one concurrent
596 * writer, you don't need extra locking to use these macro.
597 */
598#define kfifo_out(fifo, buf, n) \
599__kfifo_uint_must_check_helper( \
600({ \
601 typeof((fifo) + 1) __tmp = (fifo); \
602 typeof(__tmp->ptr) __buf = (buf); \
603 unsigned long __n = (n); \
604 const size_t __recsize = sizeof(*__tmp->rectype); \
605 struct __kfifo *__kfifo = &__tmp->kfifo; \
606 (__recsize) ?\
607 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
608 __kfifo_out(__kfifo, __buf, __n); \
609}) \
610)
611
612/**
613 * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
614 * @fifo: address of the fifo to be used
615 * @buf: pointer to the storage buffer
616 * @n: max. number of elements to get
617 * @lock: pointer to the spinlock to use for locking
618 *
619 * This macro gets the data from the fifo and returns the numbers of elements
620 * copied.
621 */
622#define kfifo_out_spinlocked(fifo, buf, n, lock) \
623__kfifo_uint_must_check_helper( \
624({ \
625 unsigned long __flags; \
626 unsigned int __ret; \
627 spin_lock_irqsave(lock, __flags); \
628 __ret = kfifo_out(fifo, buf, n); \
629 spin_unlock_irqrestore(lock, __flags); \
630 __ret; \
631}) \
632)
633
634/**
635 * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock
636 * for locking, don't disable interrupts
637 * @fifo: address of the fifo to be used
638 * @buf: pointer to the storage buffer
639 * @n: max. number of elements to get
640 * @lock: pointer to the spinlock to use for locking
641 *
642 * This is a variant of kfifo_out_spinlocked() which uses spin_lock/unlock()
643 * for locking and doesn't disable interrupts.
644 */
645#define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \
646__kfifo_uint_must_check_helper( \
647({ \
648 unsigned int __ret; \
649 spin_lock(lock); \
650 __ret = kfifo_out(fifo, buf, n); \
651 spin_unlock(lock); \
652 __ret; \
653}) \
654)
655
656/* alias for kfifo_out_spinlocked, will be removed in a future release */
657#define kfifo_out_locked(fifo, buf, n, lock) \
658 kfifo_out_spinlocked(fifo, buf, n, lock)
659
660/**
661 * kfifo_from_user - puts some data from user space into the fifo
662 * @fifo: address of the fifo to be used
663 * @from: pointer to the data to be added
664 * @len: the length of the data to be added
665 * @copied: pointer to output variable to store the number of copied bytes
666 *
667 * This macro copies at most @len bytes from the @from into the
668 * fifo, depending of the available space and returns -EFAULT/0.
669 *
670 * Note that with only one concurrent reader and one concurrent
671 * writer, you don't need extra locking to use these macro.
672 */
673#define kfifo_from_user(fifo, from, len, copied) \
674__kfifo_uint_must_check_helper( \
675({ \
676 typeof((fifo) + 1) __tmp = (fifo); \
677 const void __user *__from = (from); \
678 unsigned int __len = (len); \
679 unsigned int *__copied = (copied); \
680 const size_t __recsize = sizeof(*__tmp->rectype); \
681 struct __kfifo *__kfifo = &__tmp->kfifo; \
682 (__recsize) ? \
683 __kfifo_from_user_r(__kfifo, __from, __len, __copied, __recsize) : \
684 __kfifo_from_user(__kfifo, __from, __len, __copied); \
685}) \
686)
687
688/**
689 * kfifo_to_user - copies data from the fifo into user space
690 * @fifo: address of the fifo to be used
691 * @to: where the data must be copied
692 * @len: the size of the destination buffer
693 * @copied: pointer to output variable to store the number of copied bytes
694 *
695 * This macro copies at most @len bytes from the fifo into the
696 * @to buffer and returns -EFAULT/0.
697 *
698 * Note that with only one concurrent reader and one concurrent
699 * writer, you don't need extra locking to use these macro.
700 */
701#define kfifo_to_user(fifo, to, len, copied) \
702__kfifo_int_must_check_helper( \
703({ \
704 typeof((fifo) + 1) __tmp = (fifo); \
705 void __user *__to = (to); \
706 unsigned int __len = (len); \
707 unsigned int *__copied = (copied); \
708 const size_t __recsize = sizeof(*__tmp->rectype); \
709 struct __kfifo *__kfifo = &__tmp->kfifo; \
710 (__recsize) ? \
711 __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
712 __kfifo_to_user(__kfifo, __to, __len, __copied); \
713}) \
714)
715
716/**
717 * kfifo_dma_in_prepare_mapped - setup a scatterlist for DMA input
718 * @fifo: address of the fifo to be used
719 * @sgl: pointer to the scatterlist array
720 * @nents: number of entries in the scatterlist array
721 * @len: number of elements to transfer
722 * @dma: mapped dma address to fill into @sgl
723 *
724 * This macro fills a scatterlist for DMA input.
725 * It returns the number entries in the scatterlist array.
726 *
727 * Note that with only one concurrent reader and one concurrent
728 * writer, you don't need extra locking to use these macros.
729 */
730#define kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, dma) \
731({ \
732 typeof((fifo) + 1) __tmp = (fifo); \
733 struct scatterlist *__sgl = (sgl); \
734 int __nents = (nents); \
735 unsigned int __len = (len); \
736 const size_t __recsize = sizeof(*__tmp->rectype); \
737 struct __kfifo *__kfifo = &__tmp->kfifo; \
738 (__recsize) ? \
739 __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize, \
740 dma) : \
741 __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len, dma); \
742})
743
744#define kfifo_dma_in_prepare(fifo, sgl, nents, len) \
745 kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR)
746
747/**
748 * kfifo_dma_in_finish - finish a DMA IN operation
749 * @fifo: address of the fifo to be used
750 * @len: number of bytes to received
751 *
752 * This macro finishes a DMA IN operation. The in counter will be updated by
753 * the len parameter. No error checking will be done.
754 *
755 * Note that with only one concurrent reader and one concurrent
756 * writer, you don't need extra locking to use these macros.
757 */
758#define kfifo_dma_in_finish(fifo, len) \
759(void)({ \
760 typeof((fifo) + 1) __tmp = (fifo); \
761 unsigned int __len = (len); \
762 const size_t __recsize = sizeof(*__tmp->rectype); \
763 struct __kfifo *__kfifo = &__tmp->kfifo; \
764 if (__recsize) \
765 __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
766 else \
767 __kfifo->in += __len / sizeof(*__tmp->type); \
768})
769
770/**
771 * kfifo_dma_out_prepare_mapped - setup a scatterlist for DMA output
772 * @fifo: address of the fifo to be used
773 * @sgl: pointer to the scatterlist array
774 * @nents: number of entries in the scatterlist array
775 * @len: number of elements to transfer
776 * @dma: mapped dma address to fill into @sgl
777 *
778 * This macro fills a scatterlist for DMA output which at most @len bytes
779 * to transfer.
780 * It returns the number entries in the scatterlist array.
781 * A zero means there is no space available and the scatterlist is not filled.
782 *
783 * Note that with only one concurrent reader and one concurrent
784 * writer, you don't need extra locking to use these macros.
785 */
786#define kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, dma) \
787({ \
788 typeof((fifo) + 1) __tmp = (fifo); \
789 struct scatterlist *__sgl = (sgl); \
790 int __nents = (nents); \
791 unsigned int __len = (len); \
792 const size_t __recsize = sizeof(*__tmp->rectype); \
793 struct __kfifo *__kfifo = &__tmp->kfifo; \
794 (__recsize) ? \
795 __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize, \
796 dma) : \
797 __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len, dma); \
798})
799
800#define kfifo_dma_out_prepare(fifo, sgl, nents, len) \
801 kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR)
802
803/**
804 * kfifo_dma_out_finish - finish a DMA OUT operation
805 * @fifo: address of the fifo to be used
806 * @len: number of bytes transferred
807 *
808 * This macro finishes a DMA OUT operation. The out counter will be updated by
809 * the len parameter. No error checking will be done.
810 *
811 * Note that with only one concurrent reader and one concurrent
812 * writer, you don't need extra locking to use these macros.
813 */
814#define kfifo_dma_out_finish(fifo, len) do { \
815 typeof((fifo) + 1) ___tmp = (fifo); \
816 kfifo_skip_count(___tmp, (len) / sizeof(*___tmp->type)); \
817} while (0)
818
819/**
820 * kfifo_out_peek - gets some data from the fifo
821 * @fifo: address of the fifo to be used
822 * @buf: pointer to the storage buffer
823 * @n: max. number of elements to get
824 *
825 * This macro gets the data from the fifo and returns the numbers of elements
826 * copied. The data is not removed from the fifo.
827 *
828 * Note that with only one concurrent reader and one concurrent
829 * writer, you don't need extra locking to use these macro.
830 */
831#define kfifo_out_peek(fifo, buf, n) \
832__kfifo_uint_must_check_helper( \
833({ \
834 typeof((fifo) + 1) __tmp = (fifo); \
835 typeof(__tmp->ptr) __buf = (buf); \
836 unsigned long __n = (n); \
837 const size_t __recsize = sizeof(*__tmp->rectype); \
838 struct __kfifo *__kfifo = &__tmp->kfifo; \
839 (__recsize) ? \
840 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
841 __kfifo_out_peek(__kfifo, __buf, __n); \
842}) \
843)
844
845/**
846 * kfifo_out_linear - gets a tail of/offset to available data
847 * @fifo: address of the fifo to be used
848 * @tail: pointer to an unsigned int to store the value of tail
849 * @n: max. number of elements to point at
850 *
851 * This macro obtains the offset (tail) to the available data in the fifo
852 * buffer and returns the
853 * numbers of elements available. It returns the available count till the end
854 * of data or till the end of the buffer. So that it can be used for linear
855 * data processing (like memcpy() of (@fifo->data + @tail) with count
856 * returned).
857 *
858 * Note that with only one concurrent reader and one concurrent
859 * writer, you don't need extra locking to use these macro.
860 */
861#define kfifo_out_linear(fifo, tail, n) \
862__kfifo_uint_must_check_helper( \
863({ \
864 typeof((fifo) + 1) __tmp = (fifo); \
865 unsigned int *__tail = (tail); \
866 unsigned long __n = (n); \
867 const size_t __recsize = sizeof(*__tmp->rectype); \
868 struct __kfifo *__kfifo = &__tmp->kfifo; \
869 (__recsize) ? \
870 __kfifo_out_linear_r(__kfifo, __tail, __n, __recsize) : \
871 __kfifo_out_linear(__kfifo, __tail, __n); \
872}) \
873)
874
875/**
876 * kfifo_out_linear_ptr - gets a pointer to the available data
877 * @fifo: address of the fifo to be used
878 * @ptr: pointer to data to store the pointer to tail
879 * @n: max. number of elements to point at
880 *
881 * Similarly to kfifo_out_linear(), this macro obtains the pointer to the
882 * available data in the fifo buffer and returns the numbers of elements
883 * available. It returns the available count till the end of available data or
884 * till the end of the buffer. So that it can be used for linear data
885 * processing (like memcpy() of @ptr with count returned).
886 *
887 * Note that with only one concurrent reader and one concurrent
888 * writer, you don't need extra locking to use these macro.
889 */
890#define kfifo_out_linear_ptr(fifo, ptr, n) \
891__kfifo_uint_must_check_helper( \
892({ \
893 typeof((fifo) + 1) ___tmp = (fifo); \
894 unsigned int ___tail; \
895 unsigned int ___n = kfifo_out_linear(___tmp, &___tail, (n)); \
896 *(ptr) = ___tmp->kfifo.data + ___tail * kfifo_esize(___tmp); \
897 ___n; \
898}) \
899)
900
901
902extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
903 size_t esize, gfp_t gfp_mask);
904
905extern void __kfifo_free(struct __kfifo *fifo);
906
907extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
908 unsigned int size, size_t esize);
909
910extern unsigned int __kfifo_in(struct __kfifo *fifo,
911 const void *buf, unsigned int len);
912
913extern unsigned int __kfifo_out(struct __kfifo *fifo,
914 void *buf, unsigned int len);
915
916extern int __kfifo_from_user(struct __kfifo *fifo,
917 const void __user *from, unsigned long len, unsigned int *copied);
918
919extern int __kfifo_to_user(struct __kfifo *fifo,
920 void __user *to, unsigned long len, unsigned int *copied);
921
922extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
923 struct scatterlist *sgl, int nents, unsigned int len, dma_addr_t dma);
924
925extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
926 struct scatterlist *sgl, int nents, unsigned int len, dma_addr_t dma);
927
928extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
929 void *buf, unsigned int len);
930
931extern unsigned int __kfifo_out_linear(struct __kfifo *fifo,
932 unsigned int *tail, unsigned int n);
933
934extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
935 const void *buf, unsigned int len, size_t recsize);
936
937extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
938 void *buf, unsigned int len, size_t recsize);
939
940extern int __kfifo_from_user_r(struct __kfifo *fifo,
941 const void __user *from, unsigned long len, unsigned int *copied,
942 size_t recsize);
943
944extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
945 unsigned long len, unsigned int *copied, size_t recsize);
946
947extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
948 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize,
949 dma_addr_t dma);
950
951extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
952 unsigned int len, size_t recsize);
953
954extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
955 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize,
956 dma_addr_t dma);
957
958extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
959
960extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
961
962extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
963 void *buf, unsigned int len, size_t recsize);
964
965extern unsigned int __kfifo_out_linear_r(struct __kfifo *fifo,
966 unsigned int *tail, unsigned int n, size_t recsize);
967
968extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
969
970#endif
971