1// SPDX-License-Identifier: GPL-2.0
2/*
3 * event tracer
4 *
5 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
6 *
7 * - Added format output of fields of the trace point.
8 * This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
9 *
10 */
11
12#define pr_fmt(fmt) fmt
13
14#include <linux/workqueue.h>
15#include <linux/security.h>
16#include <linux/spinlock.h>
17#include <linux/kthread.h>
18#include <linux/tracefs.h>
19#include <linux/uaccess.h>
20#include <linux/module.h>
21#include <linux/ctype.h>
22#include <linux/sort.h>
23#include <linux/slab.h>
24#include <linux/delay.h>
25
26#include <trace/events/sched.h>
27#include <trace/syscall.h>
28
29#include <asm/setup.h>
30
31#include "trace_output.h"
32
33#undef TRACE_SYSTEM
34#define TRACE_SYSTEM "TRACE_SYSTEM"
35
36DEFINE_MUTEX(event_mutex);
37
38LIST_HEAD(ftrace_events);
39static LIST_HEAD(ftrace_generic_fields);
40static LIST_HEAD(ftrace_common_fields);
41static bool eventdir_initialized;
42
43static LIST_HEAD(module_strings);
44
45struct module_string {
46 struct list_head next;
47 struct module *module;
48 char *str;
49};
50
51#define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
52
53static struct kmem_cache *field_cachep;
54static struct kmem_cache *file_cachep;
55
56static inline int system_refcount(struct event_subsystem *system)
57{
58 return system->ref_count;
59}
60
61static int system_refcount_inc(struct event_subsystem *system)
62{
63 return system->ref_count++;
64}
65
66static int system_refcount_dec(struct event_subsystem *system)
67{
68 return --system->ref_count;
69}
70
71/* Double loops, do not use break, only goto's work */
72#define do_for_each_event_file(tr, file) \
73 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
74 list_for_each_entry(file, &tr->events, list)
75
76#define do_for_each_event_file_safe(tr, file) \
77 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
78 struct trace_event_file *___n; \
79 list_for_each_entry_safe(file, ___n, &tr->events, list)
80
81#define while_for_each_event_file() \
82 }
83
84static struct ftrace_event_field *
85__find_event_field(struct list_head *head, const char *name)
86{
87 struct ftrace_event_field *field;
88
89 list_for_each_entry(field, head, link) {
90 if (!strcmp(field->name, name))
91 return field;
92 }
93
94 return NULL;
95}
96
97struct ftrace_event_field *
98trace_find_event_field(struct trace_event_call *call, char *name)
99{
100 struct ftrace_event_field *field;
101 struct list_head *head;
102
103 head = trace_get_fields(event_call: call);
104 field = __find_event_field(head, name);
105 if (field)
106 return field;
107
108 field = __find_event_field(head: &ftrace_generic_fields, name);
109 if (field)
110 return field;
111
112 return __find_event_field(head: &ftrace_common_fields, name);
113}
114
115static int __trace_define_field(struct list_head *head, const char *type,
116 const char *name, int offset, int size,
117 int is_signed, int filter_type, int len,
118 int need_test)
119{
120 struct ftrace_event_field *field;
121
122 field = kmem_cache_alloc(field_cachep, GFP_TRACE);
123 if (!field)
124 return -ENOMEM;
125
126 field->name = name;
127 field->type = type;
128
129 if (filter_type == FILTER_OTHER)
130 field->filter_type = filter_assign_type(type);
131 else
132 field->filter_type = filter_type;
133
134 field->offset = offset;
135 field->size = size;
136 field->is_signed = is_signed;
137 field->needs_test = need_test;
138 field->len = len;
139
140 list_add(new: &field->link, head);
141
142 return 0;
143}
144
145int trace_define_field(struct trace_event_call *call, const char *type,
146 const char *name, int offset, int size, int is_signed,
147 int filter_type)
148{
149 struct list_head *head;
150
151 if (WARN_ON(!call->class))
152 return 0;
153
154 head = trace_get_fields(event_call: call);
155 return __trace_define_field(head, type, name, offset, size,
156 is_signed, filter_type, len: 0, need_test: 0);
157}
158EXPORT_SYMBOL_GPL(trace_define_field);
159
160static int trace_define_field_ext(struct trace_event_call *call, const char *type,
161 const char *name, int offset, int size, int is_signed,
162 int filter_type, int len, int need_test)
163{
164 struct list_head *head;
165
166 if (WARN_ON(!call->class))
167 return 0;
168
169 head = trace_get_fields(event_call: call);
170 return __trace_define_field(head, type, name, offset, size,
171 is_signed, filter_type, len, need_test);
172}
173
174#define __generic_field(type, item, filter_type) \
175 ret = __trace_define_field(&ftrace_generic_fields, #type, \
176 #item, 0, 0, is_signed_type(type), \
177 filter_type, 0, 0); \
178 if (ret) \
179 return ret;
180
181#define __common_field(type, item) \
182 ret = __trace_define_field(&ftrace_common_fields, #type, \
183 "common_" #item, \
184 offsetof(typeof(ent), item), \
185 sizeof(ent.item), \
186 is_signed_type(type), FILTER_OTHER, \
187 0, 0); \
188 if (ret) \
189 return ret;
190
191static int trace_define_generic_fields(void)
192{
193 int ret;
194
195 __generic_field(int, CPU, FILTER_CPU);
196 __generic_field(int, cpu, FILTER_CPU);
197 __generic_field(int, common_cpu, FILTER_CPU);
198 __generic_field(char *, COMM, FILTER_COMM);
199 __generic_field(char *, comm, FILTER_COMM);
200 __generic_field(char *, stacktrace, FILTER_STACKTRACE);
201 __generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
202
203 return ret;
204}
205
206static int trace_define_common_fields(void)
207{
208 int ret;
209 struct trace_entry ent;
210
211 __common_field(unsigned short, type);
212 __common_field(unsigned char, flags);
213 /* Holds both preempt_count and migrate_disable */
214 __common_field(unsigned char, preempt_count);
215 __common_field(int, pid);
216
217 return ret;
218}
219
220static void trace_destroy_fields(struct trace_event_call *call)
221{
222 struct ftrace_event_field *field, *next;
223 struct list_head *head;
224
225 head = trace_get_fields(event_call: call);
226 list_for_each_entry_safe(field, next, head, link) {
227 list_del(entry: &field->link);
228 kmem_cache_free(s: field_cachep, objp: field);
229 }
230}
231
232/*
233 * run-time version of trace_event_get_offsets_<call>() that returns the last
234 * accessible offset of trace fields excluding __dynamic_array bytes
235 */
236int trace_event_get_offsets(struct trace_event_call *call)
237{
238 struct ftrace_event_field *tail;
239 struct list_head *head;
240
241 head = trace_get_fields(event_call: call);
242 /*
243 * head->next points to the last field with the largest offset,
244 * since it was added last by trace_define_field()
245 */
246 tail = list_first_entry(head, struct ftrace_event_field, link);
247 return tail->offset + tail->size;
248}
249
250
251static struct trace_event_fields *find_event_field(const char *fmt,
252 struct trace_event_call *call)
253{
254 struct trace_event_fields *field = call->class->fields_array;
255 const char *p = fmt;
256 int len;
257
258 if (!(len = str_has_prefix(str: fmt, prefix: "REC->")))
259 return NULL;
260 fmt += len;
261 for (p = fmt; *p; p++) {
262 if (!isalnum(*p) && *p != '_')
263 break;
264 }
265 len = p - fmt;
266
267 for (; field->type; field++) {
268 if (strncmp(field->name, fmt, len) || field->name[len])
269 continue;
270
271 return field;
272 }
273 return NULL;
274}
275
276/*
277 * Check if the referenced field is an array and return true,
278 * as arrays are OK to dereference.
279 */
280static bool test_field(const char *fmt, struct trace_event_call *call)
281{
282 struct trace_event_fields *field;
283
284 field = find_event_field(fmt, call);
285 if (!field)
286 return false;
287
288 /* This is an array and is OK to dereference. */
289 return strchr(field->type, '[') != NULL;
290}
291
292/* Look for a string within an argument */
293static bool find_print_string(const char *arg, const char *str, const char *end)
294{
295 const char *r;
296
297 r = strstr(arg, str);
298 return r && r < end;
299}
300
301/* Return true if the argument pointer is safe */
302static bool process_pointer(const char *fmt, int len, struct trace_event_call *call)
303{
304 const char *r, *e, *a;
305
306 e = fmt + len;
307
308 /* Find the REC-> in the argument */
309 r = strstr(fmt, "REC->");
310 if (r && r < e) {
311 /*
312 * Addresses of events on the buffer, or an array on the buffer is
313 * OK to dereference. There's ways to fool this, but
314 * this is to catch common mistakes, not malicious code.
315 */
316 a = strchr(fmt, '&');
317 if ((a && (a < r)) || test_field(fmt: r, call))
318 return true;
319 } else if (find_print_string(arg: fmt, str: "__get_dynamic_array(", end: e)) {
320 return true;
321 } else if (find_print_string(arg: fmt, str: "__get_rel_dynamic_array(", end: e)) {
322 return true;
323 } else if (find_print_string(arg: fmt, str: "__get_dynamic_array_len(", end: e)) {
324 return true;
325 } else if (find_print_string(arg: fmt, str: "__get_rel_dynamic_array_len(", end: e)) {
326 return true;
327 } else if (find_print_string(arg: fmt, str: "__get_sockaddr(", end: e)) {
328 return true;
329 } else if (find_print_string(arg: fmt, str: "__get_rel_sockaddr(", end: e)) {
330 return true;
331 }
332 return false;
333}
334
335/* Return true if the string is safe */
336static bool process_string(const char *fmt, int len, struct trace_event_call *call)
337{
338 struct trace_event_fields *field;
339 const char *r, *e, *s;
340
341 e = fmt + len;
342
343 /*
344 * There are several helper functions that return strings.
345 * If the argument contains a function, then assume its field is valid.
346 * It is considered that the argument has a function if it has:
347 * alphanumeric or '_' before a parenthesis.
348 */
349 s = fmt;
350 do {
351 r = strstr(s, "(");
352 if (!r || r >= e)
353 break;
354 for (int i = 1; r - i >= s; i++) {
355 char ch = *(r - i);
356 if (isspace(ch))
357 continue;
358 if (isalnum(ch) || ch == '_')
359 return true;
360 /* Anything else, this isn't a function */
361 break;
362 }
363 /* A function could be wrapped in parethesis, try the next one */
364 s = r + 1;
365 } while (s < e);
366
367 /*
368 * Check for arrays. If the argument has: foo[REC->val]
369 * then it is very likely that foo is an array of strings
370 * that are safe to use.
371 */
372 r = strstr(s, "[");
373 if (r && r < e) {
374 r = strstr(r, "REC->");
375 if (r && r < e)
376 return true;
377 }
378
379 /*
380 * If there's any strings in the argument consider this arg OK as it
381 * could be: REC->field ? "foo" : "bar" and we don't want to get into
382 * verifying that logic here.
383 */
384 if (find_print_string(arg: fmt, str: "\"", end: e))
385 return true;
386
387 /* Dereferenced strings are also valid like any other pointer */
388 if (process_pointer(fmt, len, call))
389 return true;
390
391 /* Make sure the field is found */
392 field = find_event_field(fmt, call);
393 if (!field)
394 return false;
395
396 /* Test this field's string before printing the event */
397 call->flags |= TRACE_EVENT_FL_TEST_STR;
398 field->needs_test = 1;
399
400 return true;
401}
402
403static void handle_dereference_arg(const char *arg_str, u64 string_flags, int len,
404 u64 *dereference_flags, int arg,
405 struct trace_event_call *call)
406{
407 if (string_flags & (1ULL << arg)) {
408 if (process_string(fmt: arg_str, len, call))
409 *dereference_flags &= ~(1ULL << arg);
410 } else if (process_pointer(fmt: arg_str, len, call))
411 *dereference_flags &= ~(1ULL << arg);
412 else
413 pr_warn("TRACE EVENT ERROR: Bad dereference argument: '%.*s'\n",
414 len, arg_str);
415}
416
417/*
418 * Examine the print fmt of the event looking for unsafe dereference
419 * pointers using %p* that could be recorded in the trace event and
420 * much later referenced after the pointer was freed. Dereferencing
421 * pointers are OK, if it is dereferenced into the event itself.
422 */
423static void test_event_printk(struct trace_event_call *call)
424{
425 u64 dereference_flags = 0;
426 u64 string_flags = 0;
427 bool first = true;
428 const char *fmt;
429 int parens = 0;
430 char in_quote = 0;
431 int start_arg = 0;
432 int arg = 0;
433 int i, e;
434
435 fmt = call->print_fmt;
436
437 if (!fmt)
438 return;
439
440 for (i = 0; fmt[i]; i++) {
441 switch (fmt[i]) {
442 case '\\':
443 i++;
444 if (!fmt[i])
445 return;
446 continue;
447 case '"':
448 case '\'':
449 /*
450 * The print fmt starts with a string that
451 * is processed first to find %p* usage,
452 * then after the first string, the print fmt
453 * contains arguments that are used to check
454 * if the dereferenced %p* usage is safe.
455 */
456 if (first) {
457 if (fmt[i] == '\'')
458 continue;
459 if (in_quote) {
460 arg = 0;
461 first = false;
462 /*
463 * If there was no %p* uses
464 * the fmt is OK.
465 */
466 if (!dereference_flags)
467 return;
468 }
469 }
470 if (in_quote) {
471 if (in_quote == fmt[i])
472 in_quote = 0;
473 } else {
474 in_quote = fmt[i];
475 }
476 continue;
477 case '%':
478 if (!first || !in_quote)
479 continue;
480 i++;
481 if (!fmt[i])
482 return;
483 switch (fmt[i]) {
484 case '%':
485 continue;
486 case 'p':
487 do_pointer:
488 /* Find dereferencing fields */
489 switch (fmt[i + 1]) {
490 case 'B': case 'R': case 'r':
491 case 'b': case 'M': case 'm':
492 case 'I': case 'i': case 'E':
493 case 'U': case 'V': case 'N':
494 case 'a': case 'd': case 'D':
495 case 'g': case 't': case 'C':
496 case 'O': case 'f':
497 if (WARN_ONCE(arg == 63,
498 "Too many args for event: %s",
499 trace_event_name(call)))
500 return;
501 dereference_flags |= 1ULL << arg;
502 }
503 break;
504 default:
505 {
506 bool star = false;
507 int j;
508
509 /* Increment arg if %*s exists. */
510 for (j = 0; fmt[i + j]; j++) {
511 if (isdigit(c: fmt[i + j]) ||
512 fmt[i + j] == '.')
513 continue;
514 if (fmt[i + j] == '*') {
515 star = true;
516 /* Handle %*pbl case */
517 if (!j && fmt[i + 1] == 'p') {
518 arg++;
519 i++;
520 goto do_pointer;
521 }
522 continue;
523 }
524 if ((fmt[i + j] == 's')) {
525 if (star)
526 arg++;
527 if (WARN_ONCE(arg == 63,
528 "Too many args for event: %s",
529 trace_event_name(call)))
530 return;
531 dereference_flags |= 1ULL << arg;
532 string_flags |= 1ULL << arg;
533 }
534 break;
535 }
536 break;
537 } /* default */
538
539 } /* switch */
540 arg++;
541 continue;
542 case '(':
543 if (in_quote)
544 continue;
545 parens++;
546 continue;
547 case ')':
548 if (in_quote)
549 continue;
550 parens--;
551 if (WARN_ONCE(parens < 0,
552 "Paren mismatch for event: %s\narg='%s'\n%*s",
553 trace_event_name(call),
554 fmt + start_arg,
555 (i - start_arg) + 5, "^"))
556 return;
557 continue;
558 case ',':
559 if (in_quote || parens)
560 continue;
561 e = i;
562 i++;
563 while (isspace(fmt[i]))
564 i++;
565
566 /*
567 * If start_arg is zero, then this is the start of the
568 * first argument. The processing of the argument happens
569 * when the end of the argument is found, as it needs to
570 * handle paranthesis and such.
571 */
572 if (!start_arg) {
573 start_arg = i;
574 /* Balance out the i++ in the for loop */
575 i--;
576 continue;
577 }
578
579 if (dereference_flags & (1ULL << arg)) {
580 handle_dereference_arg(arg_str: fmt + start_arg, string_flags,
581 len: e - start_arg,
582 dereference_flags: &dereference_flags, arg, call);
583 }
584
585 start_arg = i;
586 arg++;
587 /* Balance out the i++ in the for loop */
588 i--;
589 }
590 }
591
592 if (dereference_flags & (1ULL << arg)) {
593 handle_dereference_arg(arg_str: fmt + start_arg, string_flags,
594 len: i - start_arg,
595 dereference_flags: &dereference_flags, arg, call);
596 }
597
598 /*
599 * If you triggered the below warning, the trace event reported
600 * uses an unsafe dereference pointer %p*. As the data stored
601 * at the trace event time may no longer exist when the trace
602 * event is printed, dereferencing to the original source is
603 * unsafe. The source of the dereference must be copied into the
604 * event itself, and the dereference must access the copy instead.
605 */
606 if (WARN_ON_ONCE(dereference_flags)) {
607 arg = 1;
608 while (!(dereference_flags & 1)) {
609 dereference_flags >>= 1;
610 arg++;
611 }
612 pr_warn("event %s has unsafe dereference of argument %d\n",
613 trace_event_name(call), arg);
614 pr_warn("print_fmt: %s\n", fmt);
615 }
616}
617
618int trace_event_raw_init(struct trace_event_call *call)
619{
620 int id;
621
622 id = register_trace_event(event: &call->event);
623 if (!id)
624 return -ENODEV;
625
626 test_event_printk(call);
627
628 return 0;
629}
630EXPORT_SYMBOL_GPL(trace_event_raw_init);
631
632bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
633{
634 struct trace_array *tr = trace_file->tr;
635 struct trace_pid_list *no_pid_list;
636 struct trace_pid_list *pid_list;
637
638 pid_list = rcu_dereference_raw(tr->filtered_pids);
639 no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
640
641 if (!pid_list && !no_pid_list)
642 return false;
643
644 /*
645 * This is recorded at every sched_switch for this task.
646 * Thus, even if the task migrates the ignore value will be the same.
647 */
648 return this_cpu_read(tr->array_buffer.data->ignore_pid) != 0;
649}
650EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
651
652void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
653 struct trace_event_file *trace_file,
654 unsigned long len)
655{
656 struct trace_event_call *event_call = trace_file->event_call;
657
658 if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
659 trace_event_ignore_this_pid(trace_file))
660 return NULL;
661
662 /*
663 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
664 * preemption (adding one to the preempt_count). Since we are
665 * interested in the preempt_count at the time the tracepoint was
666 * hit, we need to subtract one to offset the increment.
667 */
668 fbuffer->trace_ctx = tracing_gen_ctx_dec();
669 fbuffer->trace_file = trace_file;
670
671 fbuffer->event =
672 trace_event_buffer_lock_reserve(current_buffer: &fbuffer->buffer, trace_file,
673 type: event_call->event.type, len,
674 trace_ctx: fbuffer->trace_ctx);
675 if (!fbuffer->event)
676 return NULL;
677
678 fbuffer->regs = NULL;
679 fbuffer->entry = ring_buffer_event_data(event: fbuffer->event);
680 return fbuffer->entry;
681}
682EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
683
684int trace_event_reg(struct trace_event_call *call,
685 enum trace_reg type, void *data)
686{
687 struct trace_event_file *file = data;
688
689 WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
690 switch (type) {
691 case TRACE_REG_REGISTER:
692 return tracepoint_probe_register(tp: call->tp,
693 probe: call->class->probe,
694 data: file);
695 case TRACE_REG_UNREGISTER:
696 tracepoint_probe_unregister(tp: call->tp,
697 probe: call->class->probe,
698 data: file);
699 return 0;
700
701#ifdef CONFIG_PERF_EVENTS
702 case TRACE_REG_PERF_REGISTER:
703 return tracepoint_probe_register(tp: call->tp,
704 probe: call->class->perf_probe,
705 data: call);
706 case TRACE_REG_PERF_UNREGISTER:
707 tracepoint_probe_unregister(tp: call->tp,
708 probe: call->class->perf_probe,
709 data: call);
710 return 0;
711 case TRACE_REG_PERF_OPEN:
712 case TRACE_REG_PERF_CLOSE:
713 case TRACE_REG_PERF_ADD:
714 case TRACE_REG_PERF_DEL:
715 return 0;
716#endif
717 }
718 return 0;
719}
720EXPORT_SYMBOL_GPL(trace_event_reg);
721
722void trace_event_enable_cmd_record(bool enable)
723{
724 struct trace_event_file *file;
725 struct trace_array *tr;
726
727 lockdep_assert_held(&event_mutex);
728
729 do_for_each_event_file(tr, file) {
730
731 if (!(file->flags & EVENT_FILE_FL_ENABLED))
732 continue;
733
734 if (enable) {
735 tracing_start_cmdline_record();
736 set_bit(nr: EVENT_FILE_FL_RECORDED_CMD_BIT, addr: &file->flags);
737 } else {
738 tracing_stop_cmdline_record();
739 clear_bit(nr: EVENT_FILE_FL_RECORDED_CMD_BIT, addr: &file->flags);
740 }
741 } while_for_each_event_file();
742}
743
744void trace_event_enable_tgid_record(bool enable)
745{
746 struct trace_event_file *file;
747 struct trace_array *tr;
748
749 lockdep_assert_held(&event_mutex);
750
751 do_for_each_event_file(tr, file) {
752 if (!(file->flags & EVENT_FILE_FL_ENABLED))
753 continue;
754
755 if (enable) {
756 tracing_start_tgid_record();
757 set_bit(nr: EVENT_FILE_FL_RECORDED_TGID_BIT, addr: &file->flags);
758 } else {
759 tracing_stop_tgid_record();
760 clear_bit(nr: EVENT_FILE_FL_RECORDED_TGID_BIT,
761 addr: &file->flags);
762 }
763 } while_for_each_event_file();
764}
765
766static int __ftrace_event_enable_disable(struct trace_event_file *file,
767 int enable, int soft_disable)
768{
769 struct trace_event_call *call = file->event_call;
770 struct trace_array *tr = file->tr;
771 bool soft_mode = atomic_read(v: &file->sm_ref) != 0;
772 int ret = 0;
773 int disable;
774
775 switch (enable) {
776 case 0:
777 /*
778 * When soft_disable is set and enable is cleared, the sm_ref
779 * reference counter is decremented. If it reaches 0, we want
780 * to clear the SOFT_DISABLED flag but leave the event in the
781 * state that it was. That is, if the event was enabled and
782 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
783 * is set we do not want the event to be enabled before we
784 * clear the bit.
785 *
786 * When soft_disable is not set but the soft_mode is,
787 * we do nothing. Do not disable the tracepoint, otherwise
788 * "soft enable"s (clearing the SOFT_DISABLED bit) wont work.
789 */
790 if (soft_disable) {
791 if (atomic_dec_return(v: &file->sm_ref) > 0)
792 break;
793 disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
794 soft_mode = false;
795 /* Disable use of trace_buffered_event */
796 trace_buffered_event_disable();
797 } else
798 disable = !soft_mode;
799
800 if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
801 clear_bit(nr: EVENT_FILE_FL_ENABLED_BIT, addr: &file->flags);
802 if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
803 tracing_stop_cmdline_record();
804 clear_bit(nr: EVENT_FILE_FL_RECORDED_CMD_BIT, addr: &file->flags);
805 }
806
807 if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
808 tracing_stop_tgid_record();
809 clear_bit(nr: EVENT_FILE_FL_RECORDED_TGID_BIT, addr: &file->flags);
810 }
811
812 ret = call->class->reg(call, TRACE_REG_UNREGISTER, file);
813
814 WARN_ON_ONCE(ret);
815 }
816 /* If in soft mode, just set the SOFT_DISABLE_BIT, else clear it */
817 if (soft_mode)
818 set_bit(nr: EVENT_FILE_FL_SOFT_DISABLED_BIT, addr: &file->flags);
819 else
820 clear_bit(nr: EVENT_FILE_FL_SOFT_DISABLED_BIT, addr: &file->flags);
821 break;
822 case 1:
823 /*
824 * When soft_disable is set and enable is set, we want to
825 * register the tracepoint for the event, but leave the event
826 * as is. That means, if the event was already enabled, we do
827 * nothing (but set soft_mode). If the event is disabled, we
828 * set SOFT_DISABLED before enabling the event tracepoint, so
829 * it still seems to be disabled.
830 */
831 if (!soft_disable)
832 clear_bit(nr: EVENT_FILE_FL_SOFT_DISABLED_BIT, addr: &file->flags);
833 else {
834 if (atomic_inc_return(v: &file->sm_ref) > 1)
835 break;
836 soft_mode = true;
837 /* Enable use of trace_buffered_event */
838 trace_buffered_event_enable();
839 }
840
841 if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
842 bool cmd = false, tgid = false;
843
844 /* Keep the event disabled, when going to soft mode. */
845 if (soft_disable)
846 set_bit(nr: EVENT_FILE_FL_SOFT_DISABLED_BIT, addr: &file->flags);
847
848 if (tr->trace_flags & TRACE_ITER_RECORD_CMD) {
849 cmd = true;
850 tracing_start_cmdline_record();
851 set_bit(nr: EVENT_FILE_FL_RECORDED_CMD_BIT, addr: &file->flags);
852 }
853
854 if (tr->trace_flags & TRACE_ITER_RECORD_TGID) {
855 tgid = true;
856 tracing_start_tgid_record();
857 set_bit(nr: EVENT_FILE_FL_RECORDED_TGID_BIT, addr: &file->flags);
858 }
859
860 ret = call->class->reg(call, TRACE_REG_REGISTER, file);
861 if (ret) {
862 if (cmd)
863 tracing_stop_cmdline_record();
864 if (tgid)
865 tracing_stop_tgid_record();
866 pr_info("event trace: Could not enable event "
867 "%s\n", trace_event_name(call));
868 break;
869 }
870 set_bit(nr: EVENT_FILE_FL_ENABLED_BIT, addr: &file->flags);
871
872 /* WAS_ENABLED gets set but never cleared. */
873 set_bit(nr: EVENT_FILE_FL_WAS_ENABLED_BIT, addr: &file->flags);
874 }
875 break;
876 }
877
878 return ret;
879}
880
881int trace_event_enable_disable(struct trace_event_file *file,
882 int enable, int soft_disable)
883{
884 return __ftrace_event_enable_disable(file, enable, soft_disable);
885}
886
887static int ftrace_event_enable_disable(struct trace_event_file *file,
888 int enable)
889{
890 return __ftrace_event_enable_disable(file, enable, soft_disable: 0);
891}
892
893#ifdef CONFIG_MODULES
894struct event_mod_load {
895 struct list_head list;
896 char *module;
897 char *match;
898 char *system;
899 char *event;
900};
901
902static void free_event_mod(struct event_mod_load *event_mod)
903{
904 list_del(entry: &event_mod->list);
905 kfree(objp: event_mod->module);
906 kfree(objp: event_mod->match);
907 kfree(objp: event_mod->system);
908 kfree(objp: event_mod->event);
909 kfree(objp: event_mod);
910}
911
912static void clear_mod_events(struct trace_array *tr)
913{
914 struct event_mod_load *event_mod, *n;
915
916 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
917 free_event_mod(event_mod);
918 }
919}
920
921static int remove_cache_mod(struct trace_array *tr, const char *mod,
922 const char *match, const char *system, const char *event)
923{
924 struct event_mod_load *event_mod, *n;
925 int ret = -EINVAL;
926
927 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
928 if (strcmp(event_mod->module, mod) != 0)
929 continue;
930
931 if (match && strcmp(event_mod->match, match) != 0)
932 continue;
933
934 if (system &&
935 (!event_mod->system || strcmp(event_mod->system, system) != 0))
936 continue;
937
938 if (event &&
939 (!event_mod->event || strcmp(event_mod->event, event) != 0))
940 continue;
941
942 free_event_mod(event_mod);
943 ret = 0;
944 }
945
946 return ret;
947}
948
949static int cache_mod(struct trace_array *tr, const char *mod, int set,
950 const char *match, const char *system, const char *event)
951{
952 struct event_mod_load *event_mod;
953
954 /* If the module exists, then this just failed to find an event */
955 if (module_exists(module: mod))
956 return -EINVAL;
957
958 /* See if this is to remove a cached filter */
959 if (!set)
960 return remove_cache_mod(tr, mod, match, system, event);
961
962 event_mod = kzalloc(sizeof(*event_mod), GFP_KERNEL);
963 if (!event_mod)
964 return -ENOMEM;
965
966 INIT_LIST_HEAD(list: &event_mod->list);
967 event_mod->module = kstrdup(s: mod, GFP_KERNEL);
968 if (!event_mod->module)
969 goto out_free;
970
971 if (match) {
972 event_mod->match = kstrdup(s: match, GFP_KERNEL);
973 if (!event_mod->match)
974 goto out_free;
975 }
976
977 if (system) {
978 event_mod->system = kstrdup(s: system, GFP_KERNEL);
979 if (!event_mod->system)
980 goto out_free;
981 }
982
983 if (event) {
984 event_mod->event = kstrdup(s: event, GFP_KERNEL);
985 if (!event_mod->event)
986 goto out_free;
987 }
988
989 list_add(new: &event_mod->list, head: &tr->mod_events);
990
991 return 0;
992
993 out_free:
994 free_event_mod(event_mod);
995
996 return -ENOMEM;
997}
998#else /* CONFIG_MODULES */
999static inline void clear_mod_events(struct trace_array *tr) { }
1000static int cache_mod(struct trace_array *tr, const char *mod, int set,
1001 const char *match, const char *system, const char *event)
1002{
1003 return -EINVAL;
1004}
1005#endif
1006
1007static void ftrace_clear_events(struct trace_array *tr)
1008{
1009 struct trace_event_file *file;
1010
1011 mutex_lock(lock: &event_mutex);
1012 list_for_each_entry(file, &tr->events, list) {
1013 ftrace_event_enable_disable(file, enable: 0);
1014 }
1015 clear_mod_events(tr);
1016 mutex_unlock(lock: &event_mutex);
1017}
1018
1019static void
1020event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
1021{
1022 struct trace_pid_list *pid_list;
1023 struct trace_array *tr = data;
1024
1025 pid_list = rcu_dereference_raw(tr->filtered_pids);
1026 trace_filter_add_remove_task(pid_list, NULL, task);
1027
1028 pid_list = rcu_dereference_raw(tr->filtered_no_pids);
1029 trace_filter_add_remove_task(pid_list, NULL, task);
1030}
1031
1032static void
1033event_filter_pid_sched_process_fork(void *data,
1034 struct task_struct *self,
1035 struct task_struct *task)
1036{
1037 struct trace_pid_list *pid_list;
1038 struct trace_array *tr = data;
1039
1040 pid_list = rcu_dereference_sched(tr->filtered_pids);
1041 trace_filter_add_remove_task(pid_list, self, task);
1042
1043 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1044 trace_filter_add_remove_task(pid_list, self, task);
1045}
1046
1047void trace_event_follow_fork(struct trace_array *tr, bool enable)
1048{
1049 if (enable) {
1050 register_trace_prio_sched_process_fork(probe: event_filter_pid_sched_process_fork,
1051 data: tr, INT_MIN);
1052 register_trace_prio_sched_process_free(probe: event_filter_pid_sched_process_exit,
1053 data: tr, INT_MAX);
1054 } else {
1055 unregister_trace_sched_process_fork(probe: event_filter_pid_sched_process_fork,
1056 data: tr);
1057 unregister_trace_sched_process_free(probe: event_filter_pid_sched_process_exit,
1058 data: tr);
1059 }
1060}
1061
1062static void
1063event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
1064 struct task_struct *prev,
1065 struct task_struct *next,
1066 unsigned int prev_state)
1067{
1068 struct trace_array *tr = data;
1069 struct trace_pid_list *no_pid_list;
1070 struct trace_pid_list *pid_list;
1071 bool ret;
1072
1073 pid_list = rcu_dereference_sched(tr->filtered_pids);
1074 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1075
1076 /*
1077 * Sched switch is funny, as we only want to ignore it
1078 * in the notrace case if both prev and next should be ignored.
1079 */
1080 ret = trace_ignore_this_task(NULL, filtered_no_pids: no_pid_list, task: prev) &&
1081 trace_ignore_this_task(NULL, filtered_no_pids: no_pid_list, task: next);
1082
1083 this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
1084 (trace_ignore_this_task(pid_list, NULL, prev) &&
1085 trace_ignore_this_task(pid_list, NULL, next)));
1086}
1087
1088static void
1089event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
1090 struct task_struct *prev,
1091 struct task_struct *next,
1092 unsigned int prev_state)
1093{
1094 struct trace_array *tr = data;
1095 struct trace_pid_list *no_pid_list;
1096 struct trace_pid_list *pid_list;
1097
1098 pid_list = rcu_dereference_sched(tr->filtered_pids);
1099 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1100
1101 this_cpu_write(tr->array_buffer.data->ignore_pid,
1102 trace_ignore_this_task(pid_list, no_pid_list, next));
1103}
1104
1105static void
1106event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
1107{
1108 struct trace_array *tr = data;
1109 struct trace_pid_list *no_pid_list;
1110 struct trace_pid_list *pid_list;
1111
1112 /* Nothing to do if we are already tracing */
1113 if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
1114 return;
1115
1116 pid_list = rcu_dereference_sched(tr->filtered_pids);
1117 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1118
1119 this_cpu_write(tr->array_buffer.data->ignore_pid,
1120 trace_ignore_this_task(pid_list, no_pid_list, task));
1121}
1122
1123static void
1124event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
1125{
1126 struct trace_array *tr = data;
1127 struct trace_pid_list *no_pid_list;
1128 struct trace_pid_list *pid_list;
1129
1130 /* Nothing to do if we are not tracing */
1131 if (this_cpu_read(tr->array_buffer.data->ignore_pid))
1132 return;
1133
1134 pid_list = rcu_dereference_sched(tr->filtered_pids);
1135 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1136
1137 /* Set tracing if current is enabled */
1138 this_cpu_write(tr->array_buffer.data->ignore_pid,
1139 trace_ignore_this_task(pid_list, no_pid_list, current));
1140}
1141
1142static void unregister_pid_events(struct trace_array *tr)
1143{
1144 unregister_trace_sched_switch(probe: event_filter_pid_sched_switch_probe_pre, data: tr);
1145 unregister_trace_sched_switch(probe: event_filter_pid_sched_switch_probe_post, data: tr);
1146
1147 unregister_trace_sched_wakeup(probe: event_filter_pid_sched_wakeup_probe_pre, data: tr);
1148 unregister_trace_sched_wakeup(probe: event_filter_pid_sched_wakeup_probe_post, data: tr);
1149
1150 unregister_trace_sched_wakeup_new(probe: event_filter_pid_sched_wakeup_probe_pre, data: tr);
1151 unregister_trace_sched_wakeup_new(probe: event_filter_pid_sched_wakeup_probe_post, data: tr);
1152
1153 unregister_trace_sched_waking(probe: event_filter_pid_sched_wakeup_probe_pre, data: tr);
1154 unregister_trace_sched_waking(probe: event_filter_pid_sched_wakeup_probe_post, data: tr);
1155}
1156
1157static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
1158{
1159 struct trace_pid_list *pid_list;
1160 struct trace_pid_list *no_pid_list;
1161 struct trace_event_file *file;
1162 int cpu;
1163
1164 pid_list = rcu_dereference_protected(tr->filtered_pids,
1165 lockdep_is_held(&event_mutex));
1166 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1167 lockdep_is_held(&event_mutex));
1168
1169 /* Make sure there's something to do */
1170 if (!pid_type_enabled(type, pid_list, no_pid_list))
1171 return;
1172
1173 if (!still_need_pid_events(type, pid_list, no_pid_list)) {
1174 unregister_pid_events(tr);
1175
1176 list_for_each_entry(file, &tr->events, list) {
1177 clear_bit(nr: EVENT_FILE_FL_PID_FILTER_BIT, addr: &file->flags);
1178 }
1179
1180 for_each_possible_cpu(cpu)
1181 per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
1182 }
1183
1184 if (type & TRACE_PIDS)
1185 rcu_assign_pointer(tr->filtered_pids, NULL);
1186
1187 if (type & TRACE_NO_PIDS)
1188 rcu_assign_pointer(tr->filtered_no_pids, NULL);
1189
1190 /* Wait till all users are no longer using pid filtering */
1191 tracepoint_synchronize_unregister();
1192
1193 if ((type & TRACE_PIDS) && pid_list)
1194 trace_pid_list_free(pid_list);
1195
1196 if ((type & TRACE_NO_PIDS) && no_pid_list)
1197 trace_pid_list_free(pid_list: no_pid_list);
1198}
1199
1200static void ftrace_clear_event_pids(struct trace_array *tr, int type)
1201{
1202 mutex_lock(lock: &event_mutex);
1203 __ftrace_clear_event_pids(tr, type);
1204 mutex_unlock(lock: &event_mutex);
1205}
1206
1207static void __put_system(struct event_subsystem *system)
1208{
1209 struct event_filter *filter = system->filter;
1210
1211 WARN_ON_ONCE(system_refcount(system) == 0);
1212 if (system_refcount_dec(system))
1213 return;
1214
1215 list_del(entry: &system->list);
1216
1217 if (filter) {
1218 kfree(objp: filter->filter_string);
1219 kfree(objp: filter);
1220 }
1221 kfree_const(x: system->name);
1222 kfree(objp: system);
1223}
1224
1225static void __get_system(struct event_subsystem *system)
1226{
1227 WARN_ON_ONCE(system_refcount(system) == 0);
1228 system_refcount_inc(system);
1229}
1230
1231static void __get_system_dir(struct trace_subsystem_dir *dir)
1232{
1233 WARN_ON_ONCE(dir->ref_count == 0);
1234 dir->ref_count++;
1235 __get_system(system: dir->subsystem);
1236}
1237
1238static void __put_system_dir(struct trace_subsystem_dir *dir)
1239{
1240 WARN_ON_ONCE(dir->ref_count == 0);
1241 /* If the subsystem is about to be freed, the dir must be too */
1242 WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
1243
1244 __put_system(system: dir->subsystem);
1245 if (!--dir->ref_count)
1246 kfree(objp: dir);
1247}
1248
1249static void put_system(struct trace_subsystem_dir *dir)
1250{
1251 mutex_lock(lock: &event_mutex);
1252 __put_system_dir(dir);
1253 mutex_unlock(lock: &event_mutex);
1254}
1255
1256static void remove_subsystem(struct trace_subsystem_dir *dir)
1257{
1258 if (!dir)
1259 return;
1260
1261 if (!--dir->nr_events) {
1262 eventfs_remove_dir(ei: dir->ei);
1263 list_del(entry: &dir->list);
1264 __put_system_dir(dir);
1265 }
1266}
1267
1268void event_file_get(struct trace_event_file *file)
1269{
1270 refcount_inc(r: &file->ref);
1271}
1272
1273void event_file_put(struct trace_event_file *file)
1274{
1275 if (WARN_ON_ONCE(!refcount_read(&file->ref))) {
1276 if (file->flags & EVENT_FILE_FL_FREED)
1277 kmem_cache_free(s: file_cachep, objp: file);
1278 return;
1279 }
1280
1281 if (refcount_dec_and_test(r: &file->ref)) {
1282 /* Count should only go to zero when it is freed */
1283 if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1284 return;
1285 kmem_cache_free(s: file_cachep, objp: file);
1286 }
1287}
1288
1289static void remove_event_file_dir(struct trace_event_file *file)
1290{
1291 eventfs_remove_dir(ei: file->ei);
1292 list_del(entry: &file->list);
1293 remove_subsystem(dir: file->system);
1294 free_event_filter(filter: file->filter);
1295 file->flags |= EVENT_FILE_FL_FREED;
1296 event_file_put(file);
1297}
1298
1299/*
1300 * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1301 */
1302static int
1303__ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1304 const char *sub, const char *event, int set,
1305 const char *mod)
1306{
1307 struct trace_event_file *file;
1308 struct trace_event_call *call;
1309 char *module __free(kfree) = NULL;
1310 const char *name;
1311 int ret = -EINVAL;
1312 int eret = 0;
1313
1314 if (mod) {
1315 char *p;
1316
1317 module = kstrdup(s: mod, GFP_KERNEL);
1318 if (!module)
1319 return -ENOMEM;
1320
1321 /* Replace all '-' with '_' as that's what modules do */
1322 for (p = strchr(module, '-'); p; p = strchr(p + 1, '-'))
1323 *p = '_';
1324 }
1325
1326 list_for_each_entry(file, &tr->events, list) {
1327
1328 call = file->event_call;
1329
1330 /* If a module is specified, skip events that are not that module */
1331 if (module && (!call->module || strcmp(module_name(call->module), module)))
1332 continue;
1333
1334 name = trace_event_name(call);
1335
1336 if (!name || !call->class || !call->class->reg)
1337 continue;
1338
1339 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1340 continue;
1341
1342 if (match &&
1343 strcmp(match, name) != 0 &&
1344 strcmp(match, call->class->system) != 0)
1345 continue;
1346
1347 if (sub && strcmp(sub, call->class->system) != 0)
1348 continue;
1349
1350 if (event && strcmp(event, name) != 0)
1351 continue;
1352
1353 ret = ftrace_event_enable_disable(file, enable: set);
1354
1355 /*
1356 * Save the first error and return that. Some events
1357 * may still have been enabled, but let the user
1358 * know that something went wrong.
1359 */
1360 if (ret && !eret)
1361 eret = ret;
1362
1363 ret = eret;
1364 }
1365
1366 /*
1367 * If this is a module setting and nothing was found,
1368 * check if the module was loaded. If it wasn't cache it.
1369 */
1370 if (module && ret == -EINVAL && !eret)
1371 ret = cache_mod(tr, mod: module, set, match, system: sub, event);
1372
1373 return ret;
1374}
1375
1376static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1377 const char *sub, const char *event, int set,
1378 const char *mod)
1379{
1380 int ret;
1381
1382 mutex_lock(lock: &event_mutex);
1383 ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set, mod);
1384 mutex_unlock(lock: &event_mutex);
1385
1386 return ret;
1387}
1388
1389int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1390{
1391 char *event = NULL, *sub = NULL, *match, *mod;
1392 int ret;
1393
1394 if (!tr)
1395 return -ENOENT;
1396
1397 /* Modules events can be appened with :mod:<module> */
1398 mod = strstr(buf, ":mod:");
1399 if (mod) {
1400 *mod = '\0';
1401 /* move to the module name */
1402 mod += 5;
1403 }
1404
1405 /*
1406 * The buf format can be <subsystem>:<event-name>
1407 * *:<event-name> means any event by that name.
1408 * :<event-name> is the same.
1409 *
1410 * <subsystem>:* means all events in that subsystem
1411 * <subsystem>: means the same.
1412 *
1413 * <name> (no ':') means all events in a subsystem with
1414 * the name <name> or any event that matches <name>
1415 */
1416
1417 match = strsep(&buf, ":");
1418 if (buf) {
1419 sub = match;
1420 event = buf;
1421 match = NULL;
1422
1423 if (!strlen(sub) || strcmp(sub, "*") == 0)
1424 sub = NULL;
1425 if (!strlen(event) || strcmp(event, "*") == 0)
1426 event = NULL;
1427 } else if (mod) {
1428 /* Allow wildcard for no length or star */
1429 if (!strlen(match) || strcmp(match, "*") == 0)
1430 match = NULL;
1431 }
1432
1433 ret = __ftrace_set_clr_event(tr, match, sub, event, set, mod);
1434
1435 /* Put back the colon to allow this to be called again */
1436 if (buf)
1437 *(buf - 1) = ':';
1438
1439 return ret;
1440}
1441
1442/**
1443 * trace_set_clr_event - enable or disable an event
1444 * @system: system name to match (NULL for any system)
1445 * @event: event name to match (NULL for all events, within system)
1446 * @set: 1 to enable, 0 to disable
1447 *
1448 * This is a way for other parts of the kernel to enable or disable
1449 * event recording.
1450 *
1451 * Returns 0 on success, -EINVAL if the parameters do not match any
1452 * registered events.
1453 */
1454int trace_set_clr_event(const char *system, const char *event, int set)
1455{
1456 struct trace_array *tr = top_trace_array();
1457
1458 if (!tr)
1459 return -ENODEV;
1460
1461 return __ftrace_set_clr_event(tr, NULL, sub: system, event, set, NULL);
1462}
1463EXPORT_SYMBOL_GPL(trace_set_clr_event);
1464
1465/**
1466 * trace_array_set_clr_event - enable or disable an event for a trace array.
1467 * @tr: concerned trace array.
1468 * @system: system name to match (NULL for any system)
1469 * @event: event name to match (NULL for all events, within system)
1470 * @enable: true to enable, false to disable
1471 *
1472 * This is a way for other parts of the kernel to enable or disable
1473 * event recording.
1474 *
1475 * Returns 0 on success, -EINVAL if the parameters do not match any
1476 * registered events.
1477 */
1478int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1479 const char *event, bool enable)
1480{
1481 int set;
1482
1483 if (!tr)
1484 return -ENOENT;
1485
1486 set = (enable == true) ? 1 : 0;
1487 return __ftrace_set_clr_event(tr, NULL, sub: system, event, set, NULL);
1488}
1489EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1490
1491/* 128 should be much more than enough */
1492#define EVENT_BUF_SIZE 127
1493
1494static ssize_t
1495ftrace_event_write(struct file *file, const char __user *ubuf,
1496 size_t cnt, loff_t *ppos)
1497{
1498 struct trace_parser parser;
1499 struct seq_file *m = file->private_data;
1500 struct trace_array *tr = m->private;
1501 ssize_t read, ret;
1502
1503 if (!cnt)
1504 return 0;
1505
1506 ret = tracing_update_buffers(tr);
1507 if (ret < 0)
1508 return ret;
1509
1510 if (trace_parser_get_init(parser: &parser, EVENT_BUF_SIZE + 1))
1511 return -ENOMEM;
1512
1513 read = trace_get_user(parser: &parser, ubuf, cnt, ppos);
1514
1515 if (read >= 0 && trace_parser_loaded(parser: (&parser))) {
1516 int set = 1;
1517
1518 if (*parser.buffer == '!')
1519 set = 0;
1520
1521 ret = ftrace_set_clr_event(tr, buf: parser.buffer + !set, set);
1522 if (ret)
1523 goto out_put;
1524 }
1525
1526 ret = read;
1527
1528 out_put:
1529 trace_parser_put(parser: &parser);
1530
1531 return ret;
1532}
1533
1534static void *
1535t_next(struct seq_file *m, void *v, loff_t *pos)
1536{
1537 struct trace_event_file *file = v;
1538 struct trace_event_call *call;
1539 struct trace_array *tr = m->private;
1540
1541 (*pos)++;
1542
1543 list_for_each_entry_continue(file, &tr->events, list) {
1544 call = file->event_call;
1545 /*
1546 * The ftrace subsystem is for showing formats only.
1547 * They can not be enabled or disabled via the event files.
1548 */
1549 if (call->class && call->class->reg &&
1550 !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1551 return file;
1552 }
1553
1554 return NULL;
1555}
1556
1557static void *t_start(struct seq_file *m, loff_t *pos)
1558{
1559 struct trace_event_file *file;
1560 struct trace_array *tr = m->private;
1561 loff_t l;
1562
1563 mutex_lock(lock: &event_mutex);
1564
1565 file = list_entry(&tr->events, struct trace_event_file, list);
1566 for (l = 0; l <= *pos; ) {
1567 file = t_next(m, v: file, pos: &l);
1568 if (!file)
1569 break;
1570 }
1571 return file;
1572}
1573
1574enum set_event_iter_type {
1575 SET_EVENT_FILE,
1576 SET_EVENT_MOD,
1577};
1578
1579struct set_event_iter {
1580 enum set_event_iter_type type;
1581 union {
1582 struct trace_event_file *file;
1583 struct event_mod_load *event_mod;
1584 };
1585};
1586
1587static void *
1588s_next(struct seq_file *m, void *v, loff_t *pos)
1589{
1590 struct set_event_iter *iter = v;
1591 struct trace_event_file *file;
1592 struct trace_array *tr = m->private;
1593
1594 (*pos)++;
1595
1596 if (iter->type == SET_EVENT_FILE) {
1597 file = iter->file;
1598 list_for_each_entry_continue(file, &tr->events, list) {
1599 if (file->flags & EVENT_FILE_FL_ENABLED) {
1600 iter->file = file;
1601 return iter;
1602 }
1603 }
1604#ifdef CONFIG_MODULES
1605 iter->type = SET_EVENT_MOD;
1606 iter->event_mod = list_entry(&tr->mod_events, struct event_mod_load, list);
1607#endif
1608 }
1609
1610#ifdef CONFIG_MODULES
1611 list_for_each_entry_continue(iter->event_mod, &tr->mod_events, list)
1612 return iter;
1613#endif
1614
1615 /*
1616 * The iter is allocated in s_start() and passed via the 'v'
1617 * parameter. To stop the iterator, NULL must be returned. But
1618 * the return value is what the 'v' parameter in s_stop() receives
1619 * and frees. Free iter here as it will no longer be used.
1620 */
1621 kfree(objp: iter);
1622 return NULL;
1623}
1624
1625static void *s_start(struct seq_file *m, loff_t *pos)
1626{
1627 struct trace_array *tr = m->private;
1628 struct set_event_iter *iter;
1629 loff_t l;
1630
1631 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1632 mutex_lock(lock: &event_mutex);
1633 if (!iter)
1634 return NULL;
1635
1636 iter->type = SET_EVENT_FILE;
1637 iter->file = list_entry(&tr->events, struct trace_event_file, list);
1638
1639 for (l = 0; l <= *pos; ) {
1640 iter = s_next(m, v: iter, pos: &l);
1641 if (!iter)
1642 break;
1643 }
1644 return iter;
1645}
1646
1647static int t_show(struct seq_file *m, void *v)
1648{
1649 struct trace_event_file *file = v;
1650 struct trace_event_call *call = file->event_call;
1651
1652 if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1653 seq_printf(m, fmt: "%s:", call->class->system);
1654 seq_printf(m, fmt: "%s\n", trace_event_name(call));
1655
1656 return 0;
1657}
1658
1659static void t_stop(struct seq_file *m, void *p)
1660{
1661 mutex_unlock(lock: &event_mutex);
1662}
1663
1664#ifdef CONFIG_MODULES
1665static int s_show(struct seq_file *m, void *v)
1666{
1667 struct set_event_iter *iter = v;
1668 const char *system;
1669 const char *event;
1670
1671 if (iter->type == SET_EVENT_FILE)
1672 return t_show(m, v: iter->file);
1673
1674 /* When match is set, system and event are not */
1675 if (iter->event_mod->match) {
1676 seq_printf(m, fmt: "%s:mod:%s\n", iter->event_mod->match,
1677 iter->event_mod->module);
1678 return 0;
1679 }
1680
1681 system = iter->event_mod->system ? : "*";
1682 event = iter->event_mod->event ? : "*";
1683
1684 seq_printf(m, fmt: "%s:%s:mod:%s\n", system, event, iter->event_mod->module);
1685
1686 return 0;
1687}
1688#else /* CONFIG_MODULES */
1689static int s_show(struct seq_file *m, void *v)
1690{
1691 struct set_event_iter *iter = v;
1692
1693 return t_show(m, iter->file);
1694}
1695#endif
1696
1697static void s_stop(struct seq_file *m, void *v)
1698{
1699 kfree(objp: v);
1700 t_stop(m, NULL);
1701}
1702
1703static void *
1704__next(struct seq_file *m, void *v, loff_t *pos, int type)
1705{
1706 struct trace_array *tr = m->private;
1707 struct trace_pid_list *pid_list;
1708
1709 if (type == TRACE_PIDS)
1710 pid_list = rcu_dereference_sched(tr->filtered_pids);
1711 else
1712 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1713
1714 return trace_pid_next(pid_list, v, pos);
1715}
1716
1717static void *
1718p_next(struct seq_file *m, void *v, loff_t *pos)
1719{
1720 return __next(m, v, pos, type: TRACE_PIDS);
1721}
1722
1723static void *
1724np_next(struct seq_file *m, void *v, loff_t *pos)
1725{
1726 return __next(m, v, pos, type: TRACE_NO_PIDS);
1727}
1728
1729static void *__start(struct seq_file *m, loff_t *pos, int type)
1730 __acquires(RCU)
1731{
1732 struct trace_pid_list *pid_list;
1733 struct trace_array *tr = m->private;
1734
1735 /*
1736 * Grab the mutex, to keep calls to p_next() having the same
1737 * tr->filtered_pids as p_start() has.
1738 * If we just passed the tr->filtered_pids around, then RCU would
1739 * have been enough, but doing that makes things more complex.
1740 */
1741 mutex_lock(lock: &event_mutex);
1742 rcu_read_lock_sched();
1743
1744 if (type == TRACE_PIDS)
1745 pid_list = rcu_dereference_sched(tr->filtered_pids);
1746 else
1747 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1748
1749 if (!pid_list)
1750 return NULL;
1751
1752 return trace_pid_start(pid_list, pos);
1753}
1754
1755static void *p_start(struct seq_file *m, loff_t *pos)
1756 __acquires(RCU)
1757{
1758 return __start(m, pos, type: TRACE_PIDS);
1759}
1760
1761static void *np_start(struct seq_file *m, loff_t *pos)
1762 __acquires(RCU)
1763{
1764 return __start(m, pos, type: TRACE_NO_PIDS);
1765}
1766
1767static void p_stop(struct seq_file *m, void *p)
1768 __releases(RCU)
1769{
1770 rcu_read_unlock_sched();
1771 mutex_unlock(lock: &event_mutex);
1772}
1773
1774static ssize_t
1775event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1776 loff_t *ppos)
1777{
1778 struct trace_event_file *file;
1779 unsigned long flags;
1780 char buf[4] = "0";
1781
1782 mutex_lock(lock: &event_mutex);
1783 file = event_file_file(filp);
1784 if (likely(file))
1785 flags = file->flags;
1786 mutex_unlock(lock: &event_mutex);
1787
1788 if (!file)
1789 return -ENODEV;
1790
1791 if (flags & EVENT_FILE_FL_ENABLED &&
1792 !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1793 strcpy(buf, "1");
1794
1795 if (atomic_read(v: &file->sm_ref) != 0)
1796 strcat(buf, "*");
1797
1798 strcat(buf, "\n");
1799
1800 return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: strlen(buf));
1801}
1802
1803static ssize_t
1804event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1805 loff_t *ppos)
1806{
1807 struct trace_event_file *file;
1808 unsigned long val;
1809 int ret;
1810
1811 ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
1812 if (ret)
1813 return ret;
1814
1815 guard(mutex)(T: &event_mutex);
1816
1817 switch (val) {
1818 case 0:
1819 case 1:
1820 file = event_file_file(filp);
1821 if (!file)
1822 return -ENODEV;
1823 ret = tracing_update_buffers(tr: file->tr);
1824 if (ret < 0)
1825 return ret;
1826 ret = ftrace_event_enable_disable(file, enable: val);
1827 if (ret < 0)
1828 return ret;
1829 break;
1830
1831 default:
1832 return -EINVAL;
1833 }
1834
1835 *ppos += cnt;
1836
1837 return cnt;
1838}
1839
1840/*
1841 * Returns:
1842 * 0 : no events exist?
1843 * 1 : all events are disabled
1844 * 2 : all events are enabled
1845 * 3 : some events are enabled and some are enabled
1846 */
1847int trace_events_enabled(struct trace_array *tr, const char *system)
1848{
1849 struct trace_event_call *call;
1850 struct trace_event_file *file;
1851 int set = 0;
1852
1853 guard(mutex)(T: &event_mutex);
1854
1855 list_for_each_entry(file, &tr->events, list) {
1856 call = file->event_call;
1857 if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1858 !trace_event_name(call) || !call->class || !call->class->reg)
1859 continue;
1860
1861 if (system && strcmp(call->class->system, system) != 0)
1862 continue;
1863
1864 /*
1865 * We need to find out if all the events are set
1866 * or if all events or cleared, or if we have
1867 * a mixture.
1868 */
1869 set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1870
1871 /*
1872 * If we have a mixture, no need to look further.
1873 */
1874 if (set == 3)
1875 break;
1876 }
1877
1878 return set;
1879}
1880
1881static ssize_t
1882system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1883 loff_t *ppos)
1884{
1885 const char set_to_char[4] = { '?', '0', '1', 'X' };
1886 struct trace_subsystem_dir *dir = filp->private_data;
1887 struct event_subsystem *system = dir->subsystem;
1888 struct trace_array *tr = dir->tr;
1889 char buf[2];
1890 int set;
1891 int ret;
1892
1893 set = trace_events_enabled(tr, system: system ? system->name : NULL);
1894
1895 buf[0] = set_to_char[set];
1896 buf[1] = '\n';
1897
1898 ret = simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: 2);
1899
1900 return ret;
1901}
1902
1903static ssize_t
1904system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1905 loff_t *ppos)
1906{
1907 struct trace_subsystem_dir *dir = filp->private_data;
1908 struct event_subsystem *system = dir->subsystem;
1909 const char *name = NULL;
1910 unsigned long val;
1911 ssize_t ret;
1912
1913 ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
1914 if (ret)
1915 return ret;
1916
1917 ret = tracing_update_buffers(tr: dir->tr);
1918 if (ret < 0)
1919 return ret;
1920
1921 if (val != 0 && val != 1)
1922 return -EINVAL;
1923
1924 /*
1925 * Opening of "enable" adds a ref count to system,
1926 * so the name is safe to use.
1927 */
1928 if (system)
1929 name = system->name;
1930
1931 ret = __ftrace_set_clr_event(tr: dir->tr, NULL, sub: name, NULL, set: val, NULL);
1932 if (ret)
1933 goto out;
1934
1935 ret = cnt;
1936
1937out:
1938 *ppos += cnt;
1939
1940 return ret;
1941}
1942
1943enum {
1944 FORMAT_HEADER = 1,
1945 FORMAT_FIELD_SEPERATOR = 2,
1946 FORMAT_PRINTFMT = 3,
1947};
1948
1949static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1950{
1951 struct trace_event_file *file = event_file_data(filp: m->private);
1952 struct trace_event_call *call = file->event_call;
1953 struct list_head *common_head = &ftrace_common_fields;
1954 struct list_head *head = trace_get_fields(event_call: call);
1955 struct list_head *node = v;
1956
1957 (*pos)++;
1958
1959 switch ((unsigned long)v) {
1960 case FORMAT_HEADER:
1961 node = common_head;
1962 break;
1963
1964 case FORMAT_FIELD_SEPERATOR:
1965 node = head;
1966 break;
1967
1968 case FORMAT_PRINTFMT:
1969 /* all done */
1970 return NULL;
1971 }
1972
1973 node = node->prev;
1974 if (node == common_head)
1975 return (void *)FORMAT_FIELD_SEPERATOR;
1976 else if (node == head)
1977 return (void *)FORMAT_PRINTFMT;
1978 else
1979 return node;
1980}
1981
1982static int f_show(struct seq_file *m, void *v)
1983{
1984 struct trace_event_file *file = event_file_data(filp: m->private);
1985 struct trace_event_call *call = file->event_call;
1986 struct ftrace_event_field *field;
1987 const char *array_descriptor;
1988
1989 switch ((unsigned long)v) {
1990 case FORMAT_HEADER:
1991 seq_printf(m, fmt: "name: %s\n", trace_event_name(call));
1992 seq_printf(m, fmt: "ID: %d\n", call->event.type);
1993 seq_puts(m, s: "format:\n");
1994 return 0;
1995
1996 case FORMAT_FIELD_SEPERATOR:
1997 seq_putc(m, c: '\n');
1998 return 0;
1999
2000 case FORMAT_PRINTFMT:
2001 seq_printf(m, fmt: "\nprint fmt: %s\n",
2002 call->print_fmt);
2003 return 0;
2004 }
2005
2006 field = list_entry(v, struct ftrace_event_field, link);
2007 /*
2008 * Smartly shows the array type(except dynamic array).
2009 * Normal:
2010 * field:TYPE VAR
2011 * If TYPE := TYPE[LEN], it is shown:
2012 * field:TYPE VAR[LEN]
2013 */
2014 array_descriptor = strchr(field->type, '[');
2015
2016 if (str_has_prefix(str: field->type, prefix: "__data_loc"))
2017 array_descriptor = NULL;
2018
2019 if (!array_descriptor)
2020 seq_printf(m, fmt: "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2021 field->type, field->name, field->offset,
2022 field->size, !!field->is_signed);
2023 else if (field->len)
2024 seq_printf(m, fmt: "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2025 (int)(array_descriptor - field->type),
2026 field->type, field->name,
2027 field->len, field->offset,
2028 field->size, !!field->is_signed);
2029 else
2030 seq_printf(m, fmt: "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2031 (int)(array_descriptor - field->type),
2032 field->type, field->name,
2033 field->offset, field->size, !!field->is_signed);
2034
2035 return 0;
2036}
2037
2038static void *f_start(struct seq_file *m, loff_t *pos)
2039{
2040 struct trace_event_file *file;
2041 void *p = (void *)FORMAT_HEADER;
2042 loff_t l = 0;
2043
2044 /* ->stop() is called even if ->start() fails */
2045 mutex_lock(lock: &event_mutex);
2046 file = event_file_file(filp: m->private);
2047 if (!file)
2048 return ERR_PTR(error: -ENODEV);
2049
2050 while (l < *pos && p)
2051 p = f_next(m, v: p, pos: &l);
2052
2053 return p;
2054}
2055
2056static void f_stop(struct seq_file *m, void *p)
2057{
2058 mutex_unlock(lock: &event_mutex);
2059}
2060
2061static const struct seq_operations trace_format_seq_ops = {
2062 .start = f_start,
2063 .next = f_next,
2064 .stop = f_stop,
2065 .show = f_show,
2066};
2067
2068static int trace_format_open(struct inode *inode, struct file *file)
2069{
2070 struct seq_file *m;
2071 int ret;
2072
2073 /* Do we want to hide event format files on tracefs lockdown? */
2074
2075 ret = seq_open(file, &trace_format_seq_ops);
2076 if (ret < 0)
2077 return ret;
2078
2079 m = file->private_data;
2080 m->private = file;
2081
2082 return 0;
2083}
2084
2085#ifdef CONFIG_PERF_EVENTS
2086static ssize_t
2087event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2088{
2089 int id = (long)event_file_data(filp);
2090 char buf[32];
2091 int len;
2092
2093 if (unlikely(!id))
2094 return -ENODEV;
2095
2096 len = sprintf(buf, fmt: "%d\n", id);
2097
2098 return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: len);
2099}
2100#endif
2101
2102static ssize_t
2103event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2104 loff_t *ppos)
2105{
2106 struct trace_event_file *file;
2107 struct trace_seq *s;
2108 int r = -ENODEV;
2109
2110 if (*ppos)
2111 return 0;
2112
2113 s = kmalloc(sizeof(*s), GFP_KERNEL);
2114
2115 if (!s)
2116 return -ENOMEM;
2117
2118 trace_seq_init(s);
2119
2120 mutex_lock(lock: &event_mutex);
2121 file = event_file_file(filp);
2122 if (file)
2123 print_event_filter(file, s);
2124 mutex_unlock(lock: &event_mutex);
2125
2126 if (file)
2127 r = simple_read_from_buffer(to: ubuf, count: cnt, ppos,
2128 from: s->buffer, available: trace_seq_used(s));
2129
2130 kfree(objp: s);
2131
2132 return r;
2133}
2134
2135static ssize_t
2136event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2137 loff_t *ppos)
2138{
2139 struct trace_event_file *file;
2140 char *buf;
2141 int err = -ENODEV;
2142
2143 if (cnt >= PAGE_SIZE)
2144 return -EINVAL;
2145
2146 buf = memdup_user_nul(ubuf, cnt);
2147 if (IS_ERR(ptr: buf))
2148 return PTR_ERR(ptr: buf);
2149
2150 mutex_lock(lock: &event_mutex);
2151 file = event_file_file(filp);
2152 if (file) {
2153 if (file->flags & EVENT_FILE_FL_FREED)
2154 err = -ENODEV;
2155 else
2156 err = apply_event_filter(file, filter_string: buf);
2157 }
2158 mutex_unlock(lock: &event_mutex);
2159
2160 kfree(objp: buf);
2161 if (err < 0)
2162 return err;
2163
2164 *ppos += cnt;
2165
2166 return cnt;
2167}
2168
2169static LIST_HEAD(event_subsystems);
2170
2171static int subsystem_open(struct inode *inode, struct file *filp)
2172{
2173 struct trace_subsystem_dir *dir = NULL, *iter_dir;
2174 struct trace_array *tr = NULL, *iter_tr;
2175 struct event_subsystem *system = NULL;
2176 int ret;
2177
2178 if (tracing_is_disabled())
2179 return -ENODEV;
2180
2181 /* Make sure the system still exists */
2182 mutex_lock(lock: &event_mutex);
2183 mutex_lock(lock: &trace_types_lock);
2184 list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
2185 list_for_each_entry(iter_dir, &iter_tr->systems, list) {
2186 if (iter_dir == inode->i_private) {
2187 /* Don't open systems with no events */
2188 tr = iter_tr;
2189 dir = iter_dir;
2190 if (dir->nr_events) {
2191 __get_system_dir(dir);
2192 system = dir->subsystem;
2193 }
2194 goto exit_loop;
2195 }
2196 }
2197 }
2198 exit_loop:
2199 mutex_unlock(lock: &trace_types_lock);
2200 mutex_unlock(lock: &event_mutex);
2201
2202 if (!system)
2203 return -ENODEV;
2204
2205 /* Still need to increment the ref count of the system */
2206 if (trace_array_get(tr) < 0) {
2207 put_system(dir);
2208 return -ENODEV;
2209 }
2210
2211 ret = tracing_open_generic(inode, filp);
2212 if (ret < 0) {
2213 trace_array_put(tr);
2214 put_system(dir);
2215 }
2216
2217 return ret;
2218}
2219
2220static int system_tr_open(struct inode *inode, struct file *filp)
2221{
2222 struct trace_subsystem_dir *dir;
2223 struct trace_array *tr = inode->i_private;
2224 int ret;
2225
2226 /* Make a temporary dir that has no system but points to tr */
2227 dir = kzalloc(sizeof(*dir), GFP_KERNEL);
2228 if (!dir)
2229 return -ENOMEM;
2230
2231 ret = tracing_open_generic_tr(inode, filp);
2232 if (ret < 0) {
2233 kfree(objp: dir);
2234 return ret;
2235 }
2236 dir->tr = tr;
2237 filp->private_data = dir;
2238
2239 return 0;
2240}
2241
2242static int subsystem_release(struct inode *inode, struct file *file)
2243{
2244 struct trace_subsystem_dir *dir = file->private_data;
2245
2246 trace_array_put(tr: dir->tr);
2247
2248 /*
2249 * If dir->subsystem is NULL, then this is a temporary
2250 * descriptor that was made for a trace_array to enable
2251 * all subsystems.
2252 */
2253 if (dir->subsystem)
2254 put_system(dir);
2255 else
2256 kfree(objp: dir);
2257
2258 return 0;
2259}
2260
2261static ssize_t
2262subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2263 loff_t *ppos)
2264{
2265 struct trace_subsystem_dir *dir = filp->private_data;
2266 struct event_subsystem *system = dir->subsystem;
2267 struct trace_seq *s;
2268 int r;
2269
2270 if (*ppos)
2271 return 0;
2272
2273 s = kmalloc(sizeof(*s), GFP_KERNEL);
2274 if (!s)
2275 return -ENOMEM;
2276
2277 trace_seq_init(s);
2278
2279 print_subsystem_event_filter(system, s);
2280 r = simple_read_from_buffer(to: ubuf, count: cnt, ppos,
2281 from: s->buffer, available: trace_seq_used(s));
2282
2283 kfree(objp: s);
2284
2285 return r;
2286}
2287
2288static ssize_t
2289subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2290 loff_t *ppos)
2291{
2292 struct trace_subsystem_dir *dir = filp->private_data;
2293 char *buf;
2294 int err;
2295
2296 if (cnt >= PAGE_SIZE)
2297 return -EINVAL;
2298
2299 buf = memdup_user_nul(ubuf, cnt);
2300 if (IS_ERR(ptr: buf))
2301 return PTR_ERR(ptr: buf);
2302
2303 err = apply_subsystem_event_filter(dir, filter_string: buf);
2304 kfree(objp: buf);
2305 if (err < 0)
2306 return err;
2307
2308 *ppos += cnt;
2309
2310 return cnt;
2311}
2312
2313static ssize_t
2314show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2315{
2316 struct trace_array *tr = filp->private_data;
2317 struct trace_seq *s;
2318 int r;
2319
2320 if (*ppos)
2321 return 0;
2322
2323 s = kmalloc(sizeof(*s), GFP_KERNEL);
2324 if (!s)
2325 return -ENOMEM;
2326
2327 trace_seq_init(s);
2328
2329 ring_buffer_print_page_header(buffer: tr->array_buffer.buffer, s);
2330 r = simple_read_from_buffer(to: ubuf, count: cnt, ppos,
2331 from: s->buffer, available: trace_seq_used(s));
2332
2333 kfree(objp: s);
2334
2335 return r;
2336}
2337
2338static ssize_t
2339show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2340{
2341 struct trace_seq *s;
2342 int r;
2343
2344 if (*ppos)
2345 return 0;
2346
2347 s = kmalloc(sizeof(*s), GFP_KERNEL);
2348 if (!s)
2349 return -ENOMEM;
2350
2351 trace_seq_init(s);
2352
2353 ring_buffer_print_entry_header(s);
2354 r = simple_read_from_buffer(to: ubuf, count: cnt, ppos,
2355 from: s->buffer, available: trace_seq_used(s));
2356
2357 kfree(objp: s);
2358
2359 return r;
2360}
2361
2362static void ignore_task_cpu(void *data)
2363{
2364 struct trace_array *tr = data;
2365 struct trace_pid_list *pid_list;
2366 struct trace_pid_list *no_pid_list;
2367
2368 /*
2369 * This function is called by on_each_cpu() while the
2370 * event_mutex is held.
2371 */
2372 pid_list = rcu_dereference_protected(tr->filtered_pids,
2373 mutex_is_locked(&event_mutex));
2374 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2375 mutex_is_locked(&event_mutex));
2376
2377 this_cpu_write(tr->array_buffer.data->ignore_pid,
2378 trace_ignore_this_task(pid_list, no_pid_list, current));
2379}
2380
2381static void register_pid_events(struct trace_array *tr)
2382{
2383 /*
2384 * Register a probe that is called before all other probes
2385 * to set ignore_pid if next or prev do not match.
2386 * Register a probe this is called after all other probes
2387 * to only keep ignore_pid set if next pid matches.
2388 */
2389 register_trace_prio_sched_switch(probe: event_filter_pid_sched_switch_probe_pre,
2390 data: tr, INT_MAX);
2391 register_trace_prio_sched_switch(probe: event_filter_pid_sched_switch_probe_post,
2392 data: tr, prio: 0);
2393
2394 register_trace_prio_sched_wakeup(probe: event_filter_pid_sched_wakeup_probe_pre,
2395 data: tr, INT_MAX);
2396 register_trace_prio_sched_wakeup(probe: event_filter_pid_sched_wakeup_probe_post,
2397 data: tr, prio: 0);
2398
2399 register_trace_prio_sched_wakeup_new(probe: event_filter_pid_sched_wakeup_probe_pre,
2400 data: tr, INT_MAX);
2401 register_trace_prio_sched_wakeup_new(probe: event_filter_pid_sched_wakeup_probe_post,
2402 data: tr, prio: 0);
2403
2404 register_trace_prio_sched_waking(probe: event_filter_pid_sched_wakeup_probe_pre,
2405 data: tr, INT_MAX);
2406 register_trace_prio_sched_waking(probe: event_filter_pid_sched_wakeup_probe_post,
2407 data: tr, prio: 0);
2408}
2409
2410static ssize_t
2411event_pid_write(struct file *filp, const char __user *ubuf,
2412 size_t cnt, loff_t *ppos, int type)
2413{
2414 struct seq_file *m = filp->private_data;
2415 struct trace_array *tr = m->private;
2416 struct trace_pid_list *filtered_pids = NULL;
2417 struct trace_pid_list *other_pids = NULL;
2418 struct trace_pid_list *pid_list;
2419 struct trace_event_file *file;
2420 ssize_t ret;
2421
2422 if (!cnt)
2423 return 0;
2424
2425 ret = tracing_update_buffers(tr);
2426 if (ret < 0)
2427 return ret;
2428
2429 guard(mutex)(T: &event_mutex);
2430
2431 if (type == TRACE_PIDS) {
2432 filtered_pids = rcu_dereference_protected(tr->filtered_pids,
2433 lockdep_is_held(&event_mutex));
2434 other_pids = rcu_dereference_protected(tr->filtered_no_pids,
2435 lockdep_is_held(&event_mutex));
2436 } else {
2437 filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
2438 lockdep_is_held(&event_mutex));
2439 other_pids = rcu_dereference_protected(tr->filtered_pids,
2440 lockdep_is_held(&event_mutex));
2441 }
2442
2443 ret = trace_pid_write(filtered_pids, new_pid_list: &pid_list, ubuf, cnt);
2444 if (ret < 0)
2445 return ret;
2446
2447 if (type == TRACE_PIDS)
2448 rcu_assign_pointer(tr->filtered_pids, pid_list);
2449 else
2450 rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2451
2452 list_for_each_entry(file, &tr->events, list) {
2453 set_bit(nr: EVENT_FILE_FL_PID_FILTER_BIT, addr: &file->flags);
2454 }
2455
2456 if (filtered_pids) {
2457 tracepoint_synchronize_unregister();
2458 trace_pid_list_free(pid_list: filtered_pids);
2459 } else if (pid_list && !other_pids) {
2460 register_pid_events(tr);
2461 }
2462
2463 /*
2464 * Ignoring of pids is done at task switch. But we have to
2465 * check for those tasks that are currently running.
2466 * Always do this in case a pid was appended or removed.
2467 */
2468 on_each_cpu(func: ignore_task_cpu, info: tr, wait: 1);
2469
2470 *ppos += ret;
2471
2472 return ret;
2473}
2474
2475static ssize_t
2476ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2477 size_t cnt, loff_t *ppos)
2478{
2479 return event_pid_write(filp, ubuf, cnt, ppos, type: TRACE_PIDS);
2480}
2481
2482static ssize_t
2483ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2484 size_t cnt, loff_t *ppos)
2485{
2486 return event_pid_write(filp, ubuf, cnt, ppos, type: TRACE_NO_PIDS);
2487}
2488
2489static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2490static int ftrace_event_set_open(struct inode *inode, struct file *file);
2491static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2492static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2493static int ftrace_event_release(struct inode *inode, struct file *file);
2494
2495static const struct seq_operations show_event_seq_ops = {
2496 .start = t_start,
2497 .next = t_next,
2498 .show = t_show,
2499 .stop = t_stop,
2500};
2501
2502static const struct seq_operations show_set_event_seq_ops = {
2503 .start = s_start,
2504 .next = s_next,
2505 .show = s_show,
2506 .stop = s_stop,
2507};
2508
2509static const struct seq_operations show_set_pid_seq_ops = {
2510 .start = p_start,
2511 .next = p_next,
2512 .show = trace_pid_show,
2513 .stop = p_stop,
2514};
2515
2516static const struct seq_operations show_set_no_pid_seq_ops = {
2517 .start = np_start,
2518 .next = np_next,
2519 .show = trace_pid_show,
2520 .stop = p_stop,
2521};
2522
2523static const struct file_operations ftrace_avail_fops = {
2524 .open = ftrace_event_avail_open,
2525 .read = seq_read,
2526 .llseek = seq_lseek,
2527 .release = seq_release,
2528};
2529
2530static const struct file_operations ftrace_set_event_fops = {
2531 .open = ftrace_event_set_open,
2532 .read = seq_read,
2533 .write = ftrace_event_write,
2534 .llseek = seq_lseek,
2535 .release = ftrace_event_release,
2536};
2537
2538static const struct file_operations ftrace_set_event_pid_fops = {
2539 .open = ftrace_event_set_pid_open,
2540 .read = seq_read,
2541 .write = ftrace_event_pid_write,
2542 .llseek = seq_lseek,
2543 .release = ftrace_event_release,
2544};
2545
2546static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2547 .open = ftrace_event_set_npid_open,
2548 .read = seq_read,
2549 .write = ftrace_event_npid_write,
2550 .llseek = seq_lseek,
2551 .release = ftrace_event_release,
2552};
2553
2554static const struct file_operations ftrace_enable_fops = {
2555 .open = tracing_open_file_tr,
2556 .read = event_enable_read,
2557 .write = event_enable_write,
2558 .release = tracing_release_file_tr,
2559 .llseek = default_llseek,
2560};
2561
2562static const struct file_operations ftrace_event_format_fops = {
2563 .open = trace_format_open,
2564 .read = seq_read,
2565 .llseek = seq_lseek,
2566 .release = seq_release,
2567};
2568
2569#ifdef CONFIG_PERF_EVENTS
2570static const struct file_operations ftrace_event_id_fops = {
2571 .read = event_id_read,
2572 .llseek = default_llseek,
2573};
2574#endif
2575
2576static const struct file_operations ftrace_event_filter_fops = {
2577 .open = tracing_open_file_tr,
2578 .read = event_filter_read,
2579 .write = event_filter_write,
2580 .release = tracing_release_file_tr,
2581 .llseek = default_llseek,
2582};
2583
2584static const struct file_operations ftrace_subsystem_filter_fops = {
2585 .open = subsystem_open,
2586 .read = subsystem_filter_read,
2587 .write = subsystem_filter_write,
2588 .llseek = default_llseek,
2589 .release = subsystem_release,
2590};
2591
2592static const struct file_operations ftrace_system_enable_fops = {
2593 .open = subsystem_open,
2594 .read = system_enable_read,
2595 .write = system_enable_write,
2596 .llseek = default_llseek,
2597 .release = subsystem_release,
2598};
2599
2600static const struct file_operations ftrace_tr_enable_fops = {
2601 .open = system_tr_open,
2602 .read = system_enable_read,
2603 .write = system_enable_write,
2604 .llseek = default_llseek,
2605 .release = subsystem_release,
2606};
2607
2608static const struct file_operations ftrace_show_header_page_fops = {
2609 .open = tracing_open_generic_tr,
2610 .read = show_header_page_file,
2611 .llseek = default_llseek,
2612 .release = tracing_release_generic_tr,
2613};
2614
2615static const struct file_operations ftrace_show_header_event_fops = {
2616 .open = tracing_open_generic_tr,
2617 .read = show_header_event_file,
2618 .llseek = default_llseek,
2619 .release = tracing_release_generic_tr,
2620};
2621
2622static int
2623ftrace_event_open(struct inode *inode, struct file *file,
2624 const struct seq_operations *seq_ops)
2625{
2626 struct seq_file *m;
2627 int ret;
2628
2629 ret = security_locked_down(what: LOCKDOWN_TRACEFS);
2630 if (ret)
2631 return ret;
2632
2633 ret = seq_open(file, seq_ops);
2634 if (ret < 0)
2635 return ret;
2636 m = file->private_data;
2637 /* copy tr over to seq ops */
2638 m->private = inode->i_private;
2639
2640 return ret;
2641}
2642
2643static int ftrace_event_release(struct inode *inode, struct file *file)
2644{
2645 struct trace_array *tr = inode->i_private;
2646
2647 trace_array_put(tr);
2648
2649 return seq_release(inode, file);
2650}
2651
2652static int
2653ftrace_event_avail_open(struct inode *inode, struct file *file)
2654{
2655 const struct seq_operations *seq_ops = &show_event_seq_ops;
2656
2657 /* Checks for tracefs lockdown */
2658 return ftrace_event_open(inode, file, seq_ops);
2659}
2660
2661static int
2662ftrace_event_set_open(struct inode *inode, struct file *file)
2663{
2664 const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2665 struct trace_array *tr = inode->i_private;
2666 int ret;
2667
2668 ret = tracing_check_open_get_tr(tr);
2669 if (ret)
2670 return ret;
2671
2672 if ((file->f_mode & FMODE_WRITE) &&
2673 (file->f_flags & O_TRUNC))
2674 ftrace_clear_events(tr);
2675
2676 ret = ftrace_event_open(inode, file, seq_ops);
2677 if (ret < 0)
2678 trace_array_put(tr);
2679 return ret;
2680}
2681
2682static int
2683ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2684{
2685 const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2686 struct trace_array *tr = inode->i_private;
2687 int ret;
2688
2689 ret = tracing_check_open_get_tr(tr);
2690 if (ret)
2691 return ret;
2692
2693 if ((file->f_mode & FMODE_WRITE) &&
2694 (file->f_flags & O_TRUNC))
2695 ftrace_clear_event_pids(tr, type: TRACE_PIDS);
2696
2697 ret = ftrace_event_open(inode, file, seq_ops);
2698 if (ret < 0)
2699 trace_array_put(tr);
2700 return ret;
2701}
2702
2703static int
2704ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2705{
2706 const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2707 struct trace_array *tr = inode->i_private;
2708 int ret;
2709
2710 ret = tracing_check_open_get_tr(tr);
2711 if (ret)
2712 return ret;
2713
2714 if ((file->f_mode & FMODE_WRITE) &&
2715 (file->f_flags & O_TRUNC))
2716 ftrace_clear_event_pids(tr, type: TRACE_NO_PIDS);
2717
2718 ret = ftrace_event_open(inode, file, seq_ops);
2719 if (ret < 0)
2720 trace_array_put(tr);
2721 return ret;
2722}
2723
2724static struct event_subsystem *
2725create_new_subsystem(const char *name)
2726{
2727 struct event_subsystem *system;
2728
2729 /* need to create new entry */
2730 system = kmalloc(sizeof(*system), GFP_KERNEL);
2731 if (!system)
2732 return NULL;
2733
2734 system->ref_count = 1;
2735
2736 /* Only allocate if dynamic (kprobes and modules) */
2737 system->name = kstrdup_const(s: name, GFP_KERNEL);
2738 if (!system->name)
2739 goto out_free;
2740
2741 system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2742 if (!system->filter)
2743 goto out_free;
2744
2745 list_add(new: &system->list, head: &event_subsystems);
2746
2747 return system;
2748
2749 out_free:
2750 kfree_const(x: system->name);
2751 kfree(objp: system);
2752 return NULL;
2753}
2754
2755static int system_callback(const char *name, umode_t *mode, void **data,
2756 const struct file_operations **fops)
2757{
2758 if (strcmp(name, "filter") == 0)
2759 *fops = &ftrace_subsystem_filter_fops;
2760
2761 else if (strcmp(name, "enable") == 0)
2762 *fops = &ftrace_system_enable_fops;
2763
2764 else
2765 return 0;
2766
2767 *mode = TRACE_MODE_WRITE;
2768 return 1;
2769}
2770
2771static struct eventfs_inode *
2772event_subsystem_dir(struct trace_array *tr, const char *name,
2773 struct trace_event_file *file, struct eventfs_inode *parent)
2774{
2775 struct event_subsystem *system, *iter;
2776 struct trace_subsystem_dir *dir;
2777 struct eventfs_inode *ei;
2778 int nr_entries;
2779 static struct eventfs_entry system_entries[] = {
2780 {
2781 .name = "filter",
2782 .callback = system_callback,
2783 },
2784 {
2785 .name = "enable",
2786 .callback = system_callback,
2787 }
2788 };
2789
2790 /* First see if we did not already create this dir */
2791 list_for_each_entry(dir, &tr->systems, list) {
2792 system = dir->subsystem;
2793 if (strcmp(system->name, name) == 0) {
2794 dir->nr_events++;
2795 file->system = dir;
2796 return dir->ei;
2797 }
2798 }
2799
2800 /* Now see if the system itself exists. */
2801 system = NULL;
2802 list_for_each_entry(iter, &event_subsystems, list) {
2803 if (strcmp(iter->name, name) == 0) {
2804 system = iter;
2805 break;
2806 }
2807 }
2808
2809 dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2810 if (!dir)
2811 goto out_fail;
2812
2813 if (!system) {
2814 system = create_new_subsystem(name);
2815 if (!system)
2816 goto out_free;
2817 } else
2818 __get_system(system);
2819
2820 /* ftrace only has directories no files */
2821 if (strcmp(name, "ftrace") == 0)
2822 nr_entries = 0;
2823 else
2824 nr_entries = ARRAY_SIZE(system_entries);
2825
2826 ei = eventfs_create_dir(name, parent, entries: system_entries, size: nr_entries, data: dir);
2827 if (IS_ERR(ptr: ei)) {
2828 pr_warn("Failed to create system directory %s\n", name);
2829 __put_system(system);
2830 goto out_free;
2831 }
2832
2833 dir->ei = ei;
2834 dir->tr = tr;
2835 dir->ref_count = 1;
2836 dir->nr_events = 1;
2837 dir->subsystem = system;
2838 file->system = dir;
2839
2840 list_add(new: &dir->list, head: &tr->systems);
2841
2842 return dir->ei;
2843
2844 out_free:
2845 kfree(objp: dir);
2846 out_fail:
2847 /* Only print this message if failed on memory allocation */
2848 if (!dir || !system)
2849 pr_warn("No memory to create event subsystem %s\n", name);
2850 return NULL;
2851}
2852
2853static int
2854event_define_fields(struct trace_event_call *call)
2855{
2856 struct list_head *head;
2857 int ret = 0;
2858
2859 /*
2860 * Other events may have the same class. Only update
2861 * the fields if they are not already defined.
2862 */
2863 head = trace_get_fields(event_call: call);
2864 if (list_empty(head)) {
2865 struct trace_event_fields *field = call->class->fields_array;
2866 unsigned int offset = sizeof(struct trace_entry);
2867
2868 for (; field->type; field++) {
2869 if (field->type == TRACE_FUNCTION_TYPE) {
2870 field->define_fields(call);
2871 break;
2872 }
2873
2874 offset = ALIGN(offset, field->align);
2875 ret = trace_define_field_ext(call, type: field->type, name: field->name,
2876 offset, size: field->size,
2877 is_signed: field->is_signed, filter_type: field->filter_type,
2878 len: field->len, need_test: field->needs_test);
2879 if (WARN_ON_ONCE(ret)) {
2880 pr_err("error code is %d\n", ret);
2881 break;
2882 }
2883
2884 offset += field->size;
2885 }
2886 }
2887
2888 return ret;
2889}
2890
2891static int event_callback(const char *name, umode_t *mode, void **data,
2892 const struct file_operations **fops)
2893{
2894 struct trace_event_file *file = *data;
2895 struct trace_event_call *call = file->event_call;
2896
2897 if (strcmp(name, "format") == 0) {
2898 *mode = TRACE_MODE_READ;
2899 *fops = &ftrace_event_format_fops;
2900 return 1;
2901 }
2902
2903 /*
2904 * Only event directories that can be enabled should have
2905 * triggers or filters, with the exception of the "print"
2906 * event that can have a "trigger" file.
2907 */
2908 if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2909 if (call->class->reg && strcmp(name, "enable") == 0) {
2910 *mode = TRACE_MODE_WRITE;
2911 *fops = &ftrace_enable_fops;
2912 return 1;
2913 }
2914
2915 if (strcmp(name, "filter") == 0) {
2916 *mode = TRACE_MODE_WRITE;
2917 *fops = &ftrace_event_filter_fops;
2918 return 1;
2919 }
2920 }
2921
2922 if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
2923 strcmp(trace_event_name(call), "print") == 0) {
2924 if (strcmp(name, "trigger") == 0) {
2925 *mode = TRACE_MODE_WRITE;
2926 *fops = &event_trigger_fops;
2927 return 1;
2928 }
2929 }
2930
2931#ifdef CONFIG_PERF_EVENTS
2932 if (call->event.type && call->class->reg &&
2933 strcmp(name, "id") == 0) {
2934 *mode = TRACE_MODE_READ;
2935 *data = (void *)(long)call->event.type;
2936 *fops = &ftrace_event_id_fops;
2937 return 1;
2938 }
2939#endif
2940
2941#ifdef CONFIG_HIST_TRIGGERS
2942 if (strcmp(name, "hist") == 0) {
2943 *mode = TRACE_MODE_READ;
2944 *fops = &event_hist_fops;
2945 return 1;
2946 }
2947#endif
2948#ifdef CONFIG_HIST_TRIGGERS_DEBUG
2949 if (strcmp(name, "hist_debug") == 0) {
2950 *mode = TRACE_MODE_READ;
2951 *fops = &event_hist_debug_fops;
2952 return 1;
2953 }
2954#endif
2955#ifdef CONFIG_TRACE_EVENT_INJECT
2956 if (call->event.type && call->class->reg &&
2957 strcmp(name, "inject") == 0) {
2958 *mode = 0200;
2959 *fops = &event_inject_fops;
2960 return 1;
2961 }
2962#endif
2963 return 0;
2964}
2965
2966/* The file is incremented on creation and freeing the enable file decrements it */
2967static void event_release(const char *name, void *data)
2968{
2969 struct trace_event_file *file = data;
2970
2971 event_file_put(file);
2972}
2973
2974static int
2975event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
2976{
2977 struct trace_event_call *call = file->event_call;
2978 struct trace_array *tr = file->tr;
2979 struct eventfs_inode *e_events;
2980 struct eventfs_inode *ei;
2981 const char *name;
2982 int nr_entries;
2983 int ret;
2984 static struct eventfs_entry event_entries[] = {
2985 {
2986 .name = "enable",
2987 .callback = event_callback,
2988 .release = event_release,
2989 },
2990 {
2991 .name = "filter",
2992 .callback = event_callback,
2993 },
2994 {
2995 .name = "trigger",
2996 .callback = event_callback,
2997 },
2998 {
2999 .name = "format",
3000 .callback = event_callback,
3001 },
3002#ifdef CONFIG_PERF_EVENTS
3003 {
3004 .name = "id",
3005 .callback = event_callback,
3006 },
3007#endif
3008#ifdef CONFIG_HIST_TRIGGERS
3009 {
3010 .name = "hist",
3011 .callback = event_callback,
3012 },
3013#endif
3014#ifdef CONFIG_HIST_TRIGGERS_DEBUG
3015 {
3016 .name = "hist_debug",
3017 .callback = event_callback,
3018 },
3019#endif
3020#ifdef CONFIG_TRACE_EVENT_INJECT
3021 {
3022 .name = "inject",
3023 .callback = event_callback,
3024 },
3025#endif
3026 };
3027
3028 /*
3029 * If the trace point header did not define TRACE_SYSTEM
3030 * then the system would be called "TRACE_SYSTEM". This should
3031 * never happen.
3032 */
3033 if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
3034 return -ENODEV;
3035
3036 e_events = event_subsystem_dir(tr, name: call->class->system, file, parent);
3037 if (!e_events)
3038 return -ENOMEM;
3039
3040 nr_entries = ARRAY_SIZE(event_entries);
3041
3042 name = trace_event_name(call);
3043 ei = eventfs_create_dir(name, parent: e_events, entries: event_entries, size: nr_entries, data: file);
3044 if (IS_ERR(ptr: ei)) {
3045 pr_warn("Could not create tracefs '%s' directory\n", name);
3046 return -1;
3047 }
3048
3049 file->ei = ei;
3050
3051 ret = event_define_fields(call);
3052 if (ret < 0) {
3053 pr_warn("Could not initialize trace point events/%s\n", name);
3054 return ret;
3055 }
3056
3057 /* Gets decremented on freeing of the "enable" file */
3058 event_file_get(file);
3059
3060 return 0;
3061}
3062
3063static void remove_event_from_tracers(struct trace_event_call *call)
3064{
3065 struct trace_event_file *file;
3066 struct trace_array *tr;
3067
3068 do_for_each_event_file_safe(tr, file) {
3069 if (file->event_call != call)
3070 continue;
3071
3072 remove_event_file_dir(file);
3073 /*
3074 * The do_for_each_event_file_safe() is
3075 * a double loop. After finding the call for this
3076 * trace_array, we use break to jump to the next
3077 * trace_array.
3078 */
3079 break;
3080 } while_for_each_event_file();
3081}
3082
3083static void event_remove(struct trace_event_call *call)
3084{
3085 struct trace_array *tr;
3086 struct trace_event_file *file;
3087
3088 do_for_each_event_file(tr, file) {
3089 if (file->event_call != call)
3090 continue;
3091
3092 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3093 tr->clear_trace = true;
3094
3095 ftrace_event_enable_disable(file, enable: 0);
3096 /*
3097 * The do_for_each_event_file() is
3098 * a double loop. After finding the call for this
3099 * trace_array, we use break to jump to the next
3100 * trace_array.
3101 */
3102 break;
3103 } while_for_each_event_file();
3104
3105 if (call->event.funcs)
3106 __unregister_trace_event(event: &call->event);
3107 remove_event_from_tracers(call);
3108 list_del(entry: &call->list);
3109}
3110
3111static int event_init(struct trace_event_call *call)
3112{
3113 int ret = 0;
3114 const char *name;
3115
3116 name = trace_event_name(call);
3117 if (WARN_ON(!name))
3118 return -EINVAL;
3119
3120 if (call->class->raw_init) {
3121 ret = call->class->raw_init(call);
3122 if (ret < 0 && ret != -ENOSYS)
3123 pr_warn("Could not initialize trace events/%s\n", name);
3124 }
3125
3126 return ret;
3127}
3128
3129static int
3130__register_event(struct trace_event_call *call, struct module *mod)
3131{
3132 int ret;
3133
3134 ret = event_init(call);
3135 if (ret < 0)
3136 return ret;
3137
3138 down_write(sem: &trace_event_sem);
3139 list_add(new: &call->list, head: &ftrace_events);
3140 up_write(sem: &trace_event_sem);
3141
3142 if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3143 atomic_set(v: &call->refcnt, i: 0);
3144 else
3145 call->module = mod;
3146
3147 return 0;
3148}
3149
3150static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
3151{
3152 int rlen;
3153 int elen;
3154
3155 /* Find the length of the eval value as a string */
3156 elen = snprintf(buf: ptr, size: 0, fmt: "%ld", map->eval_value);
3157 /* Make sure there's enough room to replace the string with the value */
3158 if (len < elen)
3159 return NULL;
3160
3161 snprintf(buf: ptr, size: elen + 1, fmt: "%ld", map->eval_value);
3162
3163 /* Get the rest of the string of ptr */
3164 rlen = strlen(ptr + len);
3165 memmove(dest: ptr + elen, src: ptr + len, count: rlen);
3166 /* Make sure we end the new string */
3167 ptr[elen + rlen] = 0;
3168
3169 return ptr + elen;
3170}
3171
3172static void update_event_printk(struct trace_event_call *call,
3173 struct trace_eval_map *map)
3174{
3175 char *ptr;
3176 int quote = 0;
3177 int len = strlen(map->eval_string);
3178
3179 for (ptr = call->print_fmt; *ptr; ptr++) {
3180 if (*ptr == '\\') {
3181 ptr++;
3182 /* paranoid */
3183 if (!*ptr)
3184 break;
3185 continue;
3186 }
3187 if (*ptr == '"') {
3188 quote ^= 1;
3189 continue;
3190 }
3191 if (quote)
3192 continue;
3193 if (isdigit(c: *ptr)) {
3194 /* skip numbers */
3195 do {
3196 ptr++;
3197 /* Check for alpha chars like ULL */
3198 } while (isalnum(*ptr));
3199 if (!*ptr)
3200 break;
3201 /*
3202 * A number must have some kind of delimiter after
3203 * it, and we can ignore that too.
3204 */
3205 continue;
3206 }
3207 if (isalpha(*ptr) || *ptr == '_') {
3208 if (strncmp(map->eval_string, ptr, len) == 0 &&
3209 !isalnum(ptr[len]) && ptr[len] != '_') {
3210 ptr = eval_replace(ptr, map, len);
3211 /* enum/sizeof string smaller than value */
3212 if (WARN_ON_ONCE(!ptr))
3213 return;
3214 /*
3215 * No need to decrement here, as eval_replace()
3216 * returns the pointer to the character passed
3217 * the eval, and two evals can not be placed
3218 * back to back without something in between.
3219 * We can skip that something in between.
3220 */
3221 continue;
3222 }
3223 skip_more:
3224 do {
3225 ptr++;
3226 } while (isalnum(*ptr) || *ptr == '_');
3227 if (!*ptr)
3228 break;
3229 /*
3230 * If what comes after this variable is a '.' or
3231 * '->' then we can continue to ignore that string.
3232 */
3233 if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
3234 ptr += *ptr == '.' ? 1 : 2;
3235 if (!*ptr)
3236 break;
3237 goto skip_more;
3238 }
3239 /*
3240 * Once again, we can skip the delimiter that came
3241 * after the string.
3242 */
3243 continue;
3244 }
3245 }
3246}
3247
3248static void add_str_to_module(struct module *module, char *str)
3249{
3250 struct module_string *modstr;
3251
3252 modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
3253
3254 /*
3255 * If we failed to allocate memory here, then we'll just
3256 * let the str memory leak when the module is removed.
3257 * If this fails to allocate, there's worse problems than
3258 * a leaked string on module removal.
3259 */
3260 if (WARN_ON_ONCE(!modstr))
3261 return;
3262
3263 modstr->module = module;
3264 modstr->str = str;
3265
3266 list_add(new: &modstr->next, head: &module_strings);
3267}
3268
3269#define ATTRIBUTE_STR "__attribute__("
3270#define ATTRIBUTE_STR_LEN (sizeof(ATTRIBUTE_STR) - 1)
3271
3272/* Remove all __attribute__() from @type. Return allocated string or @type. */
3273static char *sanitize_field_type(const char *type)
3274{
3275 char *attr, *tmp, *next, *ret = (char *)type;
3276 int depth;
3277
3278 next = (char *)type;
3279 while ((attr = strstr(next, ATTRIBUTE_STR))) {
3280 /* Retry if "__attribute__(" is a part of another word. */
3281 if (attr != next && !isspace(attr[-1])) {
3282 next = attr + ATTRIBUTE_STR_LEN;
3283 continue;
3284 }
3285
3286 if (ret == type) {
3287 ret = kstrdup(s: type, GFP_KERNEL);
3288 if (WARN_ON_ONCE(!ret))
3289 return NULL;
3290 attr = ret + (attr - type);
3291 }
3292
3293 /* the ATTRIBUTE_STR already has the first '(' */
3294 depth = 1;
3295 next = attr + ATTRIBUTE_STR_LEN;
3296 do {
3297 tmp = strpbrk(next, "()");
3298 /* There is unbalanced parentheses */
3299 if (WARN_ON_ONCE(!tmp)) {
3300 kfree(objp: ret);
3301 return (char *)type;
3302 }
3303
3304 if (*tmp == '(')
3305 depth++;
3306 else
3307 depth--;
3308 next = tmp + 1;
3309 } while (depth > 0);
3310 next = skip_spaces(next);
3311 strcpy(attr, next);
3312 next = attr;
3313 }
3314 return ret;
3315}
3316
3317static char *find_replacable_eval(const char *type, const char *eval_string,
3318 int len)
3319{
3320 char *ptr;
3321
3322 if (!eval_string)
3323 return NULL;
3324
3325 ptr = strchr(type, '[');
3326 if (!ptr)
3327 return NULL;
3328 ptr++;
3329
3330 if (!isalpha(*ptr) && *ptr != '_')
3331 return NULL;
3332
3333 if (strncmp(eval_string, ptr, len) != 0)
3334 return NULL;
3335
3336 return ptr;
3337}
3338
3339static void update_event_fields(struct trace_event_call *call,
3340 struct trace_eval_map *map)
3341{
3342 struct ftrace_event_field *field;
3343 const char *eval_string = NULL;
3344 struct list_head *head;
3345 int len = 0;
3346 char *ptr;
3347 char *str;
3348
3349 /* Dynamic events should never have field maps */
3350 if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3351 return;
3352
3353 if (map) {
3354 eval_string = map->eval_string;
3355 len = strlen(map->eval_string);
3356 }
3357
3358 head = trace_get_fields(event_call: call);
3359 list_for_each_entry(field, head, link) {
3360 str = sanitize_field_type(type: field->type);
3361 if (!str)
3362 return;
3363
3364 ptr = find_replacable_eval(type: str, eval_string, len);
3365 if (ptr) {
3366 if (str == field->type) {
3367 str = kstrdup(s: field->type, GFP_KERNEL);
3368 if (WARN_ON_ONCE(!str))
3369 return;
3370 ptr = str + (ptr - field->type);
3371 }
3372
3373 ptr = eval_replace(ptr, map, len);
3374 /* enum/sizeof string smaller than value */
3375 if (WARN_ON_ONCE(!ptr)) {
3376 kfree(objp: str);
3377 continue;
3378 }
3379 }
3380
3381 if (str == field->type)
3382 continue;
3383 /*
3384 * If the event is part of a module, then we need to free the string
3385 * when the module is removed. Otherwise, it will stay allocated
3386 * until a reboot.
3387 */
3388 if (call->module)
3389 add_str_to_module(module: call->module, str);
3390
3391 field->type = str;
3392 if (field->filter_type == FILTER_OTHER)
3393 field->filter_type = filter_assign_type(type: field->type);
3394 }
3395}
3396
3397/* Update all events for replacing eval and sanitizing */
3398void trace_event_update_all(struct trace_eval_map **map, int len)
3399{
3400 struct trace_event_call *call, *p;
3401 const char *last_system = NULL;
3402 bool first = false;
3403 bool updated;
3404 int last_i;
3405 int i;
3406
3407 down_write(sem: &trace_event_sem);
3408 list_for_each_entry_safe(call, p, &ftrace_events, list) {
3409 /* events are usually grouped together with systems */
3410 if (!last_system || call->class->system != last_system) {
3411 first = true;
3412 last_i = 0;
3413 last_system = call->class->system;
3414 }
3415
3416 updated = false;
3417 /*
3418 * Since calls are grouped by systems, the likelihood that the
3419 * next call in the iteration belongs to the same system as the
3420 * previous call is high. As an optimization, we skip searching
3421 * for a map[] that matches the call's system if the last call
3422 * was from the same system. That's what last_i is for. If the
3423 * call has the same system as the previous call, then last_i
3424 * will be the index of the first map[] that has a matching
3425 * system.
3426 */
3427 for (i = last_i; i < len; i++) {
3428 if (call->class->system == map[i]->system) {
3429 /* Save the first system if need be */
3430 if (first) {
3431 last_i = i;
3432 first = false;
3433 }
3434 update_event_printk(call, map: map[i]);
3435 update_event_fields(call, map: map[i]);
3436 updated = true;
3437 }
3438 }
3439 /* If not updated yet, update field for sanitizing. */
3440 if (!updated)
3441 update_event_fields(call, NULL);
3442 cond_resched();
3443 }
3444 up_write(sem: &trace_event_sem);
3445}
3446
3447static bool event_in_systems(struct trace_event_call *call,
3448 const char *systems)
3449{
3450 const char *system;
3451 const char *p;
3452
3453 if (!systems)
3454 return true;
3455
3456 system = call->class->system;
3457 p = strstr(systems, system);
3458 if (!p)
3459 return false;
3460
3461 if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
3462 return false;
3463
3464 p += strlen(system);
3465 return !*p || isspace(*p) || *p == ',';
3466}
3467
3468#ifdef CONFIG_HIST_TRIGGERS
3469/*
3470 * Wake up waiter on the hist_poll_wq from irq_work because the hist trigger
3471 * may happen in any context.
3472 */
3473static void hist_poll_event_irq_work(struct irq_work *work)
3474{
3475 wake_up_all(&hist_poll_wq);
3476}
3477
3478DEFINE_IRQ_WORK(hist_poll_work, hist_poll_event_irq_work);
3479DECLARE_WAIT_QUEUE_HEAD(hist_poll_wq);
3480#endif
3481
3482static struct trace_event_file *
3483trace_create_new_event(struct trace_event_call *call,
3484 struct trace_array *tr)
3485{
3486 struct trace_pid_list *no_pid_list;
3487 struct trace_pid_list *pid_list;
3488 struct trace_event_file *file;
3489 unsigned int first;
3490
3491 if (!event_in_systems(call, systems: tr->system_names))
3492 return NULL;
3493
3494 file = kmem_cache_alloc(file_cachep, GFP_TRACE);
3495 if (!file)
3496 return ERR_PTR(error: -ENOMEM);
3497
3498 pid_list = rcu_dereference_protected(tr->filtered_pids,
3499 lockdep_is_held(&event_mutex));
3500 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
3501 lockdep_is_held(&event_mutex));
3502
3503 if (!trace_pid_list_first(pid_list, pid: &first) ||
3504 !trace_pid_list_first(pid_list: no_pid_list, pid: &first))
3505 file->flags |= EVENT_FILE_FL_PID_FILTER;
3506
3507 file->event_call = call;
3508 file->tr = tr;
3509 atomic_set(v: &file->sm_ref, i: 0);
3510 atomic_set(v: &file->tm_ref, i: 0);
3511 INIT_LIST_HEAD(list: &file->triggers);
3512 list_add(new: &file->list, head: &tr->events);
3513 refcount_set(r: &file->ref, n: 1);
3514
3515 return file;
3516}
3517
3518#define MAX_BOOT_TRIGGERS 32
3519
3520static struct boot_triggers {
3521 const char *event;
3522 char *trigger;
3523} bootup_triggers[MAX_BOOT_TRIGGERS];
3524
3525static char bootup_trigger_buf[COMMAND_LINE_SIZE];
3526static int nr_boot_triggers;
3527
3528static __init int setup_trace_triggers(char *str)
3529{
3530 char *trigger;
3531 char *buf;
3532 int i;
3533
3534 strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
3535 trace_set_ring_buffer_expanded(NULL);
3536 disable_tracing_selftest(reason: "running event triggers");
3537
3538 buf = bootup_trigger_buf;
3539 for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
3540 trigger = strsep(&buf, ",");
3541 if (!trigger)
3542 break;
3543 bootup_triggers[i].event = strsep(&trigger, ".");
3544 bootup_triggers[i].trigger = trigger;
3545 if (!bootup_triggers[i].trigger)
3546 break;
3547 }
3548
3549 nr_boot_triggers = i;
3550 return 1;
3551}
3552__setup("trace_trigger=", setup_trace_triggers);
3553
3554/* Add an event to a trace directory */
3555static int
3556__trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
3557{
3558 struct trace_event_file *file;
3559
3560 file = trace_create_new_event(call, tr);
3561 /*
3562 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3563 * allocation, or NULL if the event is not part of the tr->system_names.
3564 * When the event is not part of the tr->system_names, return zero, not
3565 * an error.
3566 */
3567 if (!file)
3568 return 0;
3569
3570 if (IS_ERR(ptr: file))
3571 return PTR_ERR(ptr: file);
3572
3573 if (eventdir_initialized)
3574 return event_create_dir(parent: tr->event_dir, file);
3575 else
3576 return event_define_fields(call);
3577}
3578
3579static void trace_early_triggers(struct trace_event_file *file, const char *name)
3580{
3581 int ret;
3582 int i;
3583
3584 for (i = 0; i < nr_boot_triggers; i++) {
3585 if (strcmp(name, bootup_triggers[i].event))
3586 continue;
3587 mutex_lock(lock: &event_mutex);
3588 ret = trigger_process_regex(file, buff: bootup_triggers[i].trigger);
3589 mutex_unlock(lock: &event_mutex);
3590 if (ret)
3591 pr_err("Failed to register trigger '%s' on event %s\n",
3592 bootup_triggers[i].trigger,
3593 bootup_triggers[i].event);
3594 }
3595}
3596
3597/*
3598 * Just create a descriptor for early init. A descriptor is required
3599 * for enabling events at boot. We want to enable events before
3600 * the filesystem is initialized.
3601 */
3602static int
3603__trace_early_add_new_event(struct trace_event_call *call,
3604 struct trace_array *tr)
3605{
3606 struct trace_event_file *file;
3607 int ret;
3608
3609 file = trace_create_new_event(call, tr);
3610 /*
3611 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3612 * allocation, or NULL if the event is not part of the tr->system_names.
3613 * When the event is not part of the tr->system_names, return zero, not
3614 * an error.
3615 */
3616 if (!file)
3617 return 0;
3618
3619 if (IS_ERR(ptr: file))
3620 return PTR_ERR(ptr: file);
3621
3622 ret = event_define_fields(call);
3623 if (ret)
3624 return ret;
3625
3626 trace_early_triggers(file, name: trace_event_name(call));
3627
3628 return 0;
3629}
3630
3631struct ftrace_module_file_ops;
3632static void __add_event_to_tracers(struct trace_event_call *call);
3633
3634/* Add an additional event_call dynamically */
3635int trace_add_event_call(struct trace_event_call *call)
3636{
3637 int ret;
3638 lockdep_assert_held(&event_mutex);
3639
3640 guard(mutex)(T: &trace_types_lock);
3641
3642 ret = __register_event(call, NULL);
3643 if (ret < 0)
3644 return ret;
3645
3646 __add_event_to_tracers(call);
3647 return ret;
3648}
3649EXPORT_SYMBOL_GPL(trace_add_event_call);
3650
3651/*
3652 * Must be called under locking of trace_types_lock, event_mutex and
3653 * trace_event_sem.
3654 */
3655static void __trace_remove_event_call(struct trace_event_call *call)
3656{
3657 event_remove(call);
3658 trace_destroy_fields(call);
3659}
3660
3661static int probe_remove_event_call(struct trace_event_call *call)
3662{
3663 struct trace_array *tr;
3664 struct trace_event_file *file;
3665
3666#ifdef CONFIG_PERF_EVENTS
3667 if (call->perf_refcount)
3668 return -EBUSY;
3669#endif
3670 do_for_each_event_file(tr, file) {
3671 if (file->event_call != call)
3672 continue;
3673 /*
3674 * We can't rely on ftrace_event_enable_disable(enable => 0)
3675 * we are going to do, soft mode can suppress
3676 * TRACE_REG_UNREGISTER.
3677 */
3678 if (file->flags & EVENT_FILE_FL_ENABLED)
3679 goto busy;
3680
3681 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3682 tr->clear_trace = true;
3683 /*
3684 * The do_for_each_event_file_safe() is
3685 * a double loop. After finding the call for this
3686 * trace_array, we use break to jump to the next
3687 * trace_array.
3688 */
3689 break;
3690 } while_for_each_event_file();
3691
3692 __trace_remove_event_call(call);
3693
3694 return 0;
3695 busy:
3696 /* No need to clear the trace now */
3697 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
3698 tr->clear_trace = false;
3699 }
3700 return -EBUSY;
3701}
3702
3703/* Remove an event_call */
3704int trace_remove_event_call(struct trace_event_call *call)
3705{
3706 int ret;
3707
3708 lockdep_assert_held(&event_mutex);
3709
3710 mutex_lock(lock: &trace_types_lock);
3711 down_write(sem: &trace_event_sem);
3712 ret = probe_remove_event_call(call);
3713 up_write(sem: &trace_event_sem);
3714 mutex_unlock(lock: &trace_types_lock);
3715
3716 return ret;
3717}
3718EXPORT_SYMBOL_GPL(trace_remove_event_call);
3719
3720#define for_each_event(event, start, end) \
3721 for (event = start; \
3722 (unsigned long)event < (unsigned long)end; \
3723 event++)
3724
3725#ifdef CONFIG_MODULES
3726static void update_mod_cache(struct trace_array *tr, struct module *mod)
3727{
3728 struct event_mod_load *event_mod, *n;
3729
3730 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
3731 if (strcmp(event_mod->module, mod->name) != 0)
3732 continue;
3733
3734 __ftrace_set_clr_event_nolock(tr, match: event_mod->match,
3735 sub: event_mod->system,
3736 event: event_mod->event, set: 1, mod: mod->name);
3737 free_event_mod(event_mod);
3738 }
3739}
3740
3741static void update_cache_events(struct module *mod)
3742{
3743 struct trace_array *tr;
3744
3745 list_for_each_entry(tr, &ftrace_trace_arrays, list)
3746 update_mod_cache(tr, mod);
3747}
3748
3749static void trace_module_add_events(struct module *mod)
3750{
3751 struct trace_event_call **call, **start, **end;
3752
3753 if (!mod->num_trace_events)
3754 return;
3755
3756 /* Don't add infrastructure for mods without tracepoints */
3757 if (trace_module_has_bad_taint(mod)) {
3758 pr_err("%s: module has bad taint, not creating trace events\n",
3759 mod->name);
3760 return;
3761 }
3762
3763 start = mod->trace_events;
3764 end = mod->trace_events + mod->num_trace_events;
3765
3766 for_each_event(call, start, end) {
3767 __register_event(call: *call, mod);
3768 __add_event_to_tracers(call: *call);
3769 }
3770
3771 update_cache_events(mod);
3772}
3773
3774static void trace_module_remove_events(struct module *mod)
3775{
3776 struct trace_event_call *call, *p;
3777 struct module_string *modstr, *m;
3778
3779 down_write(sem: &trace_event_sem);
3780 list_for_each_entry_safe(call, p, &ftrace_events, list) {
3781 if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3782 continue;
3783 if (call->module == mod)
3784 __trace_remove_event_call(call);
3785 }
3786 /* Check for any strings allocated for this module */
3787 list_for_each_entry_safe(modstr, m, &module_strings, next) {
3788 if (modstr->module != mod)
3789 continue;
3790 list_del(entry: &modstr->next);
3791 kfree(objp: modstr->str);
3792 kfree(objp: modstr);
3793 }
3794 up_write(sem: &trace_event_sem);
3795
3796 /*
3797 * It is safest to reset the ring buffer if the module being unloaded
3798 * registered any events that were used. The only worry is if
3799 * a new module gets loaded, and takes on the same id as the events
3800 * of this module. When printing out the buffer, traced events left
3801 * over from this module may be passed to the new module events and
3802 * unexpected results may occur.
3803 */
3804 tracing_reset_all_online_cpus_unlocked();
3805}
3806
3807static int trace_module_notify(struct notifier_block *self,
3808 unsigned long val, void *data)
3809{
3810 struct module *mod = data;
3811
3812 mutex_lock(lock: &event_mutex);
3813 mutex_lock(lock: &trace_types_lock);
3814 switch (val) {
3815 case MODULE_STATE_COMING:
3816 trace_module_add_events(mod);
3817 break;
3818 case MODULE_STATE_GOING:
3819 trace_module_remove_events(mod);
3820 break;
3821 }
3822 mutex_unlock(lock: &trace_types_lock);
3823 mutex_unlock(lock: &event_mutex);
3824
3825 return NOTIFY_OK;
3826}
3827
3828static struct notifier_block trace_module_nb = {
3829 .notifier_call = trace_module_notify,
3830 .priority = 1, /* higher than trace.c module notify */
3831};
3832#endif /* CONFIG_MODULES */
3833
3834/* Create a new event directory structure for a trace directory. */
3835static void
3836__trace_add_event_dirs(struct trace_array *tr)
3837{
3838 struct trace_event_call *call;
3839 int ret;
3840
3841 lockdep_assert_held(&trace_event_sem);
3842
3843 list_for_each_entry(call, &ftrace_events, list) {
3844 ret = __trace_add_new_event(call, tr);
3845 if (ret < 0)
3846 pr_warn("Could not create directory for event %s\n",
3847 trace_event_name(call));
3848 }
3849}
3850
3851/* Returns any file that matches the system and event */
3852struct trace_event_file *
3853__find_event_file(struct trace_array *tr, const char *system, const char *event)
3854{
3855 struct trace_event_file *file;
3856 struct trace_event_call *call;
3857 const char *name;
3858
3859 list_for_each_entry(file, &tr->events, list) {
3860
3861 call = file->event_call;
3862 name = trace_event_name(call);
3863
3864 if (!name || !call->class)
3865 continue;
3866
3867 if (strcmp(event, name) == 0 &&
3868 strcmp(system, call->class->system) == 0)
3869 return file;
3870 }
3871 return NULL;
3872}
3873
3874/* Returns valid trace event files that match system and event */
3875struct trace_event_file *
3876find_event_file(struct trace_array *tr, const char *system, const char *event)
3877{
3878 struct trace_event_file *file;
3879
3880 file = __find_event_file(tr, system, event);
3881 if (!file || !file->event_call->class->reg ||
3882 file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3883 return NULL;
3884
3885 return file;
3886}
3887
3888/**
3889 * trace_get_event_file - Find and return a trace event file
3890 * @instance: The name of the trace instance containing the event
3891 * @system: The name of the system containing the event
3892 * @event: The name of the event
3893 *
3894 * Return a trace event file given the trace instance name, trace
3895 * system, and trace event name. If the instance name is NULL, it
3896 * refers to the top-level trace array.
3897 *
3898 * This function will look it up and return it if found, after calling
3899 * trace_array_get() to prevent the instance from going away, and
3900 * increment the event's module refcount to prevent it from being
3901 * removed.
3902 *
3903 * To release the file, call trace_put_event_file(), which will call
3904 * trace_array_put() and decrement the event's module refcount.
3905 *
3906 * Return: The trace event on success, ERR_PTR otherwise.
3907 */
3908struct trace_event_file *trace_get_event_file(const char *instance,
3909 const char *system,
3910 const char *event)
3911{
3912 struct trace_array *tr = top_trace_array();
3913 struct trace_event_file *file = NULL;
3914 int ret = -EINVAL;
3915
3916 if (instance) {
3917 tr = trace_array_find_get(instance);
3918 if (!tr)
3919 return ERR_PTR(error: -ENOENT);
3920 } else {
3921 ret = trace_array_get(tr);
3922 if (ret)
3923 return ERR_PTR(error: ret);
3924 }
3925
3926 guard(mutex)(T: &event_mutex);
3927
3928 file = find_event_file(tr, system, event);
3929 if (!file) {
3930 trace_array_put(tr);
3931 return ERR_PTR(error: -EINVAL);
3932 }
3933
3934 /* Don't let event modules unload while in use */
3935 ret = trace_event_try_get_ref(call: file->event_call);
3936 if (!ret) {
3937 trace_array_put(tr);
3938 return ERR_PTR(error: -EBUSY);
3939 }
3940
3941 return file;
3942}
3943EXPORT_SYMBOL_GPL(trace_get_event_file);
3944
3945/**
3946 * trace_put_event_file - Release a file from trace_get_event_file()
3947 * @file: The trace event file
3948 *
3949 * If a file was retrieved using trace_get_event_file(), this should
3950 * be called when it's no longer needed. It will cancel the previous
3951 * trace_array_get() called by that function, and decrement the
3952 * event's module refcount.
3953 */
3954void trace_put_event_file(struct trace_event_file *file)
3955{
3956 mutex_lock(lock: &event_mutex);
3957 trace_event_put_ref(call: file->event_call);
3958 mutex_unlock(lock: &event_mutex);
3959
3960 trace_array_put(tr: file->tr);
3961}
3962EXPORT_SYMBOL_GPL(trace_put_event_file);
3963
3964#ifdef CONFIG_DYNAMIC_FTRACE
3965
3966/* Avoid typos */
3967#define ENABLE_EVENT_STR "enable_event"
3968#define DISABLE_EVENT_STR "disable_event"
3969
3970struct event_probe_data {
3971 struct trace_event_file *file;
3972 unsigned long count;
3973 int ref;
3974 bool enable;
3975};
3976
3977static void update_event_probe(struct event_probe_data *data)
3978{
3979 if (data->enable)
3980 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3981 else
3982 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3983}
3984
3985static void
3986event_enable_probe(unsigned long ip, unsigned long parent_ip,
3987 struct trace_array *tr, struct ftrace_probe_ops *ops,
3988 void *data)
3989{
3990 struct ftrace_func_mapper *mapper = data;
3991 struct event_probe_data *edata;
3992 void **pdata;
3993
3994 pdata = ftrace_func_mapper_find_ip(mapper, ip);
3995 if (!pdata || !*pdata)
3996 return;
3997
3998 edata = *pdata;
3999 update_event_probe(edata);
4000}
4001
4002static void
4003event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
4004 struct trace_array *tr, struct ftrace_probe_ops *ops,
4005 void *data)
4006{
4007 struct ftrace_func_mapper *mapper = data;
4008 struct event_probe_data *edata;
4009 void **pdata;
4010
4011 pdata = ftrace_func_mapper_find_ip(mapper, ip);
4012 if (!pdata || !*pdata)
4013 return;
4014
4015 edata = *pdata;
4016
4017 if (!edata->count)
4018 return;
4019
4020 /* Skip if the event is in a state we want to switch to */
4021 if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
4022 return;
4023
4024 if (edata->count != -1)
4025 (edata->count)--;
4026
4027 update_event_probe(edata);
4028}
4029
4030static int
4031event_enable_print(struct seq_file *m, unsigned long ip,
4032 struct ftrace_probe_ops *ops, void *data)
4033{
4034 struct ftrace_func_mapper *mapper = data;
4035 struct event_probe_data *edata;
4036 void **pdata;
4037
4038 pdata = ftrace_func_mapper_find_ip(mapper, ip);
4039
4040 if (WARN_ON_ONCE(!pdata || !*pdata))
4041 return 0;
4042
4043 edata = *pdata;
4044
4045 seq_printf(m, "%ps:", (void *)ip);
4046
4047 seq_printf(m, "%s:%s:%s",
4048 edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
4049 edata->file->event_call->class->system,
4050 trace_event_name(edata->file->event_call));
4051
4052 if (edata->count == -1)
4053 seq_puts(m, ":unlimited\n");
4054 else
4055 seq_printf(m, ":count=%ld\n", edata->count);
4056
4057 return 0;
4058}
4059
4060static int
4061event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
4062 unsigned long ip, void *init_data, void **data)
4063{
4064 struct ftrace_func_mapper *mapper = *data;
4065 struct event_probe_data *edata = init_data;
4066 int ret;
4067
4068 if (!mapper) {
4069 mapper = allocate_ftrace_func_mapper();
4070 if (!mapper)
4071 return -ENODEV;
4072 *data = mapper;
4073 }
4074
4075 ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
4076 if (ret < 0)
4077 return ret;
4078
4079 edata->ref++;
4080
4081 return 0;
4082}
4083
4084static int free_probe_data(void *data)
4085{
4086 struct event_probe_data *edata = data;
4087
4088 edata->ref--;
4089 if (!edata->ref) {
4090 /* Remove soft mode */
4091 __ftrace_event_enable_disable(edata->file, 0, 1);
4092 trace_event_put_ref(edata->file->event_call);
4093 kfree(edata);
4094 }
4095 return 0;
4096}
4097
4098static void
4099event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
4100 unsigned long ip, void *data)
4101{
4102 struct ftrace_func_mapper *mapper = data;
4103 struct event_probe_data *edata;
4104
4105 if (!ip) {
4106 if (!mapper)
4107 return;
4108 free_ftrace_func_mapper(mapper, free_probe_data);
4109 return;
4110 }
4111
4112 edata = ftrace_func_mapper_remove_ip(mapper, ip);
4113
4114 if (WARN_ON_ONCE(!edata))
4115 return;
4116
4117 if (WARN_ON_ONCE(edata->ref <= 0))
4118 return;
4119
4120 free_probe_data(edata);
4121}
4122
4123static struct ftrace_probe_ops event_enable_probe_ops = {
4124 .func = event_enable_probe,
4125 .print = event_enable_print,
4126 .init = event_enable_init,
4127 .free = event_enable_free,
4128};
4129
4130static struct ftrace_probe_ops event_enable_count_probe_ops = {
4131 .func = event_enable_count_probe,
4132 .print = event_enable_print,
4133 .init = event_enable_init,
4134 .free = event_enable_free,
4135};
4136
4137static struct ftrace_probe_ops event_disable_probe_ops = {
4138 .func = event_enable_probe,
4139 .print = event_enable_print,
4140 .init = event_enable_init,
4141 .free = event_enable_free,
4142};
4143
4144static struct ftrace_probe_ops event_disable_count_probe_ops = {
4145 .func = event_enable_count_probe,
4146 .print = event_enable_print,
4147 .init = event_enable_init,
4148 .free = event_enable_free,
4149};
4150
4151static int
4152event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
4153 char *glob, char *cmd, char *param, int enabled)
4154{
4155 struct trace_event_file *file;
4156 struct ftrace_probe_ops *ops;
4157 struct event_probe_data *data;
4158 unsigned long count = -1;
4159 const char *system;
4160 const char *event;
4161 char *number;
4162 bool enable;
4163 int ret;
4164
4165 if (!tr)
4166 return -ENODEV;
4167
4168 /* hash funcs only work with set_ftrace_filter */
4169 if (!enabled || !param)
4170 return -EINVAL;
4171
4172 system = strsep(&param, ":");
4173 if (!param)
4174 return -EINVAL;
4175
4176 event = strsep(&param, ":");
4177
4178 guard(mutex)(&event_mutex);
4179
4180 file = find_event_file(tr, system, event);
4181 if (!file)
4182 return -EINVAL;
4183
4184 enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
4185
4186 if (enable)
4187 ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
4188 else
4189 ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
4190
4191 if (glob[0] == '!')
4192 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
4193
4194 if (param) {
4195 number = strsep(&param, ":");
4196
4197 if (!strlen(number))
4198 return -EINVAL;
4199
4200 /*
4201 * We use the callback data field (which is a pointer)
4202 * as our counter.
4203 */
4204 ret = kstrtoul(number, 0, &count);
4205 if (ret)
4206 return ret;
4207 }
4208
4209 /* Don't let event modules unload while probe registered */
4210 ret = trace_event_try_get_ref(file->event_call);
4211 if (!ret)
4212 return -EBUSY;
4213
4214 ret = __ftrace_event_enable_disable(file, 1, 1);
4215 if (ret < 0)
4216 goto out_put;
4217
4218 ret = -ENOMEM;
4219 data = kzalloc(sizeof(*data), GFP_KERNEL);
4220 if (!data)
4221 goto out_put;
4222
4223 data->enable = enable;
4224 data->count = count;
4225 data->file = file;
4226
4227 ret = register_ftrace_function_probe(glob, tr, ops, data);
4228 /*
4229 * The above returns on success the # of functions enabled,
4230 * but if it didn't find any functions it returns zero.
4231 * Consider no functions a failure too.
4232 */
4233
4234 /* Just return zero, not the number of enabled functions */
4235 if (ret > 0)
4236 return 0;
4237
4238 kfree(data);
4239
4240 if (!ret)
4241 ret = -ENOENT;
4242
4243 __ftrace_event_enable_disable(file, 0, 1);
4244 out_put:
4245 trace_event_put_ref(file->event_call);
4246 return ret;
4247}
4248
4249static struct ftrace_func_command event_enable_cmd = {
4250 .name = ENABLE_EVENT_STR,
4251 .func = event_enable_func,
4252};
4253
4254static struct ftrace_func_command event_disable_cmd = {
4255 .name = DISABLE_EVENT_STR,
4256 .func = event_enable_func,
4257};
4258
4259static __init int register_event_cmds(void)
4260{
4261 int ret;
4262
4263 ret = register_ftrace_command(&event_enable_cmd);
4264 if (WARN_ON(ret < 0))
4265 return ret;
4266 ret = register_ftrace_command(&event_disable_cmd);
4267 if (WARN_ON(ret < 0))
4268 unregister_ftrace_command(&event_enable_cmd);
4269 return ret;
4270}
4271#else
4272static inline int register_event_cmds(void) { return 0; }
4273#endif /* CONFIG_DYNAMIC_FTRACE */
4274
4275/*
4276 * The top level array and trace arrays created by boot-time tracing
4277 * have already had its trace_event_file descriptors created in order
4278 * to allow for early events to be recorded.
4279 * This function is called after the tracefs has been initialized,
4280 * and we now have to create the files associated to the events.
4281 */
4282static void __trace_early_add_event_dirs(struct trace_array *tr)
4283{
4284 struct trace_event_file *file;
4285 int ret;
4286
4287
4288 list_for_each_entry(file, &tr->events, list) {
4289 ret = event_create_dir(parent: tr->event_dir, file);
4290 if (ret < 0)
4291 pr_warn("Could not create directory for event %s\n",
4292 trace_event_name(file->event_call));
4293 }
4294}
4295
4296/*
4297 * For early boot up, the top trace array and the trace arrays created
4298 * by boot-time tracing require to have a list of events that can be
4299 * enabled. This must be done before the filesystem is set up in order
4300 * to allow events to be traced early.
4301 */
4302void __trace_early_add_events(struct trace_array *tr)
4303{
4304 struct trace_event_call *call;
4305 int ret;
4306
4307 list_for_each_entry(call, &ftrace_events, list) {
4308 /* Early boot up should not have any modules loaded */
4309 if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
4310 WARN_ON_ONCE(call->module))
4311 continue;
4312
4313 ret = __trace_early_add_new_event(call, tr);
4314 if (ret < 0)
4315 pr_warn("Could not create early event %s\n",
4316 trace_event_name(call));
4317 }
4318}
4319
4320/* Remove the event directory structure for a trace directory. */
4321static void
4322__trace_remove_event_dirs(struct trace_array *tr)
4323{
4324 struct trace_event_file *file, *next;
4325
4326 list_for_each_entry_safe(file, next, &tr->events, list)
4327 remove_event_file_dir(file);
4328}
4329
4330static void __add_event_to_tracers(struct trace_event_call *call)
4331{
4332 struct trace_array *tr;
4333
4334 list_for_each_entry(tr, &ftrace_trace_arrays, list)
4335 __trace_add_new_event(call, tr);
4336}
4337
4338extern struct trace_event_call *__start_ftrace_events[];
4339extern struct trace_event_call *__stop_ftrace_events[];
4340
4341static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
4342
4343static __init int setup_trace_event(char *str)
4344{
4345 strscpy(bootup_event_buf, str, COMMAND_LINE_SIZE);
4346 trace_set_ring_buffer_expanded(NULL);
4347 disable_tracing_selftest(reason: "running event tracing");
4348
4349 return 1;
4350}
4351__setup("trace_event=", setup_trace_event);
4352
4353static int events_callback(const char *name, umode_t *mode, void **data,
4354 const struct file_operations **fops)
4355{
4356 if (strcmp(name, "enable") == 0) {
4357 *mode = TRACE_MODE_WRITE;
4358 *fops = &ftrace_tr_enable_fops;
4359 return 1;
4360 }
4361
4362 if (strcmp(name, "header_page") == 0) {
4363 *mode = TRACE_MODE_READ;
4364 *fops = &ftrace_show_header_page_fops;
4365
4366 } else if (strcmp(name, "header_event") == 0) {
4367 *mode = TRACE_MODE_READ;
4368 *fops = &ftrace_show_header_event_fops;
4369 } else
4370 return 0;
4371
4372 return 1;
4373}
4374
4375/* Expects to have event_mutex held when called */
4376static int
4377create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
4378{
4379 struct eventfs_inode *e_events;
4380 struct dentry *entry;
4381 int nr_entries;
4382 static struct eventfs_entry events_entries[] = {
4383 {
4384 .name = "enable",
4385 .callback = events_callback,
4386 },
4387 {
4388 .name = "header_page",
4389 .callback = events_callback,
4390 },
4391 {
4392 .name = "header_event",
4393 .callback = events_callback,
4394 },
4395 };
4396
4397 entry = trace_create_file(name: "set_event", TRACE_MODE_WRITE, parent,
4398 data: tr, fops: &ftrace_set_event_fops);
4399 if (!entry)
4400 return -ENOMEM;
4401
4402 nr_entries = ARRAY_SIZE(events_entries);
4403
4404 e_events = eventfs_create_events_dir(name: "events", parent, entries: events_entries,
4405 size: nr_entries, data: tr);
4406 if (IS_ERR(ptr: e_events)) {
4407 pr_warn("Could not create tracefs 'events' directory\n");
4408 return -ENOMEM;
4409 }
4410
4411 /* There are not as crucial, just warn if they are not created */
4412
4413 trace_create_file(name: "set_event_pid", TRACE_MODE_WRITE, parent,
4414 data: tr, fops: &ftrace_set_event_pid_fops);
4415
4416 trace_create_file(name: "set_event_notrace_pid",
4417 TRACE_MODE_WRITE, parent, data: tr,
4418 fops: &ftrace_set_event_notrace_pid_fops);
4419
4420 tr->event_dir = e_events;
4421
4422 return 0;
4423}
4424
4425/**
4426 * event_trace_add_tracer - add a instance of a trace_array to events
4427 * @parent: The parent dentry to place the files/directories for events in
4428 * @tr: The trace array associated with these events
4429 *
4430 * When a new instance is created, it needs to set up its events
4431 * directory, as well as other files associated with events. It also
4432 * creates the event hierarchy in the @parent/events directory.
4433 *
4434 * Returns 0 on success.
4435 *
4436 * Must be called with event_mutex held.
4437 */
4438int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
4439{
4440 int ret;
4441
4442 lockdep_assert_held(&event_mutex);
4443
4444 ret = create_event_toplevel_files(parent, tr);
4445 if (ret)
4446 goto out;
4447
4448 down_write(sem: &trace_event_sem);
4449 /* If tr already has the event list, it is initialized in early boot. */
4450 if (unlikely(!list_empty(&tr->events)))
4451 __trace_early_add_event_dirs(tr);
4452 else
4453 __trace_add_event_dirs(tr);
4454 up_write(sem: &trace_event_sem);
4455
4456 out:
4457 return ret;
4458}
4459
4460/*
4461 * The top trace array already had its file descriptors created.
4462 * Now the files themselves need to be created.
4463 */
4464static __init int
4465early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
4466{
4467 int ret;
4468
4469 guard(mutex)(T: &event_mutex);
4470
4471 ret = create_event_toplevel_files(parent, tr);
4472 if (ret)
4473 return ret;
4474
4475 down_write(sem: &trace_event_sem);
4476 __trace_early_add_event_dirs(tr);
4477 up_write(sem: &trace_event_sem);
4478
4479 return 0;
4480}
4481
4482/* Must be called with event_mutex held */
4483int event_trace_del_tracer(struct trace_array *tr)
4484{
4485 lockdep_assert_held(&event_mutex);
4486
4487 /* Disable any event triggers and associated soft-disabled events */
4488 clear_event_triggers(tr);
4489
4490 /* Clear the pid list */
4491 __ftrace_clear_event_pids(tr, type: TRACE_PIDS | TRACE_NO_PIDS);
4492
4493 /* Disable any running events */
4494 __ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, set: 0, NULL);
4495
4496 /* Make sure no more events are being executed */
4497 tracepoint_synchronize_unregister();
4498
4499 down_write(sem: &trace_event_sem);
4500 __trace_remove_event_dirs(tr);
4501 eventfs_remove_events_dir(ei: tr->event_dir);
4502 up_write(sem: &trace_event_sem);
4503
4504 tr->event_dir = NULL;
4505
4506 return 0;
4507}
4508
4509static __init int event_trace_memsetup(void)
4510{
4511 field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
4512 file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
4513 return 0;
4514}
4515
4516__init void
4517early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
4518{
4519 char *token;
4520 int ret;
4521
4522 while (true) {
4523 token = strsep(&buf, ",");
4524
4525 if (!token)
4526 break;
4527
4528 if (*token) {
4529 /* Restarting syscalls requires that we stop them first */
4530 if (disable_first)
4531 ftrace_set_clr_event(tr, buf: token, set: 0);
4532
4533 ret = ftrace_set_clr_event(tr, buf: token, set: 1);
4534 if (ret)
4535 pr_warn("Failed to enable trace event: %s\n", token);
4536 }
4537
4538 /* Put back the comma to allow this to be called again */
4539 if (buf)
4540 *(buf - 1) = ',';
4541 }
4542}
4543
4544static __init int event_trace_enable(void)
4545{
4546 struct trace_array *tr = top_trace_array();
4547 struct trace_event_call **iter, *call;
4548 int ret;
4549
4550 if (!tr)
4551 return -ENODEV;
4552
4553 for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
4554
4555 call = *iter;
4556 ret = event_init(call);
4557 if (!ret)
4558 list_add(new: &call->list, head: &ftrace_events);
4559 }
4560
4561 register_trigger_cmds();
4562
4563 /*
4564 * We need the top trace array to have a working set of trace
4565 * points at early init, before the debug files and directories
4566 * are created. Create the file entries now, and attach them
4567 * to the actual file dentries later.
4568 */
4569 __trace_early_add_events(tr);
4570
4571 early_enable_events(tr, buf: bootup_event_buf, disable_first: false);
4572
4573 trace_printk_start_comm();
4574
4575 register_event_cmds();
4576
4577
4578 return 0;
4579}
4580
4581/*
4582 * event_trace_enable() is called from trace_event_init() first to
4583 * initialize events and perhaps start any events that are on the
4584 * command line. Unfortunately, there are some events that will not
4585 * start this early, like the system call tracepoints that need
4586 * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
4587 * event_trace_enable() is called before pid 1 starts, and this flag
4588 * is never set, making the syscall tracepoint never get reached, but
4589 * the event is enabled regardless (and not doing anything).
4590 */
4591static __init int event_trace_enable_again(void)
4592{
4593 struct trace_array *tr;
4594
4595 tr = top_trace_array();
4596 if (!tr)
4597 return -ENODEV;
4598
4599 early_enable_events(tr, buf: bootup_event_buf, disable_first: true);
4600
4601 return 0;
4602}
4603
4604early_initcall(event_trace_enable_again);
4605
4606/* Init fields which doesn't related to the tracefs */
4607static __init int event_trace_init_fields(void)
4608{
4609 if (trace_define_generic_fields())
4610 pr_warn("tracing: Failed to allocated generic fields");
4611
4612 if (trace_define_common_fields())
4613 pr_warn("tracing: Failed to allocate common fields");
4614
4615 return 0;
4616}
4617
4618__init int event_trace_init(void)
4619{
4620 struct trace_array *tr;
4621 int ret;
4622
4623 tr = top_trace_array();
4624 if (!tr)
4625 return -ENODEV;
4626
4627 trace_create_file(name: "available_events", TRACE_MODE_READ,
4628 NULL, data: tr, fops: &ftrace_avail_fops);
4629
4630 ret = early_event_add_tracer(NULL, tr);
4631 if (ret)
4632 return ret;
4633
4634#ifdef CONFIG_MODULES
4635 ret = register_module_notifier(nb: &trace_module_nb);
4636 if (ret)
4637 pr_warn("Failed to register trace events module notifier\n");
4638#endif
4639
4640 eventdir_initialized = true;
4641
4642 return 0;
4643}
4644
4645void __init trace_event_init(void)
4646{
4647 event_trace_memsetup();
4648 init_ftrace_syscalls();
4649 event_trace_enable();
4650 event_trace_init_fields();
4651}
4652
4653#ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4654
4655static DEFINE_SPINLOCK(test_spinlock);
4656static DEFINE_SPINLOCK(test_spinlock_irq);
4657static DEFINE_MUTEX(test_mutex);
4658
4659static __init void test_work(struct work_struct *dummy)
4660{
4661 spin_lock(&test_spinlock);
4662 spin_lock_irq(&test_spinlock_irq);
4663 udelay(1);
4664 spin_unlock_irq(&test_spinlock_irq);
4665 spin_unlock(&test_spinlock);
4666
4667 mutex_lock(&test_mutex);
4668 msleep(1);
4669 mutex_unlock(&test_mutex);
4670}
4671
4672static __init int event_test_thread(void *unused)
4673{
4674 void *test_malloc;
4675
4676 test_malloc = kmalloc(1234, GFP_KERNEL);
4677 if (!test_malloc)
4678 pr_info("failed to kmalloc\n");
4679
4680 schedule_on_each_cpu(test_work);
4681
4682 kfree(test_malloc);
4683
4684 set_current_state(TASK_INTERRUPTIBLE);
4685 while (!kthread_should_stop()) {
4686 schedule();
4687 set_current_state(TASK_INTERRUPTIBLE);
4688 }
4689 __set_current_state(TASK_RUNNING);
4690
4691 return 0;
4692}
4693
4694/*
4695 * Do various things that may trigger events.
4696 */
4697static __init void event_test_stuff(void)
4698{
4699 struct task_struct *test_thread;
4700
4701 test_thread = kthread_run(event_test_thread, NULL, "test-events");
4702 msleep(1);
4703 kthread_stop(test_thread);
4704}
4705
4706/*
4707 * For every trace event defined, we will test each trace point separately,
4708 * and then by groups, and finally all trace points.
4709 */
4710static __init void event_trace_self_tests(void)
4711{
4712 struct trace_subsystem_dir *dir;
4713 struct trace_event_file *file;
4714 struct trace_event_call *call;
4715 struct event_subsystem *system;
4716 struct trace_array *tr;
4717 int ret;
4718
4719 tr = top_trace_array();
4720 if (!tr)
4721 return;
4722
4723 pr_info("Running tests on trace events:\n");
4724
4725 list_for_each_entry(file, &tr->events, list) {
4726
4727 call = file->event_call;
4728
4729 /* Only test those that have a probe */
4730 if (!call->class || !call->class->probe)
4731 continue;
4732
4733/*
4734 * Testing syscall events here is pretty useless, but
4735 * we still do it if configured. But this is time consuming.
4736 * What we really need is a user thread to perform the
4737 * syscalls as we test.
4738 */
4739#ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
4740 if (call->class->system &&
4741 strcmp(call->class->system, "syscalls") == 0)
4742 continue;
4743#endif
4744
4745 pr_info("Testing event %s: ", trace_event_name(call));
4746
4747 /*
4748 * If an event is already enabled, someone is using
4749 * it and the self test should not be on.
4750 */
4751 if (file->flags & EVENT_FILE_FL_ENABLED) {
4752 pr_warn("Enabled event during self test!\n");
4753 WARN_ON_ONCE(1);
4754 continue;
4755 }
4756
4757 ftrace_event_enable_disable(file, 1);
4758 event_test_stuff();
4759 ftrace_event_enable_disable(file, 0);
4760
4761 pr_cont("OK\n");
4762 }
4763
4764 /* Now test at the sub system level */
4765
4766 pr_info("Running tests on trace event systems:\n");
4767
4768 list_for_each_entry(dir, &tr->systems, list) {
4769
4770 system = dir->subsystem;
4771
4772 /* the ftrace system is special, skip it */
4773 if (strcmp(system->name, "ftrace") == 0)
4774 continue;
4775
4776 pr_info("Testing event system %s: ", system->name);
4777
4778 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1, NULL);
4779 if (WARN_ON_ONCE(ret)) {
4780 pr_warn("error enabling system %s\n",
4781 system->name);
4782 continue;
4783 }
4784
4785 event_test_stuff();
4786
4787 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0, NULL);
4788 if (WARN_ON_ONCE(ret)) {
4789 pr_warn("error disabling system %s\n",
4790 system->name);
4791 continue;
4792 }
4793
4794 pr_cont("OK\n");
4795 }
4796
4797 /* Test with all events enabled */
4798
4799 pr_info("Running tests on all trace events:\n");
4800 pr_info("Testing all events: ");
4801
4802 ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1, NULL);
4803 if (WARN_ON_ONCE(ret)) {
4804 pr_warn("error enabling all events\n");
4805 return;
4806 }
4807
4808 event_test_stuff();
4809
4810 /* reset sysname */
4811 ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0, NULL);
4812 if (WARN_ON_ONCE(ret)) {
4813 pr_warn("error disabling all events\n");
4814 return;
4815 }
4816
4817 pr_cont("OK\n");
4818}
4819
4820#ifdef CONFIG_FUNCTION_TRACER
4821
4822static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4823
4824static struct trace_event_file event_trace_file __initdata;
4825
4826static void __init
4827function_test_events_call(unsigned long ip, unsigned long parent_ip,
4828 struct ftrace_ops *op, struct ftrace_regs *regs)
4829{
4830 struct trace_buffer *buffer;
4831 struct ring_buffer_event *event;
4832 struct ftrace_entry *entry;
4833 unsigned int trace_ctx;
4834 long disabled;
4835 int cpu;
4836
4837 trace_ctx = tracing_gen_ctx();
4838 preempt_disable_notrace();
4839 cpu = raw_smp_processor_id();
4840 disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4841
4842 if (disabled != 1)
4843 goto out;
4844
4845 event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4846 TRACE_FN, sizeof(*entry),
4847 trace_ctx);
4848 if (!event)
4849 goto out;
4850 entry = ring_buffer_event_data(event);
4851 entry->ip = ip;
4852 entry->parent_ip = parent_ip;
4853
4854 event_trigger_unlock_commit(&event_trace_file, buffer, event,
4855 entry, trace_ctx);
4856 out:
4857 atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4858 preempt_enable_notrace();
4859}
4860
4861static struct ftrace_ops trace_ops __initdata =
4862{
4863 .func = function_test_events_call,
4864};
4865
4866static __init void event_trace_self_test_with_function(void)
4867{
4868 int ret;
4869
4870 event_trace_file.tr = top_trace_array();
4871 if (WARN_ON(!event_trace_file.tr))
4872 return;
4873
4874 ret = register_ftrace_function(&trace_ops);
4875 if (WARN_ON(ret < 0)) {
4876 pr_info("Failed to enable function tracer for event tests\n");
4877 return;
4878 }
4879 pr_info("Running tests again, along with the function tracer\n");
4880 event_trace_self_tests();
4881 unregister_ftrace_function(&trace_ops);
4882}
4883#else
4884static __init void event_trace_self_test_with_function(void)
4885{
4886}
4887#endif
4888
4889static __init int event_trace_self_tests_init(void)
4890{
4891 if (!tracing_selftest_disabled) {
4892 event_trace_self_tests();
4893 event_trace_self_test_with_function();
4894 }
4895
4896 return 0;
4897}
4898
4899late_initcall(event_trace_self_tests_init);
4900
4901#endif
4902