1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* ring buffer based function tracer
4	*
5	* Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
6	* Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7	*
8	* Originally taken from the RT patch by:
9	* Arnaldo Carvalho de Melo <acme@redhat.com>
10	*
11	* Based on code from the latency_tracer, that is:
12	* Copyright (C) 2004-2006 Ingo Molnar
13	* Copyright (C) 2004 Nadia Yvette Chambers
14	*/
15	#include <linux/ring_buffer.h>
16	#include <linux/utsname.h>
17	#include <linux/stacktrace.h>
18	#include <linux/writeback.h>
19	#include <linux/kallsyms.h>
20	#include <linux/security.h>
21	#include <linux/seq_file.h>
22	#include <linux/irqflags.h>
23	#include <linux/debugfs.h>
24	#include <linux/tracefs.h>
25	#include <linux/pagemap.h>
26	#include <linux/hardirq.h>
27	#include <linux/linkage.h>
28	#include <linux/uaccess.h>
29	#include <linux/vmalloc.h>
30	#include <linux/ftrace.h>
31	#include <linux/module.h>
32	#include <linux/percpu.h>
33	#include <linux/splice.h>
34	#include <linux/kdebug.h>
35	#include <linux/string.h>
36	#include <linux/mount.h>
37	#include <linux/rwsem.h>
38	#include <linux/slab.h>
39	#include <linux/ctype.h>
40	#include <linux/init.h>
41	#include <linux/panic_notifier.h>
42	#include <linux/poll.h>
43	#include <linux/nmi.h>
44	#include <linux/fs.h>
45	#include <linux/trace.h>
46	#include <linux/sched/clock.h>
47	#include <linux/sched/rt.h>
48	#include <linux/fsnotify.h>
49	#include <linux/irq_work.h>
50	#include <linux/workqueue.h>
51
52	#include <asm/setup.h> /* COMMAND_LINE_SIZE */
53
54	#include "trace.h"
55	#include "trace_output.h"
56
57	#ifdef CONFIG_FTRACE_STARTUP_TEST
58	/*
59	* We need to change this state when a selftest is running.
60	* A selftest will lurk into the ring-buffer to count the
61	* entries inserted during the selftest although some concurrent
62	* insertions into the ring-buffer such as trace_printk could occurred
63	* at the same time, giving false positive or negative results.
64	*/
65	static bool __read_mostly tracing_selftest_running;
66
67	/*
68	* If boot-time tracing including tracers/events via kernel cmdline
69	* is running, we do not want to run SELFTEST.
70	*/
71	bool __read_mostly tracing_selftest_disabled;
72
73	void __init disable_tracing_selftest(const char *reason)
74	{
75	if (!tracing_selftest_disabled) {
76	tracing_selftest_disabled = true;
77	pr_info("Ftrace startup test is disabled due to %s\n", reason);
78	}
79	}
80	#else
81	#define tracing_selftest_running 0
82	#define tracing_selftest_disabled 0
83	#endif
84
85	/* Pipe tracepoints to printk */
86	static struct trace_iterator *tracepoint_print_iter;
87	int tracepoint_printk;
88	static bool tracepoint_printk_stop_on_boot __initdata;
89	static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
90
91	/* For tracers that don't implement custom flags */
92	static struct tracer_opt dummy_tracer_opt[] = {
93	{ }
94	};
95
96	static int
97	dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
98	{
99	return 0;
100	}
101
102	/*
103	* To prevent the comm cache from being overwritten when no
104	* tracing is active, only save the comm when a trace event
105	* occurred.
106	*/
107	DEFINE_PER_CPU(bool, trace_taskinfo_save);
108
109	/*
110	* Kill all tracing for good (never come back).
111	* It is initialized to 1 but will turn to zero if the initialization
112	* of the tracer is successful. But that is the only place that sets
113	* this back to zero.
114	*/
115	static int tracing_disabled = 1;
116
117	cpumask_var_t __read_mostly tracing_buffer_mask;
118
119	/*
120	* ftrace_dump_on_oops - variable to dump ftrace buffer on oops
121	*
122	* If there is an oops (or kernel panic) and the ftrace_dump_on_oops
123	* is set, then ftrace_dump is called. This will output the contents
124	* of the ftrace buffers to the console. This is very useful for
125	* capturing traces that lead to crashes and outputing it to a
126	* serial console.
127	*
128	* It is default off, but you can enable it with either specifying
129	* "ftrace_dump_on_oops" in the kernel command line, or setting
130	* /proc/sys/kernel/ftrace_dump_on_oops
131	* Set 1 if you want to dump buffers of all CPUs
132	* Set 2 if you want to dump the buffer of the CPU that triggered oops
133	* Set instance name if you want to dump the specific trace instance
134	* Multiple instance dump is also supported, and instances are seperated
135	* by commas.
136	*/
137	/* Set to string format zero to disable by default */
138	char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0";
139
140	/* When set, tracing will stop when a WARN*() is hit */
141	int __disable_trace_on_warning;
142
143	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
144	/* Map of enums to their values, for "eval_map" file */
145	struct trace_eval_map_head {
146	struct module *mod;
147	unsigned long length;
148	};
149
150	union trace_eval_map_item;
151
152	struct trace_eval_map_tail {
153	/*
154	* "end" is first and points to NULL as it must be different
155	* than "mod" or "eval_string"
156	*/
157	union trace_eval_map_item *next;
158	const char *end; /* points to NULL */
159	};
160
161	static DEFINE_MUTEX(trace_eval_mutex);
162
163	/*
164	* The trace_eval_maps are saved in an array with two extra elements,
165	* one at the beginning, and one at the end. The beginning item contains
166	* the count of the saved maps (head.length), and the module they
167	* belong to if not built in (head.mod). The ending item contains a
168	* pointer to the next array of saved eval_map items.
169	*/
170	union trace_eval_map_item {
171	struct trace_eval_map map;
172	struct trace_eval_map_head head;
173	struct trace_eval_map_tail tail;
174	};
175
176	static union trace_eval_map_item *trace_eval_maps;
177	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
178
179	int tracing_set_tracer(struct trace_array *tr, const char *buf);
180	static void ftrace_trace_userstack(struct trace_array *tr,
181	struct trace_buffer *buffer,
182	unsigned int trace_ctx);
183
184	static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
185	static char *default_bootup_tracer;
186
187	static bool allocate_snapshot;
188	static bool snapshot_at_boot;
189
190	static char boot_instance_info[COMMAND_LINE_SIZE] __initdata;
191	static int boot_instance_index;
192
193	static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata;
194	static int boot_snapshot_index;
195
196	static int __init set_cmdline_ftrace(char *str)
197	{
198	strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
199	default_bootup_tracer = bootup_tracer_buf;
200	/* We are using ftrace early, expand it */
201	trace_set_ring_buffer_expanded(NULL);
202	return 1;
203	}
204	__setup("ftrace=", set_cmdline_ftrace);
205
206	int ftrace_dump_on_oops_enabled(void)
207	{
208	if (!strcmp("0", ftrace_dump_on_oops))
209	return 0;
210	else
211	return 1;
212	}
213
214	static int __init set_ftrace_dump_on_oops(char *str)
215	{
216	if (!*str) {
217	strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
218	return 1;
219	}
220
221	if (*str == ',') {
222	strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
223	strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);
224	return 1;
225	}
226
227	if (*str++ == '=') {
228	strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE);
229	return 1;
230	}
231
232	return 0;
233	}
234	__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
235
236	static int __init stop_trace_on_warning(char *str)
237	{
238	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
239	__disable_trace_on_warning = 1;
240	return 1;
241	}
242	__setup("traceoff_on_warning", stop_trace_on_warning);
243
244	static int __init boot_alloc_snapshot(char *str)
245	{
246	char *slot = boot_snapshot_info + boot_snapshot_index;
247	int left = sizeof(boot_snapshot_info) - boot_snapshot_index;
248	int ret;
249
250	if (str[0] == '=') {
251	str++;
252	if (strlen(str) >= left)
253	return -1;
254
255	ret = snprintf(buf: slot, size: left, fmt: "%s\t", str);
256	boot_snapshot_index += ret;
257	} else {
258	allocate_snapshot = true;
259	/* We also need the main ring buffer expanded */
260	trace_set_ring_buffer_expanded(NULL);
261	}
262	return 1;
263	}
264	__setup("alloc_snapshot", boot_alloc_snapshot);
265
266
267	static int __init boot_snapshot(char *str)
268	{
269	snapshot_at_boot = true;
270	boot_alloc_snapshot(str);
271	return 1;
272	}
273	__setup("ftrace_boot_snapshot", boot_snapshot);
274
275
276	static int __init boot_instance(char *str)
277	{
278	char *slot = boot_instance_info + boot_instance_index;
279	int left = sizeof(boot_instance_info) - boot_instance_index;
280	int ret;
281
282	if (strlen(str) >= left)
283	return -1;
284
285	ret = snprintf(buf: slot, size: left, fmt: "%s\t", str);
286	boot_instance_index += ret;
287
288	return 1;
289	}
290	__setup("trace_instance=", boot_instance);
291
292
293	static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
294
295	static int __init set_trace_boot_options(char *str)
296	{
297	strscpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
298	return 1;
299	}
300	__setup("trace_options=", set_trace_boot_options);
301
302	static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
303	static char *trace_boot_clock __initdata;
304
305	static int __init set_trace_boot_clock(char *str)
306	{
307	strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
308	trace_boot_clock = trace_boot_clock_buf;
309	return 1;
310	}
311	__setup("trace_clock=", set_trace_boot_clock);
312
313	static int __init set_tracepoint_printk(char *str)
314	{
315	/* Ignore the "tp_printk_stop_on_boot" param */
316	if (*str == '_')
317	return 0;
318
319	if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
320	tracepoint_printk = 1;
321	return 1;
322	}
323	__setup("tp_printk", set_tracepoint_printk);
324
325	static int __init set_tracepoint_printk_stop(char *str)
326	{
327	tracepoint_printk_stop_on_boot = true;
328	return 1;
329	}
330	__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop);
331
332	unsigned long long ns2usecs(u64 nsec)
333	{
334	nsec += 500;
335	do_div(nsec, 1000);
336	return nsec;
337	}
338
339	static void
340	trace_process_export(struct trace_export *export,
341	struct ring_buffer_event *event, int flag)
342	{
343	struct trace_entry *entry;
344	unsigned int size = 0;
345
346	if (export->flags & flag) {
347	entry = ring_buffer_event_data(event);
348	size = ring_buffer_event_length(event);
349	export->write(export, entry, size);
350	}
351	}
352
353	static DEFINE_MUTEX(ftrace_export_lock);
354
355	static struct trace_export __rcu *ftrace_exports_list __read_mostly;
356
357	static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled);
358	static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled);
359	static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled);
360
361	static inline void ftrace_exports_enable(struct trace_export *export)
362	{
363	if (export->flags & TRACE_EXPORT_FUNCTION)
364	static_branch_inc(&trace_function_exports_enabled);
365
366	if (export->flags & TRACE_EXPORT_EVENT)
367	static_branch_inc(&trace_event_exports_enabled);
368
369	if (export->flags & TRACE_EXPORT_MARKER)
370	static_branch_inc(&trace_marker_exports_enabled);
371	}
372
373	static inline void ftrace_exports_disable(struct trace_export *export)
374	{
375	if (export->flags & TRACE_EXPORT_FUNCTION)
376	static_branch_dec(&trace_function_exports_enabled);
377
378	if (export->flags & TRACE_EXPORT_EVENT)
379	static_branch_dec(&trace_event_exports_enabled);
380
381	if (export->flags & TRACE_EXPORT_MARKER)
382	static_branch_dec(&trace_marker_exports_enabled);
383	}
384
385	static void ftrace_exports(struct ring_buffer_event *event, int flag)
386	{
387	struct trace_export *export;
388
389	preempt_disable_notrace();
390
391	export = rcu_dereference_raw_check(ftrace_exports_list);
392	while (export) {
393	trace_process_export(export, event, flag);
394	export = rcu_dereference_raw_check(export->next);
395	}
396
397	preempt_enable_notrace();
398	}
399
400	static inline void
401	add_trace_export(struct trace_export **list, struct trace_export *export)
402	{
403	rcu_assign_pointer(export->next, *list);
404	/*
405	* We are entering export into the list but another
406	* CPU might be walking that list. We need to make sure
407	* the export->next pointer is valid before another CPU sees
408	* the export pointer included into the list.
409	*/
410	rcu_assign_pointer(*list, export);
411	}
412
413	static inline int
414	rm_trace_export(struct trace_export **list, struct trace_export *export)
415	{
416	struct trace_export **p;
417
418	for (p = list; *p != NULL; p = &(*p)->next)
419	if (*p == export)
420	break;
421
422	if (*p != export)
423	return -1;
424
425	rcu_assign_pointer(*p, (*p)->next);
426
427	return 0;
428	}
429
430	static inline void
431	add_ftrace_export(struct trace_export **list, struct trace_export *export)
432	{
433	ftrace_exports_enable(export);
434
435	add_trace_export(list, export);
436	}
437
438	static inline int
439	rm_ftrace_export(struct trace_export **list, struct trace_export *export)
440	{
441	int ret;
442
443	ret = rm_trace_export(list, export);
444	ftrace_exports_disable(export);
445
446	return ret;
447	}
448
449	int register_ftrace_export(struct trace_export *export)
450	{
451	if (WARN_ON_ONCE(!export->write))
452	return -1;
453
454	mutex_lock(&ftrace_export_lock);
455
456	add_ftrace_export(list: &ftrace_exports_list, export);
457
458	mutex_unlock(lock: &ftrace_export_lock);
459
460	return 0;
461	}
462	EXPORT_SYMBOL_GPL(register_ftrace_export);
463
464	int unregister_ftrace_export(struct trace_export *export)
465	{
466	int ret;
467
468	mutex_lock(&ftrace_export_lock);
469
470	ret = rm_ftrace_export(list: &ftrace_exports_list, export);
471
472	mutex_unlock(lock: &ftrace_export_lock);
473
474	return ret;
475	}
476	EXPORT_SYMBOL_GPL(unregister_ftrace_export);
477
478	/* trace_flags holds trace_options default values */
479	#define TRACE_DEFAULT_FLAGS \
480	(FUNCTION_DEFAULT_FLAGS | \
481	TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
482	TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
483	TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
484	TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | \
485	TRACE_ITER_HASH_PTR)
486
487	/* trace_options that are only supported by global_trace */
488	#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
489	TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
490
491	/* trace_flags that are default zero for instances */
492	#define ZEROED_TRACE_FLAGS \
493	(TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
494
495	/*
496	* The global_trace is the descriptor that holds the top-level tracing
497	* buffers for the live tracing.
498	*/
499	static struct trace_array global_trace = {
500	.trace_flags = TRACE_DEFAULT_FLAGS,
501	};
502
503	void trace_set_ring_buffer_expanded(struct trace_array *tr)
504	{
505	if (!tr)
506	tr = &global_trace;
507	tr->ring_buffer_expanded = true;
508	}
509
510	LIST_HEAD(ftrace_trace_arrays);
511
512	int trace_array_get(struct trace_array *this_tr)
513	{
514	struct trace_array *tr;
515	int ret = -ENODEV;
516
517	mutex_lock(&trace_types_lock);
518	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
519	if (tr == this_tr) {
520	tr->ref++;
521	ret = 0;
522	break;
523	}
524	}
525	mutex_unlock(lock: &trace_types_lock);
526
527	return ret;
528	}
529
530	static void __trace_array_put(struct trace_array *this_tr)
531	{
532	WARN_ON(!this_tr->ref);
533	this_tr->ref--;
534	}
535
536	/**
537	* trace_array_put - Decrement the reference counter for this trace array.
538	* @this_tr : pointer to the trace array
539	*
540	* NOTE: Use this when we no longer need the trace array returned by
541	* trace_array_get_by_name(). This ensures the trace array can be later
542	* destroyed.
543	*
544	*/
545	void trace_array_put(struct trace_array *this_tr)
546	{
547	if (!this_tr)
548	return;
549
550	mutex_lock(&trace_types_lock);
551	__trace_array_put(this_tr);
552	mutex_unlock(lock: &trace_types_lock);
553	}
554	EXPORT_SYMBOL_GPL(trace_array_put);
555
556	int tracing_check_open_get_tr(struct trace_array *tr)
557	{
558	int ret;
559
560	ret = security_locked_down(what: LOCKDOWN_TRACEFS);
561	if (ret)
562	return ret;
563
564	if (tracing_disabled)
565	return -ENODEV;
566
567	if (tr && trace_array_get(this_tr: tr) < 0)
568	return -ENODEV;
569
570	return 0;
571	}
572
573	int call_filter_check_discard(struct trace_event_call *call, void *rec,
574	struct trace_buffer *buffer,
575	struct ring_buffer_event *event)
576	{
577	if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
578	!filter_match_preds(filter: call->filter, rec)) {
579	__trace_event_discard_commit(buffer, event);
580	return 1;
581	}
582
583	return 0;
584	}
585
586	/**
587	* trace_find_filtered_pid - check if a pid exists in a filtered_pid list
588	* @filtered_pids: The list of pids to check
589	* @search_pid: The PID to find in @filtered_pids
590	*
591	* Returns true if @search_pid is found in @filtered_pids, and false otherwise.
592	*/
593	bool
594	trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
595	{
596	return trace_pid_list_is_set(pid_list: filtered_pids, pid: search_pid);
597	}
598
599	/**
600	* trace_ignore_this_task - should a task be ignored for tracing
601	* @filtered_pids: The list of pids to check
602	* @filtered_no_pids: The list of pids not to be traced
603	* @task: The task that should be ignored if not filtered
604	*
605	* Checks if @task should be traced or not from @filtered_pids.
606	* Returns true if @task should *NOT* be traced.
607	* Returns false if @task should be traced.
608	*/
609	bool
610	trace_ignore_this_task(struct trace_pid_list *filtered_pids,
611	struct trace_pid_list *filtered_no_pids,
612	struct task_struct *task)
613	{
614	/*
615	* If filtered_no_pids is not empty, and the task's pid is listed
616	* in filtered_no_pids, then return true.
617	* Otherwise, if filtered_pids is empty, that means we can
618	* trace all tasks. If it has content, then only trace pids
619	* within filtered_pids.
620	*/
621
622	return (filtered_pids &&
623	!trace_find_filtered_pid(filtered_pids, search_pid: task->pid)) ||
624	(filtered_no_pids &&
625	trace_find_filtered_pid(filtered_pids: filtered_no_pids, search_pid: task->pid));
626	}
627
628	/**
629	* trace_filter_add_remove_task - Add or remove a task from a pid_list
630	* @pid_list: The list to modify
631	* @self: The current task for fork or NULL for exit
632	* @task: The task to add or remove
633	*
634	* If adding a task, if @self is defined, the task is only added if @self
635	* is also included in @pid_list. This happens on fork and tasks should
636	* only be added when the parent is listed. If @self is NULL, then the
637	* @task pid will be removed from the list, which would happen on exit
638	* of a task.
639	*/
640	void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
641	struct task_struct *self,
642	struct task_struct *task)
643	{
644	if (!pid_list)
645	return;
646
647	/* For forks, we only add if the forking task is listed */
648	if (self) {
649	if (!trace_find_filtered_pid(filtered_pids: pid_list, search_pid: self->pid))
650	return;
651	}
652
653	/* "self" is set for forks, and NULL for exits */
654	if (self)
655	trace_pid_list_set(pid_list, pid: task->pid);
656	else
657	trace_pid_list_clear(pid_list, pid: task->pid);
658	}
659
660	/**
661	* trace_pid_next - Used for seq_file to get to the next pid of a pid_list
662	* @pid_list: The pid list to show
663	* @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
664	* @pos: The position of the file
665	*
666	* This is used by the seq_file "next" operation to iterate the pids
667	* listed in a trace_pid_list structure.
668	*
669	* Returns the pid+1 as we want to display pid of zero, but NULL would
670	* stop the iteration.
671	*/
672	void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
673	{
674	long pid = (unsigned long)v;
675	unsigned int next;
676
677	(*pos)++;
678
679	/* pid already is +1 of the actual previous bit */
680	if (trace_pid_list_next(pid_list, pid, next: &next) < 0)
681	return NULL;
682
683	pid = next;
684
685	/* Return pid + 1 to allow zero to be represented */
686	return (void *)(pid + 1);
687	}
688
689	/**
690	* trace_pid_start - Used for seq_file to start reading pid lists
691	* @pid_list: The pid list to show
692	* @pos: The position of the file
693	*
694	* This is used by seq_file "start" operation to start the iteration
695	* of listing pids.
696	*
697	* Returns the pid+1 as we want to display pid of zero, but NULL would
698	* stop the iteration.
699	*/
700	void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
701	{
702	unsigned long pid;
703	unsigned int first;
704	loff_t l = 0;
705
706	if (trace_pid_list_first(pid_list, pid: &first) < 0)
707	return NULL;
708
709	pid = first;
710
711	/* Return pid + 1 so that zero can be the exit value */
712	for (pid++; pid && l < *pos;
713	pid = (unsigned long)trace_pid_next(pid_list, v: (void *)pid, pos: &l))
714	;
715	return (void *)pid;
716	}
717
718	/**
719	* trace_pid_show - show the current pid in seq_file processing
720	* @m: The seq_file structure to write into
721	* @v: A void pointer of the pid (+1) value to display
722	*
723	* Can be directly used by seq_file operations to display the current
724	* pid value.
725	*/
726	int trace_pid_show(struct seq_file *m, void *v)
727	{
728	unsigned long pid = (unsigned long)v - 1;
729
730	seq_printf(m, fmt: "%lu\n", pid);
731	return 0;
732	}
733
734	/* 128 should be much more than enough */
735	#define PID_BUF_SIZE 127
736
737	int trace_pid_write(struct trace_pid_list *filtered_pids,
738	struct trace_pid_list **new_pid_list,
739	const char __user *ubuf, size_t cnt)
740	{
741	struct trace_pid_list *pid_list;
742	struct trace_parser parser;
743	unsigned long val;
744	int nr_pids = 0;
745	ssize_t read = 0;
746	ssize_t ret;
747	loff_t pos;
748	pid_t pid;
749
750	if (trace_parser_get_init(parser: &parser, PID_BUF_SIZE + 1))
751	return -ENOMEM;
752
753	/*
754	* Always recreate a new array. The write is an all or nothing
755	* operation. Always create a new array when adding new pids by
756	* the user. If the operation fails, then the current list is
757	* not modified.
758	*/
759	pid_list = trace_pid_list_alloc();
760	if (!pid_list) {
761	trace_parser_put(parser: &parser);
762	return -ENOMEM;
763	}
764
765	if (filtered_pids) {
766	/* copy the current bits to the new max */
767	ret = trace_pid_list_first(pid_list: filtered_pids, pid: &pid);
768	while (!ret) {
769	trace_pid_list_set(pid_list, pid);
770	ret = trace_pid_list_next(pid_list: filtered_pids, pid: pid + 1, next: &pid);
771	nr_pids++;
772	}
773	}
774
775	ret = 0;
776	while (cnt > 0) {
777
778	pos = 0;
779
780	ret = trace_get_user(parser: &parser, ubuf, cnt, ppos: &pos);
781	if (ret < 0)
782	break;
783
784	read += ret;
785	ubuf += ret;
786	cnt -= ret;
787
788	if (!trace_parser_loaded(parser: &parser))
789	break;
790
791	ret = -EINVAL;
792	if (kstrtoul(s: parser.buffer, base: 0, res: &val))
793	break;
794
795	pid = (pid_t)val;
796
797	if (trace_pid_list_set(pid_list, pid) < 0) {
798	ret = -1;
799	break;
800	}
801	nr_pids++;
802
803	trace_parser_clear(parser: &parser);
804	ret = 0;
805	}
806	trace_parser_put(parser: &parser);
807
808	if (ret < 0) {
809	trace_pid_list_free(pid_list);
810	return ret;
811	}
812
813	if (!nr_pids) {
814	/* Cleared the list of pids */
815	trace_pid_list_free(pid_list);
816	pid_list = NULL;
817	}
818
819	*new_pid_list = pid_list;
820
821	return read;
822	}
823
824	static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu)
825	{
826	u64 ts;
827
828	/* Early boot up does not have a buffer yet */
829	if (!buf->buffer)
830	return trace_clock_local();
831
832	ts = ring_buffer_time_stamp(buffer: buf->buffer);
833	ring_buffer_normalize_time_stamp(buffer: buf->buffer, cpu, ts: &ts);
834
835	return ts;
836	}
837
838	u64 ftrace_now(int cpu)
839	{
840	return buffer_ftrace_now(buf: &global_trace.array_buffer, cpu);
841	}
842
843	/**
844	* tracing_is_enabled - Show if global_trace has been enabled
845	*
846	* Shows if the global trace has been enabled or not. It uses the
847	* mirror flag "buffer_disabled" to be used in fast paths such as for
848	* the irqsoff tracer. But it may be inaccurate due to races. If you
849	* need to know the accurate state, use tracing_is_on() which is a little
850	* slower, but accurate.
851	*/
852	int tracing_is_enabled(void)
853	{
854	/*
855	* For quick access (irqsoff uses this in fast path), just
856	* return the mirror variable of the state of the ring buffer.
857	* It's a little racy, but we don't really care.
858	*/
859	smp_rmb();
860	return !global_trace.buffer_disabled;
861	}
862
863	/*
864	* trace_buf_size is the size in bytes that is allocated
865	* for a buffer. Note, the number of bytes is always rounded
866	* to page size.
867	*
868	* This number is purposely set to a low number of 16384.
869	* If the dump on oops happens, it will be much appreciated
870	* to not have to wait for all that output. Anyway this can be
871	* boot time and run time configurable.
872	*/
873	#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
874
875	static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
876
877	/* trace_types holds a link list of available tracers. */
878	static struct tracer *trace_types __read_mostly;
879
880	/*
881	* trace_types_lock is used to protect the trace_types list.
882	*/
883	DEFINE_MUTEX(trace_types_lock);
884
885	/*
886	* serialize the access of the ring buffer
887	*
888	* ring buffer serializes readers, but it is low level protection.
889	* The validity of the events (which returns by ring_buffer_peek() ..etc)
890	* are not protected by ring buffer.
891	*
892	* The content of events may become garbage if we allow other process consumes
893	* these events concurrently:
894	* A) the page of the consumed events may become a normal page
895	* (not reader page) in ring buffer, and this page will be rewritten
896	* by events producer.
897	* B) The page of the consumed events may become a page for splice_read,
898	* and this page will be returned to system.
899	*
900	* These primitives allow multi process access to different cpu ring buffer
901	* concurrently.
902	*
903	* These primitives don't distinguish read-only and read-consume access.
904	* Multi read-only access are also serialized.
905	*/
906
907	#ifdef CONFIG_SMP
908	static DECLARE_RWSEM(all_cpu_access_lock);
909	static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
910
911	static inline void trace_access_lock(int cpu)
912	{
913	if (cpu == RING_BUFFER_ALL_CPUS) {
914	/* gain it for accessing the whole ring buffer. */
915	down_write(sem: &all_cpu_access_lock);
916	} else {
917	/* gain it for accessing a cpu ring buffer. */
918
919	/* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
920	down_read(sem: &all_cpu_access_lock);
921
922	/* Secondly block other access to this @cpu ring buffer. */
923	mutex_lock(&per_cpu(cpu_access_lock, cpu));
924	}
925	}
926
927	static inline void trace_access_unlock(int cpu)
928	{
929	if (cpu == RING_BUFFER_ALL_CPUS) {
930	up_write(sem: &all_cpu_access_lock);
931	} else {
932	mutex_unlock(lock: &per_cpu(cpu_access_lock, cpu));
933	up_read(sem: &all_cpu_access_lock);
934	}
935	}
936
937	static inline void trace_access_lock_init(void)
938	{
939	int cpu;
940
941	for_each_possible_cpu(cpu)
942	mutex_init(&per_cpu(cpu_access_lock, cpu));
943	}
944
945	#else
946
947	static DEFINE_MUTEX(access_lock);
948
949	static inline void trace_access_lock(int cpu)
950	{
951	(void)cpu;
952	mutex_lock(&access_lock);
953	}
954
955	static inline void trace_access_unlock(int cpu)
956	{
957	(void)cpu;
958	mutex_unlock(&access_lock);
959	}
960
961	static inline void trace_access_lock_init(void)
962	{
963	}
964
965	#endif
966
967	#ifdef CONFIG_STACKTRACE
968	static void __ftrace_trace_stack(struct trace_buffer *buffer,
969	unsigned int trace_ctx,
970	int skip, struct pt_regs *regs);
971	static inline void ftrace_trace_stack(struct trace_array *tr,
972	struct trace_buffer *buffer,
973	unsigned int trace_ctx,
974	int skip, struct pt_regs *regs);
975
976	#else
977	static inline void __ftrace_trace_stack(struct trace_buffer *buffer,
978	unsigned int trace_ctx,
979	int skip, struct pt_regs *regs)
980	{
981	}
982	static inline void ftrace_trace_stack(struct trace_array *tr,
983	struct trace_buffer *buffer,
984	unsigned long trace_ctx,
985	int skip, struct pt_regs *regs)
986	{
987	}
988
989	#endif
990
991	static __always_inline void
992	trace_event_setup(struct ring_buffer_event *event,
993	int type, unsigned int trace_ctx)
994	{
995	struct trace_entry *ent = ring_buffer_event_data(event);
996
997	tracing_generic_entry_update(entry: ent, type, trace_ctx);
998	}
999
1000	static __always_inline struct ring_buffer_event *
1001	__trace_buffer_lock_reserve(struct trace_buffer *buffer,
1002	int type,
1003	unsigned long len,
1004	unsigned int trace_ctx)
1005	{
1006	struct ring_buffer_event *event;
1007
1008	event = ring_buffer_lock_reserve(buffer, length: len);
1009	if (event != NULL)
1010	trace_event_setup(event, type, trace_ctx);
1011
1012	return event;
1013	}
1014
1015	void tracer_tracing_on(struct trace_array *tr)
1016	{
1017	if (tr->array_buffer.buffer)
1018	ring_buffer_record_on(buffer: tr->array_buffer.buffer);
1019	/*
1020	* This flag is looked at when buffers haven't been allocated
1021	* yet, or by some tracers (like irqsoff), that just want to
1022	* know if the ring buffer has been disabled, but it can handle
1023	* races of where it gets disabled but we still do a record.
1024	* As the check is in the fast path of the tracers, it is more
1025	* important to be fast than accurate.
1026	*/
1027	tr->buffer_disabled = 0;
1028	/* Make the flag seen by readers */
1029	smp_wmb();
1030	}
1031
1032	/**
1033	* tracing_on - enable tracing buffers
1034	*
1035	* This function enables tracing buffers that may have been
1036	* disabled with tracing_off.
1037	*/
1038	void tracing_on(void)
1039	{
1040	tracer_tracing_on(tr: &global_trace);
1041	}
1042	EXPORT_SYMBOL_GPL(tracing_on);
1043
1044
1045	static __always_inline void
1046	__buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event)
1047	{
1048	__this_cpu_write(trace_taskinfo_save, true);
1049
1050	/* If this is the temp buffer, we need to commit fully */
1051	if (this_cpu_read(trace_buffered_event) == event) {
1052	/* Length is in event->array[0] */
1053	ring_buffer_write(buffer, length: event->array[0], data: &event->array[1]);
1054	/* Release the temp buffer */
1055	this_cpu_dec(trace_buffered_event_cnt);
1056	/* ring_buffer_unlock_commit() enables preemption */
1057	preempt_enable_notrace();
1058	} else
1059	ring_buffer_unlock_commit(buffer);
1060	}
1061
1062	int __trace_array_puts(struct trace_array *tr, unsigned long ip,
1063	const char *str, int size)
1064	{
1065	struct ring_buffer_event *event;
1066	struct trace_buffer *buffer;
1067	struct print_entry *entry;
1068	unsigned int trace_ctx;
1069	int alloc;
1070
1071	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
1072	return 0;
1073
1074	if (unlikely(tracing_selftest_running && tr == &global_trace))
1075	return 0;
1076
1077	if (unlikely(tracing_disabled))
1078	return 0;
1079
1080	alloc = sizeof(*entry) + size + 2; /* possible \n added */
1081
1082	trace_ctx = tracing_gen_ctx();
1083	buffer = tr->array_buffer.buffer;
1084	ring_buffer_nest_start(buffer);
1085	event = __trace_buffer_lock_reserve(buffer, type: TRACE_PRINT, len: alloc,
1086	trace_ctx);
1087	if (!event) {
1088	size = 0;
1089	goto out;
1090	}
1091
1092	entry = ring_buffer_event_data(event);
1093	entry->ip = ip;
1094
1095	memcpy(&entry->buf, str, size);
1096
1097	/* Add a newline if necessary */
1098	if (entry->buf[size - 1] != '\n') {
1099	entry->buf[size] = '\n';
1100	entry->buf[size + 1] = '\0';
1101	} else
1102	entry->buf[size] = '\0';
1103
1104	__buffer_unlock_commit(buffer, event);
1105	ftrace_trace_stack(tr, buffer, trace_ctx, skip: 4, NULL);
1106	out:
1107	ring_buffer_nest_end(buffer);
1108	return size;
1109	}
1110	EXPORT_SYMBOL_GPL(__trace_array_puts);
1111
1112	/**
1113	* __trace_puts - write a constant string into the trace buffer.
1114	* @ip: The address of the caller
1115	* @str: The constant string to write
1116	* @size: The size of the string.
1117	*/
1118	int __trace_puts(unsigned long ip, const char *str, int size)
1119	{
1120	return __trace_array_puts(&global_trace, ip, str, size);
1121	}
1122	EXPORT_SYMBOL_GPL(__trace_puts);
1123
1124	/**
1125	* __trace_bputs - write the pointer to a constant string into trace buffer
1126	* @ip: The address of the caller
1127	* @str: The constant string to write to the buffer to
1128	*/
1129	int __trace_bputs(unsigned long ip, const char *str)
1130	{
1131	struct ring_buffer_event *event;
1132	struct trace_buffer *buffer;
1133	struct bputs_entry *entry;
1134	unsigned int trace_ctx;
1135	int size = sizeof(struct bputs_entry);
1136	int ret = 0;
1137
1138	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
1139	return 0;
1140
1141	if (unlikely(tracing_selftest_running || tracing_disabled))
1142	return 0;
1143
1144	trace_ctx = tracing_gen_ctx();
1145	buffer = global_trace.array_buffer.buffer;
1146
1147	ring_buffer_nest_start(buffer);
1148	event = __trace_buffer_lock_reserve(buffer, type: TRACE_BPUTS, len: size,
1149	trace_ctx);
1150	if (!event)
1151	goto out;
1152
1153	entry = ring_buffer_event_data(event);
1154	entry->ip = ip;
1155	entry->str = str;
1156
1157	__buffer_unlock_commit(buffer, event);
1158	ftrace_trace_stack(tr: &global_trace, buffer, trace_ctx, skip: 4, NULL);
1159
1160	ret = 1;
1161	out:
1162	ring_buffer_nest_end(buffer);
1163	return ret;
1164	}
1165	EXPORT_SYMBOL_GPL(__trace_bputs);
1166
1167	#ifdef CONFIG_TRACER_SNAPSHOT
1168	static void tracing_snapshot_instance_cond(struct trace_array *tr,
1169	void *cond_data)
1170	{
1171	struct tracer *tracer = tr->current_trace;
1172	unsigned long flags;
1173
1174	if (in_nmi()) {
1175	trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
1176	trace_array_puts(tr, "*** snapshot is being ignored ***\n");
1177	return;
1178	}
1179
1180	if (!tr->allocated_snapshot) {
1181	trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n");
1182	trace_array_puts(tr, "*** stopping trace here! ***\n");
1183	tracer_tracing_off(tr);
1184	return;
1185	}
1186
1187	/* Note, snapshot can not be used when the tracer uses it */
1188	if (tracer->use_max_tr) {
1189	trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n");
1190	trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n");
1191	return;
1192	}
1193
1194	local_irq_save(flags);
1195	update_max_tr(tr, current, smp_processor_id(), cond_data);
1196	local_irq_restore(flags);
1197	}
1198
1199	void tracing_snapshot_instance(struct trace_array *tr)
1200	{
1201	tracing_snapshot_instance_cond(tr, NULL);
1202	}
1203
1204	/**
1205	* tracing_snapshot - take a snapshot of the current buffer.
1206	*
1207	* This causes a swap between the snapshot buffer and the current live
1208	* tracing buffer. You can use this to take snapshots of the live
1209	* trace when some condition is triggered, but continue to trace.
1210	*
1211	* Note, make sure to allocate the snapshot with either
1212	* a tracing_snapshot_alloc(), or by doing it manually
1213	* with: echo 1 > /sys/kernel/tracing/snapshot
1214	*
1215	* If the snapshot buffer is not allocated, it will stop tracing.
1216	* Basically making a permanent snapshot.
1217	*/
1218	void tracing_snapshot(void)
1219	{
1220	struct trace_array *tr = &global_trace;
1221
1222	tracing_snapshot_instance(tr);
1223	}
1224	EXPORT_SYMBOL_GPL(tracing_snapshot);
1225
1226	/**
1227	* tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1228	* @tr: The tracing instance to snapshot
1229	* @cond_data: The data to be tested conditionally, and possibly saved
1230	*
1231	* This is the same as tracing_snapshot() except that the snapshot is
1232	* conditional - the snapshot will only happen if the
1233	* cond_snapshot.update() implementation receiving the cond_data
1234	* returns true, which means that the trace array's cond_snapshot
1235	* update() operation used the cond_data to determine whether the
1236	* snapshot should be taken, and if it was, presumably saved it along
1237	* with the snapshot.
1238	*/
1239	void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1240	{
1241	tracing_snapshot_instance_cond(tr, cond_data);
1242	}
1243	EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1244
1245	/**
1246	* tracing_cond_snapshot_data - get the user data associated with a snapshot
1247	* @tr: The tracing instance
1248	*
1249	* When the user enables a conditional snapshot using
1250	* tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1251	* with the snapshot. This accessor is used to retrieve it.
1252	*
1253	* Should not be called from cond_snapshot.update(), since it takes
1254	* the tr->max_lock lock, which the code calling
1255	* cond_snapshot.update() has already done.
1256	*
1257	* Returns the cond_data associated with the trace array's snapshot.
1258	*/
1259	void *tracing_cond_snapshot_data(struct trace_array *tr)
1260	{
1261	void *cond_data = NULL;
1262
1263	local_irq_disable();
1264	arch_spin_lock(&tr->max_lock);
1265
1266	if (tr->cond_snapshot)
1267	cond_data = tr->cond_snapshot->cond_data;
1268
1269	arch_spin_unlock(&tr->max_lock);
1270	local_irq_enable();
1271
1272	return cond_data;
1273	}
1274	EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1275
1276	static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
1277	struct array_buffer *size_buf, int cpu_id);
1278	static void set_buffer_entries(struct array_buffer *buf, unsigned long val);
1279
1280	int tracing_alloc_snapshot_instance(struct trace_array *tr)
1281	{
1282	int order;
1283	int ret;
1284
1285	if (!tr->allocated_snapshot) {
1286
1287	/* Make the snapshot buffer have the same order as main buffer */
1288	order = ring_buffer_subbuf_order_get(buffer: tr->array_buffer.buffer);
1289	ret = ring_buffer_subbuf_order_set(buffer: tr->max_buffer.buffer, order);
1290	if (ret < 0)
1291	return ret;
1292
1293	/* allocate spare buffer */
1294	ret = resize_buffer_duplicate_size(trace_buf: &tr->max_buffer,
1295	size_buf: &tr->array_buffer, RING_BUFFER_ALL_CPUS);
1296	if (ret < 0)
1297	return ret;
1298
1299	tr->allocated_snapshot = true;
1300	}
1301
1302	return 0;
1303	}
1304
1305	static void free_snapshot(struct trace_array *tr)
1306	{
1307	/*
1308	* We don't free the ring buffer. instead, resize it because
1309	* The max_tr ring buffer has some state (e.g. ring->clock) and
1310	* we want preserve it.
1311	*/
1312	ring_buffer_subbuf_order_set(buffer: tr->max_buffer.buffer, order: 0);
1313	ring_buffer_resize(buffer: tr->max_buffer.buffer, size: 1, RING_BUFFER_ALL_CPUS);
1314	set_buffer_entries(buf: &tr->max_buffer, val: 1);
1315	tracing_reset_online_cpus(buf: &tr->max_buffer);
1316	tr->allocated_snapshot = false;
1317	}
1318
1319	static int tracing_arm_snapshot_locked(struct trace_array *tr)
1320	{
1321	int ret;
1322
1323	lockdep_assert_held(&trace_types_lock);
1324
1325	spin_lock(lock: &tr->snapshot_trigger_lock);
1326	if (tr->snapshot == UINT_MAX) {
1327	spin_unlock(lock: &tr->snapshot_trigger_lock);
1328	return -EBUSY;
1329	}
1330
1331	tr->snapshot++;
1332	spin_unlock(lock: &tr->snapshot_trigger_lock);
1333
1334	ret = tracing_alloc_snapshot_instance(tr);
1335	if (ret) {
1336	spin_lock(lock: &tr->snapshot_trigger_lock);
1337	tr->snapshot--;
1338	spin_unlock(lock: &tr->snapshot_trigger_lock);
1339	}
1340
1341	return ret;
1342	}
1343
1344	int tracing_arm_snapshot(struct trace_array *tr)
1345	{
1346	int ret;
1347
1348	mutex_lock(&trace_types_lock);
1349	ret = tracing_arm_snapshot_locked(tr);
1350	mutex_unlock(lock: &trace_types_lock);
1351
1352	return ret;
1353	}
1354
1355	void tracing_disarm_snapshot(struct trace_array *tr)
1356	{
1357	spin_lock(lock: &tr->snapshot_trigger_lock);
1358	if (!WARN_ON(!tr->snapshot))
1359	tr->snapshot--;
1360	spin_unlock(lock: &tr->snapshot_trigger_lock);
1361	}
1362
1363	/**
1364	* tracing_alloc_snapshot - allocate snapshot buffer.
1365	*
1366	* This only allocates the snapshot buffer if it isn't already
1367	* allocated - it doesn't also take a snapshot.
1368	*
1369	* This is meant to be used in cases where the snapshot buffer needs
1370	* to be set up for events that can't sleep but need to be able to
1371	* trigger a snapshot.
1372	*/
1373	int tracing_alloc_snapshot(void)
1374	{
1375	struct trace_array *tr = &global_trace;
1376	int ret;
1377
1378	ret = tracing_alloc_snapshot_instance(tr);
1379	WARN_ON(ret < 0);
1380
1381	return ret;
1382	}
1383	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1384
1385	/**
1386	* tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1387	*
1388	* This is similar to tracing_snapshot(), but it will allocate the
1389	* snapshot buffer if it isn't already allocated. Use this only
1390	* where it is safe to sleep, as the allocation may sleep.
1391	*
1392	* This causes a swap between the snapshot buffer and the current live
1393	* tracing buffer. You can use this to take snapshots of the live
1394	* trace when some condition is triggered, but continue to trace.
1395	*/
1396	void tracing_snapshot_alloc(void)
1397	{
1398	int ret;
1399
1400	ret = tracing_alloc_snapshot();
1401	if (ret < 0)
1402	return;
1403
1404	tracing_snapshot();
1405	}
1406	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1407
1408	/**
1409	* tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1410	* @tr: The tracing instance
1411	* @cond_data: User data to associate with the snapshot
1412	* @update: Implementation of the cond_snapshot update function
1413	*
1414	* Check whether the conditional snapshot for the given instance has
1415	* already been enabled, or if the current tracer is already using a
1416	* snapshot; if so, return -EBUSY, else create a cond_snapshot and
1417	* save the cond_data and update function inside.
1418	*
1419	* Returns 0 if successful, error otherwise.
1420	*/
1421	int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data,
1422	cond_update_fn_t update)
1423	{
1424	struct cond_snapshot *cond_snapshot;
1425	int ret = 0;
1426
1427	cond_snapshot = kzalloc(size: sizeof(*cond_snapshot), GFP_KERNEL);
1428	if (!cond_snapshot)
1429	return -ENOMEM;
1430
1431	cond_snapshot->cond_data = cond_data;
1432	cond_snapshot->update = update;
1433
1434	mutex_lock(&trace_types_lock);
1435
1436	if (tr->current_trace->use_max_tr) {
1437	ret = -EBUSY;
1438	goto fail_unlock;
1439	}
1440
1441	/*
1442	* The cond_snapshot can only change to NULL without the
1443	* trace_types_lock. We don't care if we race with it going
1444	* to NULL, but we want to make sure that it's not set to
1445	* something other than NULL when we get here, which we can
1446	* do safely with only holding the trace_types_lock and not
1447	* having to take the max_lock.
1448	*/
1449	if (tr->cond_snapshot) {
1450	ret = -EBUSY;
1451	goto fail_unlock;
1452	}
1453
1454	ret = tracing_arm_snapshot_locked(tr);
1455	if (ret)
1456	goto fail_unlock;
1457
1458	local_irq_disable();
1459	arch_spin_lock(&tr->max_lock);
1460	tr->cond_snapshot = cond_snapshot;
1461	arch_spin_unlock(&tr->max_lock);
1462	local_irq_enable();
1463
1464	mutex_unlock(lock: &trace_types_lock);
1465
1466	return ret;
1467
1468	fail_unlock:
1469	mutex_unlock(lock: &trace_types_lock);
1470	kfree(objp: cond_snapshot);
1471	return ret;
1472	}
1473	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1474
1475	/**
1476	* tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1477	* @tr: The tracing instance
1478	*
1479	* Check whether the conditional snapshot for the given instance is
1480	* enabled; if so, free the cond_snapshot associated with it,
1481	* otherwise return -EINVAL.
1482	*
1483	* Returns 0 if successful, error otherwise.
1484	*/
1485	int tracing_snapshot_cond_disable(struct trace_array *tr)
1486	{
1487	int ret = 0;
1488
1489	local_irq_disable();
1490	arch_spin_lock(&tr->max_lock);
1491
1492	if (!tr->cond_snapshot)
1493	ret = -EINVAL;
1494	else {
1495	kfree(objp: tr->cond_snapshot);
1496	tr->cond_snapshot = NULL;
1497	}
1498
1499	arch_spin_unlock(&tr->max_lock);
1500	local_irq_enable();
1501
1502	tracing_disarm_snapshot(tr);
1503
1504	return ret;
1505	}
1506	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1507	#else
1508	void tracing_snapshot(void)
1509	{
1510	WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1511	}
1512	EXPORT_SYMBOL_GPL(tracing_snapshot);
1513	void tracing_snapshot_cond(struct trace_array *tr, void *cond_data)
1514	{
1515	WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used");
1516	}
1517	EXPORT_SYMBOL_GPL(tracing_snapshot_cond);
1518	int tracing_alloc_snapshot(void)
1519	{
1520	WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1521	return -ENODEV;
1522	}
1523	EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1524	void tracing_snapshot_alloc(void)
1525	{
1526	/* Give warning */
1527	tracing_snapshot();
1528	}
1529	EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1530	void *tracing_cond_snapshot_data(struct trace_array *tr)
1531	{
1532	return NULL;
1533	}
1534	EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data);
1535	int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update)
1536	{
1537	return -ENODEV;
1538	}
1539	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable);
1540	int tracing_snapshot_cond_disable(struct trace_array *tr)
1541	{
1542	return false;
1543	}
1544	EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable);
1545	#define free_snapshot(tr) do { } while (0)
1546	#define tracing_arm_snapshot_locked(tr) ({ -EBUSY; })
1547	#endif /* CONFIG_TRACER_SNAPSHOT */
1548
1549	void tracer_tracing_off(struct trace_array *tr)
1550	{
1551	if (tr->array_buffer.buffer)
1552	ring_buffer_record_off(buffer: tr->array_buffer.buffer);
1553	/*
1554	* This flag is looked at when buffers haven't been allocated
1555	* yet, or by some tracers (like irqsoff), that just want to
1556	* know if the ring buffer has been disabled, but it can handle
1557	* races of where it gets disabled but we still do a record.
1558	* As the check is in the fast path of the tracers, it is more
1559	* important to be fast than accurate.
1560	*/
1561	tr->buffer_disabled = 1;
1562	/* Make the flag seen by readers */
1563	smp_wmb();
1564	}
1565
1566	/**
1567	* tracing_off - turn off tracing buffers
1568	*
1569	* This function stops the tracing buffers from recording data.
1570	* It does not disable any overhead the tracers themselves may
1571	* be causing. This function simply causes all recording to
1572	* the ring buffers to fail.
1573	*/
1574	void tracing_off(void)
1575	{
1576	tracer_tracing_off(tr: &global_trace);
1577	}
1578	EXPORT_SYMBOL_GPL(tracing_off);
1579
1580	void disable_trace_on_warning(void)
1581	{
1582	if (__disable_trace_on_warning) {
1583	trace_array_printk_buf(buffer: global_trace.array_buffer.buffer, _THIS_IP_,
1584	fmt: "Disabling tracing due to warning\n");
1585	tracing_off();
1586	}
1587	}
1588
1589	/**
1590	* tracer_tracing_is_on - show real state of ring buffer enabled
1591	* @tr : the trace array to know if ring buffer is enabled
1592	*
1593	* Shows real state of the ring buffer if it is enabled or not.
1594	*/
1595	bool tracer_tracing_is_on(struct trace_array *tr)
1596	{
1597	if (tr->array_buffer.buffer)
1598	return ring_buffer_record_is_set_on(buffer: tr->array_buffer.buffer);
1599	return !tr->buffer_disabled;
1600	}
1601
1602	/**
1603	* tracing_is_on - show state of ring buffers enabled
1604	*/
1605	int tracing_is_on(void)
1606	{
1607	return tracer_tracing_is_on(tr: &global_trace);
1608	}
1609	EXPORT_SYMBOL_GPL(tracing_is_on);
1610
1611	static int __init set_buf_size(char *str)
1612	{
1613	unsigned long buf_size;
1614
1615	if (!str)
1616	return 0;
1617	buf_size = memparse(ptr: str, retptr: &str);
1618	/*
1619	* nr_entries can not be zero and the startup
1620	* tests require some buffer space. Therefore
1621	* ensure we have at least 4096 bytes of buffer.
1622	*/
1623	trace_buf_size = max(4096UL, buf_size);
1624	return 1;
1625	}
1626	__setup("trace_buf_size=", set_buf_size);
1627
1628	static int __init set_tracing_thresh(char *str)
1629	{
1630	unsigned long threshold;
1631	int ret;
1632
1633	if (!str)
1634	return 0;
1635	ret = kstrtoul(s: str, base: 0, res: &threshold);
1636	if (ret < 0)
1637	return 0;
1638	tracing_thresh = threshold * 1000;
1639	return 1;
1640	}
1641	__setup("tracing_thresh=", set_tracing_thresh);
1642
1643	unsigned long nsecs_to_usecs(unsigned long nsecs)
1644	{
1645	return nsecs / 1000;
1646	}
1647
1648	/*
1649	* TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1650	* It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1651	* matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1652	* of strings in the order that the evals (enum) were defined.
1653	*/
1654	#undef C
1655	#define C(a, b) b
1656
1657	/* These must match the bit positions in trace_iterator_flags */
1658	static const char *trace_options[] = {
1659	TRACE_FLAGS
1660	NULL
1661	};
1662
1663	static struct {
1664	u64 (*func)(void);
1665	const char *name;
1666	int in_ns; /* is this clock in nanoseconds? */
1667	} trace_clocks[] = {
1668	{ trace_clock_local, "local", 1 },
1669	{ trace_clock_global, "global", 1 },
1670	{ trace_clock_counter, "counter", 0 },
1671	{ trace_clock_jiffies, "uptime", 0 },
1672	{ trace_clock, "perf", 1 },
1673	{ ktime_get_mono_fast_ns, "mono", 1 },
1674	{ ktime_get_raw_fast_ns, "mono_raw", 1 },
1675	{ ktime_get_boot_fast_ns, "boot", 1 },
1676	{ ktime_get_tai_fast_ns, "tai", 1 },
1677	ARCH_TRACE_CLOCKS
1678	};
1679
1680	bool trace_clock_in_ns(struct trace_array *tr)
1681	{
1682	if (trace_clocks[tr->clock_id].in_ns)
1683	return true;
1684
1685	return false;
1686	}
1687
1688	/*
1689	* trace_parser_get_init - gets the buffer for trace parser
1690	*/
1691	int trace_parser_get_init(struct trace_parser *parser, int size)
1692	{
1693	memset(parser, 0, sizeof(*parser));
1694
1695	parser->buffer = kmalloc(size, GFP_KERNEL);
1696	if (!parser->buffer)
1697	return 1;
1698
1699	parser->size = size;
1700	return 0;
1701	}
1702
1703	/*
1704	* trace_parser_put - frees the buffer for trace parser
1705	*/
1706	void trace_parser_put(struct trace_parser *parser)
1707	{
1708	kfree(objp: parser->buffer);
1709	parser->buffer = NULL;
1710	}
1711
1712	/*
1713	* trace_get_user - reads the user input string separated by space
1714	* (matched by isspace(ch))
1715	*
1716	* For each string found the 'struct trace_parser' is updated,
1717	* and the function returns.
1718	*
1719	* Returns number of bytes read.
1720	*
1721	* See kernel/trace/trace.h for 'struct trace_parser' details.
1722	*/
1723	int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1724	size_t cnt, loff_t *ppos)
1725	{
1726	char ch;
1727	size_t read = 0;
1728	ssize_t ret;
1729
1730	if (!*ppos)
1731	trace_parser_clear(parser);
1732
1733	ret = get_user(ch, ubuf++);
1734	if (ret)
1735	goto out;
1736
1737	read++;
1738	cnt--;
1739
1740	/*
1741	* The parser is not finished with the last write,
1742	* continue reading the user input without skipping spaces.
1743	*/
1744	if (!parser->cont) {
1745	/* skip white space */
1746	while (cnt && isspace(ch)) {
1747	ret = get_user(ch, ubuf++);
1748	if (ret)
1749	goto out;
1750	read++;
1751	cnt--;
1752	}
1753
1754	parser->idx = 0;
1755
1756	/* only spaces were written */
1757	if (isspace(ch) || !ch) {
1758	*ppos += read;
1759	ret = read;
1760	goto out;
1761	}
1762	}
1763
1764	/* read the non-space input */
1765	while (cnt && !isspace(ch) && ch) {
1766	if (parser->idx < parser->size - 1)
1767	parser->buffer[parser->idx++] = ch;
1768	else {
1769	ret = -EINVAL;
1770	goto out;
1771	}
1772	ret = get_user(ch, ubuf++);
1773	if (ret)
1774	goto out;
1775	read++;
1776	cnt--;
1777	}
1778
1779	/* We either got finished input or we have to wait for another call. */
1780	if (isspace(ch) || !ch) {
1781	parser->buffer[parser->idx] = 0;
1782	parser->cont = false;
1783	} else if (parser->idx < parser->size - 1) {
1784	parser->cont = true;
1785	parser->buffer[parser->idx++] = ch;
1786	/* Make sure the parsed string always terminates with '\0'. */
1787	parser->buffer[parser->idx] = 0;
1788	} else {
1789	ret = -EINVAL;
1790	goto out;
1791	}
1792
1793	*ppos += read;
1794	ret = read;
1795
1796	out:
1797	return ret;
1798	}
1799
1800	/* TODO add a seq_buf_to_buffer() */
1801	static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1802	{
1803	int len;
1804
1805	if (trace_seq_used(s) <= s->readpos)
1806	return -EBUSY;
1807
1808	len = trace_seq_used(s) - s->readpos;
1809	if (cnt > len)
1810	cnt = len;
1811	memcpy(buf, s->buffer + s->readpos, cnt);
1812
1813	s->readpos += cnt;
1814	return cnt;
1815	}
1816
1817	unsigned long __read_mostly tracing_thresh;
1818
1819	#ifdef CONFIG_TRACER_MAX_TRACE
1820	static const struct file_operations tracing_max_lat_fops;
1821
1822	#ifdef LATENCY_FS_NOTIFY
1823
1824	static struct workqueue_struct *fsnotify_wq;
1825
1826	static void latency_fsnotify_workfn(struct work_struct *work)
1827	{
1828	struct trace_array *tr = container_of(work, struct trace_array,
1829	fsnotify_work);
1830	fsnotify_inode(inode: tr->d_max_latency->d_inode, FS_MODIFY);
1831	}
1832
1833	static void latency_fsnotify_workfn_irq(struct irq_work *iwork)
1834	{
1835	struct trace_array *tr = container_of(iwork, struct trace_array,
1836	fsnotify_irqwork);
1837	queue_work(wq: fsnotify_wq, work: &tr->fsnotify_work);
1838	}
1839
1840	static void trace_create_maxlat_file(struct trace_array *tr,
1841	struct dentry *d_tracer)
1842	{
1843	INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);
1844	init_irq_work(work: &tr->fsnotify_irqwork, func: latency_fsnotify_workfn_irq);
1845	tr->d_max_latency = trace_create_file(name: "tracing_max_latency",
1846	TRACE_MODE_WRITE,
1847	parent: d_tracer, data: tr,
1848	fops: &tracing_max_lat_fops);
1849	}
1850
1851	__init static int latency_fsnotify_init(void)
1852	{
1853	fsnotify_wq = alloc_workqueue(fmt: "tr_max_lat_wq",
1854	flags: WQ_UNBOUND | WQ_HIGHPRI, max_active: 0);
1855	if (!fsnotify_wq) {
1856	pr_err("Unable to allocate tr_max_lat_wq\n");
1857	return -ENOMEM;
1858	}
1859	return 0;
1860	}
1861
1862	late_initcall_sync(latency_fsnotify_init);
1863
1864	void latency_fsnotify(struct trace_array *tr)
1865	{
1866	if (!fsnotify_wq)
1867	return;
1868	/*
1869	* We cannot call queue_work(&tr->fsnotify_work) from here because it's
1870	* possible that we are called from __schedule() or do_idle(), which
1871	* could cause a deadlock.
1872	*/
1873	irq_work_queue(work: &tr->fsnotify_irqwork);
1874	}
1875
1876	#else /* !LATENCY_FS_NOTIFY */
1877
1878	#define trace_create_maxlat_file(tr, d_tracer) \
1879	trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \
1880	d_tracer, tr, &tracing_max_lat_fops)
1881
1882	#endif
1883
1884	/*
1885	* Copy the new maximum trace into the separate maximum-trace
1886	* structure. (this way the maximum trace is permanently saved,
1887	* for later retrieval via /sys/kernel/tracing/tracing_max_latency)
1888	*/
1889	static void
1890	__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1891	{
1892	struct array_buffer *trace_buf = &tr->array_buffer;
1893	struct array_buffer *max_buf = &tr->max_buffer;
1894	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1895	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1896
1897	max_buf->cpu = cpu;
1898	max_buf->time_start = data->preempt_timestamp;
1899
1900	max_data->saved_latency = tr->max_latency;
1901	max_data->critical_start = data->critical_start;
1902	max_data->critical_end = data->critical_end;
1903
1904	strncpy(p: max_data->comm, q: tsk->comm, size: TASK_COMM_LEN);
1905	max_data->pid = tsk->pid;
1906	/*
1907	* If tsk == current, then use current_uid(), as that does not use
1908	* RCU. The irq tracer can be called out of RCU scope.
1909	*/
1910	if (tsk == current)
1911	max_data->uid = current_uid();
1912	else
1913	max_data->uid = task_uid(tsk);
1914
1915	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1916	max_data->policy = tsk->policy;
1917	max_data->rt_priority = tsk->rt_priority;
1918
1919	/* record this tasks comm */
1920	tracing_record_cmdline(task: tsk);
1921	latency_fsnotify(tr);
1922	}
1923
1924	/**
1925	* update_max_tr - snapshot all trace buffers from global_trace to max_tr
1926	* @tr: tracer
1927	* @tsk: the task with the latency
1928	* @cpu: The cpu that initiated the trace.
1929	* @cond_data: User data associated with a conditional snapshot
1930	*
1931	* Flip the buffers between the @tr and the max_tr and record information
1932	* about which task was the cause of this latency.
1933	*/
1934	void
1935	update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
1936	void *cond_data)
1937	{
1938	if (tr->stop_count)
1939	return;
1940
1941	WARN_ON_ONCE(!irqs_disabled());
1942
1943	if (!tr->allocated_snapshot) {
1944	/* Only the nop tracer should hit this when disabling */
1945	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1946	return;
1947	}
1948
1949	arch_spin_lock(&tr->max_lock);
1950
1951	/* Inherit the recordable setting from array_buffer */
1952	if (ring_buffer_record_is_set_on(buffer: tr->array_buffer.buffer))
1953	ring_buffer_record_on(buffer: tr->max_buffer.buffer);
1954	else
1955	ring_buffer_record_off(buffer: tr->max_buffer.buffer);
1956
1957	#ifdef CONFIG_TRACER_SNAPSHOT
1958	if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) {
1959	arch_spin_unlock(&tr->max_lock);
1960	return;
1961	}
1962	#endif
1963	swap(tr->array_buffer.buffer, tr->max_buffer.buffer);
1964
1965	__update_max_tr(tr, tsk, cpu);
1966
1967	arch_spin_unlock(&tr->max_lock);
1968
1969	/* Any waiters on the old snapshot buffer need to wake up */
1970	ring_buffer_wake_waiters(buffer: tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS);
1971	}
1972
1973	/**
1974	* update_max_tr_single - only copy one trace over, and reset the rest
1975	* @tr: tracer
1976	* @tsk: task with the latency
1977	* @cpu: the cpu of the buffer to copy.
1978	*
1979	* Flip the trace of a single CPU buffer between the @tr and the max_tr.
1980	*/
1981	void
1982	update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1983	{
1984	int ret;
1985
1986	if (tr->stop_count)
1987	return;
1988
1989	WARN_ON_ONCE(!irqs_disabled());
1990	if (!tr->allocated_snapshot) {
1991	/* Only the nop tracer should hit this when disabling */
1992	WARN_ON_ONCE(tr->current_trace != &nop_trace);
1993	return;
1994	}
1995
1996	arch_spin_lock(&tr->max_lock);
1997
1998	ret = ring_buffer_swap_cpu(buffer_a: tr->max_buffer.buffer, buffer_b: tr->array_buffer.buffer, cpu);
1999
2000	if (ret == -EBUSY) {
2001	/*
2002	* We failed to swap the buffer due to a commit taking
2003	* place on this CPU. We fail to record, but we reset
2004	* the max trace buffer (no one writes directly to it)
2005	* and flag that it failed.
2006	* Another reason is resize is in progress.
2007	*/
2008	trace_array_printk_buf(buffer: tr->max_buffer.buffer, _THIS_IP_,
2009	fmt: "Failed to swap buffers due to commit or resize in progress\n");
2010	}
2011
2012	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
2013
2014	__update_max_tr(tr, tsk, cpu);
2015	arch_spin_unlock(&tr->max_lock);
2016	}
2017
2018	#endif /* CONFIG_TRACER_MAX_TRACE */
2019
2020	struct pipe_wait {
2021	struct trace_iterator *iter;
2022	int wait_index;
2023	};
2024
2025	static bool wait_pipe_cond(void *data)
2026	{
2027	struct pipe_wait *pwait = data;
2028	struct trace_iterator *iter = pwait->iter;
2029
2030	if (atomic_read_acquire(v: &iter->wait_index) != pwait->wait_index)
2031	return true;
2032
2033	return iter->closed;
2034	}
2035
2036	static int wait_on_pipe(struct trace_iterator *iter, int full)
2037	{
2038	struct pipe_wait pwait;
2039	int ret;
2040
2041	/* Iterators are static, they should be filled or empty */
2042	if (trace_buffer_iter(iter, cpu: iter->cpu_file))
2043	return 0;
2044
2045	pwait.wait_index = atomic_read_acquire(v: &iter->wait_index);
2046	pwait.iter = iter;
2047
2048	ret = ring_buffer_wait(buffer: iter->array_buffer->buffer, cpu: iter->cpu_file, full,
2049	cond: wait_pipe_cond, data: &pwait);
2050
2051	#ifdef CONFIG_TRACER_MAX_TRACE
2052	/*
2053	* Make sure this is still the snapshot buffer, as if a snapshot were
2054	* to happen, this would now be the main buffer.
2055	*/
2056	if (iter->snapshot)
2057	iter->array_buffer = &iter->tr->max_buffer;
2058	#endif
2059	return ret;
2060	}
2061
2062	#ifdef CONFIG_FTRACE_STARTUP_TEST
2063	static bool selftests_can_run;
2064
2065	struct trace_selftests {
2066	struct list_head list;
2067	struct tracer *type;
2068	};
2069
2070	static LIST_HEAD(postponed_selftests);
2071
2072	static int save_selftest(struct tracer *type)
2073	{
2074	struct trace_selftests *selftest;
2075
2076	selftest = kmalloc(size: sizeof(*selftest), GFP_KERNEL);
2077	if (!selftest)
2078	return -ENOMEM;
2079
2080	selftest->type = type;
2081	list_add(new: &selftest->list, head: &postponed_selftests);
2082	return 0;
2083	}
2084
2085	static int run_tracer_selftest(struct tracer *type)
2086	{
2087	struct trace_array *tr = &global_trace;
2088	struct tracer *saved_tracer = tr->current_trace;
2089	int ret;
2090
2091	if (!type->selftest || tracing_selftest_disabled)
2092	return 0;
2093
2094	/*
2095	* If a tracer registers early in boot up (before scheduling is
2096	* initialized and such), then do not run its selftests yet.
2097	* Instead, run it a little later in the boot process.
2098	*/
2099	if (!selftests_can_run)
2100	return save_selftest(type);
2101
2102	if (!tracing_is_on()) {
2103	pr_warn("Selftest for tracer %s skipped due to tracing disabled\n",
2104	type->name);
2105	return 0;
2106	}
2107
2108	/*
2109	* Run a selftest on this tracer.
2110	* Here we reset the trace buffer, and set the current
2111	* tracer to be this tracer. The tracer can then run some
2112	* internal tracing to verify that everything is in order.
2113	* If we fail, we do not register this tracer.
2114	*/
2115	tracing_reset_online_cpus(buf: &tr->array_buffer);
2116
2117	tr->current_trace = type;
2118
2119	#ifdef CONFIG_TRACER_MAX_TRACE
2120	if (type->use_max_tr) {
2121	/* If we expanded the buffers, make sure the max is expanded too */
2122	if (tr->ring_buffer_expanded)
2123	ring_buffer_resize(buffer: tr->max_buffer.buffer, size: trace_buf_size,
2124	RING_BUFFER_ALL_CPUS);
2125	tr->allocated_snapshot = true;
2126	}
2127	#endif
2128
2129	/* the test is responsible for initializing and enabling */
2130	pr_info("Testing tracer %s: ", type->name);
2131	ret = type->selftest(type, tr);
2132	/* the test is responsible for resetting too */
2133	tr->current_trace = saved_tracer;
2134	if (ret) {
2135	printk(KERN_CONT "FAILED!\n");
2136	/* Add the warning after printing 'FAILED' */
2137	WARN_ON(1);
2138	return -1;
2139	}
2140	/* Only reset on passing, to avoid touching corrupted buffers */
2141	tracing_reset_online_cpus(buf: &tr->array_buffer);
2142
2143	#ifdef CONFIG_TRACER_MAX_TRACE
2144	if (type->use_max_tr) {
2145	tr->allocated_snapshot = false;
2146
2147	/* Shrink the max buffer again */
2148	if (tr->ring_buffer_expanded)
2149	ring_buffer_resize(buffer: tr->max_buffer.buffer, size: 1,
2150	RING_BUFFER_ALL_CPUS);
2151	}
2152	#endif
2153
2154	printk(KERN_CONT "PASSED\n");
2155	return 0;
2156	}
2157
2158	static int do_run_tracer_selftest(struct tracer *type)
2159	{
2160	int ret;
2161
2162	/*
2163	* Tests can take a long time, especially if they are run one after the
2164	* other, as does happen during bootup when all the tracers are
2165	* registered. This could cause the soft lockup watchdog to trigger.
2166	*/
2167	cond_resched();
2168
2169	tracing_selftest_running = true;
2170	ret = run_tracer_selftest(type);
2171	tracing_selftest_running = false;
2172
2173	return ret;
2174	}
2175
2176	static __init int init_trace_selftests(void)
2177	{
2178	struct trace_selftests *p, *n;
2179	struct tracer *t, **last;
2180	int ret;
2181
2182	selftests_can_run = true;
2183
2184	mutex_lock(&trace_types_lock);
2185
2186	if (list_empty(head: &postponed_selftests))
2187	goto out;
2188
2189	pr_info("Running postponed tracer tests:\n");
2190
2191	tracing_selftest_running = true;
2192	list_for_each_entry_safe(p, n, &postponed_selftests, list) {
2193	/* This loop can take minutes when sanitizers are enabled, so
2194	* lets make sure we allow RCU processing.
2195	*/
2196	cond_resched();
2197	ret = run_tracer_selftest(type: p->type);
2198	/* If the test fails, then warn and remove from available_tracers */
2199	if (ret < 0) {
2200	WARN(1, "tracer: %s failed selftest, disabling\n",
2201	p->type->name);
2202	last = &trace_types;
2203	for (t = trace_types; t; t = t->next) {
2204	if (t == p->type) {
2205	*last = t->next;
2206	break;
2207	}
2208	last = &t->next;
2209	}
2210	}
2211	list_del(entry: &p->list);
2212	kfree(objp: p);
2213	}
2214	tracing_selftest_running = false;
2215
2216	out:
2217	mutex_unlock(lock: &trace_types_lock);
2218
2219	return 0;
2220	}
2221	core_initcall(init_trace_selftests);
2222	#else
2223	static inline int run_tracer_selftest(struct tracer *type)
2224	{
2225	return 0;
2226	}
2227	static inline int do_run_tracer_selftest(struct tracer *type)
2228	{
2229	return 0;
2230	}
2231	#endif /* CONFIG_FTRACE_STARTUP_TEST */
2232
2233	static void add_tracer_options(struct trace_array *tr, struct tracer *t);
2234
2235	static void __init apply_trace_boot_options(void);
2236
2237	/**
2238	* register_tracer - register a tracer with the ftrace system.
2239	* @type: the plugin for the tracer
2240	*
2241	* Register a new plugin tracer.
2242	*/
2243	int __init register_tracer(struct tracer *type)
2244	{
2245	struct tracer *t;
2246	int ret = 0;
2247
2248	if (!type->name) {
2249	pr_info("Tracer must have a name\n");
2250	return -1;
2251	}
2252
2253	if (strlen(type->name) >= MAX_TRACER_SIZE) {
2254	pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
2255	return -1;
2256	}
2257
2258	if (security_locked_down(what: LOCKDOWN_TRACEFS)) {
2259	pr_warn("Can not register tracer %s due to lockdown\n",
2260	type->name);
2261	return -EPERM;
2262	}
2263
2264	mutex_lock(&trace_types_lock);
2265
2266	for (t = trace_types; t; t = t->next) {
2267	if (strcmp(type->name, t->name) == 0) {
2268	/* already found */
2269	pr_info("Tracer %s already registered\n",
2270	type->name);
2271	ret = -1;
2272	goto out;
2273	}
2274	}
2275
2276	if (!type->set_flag)
2277	type->set_flag = &dummy_set_flag;
2278	if (!type->flags) {
2279	/*allocate a dummy tracer_flags*/
2280	type->flags = kmalloc(size: sizeof(*type->flags), GFP_KERNEL);
2281	if (!type->flags) {
2282	ret = -ENOMEM;
2283	goto out;
2284	}
2285	type->flags->val = 0;
2286	type->flags->opts = dummy_tracer_opt;
2287	} else
2288	if (!type->flags->opts)
2289	type->flags->opts = dummy_tracer_opt;
2290
2291	/* store the tracer for __set_tracer_option */
2292	type->flags->trace = type;
2293
2294	ret = do_run_tracer_selftest(type);
2295	if (ret < 0)
2296	goto out;
2297
2298	type->next = trace_types;
2299	trace_types = type;
2300	add_tracer_options(tr: &global_trace, t: type);
2301
2302	out:
2303	mutex_unlock(lock: &trace_types_lock);
2304
2305	if (ret || !default_bootup_tracer)
2306	goto out_unlock;
2307
2308	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
2309	goto out_unlock;
2310
2311	printk(KERN_INFO "Starting tracer '%s'\n", type->name);
2312	/* Do we want this tracer to start on bootup? */
2313	tracing_set_tracer(tr: &global_trace, buf: type->name);
2314	default_bootup_tracer = NULL;
2315
2316	apply_trace_boot_options();
2317
2318	/* disable other selftests, since this will break it. */
2319	disable_tracing_selftest(reason: "running a tracer");
2320
2321	out_unlock:
2322	return ret;
2323	}
2324
2325	static void tracing_reset_cpu(struct array_buffer *buf, int cpu)
2326	{
2327	struct trace_buffer *buffer = buf->buffer;
2328
2329	if (!buffer)
2330	return;
2331
2332	ring_buffer_record_disable(buffer);
2333
2334	/* Make sure all commits have finished */
2335	synchronize_rcu();
2336	ring_buffer_reset_cpu(buffer, cpu);
2337
2338	ring_buffer_record_enable(buffer);
2339	}
2340
2341	void tracing_reset_online_cpus(struct array_buffer *buf)
2342	{
2343	struct trace_buffer *buffer = buf->buffer;
2344
2345	if (!buffer)
2346	return;
2347
2348	ring_buffer_record_disable(buffer);
2349
2350	/* Make sure all commits have finished */
2351	synchronize_rcu();
2352
2353	buf->time_start = buffer_ftrace_now(buf, cpu: buf->cpu);
2354
2355	ring_buffer_reset_online_cpus(buffer);
2356
2357	ring_buffer_record_enable(buffer);
2358	}
2359
2360	/* Must have trace_types_lock held */
2361	void tracing_reset_all_online_cpus_unlocked(void)
2362	{
2363	struct trace_array *tr;
2364
2365	lockdep_assert_held(&trace_types_lock);
2366
2367	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
2368	if (!tr->clear_trace)
2369	continue;
2370	tr->clear_trace = false;
2371	tracing_reset_online_cpus(buf: &tr->array_buffer);
2372	#ifdef CONFIG_TRACER_MAX_TRACE
2373	tracing_reset_online_cpus(buf: &tr->max_buffer);
2374	#endif
2375	}
2376	}
2377
2378	void tracing_reset_all_online_cpus(void)
2379	{
2380	mutex_lock(&trace_types_lock);
2381	tracing_reset_all_online_cpus_unlocked();
2382	mutex_unlock(lock: &trace_types_lock);
2383	}
2384
2385	int is_tracing_stopped(void)
2386	{
2387	return global_trace.stop_count;
2388	}
2389
2390	static void tracing_start_tr(struct trace_array *tr)
2391	{
2392	struct trace_buffer *buffer;
2393	unsigned long flags;
2394
2395	if (tracing_disabled)
2396	return;
2397
2398	raw_spin_lock_irqsave(&tr->start_lock, flags);
2399	if (--tr->stop_count) {
2400	if (WARN_ON_ONCE(tr->stop_count < 0)) {
2401	/* Someone screwed up their debugging */
2402	tr->stop_count = 0;
2403	}
2404	goto out;
2405	}
2406
2407	/* Prevent the buffers from switching */
2408	arch_spin_lock(&tr->max_lock);
2409
2410	buffer = tr->array_buffer.buffer;
2411	if (buffer)
2412	ring_buffer_record_enable(buffer);
2413
2414	#ifdef CONFIG_TRACER_MAX_TRACE
2415	buffer = tr->max_buffer.buffer;
2416	if (buffer)
2417	ring_buffer_record_enable(buffer);
2418	#endif
2419
2420	arch_spin_unlock(&tr->max_lock);
2421
2422	out:
2423	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2424	}
2425
2426	/**
2427	* tracing_start - quick start of the tracer
2428	*
2429	* If tracing is enabled but was stopped by tracing_stop,
2430	* this will start the tracer back up.
2431	*/
2432	void tracing_start(void)
2433
2434	{
2435	return tracing_start_tr(tr: &global_trace);
2436	}
2437
2438	static void tracing_stop_tr(struct trace_array *tr)
2439	{
2440	struct trace_buffer *buffer;
2441	unsigned long flags;
2442
2443	raw_spin_lock_irqsave(&tr->start_lock, flags);
2444	if (tr->stop_count++)
2445	goto out;
2446
2447	/* Prevent the buffers from switching */
2448	arch_spin_lock(&tr->max_lock);
2449
2450	buffer = tr->array_buffer.buffer;
2451	if (buffer)
2452	ring_buffer_record_disable(buffer);
2453
2454	#ifdef CONFIG_TRACER_MAX_TRACE
2455	buffer = tr->max_buffer.buffer;
2456	if (buffer)
2457	ring_buffer_record_disable(buffer);
2458	#endif
2459
2460	arch_spin_unlock(&tr->max_lock);
2461
2462	out:
2463	raw_spin_unlock_irqrestore(&tr->start_lock, flags);
2464	}
2465
2466	/**
2467	* tracing_stop - quick stop of the tracer
2468	*
2469	* Light weight way to stop tracing. Use in conjunction with
2470	* tracing_start.
2471	*/
2472	void tracing_stop(void)
2473	{
2474	return tracing_stop_tr(tr: &global_trace);
2475	}
2476
2477	/*
2478	* Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2479	* overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2480	* simplifies those functions and keeps them in sync.
2481	*/
2482	enum print_line_t trace_handle_return(struct trace_seq *s)
2483	{
2484	return trace_seq_has_overflowed(s) ?
2485	TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2486	}
2487	EXPORT_SYMBOL_GPL(trace_handle_return);
2488
2489	static unsigned short migration_disable_value(void)
2490	{
2491	#if defined(CONFIG_SMP)
2492	return current->migration_disabled;
2493	#else
2494	return 0;
2495	#endif
2496	}
2497
2498	unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status)
2499	{
2500	unsigned int trace_flags = irqs_status;
2501	unsigned int pc;
2502
2503	pc = preempt_count();
2504
2505	if (pc & NMI_MASK)
2506	trace_flags |= TRACE_FLAG_NMI;
2507	if (pc & HARDIRQ_MASK)
2508	trace_flags |= TRACE_FLAG_HARDIRQ;
2509	if (in_serving_softirq())
2510	trace_flags |= TRACE_FLAG_SOFTIRQ;
2511	if (softirq_count() >> (SOFTIRQ_SHIFT + 1))
2512	trace_flags |= TRACE_FLAG_BH_OFF;
2513
2514	if (tif_need_resched())
2515	trace_flags |= TRACE_FLAG_NEED_RESCHED;
2516	if (test_preempt_need_resched())
2517	trace_flags |= TRACE_FLAG_PREEMPT_RESCHED;
2518	return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) |
2519	(min_t(unsigned int, migration_disable_value(), 0xf)) << 4;
2520	}
2521
2522	struct ring_buffer_event *
2523	trace_buffer_lock_reserve(struct trace_buffer *buffer,
2524	int type,
2525	unsigned long len,
2526	unsigned int trace_ctx)
2527	{
2528	return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx);
2529	}
2530
2531	DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2532	DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2533	static int trace_buffered_event_ref;
2534
2535	/**
2536	* trace_buffered_event_enable - enable buffering events
2537	*
2538	* When events are being filtered, it is quicker to use a temporary
2539	* buffer to write the event data into if there's a likely chance
2540	* that it will not be committed. The discard of the ring buffer
2541	* is not as fast as committing, and is much slower than copying
2542	* a commit.
2543	*
2544	* When an event is to be filtered, allocate per cpu buffers to
2545	* write the event data into, and if the event is filtered and discarded
2546	* it is simply dropped, otherwise, the entire data is to be committed
2547	* in one shot.
2548	*/
2549	void trace_buffered_event_enable(void)
2550	{
2551	struct ring_buffer_event *event;
2552	struct page *page;
2553	int cpu;
2554
2555	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2556
2557	if (trace_buffered_event_ref++)
2558	return;
2559
2560	for_each_tracing_cpu(cpu) {
2561	page = alloc_pages_node(cpu_to_node(cpu),
2562	GFP_KERNEL | __GFP_NORETRY, order: 0);
2563	/* This is just an optimization and can handle failures */
2564	if (!page) {
2565	pr_err("Failed to allocate event buffer\n");
2566	break;
2567	}
2568
2569	event = page_address(page);
2570	memset(event, 0, sizeof(*event));
2571
2572	per_cpu(trace_buffered_event, cpu) = event;
2573
2574	preempt_disable();
2575	if (cpu == smp_processor_id() &&
2576	__this_cpu_read(trace_buffered_event) !=
2577	per_cpu(trace_buffered_event, cpu))
2578	WARN_ON_ONCE(1);
2579	preempt_enable();
2580	}
2581	}
2582
2583	static void enable_trace_buffered_event(void *data)
2584	{
2585	/* Probably not needed, but do it anyway */
2586	smp_rmb();
2587	this_cpu_dec(trace_buffered_event_cnt);
2588	}
2589
2590	static void disable_trace_buffered_event(void *data)
2591	{
2592	this_cpu_inc(trace_buffered_event_cnt);
2593	}
2594
2595	/**
2596	* trace_buffered_event_disable - disable buffering events
2597	*
2598	* When a filter is removed, it is faster to not use the buffered
2599	* events, and to commit directly into the ring buffer. Free up
2600	* the temp buffers when there are no more users. This requires
2601	* special synchronization with current events.
2602	*/
2603	void trace_buffered_event_disable(void)
2604	{
2605	int cpu;
2606
2607	WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2608
2609	if (WARN_ON_ONCE(!trace_buffered_event_ref))
2610	return;
2611
2612	if (--trace_buffered_event_ref)
2613	return;
2614
2615	/* For each CPU, set the buffer as used. */
2616	on_each_cpu_mask(mask: tracing_buffer_mask, func: disable_trace_buffered_event,
2617	NULL, wait: true);
2618
2619	/* Wait for all current users to finish */
2620	synchronize_rcu();
2621
2622	for_each_tracing_cpu(cpu) {
2623	free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2624	per_cpu(trace_buffered_event, cpu) = NULL;
2625	}
2626
2627	/*
2628	* Wait for all CPUs that potentially started checking if they can use
2629	* their event buffer only after the previous synchronize_rcu() call and
2630	* they still read a valid pointer from trace_buffered_event. It must be
2631	* ensured they don't see cleared trace_buffered_event_cnt else they
2632	* could wrongly decide to use the pointed-to buffer which is now freed.
2633	*/
2634	synchronize_rcu();
2635
2636	/* For each CPU, relinquish the buffer */
2637	on_each_cpu_mask(mask: tracing_buffer_mask, func: enable_trace_buffered_event, NULL,
2638	wait: true);
2639	}
2640
2641	static struct trace_buffer *temp_buffer;
2642
2643	struct ring_buffer_event *
2644	trace_event_buffer_lock_reserve(struct trace_buffer **current_rb,
2645	struct trace_event_file *trace_file,
2646	int type, unsigned long len,
2647	unsigned int trace_ctx)
2648	{
2649	struct ring_buffer_event *entry;
2650	struct trace_array *tr = trace_file->tr;
2651	int val;
2652
2653	*current_rb = tr->array_buffer.buffer;
2654
2655	if (!tr->no_filter_buffering_ref &&
2656	(trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {
2657	preempt_disable_notrace();
2658	/*
2659	* Filtering is on, so try to use the per cpu buffer first.
2660	* This buffer will simulate a ring_buffer_event,
2661	* where the type_len is zero and the array[0] will
2662	* hold the full length.
2663	* (see include/linux/ring-buffer.h for details on
2664	* how the ring_buffer_event is structured).
2665	*
2666	* Using a temp buffer during filtering and copying it
2667	* on a matched filter is quicker than writing directly
2668	* into the ring buffer and then discarding it when
2669	* it doesn't match. That is because the discard
2670	* requires several atomic operations to get right.
2671	* Copying on match and doing nothing on a failed match
2672	* is still quicker than no copy on match, but having
2673	* to discard out of the ring buffer on a failed match.
2674	*/
2675	if ((entry = __this_cpu_read(trace_buffered_event))) {
2676	int max_len = PAGE_SIZE - struct_size(entry, array, 1);
2677
2678	val = this_cpu_inc_return(trace_buffered_event_cnt);
2679
2680	/*
2681	* Preemption is disabled, but interrupts and NMIs
2682	* can still come in now. If that happens after
2683	* the above increment, then it will have to go
2684	* back to the old method of allocating the event
2685	* on the ring buffer, and if the filter fails, it
2686	* will have to call ring_buffer_discard_commit()
2687	* to remove it.
2688	*
2689	* Need to also check the unlikely case that the
2690	* length is bigger than the temp buffer size.
2691	* If that happens, then the reserve is pretty much
2692	* guaranteed to fail, as the ring buffer currently
2693	* only allows events less than a page. But that may
2694	* change in the future, so let the ring buffer reserve
2695	* handle the failure in that case.
2696	*/
2697	if (val == 1 && likely(len <= max_len)) {
2698	trace_event_setup(event: entry, type, trace_ctx);
2699	entry->array[0] = len;
2700	/* Return with preemption disabled */
2701	return entry;
2702	}
2703	this_cpu_dec(trace_buffered_event_cnt);
2704	}
2705	/* __trace_buffer_lock_reserve() disables preemption */
2706	preempt_enable_notrace();
2707	}
2708
2709	entry = __trace_buffer_lock_reserve(buffer: *current_rb, type, len,
2710	trace_ctx);
2711	/*
2712	* If tracing is off, but we have triggers enabled
2713	* we still need to look at the event data. Use the temp_buffer
2714	* to store the trace event for the trigger to use. It's recursive
2715	* safe and will not be recorded anywhere.
2716	*/
2717	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2718	*current_rb = temp_buffer;
2719	entry = __trace_buffer_lock_reserve(buffer: *current_rb, type, len,
2720	trace_ctx);
2721	}
2722	return entry;
2723	}
2724	EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2725
2726	static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock);
2727	static DEFINE_MUTEX(tracepoint_printk_mutex);
2728
2729	static void output_printk(struct trace_event_buffer *fbuffer)
2730	{
2731	struct trace_event_call *event_call;
2732	struct trace_event_file *file;
2733	struct trace_event *event;
2734	unsigned long flags;
2735	struct trace_iterator *iter = tracepoint_print_iter;
2736
2737	/* We should never get here if iter is NULL */
2738	if (WARN_ON_ONCE(!iter))
2739	return;
2740
2741	event_call = fbuffer->trace_file->event_call;
2742	if (!event_call || !event_call->event.funcs ||
2743	!event_call->event.funcs->trace)
2744	return;
2745
2746	file = fbuffer->trace_file;
2747	if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||
2748	(unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
2749	!filter_match_preds(filter: file->filter, rec: fbuffer->entry)))
2750	return;
2751
2752	event = &fbuffer->trace_file->event_call->event;
2753
2754	raw_spin_lock_irqsave(&tracepoint_iter_lock, flags);
2755	trace_seq_init(s: &iter->seq);
2756	iter->ent = fbuffer->entry;
2757	event_call->event.funcs->trace(iter, 0, event);
2758	trace_seq_putc(s: &iter->seq, c: 0);
2759	printk("%s", iter->seq.buffer);
2760
2761	raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2762	}
2763
2764	int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2765	void *buffer, size_t *lenp,
2766	loff_t *ppos)
2767	{
2768	int save_tracepoint_printk;
2769	int ret;
2770
2771	mutex_lock(&tracepoint_printk_mutex);
2772	save_tracepoint_printk = tracepoint_printk;
2773
2774	ret = proc_dointvec(table, write, buffer, lenp, ppos);
2775
2776	/*
2777	* This will force exiting early, as tracepoint_printk
2778	* is always zero when tracepoint_printk_iter is not allocated
2779	*/
2780	if (!tracepoint_print_iter)
2781	tracepoint_printk = 0;
2782
2783	if (save_tracepoint_printk == tracepoint_printk)
2784	goto out;
2785
2786	if (tracepoint_printk)
2787	static_key_enable(key: &tracepoint_printk_key.key);
2788	else
2789	static_key_disable(key: &tracepoint_printk_key.key);
2790
2791	out:
2792	mutex_unlock(lock: &tracepoint_printk_mutex);
2793
2794	return ret;
2795	}
2796
2797	void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2798	{
2799	enum event_trigger_type tt = ETT_NONE;
2800	struct trace_event_file *file = fbuffer->trace_file;
2801
2802	if (__event_trigger_test_discard(file, buffer: fbuffer->buffer, event: fbuffer->event,
2803	entry: fbuffer->entry, tt: &tt))
2804	goto discard;
2805
2806	if (static_key_false(key: &tracepoint_printk_key.key))
2807	output_printk(fbuffer);
2808
2809	if (static_branch_unlikely(&trace_event_exports_enabled))
2810	ftrace_exports(event: fbuffer->event, TRACE_EXPORT_EVENT);
2811
2812	trace_buffer_unlock_commit_regs(tr: file->tr, buffer: fbuffer->buffer,
2813	event: fbuffer->event, trcace_ctx: fbuffer->trace_ctx, regs: fbuffer->regs);
2814
2815	discard:
2816	if (tt)
2817	event_triggers_post_call(file, tt);
2818
2819	}
2820	EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2821
2822	/*
2823	* Skip 3:
2824	*
2825	* trace_buffer_unlock_commit_regs()
2826	* trace_event_buffer_commit()
2827	* trace_event_raw_event_xxx()
2828	*/
2829	# define STACK_SKIP 3
2830
2831	void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2832	struct trace_buffer *buffer,
2833	struct ring_buffer_event *event,
2834	unsigned int trace_ctx,
2835	struct pt_regs *regs)
2836	{
2837	__buffer_unlock_commit(buffer, event);
2838
2839	/*
2840	* If regs is not set, then skip the necessary functions.
2841	* Note, we can still get here via blktrace, wakeup tracer
2842	* and mmiotrace, but that's ok if they lose a function or
2843	* two. They are not that meaningful.
2844	*/
2845	ftrace_trace_stack(tr, buffer, trace_ctx, skip: regs ? 0 : STACK_SKIP, regs);
2846	ftrace_trace_userstack(tr, buffer, trace_ctx);
2847	}
2848
2849	/*
2850	* Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2851	*/
2852	void
2853	trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
2854	struct ring_buffer_event *event)
2855	{
2856	__buffer_unlock_commit(buffer, event);
2857	}
2858
2859	void
2860	trace_function(struct trace_array *tr, unsigned long ip, unsigned long
2861	parent_ip, unsigned int trace_ctx)
2862	{
2863	struct trace_event_call *call = &event_function;
2864	struct trace_buffer *buffer = tr->array_buffer.buffer;
2865	struct ring_buffer_event *event;
2866	struct ftrace_entry *entry;
2867
2868	event = __trace_buffer_lock_reserve(buffer, type: TRACE_FN, len: sizeof(*entry),
2869	trace_ctx);
2870	if (!event)
2871	return;
2872	entry = ring_buffer_event_data(event);
2873	entry->ip = ip;
2874	entry->parent_ip = parent_ip;
2875
2876	if (!call_filter_check_discard(call, rec: entry, buffer, event)) {
2877	if (static_branch_unlikely(&trace_function_exports_enabled))
2878	ftrace_exports(event, TRACE_EXPORT_FUNCTION);
2879	__buffer_unlock_commit(buffer, event);
2880	}
2881	}
2882
2883	#ifdef CONFIG_STACKTRACE
2884
2885	/* Allow 4 levels of nesting: normal, softirq, irq, NMI */
2886	#define FTRACE_KSTACK_NESTING 4
2887
2888	#define FTRACE_KSTACK_ENTRIES (PAGE_SIZE / FTRACE_KSTACK_NESTING)
2889
2890	struct ftrace_stack {
2891	unsigned long calls[FTRACE_KSTACK_ENTRIES];
2892	};
2893
2894
2895	struct ftrace_stacks {
2896	struct ftrace_stack stacks[FTRACE_KSTACK_NESTING];
2897	};
2898
2899	static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks);
2900	static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2901
2902	static void __ftrace_trace_stack(struct trace_buffer *buffer,
2903	unsigned int trace_ctx,
2904	int skip, struct pt_regs *regs)
2905	{
2906	struct trace_event_call *call = &event_kernel_stack;
2907	struct ring_buffer_event *event;
2908	unsigned int size, nr_entries;
2909	struct ftrace_stack *fstack;
2910	struct stack_entry *entry;
2911	int stackidx;
2912
2913	/*
2914	* Add one, for this function and the call to save_stack_trace()
2915	* If regs is set, then these functions will not be in the way.
2916	*/
2917	#ifndef CONFIG_UNWINDER_ORC
2918	if (!regs)
2919	skip++;
2920	#endif
2921
2922	preempt_disable_notrace();
2923
2924	stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;
2925
2926	/* This should never happen. If it does, yell once and skip */
2927	if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING))
2928	goto out;
2929
2930	/*
2931	* The above __this_cpu_inc_return() is 'atomic' cpu local. An
2932	* interrupt will either see the value pre increment or post
2933	* increment. If the interrupt happens pre increment it will have
2934	* restored the counter when it returns. We just need a barrier to
2935	* keep gcc from moving things around.
2936	*/
2937	barrier();
2938
2939	fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx;
2940	size = ARRAY_SIZE(fstack->calls);
2941
2942	if (regs) {
2943	nr_entries = stack_trace_save_regs(regs, store: fstack->calls,
2944	size, skipnr: skip);
2945	} else {
2946	nr_entries = stack_trace_save(store: fstack->calls, size, skipnr: skip);
2947	}
2948
2949	event = __trace_buffer_lock_reserve(buffer, type: TRACE_STACK,
2950	struct_size(entry, caller, nr_entries),
2951	trace_ctx);
2952	if (!event)
2953	goto out;
2954	entry = ring_buffer_event_data(event);
2955
2956	entry->size = nr_entries;
2957	memcpy(&entry->caller, fstack->calls,
2958	flex_array_size(entry, caller, nr_entries));
2959
2960	if (!call_filter_check_discard(call, rec: entry, buffer, event))
2961	__buffer_unlock_commit(buffer, event);
2962
2963	out:
2964	/* Again, don't let gcc optimize things here */
2965	barrier();
2966	__this_cpu_dec(ftrace_stack_reserve);
2967	preempt_enable_notrace();
2968
2969	}
2970
2971	static inline void ftrace_trace_stack(struct trace_array *tr,
2972	struct trace_buffer *buffer,
2973	unsigned int trace_ctx,
2974	int skip, struct pt_regs *regs)
2975	{
2976	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2977	return;
2978
2979	__ftrace_trace_stack(buffer, trace_ctx, skip, regs);
2980	}
2981
2982	void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
2983	int skip)
2984	{
2985	struct trace_buffer *buffer = tr->array_buffer.buffer;
2986
2987	if (rcu_is_watching()) {
2988	__ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
2989	return;
2990	}
2991
2992	if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY)))
2993	return;
2994
2995	/*
2996	* When an NMI triggers, RCU is enabled via ct_nmi_enter(),
2997	* but if the above rcu_is_watching() failed, then the NMI
2998	* triggered someplace critical, and ct_irq_enter() should
2999	* not be called from NMI.
3000	*/
3001	if (unlikely(in_nmi()))
3002	return;
3003
3004	ct_irq_enter_irqson();
3005	__ftrace_trace_stack(buffer, trace_ctx, skip, NULL);
3006	ct_irq_exit_irqson();
3007	}
3008
3009	/**
3010	* trace_dump_stack - record a stack back trace in the trace buffer
3011	* @skip: Number of functions to skip (helper handlers)
3012	*/
3013	void trace_dump_stack(int skip)
3014	{
3015	if (tracing_disabled || tracing_selftest_running)
3016	return;
3017
3018	#ifndef CONFIG_UNWINDER_ORC
3019	/* Skip 1 to skip this function. */
3020	skip++;
3021	#endif
3022	__ftrace_trace_stack(buffer: global_trace.array_buffer.buffer,
3023	trace_ctx: tracing_gen_ctx(), skip, NULL);
3024	}
3025	EXPORT_SYMBOL_GPL(trace_dump_stack);
3026
3027	#ifdef CONFIG_USER_STACKTRACE_SUPPORT
3028	static DEFINE_PER_CPU(int, user_stack_count);
3029
3030	static void
3031	ftrace_trace_userstack(struct trace_array *tr,
3032	struct trace_buffer *buffer, unsigned int trace_ctx)
3033	{
3034	struct trace_event_call *call = &event_user_stack;
3035	struct ring_buffer_event *event;
3036	struct userstack_entry *entry;
3037
3038	if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
3039	return;
3040
3041	/*
3042	* NMIs can not handle page faults, even with fix ups.
3043	* The save user stack can (and often does) fault.
3044	*/
3045	if (unlikely(in_nmi()))
3046	return;
3047
3048	/*
3049	* prevent recursion, since the user stack tracing may
3050	* trigger other kernel events.
3051	*/
3052	preempt_disable();
3053	if (__this_cpu_read(user_stack_count))
3054	goto out;
3055
3056	__this_cpu_inc(user_stack_count);
3057
3058	event = __trace_buffer_lock_reserve(buffer, type: TRACE_USER_STACK,
3059	len: sizeof(*entry), trace_ctx);
3060	if (!event)
3061	goto out_drop_count;
3062	entry = ring_buffer_event_data(event);
3063
3064	entry->tgid = current->tgid;
3065	memset(&entry->caller, 0, sizeof(entry->caller));
3066
3067	stack_trace_save_user(store: entry->caller, FTRACE_STACK_ENTRIES);
3068	if (!call_filter_check_discard(call, rec: entry, buffer, event))
3069	__buffer_unlock_commit(buffer, event);
3070
3071	out_drop_count:
3072	__this_cpu_dec(user_stack_count);
3073	out:
3074	preempt_enable();
3075	}
3076	#else /* CONFIG_USER_STACKTRACE_SUPPORT */
3077	static void ftrace_trace_userstack(struct trace_array *tr,
3078	struct trace_buffer *buffer,
3079	unsigned int trace_ctx)
3080	{
3081	}
3082	#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */
3083
3084	#endif /* CONFIG_STACKTRACE */
3085
3086	static inline void
3087	func_repeats_set_delta_ts(struct func_repeats_entry *entry,
3088	unsigned long long delta)
3089	{
3090	entry->bottom_delta_ts = delta & U32_MAX;
3091	entry->top_delta_ts = (delta >> 32);
3092	}
3093
3094	void trace_last_func_repeats(struct trace_array *tr,
3095	struct trace_func_repeats *last_info,
3096	unsigned int trace_ctx)
3097	{
3098	struct trace_buffer *buffer = tr->array_buffer.buffer;
3099	struct func_repeats_entry *entry;
3100	struct ring_buffer_event *event;
3101	u64 delta;
3102
3103	event = __trace_buffer_lock_reserve(buffer, type: TRACE_FUNC_REPEATS,
3104	len: sizeof(*entry), trace_ctx);
3105	if (!event)
3106	return;
3107
3108	delta = ring_buffer_event_time_stamp(buffer, event) -
3109	last_info->ts_last_call;
3110
3111	entry = ring_buffer_event_data(event);
3112	entry->ip = last_info->ip;
3113	entry->parent_ip = last_info->parent_ip;
3114	entry->count = last_info->count;
3115	func_repeats_set_delta_ts(entry, delta);
3116
3117	__buffer_unlock_commit(buffer, event);
3118	}
3119
3120	/* created for use with alloc_percpu */
3121	struct trace_buffer_struct {
3122	int nesting;
3123	char buffer[4][TRACE_BUF_SIZE];
3124	};
3125
3126	static struct trace_buffer_struct __percpu *trace_percpu_buffer;
3127
3128	/*
3129	* This allows for lockless recording. If we're nested too deeply, then
3130	* this returns NULL.
3131	*/
3132	static char *get_trace_buf(void)
3133	{
3134	struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
3135
3136	if (!trace_percpu_buffer || buffer->nesting >= 4)
3137	return NULL;
3138
3139	buffer->nesting++;
3140
3141	/* Interrupts must see nesting incremented before we use the buffer */
3142	barrier();
3143	return &buffer->buffer[buffer->nesting - 1][0];
3144	}
3145
3146	static void put_trace_buf(void)
3147	{
3148	/* Don't let the decrement of nesting leak before this */
3149	barrier();
3150	this_cpu_dec(trace_percpu_buffer->nesting);
3151	}
3152
3153	static int alloc_percpu_trace_buffer(void)
3154	{
3155	struct trace_buffer_struct __percpu *buffers;
3156
3157	if (trace_percpu_buffer)
3158	return 0;
3159
3160	buffers = alloc_percpu(struct trace_buffer_struct);
3161	if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer"))
3162	return -ENOMEM;
3163
3164	trace_percpu_buffer = buffers;
3165	return 0;
3166	}
3167
3168	static int buffers_allocated;
3169
3170	void trace_printk_init_buffers(void)
3171	{
3172	if (buffers_allocated)
3173	return;
3174
3175	if (alloc_percpu_trace_buffer())
3176	return;
3177
3178	/* trace_printk() is for debug use only. Don't use it in production. */
3179
3180	pr_warn("\n");
3181	pr_warn("**********************************************************\n");
3182	pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3183	pr_warn("** **\n");
3184	pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
3185	pr_warn("** **\n");
3186	pr_warn("** This means that this is a DEBUG kernel and it is **\n");
3187	pr_warn("** unsafe for production use. **\n");
3188	pr_warn("** **\n");
3189	pr_warn("** If you see this message and you are not debugging **\n");
3190	pr_warn("** the kernel, report this immediately to your vendor! **\n");
3191	pr_warn("** **\n");
3192	pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
3193	pr_warn("**********************************************************\n");
3194
3195	/* Expand the buffers to set size */
3196	tracing_update_buffers(tr: &global_trace);
3197
3198	buffers_allocated = 1;
3199
3200	/*
3201	* trace_printk_init_buffers() can be called by modules.
3202	* If that happens, then we need to start cmdline recording
3203	* directly here. If the global_trace.buffer is already
3204	* allocated here, then this was called by module code.
3205	*/
3206	if (global_trace.array_buffer.buffer)
3207	tracing_start_cmdline_record();
3208	}
3209	EXPORT_SYMBOL_GPL(trace_printk_init_buffers);
3210
3211	void trace_printk_start_comm(void)
3212	{
3213	/* Start tracing comms if trace printk is set */
3214	if (!buffers_allocated)
3215	return;
3216	tracing_start_cmdline_record();
3217	}
3218
3219	static void trace_printk_start_stop_comm(int enabled)
3220	{
3221	if (!buffers_allocated)
3222	return;
3223
3224	if (enabled)
3225	tracing_start_cmdline_record();
3226	else
3227	tracing_stop_cmdline_record();
3228	}
3229
3230	/**
3231	* trace_vbprintk - write binary msg to tracing buffer
3232	* @ip: The address of the caller
3233	* @fmt: The string format to write to the buffer
3234	* @args: Arguments for @fmt
3235	*/
3236	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
3237	{
3238	struct trace_event_call *call = &event_bprint;
3239	struct ring_buffer_event *event;
3240	struct trace_buffer *buffer;
3241	struct trace_array *tr = &global_trace;
3242	struct bprint_entry *entry;
3243	unsigned int trace_ctx;
3244	char *tbuffer;
3245	int len = 0, size;
3246
3247	if (unlikely(tracing_selftest_running || tracing_disabled))
3248	return 0;
3249
3250	/* Don't pollute graph traces with trace_vprintk internals */
3251	pause_graph_tracing();
3252
3253	trace_ctx = tracing_gen_ctx();
3254	preempt_disable_notrace();
3255
3256	tbuffer = get_trace_buf();
3257	if (!tbuffer) {
3258	len = 0;
3259	goto out_nobuffer;
3260	}
3261
3262	len = vbin_printf(bin_buf: (u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
3263
3264	if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
3265	goto out_put;
3266
3267	size = sizeof(*entry) + sizeof(u32) * len;
3268	buffer = tr->array_buffer.buffer;
3269	ring_buffer_nest_start(buffer);
3270	event = __trace_buffer_lock_reserve(buffer, type: TRACE_BPRINT, len: size,
3271	trace_ctx);
3272	if (!event)
3273	goto out;
3274	entry = ring_buffer_event_data(event);
3275	entry->ip = ip;
3276	entry->fmt = fmt;
3277
3278	memcpy(entry->buf, tbuffer, sizeof(u32) * len);
3279	if (!call_filter_check_discard(call, rec: entry, buffer, event)) {
3280	__buffer_unlock_commit(buffer, event);
3281	ftrace_trace_stack(tr, buffer, trace_ctx, skip: 6, NULL);
3282	}
3283
3284	out:
3285	ring_buffer_nest_end(buffer);
3286	out_put:
3287	put_trace_buf();
3288
3289	out_nobuffer:
3290	preempt_enable_notrace();
3291	unpause_graph_tracing();
3292
3293	return len;
3294	}
3295	EXPORT_SYMBOL_GPL(trace_vbprintk);
3296
3297	__printf(3, 0)
3298	static int
3299	__trace_array_vprintk(struct trace_buffer *buffer,
3300	unsigned long ip, const char *fmt, va_list args)
3301	{
3302	struct trace_event_call *call = &event_print;
3303	struct ring_buffer_event *event;
3304	int len = 0, size;
3305	struct print_entry *entry;
3306	unsigned int trace_ctx;
3307	char *tbuffer;
3308
3309	if (tracing_disabled)
3310	return 0;
3311
3312	/* Don't pollute graph traces with trace_vprintk internals */
3313	pause_graph_tracing();
3314
3315	trace_ctx = tracing_gen_ctx();
3316	preempt_disable_notrace();
3317
3318
3319	tbuffer = get_trace_buf();
3320	if (!tbuffer) {
3321	len = 0;
3322	goto out_nobuffer;
3323	}
3324
3325	len = vscnprintf(buf: tbuffer, TRACE_BUF_SIZE, fmt, args);
3326
3327	size = sizeof(*entry) + len + 1;
3328	ring_buffer_nest_start(buffer);
3329	event = __trace_buffer_lock_reserve(buffer, type: TRACE_PRINT, len: size,
3330	trace_ctx);
3331	if (!event)
3332	goto out;
3333	entry = ring_buffer_event_data(event);
3334	entry->ip = ip;
3335
3336	memcpy(&entry->buf, tbuffer, len + 1);
3337	if (!call_filter_check_discard(call, rec: entry, buffer, event)) {
3338	__buffer_unlock_commit(buffer, event);
3339	ftrace_trace_stack(tr: &global_trace, buffer, trace_ctx, skip: 6, NULL);
3340	}
3341
3342	out:
3343	ring_buffer_nest_end(buffer);
3344	put_trace_buf();
3345
3346	out_nobuffer:
3347	preempt_enable_notrace();
3348	unpause_graph_tracing();
3349
3350	return len;
3351	}
3352
3353	__printf(3, 0)
3354	int trace_array_vprintk(struct trace_array *tr,
3355	unsigned long ip, const char *fmt, va_list args)
3356	{
3357	if (tracing_selftest_running && tr == &global_trace)
3358	return 0;
3359
3360	return __trace_array_vprintk(buffer: tr->array_buffer.buffer, ip, fmt, args);
3361	}
3362
3363	/**
3364	* trace_array_printk - Print a message to a specific instance
3365	* @tr: The instance trace_array descriptor
3366	* @ip: The instruction pointer that this is called from.
3367	* @fmt: The format to print (printf format)
3368	*
3369	* If a subsystem sets up its own instance, they have the right to
3370	* printk strings into their tracing instance buffer using this
3371	* function. Note, this function will not write into the top level
3372	* buffer (use trace_printk() for that), as writing into the top level
3373	* buffer should only have events that can be individually disabled.
3374	* trace_printk() is only used for debugging a kernel, and should not
3375	* be ever incorporated in normal use.
3376	*
3377	* trace_array_printk() can be used, as it will not add noise to the
3378	* top level tracing buffer.
3379	*
3380	* Note, trace_array_init_printk() must be called on @tr before this
3381	* can be used.
3382	*/
3383	__printf(3, 0)
3384	int trace_array_printk(struct trace_array *tr,
3385	unsigned long ip, const char *fmt, ...)
3386	{
3387	int ret;
3388	va_list ap;
3389
3390	if (!tr)
3391	return -ENOENT;
3392
3393	/* This is only allowed for created instances */
3394	if (tr == &global_trace)
3395	return 0;
3396
3397	if (!(tr->trace_flags & TRACE_ITER_PRINTK))
3398	return 0;
3399
3400	va_start(ap, fmt);
3401	ret = trace_array_vprintk(tr, ip, fmt, args: ap);
3402	va_end(ap);
3403	return ret;
3404	}
3405	EXPORT_SYMBOL_GPL(trace_array_printk);
3406
3407	/**
3408	* trace_array_init_printk - Initialize buffers for trace_array_printk()
3409	* @tr: The trace array to initialize the buffers for
3410	*
3411	* As trace_array_printk() only writes into instances, they are OK to
3412	* have in the kernel (unlike trace_printk()). This needs to be called
3413	* before trace_array_printk() can be used on a trace_array.
3414	*/
3415	int trace_array_init_printk(struct trace_array *tr)
3416	{
3417	if (!tr)
3418	return -ENOENT;
3419
3420	/* This is only allowed for created instances */
3421	if (tr == &global_trace)
3422	return -EINVAL;
3423
3424	return alloc_percpu_trace_buffer();
3425	}
3426	EXPORT_SYMBOL_GPL(trace_array_init_printk);
3427
3428	__printf(3, 4)
3429	int trace_array_printk_buf(struct trace_buffer *buffer,
3430	unsigned long ip, const char *fmt, ...)
3431	{
3432	int ret;
3433	va_list ap;
3434
3435	if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3436	return 0;
3437
3438	va_start(ap, fmt);
3439	ret = __trace_array_vprintk(buffer, ip, fmt, args: ap);
3440	va_end(ap);
3441	return ret;
3442	}
3443
3444	__printf(2, 0)
3445	int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3446	{
3447	return trace_array_vprintk(tr: &global_trace, ip, fmt, args);
3448	}
3449	EXPORT_SYMBOL_GPL(trace_vprintk);
3450
3451	static void trace_iterator_increment(struct trace_iterator *iter)
3452	{
3453	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu: iter->cpu);
3454
3455	iter->idx++;
3456	if (buf_iter)
3457	ring_buffer_iter_advance(iter: buf_iter);
3458	}
3459
3460	static struct trace_entry *
3461	peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3462	unsigned long *lost_events)
3463	{
3464	struct ring_buffer_event *event;
3465	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3466
3467	if (buf_iter) {
3468	event = ring_buffer_iter_peek(iter: buf_iter, ts);
3469	if (lost_events)
3470	*lost_events = ring_buffer_iter_dropped(iter: buf_iter) ?
3471	(unsigned long)-1 : 0;
3472	} else {
3473	event = ring_buffer_peek(buffer: iter->array_buffer->buffer, cpu, ts,
3474	lost_events);
3475	}
3476
3477	if (event) {
3478	iter->ent_size = ring_buffer_event_length(event);
3479	return ring_buffer_event_data(event);
3480	}
3481	iter->ent_size = 0;
3482	return NULL;
3483	}
3484
3485	static struct trace_entry *
3486	__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3487	unsigned long *missing_events, u64 *ent_ts)
3488	{
3489	struct trace_buffer *buffer = iter->array_buffer->buffer;
3490	struct trace_entry *ent, *next = NULL;
3491	unsigned long lost_events = 0, next_lost = 0;
3492	int cpu_file = iter->cpu_file;
3493	u64 next_ts = 0, ts;
3494	int next_cpu = -1;
3495	int next_size = 0;
3496	int cpu;
3497
3498	/*
3499	* If we are in a per_cpu trace file, don't bother by iterating over
3500	* all cpu and peek directly.
3501	*/
3502	if (cpu_file > RING_BUFFER_ALL_CPUS) {
3503	if (ring_buffer_empty_cpu(buffer, cpu: cpu_file))
3504	return NULL;
3505	ent = peek_next_entry(iter, cpu: cpu_file, ts: ent_ts, lost_events: missing_events);
3506	if (ent_cpu)
3507	*ent_cpu = cpu_file;
3508
3509	return ent;
3510	}
3511
3512	for_each_tracing_cpu(cpu) {
3513
3514	if (ring_buffer_empty_cpu(buffer, cpu))
3515	continue;
3516
3517	ent = peek_next_entry(iter, cpu, ts: &ts, lost_events: &lost_events);
3518
3519	/*
3520	* Pick the entry with the smallest timestamp:
3521	*/
3522	if (ent && (!next || ts < next_ts)) {
3523	next = ent;
3524	next_cpu = cpu;
3525	next_ts = ts;
3526	next_lost = lost_events;
3527	next_size = iter->ent_size;
3528	}
3529	}
3530
3531	iter->ent_size = next_size;
3532
3533	if (ent_cpu)
3534	*ent_cpu = next_cpu;
3535
3536	if (ent_ts)
3537	*ent_ts = next_ts;
3538
3539	if (missing_events)
3540	*missing_events = next_lost;
3541
3542	return next;
3543	}
3544
3545	#define STATIC_FMT_BUF_SIZE 128
3546	static char static_fmt_buf[STATIC_FMT_BUF_SIZE];
3547
3548	char *trace_iter_expand_format(struct trace_iterator *iter)
3549	{
3550	char *tmp;
3551
3552	/*
3553	* iter->tr is NULL when used with tp_printk, which makes
3554	* this get called where it is not safe to call krealloc().
3555	*/
3556	if (!iter->tr || iter->fmt == static_fmt_buf)
3557	return NULL;
3558
3559	tmp = krealloc(objp: iter->fmt, new_size: iter->fmt_size + STATIC_FMT_BUF_SIZE,
3560	GFP_KERNEL);
3561	if (tmp) {
3562	iter->fmt_size += STATIC_FMT_BUF_SIZE;
3563	iter->fmt = tmp;
3564	}
3565
3566	return tmp;
3567	}
3568
3569	/* Returns true if the string is safe to dereference from an event */
3570	static bool trace_safe_str(struct trace_iterator *iter, const char *str,
3571	bool star, int len)
3572	{
3573	unsigned long addr = (unsigned long)str;
3574	struct trace_event *trace_event;
3575	struct trace_event_call *event;
3576
3577	/* Ignore strings with no length */
3578	if (star && !len)
3579	return true;
3580
3581	/* OK if part of the event data */
3582	if ((addr >= (unsigned long)iter->ent) &&
3583	(addr < (unsigned long)iter->ent + iter->ent_size))
3584	return true;
3585
3586	/* OK if part of the temp seq buffer */
3587	if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&
3588	(addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE))
3589	return true;
3590
3591	/* Core rodata can not be freed */
3592	if (is_kernel_rodata(addr))
3593	return true;
3594
3595	if (trace_is_tracepoint_string(str))
3596	return true;
3597
3598	/*
3599	* Now this could be a module event, referencing core module
3600	* data, which is OK.
3601	*/
3602	if (!iter->ent)
3603	return false;
3604
3605	trace_event = ftrace_find_event(type: iter->ent->type);
3606	if (!trace_event)
3607	return false;
3608
3609	event = container_of(trace_event, struct trace_event_call, event);
3610	if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)
3611	return false;
3612
3613	/* Would rather have rodata, but this will suffice */
3614	if (within_module_core(addr, mod: event->module))
3615	return true;
3616
3617	return false;
3618	}
3619
3620	static DEFINE_STATIC_KEY_FALSE(trace_no_verify);
3621
3622	static int test_can_verify_check(const char *fmt, ...)
3623	{
3624	char buf[16];
3625	va_list ap;
3626	int ret;
3627
3628	/*
3629	* The verifier is dependent on vsnprintf() modifies the va_list
3630	* passed to it, where it is sent as a reference. Some architectures
3631	* (like x86_32) passes it by value, which means that vsnprintf()
3632	* does not modify the va_list passed to it, and the verifier
3633	* would then need to be able to understand all the values that
3634	* vsnprintf can use. If it is passed by value, then the verifier
3635	* is disabled.
3636	*/
3637	va_start(ap, fmt);
3638	vsnprintf(buf, size: 16, fmt: "%d", args: ap);
3639	ret = va_arg(ap, int);
3640	va_end(ap);
3641
3642	return ret;
3643	}
3644
3645	static void test_can_verify(void)
3646	{
3647	if (!test_can_verify_check(fmt: "%d %d", 0, 1)) {
3648	pr_info("trace event string verifier disabled\n");
3649	static_branch_inc(&trace_no_verify);
3650	}
3651	}
3652
3653	/**
3654	* trace_check_vprintf - Check dereferenced strings while writing to the seq buffer
3655	* @iter: The iterator that holds the seq buffer and the event being printed
3656	* @fmt: The format used to print the event
3657	* @ap: The va_list holding the data to print from @fmt.
3658	*
3659	* This writes the data into the @iter->seq buffer using the data from
3660	* @fmt and @ap. If the format has a %s, then the source of the string
3661	* is examined to make sure it is safe to print, otherwise it will
3662	* warn and print "[UNSAFE MEMORY]" in place of the dereferenced string
3663	* pointer.
3664	*/
3665	void trace_check_vprintf(struct trace_iterator *iter, const char *fmt,
3666	va_list ap)
3667	{
3668	const char *p = fmt;
3669	const char *str;
3670	int i, j;
3671
3672	if (WARN_ON_ONCE(!fmt))
3673	return;
3674
3675	if (static_branch_unlikely(&trace_no_verify))
3676	goto print;
3677
3678	/* Don't bother checking when doing a ftrace_dump() */
3679	if (iter->fmt == static_fmt_buf)
3680	goto print;
3681
3682	while (*p) {
3683	bool star = false;
3684	int len = 0;
3685
3686	j = 0;
3687
3688	/* We only care about %s and variants */
3689	for (i = 0; p[i]; i++) {
3690	if (i + 1 >= iter->fmt_size) {
3691	/*
3692	* If we can't expand the copy buffer,
3693	* just print it.
3694	*/
3695	if (!trace_iter_expand_format(iter))
3696	goto print;
3697	}
3698
3699	if (p[i] == '\\' && p[i+1]) {
3700	i++;
3701	continue;
3702	}
3703	if (p[i] == '%') {
3704	/* Need to test cases like %08.*s */
3705	for (j = 1; p[i+j]; j++) {
3706	if (isdigit(c: p[i+j]) ||
3707	p[i+j] == '.')
3708	continue;
3709	if (p[i+j] == '*') {
3710	star = true;
3711	continue;
3712	}
3713	break;
3714	}
3715	if (p[i+j] == 's')
3716	break;
3717	star = false;
3718	}
3719	j = 0;
3720	}
3721	/* If no %s found then just print normally */
3722	if (!p[i])
3723	break;
3724
3725	/* Copy up to the %s, and print that */
3726	strncpy(p: iter->fmt, q: p, size: i);
3727	iter->fmt[i] = '\0';
3728	trace_seq_vprintf(s: &iter->seq, fmt: iter->fmt, args: ap);
3729
3730	/*
3731	* If iter->seq is full, the above call no longer guarantees
3732	* that ap is in sync with fmt processing, and further calls
3733	* to va_arg() can return wrong positional arguments.
3734	*
3735	* Ensure that ap is no longer used in this case.
3736	*/
3737	if (iter->seq.full) {
3738	p = "";
3739	break;
3740	}
3741
3742	if (star)
3743	len = va_arg(ap, int);
3744
3745	/* The ap now points to the string data of the %s */
3746	str = va_arg(ap, const char *);
3747
3748	/*
3749	* If you hit this warning, it is likely that the
3750	* trace event in question used %s on a string that
3751	* was saved at the time of the event, but may not be
3752	* around when the trace is read. Use __string(),
3753	* __assign_str() and __get_str() helpers in the TRACE_EVENT()
3754	* instead. See samples/trace_events/trace-events-sample.h
3755	* for reference.
3756	*/
3757	if (WARN_ONCE(!trace_safe_str(iter, str, star, len),
3758	"fmt: '%s' current_buffer: '%s'",
3759	fmt, seq_buf_str(&iter->seq.seq))) {
3760	int ret;
3761
3762	/* Try to safely read the string */
3763	if (star) {
3764	if (len + 1 > iter->fmt_size)
3765	len = iter->fmt_size - 1;
3766	if (len < 0)
3767	len = 0;
3768	ret = copy_from_kernel_nofault(dst: iter->fmt, src: str, size: len);
3769	iter->fmt[len] = 0;
3770	star = false;
3771	} else {
3772	ret = strncpy_from_kernel_nofault(dst: iter->fmt, unsafe_addr: str,
3773	count: iter->fmt_size);
3774	}
3775	if (ret < 0)
3776	trace_seq_printf(s: &iter->seq, fmt: "(0x%px)", str);
3777	else
3778	trace_seq_printf(s: &iter->seq, fmt: "(0x%px:%s)",
3779	str, iter->fmt);
3780	str = "[UNSAFE-MEMORY]";
3781	strcpy(p: iter->fmt, q: "%s");
3782	} else {
3783	strncpy(p: iter->fmt, q: p + i, size: j + 1);
3784	iter->fmt[j+1] = '\0';
3785	}
3786	if (star)
3787	trace_seq_printf(s: &iter->seq, fmt: iter->fmt, len, str);
3788	else
3789	trace_seq_printf(s: &iter->seq, fmt: iter->fmt, str);
3790
3791	p += i + j + 1;
3792	}
3793	print:
3794	if (*p)
3795	trace_seq_vprintf(s: &iter->seq, fmt: p, args: ap);
3796	}
3797
3798	const char *trace_event_format(struct trace_iterator *iter, const char *fmt)
3799	{
3800	const char *p, *new_fmt;
3801	char *q;
3802
3803	if (WARN_ON_ONCE(!fmt))
3804	return fmt;
3805
3806	if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR)
3807	return fmt;
3808
3809	p = fmt;
3810	new_fmt = q = iter->fmt;
3811	while (*p) {
3812	if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {
3813	if (!trace_iter_expand_format(iter))
3814	return fmt;
3815
3816	q += iter->fmt - new_fmt;
3817	new_fmt = iter->fmt;
3818	}
3819
3820	*q++ = *p++;
3821
3822	/* Replace %p with %px */
3823	if (p[-1] == '%') {
3824	if (p[0] == '%') {
3825	*q++ = *p++;
3826	} else if (p[0] == 'p' && !isalnum(p[1])) {
3827	*q++ = *p++;
3828	*q++ = 'x';
3829	}
3830	}
3831	}
3832	*q = '\0';
3833
3834	return new_fmt;
3835	}
3836
3837	#define STATIC_TEMP_BUF_SIZE 128
3838	static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4);
3839
3840	/* Find the next real entry, without updating the iterator itself */
3841	struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3842	int *ent_cpu, u64 *ent_ts)
3843	{
3844	/* __find_next_entry will reset ent_size */
3845	int ent_size = iter->ent_size;
3846	struct trace_entry *entry;
3847
3848	/*
3849	* If called from ftrace_dump(), then the iter->temp buffer
3850	* will be the static_temp_buf and not created from kmalloc.
3851	* If the entry size is greater than the buffer, we can
3852	* not save it. Just return NULL in that case. This is only
3853	* used to add markers when two consecutive events' time
3854	* stamps have a large delta. See trace_print_lat_context()
3855	*/
3856	if (iter->temp == static_temp_buf &&
3857	STATIC_TEMP_BUF_SIZE < ent_size)
3858	return NULL;
3859
3860	/*
3861	* The __find_next_entry() may call peek_next_entry(), which may
3862	* call ring_buffer_peek() that may make the contents of iter->ent
3863	* undefined. Need to copy iter->ent now.
3864	*/
3865	if (iter->ent && iter->ent != iter->temp) {
3866	if ((!iter->temp || iter->temp_size < iter->ent_size) &&
3867	!WARN_ON_ONCE(iter->temp == static_temp_buf)) {
3868	void *temp;
3869	temp = kmalloc(size: iter->ent_size, GFP_KERNEL);
3870	if (!temp)
3871	return NULL;
3872	kfree(objp: iter->temp);
3873	iter->temp = temp;
3874	iter->temp_size = iter->ent_size;
3875	}
3876	memcpy(iter->temp, iter->ent, iter->ent_size);
3877	iter->ent = iter->temp;
3878	}
3879	entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3880	/* Put back the original ent_size */
3881	iter->ent_size = ent_size;
3882
3883	return entry;
3884	}
3885
3886	/* Find the next real entry, and increment the iterator to the next entry */
3887	void *trace_find_next_entry_inc(struct trace_iterator *iter)
3888	{
3889	iter->ent = __find_next_entry(iter, ent_cpu: &iter->cpu,
3890	missing_events: &iter->lost_events, ent_ts: &iter->ts);
3891
3892	if (iter->ent)
3893	trace_iterator_increment(iter);
3894
3895	return iter->ent ? iter : NULL;
3896	}
3897
3898	static void trace_consume(struct trace_iterator *iter)
3899	{
3900	ring_buffer_consume(buffer: iter->array_buffer->buffer, cpu: iter->cpu, ts: &iter->ts,
3901	lost_events: &iter->lost_events);
3902	}
3903
3904	static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3905	{
3906	struct trace_iterator *iter = m->private;
3907	int i = (int)*pos;
3908	void *ent;
3909
3910	WARN_ON_ONCE(iter->leftover);
3911
3912	(*pos)++;
3913
3914	/* can't go backwards */
3915	if (iter->idx > i)
3916	return NULL;
3917
3918	if (iter->idx < 0)
3919	ent = trace_find_next_entry_inc(iter);
3920	else
3921	ent = iter;
3922
3923	while (ent && iter->idx < i)
3924	ent = trace_find_next_entry_inc(iter);
3925
3926	iter->pos = *pos;
3927
3928	return ent;
3929	}
3930
3931	void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3932	{
3933	struct ring_buffer_iter *buf_iter;
3934	unsigned long entries = 0;
3935	u64 ts;
3936
3937	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;
3938
3939	buf_iter = trace_buffer_iter(iter, cpu);
3940	if (!buf_iter)
3941	return;
3942
3943	ring_buffer_iter_reset(iter: buf_iter);
3944
3945	/*
3946	* We could have the case with the max latency tracers
3947	* that a reset never took place on a cpu. This is evident
3948	* by the timestamp being before the start of the buffer.
3949	*/
3950	while (ring_buffer_iter_peek(iter: buf_iter, ts: &ts)) {
3951	if (ts >= iter->array_buffer->time_start)
3952	break;
3953	entries++;
3954	ring_buffer_iter_advance(iter: buf_iter);
3955	}
3956
3957	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;
3958	}
3959
3960	/*
3961	* The current tracer is copied to avoid a global locking
3962	* all around.
3963	*/
3964	static void *s_start(struct seq_file *m, loff_t *pos)
3965	{
3966	struct trace_iterator *iter = m->private;
3967	struct trace_array *tr = iter->tr;
3968	int cpu_file = iter->cpu_file;
3969	void *p = NULL;
3970	loff_t l = 0;
3971	int cpu;
3972
3973	mutex_lock(&trace_types_lock);
3974	if (unlikely(tr->current_trace != iter->trace)) {
3975	/* Close iter->trace before switching to the new current tracer */
3976	if (iter->trace->close)
3977	iter->trace->close(iter);
3978	iter->trace = tr->current_trace;
3979	/* Reopen the new current tracer */
3980	if (iter->trace->open)
3981	iter->trace->open(iter);
3982	}
3983	mutex_unlock(lock: &trace_types_lock);
3984
3985	#ifdef CONFIG_TRACER_MAX_TRACE
3986	if (iter->snapshot && iter->trace->use_max_tr)
3987	return ERR_PTR(error: -EBUSY);
3988	#endif
3989
3990	if (*pos != iter->pos) {
3991	iter->ent = NULL;
3992	iter->cpu = 0;
3993	iter->idx = -1;
3994
3995	if (cpu_file == RING_BUFFER_ALL_CPUS) {
3996	for_each_tracing_cpu(cpu)
3997	tracing_iter_reset(iter, cpu);
3998	} else
3999	tracing_iter_reset(iter, cpu: cpu_file);
4000
4001	iter->leftover = 0;
4002	for (p = iter; p && l < *pos; p = s_next(m, v: p, pos: &l))
4003	;
4004
4005	} else {
4006	/*
4007	* If we overflowed the seq_file before, then we want
4008	* to just reuse the trace_seq buffer again.
4009	*/
4010	if (iter->leftover)
4011	p = iter;
4012	else {
4013	l = *pos - 1;
4014	p = s_next(m, v: p, pos: &l);
4015	}
4016	}
4017
4018	trace_event_read_lock();
4019	trace_access_lock(cpu: cpu_file);
4020	return p;
4021	}
4022
4023	static void s_stop(struct seq_file *m, void *p)
4024	{
4025	struct trace_iterator *iter = m->private;
4026
4027	#ifdef CONFIG_TRACER_MAX_TRACE
4028	if (iter->snapshot && iter->trace->use_max_tr)
4029	return;
4030	#endif
4031
4032	trace_access_unlock(cpu: iter->cpu_file);
4033	trace_event_read_unlock();
4034	}
4035
4036	static void
4037	get_total_entries_cpu(struct array_buffer *buf, unsigned long *total,
4038	unsigned long *entries, int cpu)
4039	{
4040	unsigned long count;
4041
4042	count = ring_buffer_entries_cpu(buffer: buf->buffer, cpu);
4043	/*
4044	* If this buffer has skipped entries, then we hold all
4045	* entries for the trace and we need to ignore the
4046	* ones before the time stamp.
4047	*/
4048	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
4049	count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
4050	/* total is the same as the entries */
4051	*total = count;
4052	} else
4053	*total = count +
4054	ring_buffer_overrun_cpu(buffer: buf->buffer, cpu);
4055	*entries = count;
4056	}
4057
4058	static void
4059	get_total_entries(struct array_buffer *buf,
4060	unsigned long *total, unsigned long *entries)
4061	{
4062	unsigned long t, e;
4063	int cpu;
4064
4065	*total = 0;
4066	*entries = 0;
4067
4068	for_each_tracing_cpu(cpu) {
4069	get_total_entries_cpu(buf, total: &t, entries: &e, cpu);
4070	*total += t;
4071	*entries += e;
4072	}
4073	}
4074
4075	unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu)
4076	{
4077	unsigned long total, entries;
4078
4079	if (!tr)
4080	tr = &global_trace;
4081
4082	get_total_entries_cpu(buf: &tr->array_buffer, total: &total, entries: &entries, cpu);
4083
4084	return entries;
4085	}
4086
4087	unsigned long trace_total_entries(struct trace_array *tr)
4088	{
4089	unsigned long total, entries;
4090
4091	if (!tr)
4092	tr = &global_trace;
4093
4094	get_total_entries(buf: &tr->array_buffer, total: &total, entries: &entries);
4095
4096	return entries;
4097	}
4098
4099	static void print_lat_help_header(struct seq_file *m)
4100	{
4101	seq_puts(m, s: "# _------=> CPU# \n"
4102	"# / _-----=> irqs-off/BH-disabled\n"
4103	"# | / _----=> need-resched \n"
4104	"# || / _---=> hardirq/softirq \n"
4105	"# ||| / _--=> preempt-depth \n"
4106	"# |||| / _-=> migrate-disable \n"
4107	"# ||||| / delay \n"
4108	"# cmd pid |||||| time | caller \n"
4109	"# \\ / |||||| \\ | / \n");
4110	}
4111
4112	static void print_event_info(struct array_buffer *buf, struct seq_file *m)
4113	{
4114	unsigned long total;
4115	unsigned long entries;
4116
4117	get_total_entries(buf, total: &total, entries: &entries);
4118	seq_printf(m, fmt: "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
4119	entries, total, num_online_cpus());
4120	seq_puts(m, s: "#\n");
4121	}
4122
4123	static void print_func_help_header(struct array_buffer *buf, struct seq_file *m,
4124	unsigned int flags)
4125	{
4126	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4127
4128	print_event_info(buf, m);
4129
4130	seq_printf(m, fmt: "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : "");
4131	seq_printf(m, fmt: "# | | %s | | |\n", tgid ? " | " : "");
4132	}
4133
4134	static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m,
4135	unsigned int flags)
4136	{
4137	bool tgid = flags & TRACE_ITER_RECORD_TGID;
4138	static const char space[] = " ";
4139	int prec = tgid ? 12 : 2;
4140
4141	print_event_info(buf, m);
4142
4143	seq_printf(m, fmt: "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space);
4144	seq_printf(m, fmt: "# %.*s / _----=> need-resched\n", prec, space);
4145	seq_printf(m, fmt: "# %.*s| / _---=> hardirq/softirq\n", prec, space);
4146	seq_printf(m, fmt: "# %.*s|| / _--=> preempt-depth\n", prec, space);
4147	seq_printf(m, fmt: "# %.*s||| / _-=> migrate-disable\n", prec, space);
4148	seq_printf(m, fmt: "# %.*s|||| / delay\n", prec, space);
4149	seq_printf(m, fmt: "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID ");
4150	seq_printf(m, fmt: "# | | %.*s | ||||| | |\n", prec, " | ");
4151	}
4152
4153	void
4154	print_trace_header(struct seq_file *m, struct trace_iterator *iter)
4155	{
4156	unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
4157	struct array_buffer *buf = iter->array_buffer;
4158	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
4159	struct tracer *type = iter->trace;
4160	unsigned long entries;
4161	unsigned long total;
4162	const char *name = type->name;
4163
4164	get_total_entries(buf, total: &total, entries: &entries);
4165
4166	seq_printf(m, fmt: "# %s latency trace v1.1.5 on %s\n",
4167	name, init_utsname()->release);
4168	seq_puts(m, s: "# -----------------------------------"
4169	"---------------------------------\n");
4170	seq_printf(m, fmt: "# latency: %lu us, #%lu/%lu, CPU#%d |"
4171	" (M:%s VP:%d, KP:%d, SP:%d HP:%d",
4172	nsecs_to_usecs(nsecs: data->saved_latency),
4173	entries,
4174	total,
4175	buf->cpu,
4176	preempt_model_none() ? "server" :
4177	preempt_model_voluntary() ? "desktop" :
4178	preempt_model_full() ? "preempt" :
4179	preempt_model_rt() ? "preempt_rt" :
4180	"unknown",
4181	/* These are reserved for later use */
4182	0, 0, 0, 0);
4183	#ifdef CONFIG_SMP
4184	seq_printf(m, fmt: " #P:%d)\n", num_online_cpus());
4185	#else
4186	seq_puts(m, ")\n");
4187	#endif
4188	seq_puts(m, s: "# -----------------\n");
4189	seq_printf(m, fmt: "# | task: %.16s-%d "
4190	"(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
4191	data->comm, data->pid,
4192	from_kuid_munged(to: seq_user_ns(seq: m), uid: data->uid), data->nice,
4193	data->policy, data->rt_priority);
4194	seq_puts(m, s: "# -----------------\n");
4195
4196	if (data->critical_start) {
4197	seq_puts(m, s: "# => started at: ");
4198	seq_print_ip_sym(s: &iter->seq, ip: data->critical_start, sym_flags);
4199	trace_print_seq(m, s: &iter->seq);
4200	seq_puts(m, s: "\n# => ended at: ");
4201	seq_print_ip_sym(s: &iter->seq, ip: data->critical_end, sym_flags);
4202	trace_print_seq(m, s: &iter->seq);
4203	seq_puts(m, s: "\n#\n");
4204	}
4205
4206	seq_puts(m, s: "#\n");
4207	}
4208
4209	static void test_cpu_buff_start(struct trace_iterator *iter)
4210	{
4211	struct trace_seq *s = &iter->seq;
4212	struct trace_array *tr = iter->tr;
4213
4214	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
4215	return;
4216
4217	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
4218	return;
4219
4220	if (cpumask_available(mask: iter->started) &&
4221	cpumask_test_cpu(cpu: iter->cpu, cpumask: iter->started))
4222	return;
4223
4224	if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)
4225	return;
4226
4227	if (cpumask_available(mask: iter->started))
4228	cpumask_set_cpu(cpu: iter->cpu, dstp: iter->started);
4229
4230	/* Don't print started cpu buffer for the first entry of the trace */
4231	if (iter->idx > 1)
4232	trace_seq_printf(s, fmt: "##### CPU %u buffer started ####\n",
4233	iter->cpu);
4234	}
4235
4236	static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
4237	{
4238	struct trace_array *tr = iter->tr;
4239	struct trace_seq *s = &iter->seq;
4240	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
4241	struct trace_entry *entry;
4242	struct trace_event *event;
4243
4244	entry = iter->ent;
4245
4246	test_cpu_buff_start(iter);
4247
4248	event = ftrace_find_event(type: entry->type);
4249
4250	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4251	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4252	trace_print_lat_context(iter);
4253	else
4254	trace_print_context(iter);
4255	}
4256
4257	if (trace_seq_has_overflowed(s))
4258	return TRACE_TYPE_PARTIAL_LINE;
4259
4260	if (event) {
4261	if (tr->trace_flags & TRACE_ITER_FIELDS)
4262	return print_event_fields(iter, event);
4263	return event->funcs->trace(iter, sym_flags, event);
4264	}
4265
4266	trace_seq_printf(s, fmt: "Unknown type %d\n", entry->type);
4267
4268	return trace_handle_return(s);
4269	}
4270
4271	static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
4272	{
4273	struct trace_array *tr = iter->tr;
4274	struct trace_seq *s = &iter->seq;
4275	struct trace_entry *entry;
4276	struct trace_event *event;
4277
4278	entry = iter->ent;
4279
4280	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
4281	trace_seq_printf(s, fmt: "%d %d %llu ",
4282	entry->pid, iter->cpu, iter->ts);
4283
4284	if (trace_seq_has_overflowed(s))
4285	return TRACE_TYPE_PARTIAL_LINE;
4286
4287	event = ftrace_find_event(type: entry->type);
4288	if (event)
4289	return event->funcs->raw(iter, 0, event);
4290
4291	trace_seq_printf(s, fmt: "%d ?\n", entry->type);
4292
4293	return trace_handle_return(s);
4294	}
4295
4296	static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
4297	{
4298	struct trace_array *tr = iter->tr;
4299	struct trace_seq *s = &iter->seq;
4300	unsigned char newline = '\n';
4301	struct trace_entry *entry;
4302	struct trace_event *event;
4303
4304	entry = iter->ent;
4305
4306	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4307	SEQ_PUT_HEX_FIELD(s, entry->pid);
4308	SEQ_PUT_HEX_FIELD(s, iter->cpu);
4309	SEQ_PUT_HEX_FIELD(s, iter->ts);
4310	if (trace_seq_has_overflowed(s))
4311	return TRACE_TYPE_PARTIAL_LINE;
4312	}
4313
4314	event = ftrace_find_event(type: entry->type);
4315	if (event) {
4316	enum print_line_t ret = event->funcs->hex(iter, 0, event);
4317	if (ret != TRACE_TYPE_HANDLED)
4318	return ret;
4319	}
4320
4321	SEQ_PUT_FIELD(s, newline);
4322
4323	return trace_handle_return(s);
4324	}
4325
4326	static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
4327	{
4328	struct trace_array *tr = iter->tr;
4329	struct trace_seq *s = &iter->seq;
4330	struct trace_entry *entry;
4331	struct trace_event *event;
4332
4333	entry = iter->ent;
4334
4335	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
4336	SEQ_PUT_FIELD(s, entry->pid);
4337	SEQ_PUT_FIELD(s, iter->cpu);
4338	SEQ_PUT_FIELD(s, iter->ts);
4339	if (trace_seq_has_overflowed(s))
4340	return TRACE_TYPE_PARTIAL_LINE;
4341	}
4342
4343	event = ftrace_find_event(type: entry->type);
4344	return event ? event->funcs->binary(iter, 0, event) :
4345	TRACE_TYPE_HANDLED;
4346	}
4347
4348	int trace_empty(struct trace_iterator *iter)
4349	{
4350	struct ring_buffer_iter *buf_iter;
4351	int cpu;
4352
4353	/* If we are looking at one CPU buffer, only check that one */
4354	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4355	cpu = iter->cpu_file;
4356	buf_iter = trace_buffer_iter(iter, cpu);
4357	if (buf_iter) {
4358	if (!ring_buffer_iter_empty(iter: buf_iter))
4359	return 0;
4360	} else {
4361	if (!ring_buffer_empty_cpu(buffer: iter->array_buffer->buffer, cpu))
4362	return 0;
4363	}
4364	return 1;
4365	}
4366
4367	for_each_tracing_cpu(cpu) {
4368	buf_iter = trace_buffer_iter(iter, cpu);
4369	if (buf_iter) {
4370	if (!ring_buffer_iter_empty(iter: buf_iter))
4371	return 0;
4372	} else {
4373	if (!ring_buffer_empty_cpu(buffer: iter->array_buffer->buffer, cpu))
4374	return 0;
4375	}
4376	}
4377
4378	return 1;
4379	}
4380
4381	/* Called with trace_event_read_lock() held. */
4382	enum print_line_t print_trace_line(struct trace_iterator *iter)
4383	{
4384	struct trace_array *tr = iter->tr;
4385	unsigned long trace_flags = tr->trace_flags;
4386	enum print_line_t ret;
4387
4388	if (iter->lost_events) {
4389	if (iter->lost_events == (unsigned long)-1)
4390	trace_seq_printf(s: &iter->seq, fmt: "CPU:%d [LOST EVENTS]\n",
4391	iter->cpu);
4392	else
4393	trace_seq_printf(s: &iter->seq, fmt: "CPU:%d [LOST %lu EVENTS]\n",
4394	iter->cpu, iter->lost_events);
4395	if (trace_seq_has_overflowed(s: &iter->seq))
4396	return TRACE_TYPE_PARTIAL_LINE;
4397	}
4398
4399	if (iter->trace && iter->trace->print_line) {
4400	ret = iter->trace->print_line(iter);
4401	if (ret != TRACE_TYPE_UNHANDLED)
4402	return ret;
4403	}
4404
4405	if (iter->ent->type == TRACE_BPUTS &&
4406	trace_flags & TRACE_ITER_PRINTK &&
4407	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4408	return trace_print_bputs_msg_only(iter);
4409
4410	if (iter->ent->type == TRACE_BPRINT &&
4411	trace_flags & TRACE_ITER_PRINTK &&
4412	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4413	return trace_print_bprintk_msg_only(iter);
4414
4415	if (iter->ent->type == TRACE_PRINT &&
4416	trace_flags & TRACE_ITER_PRINTK &&
4417	trace_flags & TRACE_ITER_PRINTK_MSGONLY)
4418	return trace_print_printk_msg_only(iter);
4419
4420	if (trace_flags & TRACE_ITER_BIN)
4421	return print_bin_fmt(iter);
4422
4423	if (trace_flags & TRACE_ITER_HEX)
4424	return print_hex_fmt(iter);
4425
4426	if (trace_flags & TRACE_ITER_RAW)
4427	return print_raw_fmt(iter);
4428
4429	return print_trace_fmt(iter);
4430	}
4431
4432	void trace_latency_header(struct seq_file *m)
4433	{
4434	struct trace_iterator *iter = m->private;
4435	struct trace_array *tr = iter->tr;
4436
4437	/* print nothing if the buffers are empty */
4438	if (trace_empty(iter))
4439	return;
4440
4441	if (iter->iter_flags & TRACE_FILE_LAT_FMT)
4442	print_trace_header(m, iter);
4443
4444	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
4445	print_lat_help_header(m);
4446	}
4447
4448	void trace_default_header(struct seq_file *m)
4449	{
4450	struct trace_iterator *iter = m->private;
4451	struct trace_array *tr = iter->tr;
4452	unsigned long trace_flags = tr->trace_flags;
4453
4454	if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
4455	return;
4456
4457	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
4458	/* print nothing if the buffers are empty */
4459	if (trace_empty(iter))
4460	return;
4461	print_trace_header(m, iter);
4462	if (!(trace_flags & TRACE_ITER_VERBOSE))
4463	print_lat_help_header(m);
4464	} else {
4465	if (!(trace_flags & TRACE_ITER_VERBOSE)) {
4466	if (trace_flags & TRACE_ITER_IRQ_INFO)
4467	print_func_help_header_irq(buf: iter->array_buffer,
4468	m, flags: trace_flags);
4469	else
4470	print_func_help_header(buf: iter->array_buffer, m,
4471	flags: trace_flags);
4472	}
4473	}
4474	}
4475
4476	static void test_ftrace_alive(struct seq_file *m)
4477	{
4478	if (!ftrace_is_dead())
4479	return;
4480	seq_puts(m, s: "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
4481	"# MAY BE MISSING FUNCTION EVENTS\n");
4482	}
4483
4484	#ifdef CONFIG_TRACER_MAX_TRACE
4485	static void show_snapshot_main_help(struct seq_file *m)
4486	{
4487	seq_puts(m, s: "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
4488	"# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4489	"# Takes a snapshot of the main buffer.\n"
4490	"# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
4491	"# (Doesn't have to be '2' works with any number that\n"
4492	"# is not a '0' or '1')\n");
4493	}
4494
4495	static void show_snapshot_percpu_help(struct seq_file *m)
4496	{
4497	seq_puts(m, s: "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
4498	#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4499	seq_puts(m, s: "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
4500	"# Takes a snapshot of the main buffer for this cpu.\n");
4501	#else
4502	seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
4503	"# Must use main snapshot file to allocate.\n");
4504	#endif
4505	seq_puts(m, s: "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
4506	"# (Doesn't have to be '2' works with any number that\n"
4507	"# is not a '0' or '1')\n");
4508	}
4509
4510	static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
4511	{
4512	if (iter->tr->allocated_snapshot)
4513	seq_puts(m, s: "#\n# * Snapshot is allocated *\n#\n");
4514	else
4515	seq_puts(m, s: "#\n# * Snapshot is freed *\n#\n");
4516
4517	seq_puts(m, s: "# Snapshot commands:\n");
4518	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4519	show_snapshot_main_help(m);
4520	else
4521	show_snapshot_percpu_help(m);
4522	}
4523	#else
4524	/* Should never be called */
4525	static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
4526	#endif
4527
4528	static int s_show(struct seq_file *m, void *v)
4529	{
4530	struct trace_iterator *iter = v;
4531	int ret;
4532
4533	if (iter->ent == NULL) {
4534	if (iter->tr) {
4535	seq_printf(m, fmt: "# tracer: %s\n", iter->trace->name);
4536	seq_puts(m, s: "#\n");
4537	test_ftrace_alive(m);
4538	}
4539	if (iter->snapshot && trace_empty(iter))
4540	print_snapshot_help(m, iter);
4541	else if (iter->trace && iter->trace->print_header)
4542	iter->trace->print_header(m);
4543	else
4544	trace_default_header(m);
4545
4546	} else if (iter->leftover) {
4547	/*
4548	* If we filled the seq_file buffer earlier, we
4549	* want to just show it now.
4550	*/
4551	ret = trace_print_seq(m, s: &iter->seq);
4552
4553	/* ret should this time be zero, but you never know */
4554	iter->leftover = ret;
4555
4556	} else {
4557	ret = print_trace_line(iter);
4558	if (ret == TRACE_TYPE_PARTIAL_LINE) {
4559	iter->seq.full = 0;
4560	trace_seq_puts(s: &iter->seq, str: "[LINE TOO BIG]\n");
4561	}
4562	ret = trace_print_seq(m, s: &iter->seq);
4563	/*
4564	* If we overflow the seq_file buffer, then it will
4565	* ask us for this data again at start up.
4566	* Use that instead.
4567	* ret is 0 if seq_file write succeeded.
4568	* -1 otherwise.
4569	*/
4570	iter->leftover = ret;
4571	}
4572
4573	return 0;
4574	}
4575
4576	/*
4577	* Should be used after trace_array_get(), trace_types_lock
4578	* ensures that i_cdev was already initialized.
4579	*/
4580	static inline int tracing_get_cpu(struct inode *inode)
4581	{
4582	if (inode->i_cdev) /* See trace_create_cpu_file() */
4583	return (long)inode->i_cdev - 1;
4584	return RING_BUFFER_ALL_CPUS;
4585	}
4586
4587	static const struct seq_operations tracer_seq_ops = {
4588	.start = s_start,
4589	.next = s_next,
4590	.stop = s_stop,
4591	.show = s_show,
4592	};
4593
4594	/*
4595	* Note, as iter itself can be allocated and freed in different
4596	* ways, this function is only used to free its content, and not
4597	* the iterator itself. The only requirement to all the allocations
4598	* is that it must zero all fields (kzalloc), as freeing works with
4599	* ethier allocated content or NULL.
4600	*/
4601	static void free_trace_iter_content(struct trace_iterator *iter)
4602	{
4603	/* The fmt is either NULL, allocated or points to static_fmt_buf */
4604	if (iter->fmt != static_fmt_buf)
4605	kfree(objp: iter->fmt);
4606
4607	kfree(objp: iter->temp);
4608	kfree(objp: iter->buffer_iter);
4609	mutex_destroy(lock: &iter->mutex);
4610	free_cpumask_var(mask: iter->started);
4611	}
4612
4613	static struct trace_iterator *
4614	__tracing_open(struct inode *inode, struct file *file, bool snapshot)
4615	{
4616	struct trace_array *tr = inode->i_private;
4617	struct trace_iterator *iter;
4618	int cpu;
4619
4620	if (tracing_disabled)
4621	return ERR_PTR(error: -ENODEV);
4622
4623	iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
4624	if (!iter)
4625	return ERR_PTR(error: -ENOMEM);
4626
4627	iter->buffer_iter = kcalloc(n: nr_cpu_ids, size: sizeof(*iter->buffer_iter),
4628	GFP_KERNEL);
4629	if (!iter->buffer_iter)
4630	goto release;
4631
4632	/*
4633	* trace_find_next_entry() may need to save off iter->ent.
4634	* It will place it into the iter->temp buffer. As most
4635	* events are less than 128, allocate a buffer of that size.
4636	* If one is greater, then trace_find_next_entry() will
4637	* allocate a new buffer to adjust for the bigger iter->ent.
4638	* It's not critical if it fails to get allocated here.
4639	*/
4640	iter->temp = kmalloc(size: 128, GFP_KERNEL);
4641	if (iter->temp)
4642	iter->temp_size = 128;
4643
4644	/*
4645	* trace_event_printf() may need to modify given format
4646	* string to replace %p with %px so that it shows real address
4647	* instead of hash value. However, that is only for the event
4648	* tracing, other tracer may not need. Defer the allocation
4649	* until it is needed.
4650	*/
4651	iter->fmt = NULL;
4652	iter->fmt_size = 0;
4653
4654	mutex_lock(&trace_types_lock);
4655	iter->trace = tr->current_trace;
4656
4657	if (!zalloc_cpumask_var(mask: &iter->started, GFP_KERNEL))
4658	goto fail;
4659
4660	iter->tr = tr;
4661
4662	#ifdef CONFIG_TRACER_MAX_TRACE
4663	/* Currently only the top directory has a snapshot */
4664	if (tr->current_trace->print_max || snapshot)
4665	iter->array_buffer = &tr->max_buffer;
4666	else
4667	#endif
4668	iter->array_buffer = &tr->array_buffer;
4669	iter->snapshot = snapshot;
4670	iter->pos = -1;
4671	iter->cpu_file = tracing_get_cpu(inode);
4672	mutex_init(&iter->mutex);
4673
4674	/* Notify the tracer early; before we stop tracing. */
4675	if (iter->trace->open)
4676	iter->trace->open(iter);
4677
4678	/* Annotate start of buffers if we had overruns */
4679	if (ring_buffer_overruns(buffer: iter->array_buffer->buffer))
4680	iter->iter_flags |= TRACE_FILE_ANNOTATE;
4681
4682	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
4683	if (trace_clocks[tr->clock_id].in_ns)
4684	iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4685
4686	/*
4687	* If pause-on-trace is enabled, then stop the trace while
4688	* dumping, unless this is the "snapshot" file
4689	*/
4690	if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))
4691	tracing_stop_tr(tr);
4692
4693	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
4694	for_each_tracing_cpu(cpu) {
4695	iter->buffer_iter[cpu] =
4696	ring_buffer_read_prepare(buffer: iter->array_buffer->buffer,
4697	cpu, GFP_KERNEL);
4698	}
4699	ring_buffer_read_prepare_sync();
4700	for_each_tracing_cpu(cpu) {
4701	ring_buffer_read_start(iter: iter->buffer_iter[cpu]);
4702	tracing_iter_reset(iter, cpu);
4703	}
4704	} else {
4705	cpu = iter->cpu_file;
4706	iter->buffer_iter[cpu] =
4707	ring_buffer_read_prepare(buffer: iter->array_buffer->buffer,
4708	cpu, GFP_KERNEL);
4709	ring_buffer_read_prepare_sync();
4710	ring_buffer_read_start(iter: iter->buffer_iter[cpu]);
4711	tracing_iter_reset(iter, cpu);
4712	}
4713
4714	mutex_unlock(lock: &trace_types_lock);
4715
4716	return iter;
4717
4718	fail:
4719	mutex_unlock(lock: &trace_types_lock);
4720	free_trace_iter_content(iter);
4721	release:
4722	seq_release_private(inode, file);
4723	return ERR_PTR(error: -ENOMEM);
4724	}
4725
4726	int tracing_open_generic(struct inode *inode, struct file *filp)
4727	{
4728	int ret;
4729
4730	ret = tracing_check_open_get_tr(NULL);
4731	if (ret)
4732	return ret;
4733
4734	filp->private_data = inode->i_private;
4735	return 0;
4736	}
4737
4738	bool tracing_is_disabled(void)
4739	{
4740	return (tracing_disabled) ? true: false;
4741	}
4742
4743	/*
4744	* Open and update trace_array ref count.
4745	* Must have the current trace_array passed to it.
4746	*/
4747	int tracing_open_generic_tr(struct inode *inode, struct file *filp)
4748	{
4749	struct trace_array *tr = inode->i_private;
4750	int ret;
4751
4752	ret = tracing_check_open_get_tr(tr);
4753	if (ret)
4754	return ret;
4755
4756	filp->private_data = inode->i_private;
4757
4758	return 0;
4759	}
4760
4761	/*
4762	* The private pointer of the inode is the trace_event_file.
4763	* Update the tr ref count associated to it.
4764	*/
4765	int tracing_open_file_tr(struct inode *inode, struct file *filp)
4766	{
4767	struct trace_event_file *file = inode->i_private;
4768	int ret;
4769
4770	ret = tracing_check_open_get_tr(tr: file->tr);
4771	if (ret)
4772	return ret;
4773
4774	mutex_lock(&event_mutex);
4775
4776	/* Fail if the file is marked for removal */
4777	if (file->flags & EVENT_FILE_FL_FREED) {
4778	trace_array_put(file->tr);
4779	ret = -ENODEV;
4780	} else {
4781	event_file_get(file);
4782	}
4783
4784	mutex_unlock(lock: &event_mutex);
4785	if (ret)
4786	return ret;
4787
4788	filp->private_data = inode->i_private;
4789
4790	return 0;
4791	}
4792
4793	int tracing_release_file_tr(struct inode *inode, struct file *filp)
4794	{
4795	struct trace_event_file *file = inode->i_private;
4796
4797	trace_array_put(file->tr);
4798	event_file_put(file);
4799
4800	return 0;
4801	}
4802
4803	int tracing_single_release_file_tr(struct inode *inode, struct file *filp)
4804	{
4805	tracing_release_file_tr(inode, filp);
4806	return single_release(inode, filp);
4807	}
4808
4809	static int tracing_mark_open(struct inode *inode, struct file *filp)
4810	{
4811	stream_open(inode, filp);
4812	return tracing_open_generic_tr(inode, filp);
4813	}
4814
4815	static int tracing_release(struct inode *inode, struct file *file)
4816	{
4817	struct trace_array *tr = inode->i_private;
4818	struct seq_file *m = file->private_data;
4819	struct trace_iterator *iter;
4820	int cpu;
4821
4822	if (!(file->f_mode & FMODE_READ)) {
4823	trace_array_put(tr);
4824	return 0;
4825	}
4826
4827	/* Writes do not use seq_file */
4828	iter = m->private;
4829	mutex_lock(&trace_types_lock);
4830
4831	for_each_tracing_cpu(cpu) {
4832	if (iter->buffer_iter[cpu])
4833	ring_buffer_read_finish(iter: iter->buffer_iter[cpu]);
4834	}
4835
4836	if (iter->trace && iter->trace->close)
4837	iter->trace->close(iter);
4838
4839	if (!iter->snapshot && tr->stop_count)
4840	/* reenable tracing if it was previously enabled */
4841	tracing_start_tr(tr);
4842
4843	__trace_array_put(this_tr: tr);
4844
4845	mutex_unlock(lock: &trace_types_lock);
4846
4847	free_trace_iter_content(iter);
4848	seq_release_private(inode, file);
4849
4850	return 0;
4851	}
4852
4853	int tracing_release_generic_tr(struct inode *inode, struct file *file)
4854	{
4855	struct trace_array *tr = inode->i_private;
4856
4857	trace_array_put(tr);
4858	return 0;
4859	}
4860
4861	static int tracing_single_release_tr(struct inode *inode, struct file *file)
4862	{
4863	struct trace_array *tr = inode->i_private;
4864
4865	trace_array_put(tr);
4866
4867	return single_release(inode, file);
4868	}
4869
4870	static int tracing_open(struct inode *inode, struct file *file)
4871	{
4872	struct trace_array *tr = inode->i_private;
4873	struct trace_iterator *iter;
4874	int ret;
4875
4876	ret = tracing_check_open_get_tr(tr);
4877	if (ret)
4878	return ret;
4879
4880	/* If this file was open for write, then erase contents */
4881	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4882	int cpu = tracing_get_cpu(inode);
4883	struct array_buffer *trace_buf = &tr->array_buffer;
4884
4885	#ifdef CONFIG_TRACER_MAX_TRACE
4886	if (tr->current_trace->print_max)
4887	trace_buf = &tr->max_buffer;
4888	#endif
4889
4890	if (cpu == RING_BUFFER_ALL_CPUS)
4891	tracing_reset_online_cpus(buf: trace_buf);
4892	else
4893	tracing_reset_cpu(buf: trace_buf, cpu);
4894	}
4895
4896	if (file->f_mode & FMODE_READ) {
4897	iter = __tracing_open(inode, file, snapshot: false);
4898	if (IS_ERR(ptr: iter))
4899	ret = PTR_ERR(ptr: iter);
4900	else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4901	iter->iter_flags |= TRACE_FILE_LAT_FMT;
4902	}
4903
4904	if (ret < 0)
4905	trace_array_put(tr);
4906
4907	return ret;
4908	}
4909
4910	/*
4911	* Some tracers are not suitable for instance buffers.
4912	* A tracer is always available for the global array (toplevel)
4913	* or if it explicitly states that it is.
4914	*/
4915	static bool
4916	trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4917	{
4918	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4919	}
4920
4921	/* Find the next tracer that this trace array may use */
4922	static struct tracer *
4923	get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4924	{
4925	while (t && !trace_ok_for_array(t, tr))
4926	t = t->next;
4927
4928	return t;
4929	}
4930
4931	static void *
4932	t_next(struct seq_file *m, void *v, loff_t *pos)
4933	{
4934	struct trace_array *tr = m->private;
4935	struct tracer *t = v;
4936
4937	(*pos)++;
4938
4939	if (t)
4940	t = get_tracer_for_array(tr, t: t->next);
4941
4942	return t;
4943	}
4944
4945	static void *t_start(struct seq_file *m, loff_t *pos)
4946	{
4947	struct trace_array *tr = m->private;
4948	struct tracer *t;
4949	loff_t l = 0;
4950
4951	mutex_lock(&trace_types_lock);
4952
4953	t = get_tracer_for_array(tr, t: trace_types);
4954	for (; t && l < *pos; t = t_next(m, v: t, pos: &l))
4955	;
4956
4957	return t;
4958	}
4959
4960	static void t_stop(struct seq_file *m, void *p)
4961	{
4962	mutex_unlock(lock: &trace_types_lock);
4963	}
4964
4965	static int t_show(struct seq_file *m, void *v)
4966	{
4967	struct tracer *t = v;
4968
4969	if (!t)
4970	return 0;
4971
4972	seq_puts(m, s: t->name);
4973	if (t->next)
4974	seq_putc(m, c: ' ');
4975	else
4976	seq_putc(m, c: '\n');
4977
4978	return 0;
4979	}
4980
4981	static const struct seq_operations show_traces_seq_ops = {
4982	.start = t_start,
4983	.next = t_next,
4984	.stop = t_stop,
4985	.show = t_show,
4986	};
4987
4988	static int show_traces_open(struct inode *inode, struct file *file)
4989	{
4990	struct trace_array *tr = inode->i_private;
4991	struct seq_file *m;
4992	int ret;
4993
4994	ret = tracing_check_open_get_tr(tr);
4995	if (ret)
4996	return ret;
4997
4998	ret = seq_open(file, &show_traces_seq_ops);
4999	if (ret) {
5000	trace_array_put(tr);
5001	return ret;
5002	}
5003
5004	m = file->private_data;
5005	m->private = tr;
5006
5007	return 0;
5008	}
5009
5010	static int show_traces_release(struct inode *inode, struct file *file)
5011	{
5012	struct trace_array *tr = inode->i_private;
5013
5014	trace_array_put(tr);
5015	return seq_release(inode, file);
5016	}
5017
5018	static ssize_t
5019	tracing_write_stub(struct file *filp, const char __user *ubuf,
5020	size_t count, loff_t *ppos)
5021	{
5022	return count;
5023	}
5024
5025	loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
5026	{
5027	int ret;
5028
5029	if (file->f_mode & FMODE_READ)
5030	ret = seq_lseek(file, offset, whence);
5031	else
5032	file->f_pos = ret = 0;
5033
5034	return ret;
5035	}
5036
5037	static const struct file_operations tracing_fops = {
5038	.open = tracing_open,
5039	.read = seq_read,
5040	.read_iter = seq_read_iter,
5041	.splice_read = copy_splice_read,
5042	.write = tracing_write_stub,
5043	.llseek = tracing_lseek,
5044	.release = tracing_release,
5045	};
5046
5047	static const struct file_operations show_traces_fops = {
5048	.open = show_traces_open,
5049	.read = seq_read,
5050	.llseek = seq_lseek,
5051	.release = show_traces_release,
5052	};
5053
5054	static ssize_t
5055	tracing_cpumask_read(struct file *filp, char __user *ubuf,
5056	size_t count, loff_t *ppos)
5057	{
5058	struct trace_array *tr = file_inode(f: filp)->i_private;
5059	char *mask_str;
5060	int len;
5061
5062	len = snprintf(NULL, size: 0, fmt: "%*pb\n",
5063	cpumask_pr_args(tr->tracing_cpumask)) + 1;
5064	mask_str = kmalloc(size: len, GFP_KERNEL);
5065	if (!mask_str)
5066	return -ENOMEM;
5067
5068	len = snprintf(buf: mask_str, size: len, fmt: "%*pb\n",
5069	cpumask_pr_args(tr->tracing_cpumask));
5070	if (len >= count) {
5071	count = -EINVAL;
5072	goto out_err;
5073	}
5074	count = simple_read_from_buffer(to: ubuf, count, ppos, from: mask_str, available: len);
5075
5076	out_err:
5077	kfree(objp: mask_str);
5078
5079	return count;
5080	}
5081
5082	int tracing_set_cpumask(struct trace_array *tr,
5083	cpumask_var_t tracing_cpumask_new)
5084	{
5085	int cpu;
5086
5087	if (!tr)
5088	return -EINVAL;
5089
5090	local_irq_disable();
5091	arch_spin_lock(&tr->max_lock);
5092	for_each_tracing_cpu(cpu) {
5093	/*
5094	* Increase/decrease the disabled counter if we are
5095	* about to flip a bit in the cpumask:
5096	*/
5097	if (cpumask_test_cpu(cpu, cpumask: tr->tracing_cpumask) &&
5098	!cpumask_test_cpu(cpu, cpumask: tracing_cpumask_new)) {
5099	atomic_inc(v: &per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5100	ring_buffer_record_disable_cpu(buffer: tr->array_buffer.buffer, cpu);
5101	#ifdef CONFIG_TRACER_MAX_TRACE
5102	ring_buffer_record_disable_cpu(buffer: tr->max_buffer.buffer, cpu);
5103	#endif
5104	}
5105	if (!cpumask_test_cpu(cpu, cpumask: tr->tracing_cpumask) &&
5106	cpumask_test_cpu(cpu, cpumask: tracing_cpumask_new)) {
5107	atomic_dec(v: &per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);
5108	ring_buffer_record_enable_cpu(buffer: tr->array_buffer.buffer, cpu);
5109	#ifdef CONFIG_TRACER_MAX_TRACE
5110	ring_buffer_record_enable_cpu(buffer: tr->max_buffer.buffer, cpu);
5111	#endif
5112	}
5113	}
5114	arch_spin_unlock(&tr->max_lock);
5115	local_irq_enable();
5116
5117	cpumask_copy(dstp: tr->tracing_cpumask, srcp: tracing_cpumask_new);
5118
5119	return 0;
5120	}
5121
5122	static ssize_t
5123	tracing_cpumask_write(struct file *filp, const char __user *ubuf,
5124	size_t count, loff_t *ppos)
5125	{
5126	struct trace_array *tr = file_inode(f: filp)->i_private;
5127	cpumask_var_t tracing_cpumask_new;
5128	int err;
5129
5130	if (!zalloc_cpumask_var(mask: &tracing_cpumask_new, GFP_KERNEL))
5131	return -ENOMEM;
5132
5133	err = cpumask_parse_user(buf: ubuf, len: count, dstp: tracing_cpumask_new);
5134	if (err)
5135	goto err_free;
5136
5137	err = tracing_set_cpumask(tr, tracing_cpumask_new);
5138	if (err)
5139	goto err_free;
5140
5141	free_cpumask_var(mask: tracing_cpumask_new);
5142
5143	return count;
5144
5145	err_free:
5146	free_cpumask_var(mask: tracing_cpumask_new);
5147
5148	return err;
5149	}
5150
5151	static const struct file_operations tracing_cpumask_fops = {
5152	.open = tracing_open_generic_tr,
5153	.read = tracing_cpumask_read,
5154	.write = tracing_cpumask_write,
5155	.release = tracing_release_generic_tr,
5156	.llseek = generic_file_llseek,
5157	};
5158
5159	static int tracing_trace_options_show(struct seq_file *m, void *v)
5160	{
5161	struct tracer_opt *trace_opts;
5162	struct trace_array *tr = m->private;
5163	u32 tracer_flags;
5164	int i;
5165
5166	mutex_lock(&trace_types_lock);
5167	tracer_flags = tr->current_trace->flags->val;
5168	trace_opts = tr->current_trace->flags->opts;
5169
5170	for (i = 0; trace_options[i]; i++) {
5171	if (tr->trace_flags & (1 << i))
5172	seq_printf(m, fmt: "%s\n", trace_options[i]);
5173	else
5174	seq_printf(m, fmt: "no%s\n", trace_options[i]);
5175	}
5176
5177	for (i = 0; trace_opts[i].name; i++) {
5178	if (tracer_flags & trace_opts[i].bit)
5179	seq_printf(m, fmt: "%s\n", trace_opts[i].name);
5180	else
5181	seq_printf(m, fmt: "no%s\n", trace_opts[i].name);
5182	}
5183	mutex_unlock(lock: &trace_types_lock);
5184
5185	return 0;
5186	}
5187
5188	static int __set_tracer_option(struct trace_array *tr,
5189	struct tracer_flags *tracer_flags,
5190	struct tracer_opt *opts, int neg)
5191	{
5192	struct tracer *trace = tracer_flags->trace;
5193	int ret;
5194
5195	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
5196	if (ret)
5197	return ret;
5198
5199	if (neg)
5200	tracer_flags->val &= ~opts->bit;
5201	else
5202	tracer_flags->val |= opts->bit;
5203	return 0;
5204	}
5205
5206	/* Try to assign a tracer specific option */
5207	static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
5208	{
5209	struct tracer *trace = tr->current_trace;
5210	struct tracer_flags *tracer_flags = trace->flags;
5211	struct tracer_opt *opts = NULL;
5212	int i;
5213
5214	for (i = 0; tracer_flags->opts[i].name; i++) {
5215	opts = &tracer_flags->opts[i];
5216
5217	if (strcmp(cmp, opts->name) == 0)
5218	return __set_tracer_option(tr, tracer_flags: trace->flags, opts, neg);
5219	}
5220
5221	return -EINVAL;
5222	}
5223
5224	/* Some tracers require overwrite to stay enabled */
5225	int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
5226	{
5227	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
5228	return -1;
5229
5230	return 0;
5231	}
5232
5233	int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
5234	{
5235	if ((mask == TRACE_ITER_RECORD_TGID) ||
5236	(mask == TRACE_ITER_RECORD_CMD))
5237	lockdep_assert_held(&event_mutex);
5238
5239	/* do nothing if flag is already set */
5240	if (!!(tr->trace_flags & mask) == !!enabled)
5241	return 0;
5242
5243	/* Give the tracer a chance to approve the change */
5244	if (tr->current_trace->flag_changed)
5245	if (tr->current_trace->flag_changed(tr, mask, !!enabled))
5246	return -EINVAL;
5247
5248	if (enabled)
5249	tr->trace_flags |= mask;
5250	else
5251	tr->trace_flags &= ~mask;
5252
5253	if (mask == TRACE_ITER_RECORD_CMD)
5254	trace_event_enable_cmd_record(enable: enabled);
5255
5256	if (mask == TRACE_ITER_RECORD_TGID) {
5257
5258	if (trace_alloc_tgid_map() < 0) {
5259	tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
5260	return -ENOMEM;
5261	}
5262
5263	trace_event_enable_tgid_record(enable: enabled);
5264	}
5265
5266	if (mask == TRACE_ITER_EVENT_FORK)
5267	trace_event_follow_fork(tr, enable: enabled);
5268
5269	if (mask == TRACE_ITER_FUNC_FORK)
5270	ftrace_pid_follow_fork(tr, enable: enabled);
5271
5272	if (mask == TRACE_ITER_OVERWRITE) {
5273	ring_buffer_change_overwrite(buffer: tr->array_buffer.buffer, val: enabled);
5274	#ifdef CONFIG_TRACER_MAX_TRACE
5275	ring_buffer_change_overwrite(buffer: tr->max_buffer.buffer, val: enabled);
5276	#endif
5277	}
5278
5279	if (mask == TRACE_ITER_PRINTK) {
5280	trace_printk_start_stop_comm(enabled);
5281	trace_printk_control(enabled);
5282	}
5283
5284	return 0;
5285	}
5286
5287	int trace_set_options(struct trace_array *tr, char *option)
5288	{
5289	char *cmp;
5290	int neg = 0;
5291	int ret;
5292	size_t orig_len = strlen(option);
5293	int len;
5294
5295	cmp = strstrip(str: option);
5296
5297	len = str_has_prefix(str: cmp, prefix: "no");
5298	if (len)
5299	neg = 1;
5300
5301	cmp += len;
5302
5303	mutex_lock(&event_mutex);
5304	mutex_lock(&trace_types_lock);
5305
5306	ret = match_string(array: trace_options, n: -1, string: cmp);
5307	/* If no option could be set, test the specific tracer options */
5308	if (ret < 0)
5309	ret = set_tracer_option(tr, cmp, neg);
5310	else
5311	ret = set_tracer_flag(tr, mask: 1 << ret, enabled: !neg);
5312
5313	mutex_unlock(lock: &trace_types_lock);
5314	mutex_unlock(lock: &event_mutex);
5315
5316	/*
5317	* If the first trailing whitespace is replaced with '\0' by strstrip,
5318	* turn it back into a space.
5319	*/
5320	if (orig_len > strlen(option))
5321	option[strlen(option)] = ' ';
5322
5323	return ret;
5324	}
5325
5326	static void __init apply_trace_boot_options(void)
5327	{
5328	char *buf = trace_boot_options_buf;
5329	char *option;
5330
5331	while (true) {
5332	option = strsep(&buf, ",");
5333
5334	if (!option)
5335	break;
5336
5337	if (*option)
5338	trace_set_options(tr: &global_trace, option);
5339
5340	/* Put back the comma to allow this to be called again */
5341	if (buf)
5342	*(buf - 1) = ',';
5343	}
5344	}
5345
5346	static ssize_t
5347	tracing_trace_options_write(struct file *filp, const char __user *ubuf,
5348	size_t cnt, loff_t *ppos)
5349	{
5350	struct seq_file *m = filp->private_data;
5351	struct trace_array *tr = m->private;
5352	char buf[64];
5353	int ret;
5354
5355	if (cnt >= sizeof(buf))
5356	return -EINVAL;
5357
5358	if (copy_from_user(to: buf, from: ubuf, n: cnt))
5359	return -EFAULT;
5360
5361	buf[cnt] = 0;
5362
5363	ret = trace_set_options(tr, option: buf);
5364	if (ret < 0)
5365	return ret;
5366
5367	*ppos += cnt;
5368
5369	return cnt;
5370	}
5371
5372	static int tracing_trace_options_open(struct inode *inode, struct file *file)
5373	{
5374	struct trace_array *tr = inode->i_private;
5375	int ret;
5376
5377	ret = tracing_check_open_get_tr(tr);
5378	if (ret)
5379	return ret;
5380
5381	ret = single_open(file, tracing_trace_options_show, inode->i_private);
5382	if (ret < 0)
5383	trace_array_put(tr);
5384
5385	return ret;
5386	}
5387
5388	static const struct file_operations tracing_iter_fops = {
5389	.open = tracing_trace_options_open,
5390	.read = seq_read,
5391	.llseek = seq_lseek,
5392	.release = tracing_single_release_tr,
5393	.write = tracing_trace_options_write,
5394	};
5395
5396	static const char readme_msg[] =
5397	"tracing mini-HOWTO:\n\n"
5398	"# echo 0 > tracing_on : quick way to disable tracing\n"
5399	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5400	" Important files:\n"
5401	" trace\t\t\t- The static contents of the buffer\n"
5402	"\t\t\t To clear the buffer write into this file: echo > trace\n"
5403	" trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5404	" current_tracer\t- function and latency tracers\n"
5405	" available_tracers\t- list of configured tracers for current_tracer\n"
5406	" error_log\t- error log for failed commands (that support it)\n"
5407	" buffer_size_kb\t- view and modify size of per cpu buffer\n"
5408	" buffer_total_size_kb - view total size of all cpu buffers\n\n"
5409	" trace_clock\t\t- change the clock used to order events\n"
5410	" local: Per cpu clock but may not be synced across CPUs\n"
5411	" global: Synced across CPUs but slows tracing down.\n"
5412	" counter: Not a clock, but just an increment\n"
5413	" uptime: Jiffy counter from time of boot\n"
5414	" perf: Same clock that perf events use\n"
5415	#ifdef CONFIG_X86_64
5416	" x86-tsc: TSC cycle counter\n"
5417	#endif
5418	"\n timestamp_mode\t- view the mode used to timestamp events\n"
5419	" delta: Delta difference against a buffer-wide timestamp\n"
5420	" absolute: Absolute (standalone) timestamp\n"
5421	"\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5422	"\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5423	" tracing_cpumask\t- Limit which CPUs to trace\n"
5424	" instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5425	"\t\t\t Remove sub-buffer with rmdir\n"
5426	" trace_options\t\t- Set format or modify how tracing happens\n"
5427	"\t\t\t Disable an option by prefixing 'no' to the\n"
5428	"\t\t\t option name\n"
5429	" saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5430	#ifdef CONFIG_DYNAMIC_FTRACE
5431	"\n available_filter_functions - list of functions that can be filtered on\n"
5432	" set_ftrace_filter\t- echo function name in here to only trace these\n"
5433	"\t\t\t functions\n"
5434	"\t accepts: func_full_name or glob-matching-pattern\n"
5435	"\t modules: Can select a group via module\n"
5436	"\t Format: :mod:<module-name>\n"
5437	"\t example: echo :mod:ext3 > set_ftrace_filter\n"
5438	"\t triggers: a command to perform when function is hit\n"
5439	"\t Format: <function>:<trigger>[:count]\n"
5440	"\t trigger: traceon, traceoff\n"
5441	"\t\t enable_event:<system>:<event>\n"
5442	"\t\t disable_event:<system>:<event>\n"
5443	#ifdef CONFIG_STACKTRACE
5444	"\t\t stacktrace\n"
5445	#endif
5446	#ifdef CONFIG_TRACER_SNAPSHOT
5447	"\t\t snapshot\n"
5448	#endif
5449	"\t\t dump\n"
5450	"\t\t cpudump\n"
5451	"\t example: echo do_fault:traceoff > set_ftrace_filter\n"
5452	"\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
5453	"\t The first one will disable tracing every time do_fault is hit\n"
5454	"\t The second will disable tracing at most 3 times when do_trap is hit\n"
5455	"\t The first time do trap is hit and it disables tracing, the\n"
5456	"\t counter will decrement to 2. If tracing is already disabled,\n"
5457	"\t the counter will not decrement. It only decrements when the\n"
5458	"\t trigger did work\n"
5459	"\t To remove trigger without count:\n"
5460	"\t echo '!<function>:<trigger> > set_ftrace_filter\n"
5461	"\t To remove trigger with a count:\n"
5462	"\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
5463	" set_ftrace_notrace\t- echo function name in here to never trace.\n"
5464	"\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
5465	"\t modules: Can select a group via module command :mod:\n"
5466	"\t Does not accept triggers\n"
5467	#endif /* CONFIG_DYNAMIC_FTRACE */
5468	#ifdef CONFIG_FUNCTION_TRACER
5469	" set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5470	"\t\t (function)\n"
5471	" set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5472	"\t\t (function)\n"
5473	#endif
5474	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
5475	" set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5476	" set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5477	" max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5478	#endif
5479	#ifdef CONFIG_TRACER_SNAPSHOT
5480	"\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
5481	"\t\t\t snapshot buffer. Read the contents for more\n"
5482	"\t\t\t information\n"
5483	#endif
5484	#ifdef CONFIG_STACK_TRACER
5485	" stack_trace\t\t- Shows the max stack trace when active\n"
5486	" stack_max_size\t- Shows current max stack size that was traced\n"
5487	"\t\t\t Write into this file to reset the max size (trigger a\n"
5488	"\t\t\t new trace)\n"
5489	#ifdef CONFIG_DYNAMIC_FTRACE
5490	" stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5491	"\t\t\t traces\n"
5492	#endif
5493	#endif /* CONFIG_STACK_TRACER */
5494	#ifdef CONFIG_DYNAMIC_EVENTS
5495	" dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5496	"\t\t\t Write into this file to define/undefine new trace events.\n"
5497	#endif
5498	#ifdef CONFIG_KPROBE_EVENTS
5499	" kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5500	"\t\t\t Write into this file to define/undefine new trace events.\n"
5501	#endif
5502	#ifdef CONFIG_UPROBE_EVENTS
5503	" uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5504	"\t\t\t Write into this file to define/undefine new trace events.\n"
5505	#endif
5506	#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \
5507	defined(CONFIG_FPROBE_EVENTS)
5508	"\t accepts: event-definitions (one definition per line)\n"
5509	#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
5510	"\t Format: p[:[<group>/][<event>]] <place> [<args>]\n"
5511	"\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n"
5512	#endif
5513	#ifdef CONFIG_FPROBE_EVENTS
5514	"\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
5515	"\t t[:[<group>/][<event>]] <tracepoint> [<args>]\n"
5516	#endif
5517	#ifdef CONFIG_HIST_TRIGGERS
5518	"\t s:[synthetic/]<event> <field> [<field>]\n"
5519	#endif
5520	"\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
5521	"\t -:[<group>/][<event>]\n"
5522	#ifdef CONFIG_KPROBE_EVENTS
5523	"\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
5524	"place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n"
5525	#endif
5526	#ifdef CONFIG_UPROBE_EVENTS
5527	" place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n"
5528	#endif
5529	"\t args: <name>=fetcharg[:type]\n"
5530	"\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5531	#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
5532	"\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
5533	#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
5534	"\t <argname>[->field[->field|.field...]],\n"
5535	#endif
5536	#else
5537	"\t $stack<index>, $stack, $retval, $comm,\n"
5538	#endif
5539	"\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5540	"\t kernel return probes support: $retval, $arg<N>, $comm\n"
5541	"\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
5542	"\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5543	"\t symstr, <type>\\[<array-size>\\]\n"
5544	#ifdef CONFIG_HIST_TRIGGERS
5545	"\t field: <stype> <name>;\n"
5546	"\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n"
5547	"\t [unsigned] char/int/long\n"
5548	#endif
5549	"\t efield: For event probes ('e' types), the field is on of the fields\n"
5550	"\t of the <attached-group>/<attached-event>.\n"
5551	#endif
5552	" events/\t\t- Directory containing all trace event subsystems:\n"
5553	" enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5554	" events/<system>/\t- Directory containing all trace events for <system>:\n"
5555	" enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5556	"\t\t\t events\n"
5557	" filter\t\t- If set, only events passing filter are traced\n"
5558	" events/<system>/<event>/\t- Directory containing control files for\n"
5559	"\t\t\t <event>:\n"
5560	" enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5561	" filter\t\t- If set, only events passing filter are traced\n"
5562	" trigger\t\t- If set, a command to perform when event is hit\n"
5563	"\t Format: <trigger>[:count][if <filter>]\n"
5564	"\t trigger: traceon, traceoff\n"
5565	"\t enable_event:<system>:<event>\n"
5566	"\t disable_event:<system>:<event>\n"
5567	#ifdef CONFIG_HIST_TRIGGERS
5568	"\t enable_hist:<system>:<event>\n"
5569	"\t disable_hist:<system>:<event>\n"
5570	#endif
5571	#ifdef CONFIG_STACKTRACE
5572	"\t\t stacktrace\n"
5573	#endif
5574	#ifdef CONFIG_TRACER_SNAPSHOT
5575	"\t\t snapshot\n"
5576	#endif
5577	#ifdef CONFIG_HIST_TRIGGERS
5578	"\t\t hist (see below)\n"
5579	#endif
5580	"\t example: echo traceoff > events/block/block_unplug/trigger\n"
5581	"\t echo traceoff:3 > events/block/block_unplug/trigger\n"
5582	"\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
5583	"\t events/block/block_unplug/trigger\n"
5584	"\t The first disables tracing every time block_unplug is hit.\n"
5585	"\t The second disables tracing the first 3 times block_unplug is hit.\n"
5586	"\t The third enables the kmalloc event the first 3 times block_unplug\n"
5587	"\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
5588	"\t Like function triggers, the counter is only decremented if it\n"
5589	"\t enabled or disabled tracing.\n"
5590	"\t To remove a trigger without a count:\n"
5591	"\t echo '!<trigger> > <system>/<event>/trigger\n"
5592	"\t To remove a trigger with a count:\n"
5593	"\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
5594	"\t Filters can be ignored when removing a trigger.\n"
5595	#ifdef CONFIG_HIST_TRIGGERS
5596	" hist trigger\t- If set, event hits are aggregated into a hash table\n"
5597	"\t Format: hist:keys=<field1[,field2,...]>\n"
5598	"\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n"
5599	"\t [:values=<field1[,field2,...]>]\n"
5600	"\t [:sort=<field1[,field2,...]>]\n"
5601	"\t [:size=#entries]\n"
5602	"\t [:pause][:continue][:clear]\n"
5603	"\t [:name=histname1]\n"
5604	"\t [:nohitcount]\n"
5605	"\t [:<handler>.<action>]\n"
5606	"\t [if <filter>]\n\n"
5607	"\t Note, special fields can be used as well:\n"
5608	"\t common_timestamp - to record current timestamp\n"
5609	"\t common_cpu - to record the CPU the event happened on\n"
5610	"\n"
5611	"\t A hist trigger variable can be:\n"
5612	"\t - a reference to a field e.g. x=current_timestamp,\n"
5613	"\t - a reference to another variable e.g. y=$x,\n"
5614	"\t - a numeric literal: e.g. ms_per_sec=1000,\n"
5615	"\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5616	"\n"
5617	"\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5618	"\t multiplication(*) and division(/) operators. An operand can be either a\n"
5619	"\t variable reference, field or numeric literal.\n"
5620	"\n"
5621	"\t When a matching event is hit, an entry is added to a hash\n"
5622	"\t table using the key(s) and value(s) named, and the value of a\n"
5623	"\t sum called 'hitcount' is incremented. Keys and values\n"
5624	"\t correspond to fields in the event's format description. Keys\n"
5625	"\t can be any field, or the special string 'common_stacktrace'.\n"
5626	"\t Compound keys consisting of up to two fields can be specified\n"
5627	"\t by the 'keys' keyword. Values must correspond to numeric\n"
5628	"\t fields. Sort keys consisting of up to two fields can be\n"
5629	"\t specified using the 'sort' keyword. The sort direction can\n"
5630	"\t be modified by appending '.descending' or '.ascending' to a\n"
5631	"\t sort field. The 'size' parameter can be used to specify more\n"
5632	"\t or fewer than the default 2048 entries for the hashtable size.\n"
5633	"\t If a hist trigger is given a name using the 'name' parameter,\n"
5634	"\t its histogram data will be shared with other triggers of the\n"
5635	"\t same name, and trigger hits will update this common data.\n\n"
5636	"\t Reading the 'hist' file for the event will dump the hash\n"
5637	"\t table in its entirety to stdout. If there are multiple hist\n"
5638	"\t triggers attached to an event, there will be a table for each\n"
5639	"\t trigger in the output. The table displayed for a named\n"
5640	"\t trigger will be the same as any other instance having the\n"
5641	"\t same name. The default format used to display a given field\n"
5642	"\t can be modified by appending any of the following modifiers\n"
5643	"\t to the field name, as applicable:\n\n"
5644	"\t .hex display a number as a hex value\n"
5645	"\t .sym display an address as a symbol\n"
5646	"\t .sym-offset display an address as a symbol and offset\n"
5647	"\t .execname display a common_pid as a program name\n"
5648	"\t .syscall display a syscall id as a syscall name\n"
5649	"\t .log2 display log2 value rather than raw number\n"
5650	"\t .buckets=size display values in groups of size rather than raw number\n"
5651	"\t .usecs display a common_timestamp in microseconds\n"
5652	"\t .percent display a number of percentage value\n"
5653	"\t .graph display a bar-graph of a value\n\n"
5654	"\t The 'pause' parameter can be used to pause an existing hist\n"
5655	"\t trigger or to start a hist trigger but not log any events\n"
5656	"\t until told to do so. 'continue' can be used to start or\n"
5657	"\t restart a paused hist trigger.\n\n"
5658	"\t The 'clear' parameter will clear the contents of a running\n"
5659	"\t hist trigger and leave its current paused/active state\n"
5660	"\t unchanged.\n\n"
5661	"\t The 'nohitcount' (or NOHC) parameter will suppress display of\n"
5662	"\t raw hitcount in the histogram.\n\n"
5663	"\t The enable_hist and disable_hist triggers can be used to\n"
5664	"\t have one event conditionally start and stop another event's\n"
5665	"\t already-attached hist trigger. The syntax is analogous to\n"
5666	"\t the enable_event and disable_event triggers.\n\n"
5667	"\t Hist trigger handlers and actions are executed whenever a\n"
5668	"\t a histogram entry is added or updated. They take the form:\n\n"
5669	"\t <handler>.<action>\n\n"
5670	"\t The available handlers are:\n\n"
5671	"\t onmatch(matching.event) - invoke on addition or update\n"
5672	"\t onmax(var) - invoke if var exceeds current max\n"
5673	"\t onchange(var) - invoke action if var changes\n\n"
5674	"\t The available actions are:\n\n"
5675	"\t trace(<synthetic_event>,param list) - generate synthetic event\n"
5676	"\t save(field,...) - save current event fields\n"
5677	#ifdef CONFIG_TRACER_SNAPSHOT
5678	"\t snapshot() - snapshot the trace buffer\n\n"
5679	#endif
5680	#ifdef CONFIG_SYNTH_EVENTS
5681	" events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5682	"\t Write into this file to define/undefine new synthetic events.\n"
5683	"\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
5684	#endif
5685	#endif
5686	;
5687
5688	static ssize_t
5689	tracing_readme_read(struct file *filp, char __user *ubuf,
5690	size_t cnt, loff_t *ppos)
5691	{
5692	return simple_read_from_buffer(to: ubuf, count: cnt, ppos,
5693	from: readme_msg, strlen(readme_msg));
5694	}
5695
5696	static const struct file_operations tracing_readme_fops = {
5697	.open = tracing_open_generic,
5698	.read = tracing_readme_read,
5699	.llseek = generic_file_llseek,
5700	};
5701
5702	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
5703	static union trace_eval_map_item *
5704	update_eval_map(union trace_eval_map_item *ptr)
5705	{
5706	if (!ptr->map.eval_string) {
5707	if (ptr->tail.next) {
5708	ptr = ptr->tail.next;
5709	/* Set ptr to the next real item (skip head) */
5710	ptr++;
5711	} else
5712	return NULL;
5713	}
5714	return ptr;
5715	}
5716
5717	static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5718	{
5719	union trace_eval_map_item *ptr = v;
5720
5721	/*
5722	* Paranoid! If ptr points to end, we don't want to increment past it.
5723	* This really should never happen.
5724	*/
5725	(*pos)++;
5726	ptr = update_eval_map(ptr);
5727	if (WARN_ON_ONCE(!ptr))
5728	return NULL;
5729
5730	ptr++;
5731	ptr = update_eval_map(ptr);
5732
5733	return ptr;
5734	}
5735
5736	static void *eval_map_start(struct seq_file *m, loff_t *pos)
5737	{
5738	union trace_eval_map_item *v;
5739	loff_t l = 0;
5740
5741	mutex_lock(&trace_eval_mutex);
5742
5743	v = trace_eval_maps;
5744	if (v)
5745	v++;
5746
5747	while (v && l < *pos) {
5748	v = eval_map_next(m, v, pos: &l);
5749	}
5750
5751	return v;
5752	}
5753
5754	static void eval_map_stop(struct seq_file *m, void *v)
5755	{
5756	mutex_unlock(lock: &trace_eval_mutex);
5757	}
5758
5759	static int eval_map_show(struct seq_file *m, void *v)
5760	{
5761	union trace_eval_map_item *ptr = v;
5762
5763	seq_printf(m, fmt: "%s %ld (%s)\n",
5764	ptr->map.eval_string, ptr->map.eval_value,
5765	ptr->map.system);
5766
5767	return 0;
5768	}
5769
5770	static const struct seq_operations tracing_eval_map_seq_ops = {
5771	.start = eval_map_start,
5772	.next = eval_map_next,
5773	.stop = eval_map_stop,
5774	.show = eval_map_show,
5775	};
5776
5777	static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5778	{
5779	int ret;
5780
5781	ret = tracing_check_open_get_tr(NULL);
5782	if (ret)
5783	return ret;
5784
5785	return seq_open(filp, &tracing_eval_map_seq_ops);
5786	}
5787
5788	static const struct file_operations tracing_eval_map_fops = {
5789	.open = tracing_eval_map_open,
5790	.read = seq_read,
5791	.llseek = seq_lseek,
5792	.release = seq_release,
5793	};
5794
5795	static inline union trace_eval_map_item *
5796	trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5797	{
5798	/* Return tail of array given the head */
5799	return ptr + ptr->head.length + 1;
5800	}
5801
5802	static void
5803	trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5804	int len)
5805	{
5806	struct trace_eval_map **stop;
5807	struct trace_eval_map **map;
5808	union trace_eval_map_item *map_array;
5809	union trace_eval_map_item *ptr;
5810
5811	stop = start + len;
5812
5813	/*
5814	* The trace_eval_maps contains the map plus a head and tail item,
5815	* where the head holds the module and length of array, and the
5816	* tail holds a pointer to the next list.
5817	*/
5818	map_array = kmalloc_array(n: len + 2, size: sizeof(*map_array), GFP_KERNEL);
5819	if (!map_array) {
5820	pr_warn("Unable to allocate trace eval mapping\n");
5821	return;
5822	}
5823
5824	mutex_lock(&trace_eval_mutex);
5825
5826	if (!trace_eval_maps)
5827	trace_eval_maps = map_array;
5828	else {
5829	ptr = trace_eval_maps;
5830	for (;;) {
5831	ptr = trace_eval_jmp_to_tail(ptr);
5832	if (!ptr->tail.next)
5833	break;
5834	ptr = ptr->tail.next;
5835
5836	}
5837	ptr->tail.next = map_array;
5838	}
5839	map_array->head.mod = mod;
5840	map_array->head.length = len;
5841	map_array++;
5842
5843	for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5844	map_array->map = **map;
5845	map_array++;
5846	}
5847	memset(map_array, 0, sizeof(*map_array));
5848
5849	mutex_unlock(lock: &trace_eval_mutex);
5850	}
5851
5852	static void trace_create_eval_file(struct dentry *d_tracer)
5853	{
5854	trace_create_file(name: "eval_map", TRACE_MODE_READ, parent: d_tracer,
5855	NULL, fops: &tracing_eval_map_fops);
5856	}
5857
5858	#else /* CONFIG_TRACE_EVAL_MAP_FILE */
5859	static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5860	static inline void trace_insert_eval_map_file(struct module *mod,
5861	struct trace_eval_map **start, int len) { }
5862	#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5863
5864	static void trace_insert_eval_map(struct module *mod,
5865	struct trace_eval_map **start, int len)
5866	{
5867	struct trace_eval_map **map;
5868
5869	if (len <= 0)
5870	return;
5871
5872	map = start;
5873
5874	trace_event_eval_update(map, len);
5875
5876	trace_insert_eval_map_file(mod, start, len);
5877	}
5878
5879	static ssize_t
5880	tracing_set_trace_read(struct file *filp, char __user *ubuf,
5881	size_t cnt, loff_t *ppos)
5882	{
5883	struct trace_array *tr = filp->private_data;
5884	char buf[MAX_TRACER_SIZE+2];
5885	int r;
5886
5887	mutex_lock(&trace_types_lock);
5888	r = sprintf(buf, fmt: "%s\n", tr->current_trace->name);
5889	mutex_unlock(lock: &trace_types_lock);
5890
5891	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
5892	}
5893
5894	int tracer_init(struct tracer *t, struct trace_array *tr)
5895	{
5896	tracing_reset_online_cpus(buf: &tr->array_buffer);
5897	return t->init(tr);
5898	}
5899
5900	static void set_buffer_entries(struct array_buffer *buf, unsigned long val)
5901	{
5902	int cpu;
5903
5904	for_each_tracing_cpu(cpu)
5905	per_cpu_ptr(buf->data, cpu)->entries = val;
5906	}
5907
5908	static void update_buffer_entries(struct array_buffer *buf, int cpu)
5909	{
5910	if (cpu == RING_BUFFER_ALL_CPUS) {
5911	set_buffer_entries(buf, val: ring_buffer_size(buffer: buf->buffer, cpu: 0));
5912	} else {
5913	per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buffer: buf->buffer, cpu);
5914	}
5915	}
5916
5917	#ifdef CONFIG_TRACER_MAX_TRACE
5918	/* resize @tr's buffer to the size of @size_tr's entries */
5919	static int resize_buffer_duplicate_size(struct array_buffer *trace_buf,
5920	struct array_buffer *size_buf, int cpu_id)
5921	{
5922	int cpu, ret = 0;
5923
5924	if (cpu_id == RING_BUFFER_ALL_CPUS) {
5925	for_each_tracing_cpu(cpu) {
5926	ret = ring_buffer_resize(buffer: trace_buf->buffer,
5927	per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5928	if (ret < 0)
5929	break;
5930	per_cpu_ptr(trace_buf->data, cpu)->entries =
5931	per_cpu_ptr(size_buf->data, cpu)->entries;
5932	}
5933	} else {
5934	ret = ring_buffer_resize(buffer: trace_buf->buffer,
5935	per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu: cpu_id);
5936	if (ret == 0)
5937	per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5938	per_cpu_ptr(size_buf->data, cpu_id)->entries;
5939	}
5940
5941	return ret;
5942	}
5943	#endif /* CONFIG_TRACER_MAX_TRACE */
5944
5945	static int __tracing_resize_ring_buffer(struct trace_array *tr,
5946	unsigned long size, int cpu)
5947	{
5948	int ret;
5949
5950	/*
5951	* If kernel or user changes the size of the ring buffer
5952	* we use the size that was given, and we can forget about
5953	* expanding it later.
5954	*/
5955	trace_set_ring_buffer_expanded(tr);
5956
5957	/* May be called before buffers are initialized */
5958	if (!tr->array_buffer.buffer)
5959	return 0;
5960
5961	/* Do not allow tracing while resizing ring buffer */
5962	tracing_stop_tr(tr);
5963
5964	ret = ring_buffer_resize(buffer: tr->array_buffer.buffer, size, cpu);
5965	if (ret < 0)
5966	goto out_start;
5967
5968	#ifdef CONFIG_TRACER_MAX_TRACE
5969	if (!tr->allocated_snapshot)
5970	goto out;
5971
5972	ret = ring_buffer_resize(buffer: tr->max_buffer.buffer, size, cpu);
5973	if (ret < 0) {
5974	int r = resize_buffer_duplicate_size(trace_buf: &tr->array_buffer,
5975	size_buf: &tr->array_buffer, cpu_id: cpu);
5976	if (r < 0) {
5977	/*
5978	* AARGH! We are left with different
5979	* size max buffer!!!!
5980	* The max buffer is our "snapshot" buffer.
5981	* When a tracer needs a snapshot (one of the
5982	* latency tracers), it swaps the max buffer
5983	* with the saved snap shot. We succeeded to
5984	* update the size of the main buffer, but failed to
5985	* update the size of the max buffer. But when we tried
5986	* to reset the main buffer to the original size, we
5987	* failed there too. This is very unlikely to
5988	* happen, but if it does, warn and kill all
5989	* tracing.
5990	*/
5991	WARN_ON(1);
5992	tracing_disabled = 1;
5993	}
5994	goto out_start;
5995	}
5996
5997	update_buffer_entries(buf: &tr->max_buffer, cpu);
5998
5999	out:
6000	#endif /* CONFIG_TRACER_MAX_TRACE */
6001
6002	update_buffer_entries(buf: &tr->array_buffer, cpu);
6003	out_start:
6004	tracing_start_tr(tr);
6005	return ret;
6006	}
6007
6008	ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
6009	unsigned long size, int cpu_id)
6010	{
6011	int ret;
6012
6013	mutex_lock(&trace_types_lock);
6014
6015	if (cpu_id != RING_BUFFER_ALL_CPUS) {
6016	/* make sure, this cpu is enabled in the mask */
6017	if (!cpumask_test_cpu(cpu: cpu_id, cpumask: tracing_buffer_mask)) {
6018	ret = -EINVAL;
6019	goto out;
6020	}
6021	}
6022
6023	ret = __tracing_resize_ring_buffer(tr, size, cpu: cpu_id);
6024	if (ret < 0)
6025	ret = -ENOMEM;
6026
6027	out:
6028	mutex_unlock(lock: &trace_types_lock);
6029
6030	return ret;
6031	}
6032
6033
6034	/**
6035	* tracing_update_buffers - used by tracing facility to expand ring buffers
6036	* @tr: The tracing instance
6037	*
6038	* To save on memory when the tracing is never used on a system with it
6039	* configured in. The ring buffers are set to a minimum size. But once
6040	* a user starts to use the tracing facility, then they need to grow
6041	* to their default size.
6042	*
6043	* This function is to be called when a tracer is about to be used.
6044	*/
6045	int tracing_update_buffers(struct trace_array *tr)
6046	{
6047	int ret = 0;
6048
6049	mutex_lock(&trace_types_lock);
6050	if (!tr->ring_buffer_expanded)
6051	ret = __tracing_resize_ring_buffer(tr, size: trace_buf_size,
6052	RING_BUFFER_ALL_CPUS);
6053	mutex_unlock(lock: &trace_types_lock);
6054
6055	return ret;
6056	}
6057
6058	struct trace_option_dentry;
6059
6060	static void
6061	create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
6062
6063	/*
6064	* Used to clear out the tracer before deletion of an instance.
6065	* Must have trace_types_lock held.
6066	*/
6067	static void tracing_set_nop(struct trace_array *tr)
6068	{
6069	if (tr->current_trace == &nop_trace)
6070	return;
6071
6072	tr->current_trace->enabled--;
6073
6074	if (tr->current_trace->reset)
6075	tr->current_trace->reset(tr);
6076
6077	tr->current_trace = &nop_trace;
6078	}
6079
6080	static bool tracer_options_updated;
6081
6082	static void add_tracer_options(struct trace_array *tr, struct tracer *t)
6083	{
6084	/* Only enable if the directory has been created already. */
6085	if (!tr->dir)
6086	return;
6087
6088	/* Only create trace option files after update_tracer_options finish */
6089	if (!tracer_options_updated)
6090	return;
6091
6092	create_trace_option_files(tr, tracer: t);
6093	}
6094
6095	int tracing_set_tracer(struct trace_array *tr, const char *buf)
6096	{
6097	struct tracer *t;
6098	#ifdef CONFIG_TRACER_MAX_TRACE
6099	bool had_max_tr;
6100	#endif
6101	int ret = 0;
6102
6103	mutex_lock(&trace_types_lock);
6104
6105	if (!tr->ring_buffer_expanded) {
6106	ret = __tracing_resize_ring_buffer(tr, size: trace_buf_size,
6107	RING_BUFFER_ALL_CPUS);
6108	if (ret < 0)
6109	goto out;
6110	ret = 0;
6111	}
6112
6113	for (t = trace_types; t; t = t->next) {
6114	if (strcmp(t->name, buf) == 0)
6115	break;
6116	}
6117	if (!t) {
6118	ret = -EINVAL;
6119	goto out;
6120	}
6121	if (t == tr->current_trace)
6122	goto out;
6123
6124	#ifdef CONFIG_TRACER_SNAPSHOT
6125	if (t->use_max_tr) {
6126	local_irq_disable();
6127	arch_spin_lock(&tr->max_lock);
6128	if (tr->cond_snapshot)
6129	ret = -EBUSY;
6130	arch_spin_unlock(&tr->max_lock);
6131	local_irq_enable();
6132	if (ret)
6133	goto out;
6134	}
6135	#endif
6136	/* Some tracers won't work on kernel command line */
6137	if (system_state < SYSTEM_RUNNING && t->noboot) {
6138	pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
6139	t->name);
6140	goto out;
6141	}
6142
6143	/* Some tracers are only allowed for the top level buffer */
6144	if (!trace_ok_for_array(t, tr)) {
6145	ret = -EINVAL;
6146	goto out;
6147	}
6148
6149	/* If trace pipe files are being read, we can't change the tracer */
6150	if (tr->trace_ref) {
6151	ret = -EBUSY;
6152	goto out;
6153	}
6154
6155	trace_branch_disable();
6156
6157	tr->current_trace->enabled--;
6158
6159	if (tr->current_trace->reset)
6160	tr->current_trace->reset(tr);
6161
6162	#ifdef CONFIG_TRACER_MAX_TRACE
6163	had_max_tr = tr->current_trace->use_max_tr;
6164
6165	/* Current trace needs to be nop_trace before synchronize_rcu */
6166	tr->current_trace = &nop_trace;
6167
6168	if (had_max_tr && !t->use_max_tr) {
6169	/*
6170	* We need to make sure that the update_max_tr sees that
6171	* current_trace changed to nop_trace to keep it from
6172	* swapping the buffers after we resize it.
6173	* The update_max_tr is called from interrupts disabled
6174	* so a synchronized_sched() is sufficient.
6175	*/
6176	synchronize_rcu();
6177	free_snapshot(tr);
6178	tracing_disarm_snapshot(tr);
6179	}
6180
6181	if (!had_max_tr && t->use_max_tr) {
6182	ret = tracing_arm_snapshot_locked(tr);
6183	if (ret)
6184	goto out;
6185	}
6186	#else
6187	tr->current_trace = &nop_trace;
6188	#endif
6189
6190	if (t->init) {
6191	ret = tracer_init(t, tr);
6192	if (ret) {
6193	#ifdef CONFIG_TRACER_MAX_TRACE
6194	if (t->use_max_tr)
6195	tracing_disarm_snapshot(tr);
6196	#endif
6197	goto out;
6198	}
6199	}
6200
6201	tr->current_trace = t;
6202	tr->current_trace->enabled++;
6203	trace_branch_enable(tr);
6204	out:
6205	mutex_unlock(lock: &trace_types_lock);
6206
6207	return ret;
6208	}
6209
6210	static ssize_t
6211	tracing_set_trace_write(struct file *filp, const char __user *ubuf,
6212	size_t cnt, loff_t *ppos)
6213	{
6214	struct trace_array *tr = filp->private_data;
6215	char buf[MAX_TRACER_SIZE+1];
6216	char *name;
6217	size_t ret;
6218	int err;
6219
6220	ret = cnt;
6221
6222	if (cnt > MAX_TRACER_SIZE)
6223	cnt = MAX_TRACER_SIZE;
6224
6225	if (copy_from_user(to: buf, from: ubuf, n: cnt))
6226	return -EFAULT;
6227
6228	buf[cnt] = 0;
6229
6230	name = strim(buf);
6231
6232	err = tracing_set_tracer(tr, buf: name);
6233	if (err)
6234	return err;
6235
6236	*ppos += ret;
6237
6238	return ret;
6239	}
6240
6241	static ssize_t
6242	tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
6243	size_t cnt, loff_t *ppos)
6244	{
6245	char buf[64];
6246	int r;
6247
6248	r = snprintf(buf, size: sizeof(buf), fmt: "%ld\n",
6249	*ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(nsecs: *ptr));
6250	if (r > sizeof(buf))
6251	r = sizeof(buf);
6252	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
6253	}
6254
6255	static ssize_t
6256	tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
6257	size_t cnt, loff_t *ppos)
6258	{
6259	unsigned long val;
6260	int ret;
6261
6262	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
6263	if (ret)
6264	return ret;
6265
6266	*ptr = val * 1000;
6267
6268	return cnt;
6269	}
6270
6271	static ssize_t
6272	tracing_thresh_read(struct file *filp, char __user *ubuf,
6273	size_t cnt, loff_t *ppos)
6274	{
6275	return tracing_nsecs_read(ptr: &tracing_thresh, ubuf, cnt, ppos);
6276	}
6277
6278	static ssize_t
6279	tracing_thresh_write(struct file *filp, const char __user *ubuf,
6280	size_t cnt, loff_t *ppos)
6281	{
6282	struct trace_array *tr = filp->private_data;
6283	int ret;
6284
6285	mutex_lock(&trace_types_lock);
6286	ret = tracing_nsecs_write(ptr: &tracing_thresh, ubuf, cnt, ppos);
6287	if (ret < 0)
6288	goto out;
6289
6290	if (tr->current_trace->update_thresh) {
6291	ret = tr->current_trace->update_thresh(tr);
6292	if (ret < 0)
6293	goto out;
6294	}
6295
6296	ret = cnt;
6297	out:
6298	mutex_unlock(lock: &trace_types_lock);
6299
6300	return ret;
6301	}
6302
6303	#ifdef CONFIG_TRACER_MAX_TRACE
6304
6305	static ssize_t
6306	tracing_max_lat_read(struct file *filp, char __user *ubuf,
6307	size_t cnt, loff_t *ppos)
6308	{
6309	struct trace_array *tr = filp->private_data;
6310
6311	return tracing_nsecs_read(ptr: &tr->max_latency, ubuf, cnt, ppos);
6312	}
6313
6314	static ssize_t
6315	tracing_max_lat_write(struct file *filp, const char __user *ubuf,
6316	size_t cnt, loff_t *ppos)
6317	{
6318	struct trace_array *tr = filp->private_data;
6319
6320	return tracing_nsecs_write(ptr: &tr->max_latency, ubuf, cnt, ppos);
6321	}
6322
6323	#endif
6324
6325	static int open_pipe_on_cpu(struct trace_array *tr, int cpu)
6326	{
6327	if (cpu == RING_BUFFER_ALL_CPUS) {
6328	if (cpumask_empty(srcp: tr->pipe_cpumask)) {
6329	cpumask_setall(dstp: tr->pipe_cpumask);
6330	return 0;
6331	}
6332	} else if (!cpumask_test_cpu(cpu, cpumask: tr->pipe_cpumask)) {
6333	cpumask_set_cpu(cpu, dstp: tr->pipe_cpumask);
6334	return 0;
6335	}
6336	return -EBUSY;
6337	}
6338
6339	static void close_pipe_on_cpu(struct trace_array *tr, int cpu)
6340	{
6341	if (cpu == RING_BUFFER_ALL_CPUS) {
6342	WARN_ON(!cpumask_full(tr->pipe_cpumask));
6343	cpumask_clear(dstp: tr->pipe_cpumask);
6344	} else {
6345	WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask));
6346	cpumask_clear_cpu(cpu, dstp: tr->pipe_cpumask);
6347	}
6348	}
6349
6350	static int tracing_open_pipe(struct inode *inode, struct file *filp)
6351	{
6352	struct trace_array *tr = inode->i_private;
6353	struct trace_iterator *iter;
6354	int cpu;
6355	int ret;
6356
6357	ret = tracing_check_open_get_tr(tr);
6358	if (ret)
6359	return ret;
6360
6361	mutex_lock(&trace_types_lock);
6362	cpu = tracing_get_cpu(inode);
6363	ret = open_pipe_on_cpu(tr, cpu);
6364	if (ret)
6365	goto fail_pipe_on_cpu;
6366
6367	/* create a buffer to store the information to pass to userspace */
6368	iter = kzalloc(size: sizeof(*iter), GFP_KERNEL);
6369	if (!iter) {
6370	ret = -ENOMEM;
6371	goto fail_alloc_iter;
6372	}
6373
6374	trace_seq_init(s: &iter->seq);
6375	iter->trace = tr->current_trace;
6376
6377	if (!alloc_cpumask_var(mask: &iter->started, GFP_KERNEL)) {
6378	ret = -ENOMEM;
6379	goto fail;
6380	}
6381
6382	/* trace pipe does not show start of buffer */
6383	cpumask_setall(dstp: iter->started);
6384
6385	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
6386	iter->iter_flags |= TRACE_FILE_LAT_FMT;
6387
6388	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
6389	if (trace_clocks[tr->clock_id].in_ns)
6390	iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6391
6392	iter->tr = tr;
6393	iter->array_buffer = &tr->array_buffer;
6394	iter->cpu_file = cpu;
6395	mutex_init(&iter->mutex);
6396	filp->private_data = iter;
6397
6398	if (iter->trace->pipe_open)
6399	iter->trace->pipe_open(iter);
6400
6401	nonseekable_open(inode, filp);
6402
6403	tr->trace_ref++;
6404
6405	mutex_unlock(lock: &trace_types_lock);
6406	return ret;
6407
6408	fail:
6409	kfree(objp: iter);
6410	fail_alloc_iter:
6411	close_pipe_on_cpu(tr, cpu);
6412	fail_pipe_on_cpu:
6413	__trace_array_put(this_tr: tr);
6414	mutex_unlock(lock: &trace_types_lock);
6415	return ret;
6416	}
6417
6418	static int tracing_release_pipe(struct inode *inode, struct file *file)
6419	{
6420	struct trace_iterator *iter = file->private_data;
6421	struct trace_array *tr = inode->i_private;
6422
6423	mutex_lock(&trace_types_lock);
6424
6425	tr->trace_ref--;
6426
6427	if (iter->trace->pipe_close)
6428	iter->trace->pipe_close(iter);
6429	close_pipe_on_cpu(tr, cpu: iter->cpu_file);
6430	mutex_unlock(lock: &trace_types_lock);
6431
6432	free_trace_iter_content(iter);
6433	kfree(objp: iter);
6434
6435	trace_array_put(tr);
6436
6437	return 0;
6438	}
6439
6440	static __poll_t
6441	trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
6442	{
6443	struct trace_array *tr = iter->tr;
6444
6445	/* Iterators are static, they should be filled or empty */
6446	if (trace_buffer_iter(iter, cpu: iter->cpu_file))
6447	return EPOLLIN | EPOLLRDNORM;
6448
6449	if (tr->trace_flags & TRACE_ITER_BLOCK)
6450	/*
6451	* Always select as readable when in blocking mode
6452	*/
6453	return EPOLLIN | EPOLLRDNORM;
6454	else
6455	return ring_buffer_poll_wait(buffer: iter->array_buffer->buffer, cpu: iter->cpu_file,
6456	filp, poll_table, full: iter->tr->buffer_percent);
6457	}
6458
6459	static __poll_t
6460	tracing_poll_pipe(struct file *filp, poll_table *poll_table)
6461	{
6462	struct trace_iterator *iter = filp->private_data;
6463
6464	return trace_poll(iter, filp, poll_table);
6465	}
6466
6467	/* Must be called with iter->mutex held. */
6468	static int tracing_wait_pipe(struct file *filp)
6469	{
6470	struct trace_iterator *iter = filp->private_data;
6471	int ret;
6472
6473	while (trace_empty(iter)) {
6474
6475	if ((filp->f_flags & O_NONBLOCK)) {
6476	return -EAGAIN;
6477	}
6478
6479	/*
6480	* We block until we read something and tracing is disabled.
6481	* We still block if tracing is disabled, but we have never
6482	* read anything. This allows a user to cat this file, and
6483	* then enable tracing. But after we have read something,
6484	* we give an EOF when tracing is again disabled.
6485	*
6486	* iter->pos will be 0 if we haven't read anything.
6487	*/
6488	if (!tracer_tracing_is_on(tr: iter->tr) && iter->pos)
6489	break;
6490
6491	mutex_unlock(lock: &iter->mutex);
6492
6493	ret = wait_on_pipe(iter, full: 0);
6494
6495	mutex_lock(&iter->mutex);
6496
6497	if (ret)
6498	return ret;
6499	}
6500
6501	return 1;
6502	}
6503
6504	/*
6505	* Consumer reader.
6506	*/
6507	static ssize_t
6508	tracing_read_pipe(struct file *filp, char __user *ubuf,
6509	size_t cnt, loff_t *ppos)
6510	{
6511	struct trace_iterator *iter = filp->private_data;
6512	ssize_t sret;
6513
6514	/*
6515	* Avoid more than one consumer on a single file descriptor
6516	* This is just a matter of traces coherency, the ring buffer itself
6517	* is protected.
6518	*/
6519	mutex_lock(&iter->mutex);
6520
6521	/* return any leftover data */
6522	sret = trace_seq_to_user(s: &iter->seq, ubuf, cnt);
6523	if (sret != -EBUSY)
6524	goto out;
6525
6526	trace_seq_init(s: &iter->seq);
6527
6528	if (iter->trace->read) {
6529	sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
6530	if (sret)
6531	goto out;
6532	}
6533
6534	waitagain:
6535	sret = tracing_wait_pipe(filp);
6536	if (sret <= 0)
6537	goto out;
6538
6539	/* stop when tracing is finished */
6540	if (trace_empty(iter)) {
6541	sret = 0;
6542	goto out;
6543	}
6544
6545	if (cnt >= TRACE_SEQ_BUFFER_SIZE)
6546	cnt = TRACE_SEQ_BUFFER_SIZE - 1;
6547
6548	/* reset all but tr, trace, and overruns */
6549	trace_iterator_reset(iter);
6550	cpumask_clear(dstp: iter->started);
6551	trace_seq_init(s: &iter->seq);
6552
6553	trace_event_read_lock();
6554	trace_access_lock(cpu: iter->cpu_file);
6555	while (trace_find_next_entry_inc(iter) != NULL) {
6556	enum print_line_t ret;
6557	int save_len = iter->seq.seq.len;
6558
6559	ret = print_trace_line(iter);
6560	if (ret == TRACE_TYPE_PARTIAL_LINE) {
6561	/*
6562	* If one print_trace_line() fills entire trace_seq in one shot,
6563	* trace_seq_to_user() will returns -EBUSY because save_len == 0,
6564	* In this case, we need to consume it, otherwise, loop will peek
6565	* this event next time, resulting in an infinite loop.
6566	*/
6567	if (save_len == 0) {
6568	iter->seq.full = 0;
6569	trace_seq_puts(s: &iter->seq, str: "[LINE TOO BIG]\n");
6570	trace_consume(iter);
6571	break;
6572	}
6573
6574	/* In other cases, don't print partial lines */
6575	iter->seq.seq.len = save_len;
6576	break;
6577	}
6578	if (ret != TRACE_TYPE_NO_CONSUME)
6579	trace_consume(iter);
6580
6581	if (trace_seq_used(s: &iter->seq) >= cnt)
6582	break;
6583
6584	/*
6585	* Setting the full flag means we reached the trace_seq buffer
6586	* size and we should leave by partial output condition above.
6587	* One of the trace_seq_* functions is not used properly.
6588	*/
6589	WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
6590	iter->ent->type);
6591	}
6592	trace_access_unlock(cpu: iter->cpu_file);
6593	trace_event_read_unlock();
6594
6595	/* Now copy what we have to the user */
6596	sret = trace_seq_to_user(s: &iter->seq, ubuf, cnt);
6597	if (iter->seq.readpos >= trace_seq_used(s: &iter->seq))
6598	trace_seq_init(s: &iter->seq);
6599
6600	/*
6601	* If there was nothing to send to user, in spite of consuming trace
6602	* entries, go back to wait for more entries.
6603	*/
6604	if (sret == -EBUSY)
6605	goto waitagain;
6606
6607	out:
6608	mutex_unlock(lock: &iter->mutex);
6609
6610	return sret;
6611	}
6612
6613	static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
6614	unsigned int idx)
6615	{
6616	__free_page(spd->pages[idx]);
6617	}
6618
6619	static size_t
6620	tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
6621	{
6622	size_t count;
6623	int save_len;
6624	int ret;
6625
6626	/* Seq buffer is page-sized, exactly what we need. */
6627	for (;;) {
6628	save_len = iter->seq.seq.len;
6629	ret = print_trace_line(iter);
6630
6631	if (trace_seq_has_overflowed(s: &iter->seq)) {
6632	iter->seq.seq.len = save_len;
6633	break;
6634	}
6635
6636	/*
6637	* This should not be hit, because it should only
6638	* be set if the iter->seq overflowed. But check it
6639	* anyway to be safe.
6640	*/
6641	if (ret == TRACE_TYPE_PARTIAL_LINE) {
6642	iter->seq.seq.len = save_len;
6643	break;
6644	}
6645
6646	count = trace_seq_used(s: &iter->seq) - save_len;
6647	if (rem < count) {
6648	rem = 0;
6649	iter->seq.seq.len = save_len;
6650	break;
6651	}
6652
6653	if (ret != TRACE_TYPE_NO_CONSUME)
6654	trace_consume(iter);
6655	rem -= count;
6656	if (!trace_find_next_entry_inc(iter)) {
6657	rem = 0;
6658	iter->ent = NULL;
6659	break;
6660	}
6661	}
6662
6663	return rem;
6664	}
6665
6666	static ssize_t tracing_splice_read_pipe(struct file *filp,
6667	loff_t *ppos,
6668	struct pipe_inode_info *pipe,
6669	size_t len,
6670	unsigned int flags)
6671	{
6672	struct page *pages_def[PIPE_DEF_BUFFERS];
6673	struct partial_page partial_def[PIPE_DEF_BUFFERS];
6674	struct trace_iterator *iter = filp->private_data;
6675	struct splice_pipe_desc spd = {
6676	.pages = pages_def,
6677	.partial = partial_def,
6678	.nr_pages = 0, /* This gets updated below. */
6679	.nr_pages_max = PIPE_DEF_BUFFERS,
6680	.ops = &default_pipe_buf_ops,
6681	.spd_release = tracing_spd_release_pipe,
6682	};
6683	ssize_t ret;
6684	size_t rem;
6685	unsigned int i;
6686
6687	if (splice_grow_spd(pipe, &spd))
6688	return -ENOMEM;
6689
6690	mutex_lock(&iter->mutex);
6691
6692	if (iter->trace->splice_read) {
6693	ret = iter->trace->splice_read(iter, filp,
6694	ppos, pipe, len, flags);
6695	if (ret)
6696	goto out_err;
6697	}
6698
6699	ret = tracing_wait_pipe(filp);
6700	if (ret <= 0)
6701	goto out_err;
6702
6703	if (!iter->ent && !trace_find_next_entry_inc(iter)) {
6704	ret = -EFAULT;
6705	goto out_err;
6706	}
6707
6708	trace_event_read_lock();
6709	trace_access_lock(cpu: iter->cpu_file);
6710
6711	/* Fill as many pages as possible. */
6712	for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
6713	spd.pages[i] = alloc_page(GFP_KERNEL);
6714	if (!spd.pages[i])
6715	break;
6716
6717	rem = tracing_fill_pipe_page(rem, iter);
6718
6719	/* Copy the data into the page, so we can start over. */
6720	ret = trace_seq_to_buffer(s: &iter->seq,
6721	page_address(spd.pages[i]),
6722	cnt: trace_seq_used(s: &iter->seq));
6723	if (ret < 0) {
6724	__free_page(spd.pages[i]);
6725	break;
6726	}
6727	spd.partial[i].offset = 0;
6728	spd.partial[i].len = trace_seq_used(s: &iter->seq);
6729
6730	trace_seq_init(s: &iter->seq);
6731	}
6732
6733	trace_access_unlock(cpu: iter->cpu_file);
6734	trace_event_read_unlock();
6735	mutex_unlock(lock: &iter->mutex);
6736
6737	spd.nr_pages = i;
6738
6739	if (i)
6740	ret = splice_to_pipe(pipe, spd: &spd);
6741	else
6742	ret = 0;
6743	out:
6744	splice_shrink_spd(&spd);
6745	return ret;
6746
6747	out_err:
6748	mutex_unlock(lock: &iter->mutex);
6749	goto out;
6750	}
6751
6752	static ssize_t
6753	tracing_entries_read(struct file *filp, char __user *ubuf,
6754	size_t cnt, loff_t *ppos)
6755	{
6756	struct inode *inode = file_inode(f: filp);
6757	struct trace_array *tr = inode->i_private;
6758	int cpu = tracing_get_cpu(inode);
6759	char buf[64];
6760	int r = 0;
6761	ssize_t ret;
6762
6763	mutex_lock(&trace_types_lock);
6764
6765	if (cpu == RING_BUFFER_ALL_CPUS) {
6766	int cpu, buf_size_same;
6767	unsigned long size;
6768
6769	size = 0;
6770	buf_size_same = 1;
6771	/* check if all cpu sizes are same */
6772	for_each_tracing_cpu(cpu) {
6773	/* fill in the size from first enabled cpu */
6774	if (size == 0)
6775	size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;
6776	if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {
6777	buf_size_same = 0;
6778	break;
6779	}
6780	}
6781
6782	if (buf_size_same) {
6783	if (!tr->ring_buffer_expanded)
6784	r = sprintf(buf, fmt: "%lu (expanded: %lu)\n",
6785	size >> 10,
6786	trace_buf_size >> 10);
6787	else
6788	r = sprintf(buf, fmt: "%lu\n", size >> 10);
6789	} else
6790	r = sprintf(buf, fmt: "X\n");
6791	} else
6792	r = sprintf(buf, fmt: "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);
6793
6794	mutex_unlock(lock: &trace_types_lock);
6795
6796	ret = simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
6797	return ret;
6798	}
6799
6800	static ssize_t
6801	tracing_entries_write(struct file *filp, const char __user *ubuf,
6802	size_t cnt, loff_t *ppos)
6803	{
6804	struct inode *inode = file_inode(f: filp);
6805	struct trace_array *tr = inode->i_private;
6806	unsigned long val;
6807	int ret;
6808
6809	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
6810	if (ret)
6811	return ret;
6812
6813	/* must have at least 1 entry */
6814	if (!val)
6815	return -EINVAL;
6816
6817	/* value is in KB */
6818	val <<= 10;
6819	ret = tracing_resize_ring_buffer(tr, size: val, cpu_id: tracing_get_cpu(inode));
6820	if (ret < 0)
6821	return ret;
6822
6823	*ppos += cnt;
6824
6825	return cnt;
6826	}
6827
6828	static ssize_t
6829	tracing_total_entries_read(struct file *filp, char __user *ubuf,
6830	size_t cnt, loff_t *ppos)
6831	{
6832	struct trace_array *tr = filp->private_data;
6833	char buf[64];
6834	int r, cpu;
6835	unsigned long size = 0, expanded_size = 0;
6836
6837	mutex_lock(&trace_types_lock);
6838	for_each_tracing_cpu(cpu) {
6839	size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;
6840	if (!tr->ring_buffer_expanded)
6841	expanded_size += trace_buf_size >> 10;
6842	}
6843	if (tr->ring_buffer_expanded)
6844	r = sprintf(buf, fmt: "%lu\n", size);
6845	else
6846	r = sprintf(buf, fmt: "%lu (expanded: %lu)\n", size, expanded_size);
6847	mutex_unlock(lock: &trace_types_lock);
6848
6849	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
6850	}
6851
6852	static ssize_t
6853	tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6854	size_t cnt, loff_t *ppos)
6855	{
6856	/*
6857	* There is no need to read what the user has written, this function
6858	* is just to make sure that there is no error when "echo" is used
6859	*/
6860
6861	*ppos += cnt;
6862
6863	return cnt;
6864	}
6865
6866	static int
6867	tracing_free_buffer_release(struct inode *inode, struct file *filp)
6868	{
6869	struct trace_array *tr = inode->i_private;
6870
6871	/* disable tracing ? */
6872	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6873	tracer_tracing_off(tr);
6874	/* resize the ring buffer to 0 */
6875	tracing_resize_ring_buffer(tr, size: 0, RING_BUFFER_ALL_CPUS);
6876
6877	trace_array_put(tr);
6878
6879	return 0;
6880	}
6881
6882	#define TRACE_MARKER_MAX_SIZE 4096
6883
6884	static ssize_t
6885	tracing_mark_write(struct file *filp, const char __user *ubuf,
6886	size_t cnt, loff_t *fpos)
6887	{
6888	struct trace_array *tr = filp->private_data;
6889	struct ring_buffer_event *event;
6890	enum event_trigger_type tt = ETT_NONE;
6891	struct trace_buffer *buffer;
6892	struct print_entry *entry;
6893	int meta_size;
6894	ssize_t written;
6895	size_t size;
6896	int len;
6897
6898	/* Used in tracing_mark_raw_write() as well */
6899	#define FAULTED_STR "<faulted>"
6900	#define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */
6901
6902	if (tracing_disabled)
6903	return -EINVAL;
6904
6905	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6906	return -EINVAL;
6907
6908	if ((ssize_t)cnt < 0)
6909	return -EINVAL;
6910
6911	if (cnt > TRACE_MARKER_MAX_SIZE)
6912	cnt = TRACE_MARKER_MAX_SIZE;
6913
6914	meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */
6915	again:
6916	size = cnt + meta_size;
6917
6918	/* If less than "<faulted>", then make sure we can still add that */
6919	if (cnt < FAULTED_SIZE)
6920	size += FAULTED_SIZE - cnt;
6921
6922	buffer = tr->array_buffer.buffer;
6923	event = __trace_buffer_lock_reserve(buffer, type: TRACE_PRINT, len: size,
6924	trace_ctx: tracing_gen_ctx());
6925	if (unlikely(!event)) {
6926	/*
6927	* If the size was greater than what was allowed, then
6928	* make it smaller and try again.
6929	*/
6930	if (size > ring_buffer_max_event_size(buffer)) {
6931	/* cnt < FAULTED size should never be bigger than max */
6932	if (WARN_ON_ONCE(cnt < FAULTED_SIZE))
6933	return -EBADF;
6934	cnt = ring_buffer_max_event_size(buffer) - meta_size;
6935	/* The above should only happen once */
6936	if (WARN_ON_ONCE(cnt + meta_size == size))
6937	return -EBADF;
6938	goto again;
6939	}
6940
6941	/* Ring buffer disabled, return as if not open for write */
6942	return -EBADF;
6943	}
6944
6945	entry = ring_buffer_event_data(event);
6946	entry->ip = _THIS_IP_;
6947
6948	len = __copy_from_user_inatomic(to: &entry->buf, from: ubuf, n: cnt);
6949	if (len) {
6950	memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
6951	cnt = FAULTED_SIZE;
6952	written = -EFAULT;
6953	} else
6954	written = cnt;
6955
6956	if (tr->trace_marker_file && !list_empty(head: &tr->trace_marker_file->triggers)) {
6957	/* do not add \n before testing triggers, but add \0 */
6958	entry->buf[cnt] = '\0';
6959	tt = event_triggers_call(file: tr->trace_marker_file, buffer, rec: entry, event);
6960	}
6961
6962	if (entry->buf[cnt - 1] != '\n') {
6963	entry->buf[cnt] = '\n';
6964	entry->buf[cnt + 1] = '\0';
6965	} else
6966	entry->buf[cnt] = '\0';
6967
6968	if (static_branch_unlikely(&trace_marker_exports_enabled))
6969	ftrace_exports(event, TRACE_EXPORT_MARKER);
6970	__buffer_unlock_commit(buffer, event);
6971
6972	if (tt)
6973	event_triggers_post_call(file: tr->trace_marker_file, tt);
6974
6975	return written;
6976	}
6977
6978	static ssize_t
6979	tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6980	size_t cnt, loff_t *fpos)
6981	{
6982	struct trace_array *tr = filp->private_data;
6983	struct ring_buffer_event *event;
6984	struct trace_buffer *buffer;
6985	struct raw_data_entry *entry;
6986	ssize_t written;
6987	int size;
6988	int len;
6989
6990	#define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6991
6992	if (tracing_disabled)
6993	return -EINVAL;
6994
6995	if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6996	return -EINVAL;
6997
6998	/* The marker must at least have a tag id */
6999	if (cnt < sizeof(unsigned int))
7000	return -EINVAL;
7001
7002	size = sizeof(*entry) + cnt;
7003	if (cnt < FAULT_SIZE_ID)
7004	size += FAULT_SIZE_ID - cnt;
7005
7006	buffer = tr->array_buffer.buffer;
7007
7008	if (size > ring_buffer_max_event_size(buffer))
7009	return -EINVAL;
7010
7011	event = __trace_buffer_lock_reserve(buffer, type: TRACE_RAW_DATA, len: size,
7012	trace_ctx: tracing_gen_ctx());
7013	if (!event)
7014	/* Ring buffer disabled, return as if not open for write */
7015	return -EBADF;
7016
7017	entry = ring_buffer_event_data(event);
7018
7019	len = __copy_from_user_inatomic(to: &entry->id, from: ubuf, n: cnt);
7020	if (len) {
7021	entry->id = -1;
7022	memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);
7023	written = -EFAULT;
7024	} else
7025	written = cnt;
7026
7027	__buffer_unlock_commit(buffer, event);
7028
7029	return written;
7030	}
7031
7032	static int tracing_clock_show(struct seq_file *m, void *v)
7033	{
7034	struct trace_array *tr = m->private;
7035	int i;
7036
7037	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
7038	seq_printf(m,
7039	fmt: "%s%s%s%s", i ? " " : "",
7040	i == tr->clock_id ? "[" : "", trace_clocks[i].name,
7041	i == tr->clock_id ? "]" : "");
7042	seq_putc(m, c: '\n');
7043
7044	return 0;
7045	}
7046
7047	int tracing_set_clock(struct trace_array *tr, const char *clockstr)
7048	{
7049	int i;
7050
7051	for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
7052	if (strcmp(trace_clocks[i].name, clockstr) == 0)
7053	break;
7054	}
7055	if (i == ARRAY_SIZE(trace_clocks))
7056	return -EINVAL;
7057
7058	mutex_lock(&trace_types_lock);
7059
7060	tr->clock_id = i;
7061
7062	ring_buffer_set_clock(buffer: tr->array_buffer.buffer, clock: trace_clocks[i].func);
7063
7064	/*
7065	* New clock may not be consistent with the previous clock.
7066	* Reset the buffer so that it doesn't have incomparable timestamps.
7067	*/
7068	tracing_reset_online_cpus(buf: &tr->array_buffer);
7069
7070	#ifdef CONFIG_TRACER_MAX_TRACE
7071	if (tr->max_buffer.buffer)
7072	ring_buffer_set_clock(buffer: tr->max_buffer.buffer, clock: trace_clocks[i].func);
7073	tracing_reset_online_cpus(buf: &tr->max_buffer);
7074	#endif
7075
7076	mutex_unlock(lock: &trace_types_lock);
7077
7078	return 0;
7079	}
7080
7081	static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
7082	size_t cnt, loff_t *fpos)
7083	{
7084	struct seq_file *m = filp->private_data;
7085	struct trace_array *tr = m->private;
7086	char buf[64];
7087	const char *clockstr;
7088	int ret;
7089
7090	if (cnt >= sizeof(buf))
7091	return -EINVAL;
7092
7093	if (copy_from_user(to: buf, from: ubuf, n: cnt))
7094	return -EFAULT;
7095
7096	buf[cnt] = 0;
7097
7098	clockstr = strstrip(str: buf);
7099
7100	ret = tracing_set_clock(tr, clockstr);
7101	if (ret)
7102	return ret;
7103
7104	*fpos += cnt;
7105
7106	return cnt;
7107	}
7108
7109	static int tracing_clock_open(struct inode *inode, struct file *file)
7110	{
7111	struct trace_array *tr = inode->i_private;
7112	int ret;
7113
7114	ret = tracing_check_open_get_tr(tr);
7115	if (ret)
7116	return ret;
7117
7118	ret = single_open(file, tracing_clock_show, inode->i_private);
7119	if (ret < 0)
7120	trace_array_put(tr);
7121
7122	return ret;
7123	}
7124
7125	static int tracing_time_stamp_mode_show(struct seq_file *m, void *v)
7126	{
7127	struct trace_array *tr = m->private;
7128
7129	mutex_lock(&trace_types_lock);
7130
7131	if (ring_buffer_time_stamp_abs(buffer: tr->array_buffer.buffer))
7132	seq_puts(m, s: "delta [absolute]\n");
7133	else
7134	seq_puts(m, s: "[delta] absolute\n");
7135
7136	mutex_unlock(lock: &trace_types_lock);
7137
7138	return 0;
7139	}
7140
7141	static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file)
7142	{
7143	struct trace_array *tr = inode->i_private;
7144	int ret;
7145
7146	ret = tracing_check_open_get_tr(tr);
7147	if (ret)
7148	return ret;
7149
7150	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);
7151	if (ret < 0)
7152	trace_array_put(tr);
7153
7154	return ret;
7155	}
7156
7157	u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe)
7158	{
7159	if (rbe == this_cpu_read(trace_buffered_event))
7160	return ring_buffer_time_stamp(buffer);
7161
7162	return ring_buffer_event_time_stamp(buffer, event: rbe);
7163	}
7164
7165	/*
7166	* Set or disable using the per CPU trace_buffer_event when possible.
7167	*/
7168	int tracing_set_filter_buffering(struct trace_array *tr, bool set)
7169	{
7170	int ret = 0;
7171
7172	mutex_lock(&trace_types_lock);
7173
7174	if (set && tr->no_filter_buffering_ref++)
7175	goto out;
7176
7177	if (!set) {
7178	if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) {
7179	ret = -EINVAL;
7180	goto out;
7181	}
7182
7183	--tr->no_filter_buffering_ref;
7184	}
7185	out:
7186	mutex_unlock(lock: &trace_types_lock);
7187
7188	return ret;
7189	}
7190
7191	struct ftrace_buffer_info {
7192	struct trace_iterator iter;
7193	void *spare;
7194	unsigned int spare_cpu;
7195	unsigned int spare_size;
7196	unsigned int read;
7197	};
7198
7199	#ifdef CONFIG_TRACER_SNAPSHOT
7200	static int tracing_snapshot_open(struct inode *inode, struct file *file)
7201	{
7202	struct trace_array *tr = inode->i_private;
7203	struct trace_iterator *iter;
7204	struct seq_file *m;
7205	int ret;
7206
7207	ret = tracing_check_open_get_tr(tr);
7208	if (ret)
7209	return ret;
7210
7211	if (file->f_mode & FMODE_READ) {
7212	iter = __tracing_open(inode, file, snapshot: true);
7213	if (IS_ERR(ptr: iter))
7214	ret = PTR_ERR(ptr: iter);
7215	} else {
7216	/* Writes still need the seq_file to hold the private data */
7217	ret = -ENOMEM;
7218	m = kzalloc(size: sizeof(*m), GFP_KERNEL);
7219	if (!m)
7220	goto out;
7221	iter = kzalloc(size: sizeof(*iter), GFP_KERNEL);
7222	if (!iter) {
7223	kfree(objp: m);
7224	goto out;
7225	}
7226	ret = 0;
7227
7228	iter->tr = tr;
7229	iter->array_buffer = &tr->max_buffer;
7230	iter->cpu_file = tracing_get_cpu(inode);
7231	m->private = iter;
7232	file->private_data = m;
7233	}
7234	out:
7235	if (ret < 0)
7236	trace_array_put(tr);
7237
7238	return ret;
7239	}
7240
7241	static void tracing_swap_cpu_buffer(void *tr)
7242	{
7243	update_max_tr_single(tr: (struct trace_array *)tr, current, smp_processor_id());
7244	}
7245
7246	static ssize_t
7247	tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
7248	loff_t *ppos)
7249	{
7250	struct seq_file *m = filp->private_data;
7251	struct trace_iterator *iter = m->private;
7252	struct trace_array *tr = iter->tr;
7253	unsigned long val;
7254	int ret;
7255
7256	ret = tracing_update_buffers(tr);
7257	if (ret < 0)
7258	return ret;
7259
7260	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
7261	if (ret)
7262	return ret;
7263
7264	mutex_lock(&trace_types_lock);
7265
7266	if (tr->current_trace->use_max_tr) {
7267	ret = -EBUSY;
7268	goto out;
7269	}
7270
7271	local_irq_disable();
7272	arch_spin_lock(&tr->max_lock);
7273	if (tr->cond_snapshot)
7274	ret = -EBUSY;
7275	arch_spin_unlock(&tr->max_lock);
7276	local_irq_enable();
7277	if (ret)
7278	goto out;
7279
7280	switch (val) {
7281	case 0:
7282	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7283	ret = -EINVAL;
7284	break;
7285	}
7286	if (tr->allocated_snapshot)
7287	free_snapshot(tr);
7288	break;
7289	case 1:
7290	/* Only allow per-cpu swap if the ring buffer supports it */
7291	#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
7292	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
7293	ret = -EINVAL;
7294	break;
7295	}
7296	#endif
7297	if (tr->allocated_snapshot)
7298	ret = resize_buffer_duplicate_size(trace_buf: &tr->max_buffer,
7299	size_buf: &tr->array_buffer, cpu_id: iter->cpu_file);
7300
7301	ret = tracing_arm_snapshot_locked(tr);
7302	if (ret)
7303	break;
7304
7305	/* Now, we're going to swap */
7306	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
7307	local_irq_disable();
7308	update_max_tr(tr, current, smp_processor_id(), NULL);
7309	local_irq_enable();
7310	} else {
7311	smp_call_function_single(cpuid: iter->cpu_file, func: tracing_swap_cpu_buffer,
7312	info: (void *)tr, wait: 1);
7313	}
7314	tracing_disarm_snapshot(tr);
7315	break;
7316	default:
7317	if (tr->allocated_snapshot) {
7318	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
7319	tracing_reset_online_cpus(buf: &tr->max_buffer);
7320	else
7321	tracing_reset_cpu(buf: &tr->max_buffer, cpu: iter->cpu_file);
7322	}
7323	break;
7324	}
7325
7326	if (ret >= 0) {
7327	*ppos += cnt;
7328	ret = cnt;
7329	}
7330	out:
7331	mutex_unlock(lock: &trace_types_lock);
7332	return ret;
7333	}
7334
7335	static int tracing_snapshot_release(struct inode *inode, struct file *file)
7336	{
7337	struct seq_file *m = file->private_data;
7338	int ret;
7339
7340	ret = tracing_release(inode, file);
7341
7342	if (file->f_mode & FMODE_READ)
7343	return ret;
7344
7345	/* If write only, the seq_file is just a stub */
7346	if (m)
7347	kfree(objp: m->private);
7348	kfree(objp: m);
7349
7350	return 0;
7351	}
7352
7353	static int tracing_buffers_open(struct inode *inode, struct file *filp);
7354	static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
7355	size_t count, loff_t *ppos);
7356	static int tracing_buffers_release(struct inode *inode, struct file *file);
7357	static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
7358	struct pipe_inode_info *pipe, size_t len, unsigned int flags);
7359
7360	static int snapshot_raw_open(struct inode *inode, struct file *filp)
7361	{
7362	struct ftrace_buffer_info *info;
7363	int ret;
7364
7365	/* The following checks for tracefs lockdown */
7366	ret = tracing_buffers_open(inode, filp);
7367	if (ret < 0)
7368	return ret;
7369
7370	info = filp->private_data;
7371
7372	if (info->iter.trace->use_max_tr) {
7373	tracing_buffers_release(inode, file: filp);
7374	return -EBUSY;
7375	}
7376
7377	info->iter.snapshot = true;
7378	info->iter.array_buffer = &info->iter.tr->max_buffer;
7379
7380	return ret;
7381	}
7382
7383	#endif /* CONFIG_TRACER_SNAPSHOT */
7384
7385
7386	static const struct file_operations tracing_thresh_fops = {
7387	.open = tracing_open_generic,
7388	.read = tracing_thresh_read,
7389	.write = tracing_thresh_write,
7390	.llseek = generic_file_llseek,
7391	};
7392
7393	#ifdef CONFIG_TRACER_MAX_TRACE
7394	static const struct file_operations tracing_max_lat_fops = {
7395	.open = tracing_open_generic_tr,
7396	.read = tracing_max_lat_read,
7397	.write = tracing_max_lat_write,
7398	.llseek = generic_file_llseek,
7399	.release = tracing_release_generic_tr,
7400	};
7401	#endif
7402
7403	static const struct file_operations set_tracer_fops = {
7404	.open = tracing_open_generic_tr,
7405	.read = tracing_set_trace_read,
7406	.write = tracing_set_trace_write,
7407	.llseek = generic_file_llseek,
7408	.release = tracing_release_generic_tr,
7409	};
7410
7411	static const struct file_operations tracing_pipe_fops = {
7412	.open = tracing_open_pipe,
7413	.poll = tracing_poll_pipe,
7414	.read = tracing_read_pipe,
7415	.splice_read = tracing_splice_read_pipe,
7416	.release = tracing_release_pipe,
7417	.llseek = no_llseek,
7418	};
7419
7420	static const struct file_operations tracing_entries_fops = {
7421	.open = tracing_open_generic_tr,
7422	.read = tracing_entries_read,
7423	.write = tracing_entries_write,
7424	.llseek = generic_file_llseek,
7425	.release = tracing_release_generic_tr,
7426	};
7427
7428	static const struct file_operations tracing_total_entries_fops = {
7429	.open = tracing_open_generic_tr,
7430	.read = tracing_total_entries_read,
7431	.llseek = generic_file_llseek,
7432	.release = tracing_release_generic_tr,
7433	};
7434
7435	static const struct file_operations tracing_free_buffer_fops = {
7436	.open = tracing_open_generic_tr,
7437	.write = tracing_free_buffer_write,
7438	.release = tracing_free_buffer_release,
7439	};
7440
7441	static const struct file_operations tracing_mark_fops = {
7442	.open = tracing_mark_open,
7443	.write = tracing_mark_write,
7444	.release = tracing_release_generic_tr,
7445	};
7446
7447	static const struct file_operations tracing_mark_raw_fops = {
7448	.open = tracing_mark_open,
7449	.write = tracing_mark_raw_write,
7450	.release = tracing_release_generic_tr,
7451	};
7452
7453	static const struct file_operations trace_clock_fops = {
7454	.open = tracing_clock_open,
7455	.read = seq_read,
7456	.llseek = seq_lseek,
7457	.release = tracing_single_release_tr,
7458	.write = tracing_clock_write,
7459	};
7460
7461	static const struct file_operations trace_time_stamp_mode_fops = {
7462	.open = tracing_time_stamp_mode_open,
7463	.read = seq_read,
7464	.llseek = seq_lseek,
7465	.release = tracing_single_release_tr,
7466	};
7467
7468	#ifdef CONFIG_TRACER_SNAPSHOT
7469	static const struct file_operations snapshot_fops = {
7470	.open = tracing_snapshot_open,
7471	.read = seq_read,
7472	.write = tracing_snapshot_write,
7473	.llseek = tracing_lseek,
7474	.release = tracing_snapshot_release,
7475	};
7476
7477	static const struct file_operations snapshot_raw_fops = {
7478	.open = snapshot_raw_open,
7479	.read = tracing_buffers_read,
7480	.release = tracing_buffers_release,
7481	.splice_read = tracing_buffers_splice_read,
7482	.llseek = no_llseek,
7483	};
7484
7485	#endif /* CONFIG_TRACER_SNAPSHOT */
7486
7487	/*
7488	* trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7489	* @filp: The active open file structure
7490	* @ubuf: The userspace provided buffer to read value into
7491	* @cnt: The maximum number of bytes to read
7492	* @ppos: The current "file" position
7493	*
7494	* This function implements the write interface for a struct trace_min_max_param.
7495	* The filp->private_data must point to a trace_min_max_param structure that
7496	* defines where to write the value, the min and the max acceptable values,
7497	* and a lock to protect the write.
7498	*/
7499	static ssize_t
7500	trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
7501	{
7502	struct trace_min_max_param *param = filp->private_data;
7503	u64 val;
7504	int err;
7505
7506	if (!param)
7507	return -EFAULT;
7508
7509	err = kstrtoull_from_user(s: ubuf, count: cnt, base: 10, res: &val);
7510	if (err)
7511	return err;
7512
7513	if (param->lock)
7514	mutex_lock(param->lock);
7515
7516	if (param->min && val < *param->min)
7517	err = -EINVAL;
7518
7519	if (param->max && val > *param->max)
7520	err = -EINVAL;
7521
7522	if (!err)
7523	*param->val = val;
7524
7525	if (param->lock)
7526	mutex_unlock(lock: param->lock);
7527
7528	if (err)
7529	return err;
7530
7531	return cnt;
7532	}
7533
7534	/*
7535	* trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7536	* @filp: The active open file structure
7537	* @ubuf: The userspace provided buffer to read value into
7538	* @cnt: The maximum number of bytes to read
7539	* @ppos: The current "file" position
7540	*
7541	* This function implements the read interface for a struct trace_min_max_param.
7542	* The filp->private_data must point to a trace_min_max_param struct with valid
7543	* data.
7544	*/
7545	static ssize_t
7546	trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
7547	{
7548	struct trace_min_max_param *param = filp->private_data;
7549	char buf[U64_STR_SIZE];
7550	int len;
7551	u64 val;
7552
7553	if (!param)
7554	return -EFAULT;
7555
7556	val = *param->val;
7557
7558	if (cnt > sizeof(buf))
7559	cnt = sizeof(buf);
7560
7561	len = snprintf(buf, size: sizeof(buf), fmt: "%llu\n", val);
7562
7563	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: len);
7564	}
7565
7566	const struct file_operations trace_min_max_fops = {
7567	.open = tracing_open_generic,
7568	.read = trace_min_max_read,
7569	.write = trace_min_max_write,
7570	};
7571
7572	#define TRACING_LOG_ERRS_MAX 8
7573	#define TRACING_LOG_LOC_MAX 128
7574
7575	#define CMD_PREFIX " Command: "
7576
7577	struct err_info {
7578	const char **errs; /* ptr to loc-specific array of err strings */
7579	u8 type; /* index into errs -> specific err string */
7580	u16 pos; /* caret position */
7581	u64 ts;
7582	};
7583
7584	struct tracing_log_err {
7585	struct list_head list;
7586	struct err_info info;
7587	char loc[TRACING_LOG_LOC_MAX]; /* err location */
7588	char *cmd; /* what caused err */
7589	};
7590
7591	static DEFINE_MUTEX(tracing_err_log_lock);
7592
7593	static struct tracing_log_err *alloc_tracing_log_err(int len)
7594	{
7595	struct tracing_log_err *err;
7596
7597	err = kzalloc(size: sizeof(*err), GFP_KERNEL);
7598	if (!err)
7599	return ERR_PTR(error: -ENOMEM);
7600
7601	err->cmd = kzalloc(size: len, GFP_KERNEL);
7602	if (!err->cmd) {
7603	kfree(objp: err);
7604	return ERR_PTR(error: -ENOMEM);
7605	}
7606
7607	return err;
7608	}
7609
7610	static void free_tracing_log_err(struct tracing_log_err *err)
7611	{
7612	kfree(objp: err->cmd);
7613	kfree(objp: err);
7614	}
7615
7616	static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr,
7617	int len)
7618	{
7619	struct tracing_log_err *err;
7620	char *cmd;
7621
7622	if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {
7623	err = alloc_tracing_log_err(len);
7624	if (PTR_ERR(ptr: err) != -ENOMEM)
7625	tr->n_err_log_entries++;
7626
7627	return err;
7628	}
7629	cmd = kzalloc(size: len, GFP_KERNEL);
7630	if (!cmd)
7631	return ERR_PTR(error: -ENOMEM);
7632	err = list_first_entry(&tr->err_log, struct tracing_log_err, list);
7633	kfree(objp: err->cmd);
7634	err->cmd = cmd;
7635	list_del(entry: &err->list);
7636
7637	return err;
7638	}
7639
7640	/**
7641	* err_pos - find the position of a string within a command for error careting
7642	* @cmd: The tracing command that caused the error
7643	* @str: The string to position the caret at within @cmd
7644	*
7645	* Finds the position of the first occurrence of @str within @cmd. The
7646	* return value can be passed to tracing_log_err() for caret placement
7647	* within @cmd.
7648	*
7649	* Returns the index within @cmd of the first occurrence of @str or 0
7650	* if @str was not found.
7651	*/
7652	unsigned int err_pos(char *cmd, const char *str)
7653	{
7654	char *found;
7655
7656	if (WARN_ON(!strlen(cmd)))
7657	return 0;
7658
7659	found = strstr(cmd, str);
7660	if (found)
7661	return found - cmd;
7662
7663	return 0;
7664	}
7665
7666	/**
7667	* tracing_log_err - write an error to the tracing error log
7668	* @tr: The associated trace array for the error (NULL for top level array)
7669	* @loc: A string describing where the error occurred
7670	* @cmd: The tracing command that caused the error
7671	* @errs: The array of loc-specific static error strings
7672	* @type: The index into errs[], which produces the specific static err string
7673	* @pos: The position the caret should be placed in the cmd
7674	*
7675	* Writes an error into tracing/error_log of the form:
7676	*
7677	* <loc>: error: <text>
7678	* Command: <cmd>
7679	* ^
7680	*
7681	* tracing/error_log is a small log file containing the last
7682	* TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated
7683	* unless there has been a tracing error, and the error log can be
7684	* cleared and have its memory freed by writing the empty string in
7685	* truncation mode to it i.e. echo > tracing/error_log.
7686	*
7687	* NOTE: the @errs array along with the @type param are used to
7688	* produce a static error string - this string is not copied and saved
7689	* when the error is logged - only a pointer to it is saved. See
7690	* existing callers for examples of how static strings are typically
7691	* defined for use with tracing_log_err().
7692	*/
7693	void tracing_log_err(struct trace_array *tr,
7694	const char *loc, const char *cmd,
7695	const char **errs, u8 type, u16 pos)
7696	{
7697	struct tracing_log_err *err;
7698	int len = 0;
7699
7700	if (!tr)
7701	tr = &global_trace;
7702
7703	len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;
7704
7705	mutex_lock(&tracing_err_log_lock);
7706	err = get_tracing_log_err(tr, len);
7707	if (PTR_ERR(ptr: err) == -ENOMEM) {
7708	mutex_unlock(lock: &tracing_err_log_lock);
7709	return;
7710	}
7711
7712	snprintf(buf: err->loc, TRACING_LOG_LOC_MAX, fmt: "%s: error: ", loc);
7713	snprintf(buf: err->cmd, size: len, fmt: "\n" CMD_PREFIX "%s\n", cmd);
7714
7715	err->info.errs = errs;
7716	err->info.type = type;
7717	err->info.pos = pos;
7718	err->info.ts = local_clock();
7719
7720	list_add_tail(new: &err->list, head: &tr->err_log);
7721	mutex_unlock(lock: &tracing_err_log_lock);
7722	}
7723
7724	static void clear_tracing_err_log(struct trace_array *tr)
7725	{
7726	struct tracing_log_err *err, *next;
7727
7728	mutex_lock(&tracing_err_log_lock);
7729	list_for_each_entry_safe(err, next, &tr->err_log, list) {
7730	list_del(entry: &err->list);
7731	free_tracing_log_err(err);
7732	}
7733
7734	tr->n_err_log_entries = 0;
7735	mutex_unlock(lock: &tracing_err_log_lock);
7736	}
7737
7738	static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos)
7739	{
7740	struct trace_array *tr = m->private;
7741
7742	mutex_lock(&tracing_err_log_lock);
7743
7744	return seq_list_start(head: &tr->err_log, pos: *pos);
7745	}
7746
7747	static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos)
7748	{
7749	struct trace_array *tr = m->private;
7750
7751	return seq_list_next(v, head: &tr->err_log, ppos: pos);
7752	}
7753
7754	static void tracing_err_log_seq_stop(struct seq_file *m, void *v)
7755	{
7756	mutex_unlock(lock: &tracing_err_log_lock);
7757	}
7758
7759	static void tracing_err_log_show_pos(struct seq_file *m, u16 pos)
7760	{
7761	u16 i;
7762
7763	for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)
7764	seq_putc(m, c: ' ');
7765	for (i = 0; i < pos; i++)
7766	seq_putc(m, c: ' ');
7767	seq_puts(m, s: "^\n");
7768	}
7769
7770	static int tracing_err_log_seq_show(struct seq_file *m, void *v)
7771	{
7772	struct tracing_log_err *err = v;
7773
7774	if (err) {
7775	const char *err_text = err->info.errs[err->info.type];
7776	u64 sec = err->info.ts;
7777	u32 nsec;
7778
7779	nsec = do_div(sec, NSEC_PER_SEC);
7780	seq_printf(m, fmt: "[%5llu.%06u] %s%s", sec, nsec / 1000,
7781	err->loc, err_text);
7782	seq_printf(m, fmt: "%s", err->cmd);
7783	tracing_err_log_show_pos(m, pos: err->info.pos);
7784	}
7785
7786	return 0;
7787	}
7788
7789	static const struct seq_operations tracing_err_log_seq_ops = {
7790	.start = tracing_err_log_seq_start,
7791	.next = tracing_err_log_seq_next,
7792	.stop = tracing_err_log_seq_stop,
7793	.show = tracing_err_log_seq_show
7794	};
7795
7796	static int tracing_err_log_open(struct inode *inode, struct file *file)
7797	{
7798	struct trace_array *tr = inode->i_private;
7799	int ret = 0;
7800
7801	ret = tracing_check_open_get_tr(tr);
7802	if (ret)
7803	return ret;
7804
7805	/* If this file was opened for write, then erase contents */
7806	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))
7807	clear_tracing_err_log(tr);
7808
7809	if (file->f_mode & FMODE_READ) {
7810	ret = seq_open(file, &tracing_err_log_seq_ops);
7811	if (!ret) {
7812	struct seq_file *m = file->private_data;
7813	m->private = tr;
7814	} else {
7815	trace_array_put(tr);
7816	}
7817	}
7818	return ret;
7819	}
7820
7821	static ssize_t tracing_err_log_write(struct file *file,
7822	const char __user *buffer,
7823	size_t count, loff_t *ppos)
7824	{
7825	return count;
7826	}
7827
7828	static int tracing_err_log_release(struct inode *inode, struct file *file)
7829	{
7830	struct trace_array *tr = inode->i_private;
7831
7832	trace_array_put(tr);
7833
7834	if (file->f_mode & FMODE_READ)
7835	seq_release(inode, file);
7836
7837	return 0;
7838	}
7839
7840	static const struct file_operations tracing_err_log_fops = {
7841	.open = tracing_err_log_open,
7842	.write = tracing_err_log_write,
7843	.read = seq_read,
7844	.llseek = tracing_lseek,
7845	.release = tracing_err_log_release,
7846	};
7847
7848	static int tracing_buffers_open(struct inode *inode, struct file *filp)
7849	{
7850	struct trace_array *tr = inode->i_private;
7851	struct ftrace_buffer_info *info;
7852	int ret;
7853
7854	ret = tracing_check_open_get_tr(tr);
7855	if (ret)
7856	return ret;
7857
7858	info = kvzalloc(size: sizeof(*info), GFP_KERNEL);
7859	if (!info) {
7860	trace_array_put(tr);
7861	return -ENOMEM;
7862	}
7863
7864	mutex_lock(&trace_types_lock);
7865
7866	info->iter.tr = tr;
7867	info->iter.cpu_file = tracing_get_cpu(inode);
7868	info->iter.trace = tr->current_trace;
7869	info->iter.array_buffer = &tr->array_buffer;
7870	info->spare = NULL;
7871	/* Force reading ring buffer for first read */
7872	info->read = (unsigned int)-1;
7873
7874	filp->private_data = info;
7875
7876	tr->trace_ref++;
7877
7878	mutex_unlock(lock: &trace_types_lock);
7879
7880	ret = nonseekable_open(inode, filp);
7881	if (ret < 0)
7882	trace_array_put(tr);
7883
7884	return ret;
7885	}
7886
7887	static __poll_t
7888	tracing_buffers_poll(struct file *filp, poll_table *poll_table)
7889	{
7890	struct ftrace_buffer_info *info = filp->private_data;
7891	struct trace_iterator *iter = &info->iter;
7892
7893	return trace_poll(iter, filp, poll_table);
7894	}
7895
7896	static ssize_t
7897	tracing_buffers_read(struct file *filp, char __user *ubuf,
7898	size_t count, loff_t *ppos)
7899	{
7900	struct ftrace_buffer_info *info = filp->private_data;
7901	struct trace_iterator *iter = &info->iter;
7902	void *trace_data;
7903	int page_size;
7904	ssize_t ret = 0;
7905	ssize_t size;
7906
7907	if (!count)
7908	return 0;
7909
7910	#ifdef CONFIG_TRACER_MAX_TRACE
7911	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
7912	return -EBUSY;
7913	#endif
7914
7915	page_size = ring_buffer_subbuf_size_get(buffer: iter->array_buffer->buffer);
7916
7917	/* Make sure the spare matches the current sub buffer size */
7918	if (info->spare) {
7919	if (page_size != info->spare_size) {
7920	ring_buffer_free_read_page(buffer: iter->array_buffer->buffer,
7921	cpu: info->spare_cpu, page: info->spare);
7922	info->spare = NULL;
7923	}
7924	}
7925
7926	if (!info->spare) {
7927	info->spare = ring_buffer_alloc_read_page(buffer: iter->array_buffer->buffer,
7928	cpu: iter->cpu_file);
7929	if (IS_ERR(ptr: info->spare)) {
7930	ret = PTR_ERR(ptr: info->spare);
7931	info->spare = NULL;
7932	} else {
7933	info->spare_cpu = iter->cpu_file;
7934	info->spare_size = page_size;
7935	}
7936	}
7937	if (!info->spare)
7938	return ret;
7939
7940	/* Do we have previous read data to read? */
7941	if (info->read < page_size)
7942	goto read;
7943
7944	again:
7945	trace_access_lock(cpu: iter->cpu_file);
7946	ret = ring_buffer_read_page(buffer: iter->array_buffer->buffer,
7947	data_page: info->spare,
7948	len: count,
7949	cpu: iter->cpu_file, full: 0);
7950	trace_access_unlock(cpu: iter->cpu_file);
7951
7952	if (ret < 0) {
7953	if (trace_empty(iter)) {
7954	if ((filp->f_flags & O_NONBLOCK))
7955	return -EAGAIN;
7956
7957	ret = wait_on_pipe(iter, full: 0);
7958	if (ret)
7959	return ret;
7960
7961	goto again;
7962	}
7963	return 0;
7964	}
7965
7966	info->read = 0;
7967	read:
7968	size = page_size - info->read;
7969	if (size > count)
7970	size = count;
7971	trace_data = ring_buffer_read_page_data(page: info->spare);
7972	ret = copy_to_user(to: ubuf, from: trace_data + info->read, n: size);
7973	if (ret == size)
7974	return -EFAULT;
7975
7976	size -= ret;
7977
7978	*ppos += size;
7979	info->read += size;
7980
7981	return size;
7982	}
7983
7984	static int tracing_buffers_flush(struct file *file, fl_owner_t id)
7985	{
7986	struct ftrace_buffer_info *info = file->private_data;
7987	struct trace_iterator *iter = &info->iter;
7988
7989	iter->closed = true;
7990	/* Make sure the waiters see the new wait_index */
7991	(void)atomic_fetch_inc_release(v: &iter->wait_index);
7992
7993	ring_buffer_wake_waiters(buffer: iter->array_buffer->buffer, cpu: iter->cpu_file);
7994
7995	return 0;
7996	}
7997
7998	static int tracing_buffers_release(struct inode *inode, struct file *file)
7999	{
8000	struct ftrace_buffer_info *info = file->private_data;
8001	struct trace_iterator *iter = &info->iter;
8002
8003	mutex_lock(&trace_types_lock);
8004
8005	iter->tr->trace_ref--;
8006
8007	__trace_array_put(this_tr: iter->tr);
8008
8009	if (info->spare)
8010	ring_buffer_free_read_page(buffer: iter->array_buffer->buffer,
8011	cpu: info->spare_cpu, page: info->spare);
8012	kvfree(addr: info);
8013
8014	mutex_unlock(lock: &trace_types_lock);
8015
8016	return 0;
8017	}
8018
8019	struct buffer_ref {
8020	struct trace_buffer *buffer;
8021	void *page;
8022	int cpu;
8023	refcount_t refcount;
8024	};
8025
8026	static void buffer_ref_release(struct buffer_ref *ref)
8027	{
8028	if (!refcount_dec_and_test(r: &ref->refcount))
8029	return;
8030	ring_buffer_free_read_page(buffer: ref->buffer, cpu: ref->cpu, page: ref->page);
8031	kfree(objp: ref);
8032	}
8033
8034	static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
8035	struct pipe_buffer *buf)
8036	{
8037	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8038
8039	buffer_ref_release(ref);
8040	buf->private = 0;
8041	}
8042
8043	static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
8044	struct pipe_buffer *buf)
8045	{
8046	struct buffer_ref *ref = (struct buffer_ref *)buf->private;
8047
8048	if (refcount_read(r: &ref->refcount) > INT_MAX/2)
8049	return false;
8050
8051	refcount_inc(r: &ref->refcount);
8052	return true;
8053	}
8054
8055	/* Pipe buffer operations for a buffer. */
8056	static const struct pipe_buf_operations buffer_pipe_buf_ops = {
8057	.release = buffer_pipe_buf_release,
8058	.get = buffer_pipe_buf_get,
8059	};
8060
8061	/*
8062	* Callback from splice_to_pipe(), if we need to release some pages
8063	* at the end of the spd in case we error'ed out in filling the pipe.
8064	*/
8065	static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
8066	{
8067	struct buffer_ref *ref =
8068	(struct buffer_ref *)spd->partial[i].private;
8069
8070	buffer_ref_release(ref);
8071	spd->partial[i].private = 0;
8072	}
8073
8074	static ssize_t
8075	tracing_buffers_splice_read(struct file *file, loff_t *ppos,
8076	struct pipe_inode_info *pipe, size_t len,
8077	unsigned int flags)
8078	{
8079	struct ftrace_buffer_info *info = file->private_data;
8080	struct trace_iterator *iter = &info->iter;
8081	struct partial_page partial_def[PIPE_DEF_BUFFERS];
8082	struct page *pages_def[PIPE_DEF_BUFFERS];
8083	struct splice_pipe_desc spd = {
8084	.pages = pages_def,
8085	.partial = partial_def,
8086	.nr_pages_max = PIPE_DEF_BUFFERS,
8087	.ops = &buffer_pipe_buf_ops,
8088	.spd_release = buffer_spd_release,
8089	};
8090	struct buffer_ref *ref;
8091	bool woken = false;
8092	int page_size;
8093	int entries, i;
8094	ssize_t ret = 0;
8095
8096	#ifdef CONFIG_TRACER_MAX_TRACE
8097	if (iter->snapshot && iter->tr->current_trace->use_max_tr)
8098	return -EBUSY;
8099	#endif
8100
8101	page_size = ring_buffer_subbuf_size_get(buffer: iter->array_buffer->buffer);
8102	if (*ppos & (page_size - 1))
8103	return -EINVAL;
8104
8105	if (len & (page_size - 1)) {
8106	if (len < page_size)
8107	return -EINVAL;
8108	len &= (~(page_size - 1));
8109	}
8110
8111	if (splice_grow_spd(pipe, &spd))
8112	return -ENOMEM;
8113
8114	again:
8115	trace_access_lock(cpu: iter->cpu_file);
8116	entries = ring_buffer_entries_cpu(buffer: iter->array_buffer->buffer, cpu: iter->cpu_file);
8117
8118	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) {
8119	struct page *page;
8120	int r;
8121
8122	ref = kzalloc(size: sizeof(*ref), GFP_KERNEL);
8123	if (!ref) {
8124	ret = -ENOMEM;
8125	break;
8126	}
8127
8128	refcount_set(r: &ref->refcount, n: 1);
8129	ref->buffer = iter->array_buffer->buffer;
8130	ref->page = ring_buffer_alloc_read_page(buffer: ref->buffer, cpu: iter->cpu_file);
8131	if (IS_ERR(ptr: ref->page)) {
8132	ret = PTR_ERR(ptr: ref->page);
8133	ref->page = NULL;
8134	kfree(objp: ref);
8135	break;
8136	}
8137	ref->cpu = iter->cpu_file;
8138
8139	r = ring_buffer_read_page(buffer: ref->buffer, data_page: ref->page,
8140	len, cpu: iter->cpu_file, full: 1);
8141	if (r < 0) {
8142	ring_buffer_free_read_page(buffer: ref->buffer, cpu: ref->cpu,
8143	page: ref->page);
8144	kfree(objp: ref);
8145	break;
8146	}
8147
8148	page = virt_to_page(ring_buffer_read_page_data(ref->page));
8149
8150	spd.pages[i] = page;
8151	spd.partial[i].len = page_size;
8152	spd.partial[i].offset = 0;
8153	spd.partial[i].private = (unsigned long)ref;
8154	spd.nr_pages++;
8155	*ppos += page_size;
8156
8157	entries = ring_buffer_entries_cpu(buffer: iter->array_buffer->buffer, cpu: iter->cpu_file);
8158	}
8159
8160	trace_access_unlock(cpu: iter->cpu_file);
8161	spd.nr_pages = i;
8162
8163	/* did we read anything? */
8164	if (!spd.nr_pages) {
8165
8166	if (ret)
8167	goto out;
8168
8169	if (woken)
8170	goto out;
8171
8172	ret = -EAGAIN;
8173	if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
8174	goto out;
8175
8176	ret = wait_on_pipe(iter, full: iter->snapshot ? 0 : iter->tr->buffer_percent);
8177	if (ret)
8178	goto out;
8179
8180	/* No need to wait after waking up when tracing is off */
8181	if (!tracer_tracing_is_on(tr: iter->tr))
8182	goto out;
8183
8184	/* Iterate one more time to collect any new data then exit */
8185	woken = true;
8186
8187	goto again;
8188	}
8189
8190	ret = splice_to_pipe(pipe, spd: &spd);
8191	out:
8192	splice_shrink_spd(&spd);
8193
8194	return ret;
8195	}
8196
8197	/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */
8198	static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
8199	{
8200	struct ftrace_buffer_info *info = file->private_data;
8201	struct trace_iterator *iter = &info->iter;
8202
8203	if (cmd)
8204	return -ENOIOCTLCMD;
8205
8206	mutex_lock(&trace_types_lock);
8207
8208	/* Make sure the waiters see the new wait_index */
8209	(void)atomic_fetch_inc_release(v: &iter->wait_index);
8210
8211	ring_buffer_wake_waiters(buffer: iter->array_buffer->buffer, cpu: iter->cpu_file);
8212
8213	mutex_unlock(lock: &trace_types_lock);
8214	return 0;
8215	}
8216
8217	static const struct file_operations tracing_buffers_fops = {
8218	.open = tracing_buffers_open,
8219	.read = tracing_buffers_read,
8220	.poll = tracing_buffers_poll,
8221	.release = tracing_buffers_release,
8222	.flush = tracing_buffers_flush,
8223	.splice_read = tracing_buffers_splice_read,
8224	.unlocked_ioctl = tracing_buffers_ioctl,
8225	.llseek = no_llseek,
8226	};
8227
8228	static ssize_t
8229	tracing_stats_read(struct file *filp, char __user *ubuf,
8230	size_t count, loff_t *ppos)
8231	{
8232	struct inode *inode = file_inode(f: filp);
8233	struct trace_array *tr = inode->i_private;
8234	struct array_buffer *trace_buf = &tr->array_buffer;
8235	int cpu = tracing_get_cpu(inode);
8236	struct trace_seq *s;
8237	unsigned long cnt;
8238	unsigned long long t;
8239	unsigned long usec_rem;
8240
8241	s = kmalloc(size: sizeof(*s), GFP_KERNEL);
8242	if (!s)
8243	return -ENOMEM;
8244
8245	trace_seq_init(s);
8246
8247	cnt = ring_buffer_entries_cpu(buffer: trace_buf->buffer, cpu);
8248	trace_seq_printf(s, fmt: "entries: %ld\n", cnt);
8249
8250	cnt = ring_buffer_overrun_cpu(buffer: trace_buf->buffer, cpu);
8251	trace_seq_printf(s, fmt: "overrun: %ld\n", cnt);
8252
8253	cnt = ring_buffer_commit_overrun_cpu(buffer: trace_buf->buffer, cpu);
8254	trace_seq_printf(s, fmt: "commit overrun: %ld\n", cnt);
8255
8256	cnt = ring_buffer_bytes_cpu(buffer: trace_buf->buffer, cpu);
8257	trace_seq_printf(s, fmt: "bytes: %ld\n", cnt);
8258
8259	if (trace_clocks[tr->clock_id].in_ns) {
8260	/* local or global for trace_clock */
8261	t = ns2usecs(nsec: ring_buffer_oldest_event_ts(buffer: trace_buf->buffer, cpu));
8262	usec_rem = do_div(t, USEC_PER_SEC);
8263	trace_seq_printf(s, fmt: "oldest event ts: %5llu.%06lu\n",
8264	t, usec_rem);
8265
8266	t = ns2usecs(nsec: ring_buffer_time_stamp(buffer: trace_buf->buffer));
8267	usec_rem = do_div(t, USEC_PER_SEC);
8268	trace_seq_printf(s, fmt: "now ts: %5llu.%06lu\n", t, usec_rem);
8269	} else {
8270	/* counter or tsc mode for trace_clock */
8271	trace_seq_printf(s, fmt: "oldest event ts: %llu\n",
8272	ring_buffer_oldest_event_ts(buffer: trace_buf->buffer, cpu));
8273
8274	trace_seq_printf(s, fmt: "now ts: %llu\n",
8275	ring_buffer_time_stamp(buffer: trace_buf->buffer));
8276	}
8277
8278	cnt = ring_buffer_dropped_events_cpu(buffer: trace_buf->buffer, cpu);
8279	trace_seq_printf(s, fmt: "dropped events: %ld\n", cnt);
8280
8281	cnt = ring_buffer_read_events_cpu(buffer: trace_buf->buffer, cpu);
8282	trace_seq_printf(s, fmt: "read events: %ld\n", cnt);
8283
8284	count = simple_read_from_buffer(to: ubuf, count, ppos,
8285	from: s->buffer, available: trace_seq_used(s));
8286
8287	kfree(objp: s);
8288
8289	return count;
8290	}
8291
8292	static const struct file_operations tracing_stats_fops = {
8293	.open = tracing_open_generic_tr,
8294	.read = tracing_stats_read,
8295	.llseek = generic_file_llseek,
8296	.release = tracing_release_generic_tr,
8297	};
8298
8299	#ifdef CONFIG_DYNAMIC_FTRACE
8300
8301	static ssize_t
8302	tracing_read_dyn_info(struct file *filp, char __user *ubuf,
8303	size_t cnt, loff_t *ppos)
8304	{
8305	ssize_t ret;
8306	char *buf;
8307	int r;
8308
8309	/* 256 should be plenty to hold the amount needed */
8310	buf = kmalloc(size: 256, GFP_KERNEL);
8311	if (!buf)
8312	return -ENOMEM;
8313
8314	r = scnprintf(buf, size: 256, fmt: "%ld pages:%ld groups: %ld\n",
8315	ftrace_update_tot_cnt,
8316	ftrace_number_of_pages,
8317	ftrace_number_of_groups);
8318
8319	ret = simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
8320	kfree(objp: buf);
8321	return ret;
8322	}
8323
8324	static const struct file_operations tracing_dyn_info_fops = {
8325	.open = tracing_open_generic,
8326	.read = tracing_read_dyn_info,
8327	.llseek = generic_file_llseek,
8328	};
8329	#endif /* CONFIG_DYNAMIC_FTRACE */
8330
8331	#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
8332	static void
8333	ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
8334	struct trace_array *tr, struct ftrace_probe_ops *ops,
8335	void *data)
8336	{
8337	tracing_snapshot_instance(tr);
8338	}
8339
8340	static void
8341	ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
8342	struct trace_array *tr, struct ftrace_probe_ops *ops,
8343	void *data)
8344	{
8345	struct ftrace_func_mapper *mapper = data;
8346	long *count = NULL;
8347
8348	if (mapper)
8349	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8350
8351	if (count) {
8352
8353	if (*count <= 0)
8354	return;
8355
8356	(*count)--;
8357	}
8358
8359	tracing_snapshot_instance(tr);
8360	}
8361
8362	static int
8363	ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
8364	struct ftrace_probe_ops *ops, void *data)
8365	{
8366	struct ftrace_func_mapper *mapper = data;
8367	long *count = NULL;
8368
8369	seq_printf(m, fmt: "%ps:", (void *)ip);
8370
8371	seq_puts(m, s: "snapshot");
8372
8373	if (mapper)
8374	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
8375
8376	if (count)
8377	seq_printf(m, fmt: ":count=%ld\n", *count);
8378	else
8379	seq_puts(m, s: ":unlimited\n");
8380
8381	return 0;
8382	}
8383
8384	static int
8385	ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
8386	unsigned long ip, void *init_data, void **data)
8387	{
8388	struct ftrace_func_mapper *mapper = *data;
8389
8390	if (!mapper) {
8391	mapper = allocate_ftrace_func_mapper();
8392	if (!mapper)
8393	return -ENOMEM;
8394	*data = mapper;
8395	}
8396
8397	return ftrace_func_mapper_add_ip(mapper, ip, data: init_data);
8398	}
8399
8400	static void
8401	ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
8402	unsigned long ip, void *data)
8403	{
8404	struct ftrace_func_mapper *mapper = data;
8405
8406	if (!ip) {
8407	if (!mapper)
8408	return;
8409	free_ftrace_func_mapper(mapper, NULL);
8410	return;
8411	}
8412
8413	ftrace_func_mapper_remove_ip(mapper, ip);
8414	}
8415
8416	static struct ftrace_probe_ops snapshot_probe_ops = {
8417	.func = ftrace_snapshot,
8418	.print = ftrace_snapshot_print,
8419	};
8420
8421	static struct ftrace_probe_ops snapshot_count_probe_ops = {
8422	.func = ftrace_count_snapshot,
8423	.print = ftrace_snapshot_print,
8424	.init = ftrace_snapshot_init,
8425	.free = ftrace_snapshot_free,
8426	};
8427
8428	static int
8429	ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
8430	char *glob, char *cmd, char *param, int enable)
8431	{
8432	struct ftrace_probe_ops *ops;
8433	void *count = (void *)-1;
8434	char *number;
8435	int ret;
8436
8437	if (!tr)
8438	return -ENODEV;
8439
8440	/* hash funcs only work with set_ftrace_filter */
8441	if (!enable)
8442	return -EINVAL;
8443
8444	ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
8445
8446	if (glob[0] == '!') {
8447	ret = unregister_ftrace_function_probe_func(glob: glob+1, tr, ops);
8448	if (!ret)
8449	tracing_disarm_snapshot(tr);
8450
8451	return ret;
8452	}
8453
8454	if (!param)
8455	goto out_reg;
8456
8457	number = strsep(&param, ":");
8458
8459	if (!strlen(number))
8460	goto out_reg;
8461
8462	/*
8463	* We use the callback data field (which is a pointer)
8464	* as our counter.
8465	*/
8466	ret = kstrtoul(s: number, base: 0, res: (unsigned long *)&count);
8467	if (ret)
8468	return ret;
8469
8470	out_reg:
8471	ret = tracing_arm_snapshot(tr);
8472	if (ret < 0)
8473	goto out;
8474
8475	ret = register_ftrace_function_probe(glob, tr, ops, data: count);
8476	if (ret < 0)
8477	tracing_disarm_snapshot(tr);
8478	out:
8479	return ret < 0 ? ret : 0;
8480	}
8481
8482	static struct ftrace_func_command ftrace_snapshot_cmd = {
8483	.name = "snapshot",
8484	.func = ftrace_trace_snapshot_callback,
8485	};
8486
8487	static __init int register_snapshot_cmd(void)
8488	{
8489	return register_ftrace_command(cmd: &ftrace_snapshot_cmd);
8490	}
8491	#else
8492	static inline __init int register_snapshot_cmd(void) { return 0; }
8493	#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
8494
8495	static struct dentry *tracing_get_dentry(struct trace_array *tr)
8496	{
8497	if (WARN_ON(!tr->dir))
8498	return ERR_PTR(error: -ENODEV);
8499
8500	/* Top directory uses NULL as the parent */
8501	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
8502	return NULL;
8503
8504	/* All sub buffers have a descriptor */
8505	return tr->dir;
8506	}
8507
8508	static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
8509	{
8510	struct dentry *d_tracer;
8511
8512	if (tr->percpu_dir)
8513	return tr->percpu_dir;
8514
8515	d_tracer = tracing_get_dentry(tr);
8516	if (IS_ERR(ptr: d_tracer))
8517	return NULL;
8518
8519	tr->percpu_dir = tracefs_create_dir(name: "per_cpu", parent: d_tracer);
8520
8521	MEM_FAIL(!tr->percpu_dir,
8522	"Could not create tracefs directory 'per_cpu/%d'\n", cpu);
8523
8524	return tr->percpu_dir;
8525	}
8526
8527	static struct dentry *
8528	trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
8529	void *data, long cpu, const struct file_operations *fops)
8530	{
8531	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
8532
8533	if (ret) /* See tracing_get_cpu() */
8534	d_inode(dentry: ret)->i_cdev = (void *)(cpu + 1);
8535	return ret;
8536	}
8537
8538	static void
8539	tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
8540	{
8541	struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
8542	struct dentry *d_cpu;
8543	char cpu_dir[30]; /* 30 characters should be more than enough */
8544
8545	if (!d_percpu)
8546	return;
8547
8548	snprintf(buf: cpu_dir, size: 30, fmt: "cpu%ld", cpu);
8549	d_cpu = tracefs_create_dir(name: cpu_dir, parent: d_percpu);
8550	if (!d_cpu) {
8551	pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
8552	return;
8553	}
8554
8555	/* per cpu trace_pipe */
8556	trace_create_cpu_file(name: "trace_pipe", TRACE_MODE_READ, parent: d_cpu,
8557	data: tr, cpu, fops: &tracing_pipe_fops);
8558
8559	/* per cpu trace */
8560	trace_create_cpu_file(name: "trace", TRACE_MODE_WRITE, parent: d_cpu,
8561	data: tr, cpu, fops: &tracing_fops);
8562
8563	trace_create_cpu_file(name: "trace_pipe_raw", TRACE_MODE_READ, parent: d_cpu,
8564	data: tr, cpu, fops: &tracing_buffers_fops);
8565
8566	trace_create_cpu_file(name: "stats", TRACE_MODE_READ, parent: d_cpu,
8567	data: tr, cpu, fops: &tracing_stats_fops);
8568
8569	trace_create_cpu_file(name: "buffer_size_kb", TRACE_MODE_READ, parent: d_cpu,
8570	data: tr, cpu, fops: &tracing_entries_fops);
8571
8572	#ifdef CONFIG_TRACER_SNAPSHOT
8573	trace_create_cpu_file(name: "snapshot", TRACE_MODE_WRITE, parent: d_cpu,
8574	data: tr, cpu, fops: &snapshot_fops);
8575
8576	trace_create_cpu_file(name: "snapshot_raw", TRACE_MODE_READ, parent: d_cpu,
8577	data: tr, cpu, fops: &snapshot_raw_fops);
8578	#endif
8579	}
8580
8581	#ifdef CONFIG_FTRACE_SELFTEST
8582	/* Let selftest have access to static functions in this file */
8583	#include "trace_selftest.c"
8584	#endif
8585
8586	static ssize_t
8587	trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
8588	loff_t *ppos)
8589	{
8590	struct trace_option_dentry *topt = filp->private_data;
8591	char *buf;
8592
8593	if (topt->flags->val & topt->opt->bit)
8594	buf = "1\n";
8595	else
8596	buf = "0\n";
8597
8598	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: 2);
8599	}
8600
8601	static ssize_t
8602	trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
8603	loff_t *ppos)
8604	{
8605	struct trace_option_dentry *topt = filp->private_data;
8606	unsigned long val;
8607	int ret;
8608
8609	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
8610	if (ret)
8611	return ret;
8612
8613	if (val != 0 && val != 1)
8614	return -EINVAL;
8615
8616	if (!!(topt->flags->val & topt->opt->bit) != val) {
8617	mutex_lock(&trace_types_lock);
8618	ret = __set_tracer_option(tr: topt->tr, tracer_flags: topt->flags,
8619	opts: topt->opt, neg: !val);
8620	mutex_unlock(lock: &trace_types_lock);
8621	if (ret)
8622	return ret;
8623	}
8624
8625	*ppos += cnt;
8626
8627	return cnt;
8628	}
8629
8630	static int tracing_open_options(struct inode *inode, struct file *filp)
8631	{
8632	struct trace_option_dentry *topt = inode->i_private;
8633	int ret;
8634
8635	ret = tracing_check_open_get_tr(tr: topt->tr);
8636	if (ret)
8637	return ret;
8638
8639	filp->private_data = inode->i_private;
8640	return 0;
8641	}
8642
8643	static int tracing_release_options(struct inode *inode, struct file *file)
8644	{
8645	struct trace_option_dentry *topt = file->private_data;
8646
8647	trace_array_put(topt->tr);
8648	return 0;
8649	}
8650
8651	static const struct file_operations trace_options_fops = {
8652	.open = tracing_open_options,
8653	.read = trace_options_read,
8654	.write = trace_options_write,
8655	.llseek = generic_file_llseek,
8656	.release = tracing_release_options,
8657	};
8658
8659	/*
8660	* In order to pass in both the trace_array descriptor as well as the index
8661	* to the flag that the trace option file represents, the trace_array
8662	* has a character array of trace_flags_index[], which holds the index
8663	* of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
8664	* The address of this character array is passed to the flag option file
8665	* read/write callbacks.
8666	*
8667	* In order to extract both the index and the trace_array descriptor,
8668	* get_tr_index() uses the following algorithm.
8669	*
8670	* idx = *ptr;
8671	*
8672	* As the pointer itself contains the address of the index (remember
8673	* index[1] == 1).
8674	*
8675	* Then to get the trace_array descriptor, by subtracting that index
8676	* from the ptr, we get to the start of the index itself.
8677	*
8678	* ptr - idx == &index[0]
8679	*
8680	* Then a simple container_of() from that pointer gets us to the
8681	* trace_array descriptor.
8682	*/
8683	static void get_tr_index(void *data, struct trace_array **ptr,
8684	unsigned int *pindex)
8685	{
8686	*pindex = *(unsigned char *)data;
8687
8688	*ptr = container_of(data - *pindex, struct trace_array,
8689	trace_flags_index);
8690	}
8691
8692	static ssize_t
8693	trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
8694	loff_t *ppos)
8695	{
8696	void *tr_index = filp->private_data;
8697	struct trace_array *tr;
8698	unsigned int index;
8699	char *buf;
8700
8701	get_tr_index(data: tr_index, ptr: &tr, pindex: &index);
8702
8703	if (tr->trace_flags & (1 << index))
8704	buf = "1\n";
8705	else
8706	buf = "0\n";
8707
8708	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: 2);
8709	}
8710
8711	static ssize_t
8712	trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
8713	loff_t *ppos)
8714	{
8715	void *tr_index = filp->private_data;
8716	struct trace_array *tr;
8717	unsigned int index;
8718	unsigned long val;
8719	int ret;
8720
8721	get_tr_index(data: tr_index, ptr: &tr, pindex: &index);
8722
8723	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
8724	if (ret)
8725	return ret;
8726
8727	if (val != 0 && val != 1)
8728	return -EINVAL;
8729
8730	mutex_lock(&event_mutex);
8731	mutex_lock(&trace_types_lock);
8732	ret = set_tracer_flag(tr, mask: 1 << index, enabled: val);
8733	mutex_unlock(lock: &trace_types_lock);
8734	mutex_unlock(lock: &event_mutex);
8735
8736	if (ret < 0)
8737	return ret;
8738
8739	*ppos += cnt;
8740
8741	return cnt;
8742	}
8743
8744	static const struct file_operations trace_options_core_fops = {
8745	.open = tracing_open_generic,
8746	.read = trace_options_core_read,
8747	.write = trace_options_core_write,
8748	.llseek = generic_file_llseek,
8749	};
8750
8751	struct dentry *trace_create_file(const char *name,
8752	umode_t mode,
8753	struct dentry *parent,
8754	void *data,
8755	const struct file_operations *fops)
8756	{
8757	struct dentry *ret;
8758
8759	ret = tracefs_create_file(name, mode, parent, data, fops);
8760	if (!ret)
8761	pr_warn("Could not create tracefs '%s' entry\n", name);
8762
8763	return ret;
8764	}
8765
8766
8767	static struct dentry *trace_options_init_dentry(struct trace_array *tr)
8768	{
8769	struct dentry *d_tracer;
8770
8771	if (tr->options)
8772	return tr->options;
8773
8774	d_tracer = tracing_get_dentry(tr);
8775	if (IS_ERR(ptr: d_tracer))
8776	return NULL;
8777
8778	tr->options = tracefs_create_dir(name: "options", parent: d_tracer);
8779	if (!tr->options) {
8780	pr_warn("Could not create tracefs directory 'options'\n");
8781	return NULL;
8782	}
8783
8784	return tr->options;
8785	}
8786
8787	static void
8788	create_trace_option_file(struct trace_array *tr,
8789	struct trace_option_dentry *topt,
8790	struct tracer_flags *flags,
8791	struct tracer_opt *opt)
8792	{
8793	struct dentry *t_options;
8794
8795	t_options = trace_options_init_dentry(tr);
8796	if (!t_options)
8797	return;
8798
8799	topt->flags = flags;
8800	topt->opt = opt;
8801	topt->tr = tr;
8802
8803	topt->entry = trace_create_file(name: opt->name, TRACE_MODE_WRITE,
8804	parent: t_options, data: topt, fops: &trace_options_fops);
8805
8806	}
8807
8808	static void
8809	create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
8810	{
8811	struct trace_option_dentry *topts;
8812	struct trace_options *tr_topts;
8813	struct tracer_flags *flags;
8814	struct tracer_opt *opts;
8815	int cnt;
8816	int i;
8817
8818	if (!tracer)
8819	return;
8820
8821	flags = tracer->flags;
8822
8823	if (!flags || !flags->opts)
8824	return;
8825
8826	/*
8827	* If this is an instance, only create flags for tracers
8828	* the instance may have.
8829	*/
8830	if (!trace_ok_for_array(t: tracer, tr))
8831	return;
8832
8833	for (i = 0; i < tr->nr_topts; i++) {
8834	/* Make sure there's no duplicate flags. */
8835	if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
8836	return;
8837	}
8838
8839	opts = flags->opts;
8840
8841	for (cnt = 0; opts[cnt].name; cnt++)
8842	;
8843
8844	topts = kcalloc(n: cnt + 1, size: sizeof(*topts), GFP_KERNEL);
8845	if (!topts)
8846	return;
8847
8848	tr_topts = krealloc(objp: tr->topts, new_size: sizeof(*tr->topts) * (tr->nr_topts + 1),
8849	GFP_KERNEL);
8850	if (!tr_topts) {
8851	kfree(objp: topts);
8852	return;
8853	}
8854
8855	tr->topts = tr_topts;
8856	tr->topts[tr->nr_topts].tracer = tracer;
8857	tr->topts[tr->nr_topts].topts = topts;
8858	tr->nr_topts++;
8859
8860	for (cnt = 0; opts[cnt].name; cnt++) {
8861	create_trace_option_file(tr, topt: &topts[cnt], flags,
8862	opt: &opts[cnt]);
8863	MEM_FAIL(topts[cnt].entry == NULL,
8864	"Failed to create trace option: %s",
8865	opts[cnt].name);
8866	}
8867	}
8868
8869	static struct dentry *
8870	create_trace_option_core_file(struct trace_array *tr,
8871	const char *option, long index)
8872	{
8873	struct dentry *t_options;
8874
8875	t_options = trace_options_init_dentry(tr);
8876	if (!t_options)
8877	return NULL;
8878
8879	return trace_create_file(name: option, TRACE_MODE_WRITE, parent: t_options,
8880	data: (void *)&tr->trace_flags_index[index],
8881	fops: &trace_options_core_fops);
8882	}
8883
8884	static void create_trace_options_dir(struct trace_array *tr)
8885	{
8886	struct dentry *t_options;
8887	bool top_level = tr == &global_trace;
8888	int i;
8889
8890	t_options = trace_options_init_dentry(tr);
8891	if (!t_options)
8892	return;
8893
8894	for (i = 0; trace_options[i]; i++) {
8895	if (top_level ||
8896	!((1 << i) & TOP_LEVEL_TRACE_FLAGS))
8897	create_trace_option_core_file(tr, option: trace_options[i], index: i);
8898	}
8899	}
8900
8901	static ssize_t
8902	rb_simple_read(struct file *filp, char __user *ubuf,
8903	size_t cnt, loff_t *ppos)
8904	{
8905	struct trace_array *tr = filp->private_data;
8906	char buf[64];
8907	int r;
8908
8909	r = tracer_tracing_is_on(tr);
8910	r = sprintf(buf, fmt: "%d\n", r);
8911
8912	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
8913	}
8914
8915	static ssize_t
8916	rb_simple_write(struct file *filp, const char __user *ubuf,
8917	size_t cnt, loff_t *ppos)
8918	{
8919	struct trace_array *tr = filp->private_data;
8920	struct trace_buffer *buffer = tr->array_buffer.buffer;
8921	unsigned long val;
8922	int ret;
8923
8924	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
8925	if (ret)
8926	return ret;
8927
8928	if (buffer) {
8929	mutex_lock(&trace_types_lock);
8930	if (!!val == tracer_tracing_is_on(tr)) {
8931	val = 0; /* do nothing */
8932	} else if (val) {
8933	tracer_tracing_on(tr);
8934	if (tr->current_trace->start)
8935	tr->current_trace->start(tr);
8936	} else {
8937	tracer_tracing_off(tr);
8938	if (tr->current_trace->stop)
8939	tr->current_trace->stop(tr);
8940	/* Wake up any waiters */
8941	ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS);
8942	}
8943	mutex_unlock(lock: &trace_types_lock);
8944	}
8945
8946	(*ppos)++;
8947
8948	return cnt;
8949	}
8950
8951	static const struct file_operations rb_simple_fops = {
8952	.open = tracing_open_generic_tr,
8953	.read = rb_simple_read,
8954	.write = rb_simple_write,
8955	.release = tracing_release_generic_tr,
8956	.llseek = default_llseek,
8957	};
8958
8959	static ssize_t
8960	buffer_percent_read(struct file *filp, char __user *ubuf,
8961	size_t cnt, loff_t *ppos)
8962	{
8963	struct trace_array *tr = filp->private_data;
8964	char buf[64];
8965	int r;
8966
8967	r = tr->buffer_percent;
8968	r = sprintf(buf, fmt: "%d\n", r);
8969
8970	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
8971	}
8972
8973	static ssize_t
8974	buffer_percent_write(struct file *filp, const char __user *ubuf,
8975	size_t cnt, loff_t *ppos)
8976	{
8977	struct trace_array *tr = filp->private_data;
8978	unsigned long val;
8979	int ret;
8980
8981	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
8982	if (ret)
8983	return ret;
8984
8985	if (val > 100)
8986	return -EINVAL;
8987
8988	tr->buffer_percent = val;
8989
8990	(*ppos)++;
8991
8992	return cnt;
8993	}
8994
8995	static const struct file_operations buffer_percent_fops = {
8996	.open = tracing_open_generic_tr,
8997	.read = buffer_percent_read,
8998	.write = buffer_percent_write,
8999	.release = tracing_release_generic_tr,
9000	.llseek = default_llseek,
9001	};
9002
9003	static ssize_t
9004	buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
9005	{
9006	struct trace_array *tr = filp->private_data;
9007	size_t size;
9008	char buf[64];
9009	int order;
9010	int r;
9011
9012	order = ring_buffer_subbuf_order_get(buffer: tr->array_buffer.buffer);
9013	size = (PAGE_SIZE << order) / 1024;
9014
9015	r = sprintf(buf, fmt: "%zd\n", size);
9016
9017	return simple_read_from_buffer(to: ubuf, count: cnt, ppos, from: buf, available: r);
9018	}
9019
9020	static ssize_t
9021	buffer_subbuf_size_write(struct file *filp, const char __user *ubuf,
9022	size_t cnt, loff_t *ppos)
9023	{
9024	struct trace_array *tr = filp->private_data;
9025	unsigned long val;
9026	int old_order;
9027	int order;
9028	int pages;
9029	int ret;
9030
9031	ret = kstrtoul_from_user(s: ubuf, count: cnt, base: 10, res: &val);
9032	if (ret)
9033	return ret;
9034
9035	val *= 1024; /* value passed in is in KB */
9036
9037	pages = DIV_ROUND_UP(val, PAGE_SIZE);
9038	order = fls(x: pages - 1);
9039
9040	/* limit between 1 and 128 system pages */
9041	if (order < 0 || order > 7)
9042	return -EINVAL;
9043
9044	/* Do not allow tracing while changing the order of the ring buffer */
9045	tracing_stop_tr(tr);
9046
9047	old_order = ring_buffer_subbuf_order_get(buffer: tr->array_buffer.buffer);
9048	if (old_order == order)
9049	goto out;
9050
9051	ret = ring_buffer_subbuf_order_set(buffer: tr->array_buffer.buffer, order);
9052	if (ret)
9053	goto out;
9054
9055	#ifdef CONFIG_TRACER_MAX_TRACE
9056
9057	if (!tr->allocated_snapshot)
9058	goto out_max;
9059
9060	ret = ring_buffer_subbuf_order_set(buffer: tr->max_buffer.buffer, order);
9061	if (ret) {
9062	/* Put back the old order */
9063	cnt = ring_buffer_subbuf_order_set(buffer: tr->array_buffer.buffer, order: old_order);
9064	if (WARN_ON_ONCE(cnt)) {
9065	/*
9066	* AARGH! We are left with different orders!
9067	* The max buffer is our "snapshot" buffer.
9068	* When a tracer needs a snapshot (one of the
9069	* latency tracers), it swaps the max buffer
9070	* with the saved snap shot. We succeeded to
9071	* update the order of the main buffer, but failed to
9072	* update the order of the max buffer. But when we tried
9073	* to reset the main buffer to the original size, we
9074	* failed there too. This is very unlikely to
9075	* happen, but if it does, warn and kill all
9076	* tracing.
9077	*/
9078	tracing_disabled = 1;
9079	}
9080	goto out;
9081	}
9082	out_max:
9083	#endif
9084	(*ppos)++;
9085	out:
9086	if (ret)
9087	cnt = ret;
9088	tracing_start_tr(tr);
9089	return cnt;
9090	}
9091
9092	static const struct file_operations buffer_subbuf_size_fops = {
9093	.open = tracing_open_generic_tr,
9094	.read = buffer_subbuf_size_read,
9095	.write = buffer_subbuf_size_write,
9096	.release = tracing_release_generic_tr,
9097	.llseek = default_llseek,
9098	};
9099
9100	static struct dentry *trace_instance_dir;
9101
9102	static void
9103	init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
9104
9105	static int
9106	allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size)
9107	{
9108	enum ring_buffer_flags rb_flags;
9109
9110	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
9111
9112	buf->tr = tr;
9113
9114	buf->buffer = ring_buffer_alloc(size, rb_flags);
9115	if (!buf->buffer)
9116	return -ENOMEM;
9117
9118	buf->data = alloc_percpu(struct trace_array_cpu);
9119	if (!buf->data) {
9120	ring_buffer_free(buffer: buf->buffer);
9121	buf->buffer = NULL;
9122	return -ENOMEM;
9123	}
9124
9125	/* Allocate the first page for all buffers */
9126	set_buffer_entries(buf: &tr->array_buffer,
9127	val: ring_buffer_size(buffer: tr->array_buffer.buffer, cpu: 0));
9128
9129	return 0;
9130	}
9131
9132	static void free_trace_buffer(struct array_buffer *buf)
9133	{
9134	if (buf->buffer) {
9135	ring_buffer_free(buffer: buf->buffer);
9136	buf->buffer = NULL;
9137	free_percpu(pdata: buf->data);
9138	buf->data = NULL;
9139	}
9140	}
9141
9142	static int allocate_trace_buffers(struct trace_array *tr, int size)
9143	{
9144	int ret;
9145
9146	ret = allocate_trace_buffer(tr, buf: &tr->array_buffer, size);
9147	if (ret)
9148	return ret;
9149
9150	#ifdef CONFIG_TRACER_MAX_TRACE
9151	ret = allocate_trace_buffer(tr, buf: &tr->max_buffer,
9152	size: allocate_snapshot ? size : 1);
9153	if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) {
9154	free_trace_buffer(buf: &tr->array_buffer);
9155	return -ENOMEM;
9156	}
9157	tr->allocated_snapshot = allocate_snapshot;
9158
9159	allocate_snapshot = false;
9160	#endif
9161
9162	return 0;
9163	}
9164
9165	static void free_trace_buffers(struct trace_array *tr)
9166	{
9167	if (!tr)
9168	return;
9169
9170	free_trace_buffer(buf: &tr->array_buffer);
9171
9172	#ifdef CONFIG_TRACER_MAX_TRACE
9173	free_trace_buffer(buf: &tr->max_buffer);
9174	#endif
9175	}
9176
9177	static void init_trace_flags_index(struct trace_array *tr)
9178	{
9179	int i;
9180
9181	/* Used by the trace options files */
9182	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
9183	tr->trace_flags_index[i] = i;
9184	}
9185
9186	static void __update_tracer_options(struct trace_array *tr)
9187	{
9188	struct tracer *t;
9189
9190	for (t = trace_types; t; t = t->next)
9191	add_tracer_options(tr, t);
9192	}
9193
9194	static void update_tracer_options(struct trace_array *tr)
9195	{
9196	mutex_lock(&trace_types_lock);
9197	tracer_options_updated = true;
9198	__update_tracer_options(tr);
9199	mutex_unlock(lock: &trace_types_lock);
9200	}
9201
9202	/* Must have trace_types_lock held */
9203	struct trace_array *trace_array_find(const char *instance)
9204	{
9205	struct trace_array *tr, *found = NULL;
9206
9207	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9208	if (tr->name && strcmp(tr->name, instance) == 0) {
9209	found = tr;
9210	break;
9211	}
9212	}
9213
9214	return found;
9215	}
9216
9217	struct trace_array *trace_array_find_get(const char *instance)
9218	{
9219	struct trace_array *tr;
9220
9221	mutex_lock(&trace_types_lock);
9222	tr = trace_array_find(instance);
9223	if (tr)
9224	tr->ref++;
9225	mutex_unlock(lock: &trace_types_lock);
9226
9227	return tr;
9228	}
9229
9230	static int trace_array_create_dir(struct trace_array *tr)
9231	{
9232	int ret;
9233
9234	tr->dir = tracefs_create_dir(name: tr->name, parent: trace_instance_dir);
9235	if (!tr->dir)
9236	return -EINVAL;
9237
9238	ret = event_trace_add_tracer(parent: tr->dir, tr);
9239	if (ret) {
9240	tracefs_remove(dentry: tr->dir);
9241	return ret;
9242	}
9243
9244	init_tracer_tracefs(tr, d_tracer: tr->dir);
9245	__update_tracer_options(tr);
9246
9247	return ret;
9248	}
9249
9250	static struct trace_array *
9251	trace_array_create_systems(const char *name, const char *systems)
9252	{
9253	struct trace_array *tr;
9254	int ret;
9255
9256	ret = -ENOMEM;
9257	tr = kzalloc(size: sizeof(*tr), GFP_KERNEL);
9258	if (!tr)
9259	return ERR_PTR(error: ret);
9260
9261	tr->name = kstrdup(s: name, GFP_KERNEL);
9262	if (!tr->name)
9263	goto out_free_tr;
9264
9265	if (!alloc_cpumask_var(mask: &tr->tracing_cpumask, GFP_KERNEL))
9266	goto out_free_tr;
9267
9268	if (!zalloc_cpumask_var(mask: &tr->pipe_cpumask, GFP_KERNEL))
9269	goto out_free_tr;
9270
9271	if (systems) {
9272	tr->system_names = kstrdup_const(s: systems, GFP_KERNEL);
9273	if (!tr->system_names)
9274	goto out_free_tr;
9275	}
9276
9277	tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
9278
9279	cpumask_copy(dstp: tr->tracing_cpumask, cpu_all_mask);
9280
9281	raw_spin_lock_init(&tr->start_lock);
9282
9283	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
9284	#ifdef CONFIG_TRACER_MAX_TRACE
9285	spin_lock_init(&tr->snapshot_trigger_lock);
9286	#endif
9287	tr->current_trace = &nop_trace;
9288
9289	INIT_LIST_HEAD(list: &tr->systems);
9290	INIT_LIST_HEAD(list: &tr->events);
9291	INIT_LIST_HEAD(list: &tr->hist_vars);
9292	INIT_LIST_HEAD(list: &tr->err_log);
9293
9294	if (allocate_trace_buffers(tr, size: trace_buf_size) < 0)
9295	goto out_free_tr;
9296
9297	/* The ring buffer is defaultly expanded */
9298	trace_set_ring_buffer_expanded(tr);
9299
9300	if (ftrace_allocate_ftrace_ops(tr) < 0)
9301	goto out_free_tr;
9302
9303	ftrace_init_trace_array(tr);
9304
9305	init_trace_flags_index(tr);
9306
9307	if (trace_instance_dir) {
9308	ret = trace_array_create_dir(tr);
9309	if (ret)
9310	goto out_free_tr;
9311	} else
9312	__trace_early_add_events(tr);
9313
9314	list_add(new: &tr->list, head: &ftrace_trace_arrays);
9315
9316	tr->ref++;
9317
9318	return tr;
9319
9320	out_free_tr:
9321	ftrace_free_ftrace_ops(tr);
9322	free_trace_buffers(tr);
9323	free_cpumask_var(mask: tr->pipe_cpumask);
9324	free_cpumask_var(mask: tr->tracing_cpumask);
9325	kfree_const(x: tr->system_names);
9326	kfree(objp: tr->name);
9327	kfree(objp: tr);
9328
9329	return ERR_PTR(error: ret);
9330	}
9331
9332	static struct trace_array *trace_array_create(const char *name)
9333	{
9334	return trace_array_create_systems(name, NULL);
9335	}
9336
9337	static int instance_mkdir(const char *name)
9338	{
9339	struct trace_array *tr;
9340	int ret;
9341
9342	mutex_lock(&event_mutex);
9343	mutex_lock(&trace_types_lock);
9344
9345	ret = -EEXIST;
9346	if (trace_array_find(instance: name))
9347	goto out_unlock;
9348
9349	tr = trace_array_create(name);
9350
9351	ret = PTR_ERR_OR_ZERO(ptr: tr);
9352
9353	out_unlock:
9354	mutex_unlock(lock: &trace_types_lock);
9355	mutex_unlock(lock: &event_mutex);
9356	return ret;
9357	}
9358
9359	/**
9360	* trace_array_get_by_name - Create/Lookup a trace array, given its name.
9361	* @name: The name of the trace array to be looked up/created.
9362	* @systems: A list of systems to create event directories for (NULL for all)
9363	*
9364	* Returns pointer to trace array with given name.
9365	* NULL, if it cannot be created.
9366	*
9367	* NOTE: This function increments the reference counter associated with the
9368	* trace array returned. This makes sure it cannot be freed while in use.
9369	* Use trace_array_put() once the trace array is no longer needed.
9370	* If the trace_array is to be freed, trace_array_destroy() needs to
9371	* be called after the trace_array_put(), or simply let user space delete
9372	* it from the tracefs instances directory. But until the
9373	* trace_array_put() is called, user space can not delete it.
9374	*
9375	*/
9376	struct trace_array *trace_array_get_by_name(const char *name, const char *systems)
9377	{
9378	struct trace_array *tr;
9379
9380	mutex_lock(&event_mutex);
9381	mutex_lock(&trace_types_lock);
9382
9383	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9384	if (tr->name && strcmp(tr->name, name) == 0)
9385	goto out_unlock;
9386	}
9387
9388	tr = trace_array_create_systems(name, systems);
9389
9390	if (IS_ERR(ptr: tr))
9391	tr = NULL;
9392	out_unlock:
9393	if (tr)
9394	tr->ref++;
9395
9396	mutex_unlock(lock: &trace_types_lock);
9397	mutex_unlock(lock: &event_mutex);
9398	return tr;
9399	}
9400	EXPORT_SYMBOL_GPL(trace_array_get_by_name);
9401
9402	static int __remove_instance(struct trace_array *tr)
9403	{
9404	int i;
9405
9406	/* Reference counter for a newly created trace array = 1. */
9407	if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))
9408	return -EBUSY;
9409
9410	list_del(entry: &tr->list);
9411
9412	/* Disable all the flags that were enabled coming in */
9413	for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
9414	if ((1 << i) & ZEROED_TRACE_FLAGS)
9415	set_tracer_flag(tr, mask: 1 << i, enabled: 0);
9416	}
9417
9418	tracing_set_nop(tr);
9419	clear_ftrace_function_probes(tr);
9420	event_trace_del_tracer(tr);
9421	ftrace_clear_pids(tr);
9422	ftrace_destroy_function_files(tr);
9423	tracefs_remove(dentry: tr->dir);
9424	free_percpu(pdata: tr->last_func_repeats);
9425	free_trace_buffers(tr);
9426	clear_tracing_err_log(tr);
9427
9428	for (i = 0; i < tr->nr_topts; i++) {
9429	kfree(objp: tr->topts[i].topts);
9430	}
9431	kfree(objp: tr->topts);
9432
9433	free_cpumask_var(mask: tr->pipe_cpumask);
9434	free_cpumask_var(mask: tr->tracing_cpumask);
9435	kfree_const(x: tr->system_names);
9436	kfree(objp: tr->name);
9437	kfree(objp: tr);
9438
9439	return 0;
9440	}
9441
9442	int trace_array_destroy(struct trace_array *this_tr)
9443	{
9444	struct trace_array *tr;
9445	int ret;
9446
9447	if (!this_tr)
9448	return -EINVAL;
9449
9450	mutex_lock(&event_mutex);
9451	mutex_lock(&trace_types_lock);
9452
9453	ret = -ENODEV;
9454
9455	/* Making sure trace array exists before destroying it. */
9456	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9457	if (tr == this_tr) {
9458	ret = __remove_instance(tr);
9459	break;
9460	}
9461	}
9462
9463	mutex_unlock(lock: &trace_types_lock);
9464	mutex_unlock(lock: &event_mutex);
9465
9466	return ret;
9467	}
9468	EXPORT_SYMBOL_GPL(trace_array_destroy);
9469
9470	static int instance_rmdir(const char *name)
9471	{
9472	struct trace_array *tr;
9473	int ret;
9474
9475	mutex_lock(&event_mutex);
9476	mutex_lock(&trace_types_lock);
9477
9478	ret = -ENODEV;
9479	tr = trace_array_find(instance: name);
9480	if (tr)
9481	ret = __remove_instance(tr);
9482
9483	mutex_unlock(lock: &trace_types_lock);
9484	mutex_unlock(lock: &event_mutex);
9485
9486	return ret;
9487	}
9488
9489	static __init void create_trace_instances(struct dentry *d_tracer)
9490	{
9491	struct trace_array *tr;
9492
9493	trace_instance_dir = tracefs_create_instance_dir(name: "instances", parent: d_tracer,
9494	mkdir: instance_mkdir,
9495	rmdir: instance_rmdir);
9496	if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n"))
9497	return;
9498
9499	mutex_lock(&event_mutex);
9500	mutex_lock(&trace_types_lock);
9501
9502	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
9503	if (!tr->name)
9504	continue;
9505	if (MEM_FAIL(trace_array_create_dir(tr) < 0,
9506	"Failed to create instance directory\n"))
9507	break;
9508	}
9509
9510	mutex_unlock(lock: &trace_types_lock);
9511	mutex_unlock(lock: &event_mutex);
9512	}
9513
9514	static void
9515	init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
9516	{
9517	int cpu;
9518
9519	trace_create_file(name: "available_tracers", TRACE_MODE_READ, parent: d_tracer,
9520	data: tr, fops: &show_traces_fops);
9521
9522	trace_create_file(name: "current_tracer", TRACE_MODE_WRITE, parent: d_tracer,
9523	data: tr, fops: &set_tracer_fops);
9524
9525	trace_create_file(name: "tracing_cpumask", TRACE_MODE_WRITE, parent: d_tracer,
9526	data: tr, fops: &tracing_cpumask_fops);
9527
9528	trace_create_file(name: "trace_options", TRACE_MODE_WRITE, parent: d_tracer,
9529	data: tr, fops: &tracing_iter_fops);
9530
9531	trace_create_file(name: "trace", TRACE_MODE_WRITE, parent: d_tracer,
9532	data: tr, fops: &tracing_fops);
9533
9534	trace_create_file(name: "trace_pipe", TRACE_MODE_READ, parent: d_tracer,
9535	data: tr, fops: &tracing_pipe_fops);
9536
9537	trace_create_file(name: "buffer_size_kb", TRACE_MODE_WRITE, parent: d_tracer,
9538	data: tr, fops: &tracing_entries_fops);
9539
9540	trace_create_file(name: "buffer_total_size_kb", TRACE_MODE_READ, parent: d_tracer,
9541	data: tr, fops: &tracing_total_entries_fops);
9542
9543	trace_create_file(name: "free_buffer", mode: 0200, parent: d_tracer,
9544	data: tr, fops: &tracing_free_buffer_fops);
9545
9546	trace_create_file(name: "trace_marker", mode: 0220, parent: d_tracer,
9547	data: tr, fops: &tracing_mark_fops);
9548
9549	tr->trace_marker_file = __find_event_file(tr, system: "ftrace", event: "print");
9550
9551	trace_create_file(name: "trace_marker_raw", mode: 0220, parent: d_tracer,
9552	data: tr, fops: &tracing_mark_raw_fops);
9553
9554	trace_create_file(name: "trace_clock", TRACE_MODE_WRITE, parent: d_tracer, data: tr,
9555	fops: &trace_clock_fops);
9556
9557	trace_create_file(name: "tracing_on", TRACE_MODE_WRITE, parent: d_tracer,
9558	data: tr, fops: &rb_simple_fops);
9559
9560	trace_create_file(name: "timestamp_mode", TRACE_MODE_READ, parent: d_tracer, data: tr,
9561	fops: &trace_time_stamp_mode_fops);
9562
9563	tr->buffer_percent = 50;
9564
9565	trace_create_file(name: "buffer_percent", TRACE_MODE_WRITE, parent: d_tracer,
9566	data: tr, fops: &buffer_percent_fops);
9567
9568	trace_create_file(name: "buffer_subbuf_size_kb", TRACE_MODE_WRITE, parent: d_tracer,
9569	data: tr, fops: &buffer_subbuf_size_fops);
9570
9571	create_trace_options_dir(tr);
9572
9573	#ifdef CONFIG_TRACER_MAX_TRACE
9574	trace_create_maxlat_file(tr, d_tracer);
9575	#endif
9576
9577	if (ftrace_create_function_files(tr, parent: d_tracer))
9578	MEM_FAIL(1, "Could not allocate function filter files");
9579
9580	#ifdef CONFIG_TRACER_SNAPSHOT
9581	trace_create_file(name: "snapshot", TRACE_MODE_WRITE, parent: d_tracer,
9582	data: tr, fops: &snapshot_fops);
9583	#endif
9584
9585	trace_create_file(name: "error_log", TRACE_MODE_WRITE, parent: d_tracer,
9586	data: tr, fops: &tracing_err_log_fops);
9587
9588	for_each_tracing_cpu(cpu)
9589	tracing_init_tracefs_percpu(tr, cpu);
9590
9591	ftrace_init_tracefs(tr, d_tracer);
9592	}
9593
9594	static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
9595	{
9596	struct vfsmount *mnt;
9597	struct file_system_type *type;
9598
9599	/*
9600	* To maintain backward compatibility for tools that mount
9601	* debugfs to get to the tracing facility, tracefs is automatically
9602	* mounted to the debugfs/tracing directory.
9603	*/
9604	type = get_fs_type(name: "tracefs");
9605	if (!type)
9606	return NULL;
9607	mnt = vfs_submount(mountpoint: mntpt, type, name: "tracefs", NULL);
9608	put_filesystem(fs: type);
9609	if (IS_ERR(ptr: mnt))
9610	return NULL;
9611	mntget(mnt);
9612
9613	return mnt;
9614	}
9615
9616	/**
9617	* tracing_init_dentry - initialize top level trace array
9618	*
9619	* This is called when creating files or directories in the tracing
9620	* directory. It is called via fs_initcall() by any of the boot up code
9621	* and expects to return the dentry of the top level tracing directory.
9622	*/
9623	int tracing_init_dentry(void)
9624	{
9625	struct trace_array *tr = &global_trace;
9626
9627	if (security_locked_down(what: LOCKDOWN_TRACEFS)) {
9628	pr_warn("Tracing disabled due to lockdown\n");
9629	return -EPERM;
9630	}
9631
9632	/* The top level trace array uses NULL as parent */
9633	if (tr->dir)
9634	return 0;
9635
9636	if (WARN_ON(!tracefs_initialized()))
9637	return -ENODEV;
9638
9639	/*
9640	* As there may still be users that expect the tracing
9641	* files to exist in debugfs/tracing, we must automount
9642	* the tracefs file system there, so older tools still
9643	* work with the newer kernel.
9644	*/
9645	tr->dir = debugfs_create_automount(name: "tracing", NULL,
9646	f: trace_automount, NULL);
9647
9648	return 0;
9649	}
9650
9651	extern struct trace_eval_map *__start_ftrace_eval_maps[];
9652	extern struct trace_eval_map *__stop_ftrace_eval_maps[];
9653
9654	static struct workqueue_struct *eval_map_wq __initdata;
9655	static struct work_struct eval_map_work __initdata;
9656	static struct work_struct tracerfs_init_work __initdata;
9657
9658	static void __init eval_map_work_func(struct work_struct *work)
9659	{
9660	int len;
9661
9662	len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
9663	trace_insert_eval_map(NULL, start: __start_ftrace_eval_maps, len);
9664	}
9665
9666	static int __init trace_eval_init(void)
9667	{
9668	INIT_WORK(&eval_map_work, eval_map_work_func);
9669
9670	eval_map_wq = alloc_workqueue(fmt: "eval_map_wq", flags: WQ_UNBOUND, max_active: 0);
9671	if (!eval_map_wq) {
9672	pr_err("Unable to allocate eval_map_wq\n");
9673	/* Do work here */
9674	eval_map_work_func(work: &eval_map_work);
9675	return -ENOMEM;
9676	}
9677
9678	queue_work(wq: eval_map_wq, work: &eval_map_work);
9679	return 0;
9680	}
9681
9682	subsys_initcall(trace_eval_init);
9683
9684	static int __init trace_eval_sync(void)
9685	{
9686	/* Make sure the eval map updates are finished */
9687	if (eval_map_wq)
9688	destroy_workqueue(wq: eval_map_wq);
9689	return 0;
9690	}
9691
9692	late_initcall_sync(trace_eval_sync);
9693
9694
9695	#ifdef CONFIG_MODULES
9696	static void trace_module_add_evals(struct module *mod)
9697	{
9698	if (!mod->num_trace_evals)
9699	return;
9700
9701	/*
9702	* Modules with bad taint do not have events created, do
9703	* not bother with enums either.
9704	*/
9705	if (trace_module_has_bad_taint(mod))
9706	return;
9707
9708	trace_insert_eval_map(mod, start: mod->trace_evals, len: mod->num_trace_evals);
9709	}
9710
9711	#ifdef CONFIG_TRACE_EVAL_MAP_FILE
9712	static void trace_module_remove_evals(struct module *mod)
9713	{
9714	union trace_eval_map_item *map;
9715	union trace_eval_map_item **last = &trace_eval_maps;
9716
9717	if (!mod->num_trace_evals)
9718	return;
9719
9720	mutex_lock(&trace_eval_mutex);
9721
9722	map = trace_eval_maps;
9723
9724	while (map) {
9725	if (map->head.mod == mod)
9726	break;
9727	map = trace_eval_jmp_to_tail(ptr: map);
9728	last = &map->tail.next;
9729	map = map->tail.next;
9730	}
9731	if (!map)
9732	goto out;
9733
9734	*last = trace_eval_jmp_to_tail(ptr: map)->tail.next;
9735	kfree(objp: map);
9736	out:
9737	mutex_unlock(lock: &trace_eval_mutex);
9738	}
9739	#else
9740	static inline void trace_module_remove_evals(struct module *mod) { }
9741	#endif /* CONFIG_TRACE_EVAL_MAP_FILE */
9742
9743	static int trace_module_notify(struct notifier_block *self,
9744	unsigned long val, void *data)
9745	{
9746	struct module *mod = data;
9747
9748	switch (val) {
9749	case MODULE_STATE_COMING:
9750	trace_module_add_evals(mod);
9751	break;
9752	case MODULE_STATE_GOING:
9753	trace_module_remove_evals(mod);
9754	break;
9755	}
9756
9757	return NOTIFY_OK;
9758	}
9759
9760	static struct notifier_block trace_module_nb = {
9761	.notifier_call = trace_module_notify,
9762	.priority = 0,
9763	};
9764	#endif /* CONFIG_MODULES */
9765
9766	static __init void tracer_init_tracefs_work_func(struct work_struct *work)
9767	{
9768
9769	event_trace_init();
9770
9771	init_tracer_tracefs(tr: &global_trace, NULL);
9772	ftrace_init_tracefs_toplevel(tr: &global_trace, NULL);
9773
9774	trace_create_file(name: "tracing_thresh", TRACE_MODE_WRITE, NULL,
9775	data: &global_trace, fops: &tracing_thresh_fops);
9776
9777	trace_create_file(name: "README", TRACE_MODE_READ, NULL,
9778	NULL, fops: &tracing_readme_fops);
9779
9780	trace_create_file(name: "saved_cmdlines", TRACE_MODE_READ, NULL,
9781	NULL, fops: &tracing_saved_cmdlines_fops);
9782
9783	trace_create_file(name: "saved_cmdlines_size", TRACE_MODE_WRITE, NULL,
9784	NULL, fops: &tracing_saved_cmdlines_size_fops);
9785
9786	trace_create_file(name: "saved_tgids", TRACE_MODE_READ, NULL,
9787	NULL, fops: &tracing_saved_tgids_fops);
9788
9789	trace_create_eval_file(NULL);
9790
9791	#ifdef CONFIG_MODULES
9792	register_module_notifier(nb: &trace_module_nb);
9793	#endif
9794
9795	#ifdef CONFIG_DYNAMIC_FTRACE
9796	trace_create_file(name: "dyn_ftrace_total_info", TRACE_MODE_READ, NULL,
9797	NULL, fops: &tracing_dyn_info_fops);
9798	#endif
9799
9800	create_trace_instances(NULL);
9801
9802	update_tracer_options(tr: &global_trace);
9803	}
9804
9805	static __init int tracer_init_tracefs(void)
9806	{
9807	int ret;
9808
9809	trace_access_lock_init();
9810
9811	ret = tracing_init_dentry();
9812	if (ret)
9813	return 0;
9814
9815	if (eval_map_wq) {
9816	INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func);
9817	queue_work(wq: eval_map_wq, work: &tracerfs_init_work);
9818	} else {
9819	tracer_init_tracefs_work_func(NULL);
9820	}
9821
9822	rv_init_interface();
9823
9824	return 0;
9825	}
9826
9827	fs_initcall(tracer_init_tracefs);
9828
9829	static int trace_die_panic_handler(struct notifier_block *self,
9830	unsigned long ev, void *unused);
9831
9832	static struct notifier_block trace_panic_notifier = {
9833	.notifier_call = trace_die_panic_handler,
9834	.priority = INT_MAX - 1,
9835	};
9836
9837	static struct notifier_block trace_die_notifier = {
9838	.notifier_call = trace_die_panic_handler,
9839	.priority = INT_MAX - 1,
9840	};
9841
9842	/*
9843	* The idea is to execute the following die/panic callback early, in order
9844	* to avoid showing irrelevant information in the trace (like other panic
9845	* notifier functions); we are the 2nd to run, after hung_task/rcu_stall
9846	* warnings get disabled (to prevent potential log flooding).
9847	*/
9848	static int trace_die_panic_handler(struct notifier_block *self,
9849	unsigned long ev, void *unused)
9850	{
9851	if (!ftrace_dump_on_oops_enabled())
9852	return NOTIFY_DONE;
9853
9854	/* The die notifier requires DIE_OOPS to trigger */
9855	if (self == &trace_die_notifier && ev != DIE_OOPS)
9856	return NOTIFY_DONE;
9857
9858	ftrace_dump(oops_dump_mode: DUMP_PARAM);
9859
9860	return NOTIFY_DONE;
9861	}
9862
9863	/*
9864	* printk is set to max of 1024, we really don't need it that big.
9865	* Nothing should be printing 1000 characters anyway.
9866	*/
9867	#define TRACE_MAX_PRINT 1000
9868
9869	/*
9870	* Define here KERN_TRACE so that we have one place to modify
9871	* it if we decide to change what log level the ftrace dump
9872	* should be at.
9873	*/
9874	#define KERN_TRACE KERN_EMERG
9875
9876	void
9877	trace_printk_seq(struct trace_seq *s)
9878	{
9879	/* Probably should print a warning here. */
9880	if (s->seq.len >= TRACE_MAX_PRINT)
9881	s->seq.len = TRACE_MAX_PRINT;
9882
9883	/*
9884	* More paranoid code. Although the buffer size is set to
9885	* PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
9886	* an extra layer of protection.
9887	*/
9888	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
9889	s->seq.len = s->seq.size - 1;
9890
9891	/* should be zero ended, but we are paranoid. */
9892	s->buffer[s->seq.len] = 0;
9893
9894	printk(KERN_TRACE "%s", s->buffer);
9895
9896	trace_seq_init(s);
9897	}
9898
9899	static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr)
9900	{
9901	iter->tr = tr;
9902	iter->trace = iter->tr->current_trace;
9903	iter->cpu_file = RING_BUFFER_ALL_CPUS;
9904	iter->array_buffer = &tr->array_buffer;
9905
9906	if (iter->trace && iter->trace->open)
9907	iter->trace->open(iter);
9908
9909	/* Annotate start of buffers if we had overruns */
9910	if (ring_buffer_overruns(buffer: iter->array_buffer->buffer))
9911	iter->iter_flags |= TRACE_FILE_ANNOTATE;
9912
9913	/* Output in nanoseconds only if we are using a clock in nanoseconds. */
9914	if (trace_clocks[iter->tr->clock_id].in_ns)
9915	iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
9916
9917	/* Can not use kmalloc for iter.temp and iter.fmt */
9918	iter->temp = static_temp_buf;
9919	iter->temp_size = STATIC_TEMP_BUF_SIZE;
9920	iter->fmt = static_fmt_buf;
9921	iter->fmt_size = STATIC_FMT_BUF_SIZE;
9922	}
9923
9924	void trace_init_global_iter(struct trace_iterator *iter)
9925	{
9926	trace_init_iter(iter, tr: &global_trace);
9927	}
9928
9929	static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode)
9930	{
9931	/* use static because iter can be a bit big for the stack */
9932	static struct trace_iterator iter;
9933	unsigned int old_userobj;
9934	unsigned long flags;
9935	int cnt = 0, cpu;
9936
9937	/*
9938	* Always turn off tracing when we dump.
9939	* We don't need to show trace output of what happens
9940	* between multiple crashes.
9941	*
9942	* If the user does a sysrq-z, then they can re-enable
9943	* tracing with echo 1 > tracing_on.
9944	*/
9945	tracer_tracing_off(tr);
9946
9947	local_irq_save(flags);
9948
9949	/* Simulate the iterator */
9950	trace_init_iter(iter: &iter, tr);
9951
9952	for_each_tracing_cpu(cpu) {
9953	atomic_inc(v: &per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
9954	}
9955
9956	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
9957
9958	/* don't look at user memory in panic mode */
9959	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
9960
9961	if (dump_mode == DUMP_ORIG)
9962	iter.cpu_file = raw_smp_processor_id();
9963	else
9964	iter.cpu_file = RING_BUFFER_ALL_CPUS;
9965
9966	if (tr == &global_trace)
9967	printk(KERN_TRACE "Dumping ftrace buffer:\n");
9968	else
9969	printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
9970
9971	/* Did function tracer already get disabled? */
9972	if (ftrace_is_dead()) {
9973	printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
9974	printk("# MAY BE MISSING FUNCTION EVENTS\n");
9975	}
9976
9977	/*
9978	* We need to stop all tracing on all CPUS to read
9979	* the next buffer. This is a bit expensive, but is
9980	* not done often. We fill all what we can read,
9981	* and then release the locks again.
9982	*/
9983
9984	while (!trace_empty(iter: &iter)) {
9985
9986	if (!cnt)
9987	printk(KERN_TRACE "---------------------------------\n");
9988
9989	cnt++;
9990
9991	trace_iterator_reset(iter: &iter);
9992	iter.iter_flags |= TRACE_FILE_LAT_FMT;
9993
9994	if (trace_find_next_entry_inc(iter: &iter) != NULL) {
9995	int ret;
9996
9997	ret = print_trace_line(iter: &iter);
9998	if (ret != TRACE_TYPE_NO_CONSUME)
9999	trace_consume(iter: &iter);
10000	}
10001	touch_nmi_watchdog();
10002
10003	trace_printk_seq(s: &iter.seq);
10004	}
10005
10006	if (!cnt)
10007	printk(KERN_TRACE " (ftrace buffer empty)\n");
10008	else
10009	printk(KERN_TRACE "---------------------------------\n");
10010
10011	tr->trace_flags |= old_userobj;
10012
10013	for_each_tracing_cpu(cpu) {
10014	atomic_dec(v: &per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
10015	}
10016	local_irq_restore(flags);
10017	}
10018
10019	static void ftrace_dump_by_param(void)
10020	{
10021	bool first_param = true;
10022	char dump_param[MAX_TRACER_SIZE];
10023	char *buf, *token, *inst_name;
10024	struct trace_array *tr;
10025
10026	strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE);
10027	buf = dump_param;
10028
10029	while ((token = strsep(&buf, ",")) != NULL) {
10030	if (first_param) {
10031	first_param = false;
10032	if (!strcmp("0", token))
10033	continue;
10034	else if (!strcmp("1", token)) {
10035	ftrace_dump_one(tr: &global_trace, dump_mode: DUMP_ALL);
10036	continue;
10037	}
10038	else if (!strcmp("2", token) ||
10039	!strcmp("orig_cpu", token)) {
10040	ftrace_dump_one(tr: &global_trace, dump_mode: DUMP_ORIG);
10041	continue;
10042	}
10043	}
10044
10045	inst_name = strsep(&token, "=");
10046	tr = trace_array_find(instance: inst_name);
10047	if (!tr) {
10048	printk(KERN_TRACE "Instance %s not found\n", inst_name);
10049	continue;
10050	}
10051
10052	if (token && (!strcmp("2", token) ||
10053	!strcmp("orig_cpu", token)))
10054	ftrace_dump_one(tr, dump_mode: DUMP_ORIG);
10055	else
10056	ftrace_dump_one(tr, dump_mode: DUMP_ALL);
10057	}
10058	}
10059
10060	void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
10061	{
10062	static atomic_t dump_running;
10063
10064	/* Only allow one dump user at a time. */
10065	if (atomic_inc_return(v: &dump_running) != 1) {
10066	atomic_dec(v: &dump_running);
10067	return;
10068	}
10069
10070	switch (oops_dump_mode) {
10071	case DUMP_ALL:
10072	ftrace_dump_one(tr: &global_trace, dump_mode: DUMP_ALL);
10073	break;
10074	case DUMP_ORIG:
10075	ftrace_dump_one(tr: &global_trace, dump_mode: DUMP_ORIG);
10076	break;
10077	case DUMP_PARAM:
10078	ftrace_dump_by_param();
10079	break;
10080	case DUMP_NONE:
10081	break;
10082	default:
10083	printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
10084	ftrace_dump_one(tr: &global_trace, dump_mode: DUMP_ALL);
10085	}
10086
10087	atomic_dec(v: &dump_running);
10088	}
10089	EXPORT_SYMBOL_GPL(ftrace_dump);
10090
10091	#define WRITE_BUFSIZE 4096
10092
10093	ssize_t trace_parse_run_command(struct file *file, const char __user *buffer,
10094	size_t count, loff_t *ppos,
10095	int (*createfn)(const char *))
10096	{
10097	char *kbuf, *buf, *tmp;
10098	int ret = 0;
10099	size_t done = 0;
10100	size_t size;
10101
10102	kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL);
10103	if (!kbuf)
10104	return -ENOMEM;
10105
10106	while (done < count) {
10107	size = count - done;
10108
10109	if (size >= WRITE_BUFSIZE)
10110	size = WRITE_BUFSIZE - 1;
10111
10112	if (copy_from_user(to: kbuf, from: buffer + done, n: size)) {
10113	ret = -EFAULT;
10114	goto out;
10115	}
10116	kbuf[size] = '\0';
10117	buf = kbuf;
10118	do {
10119	tmp = strchr(buf, '\n');
10120	if (tmp) {
10121	*tmp = '\0';
10122	size = tmp - buf + 1;
10123	} else {
10124	size = strlen(buf);
10125	if (done + size < count) {
10126	if (buf != kbuf)
10127	break;
10128	/* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */
10129	pr_warn("Line length is too long: Should be less than %d\n",
10130	WRITE_BUFSIZE - 2);
10131	ret = -EINVAL;
10132	goto out;
10133	}
10134	}
10135	done += size;
10136
10137	/* Remove comments */
10138	tmp = strchr(buf, '#');
10139
10140	if (tmp)
10141	*tmp = '\0';
10142
10143	ret = createfn(buf);
10144	if (ret)
10145	goto out;
10146	buf += size;
10147
10148	} while (done < count);
10149	}
10150	ret = done;
10151
10152	out:
10153	kfree(objp: kbuf);
10154
10155	return ret;
10156	}
10157
10158	#ifdef CONFIG_TRACER_MAX_TRACE
10159	__init static bool tr_needs_alloc_snapshot(const char *name)
10160	{
10161	char *test;
10162	int len = strlen(name);
10163	bool ret;
10164
10165	if (!boot_snapshot_index)
10166	return false;
10167
10168	if (strncmp(name, boot_snapshot_info, len) == 0 &&
10169	boot_snapshot_info[len] == '\t')
10170	return true;
10171
10172	test = kmalloc(strlen(name) + 3, GFP_KERNEL);
10173	if (!test)
10174	return false;
10175
10176	sprintf(buf: test, fmt: "\t%s\t", name);
10177	ret = strstr(boot_snapshot_info, test) == NULL;
10178	kfree(objp: test);
10179	return ret;
10180	}
10181
10182	__init static void do_allocate_snapshot(const char *name)
10183	{
10184	if (!tr_needs_alloc_snapshot(name))
10185	return;
10186
10187	/*
10188	* When allocate_snapshot is set, the next call to
10189	* allocate_trace_buffers() (called by trace_array_get_by_name())
10190	* will allocate the snapshot buffer. That will alse clear
10191	* this flag.
10192	*/
10193	allocate_snapshot = true;
10194	}
10195	#else
10196	static inline void do_allocate_snapshot(const char *name) { }
10197	#endif
10198
10199	__init static void enable_instances(void)
10200	{
10201	struct trace_array *tr;
10202	char *curr_str;
10203	char *str;
10204	char *tok;
10205
10206	/* A tab is always appended */
10207	boot_instance_info[boot_instance_index - 1] = '\0';
10208	str = boot_instance_info;
10209
10210	while ((curr_str = strsep(&str, "\t"))) {
10211
10212	tok = strsep(&curr_str, ",");
10213
10214	if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE))
10215	do_allocate_snapshot(name: tok);
10216
10217	tr = trace_array_get_by_name(tok, NULL);
10218	if (!tr) {
10219	pr_warn("Failed to create instance buffer %s\n", curr_str);
10220	continue;
10221	}
10222	/* Allow user space to delete it */
10223	trace_array_put(tr);
10224
10225	while ((tok = strsep(&curr_str, ","))) {
10226	early_enable_events(tr, buf: tok, disable_first: true);
10227	}
10228	}
10229	}
10230
10231	__init static int tracer_alloc_buffers(void)
10232	{
10233	int ring_buf_size;
10234	int ret = -ENOMEM;
10235
10236
10237	if (security_locked_down(what: LOCKDOWN_TRACEFS)) {
10238	pr_warn("Tracing disabled due to lockdown\n");
10239	return -EPERM;
10240	}
10241
10242	/*
10243	* Make sure we don't accidentally add more trace options
10244	* than we have bits for.
10245	*/
10246	BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
10247
10248	if (!alloc_cpumask_var(mask: &tracing_buffer_mask, GFP_KERNEL))
10249	goto out;
10250
10251	if (!alloc_cpumask_var(mask: &global_trace.tracing_cpumask, GFP_KERNEL))
10252	goto out_free_buffer_mask;
10253
10254	/* Only allocate trace_printk buffers if a trace_printk exists */
10255	if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
10256	/* Must be called before global_trace.buffer is allocated */
10257	trace_printk_init_buffers();
10258
10259	/* To save memory, keep the ring buffer size to its minimum */
10260	if (global_trace.ring_buffer_expanded)
10261	ring_buf_size = trace_buf_size;
10262	else
10263	ring_buf_size = 1;
10264
10265	cpumask_copy(dstp: tracing_buffer_mask, cpu_possible_mask);
10266	cpumask_copy(dstp: global_trace.tracing_cpumask, cpu_all_mask);
10267
10268	raw_spin_lock_init(&global_trace.start_lock);
10269
10270	/*
10271	* The prepare callbacks allocates some memory for the ring buffer. We
10272	* don't free the buffer if the CPU goes down. If we were to free
10273	* the buffer, then the user would lose any trace that was in the
10274	* buffer. The memory will be removed once the "instance" is removed.
10275	*/
10276	ret = cpuhp_setup_state_multi(state: CPUHP_TRACE_RB_PREPARE,
10277	name: "trace/RB:prepare", startup: trace_rb_cpu_prepare,
10278	NULL);
10279	if (ret < 0)
10280	goto out_free_cpumask;
10281	/* Used for event triggers */
10282	ret = -ENOMEM;
10283	temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
10284	if (!temp_buffer)
10285	goto out_rm_hp_state;
10286
10287	if (trace_create_savedcmd() < 0)
10288	goto out_free_temp_buffer;
10289
10290	if (!zalloc_cpumask_var(mask: &global_trace.pipe_cpumask, GFP_KERNEL))
10291	goto out_free_savedcmd;
10292
10293	/* TODO: make the number of buffers hot pluggable with CPUS */
10294	if (allocate_trace_buffers(tr: &global_trace, size: ring_buf_size) < 0) {
10295	MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n");
10296	goto out_free_pipe_cpumask;
10297	}
10298	if (global_trace.buffer_disabled)
10299	tracing_off();
10300
10301	if (trace_boot_clock) {
10302	ret = tracing_set_clock(tr: &global_trace, clockstr: trace_boot_clock);
10303	if (ret < 0)
10304	pr_warn("Trace clock %s not defined, going back to default\n",
10305	trace_boot_clock);
10306	}
10307
10308	/*
10309	* register_tracer() might reference current_trace, so it
10310	* needs to be set before we register anything. This is
10311	* just a bootstrap of current_trace anyway.
10312	*/
10313	global_trace.current_trace = &nop_trace;
10314
10315	global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
10316	#ifdef CONFIG_TRACER_MAX_TRACE
10317	spin_lock_init(&global_trace.snapshot_trigger_lock);
10318	#endif
10319	ftrace_init_global_array_ops(tr: &global_trace);
10320
10321	init_trace_flags_index(tr: &global_trace);
10322
10323	register_tracer(type: &nop_trace);
10324
10325	/* Function tracing may start here (via kernel command line) */
10326	init_function_trace();
10327
10328	/* All seems OK, enable tracing */
10329	tracing_disabled = 0;
10330
10331	atomic_notifier_chain_register(nh: &panic_notifier_list,
10332	nb: &trace_panic_notifier);
10333
10334	register_die_notifier(nb: &trace_die_notifier);
10335
10336	global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
10337
10338	INIT_LIST_HEAD(list: &global_trace.systems);
10339	INIT_LIST_HEAD(list: &global_trace.events);
10340	INIT_LIST_HEAD(list: &global_trace.hist_vars);
10341	INIT_LIST_HEAD(list: &global_trace.err_log);
10342	list_add(new: &global_trace.list, head: &ftrace_trace_arrays);
10343
10344	apply_trace_boot_options();
10345
10346	register_snapshot_cmd();
10347
10348	test_can_verify();
10349
10350	return 0;
10351
10352	out_free_pipe_cpumask:
10353	free_cpumask_var(mask: global_trace.pipe_cpumask);
10354	out_free_savedcmd:
10355	trace_free_saved_cmdlines_buffer();
10356	out_free_temp_buffer:
10357	ring_buffer_free(buffer: temp_buffer);
10358	out_rm_hp_state:
10359	cpuhp_remove_multi_state(state: CPUHP_TRACE_RB_PREPARE);
10360	out_free_cpumask:
10361	free_cpumask_var(mask: global_trace.tracing_cpumask);
10362	out_free_buffer_mask:
10363	free_cpumask_var(mask: tracing_buffer_mask);
10364	out:
10365	return ret;
10366	}
10367
10368	void __init ftrace_boot_snapshot(void)
10369	{
10370	#ifdef CONFIG_TRACER_MAX_TRACE
10371	struct trace_array *tr;
10372
10373	if (!snapshot_at_boot)
10374	return;
10375
10376	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
10377	if (!tr->allocated_snapshot)
10378	continue;
10379
10380	tracing_snapshot_instance(tr);
10381	trace_array_puts(tr, "** Boot snapshot taken **\n");
10382	}
10383	#endif
10384	}
10385
10386	void __init early_trace_init(void)
10387	{
10388	if (tracepoint_printk) {
10389	tracepoint_print_iter =
10390	kzalloc(size: sizeof(*tracepoint_print_iter), GFP_KERNEL);
10391	if (MEM_FAIL(!tracepoint_print_iter,
10392	"Failed to allocate trace iterator\n"))
10393	tracepoint_printk = 0;
10394	else
10395	static_key_enable(key: &tracepoint_printk_key.key);
10396	}
10397	tracer_alloc_buffers();
10398
10399	init_events();
10400	}
10401
10402	void __init trace_init(void)
10403	{
10404	trace_event_init();
10405
10406	if (boot_instance_index)
10407	enable_instances();
10408	}
10409
10410	__init static void clear_boot_tracer(void)
10411	{
10412	/*
10413	* The default tracer at boot buffer is an init section.
10414	* This function is called in lateinit. If we did not
10415	* find the boot tracer, then clear it out, to prevent
10416	* later registration from accessing the buffer that is
10417	* about to be freed.
10418	*/
10419	if (!default_bootup_tracer)
10420	return;
10421
10422	printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
10423	default_bootup_tracer);
10424	default_bootup_tracer = NULL;
10425	}
10426
10427	#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
10428	__init static void tracing_set_default_clock(void)
10429	{
10430	/* sched_clock_stable() is determined in late_initcall */
10431	if (!trace_boot_clock && !sched_clock_stable()) {
10432	if (security_locked_down(what: LOCKDOWN_TRACEFS)) {
10433	pr_warn("Can not set tracing clock due to lockdown\n");
10434	return;
10435	}
10436
10437	printk(KERN_WARNING
10438	"Unstable clock detected, switching default tracing clock to \"global\"\n"
10439	"If you want to keep using the local clock, then add:\n"
10440	" \"trace_clock=local\"\n"
10441	"on the kernel command line\n");
10442	tracing_set_clock(tr: &global_trace, clockstr: "global");
10443	}
10444	}
10445	#else
10446	static inline void tracing_set_default_clock(void) { }
10447	#endif
10448
10449	__init static int late_trace_init(void)
10450	{
10451	if (tracepoint_printk && tracepoint_printk_stop_on_boot) {
10452	static_key_disable(key: &tracepoint_printk_key.key);
10453	tracepoint_printk = 0;
10454	}
10455
10456	tracing_set_default_clock();
10457	clear_boot_tracer();
10458	return 0;
10459	}
10460
10461	late_initcall_sync(late_trace_init);
10462