1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Fprobe-based tracing events
4	* Copyright (C) 2022 Google LLC.
5	*/
6	#define pr_fmt(fmt) "trace_fprobe: " fmt
7
8	#include <linux/fprobe.h>
9	#include <linux/list.h>
10	#include <linux/module.h>
11	#include <linux/mutex.h>
12	#include <linux/rculist.h>
13	#include <linux/security.h>
14	#include <linux/tracepoint.h>
15	#include <linux/uaccess.h>
16
17	#include <asm/ptrace.h>
18
19	#include "trace_dynevent.h"
20	#include "trace_probe.h"
21	#include "trace_probe_kernel.h"
22	#include "trace_probe_tmpl.h"
23
24	#define FPROBE_EVENT_SYSTEM "fprobes"
25	#define TRACEPOINT_EVENT_SYSTEM "tracepoints"
26	#define RETHOOK_MAXACTIVE_MAX 4096
27
28	static int trace_fprobe_create(const char *raw_command);
29	static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
30	static int trace_fprobe_release(struct dyn_event *ev);
31	static bool trace_fprobe_is_busy(struct dyn_event *ev);
32	static bool trace_fprobe_match(const char *system, const char *event,
33	int argc, const char **argv, struct dyn_event *ev);
34
35	static struct dyn_event_operations trace_fprobe_ops = {
36	.create = trace_fprobe_create,
37	.show = trace_fprobe_show,
38	.is_busy = trace_fprobe_is_busy,
39	.free = trace_fprobe_release,
40	.match = trace_fprobe_match,
41	};
42
43	/* List of tracepoint_user */
44	static LIST_HEAD(tracepoint_user_list);
45	static DEFINE_MUTEX(tracepoint_user_mutex);
46
47	/* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */
48	struct tracepoint_user {
49	struct list_head list;
50	const char *name;
51	struct tracepoint *tpoint;
52	unsigned int refcount;
53	};
54
55	/* NOTE: you must lock tracepoint_user_mutex. */
56	#define for_each_tracepoint_user(tuser) \
57	list_for_each_entry(tuser, &tracepoint_user_list, list)
58
59	static int tracepoint_user_register(struct tracepoint_user *tuser)
60	{
61	struct tracepoint *tpoint = tuser->tpoint;
62
63	if (!tpoint)
64	return 0;
65
66	return tracepoint_probe_register_prio_may_exist(tp: tpoint,
67	probe: tpoint->probestub, NULL, prio: 0);
68	}
69
70	static void tracepoint_user_unregister(struct tracepoint_user *tuser)
71	{
72	if (!tuser->tpoint)
73	return;
74
75	WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL));
76	tuser->tpoint = NULL;
77	}
78
79	static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser)
80	{
81	if (!tuser->tpoint)
82	return 0UL;
83
84	return (unsigned long)tuser->tpoint->probestub;
85	}
86
87	static void __tracepoint_user_free(struct tracepoint_user *tuser)
88	{
89	if (!tuser)
90	return;
91	kfree(objp: tuser->name);
92	kfree(objp: tuser);
93	}
94
95	DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T))
96
97	static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint)
98	{
99	struct tracepoint_user *tuser __free(tuser_free) = NULL;
100	int ret;
101
102	tuser = kzalloc(sizeof(*tuser), GFP_KERNEL);
103	if (!tuser)
104	return NULL;
105	tuser->name = kstrdup(s: name, GFP_KERNEL);
106	if (!tuser->name)
107	return NULL;
108
109	/* Register tracepoint if it is loaded. */
110	if (tpoint) {
111	tuser->tpoint = tpoint;
112	ret = tracepoint_user_register(tuser);
113	if (ret)
114	return ERR_PTR(error: ret);
115	}
116
117	tuser->refcount = 1;
118	INIT_LIST_HEAD(list: &tuser->list);
119	list_add(new: &tuser->list, head: &tracepoint_user_list);
120
121	return_ptr(tuser);
122	}
123
124	static struct tracepoint *find_tracepoint(const char *tp_name,
125	struct module **tp_mod);
126
127	/*
128	* Get tracepoint_user if exist, or allocate new one and register it.
129	* If tracepoint is on a module, get its refcounter too.
130	* This returns errno or NULL (not loaded yet) or tracepoint_user.
131	*/
132	static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod)
133	{
134	struct module *mod __free(module_put) = NULL;
135	struct tracepoint_user *tuser;
136	struct tracepoint *tpoint;
137
138	if (!name || !pmod)
139	return ERR_PTR(error: -EINVAL);
140
141	/* Get and lock the module which has tracepoint. */
142	tpoint = find_tracepoint(tp_name: name, tp_mod: &mod);
143
144	guard(mutex)(T: &tracepoint_user_mutex);
145	/* Search existing tracepoint_user */
146	for_each_tracepoint_user(tuser) {
147	if (!strcmp(tuser->name, name)) {
148	tuser->refcount++;
149	*pmod = no_free_ptr(mod);
150	return tuser;
151	}
152	}
153
154	/* The corresponding tracepoint_user is not found. */
155	tuser = __tracepoint_user_init(name, tpoint);
156	if (!IS_ERR_OR_NULL(ptr: tuser))
157	*pmod = no_free_ptr(mod);
158
159	return tuser;
160	}
161
162	static void tracepoint_user_put(struct tracepoint_user *tuser)
163	{
164	scoped_guard(mutex, &tracepoint_user_mutex) {
165	if (--tuser->refcount > 0)
166	return;
167
168	list_del(entry: &tuser->list);
169	tracepoint_user_unregister(tuser);
170	}
171
172	__tracepoint_user_free(tuser);
173	}
174
175	DEFINE_FREE(tuser_put, struct tracepoint_user *,
176	if (!IS_ERR_OR_NULL(_T))
177	tracepoint_user_put(_T))
178
179	/*
180	* Fprobe event core functions
181	*/
182
183	/*
184	* @tprobe is true for tracepoint probe.
185	* @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not
186	* loaded with a module. If @tuser != NULL, this trace_fprobe is enabled.
187	*/
188	struct trace_fprobe {
189	struct dyn_event devent;
190	struct fprobe fp;
191	const char *symbol;
192	bool tprobe;
193	struct tracepoint_user *tuser;
194	struct trace_probe tp;
195	};
196
197	static bool is_trace_fprobe(struct dyn_event *ev)
198	{
199	return ev->ops == &trace_fprobe_ops;
200	}
201
202	static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev)
203	{
204	return container_of(ev, struct trace_fprobe, devent);
205	}
206
207	/**
208	* for_each_trace_fprobe - iterate over the trace_fprobe list
209	* @pos: the struct trace_fprobe * for each entry
210	* @dpos: the struct dyn_event * to use as a loop cursor
211	*/
212	#define for_each_trace_fprobe(pos, dpos) \
213	for_each_dyn_event(dpos) \
214	if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
215
216	static bool trace_fprobe_is_return(struct trace_fprobe *tf)
217	{
218	return tf->fp.exit_handler != NULL;
219	}
220
221	static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)
222	{
223	return tf->tprobe;
224	}
225
226	static const char *trace_fprobe_symbol(struct trace_fprobe *tf)
227	{
228	return tf->symbol ? tf->symbol : "unknown";
229	}
230
231	static bool trace_fprobe_is_busy(struct dyn_event *ev)
232	{
233	struct trace_fprobe *tf = to_trace_fprobe(ev);
234
235	return trace_probe_is_enabled(tp: &tf->tp);
236	}
237
238	static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
239	int argc, const char **argv)
240	{
241	char buf[MAX_ARGSTR_LEN + 1];
242
243	if (!argc)
244	return true;
245
246	snprintf(buf, size: sizeof(buf), fmt: "%s", trace_fprobe_symbol(tf));
247	if (strcmp(buf, argv[0]))
248	return false;
249	argc--; argv++;
250
251	return trace_probe_match_command_args(tp: &tf->tp, argc, argv);
252	}
253
254	static bool trace_fprobe_match(const char *system, const char *event,
255	int argc, const char **argv, struct dyn_event *ev)
256	{
257	struct trace_fprobe *tf = to_trace_fprobe(ev);
258
259	if (event[0] != '\0' && strcmp(trace_probe_name(tp: &tf->tp), event))
260	return false;
261
262	if (system && strcmp(trace_probe_group_name(tp: &tf->tp), system))
263	return false;
264
265	return trace_fprobe_match_command_head(tf, argc, argv);
266	}
267
268	static bool trace_fprobe_is_registered(struct trace_fprobe *tf)
269	{
270	return fprobe_is_registered(fp: &tf->fp);
271	}
272
273	/*
274	* Note that we don't verify the fetch_insn code, since it does not come
275	* from user space.
276	*/
277	static int
278	process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
279	void *dest, void *base)
280	{
281	struct ftrace_regs *fregs = rec;
282	unsigned long val;
283	int ret;
284
285	retry:
286	/* 1st stage: get value from context */
287	switch (code->op) {
288	case FETCH_OP_STACK:
289	val = ftrace_regs_get_kernel_stack_nth(fregs, nth: code->param);
290	break;
291	case FETCH_OP_STACKP:
292	val = ftrace_regs_get_stack_pointer(fregs);
293	break;
294	case FETCH_OP_RETVAL:
295	val = ftrace_regs_get_return_value(fregs);
296	break;
297	#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
298	case FETCH_OP_ARG:
299	val = ftrace_regs_get_argument(fregs, code->param);
300	break;
301	case FETCH_OP_EDATA:
302	val = *(unsigned long *)((unsigned long)edata + code->offset);
303	break;
304	#endif
305	case FETCH_NOP_SYMBOL: /* Ignore a place holder */
306	code++;
307	goto retry;
308	default:
309	ret = process_common_fetch_insn(code, val: &val);
310	if (ret < 0)
311	return ret;
312	}
313	code++;
314
315	return process_fetch_insn_bottom(code, val, dest, base);
316	}
317	NOKPROBE_SYMBOL(process_fetch_insn)
318
319	/* function entry handler */
320	static nokprobe_inline void
321	__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
322	struct ftrace_regs *fregs,
323	struct trace_event_file *trace_file)
324	{
325	struct fentry_trace_entry_head *entry;
326	struct trace_event_call *call = trace_probe_event_call(tp: &tf->tp);
327	struct trace_event_buffer fbuffer;
328	int dsize;
329
330	if (WARN_ON_ONCE(call != trace_file->event_call))
331	return;
332
333	if (trace_trigger_soft_disabled(file: trace_file))
334	return;
335
336	dsize = __get_data_size(tp: &tf->tp, regs: fregs, NULL);
337
338	entry = trace_event_buffer_reserve(fbuffer: &fbuffer, trace_file,
339	len: sizeof(*entry) + tf->tp.size + dsize);
340	if (!entry)
341	return;
342
343	fbuffer.regs = ftrace_get_regs(fregs);
344	entry = fbuffer.entry = ring_buffer_event_data(event: fbuffer.event);
345	entry->ip = entry_ip;
346	store_trace_args(data: &entry[1], tp: &tf->tp, rec: fregs, NULL, header_size: sizeof(*entry), maxlen: dsize);
347
348	trace_event_buffer_commit(fbuffer: &fbuffer);
349	}
350
351	static void
352	fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
353	struct ftrace_regs *fregs)
354	{
355	struct event_file_link *link;
356
357	trace_probe_for_each_link_rcu(link, &tf->tp)
358	__fentry_trace_func(tf, entry_ip, fregs, trace_file: link->file);
359	}
360	NOKPROBE_SYMBOL(fentry_trace_func);
361
362	static nokprobe_inline
363	void store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs)
364	{
365	struct probe_entry_arg *earg = tp->entry_arg;
366	unsigned long val = 0;
367	int i;
368
369	if (!earg)
370	return;
371
372	for (i = 0; i < earg->size; i++) {
373	struct fetch_insn *code = &earg->code[i];
374
375	switch (code->op) {
376	case FETCH_OP_ARG:
377	val = ftrace_regs_get_argument(fregs, code->param);
378	break;
379	case FETCH_OP_ST_EDATA:
380	*(unsigned long *)((unsigned long)edata + code->offset) = val;
381	break;
382	case FETCH_OP_END:
383	goto end;
384	default:
385	break;
386	}
387	}
388	end:
389	return;
390	}
391
392	/* function exit handler */
393	static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip,
394	unsigned long ret_ip, struct ftrace_regs *fregs,
395	void *entry_data)
396	{
397	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
398
399	if (tf->tp.entry_arg)
400	store_fprobe_entry_data(edata: entry_data, tp: &tf->tp, fregs);
401
402	return 0;
403	}
404	NOKPROBE_SYMBOL(trace_fprobe_entry_handler)
405
406	static nokprobe_inline void
407	__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
408	unsigned long ret_ip, struct ftrace_regs *fregs,
409	void *entry_data, struct trace_event_file *trace_file)
410	{
411	struct fexit_trace_entry_head *entry;
412	struct trace_event_buffer fbuffer;
413	struct trace_event_call *call = trace_probe_event_call(tp: &tf->tp);
414	int dsize;
415
416	if (WARN_ON_ONCE(call != trace_file->event_call))
417	return;
418
419	if (trace_trigger_soft_disabled(file: trace_file))
420	return;
421
422	dsize = __get_data_size(tp: &tf->tp, regs: fregs, edata: entry_data);
423
424	entry = trace_event_buffer_reserve(fbuffer: &fbuffer, trace_file,
425	len: sizeof(*entry) + tf->tp.size + dsize);
426	if (!entry)
427	return;
428
429	fbuffer.regs = ftrace_get_regs(fregs);
430	entry = fbuffer.entry = ring_buffer_event_data(event: fbuffer.event);
431	entry->func = entry_ip;
432	entry->ret_ip = ret_ip;
433	store_trace_args(data: &entry[1], tp: &tf->tp, rec: fregs, edata: entry_data, header_size: sizeof(*entry), maxlen: dsize);
434
435	trace_event_buffer_commit(fbuffer: &fbuffer);
436	}
437
438	static void
439	fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
440	unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data)
441	{
442	struct event_file_link *link;
443
444	trace_probe_for_each_link_rcu(link, &tf->tp)
445	__fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, trace_file: link->file);
446	}
447	NOKPROBE_SYMBOL(fexit_trace_func);
448
449	#ifdef CONFIG_PERF_EVENTS
450
451	static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
452	struct ftrace_regs *fregs)
453	{
454	struct trace_event_call *call = trace_probe_event_call(tp: &tf->tp);
455	struct fentry_trace_entry_head *entry;
456	struct hlist_head *head;
457	int size, __size, dsize;
458	struct pt_regs *regs;
459	int rctx;
460
461	head = this_cpu_ptr(call->perf_events);
462	if (hlist_empty(h: head))
463	return 0;
464
465	dsize = __get_data_size(tp: &tf->tp, regs: fregs, NULL);
466	__size = sizeof(*entry) + tf->tp.size + dsize;
467	size = ALIGN(__size + sizeof(u32), sizeof(u64));
468	size -= sizeof(u32);
469
470	entry = perf_trace_buf_alloc(size, regs: &regs, rctxp: &rctx);
471	if (!entry)
472	return 0;
473
474	regs = ftrace_fill_perf_regs(fregs, regs);
475
476	entry->ip = entry_ip;
477	memset(&entry[1], 0, dsize);
478	store_trace_args(data: &entry[1], tp: &tf->tp, rec: fregs, NULL, header_size: sizeof(*entry), maxlen: dsize);
479	perf_trace_buf_submit(raw_data: entry, size, rctx, type: call->event.type, count: 1, regs,
480	head, NULL);
481	return 0;
482	}
483	NOKPROBE_SYMBOL(fentry_perf_func);
484
485	static void
486	fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
487	unsigned long ret_ip, struct ftrace_regs *fregs,
488	void *entry_data)
489	{
490	struct trace_event_call *call = trace_probe_event_call(tp: &tf->tp);
491	struct fexit_trace_entry_head *entry;
492	struct hlist_head *head;
493	int size, __size, dsize;
494	struct pt_regs *regs;
495	int rctx;
496
497	head = this_cpu_ptr(call->perf_events);
498	if (hlist_empty(h: head))
499	return;
500
501	dsize = __get_data_size(tp: &tf->tp, regs: fregs, edata: entry_data);
502	__size = sizeof(*entry) + tf->tp.size + dsize;
503	size = ALIGN(__size + sizeof(u32), sizeof(u64));
504	size -= sizeof(u32);
505
506	entry = perf_trace_buf_alloc(size, regs: &regs, rctxp: &rctx);
507	if (!entry)
508	return;
509
510	regs = ftrace_fill_perf_regs(fregs, regs);
511
512	entry->func = entry_ip;
513	entry->ret_ip = ret_ip;
514	store_trace_args(data: &entry[1], tp: &tf->tp, rec: fregs, edata: entry_data, header_size: sizeof(*entry), maxlen: dsize);
515	perf_trace_buf_submit(raw_data: entry, size, rctx, type: call->event.type, count: 1, regs,
516	head, NULL);
517	}
518	NOKPROBE_SYMBOL(fexit_perf_func);
519	#endif /* CONFIG_PERF_EVENTS */
520
521	static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,
522	unsigned long ret_ip, struct ftrace_regs *fregs,
523	void *entry_data)
524	{
525	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
526	unsigned int flags = trace_probe_load_flag(tp: &tf->tp);
527	int ret = 0;
528
529	if (flags & TP_FLAG_TRACE)
530	fentry_trace_func(tf, entry_ip, fregs);
531
532	#ifdef CONFIG_PERF_EVENTS
533	if (flags & TP_FLAG_PROFILE)
534	ret = fentry_perf_func(tf, entry_ip, fregs);
535	#endif
536	return ret;
537	}
538	NOKPROBE_SYMBOL(fentry_dispatcher);
539
540	static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,
541	unsigned long ret_ip, struct ftrace_regs *fregs,
542	void *entry_data)
543	{
544	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
545	unsigned int flags = trace_probe_load_flag(tp: &tf->tp);
546
547	if (flags & TP_FLAG_TRACE)
548	fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data);
549	#ifdef CONFIG_PERF_EVENTS
550	if (flags & TP_FLAG_PROFILE)
551	fexit_perf_func(tf, entry_ip, ret_ip, fregs, entry_data);
552	#endif
553	}
554	NOKPROBE_SYMBOL(fexit_dispatcher);
555
556	static void free_trace_fprobe(struct trace_fprobe *tf)
557	{
558	if (tf) {
559	trace_probe_cleanup(tp: &tf->tp);
560	if (tf->tuser)
561	tracepoint_user_put(tuser: tf->tuser);
562	kfree(objp: tf->symbol);
563	kfree(objp: tf);
564	}
565	}
566
567	/* Since alloc_trace_fprobe() can return error, check the pointer is ERR too. */
568	DEFINE_FREE(free_trace_fprobe, struct trace_fprobe *, if (!IS_ERR_OR_NULL(_T)) free_trace_fprobe(_T))
569
570	/*
571	* Allocate new trace_probe and initialize it (including fprobe).
572	*/
573	static struct trace_fprobe *alloc_trace_fprobe(const char *group,
574	const char *event,
575	const char *symbol,
576	int nargs, bool is_return,
577	bool is_tracepoint)
578	{
579	struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;
580	int ret = -ENOMEM;
581
582	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);
583	if (!tf)
584	return ERR_PTR(error: ret);
585
586	tf->symbol = kstrdup(s: symbol, GFP_KERNEL);
587	if (!tf->symbol)
588	return ERR_PTR(error: -ENOMEM);
589
590	if (is_return)
591	tf->fp.exit_handler = fexit_dispatcher;
592	else
593	tf->fp.entry_handler = fentry_dispatcher;
594
595	tf->tprobe = is_tracepoint;
596
597	ret = trace_probe_init(tp: &tf->tp, event, group, alloc_filter: false, nargs);
598	if (ret < 0)
599	return ERR_PTR(error: ret);
600
601	dyn_event_init(ev: &tf->devent, ops: &trace_fprobe_ops);
602	return_ptr(tf);
603	}
604
605	static struct trace_fprobe *find_trace_fprobe(const char *event,
606	const char *group)
607	{
608	struct dyn_event *pos;
609	struct trace_fprobe *tf;
610
611	for_each_trace_fprobe(tf, pos)
612	if (strcmp(trace_probe_name(tp: &tf->tp), event) == 0 &&
613	strcmp(trace_probe_group_name(tp: &tf->tp), group) == 0)
614	return tf;
615	return NULL;
616	}
617
618	/* Event entry printers */
619	static enum print_line_t
620	print_fentry_event(struct trace_iterator *iter, int flags,
621	struct trace_event *event)
622	{
623	struct fentry_trace_entry_head *field;
624	struct trace_seq *s = &iter->seq;
625	struct trace_probe *tp;
626
627	field = (struct fentry_trace_entry_head *)iter->ent;
628	tp = trace_probe_primary_from_call(
629	container_of(event, struct trace_event_call, event));
630	if (WARN_ON_ONCE(!tp))
631	goto out;
632
633	trace_seq_printf(s, fmt: "%s: (", trace_probe_name(tp));
634
635	if (!seq_print_ip_sym_offset(s, ip: field->ip, sym_flags: flags))
636	goto out;
637
638	trace_seq_putc(s, c: ')');
639
640	if (trace_probe_print_args(s, args: tp->args, nr_args: tp->nr_args,
641	data: (u8 *)&field[1], field) < 0)
642	goto out;
643
644	trace_seq_putc(s, c: '\n');
645	out:
646	return trace_handle_return(s);
647	}
648
649	static enum print_line_t
650	print_fexit_event(struct trace_iterator *iter, int flags,
651	struct trace_event *event)
652	{
653	struct fexit_trace_entry_head *field;
654	struct trace_seq *s = &iter->seq;
655	struct trace_probe *tp;
656
657	field = (struct fexit_trace_entry_head *)iter->ent;
658	tp = trace_probe_primary_from_call(
659	container_of(event, struct trace_event_call, event));
660	if (WARN_ON_ONCE(!tp))
661	goto out;
662
663	trace_seq_printf(s, fmt: "%s: (", trace_probe_name(tp));
664
665	if (!seq_print_ip_sym_offset(s, ip: field->ret_ip, sym_flags: flags))
666	goto out;
667
668	trace_seq_puts(s, str: " <- ");
669
670	if (!seq_print_ip_sym_no_offset(s, ip: field->func, sym_flags: flags))
671	goto out;
672
673	trace_seq_putc(s, c: ')');
674
675	if (trace_probe_print_args(s, args: tp->args, nr_args: tp->nr_args,
676	data: (u8 *)&field[1], field) < 0)
677	goto out;
678
679	trace_seq_putc(s, c: '\n');
680
681	out:
682	return trace_handle_return(s);
683	}
684
685	static int fentry_event_define_fields(struct trace_event_call *event_call)
686	{
687	int ret;
688	struct fentry_trace_entry_head field;
689	struct trace_probe *tp;
690
691	tp = trace_probe_primary_from_call(call: event_call);
692	if (WARN_ON_ONCE(!tp))
693	return -ENOENT;
694
695	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
696
697	return traceprobe_define_arg_fields(event_call, offset: sizeof(field), tp);
698	}
699
700	static int fexit_event_define_fields(struct trace_event_call *event_call)
701	{
702	int ret;
703	struct fexit_trace_entry_head field;
704	struct trace_probe *tp;
705
706	tp = trace_probe_primary_from_call(call: event_call);
707	if (WARN_ON_ONCE(!tp))
708	return -ENOENT;
709
710	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
711	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
712
713	return traceprobe_define_arg_fields(event_call, offset: sizeof(field), tp);
714	}
715
716	static struct trace_event_functions fentry_funcs = {
717	.trace = print_fentry_event
718	};
719
720	static struct trace_event_functions fexit_funcs = {
721	.trace = print_fexit_event
722	};
723
724	static struct trace_event_fields fentry_fields_array[] = {
725	{ .type = TRACE_FUNCTION_TYPE,
726	.define_fields = fentry_event_define_fields },
727	{}
728	};
729
730	static struct trace_event_fields fexit_fields_array[] = {
731	{ .type = TRACE_FUNCTION_TYPE,
732	.define_fields = fexit_event_define_fields },
733	{}
734	};
735
736	static int fprobe_register(struct trace_event_call *event,
737	enum trace_reg type, void *data);
738
739	static inline void init_trace_event_call(struct trace_fprobe *tf)
740	{
741	struct trace_event_call *call = trace_probe_event_call(tp: &tf->tp);
742
743	if (trace_fprobe_is_return(tf)) {
744	call->event.funcs = &fexit_funcs;
745	call->class->fields_array = fexit_fields_array;
746	} else {
747	call->event.funcs = &fentry_funcs;
748	call->class->fields_array = fentry_fields_array;
749	}
750
751	call->flags = TRACE_EVENT_FL_FPROBE;
752	call->class->reg = fprobe_register;
753	}
754
755	static int register_fprobe_event(struct trace_fprobe *tf)
756	{
757	init_trace_event_call(tf);
758
759	return trace_probe_register_event_call(tp: &tf->tp);
760	}
761
762	static int unregister_fprobe_event(struct trace_fprobe *tf)
763	{
764	return trace_probe_unregister_event_call(tp: &tf->tp);
765	}
766
767	static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)
768	{
769	struct tracepoint_user *tuser __free(tuser_put) = NULL;
770	struct module *mod __free(module_put) = NULL;
771	unsigned long ip;
772	int ret;
773
774	if (WARN_ON_ONCE(tf->tuser))
775	return -EINVAL;
776
777	/* If the tracepoint is in a module, it must be locked in this function. */
778	tuser = tracepoint_user_find_get(name: tf->symbol, pmod: &mod);
779	/* This tracepoint is not loaded yet */
780	if (IS_ERR(ptr: tuser))
781	return PTR_ERR(ptr: tuser);
782	if (!tuser)
783	return -ENOMEM;
784
785	/* Register fprobe only if the tracepoint is loaded. */
786	if (tuser->tpoint) {
787	ip = tracepoint_user_ip(tuser);
788	if (WARN_ON_ONCE(!ip))
789	return -ENOENT;
790
791	ret = register_fprobe_ips(fp: &tf->fp, addrs: &ip, num: 1);
792	if (ret < 0)
793	return ret;
794	}
795
796	tf->tuser = no_free_ptr(tuser);
797	return 0;
798	}
799
800	/* Returns an error if the target function is not available, or 0 */
801	static int trace_fprobe_verify_target(struct trace_fprobe *tf)
802	{
803	int ret;
804
805	/* Tracepoint should have a stub function. */
806	if (trace_fprobe_is_tracepoint(tf))
807	return 0;
808
809	/*
810	* Note: since we don't lock the module, even if this succeeded,
811	* register_fprobe() later can fail.
812	*/
813	ret = fprobe_count_ips_from_filter(filter: tf->symbol, NULL);
814	return (ret < 0) ? ret : 0;
815	}
816
817	/* Internal register function - just handle fprobe and flags */
818	static int __register_trace_fprobe(struct trace_fprobe *tf)
819	{
820	int i, ret;
821
822	/* Should we need new LOCKDOWN flag for fprobe? */
823	ret = security_locked_down(what: LOCKDOWN_KPROBES);
824	if (ret)
825	return ret;
826
827	if (trace_fprobe_is_registered(tf))
828	return -EINVAL;
829
830	for (i = 0; i < tf->tp.nr_args; i++) {
831	ret = traceprobe_update_arg(arg: &tf->tp.args[i]);
832	if (ret)
833	return ret;
834	}
835
836	tf->fp.flags &= ~FPROBE_FL_DISABLED;
837
838	if (trace_fprobe_is_tracepoint(tf))
839	return __regsiter_tracepoint_fprobe(tf);
840
841	/* TODO: handle filter, nofilter or symbol list */
842	return register_fprobe(fp: &tf->fp, filter: tf->symbol, NULL);
843	}
844
845	/* Internal unregister function - just handle fprobe and flags */
846	static void __unregister_trace_fprobe(struct trace_fprobe *tf)
847	{
848	if (trace_fprobe_is_registered(tf))
849	unregister_fprobe(fp: &tf->fp);
850	if (tf->tuser) {
851	tracepoint_user_put(tuser: tf->tuser);
852	tf->tuser = NULL;
853	}
854	}
855
856	/* TODO: make this trace_*probe common function */
857	/* Unregister a trace_probe and probe_event */
858	static int unregister_trace_fprobe(struct trace_fprobe *tf)
859	{
860	/* If other probes are on the event, just unregister fprobe */
861	if (trace_probe_has_sibling(tp: &tf->tp))
862	goto unreg;
863
864	/* Enabled event can not be unregistered */
865	if (trace_probe_is_enabled(tp: &tf->tp))
866	return -EBUSY;
867
868	/* If there's a reference to the dynamic event */
869	if (trace_event_dyn_busy(call: trace_probe_event_call(tp: &tf->tp)))
870	return -EBUSY;
871
872	/* Will fail if probe is being used by ftrace or perf */
873	if (unregister_fprobe_event(tf))
874	return -EBUSY;
875
876	unreg:
877	__unregister_trace_fprobe(tf);
878	dyn_event_remove(ev: &tf->devent);
879	trace_probe_unlink(tp: &tf->tp);
880
881	return 0;
882	}
883
884	static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig,
885	struct trace_fprobe *comp)
886	{
887	struct trace_probe_event *tpe = orig->tp.event;
888	int i;
889
890	list_for_each_entry(orig, &tpe->probes, tp.list) {
891	if (strcmp(trace_fprobe_symbol(tf: orig),
892	trace_fprobe_symbol(tf: comp)))
893	continue;
894
895	/*
896	* trace_probe_compare_arg_type() ensured that nr_args and
897	* each argument name and type are same. Let's compare comm.
898	*/
899	for (i = 0; i < orig->tp.nr_args; i++) {
900	if (strcmp(orig->tp.args[i].comm,
901	comp->tp.args[i].comm))
902	break;
903	}
904
905	if (i == orig->tp.nr_args)
906	return true;
907	}
908
909	return false;
910	}
911
912	static int append_trace_fprobe_event(struct trace_fprobe *tf, struct trace_fprobe *to)
913	{
914	int ret;
915
916	if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(tf: to) ||
917	trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(tf: to)) {
918	trace_probe_log_set_index(index: 0);
919	trace_probe_log_err(0, DIFF_PROBE_TYPE);
920	return -EEXIST;
921	}
922	ret = trace_probe_compare_arg_type(a: &tf->tp, b: &to->tp);
923	if (ret) {
924	/* Note that argument starts index = 2 */
925	trace_probe_log_set_index(index: ret + 1);
926	trace_probe_log_err(0, DIFF_ARG_TYPE);
927	return -EEXIST;
928	}
929	if (trace_fprobe_has_same_fprobe(orig: to, comp: tf)) {
930	trace_probe_log_set_index(index: 0);
931	trace_probe_log_err(0, SAME_PROBE);
932	return -EEXIST;
933	}
934
935	/* Append to existing event */
936	ret = trace_probe_append(tp: &tf->tp, to: &to->tp);
937	if (ret)
938	return ret;
939
940	ret = trace_fprobe_verify_target(tf);
941	if (ret)
942	trace_probe_unlink(tp: &tf->tp);
943	else
944	dyn_event_add(ev: &tf->devent, call: trace_probe_event_call(tp: &tf->tp));
945
946	return ret;
947	}
948
949	/* Register a trace_probe and probe_event, and check the fprobe is available. */
950	static int register_trace_fprobe_event(struct trace_fprobe *tf)
951	{
952	struct trace_fprobe *old_tf;
953	int ret;
954
955	guard(mutex)(T: &event_mutex);
956
957	old_tf = find_trace_fprobe(event: trace_probe_name(tp: &tf->tp),
958	group: trace_probe_group_name(tp: &tf->tp));
959	if (old_tf)
960	return append_trace_fprobe_event(tf, to: old_tf);
961
962	/* Register new event */
963	ret = register_fprobe_event(tf);
964	if (ret) {
965	if (ret == -EEXIST) {
966	trace_probe_log_set_index(index: 0);
967	trace_probe_log_err(0, EVENT_EXIST);
968	} else
969	pr_warn("Failed to register probe event(%d)\n", ret);
970	return ret;
971	}
972
973	/* Verify fprobe is sane. */
974	ret = trace_fprobe_verify_target(tf);
975	if (ret < 0)
976	unregister_fprobe_event(tf);
977	else
978	dyn_event_add(ev: &tf->devent, call: trace_probe_event_call(tp: &tf->tp));
979
980	return ret;
981	}
982
983	struct __find_tracepoint_cb_data {
984	const char *tp_name;
985	struct tracepoint *tpoint;
986	struct module *mod;
987	};
988
989	static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv)
990	{
991	struct __find_tracepoint_cb_data *data = priv;
992
993	if (!data->tpoint && !strcmp(data->tp_name, tp->name)) {
994	/* If module is not specified, try getting module refcount. */
995	if (!data->mod && mod) {
996	/* If failed to get refcount, ignore this tracepoint. */
997	if (!try_module_get(module: mod))
998	return;
999
1000	data->mod = mod;
1001	}
1002	data->tpoint = tp;
1003	}
1004	}
1005
1006	static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)
1007	{
1008	struct __find_tracepoint_cb_data *data = priv;
1009
1010	if (!data->tpoint && !strcmp(data->tp_name, tp->name))
1011	data->tpoint = tp;
1012	}
1013
1014	/*
1015	* Find a tracepoint from kernel and module. If the tracepoint is on the module,
1016	* the module's refcount is incremented and returned as *@tp_mod. Thus, if it is
1017	* not NULL, caller must call module_put(*tp_mod) after used the tracepoint.
1018	*/
1019	static struct tracepoint *find_tracepoint(const char *tp_name,
1020	struct module **tp_mod)
1021	{
1022	struct __find_tracepoint_cb_data data = {
1023	.tp_name = tp_name,
1024	.mod = NULL,
1025	};
1026
1027	for_each_kernel_tracepoint(fct: __find_tracepoint_cb, priv: &data);
1028
1029	if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) {
1030	for_each_module_tracepoint(fct: __find_tracepoint_module_cb, priv: &data);
1031	*tp_mod = data.mod;
1032	}
1033
1034	return data.tpoint;
1035	}
1036
1037	#ifdef CONFIG_MODULES
1038	/*
1039	* Find a tracepoint from specified module. In this case, this does not get the
1040	* module's refcount. The caller must ensure the module is not freed.
1041	*/
1042	static struct tracepoint *find_tracepoint_in_module(struct module *mod,
1043	const char *tp_name)
1044	{
1045	struct __find_tracepoint_cb_data data = {
1046	.tp_name = tp_name,
1047	.mod = mod,
1048	};
1049
1050	for_each_tracepoint_in_module(mod, fct: __find_tracepoint_module_cb, priv: &data);
1051	return data.tpoint;
1052	}
1053
1054	/* These are CONFIG_MODULES=y specific functions. */
1055	static bool tracepoint_user_within_module(struct tracepoint_user *tuser,
1056	struct module *mod)
1057	{
1058	return within_module(addr: tracepoint_user_ip(tuser), mod);
1059	}
1060
1061	static int tracepoint_user_register_again(struct tracepoint_user *tuser,
1062	struct tracepoint *tpoint)
1063	{
1064	tuser->tpoint = tpoint;
1065	return tracepoint_user_register(tuser);
1066	}
1067
1068	static void tracepoint_user_unregister_clear(struct tracepoint_user *tuser)
1069	{
1070	tracepoint_user_unregister(tuser);
1071	tuser->tpoint = NULL;
1072	}
1073
1074	/* module callback for tracepoint_user */
1075	static int __tracepoint_probe_module_cb(struct notifier_block *self,
1076	unsigned long val, void *data)
1077	{
1078	struct tp_module *tp_mod = data;
1079	struct tracepoint_user *tuser;
1080	struct tracepoint *tpoint;
1081
1082	if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
1083	return NOTIFY_DONE;
1084
1085	mutex_lock(&tracepoint_user_mutex);
1086	for_each_tracepoint_user(tuser) {
1087	if (val == MODULE_STATE_COMING) {
1088	/* This is not a tracepoint in this module. Skip it. */
1089	tpoint = find_tracepoint_in_module(mod: tp_mod->mod, tp_name: tuser->name);
1090	if (!tpoint)
1091	continue;
1092	WARN_ON_ONCE(tracepoint_user_register_again(tuser, tpoint));
1093	} else if (val == MODULE_STATE_GOING &&
1094	tracepoint_user_within_module(tuser, mod: tp_mod->mod)) {
1095	/* Unregister all tracepoint_user in this module. */
1096	tracepoint_user_unregister_clear(tuser);
1097	}
1098	}
1099	mutex_unlock(lock: &tracepoint_user_mutex);
1100
1101	return NOTIFY_DONE;
1102	}
1103
1104	static struct notifier_block tracepoint_module_nb = {
1105	.notifier_call = __tracepoint_probe_module_cb,
1106	};
1107
1108	/* module callback for tprobe events */
1109	static int __tprobe_event_module_cb(struct notifier_block *self,
1110	unsigned long val, void *data)
1111	{
1112	struct trace_fprobe *tf;
1113	struct dyn_event *pos;
1114	struct module *mod = data;
1115
1116	if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
1117	return NOTIFY_DONE;
1118
1119	mutex_lock(&event_mutex);
1120	for_each_trace_fprobe(tf, pos) {
1121	/* Skip fprobe and disabled tprobe events. */
1122	if (!trace_fprobe_is_tracepoint(tf) || !tf->tuser)
1123	continue;
1124
1125	/* Before this notification, tracepoint notifier has already done. */
1126	if (val == MODULE_STATE_COMING &&
1127	tracepoint_user_within_module(tuser: tf->tuser, mod)) {
1128	unsigned long ip = tracepoint_user_ip(tuser: tf->tuser);
1129
1130	WARN_ON_ONCE(register_fprobe_ips(&tf->fp, &ip, 1));
1131	} else if (val == MODULE_STATE_GOING &&
1132	/*
1133	* tracepoint_user_within_module() does not work here because
1134	* tracepoint_user is already unregistered and cleared tpoint.
1135	* Instead, checking whether the fprobe is registered but
1136	* tpoint is cleared(unregistered). Such unbalance probes
1137	* must be adjusted anyway.
1138	*/
1139	trace_fprobe_is_registered(tf) &&
1140	!tf->tuser->tpoint) {
1141	unregister_fprobe(fp: &tf->fp);
1142	}
1143	}
1144	mutex_unlock(lock: &event_mutex);
1145
1146	return NOTIFY_DONE;
1147	}
1148
1149	/* NOTE: this must be called after tracepoint callback */
1150	static struct notifier_block tprobe_event_module_nb = {
1151	.notifier_call = __tprobe_event_module_cb,
1152	/* Make sure this is later than tracepoint module notifier. */
1153	.priority = -10,
1154	};
1155	#endif /* CONFIG_MODULES */
1156
1157	static int parse_symbol_and_return(int argc, const char *argv[],
1158	char **symbol, bool *is_return,
1159	bool is_tracepoint)
1160	{
1161	char *tmp = strchr(argv[1], '%');
1162	int i;
1163
1164	if (tmp) {
1165	int len = tmp - argv[1];
1166
1167	if (!is_tracepoint && !strcmp(tmp, "%return")) {
1168	*is_return = true;
1169	} else {
1170	trace_probe_log_err(len, BAD_ADDR_SUFFIX);
1171	return -EINVAL;
1172	}
1173	*symbol = kmemdup_nul(s: argv[1], len, GFP_KERNEL);
1174	} else
1175	*symbol = kstrdup(s: argv[1], GFP_KERNEL);
1176	if (!*symbol)
1177	return -ENOMEM;
1178
1179	if (*is_return)
1180	return 0;
1181
1182	if (is_tracepoint) {
1183	tmp = *symbol;
1184	while (*tmp && (isalnum(*tmp) || *tmp == '_'))
1185	tmp++;
1186	if (*tmp) {
1187	/* find a wrong character. */
1188	trace_probe_log_err(tmp - *symbol, BAD_TP_NAME);
1189	kfree(objp: *symbol);
1190	*symbol = NULL;
1191	return -EINVAL;
1192	}
1193	}
1194
1195	/* If there is $retval, this should be a return fprobe. */
1196	for (i = 2; i < argc; i++) {
1197	tmp = strstr(argv[i], "$retval");
1198	if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') {
1199	if (is_tracepoint) {
1200	trace_probe_log_set_index(index: i);
1201	trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);
1202	kfree(objp: *symbol);
1203	*symbol = NULL;
1204	return -EINVAL;
1205	}
1206	*is_return = true;
1207	break;
1208	}
1209	}
1210	return 0;
1211	}
1212
1213	static int trace_fprobe_create_internal(int argc, const char *argv[],
1214	struct traceprobe_parse_context *ctx)
1215	{
1216	/*
1217	* Argument syntax:
1218	* - Add fentry probe:
1219	* f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS]
1220	* - Add fexit probe:
1221	* f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS]
1222	* - Add tracepoint probe:
1223	* t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS]
1224	*
1225	* Fetch args:
1226	* $retval : fetch return value
1227	* $stack : fetch stack address
1228	* $stackN : fetch Nth entry of stack (N:0-)
1229	* $argN : fetch Nth argument (N:1-)
1230	* $comm : fetch current task comm
1231	* @ADDR : fetch memory at ADDR (ADDR should be in kernel)
1232	* @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
1233	* Dereferencing memory fetch:
1234	* +|-offs(ARG) : fetch memory at ARG +|- offs address.
1235	* Alias name of args:
1236	* NAME=FETCHARG : set NAME as alias of FETCHARG.
1237	* Type of args:
1238	* FETCHARG:TYPE : use TYPE instead of unsigned long.
1239	*/
1240	struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;
1241	const char *event = NULL, *group = FPROBE_EVENT_SYSTEM;
1242	struct module *mod __free(module_put) = NULL;
1243	const char **new_argv __free(kfree) = NULL;
1244	char *symbol __free(kfree) = NULL;
1245	char *ebuf __free(kfree) = NULL;
1246	char *gbuf __free(kfree) = NULL;
1247	char *sbuf __free(kfree) = NULL;
1248	char *abuf __free(kfree) = NULL;
1249	char *dbuf __free(kfree) = NULL;
1250	int i, new_argc = 0, ret = 0;
1251	bool is_tracepoint = false;
1252	bool is_return = false;
1253
1254	if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2)
1255	return -ECANCELED;
1256
1257	if (argv[0][0] == 't') {
1258	is_tracepoint = true;
1259	group = TRACEPOINT_EVENT_SYSTEM;
1260	}
1261
1262	if (argv[0][1] != '\0') {
1263	if (argv[0][1] != ':') {
1264	trace_probe_log_set_index(index: 0);
1265	trace_probe_log_err(1, BAD_MAXACT);
1266	return -EINVAL;
1267	}
1268	event = &argv[0][2];
1269	}
1270
1271	trace_probe_log_set_index(index: 1);
1272
1273	/* a symbol(or tracepoint) must be specified */
1274	ret = parse_symbol_and_return(argc, argv, symbol: &symbol, is_return: &is_return, is_tracepoint);
1275	if (ret < 0)
1276	return -EINVAL;
1277
1278	trace_probe_log_set_index(index: 0);
1279	if (event) {
1280	gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1281	if (!gbuf)
1282	return -ENOMEM;
1283	ret = traceprobe_parse_event_name(pevent: &event, pgroup: &group, buf: gbuf,
1284	offset: event - argv[0]);
1285	if (ret)
1286	return -EINVAL;
1287	}
1288
1289	if (!event) {
1290	ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1291	if (!ebuf)
1292	return -ENOMEM;
1293	/* Make a new event name */
1294	if (is_tracepoint)
1295	snprintf(buf: ebuf, MAX_EVENT_NAME_LEN, fmt: "%s%s",
1296	isdigit(c: *symbol) ? "_" : "", symbol);
1297	else
1298	snprintf(buf: ebuf, MAX_EVENT_NAME_LEN, fmt: "%s__%s", symbol,
1299	is_return ? "exit" : "entry");
1300	sanitize_event_name(name: ebuf);
1301	event = ebuf;
1302	}
1303
1304	if (is_return)
1305	ctx->flags |= TPARG_FL_RETURN;
1306	else
1307	ctx->flags |= TPARG_FL_FENTRY;
1308
1309	ctx->funcname = NULL;
1310	if (is_tracepoint) {
1311	/* Get tracepoint and lock its module until the end of the registration. */
1312	struct tracepoint *tpoint;
1313
1314	ctx->flags |= TPARG_FL_TPOINT;
1315	mod = NULL;
1316	tpoint = find_tracepoint(tp_name: symbol, tp_mod: &mod);
1317	if (tpoint) {
1318	sbuf = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);
1319	if (!sbuf)
1320	return -ENOMEM;
1321	ctx->funcname = kallsyms_lookup(addr: (unsigned long)tpoint->probestub,
1322	NULL, NULL, NULL, namebuf: sbuf);
1323	}
1324	}
1325	if (!ctx->funcname)
1326	ctx->funcname = symbol;
1327
1328	abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL);
1329	if (!abuf)
1330	return -ENOMEM;
1331	argc -= 2; argv += 2;
1332	new_argv = traceprobe_expand_meta_args(argc, argv, new_argc: &new_argc,
1333	buf: abuf, MAX_BTF_ARGS_LEN, ctx);
1334	if (IS_ERR(ptr: new_argv))
1335	return PTR_ERR(ptr: new_argv);
1336	if (new_argv) {
1337	argc = new_argc;
1338	argv = new_argv;
1339	}
1340	if (argc > MAX_TRACE_ARGS) {
1341	trace_probe_log_set_index(index: 2);
1342	trace_probe_log_err(0, TOO_MANY_ARGS);
1343	return -E2BIG;
1344	}
1345
1346	ret = traceprobe_expand_dentry_args(argc, argv, buf: &dbuf);
1347	if (ret)
1348	return ret;
1349
1350	/* setup a probe */
1351	tf = alloc_trace_fprobe(group, event, symbol, nargs: argc, is_return, is_tracepoint);
1352	if (IS_ERR(ptr: tf)) {
1353	ret = PTR_ERR(ptr: tf);
1354	/* This must return -ENOMEM, else there is a bug */
1355	WARN_ON_ONCE(ret != -ENOMEM);
1356	return ret;
1357	}
1358
1359	/* parse arguments */
1360	for (i = 0; i < argc; i++) {
1361	trace_probe_log_set_index(index: i + 2);
1362	ctx->offset = 0;
1363	ret = traceprobe_parse_probe_arg(tp: &tf->tp, i, argv: argv[i], ctx);
1364	if (ret)
1365	return ret; /* This can be -ENOMEM */
1366	}
1367
1368	if (is_return && tf->tp.entry_arg) {
1369	tf->fp.entry_handler = trace_fprobe_entry_handler;
1370	tf->fp.entry_data_size = traceprobe_get_entry_data_size(tp: &tf->tp);
1371	if (ALIGN(tf->fp.entry_data_size, sizeof(long)) > MAX_FPROBE_DATA_SIZE) {
1372	trace_probe_log_set_index(index: 2);
1373	trace_probe_log_err(0, TOO_MANY_EARGS);
1374	return -E2BIG;
1375	}
1376	}
1377
1378	ret = traceprobe_set_print_fmt(tp: &tf->tp,
1379	ptype: is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL);
1380	if (ret < 0)
1381	return ret;
1382
1383	ret = register_trace_fprobe_event(tf);
1384	if (ret) {
1385	trace_probe_log_set_index(index: 1);
1386	if (ret == -EILSEQ)
1387	trace_probe_log_err(0, BAD_INSN_BNDRY);
1388	else if (ret == -ENOENT)
1389	trace_probe_log_err(0, BAD_PROBE_ADDR);
1390	else if (ret != -ENOMEM && ret != -EEXIST)
1391	trace_probe_log_err(0, FAIL_REG_PROBE);
1392	return -EINVAL;
1393	}
1394
1395	/* 'tf' is successfully registered. To avoid freeing, assign NULL. */
1396	tf = NULL;
1397
1398	return 0;
1399	}
1400
1401	static int trace_fprobe_create_cb(int argc, const char *argv[])
1402	{
1403	struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
1404	int ret;
1405
1406	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1407	if (!ctx)
1408	return -ENOMEM;
1409
1410	ctx->flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE;
1411
1412	trace_probe_log_init(subsystem: "trace_fprobe", argc, argv);
1413	ret = trace_fprobe_create_internal(argc, argv, ctx);
1414	trace_probe_log_clear();
1415	return ret;
1416	}
1417
1418	static int trace_fprobe_create(const char *raw_command)
1419	{
1420	return trace_probe_create(raw_command, createfn: trace_fprobe_create_cb);
1421	}
1422
1423	static int trace_fprobe_release(struct dyn_event *ev)
1424	{
1425	struct trace_fprobe *tf = to_trace_fprobe(ev);
1426	int ret = unregister_trace_fprobe(tf);
1427
1428	if (!ret)
1429	free_trace_fprobe(tf);
1430	return ret;
1431	}
1432
1433	static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
1434	{
1435	struct trace_fprobe *tf = to_trace_fprobe(ev);
1436	int i;
1437
1438	if (trace_fprobe_is_tracepoint(tf))
1439	seq_putc(m, c: 't');
1440	else
1441	seq_putc(m, c: 'f');
1442	seq_printf(m, fmt: ":%s/%s", trace_probe_group_name(tp: &tf->tp),
1443	trace_probe_name(tp: &tf->tp));
1444
1445	seq_printf(m, fmt: " %s%s", trace_fprobe_symbol(tf),
1446	trace_fprobe_is_return(tf) ? "%return" : "");
1447
1448	for (i = 0; i < tf->tp.nr_args; i++)
1449	seq_printf(m, fmt: " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
1450	seq_putc(m, c: '\n');
1451
1452	return 0;
1453	}
1454
1455	/*
1456	* Enable trace_probe
1457	* if the file is NULL, enable "perf" handler, or enable "trace" handler.
1458	*/
1459	static int enable_trace_fprobe(struct trace_event_call *call,
1460	struct trace_event_file *file)
1461	{
1462	struct trace_probe *tp;
1463	struct trace_fprobe *tf;
1464	bool enabled;
1465	int ret = 0;
1466
1467	tp = trace_probe_primary_from_call(call);
1468	if (WARN_ON_ONCE(!tp))
1469	return -ENODEV;
1470	enabled = trace_probe_is_enabled(tp);
1471
1472	/* This also changes "enabled" state */
1473	if (file) {
1474	ret = trace_probe_add_file(tp, file);
1475	if (ret)
1476	return ret;
1477	} else
1478	trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1479
1480	if (!enabled) {
1481	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
1482	ret = __register_trace_fprobe(tf);
1483	if (ret < 0)
1484	return ret;
1485	}
1486	}
1487
1488	return 0;
1489	}
1490
1491	/*
1492	* Disable trace_probe
1493	* if the file is NULL, disable "perf" handler, or disable "trace" handler.
1494	*/
1495	static int disable_trace_fprobe(struct trace_event_call *call,
1496	struct trace_event_file *file)
1497	{
1498	struct trace_fprobe *tf;
1499	struct trace_probe *tp;
1500
1501	tp = trace_probe_primary_from_call(call);
1502	if (WARN_ON_ONCE(!tp))
1503	return -ENODEV;
1504
1505	if (file) {
1506	if (!trace_probe_get_file_link(tp, file))
1507	return -ENOENT;
1508	if (!trace_probe_has_single_file(tp))
1509	goto out;
1510	trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1511	} else
1512	trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1513
1514	if (!trace_probe_is_enabled(tp)) {
1515	list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
1516	unregister_fprobe(fp: &tf->fp);
1517	if (tf->tuser) {
1518	tracepoint_user_put(tuser: tf->tuser);
1519	tf->tuser = NULL;
1520	}
1521	}
1522	}
1523
1524	out:
1525	if (file)
1526	/*
1527	* Synchronization is done in below function. For perf event,
1528	* file == NULL and perf_trace_event_unreg() calls
1529	* tracepoint_synchronize_unregister() to ensure synchronize
1530	* event. We don't need to care about it.
1531	*/
1532	trace_probe_remove_file(tp, file);
1533
1534	return 0;
1535	}
1536
1537	/*
1538	* called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1539	*/
1540	static int fprobe_register(struct trace_event_call *event,
1541	enum trace_reg type, void *data)
1542	{
1543	struct trace_event_file *file = data;
1544
1545	switch (type) {
1546	case TRACE_REG_REGISTER:
1547	return enable_trace_fprobe(call: event, file);
1548	case TRACE_REG_UNREGISTER:
1549	return disable_trace_fprobe(call: event, file);
1550
1551	#ifdef CONFIG_PERF_EVENTS
1552	case TRACE_REG_PERF_REGISTER:
1553	return enable_trace_fprobe(call: event, NULL);
1554	case TRACE_REG_PERF_UNREGISTER:
1555	return disable_trace_fprobe(call: event, NULL);
1556	case TRACE_REG_PERF_OPEN:
1557	case TRACE_REG_PERF_CLOSE:
1558	case TRACE_REG_PERF_ADD:
1559	case TRACE_REG_PERF_DEL:
1560	return 0;
1561	#endif
1562	}
1563	return 0;
1564	}
1565
1566	/*
1567	* Register dynevent at core_initcall. This allows kernel to setup fprobe
1568	* events in postcore_initcall without tracefs.
1569	*/
1570	static __init int init_fprobe_trace_early(void)
1571	{
1572	int ret;
1573
1574	ret = dyn_event_register(ops: &trace_fprobe_ops);
1575	if (ret)
1576	return ret;
1577
1578	#ifdef CONFIG_MODULES
1579	ret = register_tracepoint_module_notifier(nb: &tracepoint_module_nb);
1580	if (ret)
1581	return ret;
1582	ret = register_module_notifier(nb: &tprobe_event_module_nb);
1583	if (ret)
1584	return ret;
1585	#endif
1586
1587	return 0;
1588	}
1589	core_initcall(init_fprobe_trace_early);
1590