1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
4	*
5	*/
6
7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9	#include <linux/kernel.h>
10	#include <linux/blkdev.h>
11	#include <linux/blktrace_api.h>
12	#include <linux/percpu.h>
13	#include <linux/init.h>
14	#include <linux/mutex.h>
15	#include <linux/slab.h>
16	#include <linux/debugfs.h>
17	#include <linux/export.h>
18	#include <linux/time.h>
19	#include <linux/uaccess.h>
20	#include <linux/list.h>
21	#include <linux/blk-cgroup.h>
22
23	#include "../../block/blk.h"
24
25	#include <trace/events/block.h>
26
27	#include "trace_output.h"
28
29	#ifdef CONFIG_BLK_DEV_IO_TRACE
30
31	static unsigned int blktrace_seq __read_mostly = 1;
32
33	static struct trace_array *blk_tr;
34	static bool blk_tracer_enabled __read_mostly;
35
36	static LIST_HEAD(running_trace_list);
37	static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock);
38
39	/* Select an alternative, minimalistic output than the original one */
40	#define TRACE_BLK_OPT_CLASSIC 0x1
41	#define TRACE_BLK_OPT_CGROUP 0x2
42	#define TRACE_BLK_OPT_CGNAME 0x4
43
44	static struct tracer_opt blk_tracer_opts[] = {
45	/* Default disable the minimalistic output */
46	{ TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
47	#ifdef CONFIG_BLK_CGROUP
48	{ TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) },
49	{ TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) },
50	#endif
51	{ }
52	};
53
54	static struct tracer_flags blk_tracer_flags = {
55	.val = 0,
56	.opts = blk_tracer_opts,
57	};
58
59	/* Global reference count of probes */
60	static DEFINE_MUTEX(blk_probe_mutex);
61	static int blk_probes_ref;
62
63	static void blk_register_tracepoints(void);
64	static void blk_unregister_tracepoints(void);
65
66	static void record_blktrace_event(struct blk_io_trace *t, pid_t pid, int cpu,
67	sector_t sector, int bytes, u64 what,
68	dev_t dev, int error, u64 cgid,
69	ssize_t cgid_len, void *pdu_data, int pdu_len)
70
71	{
72	/*
73	* These two are not needed in ftrace as they are in the
74	* generic trace_entry, filled by tracing_generic_entry_update,
75	* but for the trace_event->bin() synthesizer benefit we do it
76	* here too.
77	*/
78	t->cpu = cpu;
79	t->pid = pid;
80
81	t->sector = sector;
82	t->bytes = bytes;
83	t->action = lower_32_bits(what);
84	t->device = dev;
85	t->error = error;
86	t->pdu_len = pdu_len + cgid_len;
87
88	if (cgid_len)
89	memcpy((void *)t + sizeof(*t), &cgid, cgid_len);
90	if (pdu_len)
91	memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len);
92	}
93
94	static void record_blktrace_event2(struct blk_io_trace2 *t2, pid_t pid, int cpu,
95	sector_t sector, int bytes, u64 what,
96	dev_t dev, int error, u64 cgid,
97	ssize_t cgid_len, void *pdu_data,
98	int pdu_len)
99	{
100	t2->pid = pid;
101	t2->cpu = cpu;
102
103	t2->sector = sector;
104	t2->bytes = bytes;
105	t2->action = what;
106	t2->device = dev;
107	t2->error = error;
108	t2->pdu_len = pdu_len + cgid_len;
109
110	if (cgid_len)
111	memcpy((void *)t2 + sizeof(*t2), &cgid, cgid_len);
112	if (pdu_len)
113	memcpy((void *)t2 + sizeof(*t2) + cgid_len, pdu_data, pdu_len);
114	}
115
116	static void relay_blktrace_event1(struct blk_trace *bt, unsigned long sequence,
117	pid_t pid, int cpu, sector_t sector, int bytes,
118	u64 what, int error, u64 cgid,
119	ssize_t cgid_len, void *pdu_data, int pdu_len)
120	{
121	struct blk_io_trace *t;
122	size_t trace_len = sizeof(*t) + pdu_len + cgid_len;
123
124	t = relay_reserve(chan: bt->rchan, length: trace_len);
125	if (!t)
126	return;
127
128	t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
129	t->sequence = sequence;
130	t->time = ktime_to_ns(kt: ktime_get());
131
132	record_blktrace_event(t, pid, cpu, sector, bytes, what, dev: bt->dev, error,
133	cgid, cgid_len, pdu_data, pdu_len);
134	}
135
136	static void relay_blktrace_event2(struct blk_trace *bt, unsigned long sequence,
137	pid_t pid, int cpu, sector_t sector,
138	int bytes, u64 what, int error, u64 cgid,
139	ssize_t cgid_len, void *pdu_data, int pdu_len)
140	{
141	struct blk_io_trace2 *t;
142	size_t trace_len = sizeof(struct blk_io_trace2) + pdu_len + cgid_len;
143
144	t = relay_reserve(chan: bt->rchan, length: trace_len);
145	if (!t)
146	return;
147
148	t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE2_VERSION;
149	t->sequence = sequence;
150	t->time = ktime_to_ns(kt: ktime_get());
151
152	record_blktrace_event2(t2: t, pid, cpu, sector, bytes, what, dev: bt->dev, error,
153	cgid, cgid_len, pdu_data, pdu_len);
154	}
155
156	static void relay_blktrace_event(struct blk_trace *bt, unsigned long sequence,
157	pid_t pid, int cpu, sector_t sector, int bytes,
158	u64 what, int error, u64 cgid,
159	ssize_t cgid_len, void *pdu_data, int pdu_len)
160	{
161	if (bt->version == 2)
162	return relay_blktrace_event2(bt, sequence, pid, cpu, sector,
163	bytes, what, error, cgid, cgid_len,
164	pdu_data, pdu_len);
165	return relay_blktrace_event1(bt, sequence, pid, cpu, sector, bytes,
166	what, error, cgid, cgid_len, pdu_data,
167	pdu_len);
168	}
169
170	/*
171	* Send out a notify message.
172	*/
173	static void trace_note(struct blk_trace *bt, pid_t pid, u64 action,
174	const void *data, size_t len, u64 cgid)
175	{
176	struct ring_buffer_event *event = NULL;
177	struct trace_buffer *buffer = NULL;
178	unsigned int trace_ctx = 0;
179	int cpu = smp_processor_id();
180	bool blk_tracer = blk_tracer_enabled;
181	ssize_t cgid_len = cgid ? sizeof(cgid) : 0;
182
183	action = lower_32_bits(action | (cgid ? __BLK_TN_CGROUP : 0));
184	if (blk_tracer) {
185	struct blk_io_trace2 *t;
186	size_t trace_len = sizeof(*t) + cgid_len + len;
187
188	buffer = blk_tr->array_buffer.buffer;
189	trace_ctx = tracing_gen_ctx_flags(irqflags: 0);
190	event = trace_buffer_lock_reserve(buffer, type: TRACE_BLK,
191	len: trace_len, trace_ctx);
192	if (!event)
193	return;
194	t = ring_buffer_event_data(event);
195	record_blktrace_event2(t2: t, pid, cpu, sector: 0, bytes: 0,
196	what: action, dev: bt->dev, error: 0, cgid, cgid_len,
197	pdu_data: (void *)data, pdu_len: len);
198	trace_buffer_unlock_commit(tr: blk_tr, buffer, event, trace_ctx);
199	return;
200	}
201
202	if (!bt->rchan)
203	return;
204
205	relay_blktrace_event(bt, sequence: 0, pid, cpu, sector: 0, bytes: 0, what: action, error: 0, cgid,
206	cgid_len, pdu_data: (void *)data, pdu_len: len);
207	}
208
209	/*
210	* Send out a notify for this process, if we haven't done so since a trace
211	* started
212	*/
213	static void trace_note_tsk(struct task_struct *tsk)
214	{
215	unsigned long flags;
216	struct blk_trace *bt;
217
218	tsk->btrace_seq = blktrace_seq;
219	raw_spin_lock_irqsave(&running_trace_lock, flags);
220	list_for_each_entry(bt, &running_trace_list, running_list) {
221	trace_note(bt, pid: tsk->pid, BLK_TN_PROCESS, data: tsk->comm,
222	len: sizeof(tsk->comm), cgid: 0);
223	}
224	raw_spin_unlock_irqrestore(&running_trace_lock, flags);
225	}
226
227	static void trace_note_time(struct blk_trace *bt)
228	{
229	struct timespec64 now;
230	unsigned long flags;
231	u32 words[2];
232
233	/* need to check user space to see if this breaks in y2038 or y2106 */
234	ktime_get_real_ts64(tv: &now);
235	words[0] = (u32)now.tv_sec;
236	words[1] = now.tv_nsec;
237
238	local_irq_save(flags);
239	trace_note(bt, pid: 0, BLK_TN_TIMESTAMP, data: words, len: sizeof(words), cgid: 0);
240	local_irq_restore(flags);
241	}
242
243	void __blk_trace_note_message(struct blk_trace *bt,
244	struct cgroup_subsys_state *css, const char *fmt, ...)
245	{
246	int n;
247	va_list args;
248	unsigned long flags;
249	char *buf;
250	u64 cgid = 0;
251
252	if (unlikely(bt->trace_state != Blktrace_running &&
253	!blk_tracer_enabled))
254	return;
255
256	/*
257	* If the BLK_TC_NOTIFY action mask isn't set, don't send any note
258	* message to the trace.
259	*/
260	if (!(bt->act_mask & BLK_TC_NOTIFY))
261	return;
262
263	local_irq_save(flags);
264	buf = this_cpu_ptr(bt->msg_data);
265	va_start(args, fmt);
266	n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
267	va_end(args);
268
269	#ifdef CONFIG_BLK_CGROUP
270	if (css && (blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))
271	cgid = cgroup_id(cgrp: css->cgroup);
272	else
273	cgid = 1;
274	#endif
275	trace_note(bt, current->pid, BLK_TN_MESSAGE, data: buf, len: n, cgid);
276	local_irq_restore(flags);
277	}
278	EXPORT_SYMBOL_GPL(__blk_trace_note_message);
279
280	static int act_log_check(struct blk_trace *bt, u64 what, sector_t sector,
281	pid_t pid)
282	{
283	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
284	return 1;
285	if (sector && (sector < bt->start_lba || sector > bt->end_lba))
286	return 1;
287	if (bt->pid && pid != bt->pid)
288	return 1;
289
290	return 0;
291	}
292
293	/*
294	* Data direction bit lookup
295	*/
296	static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
297	BLK_TC_ACT(BLK_TC_WRITE) };
298
299	#define BLK_TC_RAHEAD BLK_TC_AHEAD
300	#define BLK_TC_PREFLUSH BLK_TC_FLUSH
301
302	/* The ilog2() calls fall out because they're constant */
303	#define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) << \
304	(ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))
305
306	/*
307	* The worker for the various blk_add_trace*() types. Fills out a
308	* blk_io_trace structure and places it in a per-cpu subbuffer.
309	*/
310	static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
311	const blk_opf_t opf, u64 what, int error,
312	int pdu_len, void *pdu_data, u64 cgid)
313	{
314	struct task_struct *tsk = current;
315	struct ring_buffer_event *event = NULL;
316	struct trace_buffer *buffer = NULL;
317	unsigned long flags = 0;
318	unsigned long *sequence;
319	unsigned int trace_ctx = 0;
320	pid_t pid;
321	int cpu;
322	bool blk_tracer = blk_tracer_enabled;
323	ssize_t cgid_len = cgid ? sizeof(cgid) : 0;
324	const enum req_op op = opf & REQ_OP_MASK;
325	size_t trace_len;
326
327	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
328	return;
329
330	what |= ddir_act[op_is_write(op) ? WRITE : READ];
331	what |= MASK_TC_BIT(opf, SYNC);
332	what |= MASK_TC_BIT(opf, RAHEAD);
333	what |= MASK_TC_BIT(opf, META);
334	what |= MASK_TC_BIT(opf, PREFLUSH);
335	what |= MASK_TC_BIT(opf, FUA);
336
337	switch (op) {
338	case REQ_OP_DISCARD:
339	case REQ_OP_SECURE_ERASE:
340	what |= BLK_TC_ACT(BLK_TC_DISCARD);
341	break;
342	case REQ_OP_FLUSH:
343	what |= BLK_TC_ACT(BLK_TC_FLUSH);
344	break;
345	case REQ_OP_ZONE_APPEND:
346	what |= BLK_TC_ACT(BLK_TC_ZONE_APPEND);
347	break;
348	case REQ_OP_ZONE_RESET:
349	what |= BLK_TC_ACT(BLK_TC_ZONE_RESET);
350	break;
351	case REQ_OP_ZONE_RESET_ALL:
352	what |= BLK_TC_ACT(BLK_TC_ZONE_RESET_ALL);
353	break;
354	case REQ_OP_ZONE_FINISH:
355	what |= BLK_TC_ACT(BLK_TC_ZONE_FINISH);
356	break;
357	case REQ_OP_ZONE_OPEN:
358	what |= BLK_TC_ACT(BLK_TC_ZONE_OPEN);
359	break;
360	case REQ_OP_ZONE_CLOSE:
361	what |= BLK_TC_ACT(BLK_TC_ZONE_CLOSE);
362	break;
363	case REQ_OP_WRITE_ZEROES:
364	what |= BLK_TC_ACT(BLK_TC_WRITE_ZEROES);
365	break;
366	default:
367	break;
368	}
369
370	/* Drop trace events for zone operations with blktrace v1 */
371	if (bt->version == 1 && (what >> BLK_TC_SHIFT) > BLK_TC_END_V1) {
372	pr_debug_ratelimited("blktrace v1 cannot trace zone operation 0x%llx\n",
373	(unsigned long long)what);
374	return;
375	}
376
377	if (cgid)
378	what |= __BLK_TA_CGROUP;
379
380	pid = tsk->pid;
381	if (act_log_check(bt, what, sector, pid))
382	return;
383	cpu = raw_smp_processor_id();
384
385	if (blk_tracer) {
386	tracing_record_cmdline(current);
387
388	buffer = blk_tr->array_buffer.buffer;
389	trace_ctx = tracing_gen_ctx_flags(irqflags: 0);
390	switch (bt->version) {
391	case 1:
392	trace_len = sizeof(struct blk_io_trace);
393	break;
394	case 2:
395	default:
396	/*
397	* ftrace always uses v2 (blk_io_trace2) format.
398	*
399	* For sysfs-enabled tracing path (enabled via
400	* /sys/block/DEV/trace/enable), blk_trace_setup_queue()
401	* never initializes bt->version, leaving it 0 from
402	* kzalloc(). We must handle version==0 safely here.
403	*
404	* Fall through to default to ensure we never hit the
405	* old bug where default set trace_len=0, causing
406	* buffer underflow and memory corruption.
407	*
408	* Always use v2 format for ftrace and normalize
409	* bt->version to 2 when uninitialized.
410	*/
411	trace_len = sizeof(struct blk_io_trace2);
412	if (bt->version == 0)
413	bt->version = 2;
414	break;
415	}
416	trace_len += pdu_len + cgid_len;
417	event = trace_buffer_lock_reserve(buffer, type: TRACE_BLK,
418	len: trace_len, trace_ctx);
419	if (!event)
420	return;
421
422	switch (bt->version) {
423	case 1:
424	record_blktrace_event(t: ring_buffer_event_data(event),
425	pid, cpu, sector, bytes,
426	what, dev: bt->dev, error, cgid, cgid_len,
427	pdu_data, pdu_len);
428	break;
429	case 2:
430	default:
431	/*
432	* Use v2 recording function (record_blktrace_event2)
433	* which writes blk_io_trace2 structure with correct
434	* field layout:
435	* - 32-bit pid at offset 28
436	* - 64-bit action at offset 32
437	*
438	* Fall through to default handles version==0 case
439	* (from sysfs path), ensuring we always use correct
440	* v2 recording function to match the v2 buffer
441	* allocated above.
442	*/
443	record_blktrace_event2(t2: ring_buffer_event_data(event),
444	pid, cpu, sector, bytes,
445	what, dev: bt->dev, error, cgid, cgid_len,
446	pdu_data, pdu_len);
447	break;
448	}
449
450	trace_buffer_unlock_commit(tr: blk_tr, buffer, event, trace_ctx);
451	return;
452	}
453
454	if (unlikely(tsk->btrace_seq != blktrace_seq))
455	trace_note_tsk(tsk);
456
457	/*
458	* A word about the locking here - we disable interrupts to reserve
459	* some space in the relay per-cpu buffer, to prevent an irq
460	* from coming in and stepping on our toes.
461	*/
462	local_irq_save(flags);
463	sequence = per_cpu_ptr(bt->sequence, cpu);
464	(*sequence)++;
465	relay_blktrace_event(bt, sequence: *sequence, pid, cpu, sector, bytes,
466	what, error, cgid, cgid_len, pdu_data, pdu_len);
467	local_irq_restore(flags);
468	}
469
470	static void blk_trace_free(struct request_queue *q, struct blk_trace *bt)
471	{
472	relay_close(chan: bt->rchan);
473
474	/*
475	* If 'bt->dir' is not set, then both 'dropped' and 'msg' are created
476	* under 'q->debugfs_dir', thus lookup and remove them.
477	*/
478	if (!bt->dir) {
479	debugfs_lookup_and_remove(name: "dropped", parent: q->debugfs_dir);
480	debugfs_lookup_and_remove(name: "msg", parent: q->debugfs_dir);
481	} else {
482	debugfs_remove(dentry: bt->dir);
483	}
484	free_percpu(pdata: bt->sequence);
485	free_percpu(pdata: bt->msg_data);
486	kfree(objp: bt);
487	}
488
489	static void get_probe_ref(void)
490	{
491	mutex_lock(&blk_probe_mutex);
492	if (++blk_probes_ref == 1)
493	blk_register_tracepoints();
494	mutex_unlock(lock: &blk_probe_mutex);
495	}
496
497	static void put_probe_ref(void)
498	{
499	mutex_lock(&blk_probe_mutex);
500	if (!--blk_probes_ref)
501	blk_unregister_tracepoints();
502	mutex_unlock(lock: &blk_probe_mutex);
503	}
504
505	static int blk_trace_start(struct blk_trace *bt)
506	{
507	if (bt->trace_state != Blktrace_setup &&
508	bt->trace_state != Blktrace_stopped)
509	return -EINVAL;
510
511	blktrace_seq++;
512	smp_mb();
513	bt->trace_state = Blktrace_running;
514	raw_spin_lock_irq(&running_trace_lock);
515	list_add(new: &bt->running_list, head: &running_trace_list);
516	raw_spin_unlock_irq(&running_trace_lock);
517	trace_note_time(bt);
518
519	return 0;
520	}
521
522	static int blk_trace_stop(struct blk_trace *bt)
523	{
524	if (bt->trace_state != Blktrace_running)
525	return -EINVAL;
526
527	bt->trace_state = Blktrace_stopped;
528	raw_spin_lock_irq(&running_trace_lock);
529	list_del_init(entry: &bt->running_list);
530	raw_spin_unlock_irq(&running_trace_lock);
531	relay_flush(chan: bt->rchan);
532
533	return 0;
534	}
535
536	static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt)
537	{
538	blk_trace_stop(bt);
539	synchronize_rcu();
540	blk_trace_free(q, bt);
541	put_probe_ref();
542	}
543
544	static int __blk_trace_remove(struct request_queue *q)
545	{
546	struct blk_trace *bt;
547
548	bt = rcu_replace_pointer(q->blk_trace, NULL,
549	lockdep_is_held(&q->debugfs_mutex));
550	if (!bt)
551	return -EINVAL;
552
553	blk_trace_cleanup(q, bt);
554
555	return 0;
556	}
557
558	int blk_trace_remove(struct request_queue *q)
559	{
560	int ret;
561
562	mutex_lock(&q->debugfs_mutex);
563	ret = __blk_trace_remove(q);
564	mutex_unlock(lock: &q->debugfs_mutex);
565
566	return ret;
567	}
568	EXPORT_SYMBOL_GPL(blk_trace_remove);
569
570	static ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
571	size_t count, loff_t *ppos)
572	{
573	struct blk_trace *bt = filp->private_data;
574	size_t dropped = relay_stats(chan: bt->rchan, flags: RELAY_STATS_BUF_FULL);
575	char buf[16];
576
577	snprintf(buf, size: sizeof(buf), fmt: "%zu\n", dropped);
578
579	return simple_read_from_buffer(to: buffer, count, ppos, from: buf, strlen(buf));
580	}
581
582	static const struct file_operations blk_dropped_fops = {
583	.owner = THIS_MODULE,
584	.open = simple_open,
585	.read = blk_dropped_read,
586	.llseek = default_llseek,
587	};
588
589	static ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
590	size_t count, loff_t *ppos)
591	{
592	char *msg;
593	struct blk_trace *bt;
594
595	if (count >= BLK_TN_MAX_MSG)
596	return -EINVAL;
597
598	msg = memdup_user_nul(buffer, count);
599	if (IS_ERR(ptr: msg))
600	return PTR_ERR(ptr: msg);
601
602	bt = filp->private_data;
603	__blk_trace_note_message(bt, NULL, "%s", msg);
604	kfree(objp: msg);
605
606	return count;
607	}
608
609	static const struct file_operations blk_msg_fops = {
610	.owner = THIS_MODULE,
611	.open = simple_open,
612	.write = blk_msg_write,
613	.llseek = noop_llseek,
614	};
615
616	static int blk_remove_buf_file_callback(struct dentry *dentry)
617	{
618	debugfs_remove(dentry);
619
620	return 0;
621	}
622
623	static struct dentry *blk_create_buf_file_callback(const char *filename,
624	struct dentry *parent,
625	umode_t mode,
626	struct rchan_buf *buf,
627	int *is_global)
628	{
629	return debugfs_create_file(filename, mode, parent, buf,
630	&relay_file_operations);
631	}
632
633	static const struct rchan_callbacks blk_relay_callbacks = {
634	.create_buf_file = blk_create_buf_file_callback,
635	.remove_buf_file = blk_remove_buf_file_callback,
636	};
637
638	static void blk_trace_setup_lba(struct blk_trace *bt,
639	struct block_device *bdev)
640	{
641	if (bdev) {
642	bt->start_lba = bdev->bd_start_sect;
643	bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev);
644	} else {
645	bt->start_lba = 0;
646	bt->end_lba = -1ULL;
647	}
648	}
649
650	/*
651	* Setup everything required to start tracing
652	*/
653	static struct blk_trace *blk_trace_setup_prepare(struct request_queue *q,
654	char *name, dev_t dev,
655	u32 buf_size, u32 buf_nr,
656	struct block_device *bdev)
657	{
658	struct blk_trace *bt = NULL;
659	struct dentry *dir = NULL;
660	int ret;
661
662	lockdep_assert_held(&q->debugfs_mutex);
663
664	/*
665	* bdev can be NULL, as with scsi-generic, this is a helpful as
666	* we can be.
667	*/
668	if (rcu_dereference_protected(q->blk_trace,
669	lockdep_is_held(&q->debugfs_mutex))) {
670	pr_warn("Concurrent blktraces are not allowed on %s\n", name);
671	return ERR_PTR(error: -EBUSY);
672	}
673
674	bt = kzalloc(sizeof(*bt), GFP_KERNEL);
675	if (!bt)
676	return ERR_PTR(error: -ENOMEM);
677
678	ret = -ENOMEM;
679	bt->sequence = alloc_percpu(unsigned long);
680	if (!bt->sequence)
681	goto err;
682
683	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
684	if (!bt->msg_data)
685	goto err;
686
687	/*
688	* When tracing the whole disk reuse the existing debugfs directory
689	* created by the block layer on init. For partitions block devices,
690	* and scsi-generic block devices we create a temporary new debugfs
691	* directory that will be removed once the trace ends.
692	*/
693	if (bdev && !bdev_is_partition(bdev))
694	dir = q->debugfs_dir;
695	else
696	bt->dir = dir = debugfs_create_dir(name, parent: blk_debugfs_root);
697
698	/*
699	* As blktrace relies on debugfs for its interface the debugfs directory
700	* is required, contrary to the usual mantra of not checking for debugfs
701	* files or directories.
702	*/
703	if (IS_ERR_OR_NULL(ptr: dir)) {
704	pr_warn("debugfs_dir not present for %s so skipping\n", name);
705	ret = -ENOENT;
706	goto err;
707	}
708
709	bt->dev = dev;
710	INIT_LIST_HEAD(list: &bt->running_list);
711
712	ret = -EIO;
713	debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);
714	debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
715
716	bt->rchan = relay_open(base_filename: "trace", parent: dir, subbuf_size: buf_size, n_subbufs: buf_nr,
717	cb: &blk_relay_callbacks, private_data: bt);
718	if (!bt->rchan)
719	goto err;
720
721	blk_trace_setup_lba(bt, bdev);
722
723	return bt;
724
725	err:
726	blk_trace_free(q, bt);
727
728	return ERR_PTR(error: ret);
729	}
730
731	static void blk_trace_setup_finalize(struct request_queue *q,
732	char *name, int version,
733	struct blk_trace *bt,
734	struct blk_user_trace_setup2 *buts)
735
736	{
737	strscpy_pad(buts->name, name, BLKTRACE_BDEV_SIZE2);
738
739	/*
740	* some device names have larger paths - convert the slashes
741	* to underscores for this to work as expected
742	*/
743	strreplace(str: buts->name, old: '/', new: '_');
744
745	bt->version = version;
746	bt->act_mask = buts->act_mask;
747	if (!bt->act_mask)
748	bt->act_mask = (u16) -1;
749
750	/* overwrite with user settings */
751	if (buts->start_lba)
752	bt->start_lba = buts->start_lba;
753	if (buts->end_lba)
754	bt->end_lba = buts->end_lba;
755
756	bt->pid = buts->pid;
757	bt->trace_state = Blktrace_setup;
758
759	rcu_assign_pointer(q->blk_trace, bt);
760	get_probe_ref();
761	}
762
763	int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
764	struct block_device *bdev,
765	char __user *arg)
766	{
767	struct blk_user_trace_setup2 buts2;
768	struct blk_user_trace_setup buts;
769	struct blk_trace *bt;
770	int ret;
771
772	ret = copy_from_user(to: &buts, from: arg, n: sizeof(buts));
773	if (ret)
774	return -EFAULT;
775
776	if (!buts.buf_size || !buts.buf_nr)
777	return -EINVAL;
778
779	buts2 = (struct blk_user_trace_setup2) {
780	.act_mask = buts.act_mask,
781	.buf_size = buts.buf_size,
782	.buf_nr = buts.buf_nr,
783	.start_lba = buts.start_lba,
784	.end_lba = buts.end_lba,
785	.pid = buts.pid,
786	};
787
788	mutex_lock(&q->debugfs_mutex);
789	bt = blk_trace_setup_prepare(q, name, dev, buf_size: buts.buf_size, buf_nr: buts.buf_nr,
790	bdev);
791	if (IS_ERR(ptr: bt)) {
792	mutex_unlock(lock: &q->debugfs_mutex);
793	return PTR_ERR(ptr: bt);
794	}
795	blk_trace_setup_finalize(q, name, version: 1, bt, buts: &buts2);
796	strcpy(p: buts.name, q: buts2.name);
797	mutex_unlock(lock: &q->debugfs_mutex);
798
799	if (copy_to_user(to: arg, from: &buts, n: sizeof(buts))) {
800	blk_trace_remove(q);
801	return -EFAULT;
802	}
803	return 0;
804	}
805	EXPORT_SYMBOL_GPL(blk_trace_setup);
806
807	static int blk_trace_setup2(struct request_queue *q, char *name, dev_t dev,
808	struct block_device *bdev, char __user *arg)
809	{
810	struct blk_user_trace_setup2 buts2;
811	struct blk_trace *bt;
812
813	if (copy_from_user(to: &buts2, from: arg, n: sizeof(buts2)))
814	return -EFAULT;
815
816	if (!buts2.buf_size || !buts2.buf_nr)
817	return -EINVAL;
818
819	if (buts2.flags != 0)
820	return -EINVAL;
821
822	mutex_lock(&q->debugfs_mutex);
823	bt = blk_trace_setup_prepare(q, name, dev, buf_size: buts2.buf_size, buf_nr: buts2.buf_nr,
824	bdev);
825	if (IS_ERR(ptr: bt)) {
826	mutex_unlock(lock: &q->debugfs_mutex);
827	return PTR_ERR(ptr: bt);
828	}
829	blk_trace_setup_finalize(q, name, version: 2, bt, buts: &buts2);
830	mutex_unlock(lock: &q->debugfs_mutex);
831
832	if (copy_to_user(to: arg, from: &buts2, n: sizeof(buts2))) {
833	blk_trace_remove(q);
834	return -EFAULT;
835	}
836	return 0;
837	}
838
839	#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
840	static int compat_blk_trace_setup(struct request_queue *q, char *name,
841	dev_t dev, struct block_device *bdev,
842	char __user *arg)
843	{
844	struct blk_user_trace_setup2 buts2;
845	struct compat_blk_user_trace_setup cbuts;
846	struct blk_trace *bt;
847
848	if (copy_from_user(to: &cbuts, from: arg, n: sizeof(cbuts)))
849	return -EFAULT;
850
851	if (!cbuts.buf_size || !cbuts.buf_nr)
852	return -EINVAL;
853
854	buts2 = (struct blk_user_trace_setup2) {
855	.act_mask = cbuts.act_mask,
856	.buf_size = cbuts.buf_size,
857	.buf_nr = cbuts.buf_nr,
858	.start_lba = cbuts.start_lba,
859	.end_lba = cbuts.end_lba,
860	.pid = cbuts.pid,
861	};
862
863	mutex_lock(&q->debugfs_mutex);
864	bt = blk_trace_setup_prepare(q, name, dev, buf_size: buts2.buf_size, buf_nr: buts2.buf_nr,
865	bdev);
866	if (IS_ERR(ptr: bt)) {
867	mutex_unlock(lock: &q->debugfs_mutex);
868	return PTR_ERR(ptr: bt);
869	}
870	blk_trace_setup_finalize(q, name, version: 1, bt, buts: &buts2);
871	mutex_unlock(lock: &q->debugfs_mutex);
872
873	if (copy_to_user(to: arg, from: &buts2.name, ARRAY_SIZE(buts2.name))) {
874	blk_trace_remove(q);
875	return -EFAULT;
876	}
877
878	return 0;
879	}
880	#endif
881
882	static int __blk_trace_startstop(struct request_queue *q, int start)
883	{
884	struct blk_trace *bt;
885
886	bt = rcu_dereference_protected(q->blk_trace,
887	lockdep_is_held(&q->debugfs_mutex));
888	if (bt == NULL)
889	return -EINVAL;
890
891	if (start)
892	return blk_trace_start(bt);
893	else
894	return blk_trace_stop(bt);
895	}
896
897	int blk_trace_startstop(struct request_queue *q, int start)
898	{
899	int ret;
900
901	mutex_lock(&q->debugfs_mutex);
902	ret = __blk_trace_startstop(q, start);
903	mutex_unlock(lock: &q->debugfs_mutex);
904
905	return ret;
906	}
907	EXPORT_SYMBOL_GPL(blk_trace_startstop);
908
909	/*
910	* When reading or writing the blktrace sysfs files, the references to the
911	* opened sysfs or device files should prevent the underlying block device
912	* from being removed. So no further delete protection is really needed.
913	*/
914
915	/**
916	* blk_trace_ioctl - handle the ioctls associated with tracing
917	* @bdev: the block device
918	* @cmd: the ioctl cmd
919	* @arg: the argument data, if any
920	*
921	**/
922	int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
923	{
924	struct request_queue *q = bdev_get_queue(bdev);
925	int ret, start = 0;
926	char b[BDEVNAME_SIZE];
927
928	switch (cmd) {
929	case BLKTRACESETUP2:
930	snprintf(buf: b, size: sizeof(b), fmt: "%pg", bdev);
931	ret = blk_trace_setup2(q, name: b, dev: bdev->bd_dev, bdev, arg);
932	break;
933	case BLKTRACESETUP:
934	snprintf(buf: b, size: sizeof(b), fmt: "%pg", bdev);
935	ret = blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
936	break;
937	#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
938	case BLKTRACESETUP32:
939	snprintf(buf: b, size: sizeof(b), fmt: "%pg", bdev);
940	ret = compat_blk_trace_setup(q, name: b, dev: bdev->bd_dev, bdev, arg);
941	break;
942	#endif
943	case BLKTRACESTART:
944	start = 1;
945	fallthrough;
946	case BLKTRACESTOP:
947	ret = blk_trace_startstop(q, start);
948	break;
949	case BLKTRACETEARDOWN:
950	ret = blk_trace_remove(q);
951	break;
952	default:
953	ret = -ENOTTY;
954	break;
955	}
956	return ret;
957	}
958
959	/**
960	* blk_trace_shutdown - stop and cleanup trace structures
961	* @q: the request queue associated with the device
962	*
963	**/
964	void blk_trace_shutdown(struct request_queue *q)
965	{
966	if (rcu_dereference_protected(q->blk_trace,
967	lockdep_is_held(&q->debugfs_mutex)))
968	__blk_trace_remove(q);
969	}
970
971	#ifdef CONFIG_BLK_CGROUP
972	static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
973	{
974	struct cgroup_subsys_state *blkcg_css;
975	struct blk_trace *bt;
976
977	/* We don't use the 'bt' value here except as an optimization... */
978	bt = rcu_dereference_protected(q->blk_trace, 1);
979	if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))
980	return 0;
981
982	blkcg_css = bio_blkcg_css(bio);
983	if (!blkcg_css)
984	return 0;
985	return cgroup_id(cgrp: blkcg_css->cgroup);
986	}
987	#else
988	static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio)
989	{
990	return 0;
991	}
992	#endif
993
994	static u64
995	blk_trace_request_get_cgid(struct request *rq)
996	{
997	if (!rq->bio)
998	return 0;
999	/* Use the first bio */
1000	return blk_trace_bio_get_cgid(q: rq->q, bio: rq->bio);
1001	}
1002
1003	/*
1004	* blktrace probes
1005	*/
1006
1007	/**
1008	* blk_add_trace_rq - Add a trace for a request oriented action
1009	* @rq: the source request
1010	* @error: return status to log
1011	* @nr_bytes: number of completed bytes
1012	* @what: the action
1013	* @cgid: the cgroup info
1014	*
1015	* Description:
1016	* Records an action against a request. Will log the bio offset + size.
1017	*
1018	**/
1019	static void blk_add_trace_rq(struct request *rq, blk_status_t error,
1020	unsigned int nr_bytes, u64 what, u64 cgid)
1021	{
1022	struct blk_trace *bt;
1023
1024	rcu_read_lock();
1025	bt = rcu_dereference(rq->q->blk_trace);
1026	if (likely(!bt)) {
1027	rcu_read_unlock();
1028	return;
1029	}
1030
1031	if (blk_rq_is_passthrough(rq))
1032	what |= BLK_TC_ACT(BLK_TC_PC);
1033	else
1034	what |= BLK_TC_ACT(BLK_TC_FS);
1035
1036	__blk_add_trace(bt, sector: blk_rq_trace_sector(rq), bytes: nr_bytes, opf: rq->cmd_flags,
1037	what, error: blk_status_to_errno(status: error), pdu_len: 0, NULL, cgid);
1038	rcu_read_unlock();
1039	}
1040
1041	static void blk_add_trace_rq_insert(void *ignore, struct request *rq)
1042	{
1043	blk_add_trace_rq(rq, error: 0, nr_bytes: blk_rq_bytes(rq), BLK_TA_INSERT,
1044	cgid: blk_trace_request_get_cgid(rq));
1045	}
1046
1047	static void blk_add_trace_rq_issue(void *ignore, struct request *rq)
1048	{
1049	blk_add_trace_rq(rq, error: 0, nr_bytes: blk_rq_bytes(rq), BLK_TA_ISSUE,
1050	cgid: blk_trace_request_get_cgid(rq));
1051	}
1052
1053	static void blk_add_trace_rq_merge(void *ignore, struct request *rq)
1054	{
1055	blk_add_trace_rq(rq, error: 0, nr_bytes: blk_rq_bytes(rq), BLK_TA_BACKMERGE,
1056	cgid: blk_trace_request_get_cgid(rq));
1057	}
1058
1059	static void blk_add_trace_rq_requeue(void *ignore, struct request *rq)
1060	{
1061	blk_add_trace_rq(rq, error: 0, nr_bytes: blk_rq_bytes(rq), BLK_TA_REQUEUE,
1062	cgid: blk_trace_request_get_cgid(rq));
1063	}
1064
1065	static void blk_add_trace_rq_complete(void *ignore, struct request *rq,
1066	blk_status_t error, unsigned int nr_bytes)
1067	{
1068	blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE,
1069	cgid: blk_trace_request_get_cgid(rq));
1070	}
1071
1072	static void blk_add_trace_zone_update_request(void *ignore, struct request *rq)
1073	{
1074	struct blk_trace *bt;
1075
1076	rcu_read_lock();
1077	bt = rcu_dereference(rq->q->blk_trace);
1078	if (likely(!bt) || bt->version < 2) {
1079	rcu_read_unlock();
1080	return;
1081	}
1082	rcu_read_unlock();
1083
1084	blk_add_trace_rq(rq, error: 0, nr_bytes: blk_rq_bytes(rq), BLK_TA_ZONE_APPEND,
1085	cgid: blk_trace_request_get_cgid(rq));
1086	}
1087
1088	/**
1089	* blk_add_trace_bio - Add a trace for a bio oriented action
1090	* @q: queue the io is for
1091	* @bio: the source bio
1092	* @what: the action
1093	* @error: error, if any
1094	*
1095	* Description:
1096	* Records an action against a bio. Will log the bio offset + size.
1097	*
1098	**/
1099	static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
1100	u64 what, int error)
1101	{
1102	struct blk_trace *bt;
1103
1104	rcu_read_lock();
1105	bt = rcu_dereference(q->blk_trace);
1106	if (likely(!bt)) {
1107	rcu_read_unlock();
1108	return;
1109	}
1110
1111	__blk_add_trace(bt, sector: bio->bi_iter.bi_sector, bytes: bio->bi_iter.bi_size,
1112	opf: bio->bi_opf, what, error, pdu_len: 0, NULL,
1113	cgid: blk_trace_bio_get_cgid(q, bio));
1114	rcu_read_unlock();
1115	}
1116
1117	static void blk_add_trace_bio_complete(void *ignore,
1118	struct request_queue *q, struct bio *bio)
1119	{
1120	blk_add_trace_bio(q, bio, BLK_TA_COMPLETE,
1121	error: blk_status_to_errno(status: bio->bi_status));
1122	}
1123
1124	static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio)
1125	{
1126	blk_add_trace_bio(q: bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE,
1127	error: 0);
1128	}
1129
1130	static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio)
1131	{
1132	blk_add_trace_bio(q: bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE,
1133	error: 0);
1134	}
1135
1136	static void blk_add_trace_bio_queue(void *ignore, struct bio *bio)
1137	{
1138	blk_add_trace_bio(q: bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, error: 0);
1139	}
1140
1141	static void blk_add_trace_getrq(void *ignore, struct bio *bio)
1142	{
1143	blk_add_trace_bio(q: bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, error: 0);
1144	}
1145
1146	static void blk_add_trace_plug(void *ignore, struct request_queue *q)
1147	{
1148	struct blk_trace *bt;
1149
1150	rcu_read_lock();
1151	bt = rcu_dereference(q->blk_trace);
1152	if (bt)
1153	__blk_add_trace(bt, sector: 0, bytes: 0, opf: 0, BLK_TA_PLUG, error: 0, pdu_len: 0, NULL, cgid: 0);
1154	rcu_read_unlock();
1155	}
1156
1157	static void blk_add_trace_unplug(void *ignore, struct request_queue *q,
1158	unsigned int depth, bool explicit)
1159	{
1160	struct blk_trace *bt;
1161
1162	rcu_read_lock();
1163	bt = rcu_dereference(q->blk_trace);
1164	if (bt) {
1165	__be64 rpdu = cpu_to_be64(depth);
1166	u64 what;
1167
1168	if (explicit)
1169	what = BLK_TA_UNPLUG_IO;
1170	else
1171	what = BLK_TA_UNPLUG_TIMER;
1172
1173	__blk_add_trace(bt, sector: 0, bytes: 0, opf: 0, what, error: 0, pdu_len: sizeof(rpdu), pdu_data: &rpdu, cgid: 0);
1174	}
1175	rcu_read_unlock();
1176	}
1177
1178	static void blk_add_trace_zone_plug(void *ignore, struct request_queue *q,
1179	unsigned int zno, sector_t sector,
1180	unsigned int sectors)
1181	{
1182	struct blk_trace *bt;
1183
1184	rcu_read_lock();
1185	bt = rcu_dereference(q->blk_trace);
1186	if (bt && bt->version >= 2)
1187	__blk_add_trace(bt, sector, bytes: sectors << SECTOR_SHIFT, opf: 0,
1188	BLK_TA_ZONE_PLUG, error: 0, pdu_len: 0, NULL, cgid: 0);
1189	rcu_read_unlock();
1190
1191	return;
1192	}
1193
1194	static void blk_add_trace_zone_unplug(void *ignore, struct request_queue *q,
1195	unsigned int zno, sector_t sector,
1196	unsigned int sectors)
1197	{
1198	struct blk_trace *bt;
1199
1200	rcu_read_lock();
1201	bt = rcu_dereference(q->blk_trace);
1202	if (bt && bt->version >= 2)
1203	__blk_add_trace(bt, sector, bytes: sectors << SECTOR_SHIFT, opf: 0,
1204	BLK_TA_ZONE_UNPLUG, error: 0, pdu_len: 0, NULL, cgid: 0);
1205	rcu_read_unlock();
1206	return;
1207	}
1208
1209	static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu)
1210	{
1211	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
1212	struct blk_trace *bt;
1213
1214	rcu_read_lock();
1215	bt = rcu_dereference(q->blk_trace);
1216	if (bt) {
1217	__be64 rpdu = cpu_to_be64(pdu);
1218
1219	__blk_add_trace(bt, sector: bio->bi_iter.bi_sector,
1220	bytes: bio->bi_iter.bi_size, opf: bio->bi_opf, BLK_TA_SPLIT,
1221	error: blk_status_to_errno(status: bio->bi_status),
1222	pdu_len: sizeof(rpdu), pdu_data: &rpdu,
1223	cgid: blk_trace_bio_get_cgid(q, bio));
1224	}
1225	rcu_read_unlock();
1226	}
1227
1228	/**
1229	* blk_add_trace_bio_remap - Add a trace for a bio-remap operation
1230	* @ignore: trace callback data parameter (not used)
1231	* @bio: the source bio
1232	* @dev: source device
1233	* @from: source sector
1234	*
1235	* Called after a bio is remapped to a different device and/or sector.
1236	**/
1237	static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev,
1238	sector_t from)
1239	{
1240	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
1241	struct blk_trace *bt;
1242	struct blk_io_trace_remap r;
1243
1244	rcu_read_lock();
1245	bt = rcu_dereference(q->blk_trace);
1246	if (likely(!bt)) {
1247	rcu_read_unlock();
1248	return;
1249	}
1250
1251	r.device_from = cpu_to_be32(dev);
1252	r.device_to = cpu_to_be32(bio_dev(bio));
1253	r.sector_from = cpu_to_be64(from);
1254
1255	__blk_add_trace(bt, sector: bio->bi_iter.bi_sector, bytes: bio->bi_iter.bi_size,
1256	opf: bio->bi_opf, BLK_TA_REMAP,
1257	error: blk_status_to_errno(status: bio->bi_status),
1258	pdu_len: sizeof(r), pdu_data: &r, cgid: blk_trace_bio_get_cgid(q, bio));
1259	rcu_read_unlock();
1260	}
1261
1262	/**
1263	* blk_add_trace_rq_remap - Add a trace for a request-remap operation
1264	* @ignore: trace callback data parameter (not used)
1265	* @rq: the source request
1266	* @dev: target device
1267	* @from: source sector
1268	*
1269	* Description:
1270	* Device mapper remaps request to other devices.
1271	* Add a trace for that action.
1272	*
1273	**/
1274	static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev,
1275	sector_t from)
1276	{
1277	struct blk_trace *bt;
1278	struct blk_io_trace_remap r;
1279
1280	rcu_read_lock();
1281	bt = rcu_dereference(rq->q->blk_trace);
1282	if (likely(!bt)) {
1283	rcu_read_unlock();
1284	return;
1285	}
1286
1287	r.device_from = cpu_to_be32(dev);
1288	r.device_to = cpu_to_be32(disk_devt(rq->q->disk));
1289	r.sector_from = cpu_to_be64(from);
1290
1291	__blk_add_trace(bt, sector: blk_rq_pos(rq), bytes: blk_rq_bytes(rq),
1292	opf: rq->cmd_flags, BLK_TA_REMAP, error: 0,
1293	pdu_len: sizeof(r), pdu_data: &r, cgid: blk_trace_request_get_cgid(rq));
1294	rcu_read_unlock();
1295	}
1296
1297	/**
1298	* blk_add_driver_data - Add binary message with driver-specific data
1299	* @rq: io request
1300	* @data: driver-specific data
1301	* @len: length of driver-specific data
1302	*
1303	* Description:
1304	* Some drivers might want to write driver-specific data per request.
1305	*
1306	**/
1307	void blk_add_driver_data(struct request *rq, void *data, size_t len)
1308	{
1309	struct blk_trace *bt;
1310
1311	rcu_read_lock();
1312	bt = rcu_dereference(rq->q->blk_trace);
1313	if (likely(!bt)) {
1314	rcu_read_unlock();
1315	return;
1316	}
1317
1318	__blk_add_trace(bt, sector: blk_rq_trace_sector(rq), bytes: blk_rq_bytes(rq), opf: 0,
1319	BLK_TA_DRV_DATA, error: 0, pdu_len: len, pdu_data: data,
1320	cgid: blk_trace_request_get_cgid(rq));
1321	rcu_read_unlock();
1322	}
1323	EXPORT_SYMBOL_GPL(blk_add_driver_data);
1324
1325	static void blk_register_tracepoints(void)
1326	{
1327	int ret;
1328
1329	ret = register_trace_block_rq_insert(probe: blk_add_trace_rq_insert, NULL);
1330	WARN_ON(ret);
1331	ret = register_trace_block_rq_issue(probe: blk_add_trace_rq_issue, NULL);
1332	WARN_ON(ret);
1333	ret = register_trace_block_rq_merge(probe: blk_add_trace_rq_merge, NULL);
1334	WARN_ON(ret);
1335	ret = register_trace_block_rq_requeue(probe: blk_add_trace_rq_requeue, NULL);
1336	WARN_ON(ret);
1337	ret = register_trace_block_rq_complete(probe: blk_add_trace_rq_complete, NULL);
1338	WARN_ON(ret);
1339	ret = register_trace_block_bio_complete(probe: blk_add_trace_bio_complete, NULL);
1340	WARN_ON(ret);
1341	ret = register_trace_block_bio_backmerge(probe: blk_add_trace_bio_backmerge, NULL);
1342	WARN_ON(ret);
1343	ret = register_trace_block_bio_frontmerge(probe: blk_add_trace_bio_frontmerge, NULL);
1344	WARN_ON(ret);
1345	ret = register_trace_block_bio_queue(probe: blk_add_trace_bio_queue, NULL);
1346	WARN_ON(ret);
1347	ret = register_trace_block_getrq(probe: blk_add_trace_getrq, NULL);
1348	WARN_ON(ret);
1349	ret = register_trace_blk_zone_append_update_request_bio(
1350	probe: blk_add_trace_zone_update_request, NULL);
1351	WARN_ON(ret);
1352	ret = register_trace_disk_zone_wplug_add_bio(probe: blk_add_trace_zone_plug,
1353	NULL);
1354	WARN_ON(ret);
1355	ret = register_trace_blk_zone_wplug_bio(probe: blk_add_trace_zone_unplug,
1356	NULL);
1357	WARN_ON(ret);
1358	ret = register_trace_block_plug(probe: blk_add_trace_plug, NULL);
1359	WARN_ON(ret);
1360	ret = register_trace_block_unplug(probe: blk_add_trace_unplug, NULL);
1361	WARN_ON(ret);
1362	ret = register_trace_block_split(probe: blk_add_trace_split, NULL);
1363	WARN_ON(ret);
1364	ret = register_trace_block_bio_remap(probe: blk_add_trace_bio_remap, NULL);
1365	WARN_ON(ret);
1366	ret = register_trace_block_rq_remap(probe: blk_add_trace_rq_remap, NULL);
1367	WARN_ON(ret);
1368	}
1369
1370	static void blk_unregister_tracepoints(void)
1371	{
1372	unregister_trace_block_rq_remap(probe: blk_add_trace_rq_remap, NULL);
1373	unregister_trace_block_bio_remap(probe: blk_add_trace_bio_remap, NULL);
1374	unregister_trace_block_split(probe: blk_add_trace_split, NULL);
1375	unregister_trace_block_unplug(probe: blk_add_trace_unplug, NULL);
1376	unregister_trace_block_plug(probe: blk_add_trace_plug, NULL);
1377	unregister_trace_blk_zone_wplug_bio(probe: blk_add_trace_zone_unplug, NULL);
1378	unregister_trace_disk_zone_wplug_add_bio(probe: blk_add_trace_zone_plug, NULL);
1379	unregister_trace_blk_zone_append_update_request_bio(
1380	probe: blk_add_trace_zone_update_request, NULL);
1381	unregister_trace_block_getrq(probe: blk_add_trace_getrq, NULL);
1382	unregister_trace_block_bio_queue(probe: blk_add_trace_bio_queue, NULL);
1383	unregister_trace_block_bio_frontmerge(probe: blk_add_trace_bio_frontmerge, NULL);
1384	unregister_trace_block_bio_backmerge(probe: blk_add_trace_bio_backmerge, NULL);
1385	unregister_trace_block_bio_complete(probe: blk_add_trace_bio_complete, NULL);
1386	unregister_trace_block_rq_complete(probe: blk_add_trace_rq_complete, NULL);
1387	unregister_trace_block_rq_requeue(probe: blk_add_trace_rq_requeue, NULL);
1388	unregister_trace_block_rq_merge(probe: blk_add_trace_rq_merge, NULL);
1389	unregister_trace_block_rq_issue(probe: blk_add_trace_rq_issue, NULL);
1390	unregister_trace_block_rq_insert(probe: blk_add_trace_rq_insert, NULL);
1391
1392	tracepoint_synchronize_unregister();
1393	}
1394
1395	/*
1396	* struct blk_io_tracer formatting routines
1397	*/
1398
1399	static void fill_rwbs(char *rwbs, const struct blk_io_trace2 *t)
1400	{
1401	int i = 0;
1402	int tc = t->action >> BLK_TC_SHIFT;
1403
1404	if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) {
1405	rwbs[i++] = 'N';
1406	goto out;
1407	}
1408
1409	if (tc & BLK_TC_FLUSH)
1410	rwbs[i++] = 'F';
1411
1412	if (tc & BLK_TC_DISCARD)
1413	rwbs[i++] = 'D';
1414	else if (tc & BLK_TC_WRITE_ZEROES) {
1415	rwbs[i++] = 'W';
1416	rwbs[i++] = 'Z';
1417	} else if (tc & BLK_TC_WRITE)
1418	rwbs[i++] = 'W';
1419	else if (t->bytes)
1420	rwbs[i++] = 'R';
1421	else
1422	rwbs[i++] = 'N';
1423
1424	if (tc & BLK_TC_FUA)
1425	rwbs[i++] = 'F';
1426	if (tc & BLK_TC_AHEAD)
1427	rwbs[i++] = 'A';
1428	if (tc & BLK_TC_SYNC)
1429	rwbs[i++] = 'S';
1430	if (tc & BLK_TC_META)
1431	rwbs[i++] = 'M';
1432	out:
1433	rwbs[i] = '\0';
1434	}
1435
1436	static inline
1437	const struct blk_io_trace2 *te_blk_io_trace(const struct trace_entry *ent)
1438	{
1439	return (const struct blk_io_trace2 *)ent;
1440	}
1441
1442	static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg)
1443	{
1444	return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0);
1445	}
1446
1447	static inline u64 t_cgid(const struct trace_entry *ent)
1448	{
1449	return *(u64 *)(te_blk_io_trace(ent) + 1);
1450	}
1451
1452	static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg)
1453	{
1454	return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0);
1455	}
1456
1457	static inline u32 t_action(const struct trace_entry *ent)
1458	{
1459	return te_blk_io_trace(ent)->action;
1460	}
1461
1462	static inline u32 t_bytes(const struct trace_entry *ent)
1463	{
1464	return te_blk_io_trace(ent)->bytes;
1465	}
1466
1467	static inline u32 t_sec(const struct trace_entry *ent)
1468	{
1469	return te_blk_io_trace(ent)->bytes >> 9;
1470	}
1471
1472	static inline unsigned long long t_sector(const struct trace_entry *ent)
1473	{
1474	return te_blk_io_trace(ent)->sector;
1475	}
1476
1477	static inline __u16 t_error(const struct trace_entry *ent)
1478	{
1479	return te_blk_io_trace(ent)->error;
1480	}
1481
1482	static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg)
1483	{
1484	const __be64 *val = pdu_start(ent, has_cg);
1485	return be64_to_cpu(*val);
1486	}
1487
1488	typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act,
1489	bool has_cg);
1490
1491	static void blk_log_action_classic(struct trace_iterator *iter, const char *act,
1492	bool has_cg)
1493	{
1494	char rwbs[RWBS_LEN];
1495	unsigned long long ts = iter->ts;
1496	unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC);
1497	unsigned secs = (unsigned long)ts;
1498	const struct blk_io_trace2 *t = te_blk_io_trace(ent: iter->ent);
1499
1500	fill_rwbs(rwbs, t);
1501
1502	trace_seq_printf(s: &iter->seq,
1503	fmt: "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ",
1504	MAJOR(t->device), MINOR(t->device), iter->cpu,
1505	secs, nsec_rem, iter->ent->pid, act, rwbs);
1506	}
1507
1508	static void blk_log_action(struct trace_iterator *iter, const char *act,
1509	bool has_cg)
1510	{
1511	char rwbs[RWBS_LEN];
1512	const struct blk_io_trace2 *t = te_blk_io_trace(ent: iter->ent);
1513
1514	fill_rwbs(rwbs, t);
1515	if (has_cg) {
1516	u64 id = t_cgid(ent: iter->ent);
1517
1518	if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) {
1519	char blkcg_name_buf[NAME_MAX + 1] = "<...>";
1520
1521	cgroup_path_from_kernfs_id(id, buf: blkcg_name_buf,
1522	buflen: sizeof(blkcg_name_buf));
1523	trace_seq_printf(s: &iter->seq, fmt: "%3d,%-3d %s %2s %3s ",
1524	MAJOR(t->device), MINOR(t->device),
1525	blkcg_name_buf, act, rwbs);
1526	} else {
1527	/*
1528	* The cgid portion used to be "INO,GEN". Userland
1529	* builds a FILEID_INO32_GEN fid out of them and
1530	* opens the cgroup using open_by_handle_at(2).
1531	* While 32bit ino setups are still the same, 64bit
1532	* ones now use the 64bit ino as the whole ID and
1533	* no longer use generation.
1534	*
1535	* Regardless of the content, always output
1536	* "LOW32,HIGH32" so that FILEID_INO32_GEN fid can
1537	* be mapped back to @id on both 64 and 32bit ino
1538	* setups. See __kernfs_fh_to_dentry().
1539	*/
1540	trace_seq_printf(s: &iter->seq,
1541	fmt: "%3d,%-3d %llx,%-llx %2s %3s ",
1542	MAJOR(t->device), MINOR(t->device),
1543	id & U32_MAX, id >> 32, act, rwbs);
1544	}
1545	} else
1546	trace_seq_printf(s: &iter->seq, fmt: "%3d,%-3d %2s %3s ",
1547	MAJOR(t->device), MINOR(t->device), act, rwbs);
1548	}
1549
1550	static void blk_log_dump_pdu(struct trace_seq *s,
1551	const struct trace_entry *ent, bool has_cg)
1552	{
1553	const unsigned char *pdu_buf;
1554	int pdu_len;
1555	int i, end;
1556
1557	pdu_buf = pdu_start(ent, has_cg);
1558	pdu_len = pdu_real_len(ent, has_cg);
1559
1560	if (!pdu_len)
1561	return;
1562
1563	/* find the last zero that needs to be printed */
1564	for (end = pdu_len - 1; end >= 0; end--)
1565	if (pdu_buf[end])
1566	break;
1567	end++;
1568
1569	trace_seq_putc(s, c: '(');
1570
1571	for (i = 0; i < pdu_len; i++) {
1572
1573	trace_seq_printf(s, fmt: "%s%02x",
1574	i == 0 ? "" : " ", pdu_buf[i]);
1575
1576	/*
1577	* stop when the rest is just zeros and indicate so
1578	* with a ".." appended
1579	*/
1580	if (i == end && end != pdu_len - 1) {
1581	trace_seq_puts(s, str: " ..) ");
1582	return;
1583	}
1584	}
1585
1586	trace_seq_puts(s, str: ") ");
1587	}
1588
1589	static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1590	{
1591	char cmd[TASK_COMM_LEN];
1592
1593	trace_find_cmdline(pid: ent->pid, comm: cmd);
1594
1595	if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
1596	trace_seq_printf(s, fmt: "%u ", t_bytes(ent));
1597	blk_log_dump_pdu(s, ent, has_cg);
1598	trace_seq_printf(s, fmt: "[%s]\n", cmd);
1599	} else {
1600	if (t_sec(ent))
1601	trace_seq_printf(s, fmt: "%llu + %u [%s]\n",
1602	t_sector(ent), t_sec(ent), cmd);
1603	else
1604	trace_seq_printf(s, fmt: "[%s]\n", cmd);
1605	}
1606	}
1607
1608	static void blk_log_with_error(struct trace_seq *s,
1609	const struct trace_entry *ent, bool has_cg)
1610	{
1611	if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
1612	blk_log_dump_pdu(s, ent, has_cg);
1613	trace_seq_printf(s, fmt: "[%d]\n", t_error(ent));
1614	} else {
1615	if (t_sec(ent))
1616	trace_seq_printf(s, fmt: "%llu + %u [%d]\n",
1617	t_sector(ent),
1618	t_sec(ent), t_error(ent));
1619	else
1620	trace_seq_printf(s, fmt: "%llu [%d]\n",
1621	t_sector(ent), t_error(ent));
1622	}
1623	}
1624
1625	static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1626	{
1627	const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg);
1628
1629	trace_seq_printf(s, fmt: "%llu + %u <- (%d,%d) %llu\n",
1630	t_sector(ent), t_sec(ent),
1631	MAJOR(be32_to_cpu(__r->device_from)),
1632	MINOR(be32_to_cpu(__r->device_from)),
1633	be64_to_cpu(__r->sector_from));
1634	}
1635
1636	static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1637	{
1638	char cmd[TASK_COMM_LEN];
1639
1640	trace_find_cmdline(pid: ent->pid, comm: cmd);
1641
1642	trace_seq_printf(s, fmt: "[%s]\n", cmd);
1643	}
1644
1645	static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1646	{
1647	char cmd[TASK_COMM_LEN];
1648
1649	trace_find_cmdline(pid: ent->pid, comm: cmd);
1650
1651	trace_seq_printf(s, fmt: "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg));
1652	}
1653
1654	static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg)
1655	{
1656	char cmd[TASK_COMM_LEN];
1657
1658	trace_find_cmdline(pid: ent->pid, comm: cmd);
1659
1660	trace_seq_printf(s, fmt: "%llu / %llu [%s]\n", t_sector(ent),
1661	get_pdu_int(ent, has_cg), cmd);
1662	}
1663
1664	static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent,
1665	bool has_cg)
1666	{
1667
1668	trace_seq_putmem(s, mem: pdu_start(ent, has_cg),
1669	len: pdu_real_len(ent, has_cg));
1670	trace_seq_putc(s, c: '\n');
1671	}
1672
1673	/*
1674	* struct tracer operations
1675	*/
1676
1677	static void blk_tracer_print_header(struct seq_file *m)
1678	{
1679	if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1680	return;
1681	seq_puts(m, s: "# DEV CPU TIMESTAMP PID ACT FLG\n"
1682	"# | | | | | |\n");
1683	}
1684
1685	static void blk_tracer_start(struct trace_array *tr)
1686	{
1687	blk_tracer_enabled = true;
1688	}
1689
1690	static int blk_tracer_init(struct trace_array *tr)
1691	{
1692	blk_tr = tr;
1693	blk_tracer_start(tr);
1694	return 0;
1695	}
1696
1697	static void blk_tracer_stop(struct trace_array *tr)
1698	{
1699	blk_tracer_enabled = false;
1700	}
1701
1702	static void blk_tracer_reset(struct trace_array *tr)
1703	{
1704	blk_tracer_stop(tr);
1705	}
1706
1707	static const struct {
1708	const char *act[2];
1709	void (*print)(struct trace_seq *s, const struct trace_entry *ent,
1710	bool has_cg);
1711	} what2act[] = {
1712	[__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic },
1713	[__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic },
1714	[__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic },
1715	[__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic },
1716	[__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic },
1717	[__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error },
1718	[__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic },
1719	[__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error },
1720	[__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug },
1721	[__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug },
1722	[__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug },
1723	[__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic },
1724	[__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split },
1725	[__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap },
1726	};
1727
1728	static enum print_line_t print_one_line(struct trace_iterator *iter,
1729	bool classic)
1730	{
1731	struct trace_array *tr = iter->tr;
1732	struct trace_seq *s = &iter->seq;
1733	const struct blk_io_trace2 *t;
1734	u16 what;
1735	bool long_act;
1736	blk_log_action_t *log_action;
1737	bool has_cg;
1738
1739	t = te_blk_io_trace(ent: iter->ent);
1740	what = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP;
1741	long_act = !!(tr->trace_flags & TRACE_ITER(VERBOSE));
1742	log_action = classic ? &blk_log_action_classic : &blk_log_action;
1743	has_cg = t->action & __BLK_TA_CGROUP;
1744
1745	if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) {
1746	log_action(iter, long_act ? "message" : "m", has_cg);
1747	blk_log_msg(s, ent: iter->ent, has_cg);
1748	return trace_handle_return(s);
1749	}
1750
1751	if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act)))
1752	trace_seq_printf(s, fmt: "Unknown action %x\n", what);
1753	else {
1754	log_action(iter, what2act[what].act[long_act], has_cg);
1755	what2act[what].print(s, iter->ent, has_cg);
1756	}
1757
1758	return trace_handle_return(s);
1759	}
1760
1761	static enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
1762	int flags, struct trace_event *event)
1763	{
1764	return print_one_line(iter, classic: false);
1765	}
1766
1767	static void blk_trace_synthesize_old_trace(struct trace_iterator *iter)
1768	{
1769	struct trace_seq *s = &iter->seq;
1770	struct blk_io_trace2 *t = (struct blk_io_trace2 *)iter->ent;
1771	const int offset = offsetof(struct blk_io_trace2, sector);
1772	struct blk_io_trace old = {
1773	.magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
1774	.time = iter->ts,
1775	};
1776
1777	trace_seq_putmem(s, mem: &old, len: offset);
1778	trace_seq_putmem(s, mem: &t->sector,
1779	len: sizeof(old) - offset + t->pdu_len);
1780	}
1781
1782	static enum print_line_t
1783	blk_trace_event_print_binary(struct trace_iterator *iter, int flags,
1784	struct trace_event *event)
1785	{
1786	blk_trace_synthesize_old_trace(iter);
1787
1788	return trace_handle_return(s: &iter->seq);
1789	}
1790
1791	static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
1792	{
1793	if ((iter->ent->type != TRACE_BLK) ||
1794	!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1795	return TRACE_TYPE_UNHANDLED;
1796
1797	return print_one_line(iter, classic: true);
1798	}
1799
1800	static int
1801	blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
1802	{
1803	/* don't output context-info for blk_classic output */
1804	if (bit == TRACE_BLK_OPT_CLASSIC) {
1805	if (set)
1806	tr->trace_flags &= ~TRACE_ITER(CONTEXT_INFO);
1807	else
1808	tr->trace_flags |= TRACE_ITER(CONTEXT_INFO);
1809	}
1810	return 0;
1811	}
1812
1813	static struct tracer blk_tracer __read_mostly = {
1814	.name = "blk",
1815	.init = blk_tracer_init,
1816	.reset = blk_tracer_reset,
1817	.start = blk_tracer_start,
1818	.stop = blk_tracer_stop,
1819	.print_header = blk_tracer_print_header,
1820	.print_line = blk_tracer_print_line,
1821	.flags = &blk_tracer_flags,
1822	.set_flag = blk_tracer_set_flag,
1823	};
1824
1825	static struct trace_event_functions trace_blk_event_funcs = {
1826	.trace = blk_trace_event_print,
1827	.binary = blk_trace_event_print_binary,
1828	};
1829
1830	static struct trace_event trace_blk_event = {
1831	.type = TRACE_BLK,
1832	.funcs = &trace_blk_event_funcs,
1833	};
1834
1835	static int __init init_blk_tracer(void)
1836	{
1837	if (!register_trace_event(event: &trace_blk_event)) {
1838	pr_warn("Warning: could not register block events\n");
1839	return 1;
1840	}
1841
1842	if (register_tracer(type: &blk_tracer) != 0) {
1843	pr_warn("Warning: could not register the block tracer\n");
1844	unregister_trace_event(event: &trace_blk_event);
1845	return 1;
1846	}
1847
1848	BUILD_BUG_ON(__alignof__(struct blk_user_trace_setup2) %
1849	__alignof__(long));
1850	BUILD_BUG_ON(__alignof__(struct blk_io_trace2) % __alignof__(long));
1851
1852	return 0;
1853	}
1854
1855	device_initcall(init_blk_tracer);
1856
1857	static int blk_trace_remove_queue(struct request_queue *q)
1858	{
1859	struct blk_trace *bt;
1860
1861	bt = rcu_replace_pointer(q->blk_trace, NULL,
1862	lockdep_is_held(&q->debugfs_mutex));
1863	if (bt == NULL)
1864	return -EINVAL;
1865
1866	blk_trace_stop(bt);
1867
1868	put_probe_ref();
1869	synchronize_rcu();
1870	blk_trace_free(q, bt);
1871	return 0;
1872	}
1873
1874	/*
1875	* Setup everything required to start tracing
1876	*/
1877	static int blk_trace_setup_queue(struct request_queue *q,
1878	struct block_device *bdev)
1879	{
1880	struct blk_trace *bt = NULL;
1881	int ret = -ENOMEM;
1882
1883	bt = kzalloc(sizeof(*bt), GFP_KERNEL);
1884	if (!bt)
1885	return -ENOMEM;
1886
1887	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
1888	if (!bt->msg_data)
1889	goto free_bt;
1890
1891	bt->dev = bdev->bd_dev;
1892	bt->act_mask = (u16)-1;
1893
1894	blk_trace_setup_lba(bt, bdev);
1895
1896	rcu_assign_pointer(q->blk_trace, bt);
1897	get_probe_ref();
1898	return 0;
1899
1900	free_bt:
1901	blk_trace_free(q, bt);
1902	return ret;
1903	}
1904
1905	/*
1906	* sysfs interface to enable and configure tracing
1907	*/
1908
1909	static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1910	struct device_attribute *attr,
1911	char *buf);
1912	static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1913	struct device_attribute *attr,
1914	const char *buf, size_t count);
1915	#define BLK_TRACE_DEVICE_ATTR(_name) \
1916	DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
1917	sysfs_blk_trace_attr_show, \
1918	sysfs_blk_trace_attr_store)
1919
1920	static BLK_TRACE_DEVICE_ATTR(enable);
1921	static BLK_TRACE_DEVICE_ATTR(act_mask);
1922	static BLK_TRACE_DEVICE_ATTR(pid);
1923	static BLK_TRACE_DEVICE_ATTR(start_lba);
1924	static BLK_TRACE_DEVICE_ATTR(end_lba);
1925
1926	static struct attribute *blk_trace_attrs[] = {
1927	&dev_attr_enable.attr,
1928	&dev_attr_act_mask.attr,
1929	&dev_attr_pid.attr,
1930	&dev_attr_start_lba.attr,
1931	&dev_attr_end_lba.attr,
1932	NULL
1933	};
1934
1935	struct attribute_group blk_trace_attr_group = {
1936	.name = "trace",
1937	.attrs = blk_trace_attrs,
1938	};
1939
1940	static const struct {
1941	int mask;
1942	const char *str;
1943	} mask_maps[] = {
1944	{ BLK_TC_READ, "read" },
1945	{ BLK_TC_WRITE, "write" },
1946	{ BLK_TC_FLUSH, "flush" },
1947	{ BLK_TC_SYNC, "sync" },
1948	{ BLK_TC_QUEUE, "queue" },
1949	{ BLK_TC_REQUEUE, "requeue" },
1950	{ BLK_TC_ISSUE, "issue" },
1951	{ BLK_TC_COMPLETE, "complete" },
1952	{ BLK_TC_FS, "fs" },
1953	{ BLK_TC_PC, "pc" },
1954	{ BLK_TC_NOTIFY, "notify" },
1955	{ BLK_TC_AHEAD, "ahead" },
1956	{ BLK_TC_META, "meta" },
1957	{ BLK_TC_DISCARD, "discard" },
1958	{ BLK_TC_DRV_DATA, "drv_data" },
1959	{ BLK_TC_FUA, "fua" },
1960	{ BLK_TC_WRITE_ZEROES, "write-zeroes" },
1961	};
1962
1963	static int blk_trace_str2mask(const char *str)
1964	{
1965	int i;
1966	int mask = 0;
1967	char *buf, *s, *token;
1968
1969	buf = kstrdup(s: str, GFP_KERNEL);
1970	if (buf == NULL)
1971	return -ENOMEM;
1972	s = strstrip(str: buf);
1973
1974	while (1) {
1975	token = strsep(&s, ",");
1976	if (token == NULL)
1977	break;
1978
1979	if (*token == '\0')
1980	continue;
1981
1982	for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
1983	if (strcasecmp(s1: token, s2: mask_maps[i].str) == 0) {
1984	mask |= mask_maps[i].mask;
1985	break;
1986	}
1987	}
1988	if (i == ARRAY_SIZE(mask_maps)) {
1989	mask = -EINVAL;
1990	break;
1991	}
1992	}
1993	kfree(objp: buf);
1994
1995	return mask;
1996	}
1997
1998	static ssize_t blk_trace_mask2str(char *buf, int mask)
1999	{
2000	int i;
2001	char *p = buf;
2002
2003	for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
2004	if (mask & mask_maps[i].mask) {
2005	p += sprintf(buf: p, fmt: "%s%s",
2006	(p == buf) ? "" : ",", mask_maps[i].str);
2007	}
2008	}
2009	*p++ = '\n';
2010
2011	return p - buf;
2012	}
2013
2014	static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
2015	struct device_attribute *attr,
2016	char *buf)
2017	{
2018	struct block_device *bdev = dev_to_bdev(dev);
2019	struct request_queue *q = bdev_get_queue(bdev);
2020	struct blk_trace *bt;
2021	ssize_t ret = -ENXIO;
2022
2023	mutex_lock(&q->debugfs_mutex);
2024
2025	bt = rcu_dereference_protected(q->blk_trace,
2026	lockdep_is_held(&q->debugfs_mutex));
2027	if (attr == &dev_attr_enable) {
2028	ret = sprintf(buf, fmt: "%u\n", !!bt);
2029	goto out_unlock_bdev;
2030	}
2031
2032	if (bt == NULL)
2033	ret = sprintf(buf, fmt: "disabled\n");
2034	else if (attr == &dev_attr_act_mask)
2035	ret = blk_trace_mask2str(buf, mask: bt->act_mask);
2036	else if (attr == &dev_attr_pid)
2037	ret = sprintf(buf, fmt: "%u\n", bt->pid);
2038	else if (attr == &dev_attr_start_lba)
2039	ret = sprintf(buf, fmt: "%llu\n", bt->start_lba);
2040	else if (attr == &dev_attr_end_lba)
2041	ret = sprintf(buf, fmt: "%llu\n", bt->end_lba);
2042
2043	out_unlock_bdev:
2044	mutex_unlock(lock: &q->debugfs_mutex);
2045	return ret;
2046	}
2047
2048	static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
2049	struct device_attribute *attr,
2050	const char *buf, size_t count)
2051	{
2052	struct block_device *bdev = dev_to_bdev(dev);
2053	struct request_queue *q = bdev_get_queue(bdev);
2054	struct blk_trace *bt;
2055	u64 value;
2056	ssize_t ret = -EINVAL;
2057
2058	if (count == 0)
2059	goto out;
2060
2061	if (attr == &dev_attr_act_mask) {
2062	if (kstrtoull(s: buf, base: 0, res: &value)) {
2063	/* Assume it is a list of trace category names */
2064	ret = blk_trace_str2mask(str: buf);
2065	if (ret < 0)
2066	goto out;
2067	value = ret;
2068	}
2069	} else {
2070	if (kstrtoull(s: buf, base: 0, res: &value))
2071	goto out;
2072	}
2073
2074	mutex_lock(&q->debugfs_mutex);
2075
2076	bt = rcu_dereference_protected(q->blk_trace,
2077	lockdep_is_held(&q->debugfs_mutex));
2078	if (attr == &dev_attr_enable) {
2079	if (!!value == !!bt) {
2080	ret = 0;
2081	goto out_unlock_bdev;
2082	}
2083	if (value)
2084	ret = blk_trace_setup_queue(q, bdev);
2085	else
2086	ret = blk_trace_remove_queue(q);
2087	goto out_unlock_bdev;
2088	}
2089
2090	ret = 0;
2091	if (bt == NULL) {
2092	ret = blk_trace_setup_queue(q, bdev);
2093	bt = rcu_dereference_protected(q->blk_trace,
2094	lockdep_is_held(&q->debugfs_mutex));
2095	}
2096
2097	if (ret == 0) {
2098	if (attr == &dev_attr_act_mask)
2099	bt->act_mask = value;
2100	else if (attr == &dev_attr_pid)
2101	bt->pid = value;
2102	else if (attr == &dev_attr_start_lba)
2103	bt->start_lba = value;
2104	else if (attr == &dev_attr_end_lba)
2105	bt->end_lba = value;
2106	}
2107
2108	out_unlock_bdev:
2109	mutex_unlock(lock: &q->debugfs_mutex);
2110	out:
2111	return ret ? ret : count;
2112	}
2113	#endif /* CONFIG_BLK_DEV_IO_TRACE */
2114
2115	#ifdef CONFIG_EVENT_TRACING
2116
2117	/**
2118	* blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string.
2119	* @rwbs: buffer to be filled
2120	* @opf: request operation type (REQ_OP_XXX) and flags for the tracepoint
2121	*
2122	* Description:
2123	* Maps each request operation and flag to a single character and fills the
2124	* buffer provided by the caller with resulting string.
2125	*
2126	**/
2127	void blk_fill_rwbs(char *rwbs, blk_opf_t opf)
2128	{
2129	int i = 0;
2130
2131	if (opf & REQ_PREFLUSH)
2132	rwbs[i++] = 'F';
2133
2134	switch (opf & REQ_OP_MASK) {
2135	case REQ_OP_WRITE:
2136	rwbs[i++] = 'W';
2137	break;
2138	case REQ_OP_DISCARD:
2139	rwbs[i++] = 'D';
2140	break;
2141	case REQ_OP_SECURE_ERASE:
2142	rwbs[i++] = 'D';
2143	rwbs[i++] = 'E';
2144	break;
2145	case REQ_OP_FLUSH:
2146	rwbs[i++] = 'F';
2147	break;
2148	case REQ_OP_READ:
2149	rwbs[i++] = 'R';
2150	break;
2151	case REQ_OP_ZONE_APPEND:
2152	rwbs[i++] = 'Z';
2153	rwbs[i++] = 'A';
2154	break;
2155	case REQ_OP_ZONE_RESET:
2156	case REQ_OP_ZONE_RESET_ALL:
2157	rwbs[i++] = 'Z';
2158	rwbs[i++] = 'R';
2159	if ((opf & REQ_OP_MASK) == REQ_OP_ZONE_RESET_ALL)
2160	rwbs[i++] = 'A';
2161	break;
2162	case REQ_OP_ZONE_FINISH:
2163	rwbs[i++] = 'Z';
2164	rwbs[i++] = 'F';
2165	break;
2166	case REQ_OP_ZONE_OPEN:
2167	rwbs[i++] = 'Z';
2168	rwbs[i++] = 'O';
2169	break;
2170	case REQ_OP_ZONE_CLOSE:
2171	rwbs[i++] = 'Z';
2172	rwbs[i++] = 'C';
2173	break;
2174	case REQ_OP_WRITE_ZEROES:
2175	rwbs[i++] = 'W';
2176	rwbs[i++] = 'Z';
2177	break;
2178	default:
2179	rwbs[i++] = 'N';
2180	}
2181
2182	if (opf & REQ_FUA)
2183	rwbs[i++] = 'F';
2184	if (opf & REQ_RAHEAD)
2185	rwbs[i++] = 'A';
2186	if (opf & REQ_SYNC)
2187	rwbs[i++] = 'S';
2188	if (opf & REQ_META)
2189	rwbs[i++] = 'M';
2190	if (opf & REQ_ATOMIC)
2191	rwbs[i++] = 'U';
2192
2193	WARN_ON_ONCE(i >= RWBS_LEN);
2194
2195	rwbs[i] = '\0';
2196	}
2197	EXPORT_SYMBOL_GPL(blk_fill_rwbs);
2198
2199	#endif /* CONFIG_EVENT_TRACING */
2200
2201