1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/kernel/printk.c
4	*
5	* Copyright (C) 1991, 1992 Linus Torvalds
6	*
7	* Modified to make sys_syslog() more flexible: added commands to
8	* return the last 4k of kernel messages, regardless of whether
9	* they've been read or not. Added option to suppress kernel printk's
10	* to the console. Added hook for sending the console messages
11	* elsewhere, in preparation for a serial line console (someday).
12	* Ted Ts'o, 2/11/93.
13	* Modified for sysctl support, 1/8/97, Chris Horn.
14	* Fixed SMP synchronization, 08/08/99, Manfred Spraul
15	* manfred@colorfullife.com
16	* Rewrote bits to get rid of console_lock
17	* 01Mar01 Andrew Morton
18	*/
19
20	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
22	#include <linux/kernel.h>
23	#include <linux/mm.h>
24	#include <linux/tty.h>
25	#include <linux/tty_driver.h>
26	#include <linux/console.h>
27	#include <linux/init.h>
28	#include <linux/jiffies.h>
29	#include <linux/nmi.h>
30	#include <linux/module.h>
31	#include <linux/moduleparam.h>
32	#include <linux/delay.h>
33	#include <linux/smp.h>
34	#include <linux/security.h>
35	#include <linux/memblock.h>
36	#include <linux/syscalls.h>
37	#include <linux/syscore_ops.h>
38	#include <linux/vmcore_info.h>
39	#include <linux/ratelimit.h>
40	#include <linux/kmsg_dump.h>
41	#include <linux/syslog.h>
42	#include <linux/cpu.h>
43	#include <linux/rculist.h>
44	#include <linux/poll.h>
45	#include <linux/irq_work.h>
46	#include <linux/ctype.h>
47	#include <linux/uio.h>
48	#include <linux/sched/clock.h>
49	#include <linux/sched/debug.h>
50	#include <linux/sched/task_stack.h>
51
52	#include <linux/uaccess.h>
53	#include <asm/sections.h>
54
55	#include <trace/events/initcall.h>
56	#define CREATE_TRACE_POINTS
57	#include <trace/events/printk.h>
58
59	#include "printk_ringbuffer.h"
60	#include "console_cmdline.h"
61	#include "braille.h"
62	#include "internal.h"
63
64	int console_printk[4] = {
65	CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */
66	MESSAGE_LOGLEVEL_DEFAULT, /* default_message_loglevel */
67	CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */
68	CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */
69	};
70	EXPORT_SYMBOL_GPL(console_printk);
71
72	atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0);
73	EXPORT_SYMBOL(ignore_console_lock_warning);
74
75	EXPORT_TRACEPOINT_SYMBOL_GPL(console);
76
77	/*
78	* Low level drivers may need that to know if they can schedule in
79	* their unblank() callback or not. So let's export it.
80	*/
81	int oops_in_progress;
82	EXPORT_SYMBOL(oops_in_progress);
83
84	/*
85	* console_mutex protects console_list updates and console->flags updates.
86	* The flags are synchronized only for consoles that are registered, i.e.
87	* accessible via the console list.
88	*/
89	static DEFINE_MUTEX(console_mutex);
90
91	/*
92	* console_sem protects updates to console->seq
93	* and also provides serialization for console printing.
94	*/
95	static DEFINE_SEMAPHORE(console_sem, 1);
96	HLIST_HEAD(console_list);
97	EXPORT_SYMBOL_GPL(console_list);
98	DEFINE_STATIC_SRCU(console_srcu);
99
100	/*
101	* System may need to suppress printk message under certain
102	* circumstances, like after kernel panic happens.
103	*/
104	int __read_mostly suppress_printk;
105
106	#ifdef CONFIG_LOCKDEP
107	static struct lockdep_map console_lock_dep_map = {
108	.name = "console_lock"
109	};
110
111	void lockdep_assert_console_list_lock_held(void)
112	{
113	lockdep_assert_held(&console_mutex);
114	}
115	EXPORT_SYMBOL(lockdep_assert_console_list_lock_held);
116	#endif
117
118	#ifdef CONFIG_DEBUG_LOCK_ALLOC
119	bool console_srcu_read_lock_is_held(void)
120	{
121	return srcu_read_lock_held(ssp: &console_srcu);
122	}
123	EXPORT_SYMBOL(console_srcu_read_lock_is_held);
124	#endif
125
126	enum devkmsg_log_bits {
127	__DEVKMSG_LOG_BIT_ON = 0,
128	__DEVKMSG_LOG_BIT_OFF,
129	__DEVKMSG_LOG_BIT_LOCK,
130	};
131
132	enum devkmsg_log_masks {
133	DEVKMSG_LOG_MASK_ON = BIT(__DEVKMSG_LOG_BIT_ON),
134	DEVKMSG_LOG_MASK_OFF = BIT(__DEVKMSG_LOG_BIT_OFF),
135	DEVKMSG_LOG_MASK_LOCK = BIT(__DEVKMSG_LOG_BIT_LOCK),
136	};
137
138	/* Keep both the 'on' and 'off' bits clear, i.e. ratelimit by default: */
139	#define DEVKMSG_LOG_MASK_DEFAULT 0
140
141	static unsigned int __read_mostly devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
142
143	static int __control_devkmsg(char *str)
144	{
145	size_t len;
146
147	if (!str)
148	return -EINVAL;
149
150	len = str_has_prefix(str, prefix: "on");
151	if (len) {
152	devkmsg_log = DEVKMSG_LOG_MASK_ON;
153	return len;
154	}
155
156	len = str_has_prefix(str, prefix: "off");
157	if (len) {
158	devkmsg_log = DEVKMSG_LOG_MASK_OFF;
159	return len;
160	}
161
162	len = str_has_prefix(str, prefix: "ratelimit");
163	if (len) {
164	devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
165	return len;
166	}
167
168	return -EINVAL;
169	}
170
171	static int __init control_devkmsg(char *str)
172	{
173	if (__control_devkmsg(str) < 0) {
174	pr_warn("printk.devkmsg: bad option string '%s'\n", str);
175	return 1;
176	}
177
178	/*
179	* Set sysctl string accordingly:
180	*/
181	if (devkmsg_log == DEVKMSG_LOG_MASK_ON)
182	strscpy(devkmsg_log_str, "on");
183	else if (devkmsg_log == DEVKMSG_LOG_MASK_OFF)
184	strscpy(devkmsg_log_str, "off");
185	/* else "ratelimit" which is set by default. */
186
187	/*
188	* Sysctl cannot change it anymore. The kernel command line setting of
189	* this parameter is to force the setting to be permanent throughout the
190	* runtime of the system. This is a precation measure against userspace
191	* trying to be a smarta** and attempting to change it up on us.
192	*/
193	devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;
194
195	return 1;
196	}
197	__setup("printk.devkmsg=", control_devkmsg);
198
199	char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
200	#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
201	int devkmsg_sysctl_set_loglvl(const struct ctl_table *table, int write,
202	void *buffer, size_t *lenp, loff_t *ppos)
203	{
204	char old_str[DEVKMSG_STR_MAX_SIZE];
205	unsigned int old;
206	int err;
207
208	if (write) {
209	if (devkmsg_log & DEVKMSG_LOG_MASK_LOCK)
210	return -EINVAL;
211
212	old = devkmsg_log;
213	strscpy(old_str, devkmsg_log_str);
214	}
215
216	err = proc_dostring(table, write, buffer, lenp, ppos);
217	if (err)
218	return err;
219
220	if (write) {
221	err = __control_devkmsg(str: devkmsg_log_str);
222
223	/*
224	* Do not accept an unknown string OR a known string with
225	* trailing crap...
226	*/
227	if (err < 0 || (err + 1 != *lenp)) {
228
229	/* ... and restore old setting. */
230	devkmsg_log = old;
231	strscpy(devkmsg_log_str, old_str);
232
233	return -EINVAL;
234	}
235	}
236
237	return 0;
238	}
239	#endif /* CONFIG_PRINTK && CONFIG_SYSCTL */
240
241	/**
242	* console_list_lock - Lock the console list
243	*
244	* For console list or console->flags updates
245	*/
246	void console_list_lock(void)
247	{
248	/*
249	* In unregister_console() and console_force_preferred_locked(),
250	* synchronize_srcu() is called with the console_list_lock held.
251	* Therefore it is not allowed that the console_list_lock is taken
252	* with the srcu_lock held.
253	*
254	* Detecting if this context is really in the read-side critical
255	* section is only possible if the appropriate debug options are
256	* enabled.
257	*/
258	WARN_ON_ONCE(debug_lockdep_rcu_enabled() &&
259	srcu_read_lock_held(&console_srcu));
260
261	mutex_lock(&console_mutex);
262	}
263	EXPORT_SYMBOL(console_list_lock);
264
265	/**
266	* console_list_unlock - Unlock the console list
267	*
268	* Counterpart to console_list_lock()
269	*/
270	void console_list_unlock(void)
271	{
272	mutex_unlock(lock: &console_mutex);
273	}
274	EXPORT_SYMBOL(console_list_unlock);
275
276	/**
277	* console_srcu_read_lock - Register a new reader for the
278	* SRCU-protected console list
279	*
280	* Use for_each_console_srcu() to iterate the console list
281	*
282	* Context: Any context.
283	* Return: A cookie to pass to console_srcu_read_unlock().
284	*/
285	int console_srcu_read_lock(void)
286	__acquires(&console_srcu)
287	{
288	return srcu_read_lock_nmisafe(ssp: &console_srcu);
289	}
290	EXPORT_SYMBOL(console_srcu_read_lock);
291
292	/**
293	* console_srcu_read_unlock - Unregister an old reader from
294	* the SRCU-protected console list
295	* @cookie: cookie returned from console_srcu_read_lock()
296	*
297	* Counterpart to console_srcu_read_lock()
298	*/
299	void console_srcu_read_unlock(int cookie)
300	__releases(&console_srcu)
301	{
302	srcu_read_unlock_nmisafe(ssp: &console_srcu, idx: cookie);
303	}
304	EXPORT_SYMBOL(console_srcu_read_unlock);
305
306	/*
307	* Helper macros to handle lockdep when locking/unlocking console_sem. We use
308	* macros instead of functions so that _RET_IP_ contains useful information.
309	*/
310	#define down_console_sem() do { \
311	down(&console_sem);\
312	mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);\
313	} while (0)
314
315	static int __down_trylock_console_sem(unsigned long ip)
316	{
317	int lock_failed;
318	unsigned long flags;
319
320	/*
321	* Here and in __up_console_sem() we need to be in safe mode,
322	* because spindump/WARN/etc from under console ->lock will
323	* deadlock in printk()->down_trylock_console_sem() otherwise.
324	*/
325	printk_safe_enter_irqsave(flags);
326	lock_failed = down_trylock(sem: &console_sem);
327	printk_safe_exit_irqrestore(flags);
328
329	if (lock_failed)
330	return 1;
331	mutex_acquire(&console_lock_dep_map, 0, 1, ip);
332	return 0;
333	}
334	#define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)
335
336	static void __up_console_sem(unsigned long ip)
337	{
338	unsigned long flags;
339
340	mutex_release(&console_lock_dep_map, ip);
341
342	printk_safe_enter_irqsave(flags);
343	up(sem: &console_sem);
344	printk_safe_exit_irqrestore(flags);
345	}
346	#define up_console_sem() __up_console_sem(_RET_IP_)
347
348	static bool panic_in_progress(void)
349	{
350	return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
351	}
352
353	/* Return true if a panic is in progress on the current CPU. */
354	bool this_cpu_in_panic(void)
355	{
356	/*
357	* We can use raw_smp_processor_id() here because it is impossible for
358	* the task to be migrated to the panic_cpu, or away from it. If
359	* panic_cpu has already been set, and we're not currently executing on
360	* that CPU, then we never will be.
361	*/
362	return unlikely(atomic_read(&panic_cpu) == raw_smp_processor_id());
363	}
364
365	/*
366	* Return true if a panic is in progress on a remote CPU.
367	*
368	* On true, the local CPU should immediately release any printing resources
369	* that may be needed by the panic CPU.
370	*/
371	bool other_cpu_in_panic(void)
372	{
373	return (panic_in_progress() && !this_cpu_in_panic());
374	}
375
376	/*
377	* This is used for debugging the mess that is the VT code by
378	* keeping track if we have the console semaphore held. It's
379	* definitely not the perfect debug tool (we don't know if _WE_
380	* hold it and are racing, but it helps tracking those weird code
381	* paths in the console code where we end up in places I want
382	* locked without the console semaphore held).
383	*/
384	static int console_locked;
385
386	/*
387	* Array of consoles built from command line options (console=)
388	*/
389
390	#define MAX_CMDLINECONSOLES 8
391
392	static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
393
394	static int preferred_console = -1;
395	int console_set_on_cmdline;
396	EXPORT_SYMBOL(console_set_on_cmdline);
397
398	/* Flag: console code may call schedule() */
399	static int console_may_schedule;
400
401	enum con_msg_format_flags {
402	MSG_FORMAT_DEFAULT = 0,
403	MSG_FORMAT_SYSLOG = (1 << 0),
404	};
405
406	static int console_msg_format = MSG_FORMAT_DEFAULT;
407
408	/*
409	* The printk log buffer consists of a sequenced collection of records, each
410	* containing variable length message text. Every record also contains its
411	* own meta-data (@info).
412	*
413	* Every record meta-data carries the timestamp in microseconds, as well as
414	* the standard userspace syslog level and syslog facility. The usual kernel
415	* messages use LOG_KERN; userspace-injected messages always carry a matching
416	* syslog facility, by default LOG_USER. The origin of every message can be
417	* reliably determined that way.
418	*
419	* The human readable log message of a record is available in @text, the
420	* length of the message text in @text_len. The stored message is not
421	* terminated.
422	*
423	* Optionally, a record can carry a dictionary of properties (key/value
424	* pairs), to provide userspace with a machine-readable message context.
425	*
426	* Examples for well-defined, commonly used property names are:
427	* DEVICE=b12:8 device identifier
428	* b12:8 block dev_t
429	* c127:3 char dev_t
430	* n8 netdev ifindex
431	* +sound:card0 subsystem:devname
432	* SUBSYSTEM=pci driver-core subsystem name
433	*
434	* Valid characters in property names are [a-zA-Z0-9.-_]. Property names
435	* and values are terminated by a '\0' character.
436	*
437	* Example of record values:
438	* record.text_buf = "it's a line" (unterminated)
439	* record.info.seq = 56
440	* record.info.ts_nsec = 36863
441	* record.info.text_len = 11
442	* record.info.facility = 0 (LOG_KERN)
443	* record.info.flags = 0
444	* record.info.level = 3 (LOG_ERR)
445	* record.info.caller_id = 299 (task 299)
446	* record.info.dev_info.subsystem = "pci" (terminated)
447	* record.info.dev_info.device = "+pci:0000:00:01.0" (terminated)
448	*
449	* The 'struct printk_info' buffer must never be directly exported to
450	* userspace, it is a kernel-private implementation detail that might
451	* need to be changed in the future, when the requirements change.
452	*
453	* /dev/kmsg exports the structured data in the following line format:
454	* "<level>,<sequnum>,<timestamp>,<contflag>[,additional_values, ... ];<message text>\n"
455	*
456	* Users of the export format should ignore possible additional values
457	* separated by ',', and find the message after the ';' character.
458	*
459	* The optional key/value pairs are attached as continuation lines starting
460	* with a space character and terminated by a newline. All possible
461	* non-prinatable characters are escaped in the "\xff" notation.
462	*/
463
464	/* syslog_lock protects syslog_* variables and write access to clear_seq. */
465	static DEFINE_MUTEX(syslog_lock);
466
467	/*
468	* Specifies if a legacy console is registered. If legacy consoles are
469	* present, it is necessary to perform the console lock/unlock dance
470	* whenever console flushing should occur.
471	*/
472	bool have_legacy_console;
473
474	/*
475	* Specifies if an nbcon console is registered. If nbcon consoles are present,
476	* synchronous printing of legacy consoles will not occur during panic until
477	* the backtrace has been stored to the ringbuffer.
478	*/
479	bool have_nbcon_console;
480
481	/*
482	* Specifies if a boot console is registered. If boot consoles are present,
483	* nbcon consoles cannot print simultaneously and must be synchronized by
484	* the console lock. This is because boot consoles and nbcon consoles may
485	* have mapped the same hardware.
486	*/
487	bool have_boot_console;
488
489	/* See printk_legacy_allow_panic_sync() for details. */
490	bool legacy_allow_panic_sync;
491
492	#ifdef CONFIG_PRINTK
493	DECLARE_WAIT_QUEUE_HEAD(log_wait);
494	static DECLARE_WAIT_QUEUE_HEAD(legacy_wait);
495	/* All 3 protected by @syslog_lock. */
496	/* the next printk record to read by syslog(READ) or /proc/kmsg */
497	static u64 syslog_seq;
498	static size_t syslog_partial;
499	static bool syslog_time;
500
501	/* True when _all_ printer threads are available for printing. */
502	bool printk_kthreads_running;
503
504	struct latched_seq {
505	seqcount_latch_t latch;
506	u64 val[2];
507	};
508
509	/*
510	* The next printk record to read after the last 'clear' command. There are
511	* two copies (updated with seqcount_latch) so that reads can locklessly
512	* access a valid value. Writers are synchronized by @syslog_lock.
513	*/
514	static struct latched_seq clear_seq = {
515	.latch = SEQCNT_LATCH_ZERO(clear_seq.latch),
516	.val[0] = 0,
517	.val[1] = 0,
518	};
519
520	#define LOG_LEVEL(v) ((v) & 0x07)
521	#define LOG_FACILITY(v) ((v) >> 3 & 0xff)
522
523	/* record buffer */
524	#define LOG_ALIGN __alignof__(unsigned long)
525	#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
526	#define LOG_BUF_LEN_MAX ((u32)1 << 31)
527	static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
528	static char *log_buf = __log_buf;
529	static u32 log_buf_len = __LOG_BUF_LEN;
530
531	/*
532	* Define the average message size. This only affects the number of
533	* descriptors that will be available. Underestimating is better than
534	* overestimating (too many available descriptors is better than not enough).
535	*/
536	#define PRB_AVGBITS 5 /* 32 character average length */
537
538	#if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
539	#error CONFIG_LOG_BUF_SHIFT value too small.
540	#endif
541	_DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
542	PRB_AVGBITS, &__log_buf[0]);
543
544	static struct printk_ringbuffer printk_rb_dynamic;
545
546	struct printk_ringbuffer *prb = &printk_rb_static;
547
548	/*
549	* We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
550	* per_cpu_areas are initialised. This variable is set to true when
551	* it's safe to access per-CPU data.
552	*/
553	static bool __printk_percpu_data_ready __ro_after_init;
554
555	bool printk_percpu_data_ready(void)
556	{
557	return __printk_percpu_data_ready;
558	}
559
560	/* Must be called under syslog_lock. */
561	static void latched_seq_write(struct latched_seq *ls, u64 val)
562	{
563	write_seqcount_latch_begin(s: &ls->latch);
564	ls->val[0] = val;
565	write_seqcount_latch(s: &ls->latch);
566	ls->val[1] = val;
567	write_seqcount_latch_end(s: &ls->latch);
568	}
569
570	/* Can be called from any context. */
571	static u64 latched_seq_read_nolock(struct latched_seq *ls)
572	{
573	unsigned int seq;
574	unsigned int idx;
575	u64 val;
576
577	do {
578	seq = read_seqcount_latch(s: &ls->latch);
579	idx = seq & 0x1;
580	val = ls->val[idx];
581	} while (read_seqcount_latch_retry(s: &ls->latch, start: seq));
582
583	return val;
584	}
585
586	/* Return log buffer address */
587	char *log_buf_addr_get(void)
588	{
589	return log_buf;
590	}
591
592	/* Return log buffer size */
593	u32 log_buf_len_get(void)
594	{
595	return log_buf_len;
596	}
597
598	/*
599	* Define how much of the log buffer we could take at maximum. The value
600	* must be greater than two. Note that only half of the buffer is available
601	* when the index points to the middle.
602	*/
603	#define MAX_LOG_TAKE_PART 4
604	static const char trunc_msg[] = "<truncated>";
605
606	static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
607	{
608	/*
609	* The message should not take the whole buffer. Otherwise, it might
610	* get removed too soon.
611	*/
612	u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;
613
614	if (*text_len > max_text_len)
615	*text_len = max_text_len;
616
617	/* enable the warning message (if there is room) */
618	*trunc_msg_len = strlen(trunc_msg);
619	if (*text_len >= *trunc_msg_len)
620	*text_len -= *trunc_msg_len;
621	else
622	*trunc_msg_len = 0;
623	}
624
625	int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);
626
627	static int syslog_action_restricted(int type)
628	{
629	if (dmesg_restrict)
630	return 1;
631	/*
632	* Unless restricted, we allow "read all" and "get buffer size"
633	* for everybody.
634	*/
635	return type != SYSLOG_ACTION_READ_ALL &&
636	type != SYSLOG_ACTION_SIZE_BUFFER;
637	}
638
639	static int check_syslog_permissions(int type, int source)
640	{
641	/*
642	* If this is from /proc/kmsg and we've already opened it, then we've
643	* already done the capabilities checks at open time.
644	*/
645	if (source == SYSLOG_FROM_PROC && type != SYSLOG_ACTION_OPEN)
646	goto ok;
647
648	if (syslog_action_restricted(type)) {
649	if (capable(CAP_SYSLOG))
650	goto ok;
651	return -EPERM;
652	}
653	ok:
654	return security_syslog(type);
655	}
656
657	static void append_char(char **pp, char *e, char c)
658	{
659	if (*pp < e)
660	*(*pp)++ = c;
661	}
662
663	static ssize_t info_print_ext_header(char *buf, size_t size,
664	struct printk_info *info)
665	{
666	u64 ts_usec = info->ts_nsec;
667	char caller[20];
668	#ifdef CONFIG_PRINTK_CALLER
669	u32 id = info->caller_id;
670
671	snprintf(buf: caller, size: sizeof(caller), fmt: ",caller=%c%u",
672	id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
673	#else
674	caller[0] = '\0';
675	#endif
676
677	do_div(ts_usec, 1000);
678
679	return scnprintf(buf, size, fmt: "%u,%llu,%llu,%c%s;",
680	(info->facility << 3) | info->level, info->seq,
681	ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
682	}
683
684	static ssize_t msg_add_ext_text(char *buf, size_t size,
685	const char *text, size_t text_len,
686	unsigned char endc)
687	{
688	char *p = buf, *e = buf + size;
689	size_t i;
690
691	/* escape non-printable characters */
692	for (i = 0; i < text_len; i++) {
693	unsigned char c = text[i];
694
695	if (c < ' ' || c >= 127 || c == '\\')
696	p += scnprintf(buf: p, size: e - p, fmt: "\\x%02x", c);
697	else
698	append_char(pp: &p, e, c);
699	}
700	append_char(pp: &p, e, c: endc);
701
702	return p - buf;
703	}
704
705	static ssize_t msg_add_dict_text(char *buf, size_t size,
706	const char *key, const char *val)
707	{
708	size_t val_len = strlen(val);
709	ssize_t len;
710
711	if (!val_len)
712	return 0;
713
714	len = msg_add_ext_text(buf, size, text: "", text_len: 0, endc: ' '); /* dict prefix */
715	len += msg_add_ext_text(buf: buf + len, size: size - len, text: key, strlen(key), endc: '=');
716	len += msg_add_ext_text(buf: buf + len, size: size - len, text: val, text_len: val_len, endc: '\n');
717
718	return len;
719	}
720
721	static ssize_t msg_print_ext_body(char *buf, size_t size,
722	char *text, size_t text_len,
723	struct dev_printk_info *dev_info)
724	{
725	ssize_t len;
726
727	len = msg_add_ext_text(buf, size, text, text_len, endc: '\n');
728
729	if (!dev_info)
730	goto out;
731
732	len += msg_add_dict_text(buf: buf + len, size: size - len, key: "SUBSYSTEM",
733	val: dev_info->subsystem);
734	len += msg_add_dict_text(buf: buf + len, size: size - len, key: "DEVICE",
735	val: dev_info->device);
736	out:
737	return len;
738	}
739
740	/* /dev/kmsg - userspace message inject/listen interface */
741	struct devkmsg_user {
742	atomic64_t seq;
743	struct ratelimit_state rs;
744	struct mutex lock;
745	struct printk_buffers pbufs;
746	};
747
748	static __printf(3, 4) __cold
749	int devkmsg_emit(int facility, int level, const char *fmt, ...)
750	{
751	va_list args;
752	int r;
753
754	va_start(args, fmt);
755	r = vprintk_emit(facility, level, NULL, fmt, args);
756	va_end(args);
757
758	return r;
759	}
760
761	static ssize_t devkmsg_write(struct kiocb *iocb, struct iov_iter *from)
762	{
763	char *buf, *line;
764	int level = default_message_loglevel;
765	int facility = 1; /* LOG_USER */
766	struct file *file = iocb->ki_filp;
767	struct devkmsg_user *user = file->private_data;
768	size_t len = iov_iter_count(i: from);
769	ssize_t ret = len;
770
771	if (len > PRINTKRB_RECORD_MAX)
772	return -EINVAL;
773
774	/* Ignore when user logging is disabled. */
775	if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
776	return len;
777
778	/* Ratelimit when not explicitly enabled. */
779	if (!(devkmsg_log & DEVKMSG_LOG_MASK_ON)) {
780	if (!___ratelimit(rs: &user->rs, current->comm))
781	return ret;
782	}
783
784	buf = kmalloc(len+1, GFP_KERNEL);
785	if (buf == NULL)
786	return -ENOMEM;
787
788	buf[len] = '\0';
789	if (!copy_from_iter_full(addr: buf, bytes: len, i: from)) {
790	kfree(objp: buf);
791	return -EFAULT;
792	}
793
794	/*
795	* Extract and skip the syslog prefix <[0-9]*>. Coming from userspace
796	* the decimal value represents 32bit, the lower 3 bit are the log
797	* level, the rest are the log facility.
798	*
799	* If no prefix or no userspace facility is specified, we
800	* enforce LOG_USER, to be able to reliably distinguish
801	* kernel-generated messages from userspace-injected ones.
802	*/
803	line = buf;
804	if (line[0] == '<') {
805	char *endp = NULL;
806	unsigned int u;
807
808	u = simple_strtoul(line + 1, &endp, 10);
809	if (endp && endp[0] == '>') {
810	level = LOG_LEVEL(u);
811	if (LOG_FACILITY(u) != 0)
812	facility = LOG_FACILITY(u);
813	endp++;
814	line = endp;
815	}
816	}
817
818	devkmsg_emit(facility, level, fmt: "%s", line);
819	kfree(objp: buf);
820	return ret;
821	}
822
823	static ssize_t devkmsg_read(struct file *file, char __user *buf,
824	size_t count, loff_t *ppos)
825	{
826	struct devkmsg_user *user = file->private_data;
827	char *outbuf = &user->pbufs.outbuf[0];
828	struct printk_message pmsg = {
829	.pbufs = &user->pbufs,
830	};
831	ssize_t ret;
832
833	ret = mutex_lock_interruptible(&user->lock);
834	if (ret)
835	return ret;
836
837	if (!printk_get_next_message(pmsg: &pmsg, seq: atomic64_read(v: &user->seq), is_extended: true, may_supress: false)) {
838	if (file->f_flags & O_NONBLOCK) {
839	ret = -EAGAIN;
840	goto out;
841	}
842
843	/*
844	* Guarantee this task is visible on the waitqueue before
845	* checking the wake condition.
846	*
847	* The full memory barrier within set_current_state() of
848	* prepare_to_wait_event() pairs with the full memory barrier
849	* within wq_has_sleeper().
850	*
851	* This pairs with __wake_up_klogd:A.
852	*/
853	ret = wait_event_interruptible(log_wait,
854	printk_get_next_message(&pmsg, atomic64_read(&user->seq), true,
855	false)); /* LMM(devkmsg_read:A) */
856	if (ret)
857	goto out;
858	}
859
860	if (pmsg.dropped) {
861	/* our last seen message is gone, return error and reset */
862	atomic64_set(v: &user->seq, i: pmsg.seq);
863	ret = -EPIPE;
864	goto out;
865	}
866
867	atomic64_set(v: &user->seq, i: pmsg.seq + 1);
868
869	if (pmsg.outbuf_len > count) {
870	ret = -EINVAL;
871	goto out;
872	}
873
874	if (copy_to_user(to: buf, from: outbuf, n: pmsg.outbuf_len)) {
875	ret = -EFAULT;
876	goto out;
877	}
878	ret = pmsg.outbuf_len;
879	out:
880	mutex_unlock(lock: &user->lock);
881	return ret;
882	}
883
884	/*
885	* Be careful when modifying this function!!!
886	*
887	* Only few operations are supported because the device works only with the
888	* entire variable length messages (records). Non-standard values are
889	* returned in the other cases and has been this way for quite some time.
890	* User space applications might depend on this behavior.
891	*/
892	static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
893	{
894	struct devkmsg_user *user = file->private_data;
895	loff_t ret = 0;
896
897	if (offset)
898	return -ESPIPE;
899
900	switch (whence) {
901	case SEEK_SET:
902	/* the first record */
903	atomic64_set(v: &user->seq, i: prb_first_valid_seq(rb: prb));
904	break;
905	case SEEK_DATA:
906	/*
907	* The first record after the last SYSLOG_ACTION_CLEAR,
908	* like issued by 'dmesg -c'. Reading /dev/kmsg itself
909	* changes no global state, and does not clear anything.
910	*/
911	atomic64_set(v: &user->seq, i: latched_seq_read_nolock(ls: &clear_seq));
912	break;
913	case SEEK_END:
914	/* after the last record */
915	atomic64_set(v: &user->seq, i: prb_next_seq(rb: prb));
916	break;
917	default:
918	ret = -EINVAL;
919	}
920	return ret;
921	}
922
923	static __poll_t devkmsg_poll(struct file *file, poll_table *wait)
924	{
925	struct devkmsg_user *user = file->private_data;
926	struct printk_info info;
927	__poll_t ret = 0;
928
929	poll_wait(filp: file, wait_address: &log_wait, p: wait);
930
931	if (prb_read_valid_info(rb: prb, seq: atomic64_read(v: &user->seq), info: &info, NULL)) {
932	/* return error when data has vanished underneath us */
933	if (info.seq != atomic64_read(v: &user->seq))
934	ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
935	else
936	ret = EPOLLIN|EPOLLRDNORM;
937	}
938
939	return ret;
940	}
941
942	static int devkmsg_open(struct inode *inode, struct file *file)
943	{
944	struct devkmsg_user *user;
945	int err;
946
947	if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
948	return -EPERM;
949
950	/* write-only does not need any file context */
951	if ((file->f_flags & O_ACCMODE) != O_WRONLY) {
952	err = check_syslog_permissions(SYSLOG_ACTION_READ_ALL,
953	SYSLOG_FROM_READER);
954	if (err)
955	return err;
956	}
957
958	user = kvmalloc(sizeof(struct devkmsg_user), GFP_KERNEL);
959	if (!user)
960	return -ENOMEM;
961
962	ratelimit_default_init(rs: &user->rs);
963	ratelimit_set_flags(rs: &user->rs, RATELIMIT_MSG_ON_RELEASE);
964
965	mutex_init(&user->lock);
966
967	atomic64_set(v: &user->seq, i: prb_first_valid_seq(rb: prb));
968
969	file->private_data = user;
970	return 0;
971	}
972
973	static int devkmsg_release(struct inode *inode, struct file *file)
974	{
975	struct devkmsg_user *user = file->private_data;
976
977	ratelimit_state_exit(rs: &user->rs);
978
979	mutex_destroy(lock: &user->lock);
980	kvfree(addr: user);
981	return 0;
982	}
983
984	const struct file_operations kmsg_fops = {
985	.open = devkmsg_open,
986	.read = devkmsg_read,
987	.write_iter = devkmsg_write,
988	.llseek = devkmsg_llseek,
989	.poll = devkmsg_poll,
990	.release = devkmsg_release,
991	};
992
993	#ifdef CONFIG_VMCORE_INFO
994	/*
995	* This appends the listed symbols to /proc/vmcore
996	*
997	* /proc/vmcore is used by various utilities, like crash and makedumpfile to
998	* obtain access to symbols that are otherwise very difficult to locate. These
999	* symbols are specifically used so that utilities can access and extract the
1000	* dmesg log from a vmcore file after a crash.
1001	*/
1002	void log_buf_vmcoreinfo_setup(void)
1003	{
1004	struct dev_printk_info *dev_info = NULL;
1005
1006	VMCOREINFO_SYMBOL(prb);
1007	VMCOREINFO_SYMBOL(printk_rb_static);
1008	VMCOREINFO_SYMBOL(clear_seq);
1009
1010	/*
1011	* Export struct size and field offsets. User space tools can
1012	* parse it and detect any changes to structure down the line.
1013	*/
1014
1015	VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
1016	VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
1017	VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
1018	VMCOREINFO_OFFSET(printk_ringbuffer, fail);
1019
1020	VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
1021	VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
1022	VMCOREINFO_OFFSET(prb_desc_ring, descs);
1023	VMCOREINFO_OFFSET(prb_desc_ring, infos);
1024	VMCOREINFO_OFFSET(prb_desc_ring, head_id);
1025	VMCOREINFO_OFFSET(prb_desc_ring, tail_id);
1026
1027	VMCOREINFO_STRUCT_SIZE(prb_desc);
1028	VMCOREINFO_OFFSET(prb_desc, state_var);
1029	VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);
1030
1031	VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
1032	VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
1033	VMCOREINFO_OFFSET(prb_data_blk_lpos, next);
1034
1035	VMCOREINFO_STRUCT_SIZE(printk_info);
1036	VMCOREINFO_OFFSET(printk_info, seq);
1037	VMCOREINFO_OFFSET(printk_info, ts_nsec);
1038	VMCOREINFO_OFFSET(printk_info, text_len);
1039	VMCOREINFO_OFFSET(printk_info, caller_id);
1040	VMCOREINFO_OFFSET(printk_info, dev_info);
1041
1042	VMCOREINFO_STRUCT_SIZE(dev_printk_info);
1043	VMCOREINFO_OFFSET(dev_printk_info, subsystem);
1044	VMCOREINFO_LENGTH(printk_info_subsystem, sizeof(dev_info->subsystem));
1045	VMCOREINFO_OFFSET(dev_printk_info, device);
1046	VMCOREINFO_LENGTH(printk_info_device, sizeof(dev_info->device));
1047
1048	VMCOREINFO_STRUCT_SIZE(prb_data_ring);
1049	VMCOREINFO_OFFSET(prb_data_ring, size_bits);
1050	VMCOREINFO_OFFSET(prb_data_ring, data);
1051	VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
1052	VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);
1053
1054	VMCOREINFO_SIZE(atomic_long_t);
1055	VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);
1056
1057	VMCOREINFO_STRUCT_SIZE(latched_seq);
1058	VMCOREINFO_OFFSET(latched_seq, val);
1059	}
1060	#endif
1061
1062	/* requested log_buf_len from kernel cmdline */
1063	static unsigned long __initdata new_log_buf_len;
1064
1065	/* we practice scaling the ring buffer by powers of 2 */
1066	static void __init log_buf_len_update(u64 size)
1067	{
1068	if (size > (u64)LOG_BUF_LEN_MAX) {
1069	size = (u64)LOG_BUF_LEN_MAX;
1070	pr_err("log_buf over 2G is not supported.\n");
1071	}
1072
1073	if (size)
1074	size = roundup_pow_of_two(size);
1075	if (size > log_buf_len)
1076	new_log_buf_len = (unsigned long)size;
1077	}
1078
1079	/* save requested log_buf_len since it's too early to process it */
1080	static int __init log_buf_len_setup(char *str)
1081	{
1082	u64 size;
1083
1084	if (!str)
1085	return -EINVAL;
1086
1087	size = memparse(ptr: str, retptr: &str);
1088
1089	log_buf_len_update(size);
1090
1091	return 0;
1092	}
1093	early_param("log_buf_len", log_buf_len_setup);
1094
1095	#ifdef CONFIG_SMP
1096	#define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT)
1097
1098	static void __init log_buf_add_cpu(void)
1099	{
1100	unsigned int cpu_extra;
1101
1102	/*
1103	* archs should set up cpu_possible_bits properly with
1104	* set_cpu_possible() after setup_arch() but just in
1105	* case lets ensure this is valid.
1106	*/
1107	if (num_possible_cpus() == 1)
1108	return;
1109
1110	cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;
1111
1112	/* by default this will only continue through for large > 64 CPUs */
1113	if (cpu_extra <= __LOG_BUF_LEN / 2)
1114	return;
1115
1116	pr_info("log_buf_len individual max cpu contribution: %d bytes\n",
1117	__LOG_CPU_MAX_BUF_LEN);
1118	pr_info("log_buf_len total cpu_extra contributions: %d bytes\n",
1119	cpu_extra);
1120	pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN);
1121
1122	log_buf_len_update(size: cpu_extra + __LOG_BUF_LEN);
1123	}
1124	#else /* !CONFIG_SMP */
1125	static inline void log_buf_add_cpu(void) {}
1126	#endif /* CONFIG_SMP */
1127
1128	static void __init set_percpu_data_ready(void)
1129	{
1130	__printk_percpu_data_ready = true;
1131	}
1132
1133	static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
1134	struct printk_record *r)
1135	{
1136	struct prb_reserved_entry e;
1137	struct printk_record dest_r;
1138
1139	prb_rec_init_wr(r: &dest_r, text_buf_size: r->info->text_len);
1140
1141	if (!prb_reserve(e: &e, rb, r: &dest_r))
1142	return 0;
1143
1144	memcpy(&dest_r.text_buf[0], &r->text_buf[0], r->info->text_len);
1145	dest_r.info->text_len = r->info->text_len;
1146	dest_r.info->facility = r->info->facility;
1147	dest_r.info->level = r->info->level;
1148	dest_r.info->flags = r->info->flags;
1149	dest_r.info->ts_nsec = r->info->ts_nsec;
1150	dest_r.info->caller_id = r->info->caller_id;
1151	memcpy(&dest_r.info->dev_info, &r->info->dev_info, sizeof(dest_r.info->dev_info));
1152
1153	prb_final_commit(e: &e);
1154
1155	return prb_record_text_space(e: &e);
1156	}
1157
1158	static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata;
1159
1160	static void print_log_buf_usage_stats(void)
1161	{
1162	unsigned int descs_count = log_buf_len >> PRB_AVGBITS;
1163	size_t meta_data_size;
1164
1165	meta_data_size = descs_count * (sizeof(struct prb_desc) + sizeof(struct printk_info));
1166
1167	pr_info("log buffer data + meta data: %u + %zu = %zu bytes\n",
1168	log_buf_len, meta_data_size, log_buf_len + meta_data_size);
1169	}
1170
1171	void __init setup_log_buf(int early)
1172	{
1173	struct printk_info *new_infos;
1174	unsigned int new_descs_count;
1175	struct prb_desc *new_descs;
1176	struct printk_info info;
1177	struct printk_record r;
1178	unsigned int text_size;
1179	size_t new_descs_size;
1180	size_t new_infos_size;
1181	unsigned long flags;
1182	char *new_log_buf;
1183	unsigned int free;
1184	u64 seq;
1185
1186	/*
1187	* Some archs call setup_log_buf() multiple times - first is very
1188	* early, e.g. from setup_arch(), and second - when percpu_areas
1189	* are initialised.
1190	*/
1191	if (!early)
1192	set_percpu_data_ready();
1193
1194	if (log_buf != __log_buf)
1195	return;
1196
1197	if (!early && !new_log_buf_len)
1198	log_buf_add_cpu();
1199
1200	if (!new_log_buf_len) {
1201	/* Show the memory stats only once. */
1202	if (!early)
1203	goto out;
1204
1205	return;
1206	}
1207
1208	new_descs_count = new_log_buf_len >> PRB_AVGBITS;
1209	if (new_descs_count == 0) {
1210	pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
1211	goto out;
1212	}
1213
1214	new_log_buf = memblock_alloc(size: new_log_buf_len, LOG_ALIGN);
1215	if (unlikely(!new_log_buf)) {
1216	pr_err("log_buf_len: %lu text bytes not available\n",
1217	new_log_buf_len);
1218	goto out;
1219	}
1220
1221	new_descs_size = new_descs_count * sizeof(struct prb_desc);
1222	new_descs = memblock_alloc(size: new_descs_size, LOG_ALIGN);
1223	if (unlikely(!new_descs)) {
1224	pr_err("log_buf_len: %zu desc bytes not available\n",
1225	new_descs_size);
1226	goto err_free_log_buf;
1227	}
1228
1229	new_infos_size = new_descs_count * sizeof(struct printk_info);
1230	new_infos = memblock_alloc(size: new_infos_size, LOG_ALIGN);
1231	if (unlikely(!new_infos)) {
1232	pr_err("log_buf_len: %zu info bytes not available\n",
1233	new_infos_size);
1234	goto err_free_descs;
1235	}
1236
1237	prb_rec_init_rd(r: &r, info: &info, text_buf: &setup_text_buf[0], text_buf_size: sizeof(setup_text_buf));
1238
1239	prb_init(rb: &printk_rb_dynamic,
1240	text_buf: new_log_buf, ilog2(new_log_buf_len),
1241	descs: new_descs, ilog2(new_descs_count),
1242	infos: new_infos);
1243
1244	local_irq_save(flags);
1245
1246	log_buf_len = new_log_buf_len;
1247	log_buf = new_log_buf;
1248	new_log_buf_len = 0;
1249
1250	free = __LOG_BUF_LEN;
1251	prb_for_each_record(0, &printk_rb_static, seq, &r) {
1252	text_size = add_to_rb(rb: &printk_rb_dynamic, r: &r);
1253	if (text_size > free)
1254	free = 0;
1255	else
1256	free -= text_size;
1257	}
1258
1259	prb = &printk_rb_dynamic;
1260
1261	local_irq_restore(flags);
1262
1263	/*
1264	* Copy any remaining messages that might have appeared from
1265	* NMI context after copying but before switching to the
1266	* dynamic buffer.
1267	*/
1268	prb_for_each_record(seq, &printk_rb_static, seq, &r) {
1269	text_size = add_to_rb(rb: &printk_rb_dynamic, r: &r);
1270	if (text_size > free)
1271	free = 0;
1272	else
1273	free -= text_size;
1274	}
1275
1276	if (seq != prb_next_seq(rb: &printk_rb_static)) {
1277	pr_err("dropped %llu messages\n",
1278	prb_next_seq(&printk_rb_static) - seq);
1279	}
1280
1281	print_log_buf_usage_stats();
1282	pr_info("early log buf free: %u(%u%%)\n",
1283	free, (free * 100) / __LOG_BUF_LEN);
1284	return;
1285
1286	err_free_descs:
1287	memblock_free(ptr: new_descs, size: new_descs_size);
1288	err_free_log_buf:
1289	memblock_free(ptr: new_log_buf, size: new_log_buf_len);
1290	out:
1291	print_log_buf_usage_stats();
1292	}
1293
1294	static bool __read_mostly ignore_loglevel;
1295
1296	static int __init ignore_loglevel_setup(char *str)
1297	{
1298	ignore_loglevel = true;
1299	pr_info("debug: ignoring loglevel setting.\n");
1300
1301	return 0;
1302	}
1303
1304	early_param("ignore_loglevel", ignore_loglevel_setup);
1305	module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
1306	MODULE_PARM_DESC(ignore_loglevel,
1307	"ignore loglevel setting (prints all kernel messages to the console)");
1308
1309	static bool suppress_message_printing(int level)
1310	{
1311	return (level >= console_loglevel && !ignore_loglevel);
1312	}
1313
1314	#ifdef CONFIG_BOOT_PRINTK_DELAY
1315
1316	static int boot_delay; /* msecs delay after each printk during bootup */
1317	static unsigned long long loops_per_msec; /* based on boot_delay */
1318
1319	static int __init boot_delay_setup(char *str)
1320	{
1321	unsigned long lpj;
1322
1323	lpj = preset_lpj ? preset_lpj : 1000000; /* some guess */
1324	loops_per_msec = (unsigned long long)lpj / 1000 * HZ;
1325
1326	get_option(str: &str, pint: &boot_delay);
1327	if (boot_delay > 10 * 1000)
1328	boot_delay = 0;
1329
1330	pr_debug("boot_delay: %u, preset_lpj: %ld, lpj: %lu, "
1331	"HZ: %d, loops_per_msec: %llu\n",
1332	boot_delay, preset_lpj, lpj, HZ, loops_per_msec);
1333	return 0;
1334	}
1335	early_param("boot_delay", boot_delay_setup);
1336
1337	static void boot_delay_msec(int level)
1338	{
1339	unsigned long long k;
1340	unsigned long timeout;
1341	bool suppress = !is_printk_force_console() &&
1342	suppress_message_printing(level);
1343
1344	if ((boot_delay == 0 || system_state >= SYSTEM_RUNNING) || suppress)
1345	return;
1346
1347	k = (unsigned long long)loops_per_msec * boot_delay;
1348
1349	timeout = jiffies + msecs_to_jiffies(m: boot_delay);
1350	while (k) {
1351	k--;
1352	cpu_relax();
1353	/*
1354	* use (volatile) jiffies to prevent
1355	* compiler reduction; loop termination via jiffies
1356	* is secondary and may or may not happen.
1357	*/
1358	if (time_after(jiffies, timeout))
1359	break;
1360	touch_nmi_watchdog();
1361	}
1362	}
1363	#else
1364	static inline void boot_delay_msec(int level)
1365	{
1366	}
1367	#endif
1368
1369	static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME);
1370	module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR);
1371
1372	static size_t print_syslog(unsigned int level, char *buf)
1373	{
1374	return sprintf(buf, fmt: "<%u>", level);
1375	}
1376
1377	static size_t print_time(u64 ts, char *buf)
1378	{
1379	unsigned long rem_nsec = do_div(ts, 1000000000);
1380
1381	return sprintf(buf, fmt: "[%5lu.%06lu]",
1382	(unsigned long)ts, rem_nsec / 1000);
1383	}
1384
1385	#ifdef CONFIG_PRINTK_CALLER
1386	static size_t print_caller(u32 id, char *buf)
1387	{
1388	char caller[12];
1389
1390	snprintf(buf: caller, size: sizeof(caller), fmt: "%c%u",
1391	id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
1392	return sprintf(buf, fmt: "[%6s]", caller);
1393	}
1394	#else
1395	#define print_caller(id, buf) 0
1396	#endif
1397
1398	static size_t info_print_prefix(const struct printk_info *info, bool syslog,
1399	bool time, char *buf)
1400	{
1401	size_t len = 0;
1402
1403	if (syslog)
1404	len = print_syslog(level: (info->facility << 3) | info->level, buf);
1405
1406	if (time)
1407	len += print_time(ts: info->ts_nsec, buf: buf + len);
1408
1409	len += print_caller(id: info->caller_id, buf: buf + len);
1410
1411	if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
1412	buf[len++] = ' ';
1413	buf[len] = '\0';
1414	}
1415
1416	return len;
1417	}
1418
1419	/*
1420	* Prepare the record for printing. The text is shifted within the given
1421	* buffer to avoid a need for another one. The following operations are
1422	* done:
1423	*
1424	* - Add prefix for each line.
1425	* - Drop truncated lines that no longer fit into the buffer.
1426	* - Add the trailing newline that has been removed in vprintk_store().
1427	* - Add a string terminator.
1428	*
1429	* Since the produced string is always terminated, the maximum possible
1430	* return value is @r->text_buf_size - 1;
1431	*
1432	* Return: The length of the updated/prepared text, including the added
1433	* prefixes and the newline. The terminator is not counted. The dropped
1434	* line(s) are not counted.
1435	*/
1436	static size_t record_print_text(struct printk_record *r, bool syslog,
1437	bool time)
1438	{
1439	size_t text_len = r->info->text_len;
1440	size_t buf_size = r->text_buf_size;
1441	char *text = r->text_buf;
1442	char prefix[PRINTK_PREFIX_MAX];
1443	bool truncated = false;
1444	size_t prefix_len;
1445	size_t line_len;
1446	size_t len = 0;
1447	char *next;
1448
1449	/*
1450	* If the message was truncated because the buffer was not large
1451	* enough, treat the available text as if it were the full text.
1452	*/
1453	if (text_len > buf_size)
1454	text_len = buf_size;
1455
1456	prefix_len = info_print_prefix(info: r->info, syslog, time, buf: prefix);
1457
1458	/*
1459	* @text_len: bytes of unprocessed text
1460	* @line_len: bytes of current line _without_ newline
1461	* @text: pointer to beginning of current line
1462	* @len: number of bytes prepared in r->text_buf
1463	*/
1464	for (;;) {
1465	next = memchr(p: text, c: '\n', size: text_len);
1466	if (next) {
1467	line_len = next - text;
1468	} else {
1469	/* Drop truncated line(s). */
1470	if (truncated)
1471	break;
1472	line_len = text_len;
1473	}
1474
1475	/*
1476	* Truncate the text if there is not enough space to add the
1477	* prefix and a trailing newline and a terminator.
1478	*/
1479	if (len + prefix_len + text_len + 1 + 1 > buf_size) {
1480	/* Drop even the current line if no space. */
1481	if (len + prefix_len + line_len + 1 + 1 > buf_size)
1482	break;
1483
1484	text_len = buf_size - len - prefix_len - 1 - 1;
1485	truncated = true;
1486	}
1487
1488	memmove(text + prefix_len, text, text_len);
1489	memcpy(text, prefix, prefix_len);
1490
1491	/*
1492	* Increment the prepared length to include the text and
1493	* prefix that were just moved+copied. Also increment for the
1494	* newline at the end of this line. If this is the last line,
1495	* there is no newline, but it will be added immediately below.
1496	*/
1497	len += prefix_len + line_len + 1;
1498	if (text_len == line_len) {
1499	/*
1500	* This is the last line. Add the trailing newline
1501	* removed in vprintk_store().
1502	*/
1503	text[prefix_len + line_len] = '\n';
1504	break;
1505	}
1506
1507	/*
1508	* Advance beyond the added prefix and the related line with
1509	* its newline.
1510	*/
1511	text += prefix_len + line_len + 1;
1512
1513	/*
1514	* The remaining text has only decreased by the line with its
1515	* newline.
1516	*
1517	* Note that @text_len can become zero. It happens when @text
1518	* ended with a newline (either due to truncation or the
1519	* original string ending with "\n\n"). The loop is correctly
1520	* repeated and (if not truncated) an empty line with a prefix
1521	* will be prepared.
1522	*/
1523	text_len -= line_len + 1;
1524	}
1525
1526	/*
1527	* If a buffer was provided, it will be terminated. Space for the
1528	* string terminator is guaranteed to be available. The terminator is
1529	* not counted in the return value.
1530	*/
1531	if (buf_size > 0)
1532	r->text_buf[len] = 0;
1533
1534	return len;
1535	}
1536
1537	static size_t get_record_print_text_size(struct printk_info *info,
1538	unsigned int line_count,
1539	bool syslog, bool time)
1540	{
1541	char prefix[PRINTK_PREFIX_MAX];
1542	size_t prefix_len;
1543
1544	prefix_len = info_print_prefix(info, syslog, time, buf: prefix);
1545
1546	/*
1547	* Each line will be preceded with a prefix. The intermediate
1548	* newlines are already within the text, but a final trailing
1549	* newline will be added.
1550	*/
1551	return ((prefix_len * line_count) + info->text_len + 1);
1552	}
1553
1554	/*
1555	* Beginning with @start_seq, find the first record where it and all following
1556	* records up to (but not including) @max_seq fit into @size.
1557	*
1558	* @max_seq is simply an upper bound and does not need to exist. If the caller
1559	* does not require an upper bound, -1 can be used for @max_seq.
1560	*/
1561	static u64 find_first_fitting_seq(u64 start_seq, u64 max_seq, size_t size,
1562	bool syslog, bool time)
1563	{
1564	struct printk_info info;
1565	unsigned int line_count;
1566	size_t len = 0;
1567	u64 seq;
1568
1569	/* Determine the size of the records up to @max_seq. */
1570	prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1571	if (info.seq >= max_seq)
1572	break;
1573	len += get_record_print_text_size(info: &info, line_count, syslog, time);
1574	}
1575
1576	/*
1577	* Adjust the upper bound for the next loop to avoid subtracting
1578	* lengths that were never added.
1579	*/
1580	if (seq < max_seq)
1581	max_seq = seq;
1582
1583	/*
1584	* Move first record forward until length fits into the buffer. Ignore
1585	* newest messages that were not counted in the above cycle. Messages
1586	* might appear and get lost in the meantime. This is a best effort
1587	* that prevents an infinite loop that could occur with a retry.
1588	*/
1589	prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
1590	if (len <= size || info.seq >= max_seq)
1591	break;
1592	len -= get_record_print_text_size(info: &info, line_count, syslog, time);
1593	}
1594
1595	return seq;
1596	}
1597
1598	/* The caller is responsible for making sure @size is greater than 0. */
1599	static int syslog_print(char __user *buf, int size)
1600	{
1601	struct printk_info info;
1602	struct printk_record r;
1603	char *text;
1604	int len = 0;
1605	u64 seq;
1606
1607	text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1608	if (!text)
1609	return -ENOMEM;
1610
1611	prb_rec_init_rd(r: &r, info: &info, text_buf: text, PRINTK_MESSAGE_MAX);
1612
1613	mutex_lock(&syslog_lock);
1614
1615	/*
1616	* Wait for the @syslog_seq record to be available. @syslog_seq may
1617	* change while waiting.
1618	*/
1619	do {
1620	seq = syslog_seq;
1621
1622	mutex_unlock(lock: &syslog_lock);
1623	/*
1624	* Guarantee this task is visible on the waitqueue before
1625	* checking the wake condition.
1626	*
1627	* The full memory barrier within set_current_state() of
1628	* prepare_to_wait_event() pairs with the full memory barrier
1629	* within wq_has_sleeper().
1630	*
1631	* This pairs with __wake_up_klogd:A.
1632	*/
1633	len = wait_event_interruptible(log_wait,
1634	prb_read_valid(prb, seq, NULL)); /* LMM(syslog_print:A) */
1635	mutex_lock(&syslog_lock);
1636
1637	if (len)
1638	goto out;
1639	} while (syslog_seq != seq);
1640
1641	/*
1642	* Copy records that fit into the buffer. The above cycle makes sure
1643	* that the first record is always available.
1644	*/
1645	do {
1646	size_t n;
1647	size_t skip;
1648	int err;
1649
1650	if (!prb_read_valid(rb: prb, seq: syslog_seq, r: &r))
1651	break;
1652
1653	if (r.info->seq != syslog_seq) {
1654	/* message is gone, move to next valid one */
1655	syslog_seq = r.info->seq;
1656	syslog_partial = 0;
1657	}
1658
1659	/*
1660	* To keep reading/counting partial line consistent,
1661	* use printk_time value as of the beginning of a line.
1662	*/
1663	if (!syslog_partial)
1664	syslog_time = printk_time;
1665
1666	skip = syslog_partial;
1667	n = record_print_text(r: &r, syslog: true, time: syslog_time);
1668	if (n - syslog_partial <= size) {
1669	/* message fits into buffer, move forward */
1670	syslog_seq = r.info->seq + 1;
1671	n -= syslog_partial;
1672	syslog_partial = 0;
1673	} else if (!len){
1674	/* partial read(), remember position */
1675	n = size;
1676	syslog_partial += n;
1677	} else
1678	n = 0;
1679
1680	if (!n)
1681	break;
1682
1683	mutex_unlock(lock: &syslog_lock);
1684	err = copy_to_user(to: buf, from: text + skip, n);
1685	mutex_lock(&syslog_lock);
1686
1687	if (err) {
1688	if (!len)
1689	len = -EFAULT;
1690	break;
1691	}
1692
1693	len += n;
1694	size -= n;
1695	buf += n;
1696	} while (size);
1697	out:
1698	mutex_unlock(lock: &syslog_lock);
1699	kfree(objp: text);
1700	return len;
1701	}
1702
1703	static int syslog_print_all(char __user *buf, int size, bool clear)
1704	{
1705	struct printk_info info;
1706	struct printk_record r;
1707	char *text;
1708	int len = 0;
1709	u64 seq;
1710	bool time;
1711
1712	text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
1713	if (!text)
1714	return -ENOMEM;
1715
1716	time = printk_time;
1717	/*
1718	* Find first record that fits, including all following records,
1719	* into the user-provided buffer for this dump.
1720	*/
1721	seq = find_first_fitting_seq(start_seq: latched_seq_read_nolock(ls: &clear_seq), max_seq: -1,
1722	size, syslog: true, time);
1723
1724	prb_rec_init_rd(r: &r, info: &info, text_buf: text, PRINTK_MESSAGE_MAX);
1725
1726	prb_for_each_record(seq, prb, seq, &r) {
1727	int textlen;
1728
1729	textlen = record_print_text(r: &r, syslog: true, time);
1730
1731	if (len + textlen > size) {
1732	seq--;
1733	break;
1734	}
1735
1736	if (copy_to_user(to: buf + len, from: text, n: textlen))
1737	len = -EFAULT;
1738	else
1739	len += textlen;
1740
1741	if (len < 0)
1742	break;
1743	}
1744
1745	if (clear) {
1746	mutex_lock(&syslog_lock);
1747	latched_seq_write(ls: &clear_seq, val: seq);
1748	mutex_unlock(lock: &syslog_lock);
1749	}
1750
1751	kfree(objp: text);
1752	return len;
1753	}
1754
1755	static void syslog_clear(void)
1756	{
1757	mutex_lock(&syslog_lock);
1758	latched_seq_write(ls: &clear_seq, val: prb_next_seq(rb: prb));
1759	mutex_unlock(lock: &syslog_lock);
1760	}
1761
1762	int do_syslog(int type, char __user *buf, int len, int source)
1763	{
1764	struct printk_info info;
1765	bool clear = false;
1766	static int saved_console_loglevel = LOGLEVEL_DEFAULT;
1767	int error;
1768
1769	error = check_syslog_permissions(type, source);
1770	if (error)
1771	return error;
1772
1773	switch (type) {
1774	case SYSLOG_ACTION_CLOSE: /* Close log */
1775	break;
1776	case SYSLOG_ACTION_OPEN: /* Open log */
1777	break;
1778	case SYSLOG_ACTION_READ: /* Read from log */
1779	if (!buf || len < 0)
1780	return -EINVAL;
1781	if (!len)
1782	return 0;
1783	if (!access_ok(buf, len))
1784	return -EFAULT;
1785	error = syslog_print(buf, size: len);
1786	break;
1787	/* Read/clear last kernel messages */
1788	case SYSLOG_ACTION_READ_CLEAR:
1789	clear = true;
1790	fallthrough;
1791	/* Read last kernel messages */
1792	case SYSLOG_ACTION_READ_ALL:
1793	if (!buf || len < 0)
1794	return -EINVAL;
1795	if (!len)
1796	return 0;
1797	if (!access_ok(buf, len))
1798	return -EFAULT;
1799	error = syslog_print_all(buf, size: len, clear);
1800	break;
1801	/* Clear ring buffer */
1802	case SYSLOG_ACTION_CLEAR:
1803	syslog_clear();
1804	break;
1805	/* Disable logging to console */
1806	case SYSLOG_ACTION_CONSOLE_OFF:
1807	if (saved_console_loglevel == LOGLEVEL_DEFAULT)
1808	saved_console_loglevel = console_loglevel;
1809	console_loglevel = minimum_console_loglevel;
1810	break;
1811	/* Enable logging to console */
1812	case SYSLOG_ACTION_CONSOLE_ON:
1813	if (saved_console_loglevel != LOGLEVEL_DEFAULT) {
1814	console_loglevel = saved_console_loglevel;
1815	saved_console_loglevel = LOGLEVEL_DEFAULT;
1816	}
1817	break;
1818	/* Set level of messages printed to console */
1819	case SYSLOG_ACTION_CONSOLE_LEVEL:
1820	if (len < 1 || len > 8)
1821	return -EINVAL;
1822	if (len < minimum_console_loglevel)
1823	len = minimum_console_loglevel;
1824	console_loglevel = len;
1825	/* Implicitly re-enable logging to console */
1826	saved_console_loglevel = LOGLEVEL_DEFAULT;
1827	break;
1828	/* Number of chars in the log buffer */
1829	case SYSLOG_ACTION_SIZE_UNREAD:
1830	mutex_lock(&syslog_lock);
1831	if (!prb_read_valid_info(rb: prb, seq: syslog_seq, info: &info, NULL)) {
1832	/* No unread messages. */
1833	mutex_unlock(lock: &syslog_lock);
1834	return 0;
1835	}
1836	if (info.seq != syslog_seq) {
1837	/* messages are gone, move to first one */
1838	syslog_seq = info.seq;
1839	syslog_partial = 0;
1840	}
1841	if (source == SYSLOG_FROM_PROC) {
1842	/*
1843	* Short-cut for poll(/"proc/kmsg") which simply checks
1844	* for pending data, not the size; return the count of
1845	* records, not the length.
1846	*/
1847	error = prb_next_seq(rb: prb) - syslog_seq;
1848	} else {
1849	bool time = syslog_partial ? syslog_time : printk_time;
1850	unsigned int line_count;
1851	u64 seq;
1852
1853	prb_for_each_info(syslog_seq, prb, seq, &info,
1854	&line_count) {
1855	error += get_record_print_text_size(info: &info, line_count,
1856	syslog: true, time);
1857	time = printk_time;
1858	}
1859	error -= syslog_partial;
1860	}
1861	mutex_unlock(lock: &syslog_lock);
1862	break;
1863	/* Size of the log buffer */
1864	case SYSLOG_ACTION_SIZE_BUFFER:
1865	error = log_buf_len;
1866	break;
1867	default:
1868	error = -EINVAL;
1869	break;
1870	}
1871
1872	return error;
1873	}
1874
1875	SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
1876	{
1877	return do_syslog(type, buf, len, SYSLOG_FROM_READER);
1878	}
1879
1880	/*
1881	* Special console_lock variants that help to reduce the risk of soft-lockups.
1882	* They allow to pass console_lock to another printk() call using a busy wait.
1883	*/
1884
1885	#ifdef CONFIG_LOCKDEP
1886	static struct lockdep_map console_owner_dep_map = {
1887	.name = "console_owner"
1888	};
1889	#endif
1890
1891	static DEFINE_RAW_SPINLOCK(console_owner_lock);
1892	static struct task_struct *console_owner;
1893	static bool console_waiter;
1894
1895	/**
1896	* console_lock_spinning_enable - mark beginning of code where another
1897	* thread might safely busy wait
1898	*
1899	* This basically converts console_lock into a spinlock. This marks
1900	* the section where the console_lock owner can not sleep, because
1901	* there may be a waiter spinning (like a spinlock). Also it must be
1902	* ready to hand over the lock at the end of the section.
1903	*/
1904	void console_lock_spinning_enable(void)
1905	{
1906	/*
1907	* Do not use spinning in panic(). The panic CPU wants to keep the lock.
1908	* Non-panic CPUs abandon the flush anyway.
1909	*
1910	* Just keep the lockdep annotation. The panic-CPU should avoid
1911	* taking console_owner_lock because it might cause a deadlock.
1912	* This looks like the easiest way how to prevent false lockdep
1913	* reports without handling races a lockless way.
1914	*/
1915	if (panic_in_progress())
1916	goto lockdep;
1917
1918	raw_spin_lock(&console_owner_lock);
1919	console_owner = current;
1920	raw_spin_unlock(&console_owner_lock);
1921
1922	lockdep:
1923	/* The waiter may spin on us after setting console_owner */
1924	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
1925	}
1926
1927	/**
1928	* console_lock_spinning_disable_and_check - mark end of code where another
1929	* thread was able to busy wait and check if there is a waiter
1930	* @cookie: cookie returned from console_srcu_read_lock()
1931	*
1932	* This is called at the end of the section where spinning is allowed.
1933	* It has two functions. First, it is a signal that it is no longer
1934	* safe to start busy waiting for the lock. Second, it checks if
1935	* there is a busy waiter and passes the lock rights to her.
1936	*
1937	* Important: Callers lose both the console_lock and the SRCU read lock if
1938	* there was a busy waiter. They must not touch items synchronized by
1939	* console_lock or SRCU read lock in this case.
1940	*
1941	* Return: 1 if the lock rights were passed, 0 otherwise.
1942	*/
1943	int console_lock_spinning_disable_and_check(int cookie)
1944	{
1945	int waiter;
1946
1947	/*
1948	* Ignore spinning waiters during panic() because they might get stopped
1949	* or blocked at any time,
1950	*
1951	* It is safe because nobody is allowed to start spinning during panic
1952	* in the first place. If there has been a waiter then non panic CPUs
1953	* might stay spinning. They would get stopped anyway. The panic context
1954	* will never start spinning and an interrupted spin on panic CPU will
1955	* never continue.
1956	*/
1957	if (panic_in_progress()) {
1958	/* Keep lockdep happy. */
1959	spin_release(&console_owner_dep_map, _THIS_IP_);
1960	return 0;
1961	}
1962
1963	raw_spin_lock(&console_owner_lock);
1964	waiter = READ_ONCE(console_waiter);
1965	console_owner = NULL;
1966	raw_spin_unlock(&console_owner_lock);
1967
1968	if (!waiter) {
1969	spin_release(&console_owner_dep_map, _THIS_IP_);
1970	return 0;
1971	}
1972
1973	/* The waiter is now free to continue */
1974	WRITE_ONCE(console_waiter, false);
1975
1976	spin_release(&console_owner_dep_map, _THIS_IP_);
1977
1978	/*
1979	* Preserve lockdep lock ordering. Release the SRCU read lock before
1980	* releasing the console_lock.
1981	*/
1982	console_srcu_read_unlock(cookie);
1983
1984	/*
1985	* Hand off console_lock to waiter. The waiter will perform
1986	* the up(). After this, the waiter is the console_lock owner.
1987	*/
1988	mutex_release(&console_lock_dep_map, _THIS_IP_);
1989	return 1;
1990	}
1991
1992	/**
1993	* console_trylock_spinning - try to get console_lock by busy waiting
1994	*
1995	* This allows to busy wait for the console_lock when the current
1996	* owner is running in specially marked sections. It means that
1997	* the current owner is running and cannot reschedule until it
1998	* is ready to lose the lock.
1999	*
2000	* Return: 1 if we got the lock, 0 othrewise
2001	*/
2002	static int console_trylock_spinning(void)
2003	{
2004	struct task_struct *owner = NULL;
2005	bool waiter;
2006	bool spin = false;
2007	unsigned long flags;
2008
2009	if (console_trylock())
2010	return 1;
2011
2012	/*
2013	* It's unsafe to spin once a panic has begun. If we are the
2014	* panic CPU, we may have already halted the owner of the
2015	* console_sem. If we are not the panic CPU, then we should
2016	* avoid taking console_sem, so the panic CPU has a better
2017	* chance of cleanly acquiring it later.
2018	*/
2019	if (panic_in_progress())
2020	return 0;
2021
2022	printk_safe_enter_irqsave(flags);
2023
2024	raw_spin_lock(&console_owner_lock);
2025	owner = READ_ONCE(console_owner);
2026	waiter = READ_ONCE(console_waiter);
2027	if (!waiter && owner && owner != current) {
2028	WRITE_ONCE(console_waiter, true);
2029	spin = true;
2030	}
2031	raw_spin_unlock(&console_owner_lock);
2032
2033	/*
2034	* If there is an active printk() writing to the
2035	* consoles, instead of having it write our data too,
2036	* see if we can offload that load from the active
2037	* printer, and do some printing ourselves.
2038	* Go into a spin only if there isn't already a waiter
2039	* spinning, and there is an active printer, and
2040	* that active printer isn't us (recursive printk?).
2041	*/
2042	if (!spin) {
2043	printk_safe_exit_irqrestore(flags);
2044	return 0;
2045	}
2046
2047	/* We spin waiting for the owner to release us */
2048	spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
2049	/* Owner will clear console_waiter on hand off */
2050	while (READ_ONCE(console_waiter))
2051	cpu_relax();
2052	spin_release(&console_owner_dep_map, _THIS_IP_);
2053
2054	printk_safe_exit_irqrestore(flags);
2055	/*
2056	* The owner passed the console lock to us.
2057	* Since we did not spin on console lock, annotate
2058	* this as a trylock. Otherwise lockdep will
2059	* complain.
2060	*/
2061	mutex_acquire(&console_lock_dep_map, 0, 1, _THIS_IP_);
2062
2063	/*
2064	* Update @console_may_schedule for trylock because the previous
2065	* owner may have been schedulable.
2066	*/
2067	console_may_schedule = 0;
2068
2069	return 1;
2070	}
2071
2072	/*
2073	* Recursion is tracked separately on each CPU. If NMIs are supported, an
2074	* additional NMI context per CPU is also separately tracked. Until per-CPU
2075	* is available, a separate "early tracking" is performed.
2076	*/
2077	static DEFINE_PER_CPU(u8, printk_count);
2078	static u8 printk_count_early;
2079	#ifdef CONFIG_HAVE_NMI
2080	static DEFINE_PER_CPU(u8, printk_count_nmi);
2081	static u8 printk_count_nmi_early;
2082	#endif
2083
2084	/*
2085	* Recursion is limited to keep the output sane. printk() should not require
2086	* more than 1 level of recursion (allowing, for example, printk() to trigger
2087	* a WARN), but a higher value is used in case some printk-internal errors
2088	* exist, such as the ringbuffer validation checks failing.
2089	*/
2090	#define PRINTK_MAX_RECURSION 3
2091
2092	/*
2093	* Return a pointer to the dedicated counter for the CPU+context of the
2094	* caller.
2095	*/
2096	static u8 *__printk_recursion_counter(void)
2097	{
2098	#ifdef CONFIG_HAVE_NMI
2099	if (in_nmi()) {
2100	if (printk_percpu_data_ready())
2101	return this_cpu_ptr(&printk_count_nmi);
2102	return &printk_count_nmi_early;
2103	}
2104	#endif
2105	if (printk_percpu_data_ready())
2106	return this_cpu_ptr(&printk_count);
2107	return &printk_count_early;
2108	}
2109
2110	/*
2111	* Enter recursion tracking. Interrupts are disabled to simplify tracking.
2112	* The caller must check the boolean return value to see if the recursion is
2113	* allowed. On failure, interrupts are not disabled.
2114	*
2115	* @recursion_ptr must be a variable of type (u8 *) and is the same variable
2116	* that is passed to printk_exit_irqrestore().
2117	*/
2118	#define printk_enter_irqsave(recursion_ptr, flags) \
2119	({ \
2120	bool success = true; \
2121	\
2122	typecheck(u8 *, recursion_ptr); \
2123	local_irq_save(flags); \
2124	(recursion_ptr) = __printk_recursion_counter(); \
2125	if (*(recursion_ptr) > PRINTK_MAX_RECURSION) { \
2126	local_irq_restore(flags); \
2127	success = false; \
2128	} else { \
2129	(*(recursion_ptr))++; \
2130	} \
2131	success; \
2132	})
2133
2134	/* Exit recursion tracking, restoring interrupts. */
2135	#define printk_exit_irqrestore(recursion_ptr, flags) \
2136	do { \
2137	typecheck(u8 *, recursion_ptr); \
2138	(*(recursion_ptr))--; \
2139	local_irq_restore(flags); \
2140	} while (0)
2141
2142	int printk_delay_msec __read_mostly;
2143
2144	static inline void printk_delay(int level)
2145	{
2146	boot_delay_msec(level);
2147
2148	if (unlikely(printk_delay_msec)) {
2149	int m = printk_delay_msec;
2150
2151	while (m--) {
2152	mdelay(1);
2153	touch_nmi_watchdog();
2154	}
2155	}
2156	}
2157
2158	static inline u32 printk_caller_id(void)
2159	{
2160	return in_task() ? task_pid_nr(current) :
2161	0x80000000 + smp_processor_id();
2162	}
2163
2164	/**
2165	* printk_parse_prefix - Parse level and control flags.
2166	*
2167	* @text: The terminated text message.
2168	* @level: A pointer to the current level value, will be updated.
2169	* @flags: A pointer to the current printk_info flags, will be updated.
2170	*
2171	* @level may be NULL if the caller is not interested in the parsed value.
2172	* Otherwise the variable pointed to by @level must be set to
2173	* LOGLEVEL_DEFAULT in order to be updated with the parsed value.
2174	*
2175	* @flags may be NULL if the caller is not interested in the parsed value.
2176	* Otherwise the variable pointed to by @flags will be OR'd with the parsed
2177	* value.
2178	*
2179	* Return: The length of the parsed level and control flags.
2180	*/
2181	u16 printk_parse_prefix(const char *text, int *level,
2182	enum printk_info_flags *flags)
2183	{
2184	u16 prefix_len = 0;
2185	int kern_level;
2186
2187	while (*text) {
2188	kern_level = printk_get_level(buffer: text);
2189	if (!kern_level)
2190	break;
2191
2192	switch (kern_level) {
2193	case '0' ... '7':
2194	if (level && *level == LOGLEVEL_DEFAULT)
2195	*level = kern_level - '0';
2196	break;
2197	case 'c': /* KERN_CONT */
2198	if (flags)
2199	*flags |= LOG_CONT;
2200	}
2201
2202	prefix_len += 2;
2203	text += 2;
2204	}
2205
2206	return prefix_len;
2207	}
2208
2209	__printf(5, 0)
2210	static u16 printk_sprint(char *text, u16 size, int facility,
2211	enum printk_info_flags *flags, const char *fmt,
2212	va_list args)
2213	{
2214	u16 text_len;
2215
2216	text_len = vscnprintf(buf: text, size, fmt, args);
2217
2218	/* Mark and strip a trailing newline. */
2219	if (text_len && text[text_len - 1] == '\n') {
2220	text_len--;
2221	*flags |= LOG_NEWLINE;
2222	}
2223
2224	/* Strip log level and control flags. */
2225	if (facility == 0) {
2226	u16 prefix_len;
2227
2228	prefix_len = printk_parse_prefix(text, NULL, NULL);
2229	if (prefix_len) {
2230	text_len -= prefix_len;
2231	memmove(text, text + prefix_len, text_len);
2232	}
2233	}
2234
2235	trace_console(text, len: text_len);
2236
2237	return text_len;
2238	}
2239
2240	__printf(4, 0)
2241	int vprintk_store(int facility, int level,
2242	const struct dev_printk_info *dev_info,
2243	const char *fmt, va_list args)
2244	{
2245	struct prb_reserved_entry e;
2246	enum printk_info_flags flags = 0;
2247	struct printk_record r;
2248	unsigned long irqflags;
2249	u16 trunc_msg_len = 0;
2250	char prefix_buf[8];
2251	u8 *recursion_ptr;
2252	u16 reserve_size;
2253	va_list args2;
2254	u32 caller_id;
2255	u16 text_len;
2256	int ret = 0;
2257	u64 ts_nsec;
2258
2259	if (!printk_enter_irqsave(recursion_ptr, irqflags))
2260	return 0;
2261
2262	/*
2263	* Since the duration of printk() can vary depending on the message
2264	* and state of the ringbuffer, grab the timestamp now so that it is
2265	* close to the call of printk(). This provides a more deterministic
2266	* timestamp with respect to the caller.
2267	*/
2268	ts_nsec = local_clock();
2269
2270	caller_id = printk_caller_id();
2271
2272	/*
2273	* The sprintf needs to come first since the syslog prefix might be
2274	* passed in as a parameter. An extra byte must be reserved so that
2275	* later the vscnprintf() into the reserved buffer has room for the
2276	* terminating '\0', which is not counted by vsnprintf().
2277	*/
2278	va_copy(args2, args);
2279	reserve_size = vsnprintf(buf: &prefix_buf[0], size: sizeof(prefix_buf), fmt, args: args2) + 1;
2280	va_end(args2);
2281
2282	if (reserve_size > PRINTKRB_RECORD_MAX)
2283	reserve_size = PRINTKRB_RECORD_MAX;
2284
2285	/* Extract log level or control flags. */
2286	if (facility == 0)
2287	printk_parse_prefix(text: &prefix_buf[0], level: &level, flags: &flags);
2288
2289	if (level == LOGLEVEL_DEFAULT)
2290	level = default_message_loglevel;
2291
2292	if (dev_info)
2293	flags |= LOG_NEWLINE;
2294
2295	if (is_printk_force_console())
2296	flags |= LOG_FORCE_CON;
2297
2298	if (flags & LOG_CONT) {
2299	prb_rec_init_wr(r: &r, text_buf_size: reserve_size);
2300	if (prb_reserve_in_last(e: &e, rb: prb, r: &r, caller_id, PRINTKRB_RECORD_MAX)) {
2301	text_len = printk_sprint(text: &r.text_buf[r.info->text_len], size: reserve_size,
2302	facility, flags: &flags, fmt, args);
2303	r.info->text_len += text_len;
2304
2305	if (flags & LOG_FORCE_CON)
2306	r.info->flags |= LOG_FORCE_CON;
2307
2308	if (flags & LOG_NEWLINE) {
2309	r.info->flags |= LOG_NEWLINE;
2310	prb_final_commit(e: &e);
2311	} else {
2312	prb_commit(e: &e);
2313	}
2314
2315	ret = text_len;
2316	goto out;
2317	}
2318	}
2319
2320	/*
2321	* Explicitly initialize the record before every prb_reserve() call.
2322	* prb_reserve_in_last() and prb_reserve() purposely invalidate the
2323	* structure when they fail.
2324	*/
2325	prb_rec_init_wr(r: &r, text_buf_size: reserve_size);
2326	if (!prb_reserve(e: &e, rb: prb, r: &r)) {
2327	/* truncate the message if it is too long for empty buffer */
2328	truncate_msg(text_len: &reserve_size, trunc_msg_len: &trunc_msg_len);
2329
2330	prb_rec_init_wr(r: &r, text_buf_size: reserve_size + trunc_msg_len);
2331	if (!prb_reserve(e: &e, rb: prb, r: &r))
2332	goto out;
2333	}
2334
2335	/* fill message */
2336	text_len = printk_sprint(text: &r.text_buf[0], size: reserve_size, facility, flags: &flags, fmt, args);
2337	if (trunc_msg_len)
2338	memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
2339	r.info->text_len = text_len + trunc_msg_len;
2340	r.info->facility = facility;
2341	r.info->level = level & 7;
2342	r.info->flags = flags & 0x1f;
2343	r.info->ts_nsec = ts_nsec;
2344	r.info->caller_id = caller_id;
2345	if (dev_info)
2346	memcpy(&r.info->dev_info, dev_info, sizeof(r.info->dev_info));
2347
2348	/* A message without a trailing newline can be continued. */
2349	if (!(flags & LOG_NEWLINE))
2350	prb_commit(e: &e);
2351	else
2352	prb_final_commit(e: &e);
2353
2354	ret = text_len + trunc_msg_len;
2355	out:
2356	printk_exit_irqrestore(recursion_ptr, irqflags);
2357	return ret;
2358	}
2359
2360	/*
2361	* This acts as a one-way switch to allow legacy consoles to print from
2362	* the printk() caller context on a panic CPU. It also attempts to flush
2363	* the legacy consoles in this context.
2364	*/
2365	void printk_legacy_allow_panic_sync(void)
2366	{
2367	struct console_flush_type ft;
2368
2369	legacy_allow_panic_sync = true;
2370
2371	printk_get_console_flush_type(ft: &ft);
2372	if (ft.legacy_direct) {
2373	if (console_trylock())
2374	console_unlock();
2375	}
2376	}
2377
2378	bool __read_mostly debug_non_panic_cpus;
2379
2380	#ifdef CONFIG_PRINTK_CALLER
2381	static int __init debug_non_panic_cpus_setup(char *str)
2382	{
2383	debug_non_panic_cpus = true;
2384	pr_info("allow messages from non-panic CPUs in panic()\n");
2385
2386	return 0;
2387	}
2388	early_param("debug_non_panic_cpus", debug_non_panic_cpus_setup);
2389	module_param(debug_non_panic_cpus, bool, 0644);
2390	MODULE_PARM_DESC(debug_non_panic_cpus,
2391	"allow messages from non-panic CPUs in panic()");
2392	#endif
2393
2394	asmlinkage int vprintk_emit(int facility, int level,
2395	const struct dev_printk_info *dev_info,
2396	const char *fmt, va_list args)
2397	{
2398	struct console_flush_type ft;
2399	int printed_len;
2400
2401	/* Suppress unimportant messages after panic happens */
2402	if (unlikely(suppress_printk))
2403	return 0;
2404
2405	/*
2406	* The messages on the panic CPU are the most important. If
2407	* non-panic CPUs are generating any messages, they will be
2408	* silently dropped.
2409	*/
2410	if (other_cpu_in_panic() &&
2411	!debug_non_panic_cpus &&
2412	!panic_triggering_all_cpu_backtrace)
2413	return 0;
2414
2415	printk_get_console_flush_type(ft: &ft);
2416
2417	/* If called from the scheduler, we can not call up(). */
2418	if (level == LOGLEVEL_SCHED) {
2419	level = LOGLEVEL_DEFAULT;
2420	ft.legacy_offload |= ft.legacy_direct;
2421	ft.legacy_direct = false;
2422	}
2423
2424	printk_delay(level);
2425
2426	printed_len = vprintk_store(facility, level, dev_info, fmt, args);
2427
2428	if (ft.nbcon_atomic)
2429	nbcon_atomic_flush_pending();
2430
2431	if (ft.nbcon_offload)
2432	nbcon_kthreads_wake();
2433
2434	if (ft.legacy_direct) {
2435	/*
2436	* The caller may be holding system-critical or
2437	* timing-sensitive locks. Disable preemption during
2438	* printing of all remaining records to all consoles so that
2439	* this context can return as soon as possible. Hopefully
2440	* another printk() caller will take over the printing.
2441	*/
2442	preempt_disable();
2443	/*
2444	* Try to acquire and then immediately release the console
2445	* semaphore. The release will print out buffers. With the
2446	* spinning variant, this context tries to take over the
2447	* printing from another printing context.
2448	*/
2449	if (console_trylock_spinning())
2450	console_unlock();
2451	preempt_enable();
2452	}
2453
2454	if (ft.legacy_offload)
2455	defer_console_output();
2456	else
2457	wake_up_klogd();
2458
2459	return printed_len;
2460	}
2461	EXPORT_SYMBOL(vprintk_emit);
2462
2463	int vprintk_default(const char *fmt, va_list args)
2464	{
2465	return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, fmt, args);
2466	}
2467	EXPORT_SYMBOL_GPL(vprintk_default);
2468
2469	asmlinkage __visible int _printk(const char *fmt, ...)
2470	{
2471	va_list args;
2472	int r;
2473
2474	va_start(args, fmt);
2475	r = vprintk(fmt, args);
2476	va_end(args);
2477
2478	return r;
2479	}
2480	EXPORT_SYMBOL(_printk);
2481
2482	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);
2483
2484	#else /* CONFIG_PRINTK */
2485
2486	#define printk_time false
2487
2488	#define prb_read_valid(rb, seq, r) false
2489	#define prb_first_valid_seq(rb) 0
2490	#define prb_next_seq(rb) 0
2491
2492	static u64 syslog_seq;
2493
2494	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }
2495
2496	#endif /* CONFIG_PRINTK */
2497
2498	#ifdef CONFIG_EARLY_PRINTK
2499	struct console *early_console;
2500
2501	asmlinkage __visible void early_printk(const char *fmt, ...)
2502	{
2503	va_list ap;
2504	char buf[512];
2505	int n;
2506
2507	if (!early_console)
2508	return;
2509
2510	va_start(ap, fmt);
2511	n = vscnprintf(buf, size: sizeof(buf), fmt, args: ap);
2512	va_end(ap);
2513
2514	early_console->write(early_console, buf, n);
2515	}
2516	#endif
2517
2518	static void set_user_specified(struct console_cmdline *c, bool user_specified)
2519	{
2520	if (!user_specified)
2521	return;
2522
2523	/*
2524	* @c console was defined by the user on the command line.
2525	* Do not clear when added twice also by SPCR or the device tree.
2526	*/
2527	c->user_specified = true;
2528	/* At least one console defined by the user on the command line. */
2529	console_set_on_cmdline = 1;
2530	}
2531
2532	static int __add_preferred_console(const char *name, const short idx,
2533	const char *devname, char *options,
2534	char *brl_options, bool user_specified)
2535	{
2536	struct console_cmdline *c;
2537	int i;
2538
2539	if (!name && !devname)
2540	return -EINVAL;
2541
2542	/*
2543	* We use a signed short index for struct console for device drivers to
2544	* indicate a not yet assigned index or port. However, a negative index
2545	* value is not valid when the console name and index are defined on
2546	* the command line.
2547	*/
2548	if (name && idx < 0)
2549	return -EINVAL;
2550
2551	/*
2552	* See if this tty is not yet registered, and
2553	* if we have a slot free.
2554	*/
2555	for (i = 0, c = console_cmdline;
2556	i < MAX_CMDLINECONSOLES && (c->name[0] || c->devname[0]);
2557	i++, c++) {
2558	if ((name && strcmp(c->name, name) == 0 && c->index == idx) ||
2559	(devname && strcmp(c->devname, devname) == 0)) {
2560	if (!brl_options)
2561	preferred_console = i;
2562	set_user_specified(c, user_specified);
2563	return 0;
2564	}
2565	}
2566	if (i == MAX_CMDLINECONSOLES)
2567	return -E2BIG;
2568	if (!brl_options)
2569	preferred_console = i;
2570	if (name)
2571	strscpy(c->name, name);
2572	if (devname)
2573	strscpy(c->devname, devname);
2574	c->options = options;
2575	set_user_specified(c, user_specified);
2576	braille_set_options(c, brl_options);
2577
2578	c->index = idx;
2579	return 0;
2580	}
2581
2582	static int __init console_msg_format_setup(char *str)
2583	{
2584	if (!strcmp(str, "syslog"))
2585	console_msg_format = MSG_FORMAT_SYSLOG;
2586	if (!strcmp(str, "default"))
2587	console_msg_format = MSG_FORMAT_DEFAULT;
2588	return 1;
2589	}
2590	__setup("console_msg_format=", console_msg_format_setup);
2591
2592	/*
2593	* Set up a console. Called via do_early_param() in init/main.c
2594	* for each "console=" parameter in the boot command line.
2595	*/
2596	static int __init console_setup(char *str)
2597	{
2598	static_assert(sizeof(console_cmdline[0].devname) >= sizeof(console_cmdline[0].name) + 4);
2599	char buf[sizeof(console_cmdline[0].devname)];
2600	char *brl_options = NULL;
2601	char *ttyname = NULL;
2602	char *devname = NULL;
2603	char *options;
2604	char *s;
2605	int idx;
2606
2607	/*
2608	* console="" or console=null have been suggested as a way to
2609	* disable console output. Use ttynull that has been created
2610	* for exactly this purpose.
2611	*/
2612	if (str[0] == 0 || strcmp(str, "null") == 0) {
2613	__add_preferred_console(name: "ttynull", idx: 0, NULL, NULL, NULL, user_specified: true);
2614	return 1;
2615	}
2616
2617	if (_braille_console_setup(str: &str, brl_options: &brl_options))
2618	return 1;
2619
2620	/* For a DEVNAME:0.0 style console the character device is unknown early */
2621	if (strchr(str, ':'))
2622	devname = buf;
2623	else
2624	ttyname = buf;
2625
2626	/*
2627	* Decode str into name, index, options.
2628	*/
2629	if (ttyname && isdigit(c: str[0]))
2630	scnprintf(buf, size: sizeof(buf), fmt: "ttyS%s", str);
2631	else
2632	strscpy(buf, str);
2633
2634	options = strchr(str, ',');
2635	if (options)
2636	*(options++) = 0;
2637
2638	#ifdef __sparc__
2639	if (!strcmp(str, "ttya"))
2640	strscpy(buf, "ttyS0");
2641	if (!strcmp(str, "ttyb"))
2642	strscpy(buf, "ttyS1");
2643	#endif
2644
2645	for (s = buf; *s; s++)
2646	if ((ttyname && isdigit(c: *s)) || *s == ',')
2647	break;
2648
2649	/* @idx will get defined when devname matches. */
2650	if (devname)
2651	idx = -1;
2652	else
2653	idx = simple_strtoul(s, NULL, 10);
2654
2655	*s = 0;
2656
2657	__add_preferred_console(name: ttyname, idx, devname, options, brl_options, user_specified: true);
2658	return 1;
2659	}
2660	__setup("console=", console_setup);
2661
2662	/**
2663	* add_preferred_console - add a device to the list of preferred consoles.
2664	* @name: device name
2665	* @idx: device index
2666	* @options: options for this console
2667	*
2668	* The last preferred console added will be used for kernel messages
2669	* and stdin/out/err for init. Normally this is used by console_setup
2670	* above to handle user-supplied console arguments; however it can also
2671	* be used by arch-specific code either to override the user or more
2672	* commonly to provide a default console (ie from PROM variables) when
2673	* the user has not supplied one.
2674	*/
2675	int add_preferred_console(const char *name, const short idx, char *options)
2676	{
2677	return __add_preferred_console(name, idx, NULL, options, NULL, user_specified: false);
2678	}
2679
2680	/**
2681	* match_devname_and_update_preferred_console - Update a preferred console
2682	* when matching devname is found.
2683	* @devname: DEVNAME:0.0 style device name
2684	* @name: Name of the corresponding console driver, e.g. "ttyS"
2685	* @idx: Console index, e.g. port number.
2686	*
2687	* The function checks whether a device with the given @devname is
2688	* preferred via the console=DEVNAME:0.0 command line option.
2689	* It fills the missing console driver name and console index
2690	* so that a later register_console() call could find (match)
2691	* and enable this device.
2692	*
2693	* It might be used when a driver subsystem initializes particular
2694	* devices with already known DEVNAME:0.0 style names. And it
2695	* could predict which console driver name and index this device
2696	* would later get associated with.
2697	*
2698	* Return: 0 on success, negative error code on failure.
2699	*/
2700	int match_devname_and_update_preferred_console(const char *devname,
2701	const char *name,
2702	const short idx)
2703	{
2704	struct console_cmdline *c = console_cmdline;
2705	int i;
2706
2707	if (!devname || !strlen(devname) || !name || !strlen(name) || idx < 0)
2708	return -EINVAL;
2709
2710	for (i = 0; i < MAX_CMDLINECONSOLES && (c->name[0] || c->devname[0]);
2711	i++, c++) {
2712	if (!strcmp(devname, c->devname)) {
2713	pr_info("associate the preferred console \"%s\" with \"%s%d\"\n",
2714	devname, name, idx);
2715	strscpy(c->name, name);
2716	c->index = idx;
2717	return 0;
2718	}
2719	}
2720
2721	return -ENOENT;
2722	}
2723	EXPORT_SYMBOL_GPL(match_devname_and_update_preferred_console);
2724
2725	bool console_suspend_enabled = true;
2726	EXPORT_SYMBOL(console_suspend_enabled);
2727
2728	static int __init console_suspend_disable(char *str)
2729	{
2730	console_suspend_enabled = false;
2731	return 1;
2732	}
2733	__setup("no_console_suspend", console_suspend_disable);
2734	module_param_named(console_suspend, console_suspend_enabled,
2735	bool, S_IRUGO | S_IWUSR);
2736	MODULE_PARM_DESC(console_suspend, "suspend console during suspend"
2737	" and hibernate operations");
2738
2739	static bool printk_console_no_auto_verbose;
2740
2741	void console_verbose(void)
2742	{
2743	if (console_loglevel && !printk_console_no_auto_verbose)
2744	console_loglevel = CONSOLE_LOGLEVEL_MOTORMOUTH;
2745	}
2746	EXPORT_SYMBOL_GPL(console_verbose);
2747
2748	module_param_named(console_no_auto_verbose, printk_console_no_auto_verbose, bool, 0644);
2749	MODULE_PARM_DESC(console_no_auto_verbose, "Disable console loglevel raise to highest on oops/panic/etc");
2750
2751	/**
2752	* console_suspend_all - suspend the console subsystem
2753	*
2754	* This disables printk() while we go into suspend states
2755	*/
2756	void console_suspend_all(void)
2757	{
2758	struct console *con;
2759
2760	if (!console_suspend_enabled)
2761	return;
2762	pr_info("Suspending console(s) (use no_console_suspend to debug)\n");
2763	pr_flush(timeout_ms: 1000, reset_on_progress: true);
2764
2765	console_list_lock();
2766	for_each_console(con)
2767	console_srcu_write_flags(con, flags: con->flags | CON_SUSPENDED);
2768	console_list_unlock();
2769
2770	/*
2771	* Ensure that all SRCU list walks have completed. All printing
2772	* contexts must be able to see that they are suspended so that it
2773	* is guaranteed that all printing has stopped when this function
2774	* completes.
2775	*/
2776	synchronize_srcu(ssp: &console_srcu);
2777	}
2778
2779	void console_resume_all(void)
2780	{
2781	struct console_flush_type ft;
2782	struct console *con;
2783
2784	if (!console_suspend_enabled)
2785	return;
2786
2787	console_list_lock();
2788	for_each_console(con)
2789	console_srcu_write_flags(con, flags: con->flags & ~CON_SUSPENDED);
2790	console_list_unlock();
2791
2792	/*
2793	* Ensure that all SRCU list walks have completed. All printing
2794	* contexts must be able to see they are no longer suspended so
2795	* that they are guaranteed to wake up and resume printing.
2796	*/
2797	synchronize_srcu(ssp: &console_srcu);
2798
2799	printk_get_console_flush_type(ft: &ft);
2800	if (ft.nbcon_offload)
2801	nbcon_kthreads_wake();
2802	if (ft.legacy_offload)
2803	defer_console_output();
2804
2805	pr_flush(timeout_ms: 1000, reset_on_progress: true);
2806	}
2807
2808	/**
2809	* console_cpu_notify - print deferred console messages after CPU hotplug
2810	* @cpu: unused
2811	*
2812	* If printk() is called from a CPU that is not online yet, the messages
2813	* will be printed on the console only if there are CON_ANYTIME consoles.
2814	* This function is called when a new CPU comes online (or fails to come
2815	* up) or goes offline.
2816	*/
2817	static int console_cpu_notify(unsigned int cpu)
2818	{
2819	struct console_flush_type ft;
2820
2821	if (!cpuhp_tasks_frozen) {
2822	printk_get_console_flush_type(ft: &ft);
2823	if (ft.nbcon_atomic)
2824	nbcon_atomic_flush_pending();
2825	if (ft.legacy_direct) {
2826	if (console_trylock())
2827	console_unlock();
2828	}
2829	}
2830	return 0;
2831	}
2832
2833	/**
2834	* console_lock - block the console subsystem from printing
2835	*
2836	* Acquires a lock which guarantees that no consoles will
2837	* be in or enter their write() callback.
2838	*
2839	* Can sleep, returns nothing.
2840	*/
2841	void console_lock(void)
2842	{
2843	might_sleep();
2844
2845	/* On panic, the console_lock must be left to the panic cpu. */
2846	while (other_cpu_in_panic())
2847	msleep(msecs: 1000);
2848
2849	down_console_sem();
2850	console_locked = 1;
2851	console_may_schedule = 1;
2852	}
2853	EXPORT_SYMBOL(console_lock);
2854
2855	/**
2856	* console_trylock - try to block the console subsystem from printing
2857	*
2858	* Try to acquire a lock which guarantees that no consoles will
2859	* be in or enter their write() callback.
2860	*
2861	* returns 1 on success, and 0 on failure to acquire the lock.
2862	*/
2863	int console_trylock(void)
2864	{
2865	/* On panic, the console_lock must be left to the panic cpu. */
2866	if (other_cpu_in_panic())
2867	return 0;
2868	if (down_trylock_console_sem())
2869	return 0;
2870	console_locked = 1;
2871	console_may_schedule = 0;
2872	return 1;
2873	}
2874	EXPORT_SYMBOL(console_trylock);
2875
2876	int is_console_locked(void)
2877	{
2878	return console_locked;
2879	}
2880	EXPORT_SYMBOL(is_console_locked);
2881
2882	static void __console_unlock(void)
2883	{
2884	console_locked = 0;
2885	up_console_sem();
2886	}
2887
2888	#ifdef CONFIG_PRINTK
2889
2890	/*
2891	* Prepend the message in @pmsg->pbufs->outbuf. This is achieved by shifting
2892	* the existing message over and inserting the scratchbuf message.
2893	*
2894	* @pmsg is the original printk message.
2895	* @fmt is the printf format of the message which will prepend the existing one.
2896	*
2897	* If there is not enough space in @pmsg->pbufs->outbuf, the existing
2898	* message text will be sufficiently truncated.
2899	*
2900	* If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
2901	*/
2902	__printf(2, 3)
2903	static void console_prepend_message(struct printk_message *pmsg, const char *fmt, ...)
2904	{
2905	struct printk_buffers *pbufs = pmsg->pbufs;
2906	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2907	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2908	char *scratchbuf = &pbufs->scratchbuf[0];
2909	char *outbuf = &pbufs->outbuf[0];
2910	va_list args;
2911	size_t len;
2912
2913	va_start(args, fmt);
2914	len = vscnprintf(buf: scratchbuf, size: scratchbuf_sz, fmt, args);
2915	va_end(args);
2916
2917	/*
2918	* Make sure outbuf is sufficiently large before prepending.
2919	* Keep at least the prefix when the message must be truncated.
2920	* It is a rather theoretical problem when someone tries to
2921	* use a minimalist buffer.
2922	*/
2923	if (WARN_ON_ONCE(len + PRINTK_PREFIX_MAX >= outbuf_sz))
2924	return;
2925
2926	if (pmsg->outbuf_len + len >= outbuf_sz) {
2927	/* Truncate the message, but keep it terminated. */
2928	pmsg->outbuf_len = outbuf_sz - (len + 1);
2929	outbuf[pmsg->outbuf_len] = 0;
2930	}
2931
2932	memmove(outbuf + len, outbuf, pmsg->outbuf_len + 1);
2933	memcpy(outbuf, scratchbuf, len);
2934	pmsg->outbuf_len += len;
2935	}
2936
2937	/*
2938	* Prepend the message in @pmsg->pbufs->outbuf with a "dropped message".
2939	* @pmsg->outbuf_len is updated appropriately.
2940	*
2941	* @pmsg is the printk message to prepend.
2942	*
2943	* @dropped is the dropped count to report in the dropped message.
2944	*/
2945	void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
2946	{
2947	console_prepend_message(pmsg, fmt: "** %lu printk messages dropped **\n", dropped);
2948	}
2949
2950	/*
2951	* Prepend the message in @pmsg->pbufs->outbuf with a "replay message".
2952	* @pmsg->outbuf_len is updated appropriately.
2953	*
2954	* @pmsg is the printk message to prepend.
2955	*/
2956	void console_prepend_replay(struct printk_message *pmsg)
2957	{
2958	console_prepend_message(pmsg, fmt: "** replaying previous printk message **\n");
2959	}
2960
2961	/*
2962	* Read and format the specified record (or a later record if the specified
2963	* record is not available).
2964	*
2965	* @pmsg will contain the formatted result. @pmsg->pbufs must point to a
2966	* struct printk_buffers.
2967	*
2968	* @seq is the record to read and format. If it is not available, the next
2969	* valid record is read.
2970	*
2971	* @is_extended specifies if the message should be formatted for extended
2972	* console output.
2973	*
2974	* @may_supress specifies if records may be skipped based on loglevel.
2975	*
2976	* Returns false if no record is available. Otherwise true and all fields
2977	* of @pmsg are valid. (See the documentation of struct printk_message
2978	* for information about the @pmsg fields.)
2979	*/
2980	bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
2981	bool is_extended, bool may_suppress)
2982	{
2983	struct printk_buffers *pbufs = pmsg->pbufs;
2984	const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
2985	const size_t outbuf_sz = sizeof(pbufs->outbuf);
2986	char *scratchbuf = &pbufs->scratchbuf[0];
2987	char *outbuf = &pbufs->outbuf[0];
2988	struct printk_info info;
2989	struct printk_record r;
2990	size_t len = 0;
2991	bool force_con;
2992
2993	/*
2994	* Formatting extended messages requires a separate buffer, so use the
2995	* scratch buffer to read in the ringbuffer text.
2996	*
2997	* Formatting normal messages is done in-place, so read the ringbuffer
2998	* text directly into the output buffer.
2999	*/
3000	if (is_extended)
3001	prb_rec_init_rd(r: &r, info: &info, text_buf: scratchbuf, text_buf_size: scratchbuf_sz);
3002	else
3003	prb_rec_init_rd(r: &r, info: &info, text_buf: outbuf, text_buf_size: outbuf_sz);
3004
3005	if (!prb_read_valid(rb: prb, seq, r: &r))
3006	return false;
3007
3008	pmsg->seq = r.info->seq;
3009	pmsg->dropped = r.info->seq - seq;
3010	force_con = r.info->flags & LOG_FORCE_CON;
3011
3012	/*
3013	* Skip records that are not forced to be printed on consoles and that
3014	* has level above the console loglevel.
3015	*/
3016	if (!force_con && may_suppress && suppress_message_printing(level: r.info->level))
3017	goto out;
3018
3019	if (is_extended) {
3020	len = info_print_ext_header(buf: outbuf, size: outbuf_sz, info: r.info);
3021	len += msg_print_ext_body(buf: outbuf + len, size: outbuf_sz - len,
3022	text: &r.text_buf[0], text_len: r.info->text_len, dev_info: &r.info->dev_info);
3023	} else {
3024	len = record_print_text(r: &r, syslog: console_msg_format & MSG_FORMAT_SYSLOG, time: printk_time);
3025	}
3026	out:
3027	pmsg->outbuf_len = len;
3028	return true;
3029	}
3030
3031	/*
3032	* Legacy console printing from printk() caller context does not respect
3033	* raw_spinlock/spinlock nesting. For !PREEMPT_RT the lockdep warning is a
3034	* false positive. For PREEMPT_RT the false positive condition does not
3035	* occur.
3036	*
3037	* This map is used to temporarily establish LD_WAIT_SLEEP context for the
3038	* console write() callback when legacy printing to avoid false positive
3039	* lockdep complaints, thus allowing lockdep to continue to function for
3040	* real issues.
3041	*/
3042	#ifdef CONFIG_PREEMPT_RT
3043	static inline void printk_legacy_allow_spinlock_enter(void) { }
3044	static inline void printk_legacy_allow_spinlock_exit(void) { }
3045	#else
3046	static DEFINE_WAIT_OVERRIDE_MAP(printk_legacy_map, LD_WAIT_SLEEP);
3047
3048	static inline void printk_legacy_allow_spinlock_enter(void)
3049	{
3050	lock_map_acquire_try(&printk_legacy_map);
3051	}
3052
3053	static inline void printk_legacy_allow_spinlock_exit(void)
3054	{
3055	lock_map_release(&printk_legacy_map);
3056	}
3057	#endif /* CONFIG_PREEMPT_RT */
3058
3059	/*
3060	* Used as the printk buffers for non-panic, serialized console printing.
3061	* This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.
3062	* Its usage requires the console_lock held.
3063	*/
3064	struct printk_buffers printk_shared_pbufs;
3065
3066	/*
3067	* Print one record for the given console. The record printed is whatever
3068	* record is the next available record for the given console.
3069	*
3070	* @handover will be set to true if a printk waiter has taken over the
3071	* console_lock, in which case the caller is no longer holding both the
3072	* console_lock and the SRCU read lock. Otherwise it is set to false.
3073	*
3074	* @cookie is the cookie from the SRCU read lock.
3075	*
3076	* Returns false if the given console has no next record to print, otherwise
3077	* true.
3078	*
3079	* Requires the console_lock and the SRCU read lock.
3080	*/
3081	static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
3082	{
3083	bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
3084	char *outbuf = &printk_shared_pbufs.outbuf[0];
3085	struct printk_message pmsg = {
3086	.pbufs = &printk_shared_pbufs,
3087	};
3088	unsigned long flags;
3089
3090	*handover = false;
3091
3092	if (!printk_get_next_message(pmsg: &pmsg, seq: con->seq, is_extended, may_suppress: true))
3093	return false;
3094
3095	con->dropped += pmsg.dropped;
3096
3097	/* Skip messages of formatted length 0. */
3098	if (pmsg.outbuf_len == 0) {
3099	con->seq = pmsg.seq + 1;
3100	goto skip;
3101	}
3102
3103	if (con->dropped && !is_extended) {
3104	console_prepend_dropped(pmsg: &pmsg, dropped: con->dropped);
3105	con->dropped = 0;
3106	}
3107
3108	/* Write everything out to the hardware. */
3109
3110	if (force_legacy_kthread() && !panic_in_progress()) {
3111	/*
3112	* With forced threading this function is in a task context
3113	* (either legacy kthread or get_init_console_seq()). There
3114	* is no need for concern about printk reentrance, handovers,
3115	* or lockdep complaints.
3116	*/
3117
3118	con->write(con, outbuf, pmsg.outbuf_len);
3119	con->seq = pmsg.seq + 1;
3120	} else {
3121	/*
3122	* While actively printing out messages, if another printk()
3123	* were to occur on another CPU, it may wait for this one to
3124	* finish. This task can not be preempted if there is a
3125	* waiter waiting to take over.
3126	*
3127	* Interrupts are disabled because the hand over to a waiter
3128	* must not be interrupted until the hand over is completed
3129	* (@console_waiter is cleared).
3130	*/
3131	printk_safe_enter_irqsave(flags);
3132	console_lock_spinning_enable();
3133
3134	/* Do not trace print latency. */
3135	stop_critical_timings();
3136
3137	printk_legacy_allow_spinlock_enter();
3138	con->write(con, outbuf, pmsg.outbuf_len);
3139	printk_legacy_allow_spinlock_exit();
3140
3141	start_critical_timings();
3142
3143	con->seq = pmsg.seq + 1;
3144
3145	*handover = console_lock_spinning_disable_and_check(cookie);
3146	printk_safe_exit_irqrestore(flags);
3147	}
3148	skip:
3149	return true;
3150	}
3151
3152	#else
3153
3154	static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
3155	{
3156	*handover = false;
3157	return false;
3158	}
3159
3160	static inline void printk_kthreads_check_locked(void) { }
3161
3162	#endif /* CONFIG_PRINTK */
3163
3164	/*
3165	* Print out all remaining records to all consoles.
3166	*
3167	* @do_cond_resched is set by the caller. It can be true only in schedulable
3168	* context.
3169	*
3170	* @next_seq is set to the sequence number after the last available record.
3171	* The value is valid only when this function returns true. It means that all
3172	* usable consoles are completely flushed.
3173	*
3174	* @handover will be set to true if a printk waiter has taken over the
3175	* console_lock, in which case the caller is no longer holding the
3176	* console_lock. Otherwise it is set to false.
3177	*
3178	* Returns true when there was at least one usable console and all messages
3179	* were flushed to all usable consoles. A returned false informs the caller
3180	* that everything was not flushed (either there were no usable consoles or
3181	* another context has taken over printing or it is a panic situation and this
3182	* is not the panic CPU). Regardless the reason, the caller should assume it
3183	* is not useful to immediately try again.
3184	*
3185	* Requires the console_lock.
3186	*/
3187	static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover)
3188	{
3189	struct console_flush_type ft;
3190	bool any_usable = false;
3191	struct console *con;
3192	bool any_progress;
3193	int cookie;
3194
3195	*next_seq = 0;
3196	*handover = false;
3197
3198	do {
3199	any_progress = false;
3200
3201	printk_get_console_flush_type(ft: &ft);
3202
3203	cookie = console_srcu_read_lock();
3204	for_each_console_srcu(con) {
3205	short flags = console_srcu_read_flags(con);
3206	u64 printk_seq;
3207	bool progress;
3208
3209	/*
3210	* console_flush_all() is only responsible for nbcon
3211	* consoles when the nbcon consoles cannot print via
3212	* their atomic or threaded flushing.
3213	*/
3214	if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload))
3215	continue;
3216
3217	if (!console_is_usable(con, flags, use_atomic: !do_cond_resched))
3218	continue;
3219	any_usable = true;
3220
3221	if (flags & CON_NBCON) {
3222	progress = nbcon_legacy_emit_next_record(con, handover, cookie,
3223	use_atomic: !do_cond_resched);
3224	printk_seq = nbcon_seq_read(con);
3225	} else {
3226	progress = console_emit_next_record(con, handover, cookie);
3227	printk_seq = con->seq;
3228	}
3229
3230	/*
3231	* If a handover has occurred, the SRCU read lock
3232	* is already released.
3233	*/
3234	if (*handover)
3235	return false;
3236
3237	/* Track the next of the highest seq flushed. */
3238	if (printk_seq > *next_seq)
3239	*next_seq = printk_seq;
3240
3241	if (!progress)
3242	continue;
3243	any_progress = true;
3244
3245	/* Allow panic_cpu to take over the consoles safely. */
3246	if (other_cpu_in_panic())
3247	goto abandon;
3248
3249	if (do_cond_resched)
3250	cond_resched();
3251	}
3252	console_srcu_read_unlock(cookie);
3253	} while (any_progress);
3254
3255	return any_usable;
3256
3257	abandon:
3258	console_srcu_read_unlock(cookie);
3259	return false;
3260	}
3261
3262	static void __console_flush_and_unlock(void)
3263	{
3264	bool do_cond_resched;
3265	bool handover;
3266	bool flushed;
3267	u64 next_seq;
3268
3269	/*
3270	* Console drivers are called with interrupts disabled, so
3271	* @console_may_schedule should be cleared before; however, we may
3272	* end up dumping a lot of lines, for example, if called from
3273	* console registration path, and should invoke cond_resched()
3274	* between lines if allowable. Not doing so can cause a very long
3275	* scheduling stall on a slow console leading to RCU stall and
3276	* softlockup warnings which exacerbate the issue with more
3277	* messages practically incapacitating the system. Therefore, create
3278	* a local to use for the printing loop.
3279	*/
3280	do_cond_resched = console_may_schedule;
3281
3282	do {
3283	console_may_schedule = 0;
3284
3285	flushed = console_flush_all(do_cond_resched, next_seq: &next_seq, handover: &handover);
3286	if (!handover)
3287	__console_unlock();
3288
3289	/*
3290	* Abort if there was a failure to flush all messages to all
3291	* usable consoles. Either it is not possible to flush (in
3292	* which case it would be an infinite loop of retrying) or
3293	* another context has taken over printing.
3294	*/
3295	if (!flushed)
3296	break;
3297
3298	/*
3299	* Some context may have added new records after
3300	* console_flush_all() but before unlocking the console.
3301	* Re-check if there is a new record to flush. If the trylock
3302	* fails, another context is already handling the printing.
3303	*/
3304	} while (prb_read_valid(rb: prb, seq: next_seq, NULL) && console_trylock());
3305	}
3306
3307	/**
3308	* console_unlock - unblock the legacy console subsystem from printing
3309	*
3310	* Releases the console_lock which the caller holds to block printing of
3311	* the legacy console subsystem.
3312	*
3313	* While the console_lock was held, console output may have been buffered
3314	* by printk(). If this is the case, console_unlock() emits the output on
3315	* legacy consoles prior to releasing the lock.
3316	*
3317	* console_unlock(); may be called from any context.
3318	*/
3319	void console_unlock(void)
3320	{
3321	struct console_flush_type ft;
3322
3323	printk_get_console_flush_type(ft: &ft);
3324	if (ft.legacy_direct)
3325	__console_flush_and_unlock();
3326	else
3327	__console_unlock();
3328	}
3329	EXPORT_SYMBOL(console_unlock);
3330
3331	/**
3332	* console_conditional_schedule - yield the CPU if required
3333	*
3334	* If the console code is currently allowed to sleep, and
3335	* if this CPU should yield the CPU to another task, do
3336	* so here.
3337	*
3338	* Must be called within console_lock();.
3339	*/
3340	void __sched console_conditional_schedule(void)
3341	{
3342	if (console_may_schedule)
3343	cond_resched();
3344	}
3345	EXPORT_SYMBOL(console_conditional_schedule);
3346
3347	void console_unblank(void)
3348	{
3349	bool found_unblank = false;
3350	struct console *c;
3351	int cookie;
3352
3353	/*
3354	* First check if there are any consoles implementing the unblank()
3355	* callback. If not, there is no reason to continue and take the
3356	* console lock, which in particular can be dangerous if
3357	* @oops_in_progress is set.
3358	*/
3359	cookie = console_srcu_read_lock();
3360	for_each_console_srcu(c) {
3361	short flags = console_srcu_read_flags(con: c);
3362
3363	if (flags & CON_SUSPENDED)
3364	continue;
3365
3366	if ((flags & CON_ENABLED) && c->unblank) {
3367	found_unblank = true;
3368	break;
3369	}
3370	}
3371	console_srcu_read_unlock(cookie);
3372	if (!found_unblank)
3373	return;
3374
3375	/*
3376	* Stop console printing because the unblank() callback may
3377	* assume the console is not within its write() callback.
3378	*
3379	* If @oops_in_progress is set, this may be an atomic context.
3380	* In that case, attempt a trylock as best-effort.
3381	*/
3382	if (oops_in_progress) {
3383	/* Semaphores are not NMI-safe. */
3384	if (in_nmi())
3385	return;
3386
3387	/*
3388	* Attempting to trylock the console lock can deadlock
3389	* if another CPU was stopped while modifying the
3390	* semaphore. "Hope and pray" that this is not the
3391	* current situation.
3392	*/
3393	if (down_trylock_console_sem() != 0)
3394	return;
3395	} else
3396	console_lock();
3397
3398	console_locked = 1;
3399	console_may_schedule = 0;
3400
3401	cookie = console_srcu_read_lock();
3402	for_each_console_srcu(c) {
3403	short flags = console_srcu_read_flags(con: c);
3404
3405	if (flags & CON_SUSPENDED)
3406	continue;
3407
3408	if ((flags & CON_ENABLED) && c->unblank)
3409	c->unblank();
3410	}
3411	console_srcu_read_unlock(cookie);
3412
3413	console_unlock();
3414
3415	if (!oops_in_progress)
3416	pr_flush(timeout_ms: 1000, reset_on_progress: true);
3417	}
3418
3419	/*
3420	* Rewind all consoles to the oldest available record.
3421	*
3422	* IMPORTANT: The function is safe only when called under
3423	* console_lock(). It is not enforced because
3424	* it is used as a best effort in panic().
3425	*/
3426	static void __console_rewind_all(void)
3427	{
3428	struct console *c;
3429	short flags;
3430	int cookie;
3431	u64 seq;
3432
3433	seq = prb_first_valid_seq(rb: prb);
3434
3435	cookie = console_srcu_read_lock();
3436	for_each_console_srcu(c) {
3437	flags = console_srcu_read_flags(con: c);
3438
3439	if (flags & CON_NBCON) {
3440	nbcon_seq_force(con: c, seq);
3441	} else {
3442	/*
3443	* This assignment is safe only when called under
3444	* console_lock(). On panic, legacy consoles are
3445	* only best effort.
3446	*/
3447	c->seq = seq;
3448	}
3449	}
3450	console_srcu_read_unlock(cookie);
3451	}
3452
3453	/**
3454	* console_flush_on_panic - flush console content on panic
3455	* @mode: flush all messages in buffer or just the pending ones
3456	*
3457	* Immediately output all pending messages no matter what.
3458	*/
3459	void console_flush_on_panic(enum con_flush_mode mode)
3460	{
3461	struct console_flush_type ft;
3462	bool handover;
3463	u64 next_seq;
3464
3465	/*
3466	* Ignore the console lock and flush out the messages. Attempting a
3467	* trylock would not be useful because:
3468	*
3469	* - if it is contended, it must be ignored anyway
3470	* - console_lock() and console_trylock() block and fail
3471	* respectively in panic for non-panic CPUs
3472	* - semaphores are not NMI-safe
3473	*/
3474
3475	/*
3476	* If another context is holding the console lock,
3477	* @console_may_schedule might be set. Clear it so that
3478	* this context does not call cond_resched() while flushing.
3479	*/
3480	console_may_schedule = 0;
3481
3482	if (mode == CONSOLE_REPLAY_ALL)
3483	__console_rewind_all();
3484
3485	printk_get_console_flush_type(ft: &ft);
3486	if (ft.nbcon_atomic)
3487	nbcon_atomic_flush_pending();
3488
3489	/* Flush legacy consoles once allowed, even when dangerous. */
3490	if (legacy_allow_panic_sync)
3491	console_flush_all(do_cond_resched: false, next_seq: &next_seq, handover: &handover);
3492	}
3493
3494	/*
3495	* Return the console tty driver structure and its associated index
3496	*/
3497	struct tty_driver *console_device(int *index)
3498	{
3499	struct console *c;
3500	struct tty_driver *driver = NULL;
3501	int cookie;
3502
3503	/*
3504	* Take console_lock to serialize device() callback with
3505	* other console operations. For example, fg_console is
3506	* modified under console_lock when switching vt.
3507	*/
3508	console_lock();
3509
3510	cookie = console_srcu_read_lock();
3511	for_each_console_srcu(c) {
3512	if (!c->device)
3513	continue;
3514	driver = c->device(c, index);
3515	if (driver)
3516	break;
3517	}
3518	console_srcu_read_unlock(cookie);
3519
3520	console_unlock();
3521	return driver;
3522	}
3523
3524	/*
3525	* Prevent further output on the passed console device so that (for example)
3526	* serial drivers can suspend console output before suspending a port, and can
3527	* re-enable output afterwards.
3528	*/
3529	void console_suspend(struct console *console)
3530	{
3531	__pr_flush(con: console, timeout_ms: 1000, reset_on_progress: true);
3532	console_list_lock();
3533	console_srcu_write_flags(con: console, flags: console->flags & ~CON_ENABLED);
3534	console_list_unlock();
3535
3536	/*
3537	* Ensure that all SRCU list walks have completed. All contexts must
3538	* be able to see that this console is disabled so that (for example)
3539	* the caller can suspend the port without risk of another context
3540	* using the port.
3541	*/
3542	synchronize_srcu(ssp: &console_srcu);
3543	}
3544	EXPORT_SYMBOL(console_suspend);
3545
3546	void console_resume(struct console *console)
3547	{
3548	struct console_flush_type ft;
3549	bool is_nbcon;
3550
3551	console_list_lock();
3552	console_srcu_write_flags(con: console, flags: console->flags | CON_ENABLED);
3553	is_nbcon = console->flags & CON_NBCON;
3554	console_list_unlock();
3555
3556	/*
3557	* Ensure that all SRCU list walks have completed. The related
3558	* printing context must be able to see it is enabled so that
3559	* it is guaranteed to wake up and resume printing.
3560	*/
3561	synchronize_srcu(ssp: &console_srcu);
3562
3563	printk_get_console_flush_type(ft: &ft);
3564	if (is_nbcon && ft.nbcon_offload)
3565	nbcon_kthread_wake(con: console);
3566	else if (ft.legacy_offload)
3567	defer_console_output();
3568
3569	__pr_flush(con: console, timeout_ms: 1000, reset_on_progress: true);
3570	}
3571	EXPORT_SYMBOL(console_resume);
3572
3573	#ifdef CONFIG_PRINTK
3574	static int unregister_console_locked(struct console *console);
3575
3576	/* True when system boot is far enough to create printer threads. */
3577	static bool printk_kthreads_ready __ro_after_init;
3578
3579	static struct task_struct *printk_legacy_kthread;
3580
3581	static bool legacy_kthread_should_wakeup(void)
3582	{
3583	struct console_flush_type ft;
3584	struct console *con;
3585	bool ret = false;
3586	int cookie;
3587
3588	if (kthread_should_stop())
3589	return true;
3590
3591	printk_get_console_flush_type(ft: &ft);
3592
3593	cookie = console_srcu_read_lock();
3594	for_each_console_srcu(con) {
3595	short flags = console_srcu_read_flags(con);
3596	u64 printk_seq;
3597
3598	/*
3599	* The legacy printer thread is only responsible for nbcon
3600	* consoles when the nbcon consoles cannot print via their
3601	* atomic or threaded flushing.
3602	*/
3603	if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload))
3604	continue;
3605
3606	if (!console_is_usable(con, flags, use_atomic: false))
3607	continue;
3608
3609	if (flags & CON_NBCON) {
3610	printk_seq = nbcon_seq_read(con);
3611	} else {
3612	/*
3613	* It is safe to read @seq because only this
3614	* thread context updates @seq.
3615	*/
3616	printk_seq = con->seq;
3617	}
3618
3619	if (prb_read_valid(rb: prb, seq: printk_seq, NULL)) {
3620	ret = true;
3621	break;
3622	}
3623	}
3624	console_srcu_read_unlock(cookie);
3625
3626	return ret;
3627	}
3628
3629	static int legacy_kthread_func(void *unused)
3630	{
3631	for (;;) {
3632	wait_event_interruptible(legacy_wait, legacy_kthread_should_wakeup());
3633
3634	if (kthread_should_stop())
3635	break;
3636
3637	console_lock();
3638	__console_flush_and_unlock();
3639	}
3640
3641	return 0;
3642	}
3643
3644	static bool legacy_kthread_create(void)
3645	{
3646	struct task_struct *kt;
3647
3648	lockdep_assert_console_list_lock_held();
3649
3650	kt = kthread_run(legacy_kthread_func, NULL, "pr/legacy");
3651	if (WARN_ON(IS_ERR(kt))) {
3652	pr_err("failed to start legacy printing thread\n");
3653	return false;
3654	}
3655
3656	printk_legacy_kthread = kt;
3657
3658	/*
3659	* It is important that console printing threads are scheduled
3660	* shortly after a printk call and with generous runtime budgets.
3661	*/
3662	sched_set_normal(p: printk_legacy_kthread, nice: -20);
3663
3664	return true;
3665	}
3666
3667	/**
3668	* printk_kthreads_shutdown - shutdown all threaded printers
3669	*
3670	* On system shutdown all threaded printers are stopped. This allows printk
3671	* to transition back to atomic printing, thus providing a robust mechanism
3672	* for the final shutdown/reboot messages to be output.
3673	*/
3674	static void printk_kthreads_shutdown(void)
3675	{
3676	struct console *con;
3677
3678	console_list_lock();
3679	if (printk_kthreads_running) {
3680	printk_kthreads_running = false;
3681
3682	for_each_console(con) {
3683	if (con->flags & CON_NBCON)
3684	nbcon_kthread_stop(con);
3685	}
3686
3687	/*
3688	* The threads may have been stopped while printing a
3689	* backlog. Flush any records left over.
3690	*/
3691	nbcon_atomic_flush_pending();
3692	}
3693	console_list_unlock();
3694	}
3695
3696	static struct syscore_ops printk_syscore_ops = {
3697	.shutdown = printk_kthreads_shutdown,
3698	};
3699
3700	/*
3701	* If appropriate, start nbcon kthreads and set @printk_kthreads_running.
3702	* If any kthreads fail to start, those consoles are unregistered.
3703	*
3704	* Must be called under console_list_lock().
3705	*/
3706	static void printk_kthreads_check_locked(void)
3707	{
3708	struct hlist_node *tmp;
3709	struct console *con;
3710
3711	lockdep_assert_console_list_lock_held();
3712
3713	if (!printk_kthreads_ready)
3714	return;
3715
3716	if (have_legacy_console || have_boot_console) {
3717	if (!printk_legacy_kthread &&
3718	force_legacy_kthread() &&
3719	!legacy_kthread_create()) {
3720	/*
3721	* All legacy consoles must be unregistered. If there
3722	* are any nbcon consoles, they will set up their own
3723	* kthread.
3724	*/
3725	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3726	if (con->flags & CON_NBCON)
3727	continue;
3728
3729	unregister_console_locked(console: con);
3730	}
3731	}
3732	} else if (printk_legacy_kthread) {
3733	kthread_stop(k: printk_legacy_kthread);
3734	printk_legacy_kthread = NULL;
3735	}
3736
3737	/*
3738	* Printer threads cannot be started as long as any boot console is
3739	* registered because there is no way to synchronize the hardware
3740	* registers between boot console code and regular console code.
3741	* It can only be known that there will be no new boot consoles when
3742	* an nbcon console is registered.
3743	*/
3744	if (have_boot_console || !have_nbcon_console) {
3745	/* Clear flag in case all nbcon consoles unregistered. */
3746	printk_kthreads_running = false;
3747	return;
3748	}
3749
3750	if (printk_kthreads_running)
3751	return;
3752
3753	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
3754	if (!(con->flags & CON_NBCON))
3755	continue;
3756
3757	if (!nbcon_kthread_create(con))
3758	unregister_console_locked(console: con);
3759	}
3760
3761	printk_kthreads_running = true;
3762	}
3763
3764	static int __init printk_set_kthreads_ready(void)
3765	{
3766	register_syscore_ops(ops: &printk_syscore_ops);
3767
3768	console_list_lock();
3769	printk_kthreads_ready = true;
3770	printk_kthreads_check_locked();
3771	console_list_unlock();
3772
3773	return 0;
3774	}
3775	early_initcall(printk_set_kthreads_ready);
3776	#endif /* CONFIG_PRINTK */
3777
3778	static int __read_mostly keep_bootcon;
3779
3780	static int __init keep_bootcon_setup(char *str)
3781	{
3782	keep_bootcon = 1;
3783	pr_info("debug: skip boot console de-registration.\n");
3784
3785	return 0;
3786	}
3787
3788	early_param("keep_bootcon", keep_bootcon_setup);
3789
3790	static int console_call_setup(struct console *newcon, char *options)
3791	{
3792	int err;
3793
3794	if (!newcon->setup)
3795	return 0;
3796
3797	/* Synchronize with possible boot console. */
3798	console_lock();
3799	err = newcon->setup(newcon, options);
3800	console_unlock();
3801
3802	return err;
3803	}
3804
3805	/*
3806	* This is called by register_console() to try to match
3807	* the newly registered console with any of the ones selected
3808	* by either the command line or add_preferred_console() and
3809	* setup/enable it.
3810	*
3811	* Care need to be taken with consoles that are statically
3812	* enabled such as netconsole
3813	*/
3814	static int try_enable_preferred_console(struct console *newcon,
3815	bool user_specified)
3816	{
3817	struct console_cmdline *c;
3818	int i, err;
3819
3820	for (i = 0, c = console_cmdline;
3821	i < MAX_CMDLINECONSOLES && (c->name[0] || c->devname[0]);
3822	i++, c++) {
3823	/* Console not yet initialized? */
3824	if (!c->name[0])
3825	continue;
3826	if (c->user_specified != user_specified)
3827	continue;
3828	if (!newcon->match ||
3829	newcon->match(newcon, c->name, c->index, c->options) != 0) {
3830	/* default matching */
3831	BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
3832	if (strcmp(c->name, newcon->name) != 0)
3833	continue;
3834	if (newcon->index >= 0 &&
3835	newcon->index != c->index)
3836	continue;
3837	if (newcon->index < 0)
3838	newcon->index = c->index;
3839
3840	if (_braille_register_console(console: newcon, c))
3841	return 0;
3842
3843	err = console_call_setup(newcon, options: c->options);
3844	if (err)
3845	return err;
3846	}
3847	newcon->flags |= CON_ENABLED;
3848	if (i == preferred_console)
3849	newcon->flags |= CON_CONSDEV;
3850	return 0;
3851	}
3852
3853	/*
3854	* Some consoles, such as pstore and netconsole, can be enabled even
3855	* without matching. Accept the pre-enabled consoles only when match()
3856	* and setup() had a chance to be called.
3857	*/
3858	if (newcon->flags & CON_ENABLED && c->user_specified == user_specified)
3859	return 0;
3860
3861	return -ENOENT;
3862	}
3863
3864	/* Try to enable the console unconditionally */
3865	static void try_enable_default_console(struct console *newcon)
3866	{
3867	if (newcon->index < 0)
3868	newcon->index = 0;
3869
3870	if (console_call_setup(newcon, NULL) != 0)
3871	return;
3872
3873	newcon->flags |= CON_ENABLED;
3874
3875	if (newcon->device)
3876	newcon->flags |= CON_CONSDEV;
3877	}
3878
3879	/* Return the starting sequence number for a newly registered console. */
3880	static u64 get_init_console_seq(struct console *newcon, bool bootcon_registered)
3881	{
3882	struct console *con;
3883	bool handover;
3884	u64 init_seq;
3885
3886	if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) {
3887	/* Get a consistent copy of @syslog_seq. */
3888	mutex_lock(&syslog_lock);
3889	init_seq = syslog_seq;
3890	mutex_unlock(lock: &syslog_lock);
3891	} else {
3892	/* Begin with next message added to ringbuffer. */
3893	init_seq = prb_next_seq(rb: prb);
3894
3895	/*
3896	* If any enabled boot consoles are due to be unregistered
3897	* shortly, some may not be caught up and may be the same
3898	* device as @newcon. Since it is not known which boot console
3899	* is the same device, flush all consoles and, if necessary,
3900	* start with the message of the enabled boot console that is
3901	* the furthest behind.
3902	*/
3903	if (bootcon_registered && !keep_bootcon) {
3904	/*
3905	* Hold the console_lock to stop console printing and
3906	* guarantee safe access to console->seq.
3907	*/
3908	console_lock();
3909
3910	/*
3911	* Flush all consoles and set the console to start at
3912	* the next unprinted sequence number.
3913	*/
3914	if (!console_flush_all(do_cond_resched: true, next_seq: &init_seq, handover: &handover)) {
3915	/*
3916	* Flushing failed. Just choose the lowest
3917	* sequence of the enabled boot consoles.
3918	*/
3919
3920	/*
3921	* If there was a handover, this context no
3922	* longer holds the console_lock.
3923	*/
3924	if (handover)
3925	console_lock();
3926
3927	init_seq = prb_next_seq(rb: prb);
3928	for_each_console(con) {
3929	u64 seq;
3930
3931	if (!(con->flags & CON_BOOT) ||
3932	!(con->flags & CON_ENABLED)) {
3933	continue;
3934	}
3935
3936	if (con->flags & CON_NBCON)
3937	seq = nbcon_seq_read(con);
3938	else
3939	seq = con->seq;
3940
3941	if (seq < init_seq)
3942	init_seq = seq;
3943	}
3944	}
3945
3946	console_unlock();
3947	}
3948	}
3949
3950	return init_seq;
3951	}
3952
3953	#define console_first() \
3954	hlist_entry(console_list.first, struct console, node)
3955
3956	static int unregister_console_locked(struct console *console);
3957
3958	/*
3959	* The console driver calls this routine during kernel initialization
3960	* to register the console printing procedure with printk() and to
3961	* print any messages that were printed by the kernel before the
3962	* console driver was initialized.
3963	*
3964	* This can happen pretty early during the boot process (because of
3965	* early_printk) - sometimes before setup_arch() completes - be careful
3966	* of what kernel features are used - they may not be initialised yet.
3967	*
3968	* There are two types of consoles - bootconsoles (early_printk) and
3969	* "real" consoles (everything which is not a bootconsole) which are
3970	* handled differently.
3971	* - Any number of bootconsoles can be registered at any time.
3972	* - As soon as a "real" console is registered, all bootconsoles
3973	* will be unregistered automatically.
3974	* - Once a "real" console is registered, any attempt to register a
3975	* bootconsoles will be rejected
3976	*/
3977	void register_console(struct console *newcon)
3978	{
3979	bool use_device_lock = (newcon->flags & CON_NBCON) && newcon->write_atomic;
3980	bool bootcon_registered = false;
3981	bool realcon_registered = false;
3982	struct console *con;
3983	unsigned long flags;
3984	u64 init_seq;
3985	int err;
3986
3987	console_list_lock();
3988
3989	for_each_console(con) {
3990	if (WARN(con == newcon, "console '%s%d' already registered\n",
3991	con->name, con->index)) {
3992	goto unlock;
3993	}
3994
3995	if (con->flags & CON_BOOT)
3996	bootcon_registered = true;
3997	else
3998	realcon_registered = true;
3999	}
4000
4001	/* Do not register boot consoles when there already is a real one. */
4002	if ((newcon->flags & CON_BOOT) && realcon_registered) {
4003	pr_info("Too late to register bootconsole %s%d\n",
4004	newcon->name, newcon->index);
4005	goto unlock;
4006	}
4007
4008	if (newcon->flags & CON_NBCON) {
4009	/*
4010	* Ensure the nbcon console buffers can be allocated
4011	* before modifying any global data.
4012	*/
4013	if (!nbcon_alloc(con: newcon))
4014	goto unlock;
4015	}
4016
4017	/*
4018	* See if we want to enable this console driver by default.
4019	*
4020	* Nope when a console is preferred by the command line, device
4021	* tree, or SPCR.
4022	*
4023	* The first real console with tty binding (driver) wins. More
4024	* consoles might get enabled before the right one is found.
4025	*
4026	* Note that a console with tty binding will have CON_CONSDEV
4027	* flag set and will be first in the list.
4028	*/
4029	if (preferred_console < 0) {
4030	if (hlist_empty(h: &console_list) || !console_first()->device ||
4031	console_first()->flags & CON_BOOT) {
4032	try_enable_default_console(newcon);
4033	}
4034	}
4035
4036	/* See if this console matches one we selected on the command line */
4037	err = try_enable_preferred_console(newcon, user_specified: true);
4038
4039	/* If not, try to match against the platform default(s) */
4040	if (err == -ENOENT)
4041	err = try_enable_preferred_console(newcon, user_specified: false);
4042
4043	/* printk() messages are not printed to the Braille console. */
4044	if (err || newcon->flags & CON_BRL) {
4045	if (newcon->flags & CON_NBCON)
4046	nbcon_free(con: newcon);
4047	goto unlock;
4048	}
4049
4050	/*
4051	* If we have a bootconsole, and are switching to a real console,
4052	* don't print everything out again, since when the boot console, and
4053	* the real console are the same physical device, it's annoying to
4054	* see the beginning boot messages twice
4055	*/
4056	if (bootcon_registered &&
4057	((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
4058	newcon->flags &= ~CON_PRINTBUFFER;
4059	}
4060
4061	newcon->dropped = 0;
4062	init_seq = get_init_console_seq(newcon, bootcon_registered);
4063
4064	if (newcon->flags & CON_NBCON) {
4065	have_nbcon_console = true;
4066	nbcon_seq_force(con: newcon, seq: init_seq);
4067	} else {
4068	have_legacy_console = true;
4069	newcon->seq = init_seq;
4070	}
4071
4072	if (newcon->flags & CON_BOOT)
4073	have_boot_console = true;
4074
4075	/*
4076	* If another context is actively using the hardware of this new
4077	* console, it will not be aware of the nbcon synchronization. This
4078	* is a risk that two contexts could access the hardware
4079	* simultaneously if this new console is used for atomic printing
4080	* and the other context is still using the hardware.
4081	*
4082	* Use the driver synchronization to ensure that the hardware is not
4083	* in use while this new console transitions to being registered.
4084	*/
4085	if (use_device_lock)
4086	newcon->device_lock(newcon, &flags);
4087
4088	/*
4089	* Put this console in the list - keep the
4090	* preferred driver at the head of the list.
4091	*/
4092	if (hlist_empty(h: &console_list)) {
4093	/* Ensure CON_CONSDEV is always set for the head. */
4094	newcon->flags |= CON_CONSDEV;
4095	hlist_add_head_rcu(n: &newcon->node, h: &console_list);
4096
4097	} else if (newcon->flags & CON_CONSDEV) {
4098	/* Only the new head can have CON_CONSDEV set. */
4099	console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV);
4100	hlist_add_head_rcu(n: &newcon->node, h: &console_list);
4101
4102	} else {
4103	hlist_add_behind_rcu(n: &newcon->node, prev: console_list.first);
4104	}
4105
4106	/*
4107	* No need to synchronize SRCU here! The caller does not rely
4108	* on all contexts being able to see the new console before
4109	* register_console() completes.
4110	*/
4111
4112	/* This new console is now registered. */
4113	if (use_device_lock)
4114	newcon->device_unlock(newcon, flags);
4115
4116	console_sysfs_notify();
4117
4118	/*
4119	* By unregistering the bootconsoles after we enable the real console
4120	* we get the "console xxx enabled" message on all the consoles -
4121	* boot consoles, real consoles, etc - this is to ensure that end
4122	* users know there might be something in the kernel's log buffer that
4123	* went to the bootconsole (that they do not see on the real console)
4124	*/
4125	con_printk(KERN_INFO, newcon, "enabled\n");
4126	if (bootcon_registered &&
4127	((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) &&
4128	!keep_bootcon) {
4129	struct hlist_node *tmp;
4130
4131	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
4132	if (con->flags & CON_BOOT)
4133	unregister_console_locked(console: con);
4134	}
4135	}
4136
4137	/* Changed console list, may require printer threads to start/stop. */
4138	printk_kthreads_check_locked();
4139	unlock:
4140	console_list_unlock();
4141	}
4142	EXPORT_SYMBOL(register_console);
4143
4144	/* Must be called under console_list_lock(). */
4145	static int unregister_console_locked(struct console *console)
4146	{
4147	bool use_device_lock = (console->flags & CON_NBCON) && console->write_atomic;
4148	bool found_legacy_con = false;
4149	bool found_nbcon_con = false;
4150	bool found_boot_con = false;
4151	unsigned long flags;
4152	struct console *c;
4153	int res;
4154
4155	lockdep_assert_console_list_lock_held();
4156
4157	con_printk(KERN_INFO, console, "disabled\n");
4158
4159	res = _braille_unregister_console(console);
4160	if (res < 0)
4161	return res;
4162	if (res > 0)
4163	return 0;
4164
4165	if (!console_is_registered_locked(con: console))
4166	res = -ENODEV;
4167	else if (console_is_usable(con: console, flags: console->flags, use_atomic: true))
4168	__pr_flush(con: console, timeout_ms: 1000, reset_on_progress: true);
4169
4170	/* Disable it unconditionally */
4171	console_srcu_write_flags(con: console, flags: console->flags & ~CON_ENABLED);
4172
4173	if (res < 0)
4174	return res;
4175
4176	/*
4177	* Use the driver synchronization to ensure that the hardware is not
4178	* in use while this console transitions to being unregistered.
4179	*/
4180	if (use_device_lock)
4181	console->device_lock(console, &flags);
4182
4183	hlist_del_init_rcu(n: &console->node);
4184
4185	if (use_device_lock)
4186	console->device_unlock(console, flags);
4187
4188	/*
4189	* <HISTORICAL>
4190	* If this isn't the last console and it has CON_CONSDEV set, we
4191	* need to set it on the next preferred console.
4192	* </HISTORICAL>
4193	*
4194	* The above makes no sense as there is no guarantee that the next
4195	* console has any device attached. Oh well....
4196	*/
4197	if (!hlist_empty(h: &console_list) && console->flags & CON_CONSDEV)
4198	console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV);
4199
4200	/*
4201	* Ensure that all SRCU list walks have completed. All contexts
4202	* must not be able to see this console in the list so that any
4203	* exit/cleanup routines can be performed safely.
4204	*/
4205	synchronize_srcu(ssp: &console_srcu);
4206
4207	if (console->flags & CON_NBCON)
4208	nbcon_free(con: console);
4209
4210	console_sysfs_notify();
4211
4212	if (console->exit)
4213	res = console->exit(console);
4214
4215	/*
4216	* With this console gone, the global flags tracking registered
4217	* console types may have changed. Update them.
4218	*/
4219	for_each_console(c) {
4220	if (c->flags & CON_BOOT)
4221	found_boot_con = true;
4222
4223	if (c->flags & CON_NBCON)
4224	found_nbcon_con = true;
4225	else
4226	found_legacy_con = true;
4227	}
4228	if (!found_boot_con)
4229	have_boot_console = found_boot_con;
4230	if (!found_legacy_con)
4231	have_legacy_console = found_legacy_con;
4232	if (!found_nbcon_con)
4233	have_nbcon_console = found_nbcon_con;
4234
4235	/* Changed console list, may require printer threads to start/stop. */
4236	printk_kthreads_check_locked();
4237
4238	return res;
4239	}
4240
4241	int unregister_console(struct console *console)
4242	{
4243	int res;
4244
4245	console_list_lock();
4246	res = unregister_console_locked(console);
4247	console_list_unlock();
4248	return res;
4249	}
4250	EXPORT_SYMBOL(unregister_console);
4251
4252	/**
4253	* console_force_preferred_locked - force a registered console preferred
4254	* @con: The registered console to force preferred.
4255	*
4256	* Must be called under console_list_lock().
4257	*/
4258	void console_force_preferred_locked(struct console *con)
4259	{
4260	struct console *cur_pref_con;
4261
4262	if (!console_is_registered_locked(con))
4263	return;
4264
4265	cur_pref_con = console_first();
4266
4267	/* Already preferred? */
4268	if (cur_pref_con == con)
4269	return;
4270
4271	/*
4272	* Delete, but do not re-initialize the entry. This allows the console
4273	* to continue to appear registered (via any hlist_unhashed_lockless()
4274	* checks), even though it was briefly removed from the console list.
4275	*/
4276	hlist_del_rcu(n: &con->node);
4277
4278	/*
4279	* Ensure that all SRCU list walks have completed so that the console
4280	* can be added to the beginning of the console list and its forward
4281	* list pointer can be re-initialized.
4282	*/
4283	synchronize_srcu(ssp: &console_srcu);
4284
4285	con->flags |= CON_CONSDEV;
4286	WARN_ON(!con->device);
4287
4288	/* Only the new head can have CON_CONSDEV set. */
4289	console_srcu_write_flags(con: cur_pref_con, flags: cur_pref_con->flags & ~CON_CONSDEV);
4290	hlist_add_head_rcu(n: &con->node, h: &console_list);
4291	}
4292	EXPORT_SYMBOL(console_force_preferred_locked);
4293
4294	/*
4295	* Initialize the console device. This is called *early*, so
4296	* we can't necessarily depend on lots of kernel help here.
4297	* Just do some early initializations, and do the complex setup
4298	* later.
4299	*/
4300	void __init console_init(void)
4301	{
4302	int ret;
4303	initcall_t call;
4304	initcall_entry_t *ce;
4305
4306	#ifdef CONFIG_NULL_TTY_DEFAULT_CONSOLE
4307	if (!console_set_on_cmdline)
4308	add_preferred_console("ttynull", 0, NULL);
4309	#endif
4310
4311	/* Setup the default TTY line discipline. */
4312	n_tty_init();
4313
4314	/*
4315	* set up the console device so that later boot sequences can
4316	* inform about problems etc..
4317	*/
4318	ce = __con_initcall_start;
4319	trace_initcall_level(level: "console");
4320	while (ce < __con_initcall_end) {
4321	call = initcall_from_entry(entry: ce);
4322	trace_initcall_start(func: call);
4323	ret = call();
4324	trace_initcall_finish(func: call, ret);
4325	ce++;
4326	}
4327	}
4328
4329	/*
4330	* Some boot consoles access data that is in the init section and which will
4331	* be discarded after the initcalls have been run. To make sure that no code
4332	* will access this data, unregister the boot consoles in a late initcall.
4333	*
4334	* If for some reason, such as deferred probe or the driver being a loadable
4335	* module, the real console hasn't registered yet at this point, there will
4336	* be a brief interval in which no messages are logged to the console, which
4337	* makes it difficult to diagnose problems that occur during this time.
4338	*
4339	* To mitigate this problem somewhat, only unregister consoles whose memory
4340	* intersects with the init section. Note that all other boot consoles will
4341	* get unregistered when the real preferred console is registered.
4342	*/
4343	static int __init printk_late_init(void)
4344	{
4345	struct hlist_node *tmp;
4346	struct console *con;
4347	int ret;
4348
4349	console_list_lock();
4350	hlist_for_each_entry_safe(con, tmp, &console_list, node) {
4351	if (!(con->flags & CON_BOOT))
4352	continue;
4353
4354	/* Check addresses that might be used for enabled consoles. */
4355	if (init_section_intersects(virt: con, size: sizeof(*con)) ||
4356	init_section_contains(virt: con->write, size: 0) ||
4357	init_section_contains(virt: con->read, size: 0) ||
4358	init_section_contains(virt: con->device, size: 0) ||
4359	init_section_contains(virt: con->unblank, size: 0) ||
4360	init_section_contains(virt: con->data, size: 0)) {
4361	/*
4362	* Please, consider moving the reported consoles out
4363	* of the init section.
4364	*/
4365	pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
4366	con->name, con->index);
4367	unregister_console_locked(console: con);
4368	}
4369	}
4370	console_list_unlock();
4371
4372	ret = cpuhp_setup_state_nocalls(state: CPUHP_PRINTK_DEAD, name: "printk:dead", NULL,
4373	teardown: console_cpu_notify);
4374	WARN_ON(ret < 0);
4375	ret = cpuhp_setup_state_nocalls(state: CPUHP_AP_ONLINE_DYN, name: "printk:online",
4376	startup: console_cpu_notify, NULL);
4377	WARN_ON(ret < 0);
4378	printk_sysctl_init();
4379	return 0;
4380	}
4381	late_initcall(printk_late_init);
4382
4383	#if defined CONFIG_PRINTK
4384	/* If @con is specified, only wait for that console. Otherwise wait for all. */
4385	static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress)
4386	{
4387	unsigned long timeout_jiffies = msecs_to_jiffies(m: timeout_ms);
4388	unsigned long remaining_jiffies = timeout_jiffies;
4389	struct console_flush_type ft;
4390	struct console *c;
4391	u64 last_diff = 0;
4392	u64 printk_seq;
4393	short flags;
4394	int cookie;
4395	u64 diff;
4396	u64 seq;
4397
4398	/* Sorry, pr_flush() will not work this early. */
4399	if (system_state < SYSTEM_SCHEDULING)
4400	return false;
4401
4402	might_sleep();
4403
4404	seq = prb_next_reserve_seq(rb: prb);
4405
4406	/* Flush the consoles so that records up to @seq are printed. */
4407	printk_get_console_flush_type(ft: &ft);
4408	if (ft.nbcon_atomic)
4409	nbcon_atomic_flush_pending();
4410	if (ft.legacy_direct) {
4411	console_lock();
4412	console_unlock();
4413	}
4414
4415	for (;;) {
4416	unsigned long begin_jiffies;
4417	unsigned long slept_jiffies;
4418
4419	diff = 0;
4420
4421	/*
4422	* Hold the console_lock to guarantee safe access to
4423	* console->seq. Releasing console_lock flushes more
4424	* records in case @seq is still not printed on all
4425	* usable consoles.
4426	*
4427	* Holding the console_lock is not necessary if there
4428	* are no legacy or boot consoles. However, such a
4429	* console could register at any time. Always hold the
4430	* console_lock as a precaution rather than
4431	* synchronizing against register_console().
4432	*/
4433	console_lock();
4434
4435	cookie = console_srcu_read_lock();
4436	for_each_console_srcu(c) {
4437	if (con && con != c)
4438	continue;
4439
4440	flags = console_srcu_read_flags(con: c);
4441
4442	/*
4443	* If consoles are not usable, it cannot be expected
4444	* that they make forward progress, so only increment
4445	* @diff for usable consoles.
4446	*/
4447	if (!console_is_usable(con: c, flags, use_atomic: true) &&
4448	!console_is_usable(con: c, flags, use_atomic: false)) {
4449	continue;
4450	}
4451
4452	if (flags & CON_NBCON) {
4453	printk_seq = nbcon_seq_read(con: c);
4454	} else {
4455	printk_seq = c->seq;
4456	}
4457
4458	if (printk_seq < seq)
4459	diff += seq - printk_seq;
4460	}
4461	console_srcu_read_unlock(cookie);
4462
4463	if (diff != last_diff && reset_on_progress)
4464	remaining_jiffies = timeout_jiffies;
4465
4466	console_unlock();
4467
4468	/* Note: @diff is 0 if there are no usable consoles. */
4469	if (diff == 0 || remaining_jiffies == 0)
4470	break;
4471
4472	/* msleep(1) might sleep much longer. Check time by jiffies. */
4473	begin_jiffies = jiffies;
4474	msleep(msecs: 1);
4475	slept_jiffies = jiffies - begin_jiffies;
4476
4477	remaining_jiffies -= min(slept_jiffies, remaining_jiffies);
4478
4479	last_diff = diff;
4480	}
4481
4482	return (diff == 0);
4483	}
4484
4485	/**
4486	* pr_flush() - Wait for printing threads to catch up.
4487	*
4488	* @timeout_ms: The maximum time (in ms) to wait.
4489	* @reset_on_progress: Reset the timeout if forward progress is seen.
4490	*
4491	* A value of 0 for @timeout_ms means no waiting will occur. A value of -1
4492	* represents infinite waiting.
4493	*
4494	* If @reset_on_progress is true, the timeout will be reset whenever any
4495	* printer has been seen to make some forward progress.
4496	*
4497	* Context: Process context. May sleep while acquiring console lock.
4498	* Return: true if all usable printers are caught up.
4499	*/
4500	bool pr_flush(int timeout_ms, bool reset_on_progress)
4501	{
4502	return __pr_flush(NULL, timeout_ms, reset_on_progress);
4503	}
4504
4505	/*
4506	* Delayed printk version, for scheduler-internal messages:
4507	*/
4508	#define PRINTK_PENDING_WAKEUP 0x01
4509	#define PRINTK_PENDING_OUTPUT 0x02
4510
4511	static DEFINE_PER_CPU(int, printk_pending);
4512
4513	static void wake_up_klogd_work_func(struct irq_work *irq_work)
4514	{
4515	int pending = this_cpu_xchg(printk_pending, 0);
4516
4517	if (pending & PRINTK_PENDING_OUTPUT) {
4518	if (force_legacy_kthread()) {
4519	if (printk_legacy_kthread)
4520	wake_up_interruptible(&legacy_wait);
4521	} else {
4522	if (console_trylock())
4523	console_unlock();
4524	}
4525	}
4526
4527	if (pending & PRINTK_PENDING_WAKEUP)
4528	wake_up_interruptible(&log_wait);
4529	}
4530
4531	static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
4532	IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);
4533
4534	static void __wake_up_klogd(int val)
4535	{
4536	if (!printk_percpu_data_ready())
4537	return;
4538
4539	preempt_disable();
4540	/*
4541	* Guarantee any new records can be seen by tasks preparing to wait
4542	* before this context checks if the wait queue is empty.
4543	*
4544	* The full memory barrier within wq_has_sleeper() pairs with the full
4545	* memory barrier within set_current_state() of
4546	* prepare_to_wait_event(), which is called after ___wait_event() adds
4547	* the waiter but before it has checked the wait condition.
4548	*
4549	* This pairs with devkmsg_read:A and syslog_print:A.
4550	*/
4551	if (wq_has_sleeper(wq_head: &log_wait) || /* LMM(__wake_up_klogd:A) */
4552	(val & PRINTK_PENDING_OUTPUT)) {
4553	this_cpu_or(printk_pending, val);
4554	irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
4555	}
4556	preempt_enable();
4557	}
4558
4559	/**
4560	* wake_up_klogd - Wake kernel logging daemon
4561	*
4562	* Use this function when new records have been added to the ringbuffer
4563	* and the console printing of those records has already occurred or is
4564	* known to be handled by some other context. This function will only
4565	* wake the logging daemon.
4566	*
4567	* Context: Any context.
4568	*/
4569	void wake_up_klogd(void)
4570	{
4571	__wake_up_klogd(PRINTK_PENDING_WAKEUP);
4572	}
4573
4574	/**
4575	* defer_console_output - Wake kernel logging daemon and trigger
4576	* console printing in a deferred context
4577	*
4578	* Use this function when new records have been added to the ringbuffer,
4579	* this context is responsible for console printing those records, but
4580	* the current context is not allowed to perform the console printing.
4581	* Trigger an irq_work context to perform the console printing. This
4582	* function also wakes the logging daemon.
4583	*
4584	* Context: Any context.
4585	*/
4586	void defer_console_output(void)
4587	{
4588	/*
4589	* New messages may have been added directly to the ringbuffer
4590	* using vprintk_store(), so wake any waiters as well.
4591	*/
4592	__wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
4593	}
4594
4595	void printk_trigger_flush(void)
4596	{
4597	defer_console_output();
4598	}
4599
4600	int vprintk_deferred(const char *fmt, va_list args)
4601	{
4602	return vprintk_emit(0, LOGLEVEL_SCHED, NULL, fmt, args);
4603	}
4604
4605	int _printk_deferred(const char *fmt, ...)
4606	{
4607	va_list args;
4608	int r;
4609
4610	va_start(args, fmt);
4611	r = vprintk_deferred(fmt, args);
4612	va_end(args);
4613
4614	return r;
4615	}
4616
4617	/*
4618	* printk rate limiting, lifted from the networking subsystem.
4619	*
4620	* This enforces a rate limit: not more than 10 kernel messages
4621	* every 5s to make a denial-of-service attack impossible.
4622	*/
4623	DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);
4624
4625	int __printk_ratelimit(const char *func)
4626	{
4627	return ___ratelimit(rs: &printk_ratelimit_state, func);
4628	}
4629	EXPORT_SYMBOL(__printk_ratelimit);
4630
4631	/**
4632	* printk_timed_ratelimit - caller-controlled printk ratelimiting
4633	* @caller_jiffies: pointer to caller's state
4634	* @interval_msecs: minimum interval between prints
4635	*
4636	* printk_timed_ratelimit() returns true if more than @interval_msecs
4637	* milliseconds have elapsed since the last time printk_timed_ratelimit()
4638	* returned true.
4639	*/
4640	bool printk_timed_ratelimit(unsigned long *caller_jiffies,
4641	unsigned int interval_msecs)
4642	{
4643	unsigned long elapsed = jiffies - *caller_jiffies;
4644
4645	if (*caller_jiffies && elapsed <= msecs_to_jiffies(m: interval_msecs))
4646	return false;
4647
4648	*caller_jiffies = jiffies;
4649	return true;
4650	}
4651	EXPORT_SYMBOL(printk_timed_ratelimit);
4652
4653	static DEFINE_SPINLOCK(dump_list_lock);
4654	static LIST_HEAD(dump_list);
4655
4656	/**
4657	* kmsg_dump_register - register a kernel log dumper.
4658	* @dumper: pointer to the kmsg_dumper structure
4659	*
4660	* Adds a kernel log dumper to the system. The dump callback in the
4661	* structure will be called when the kernel oopses or panics and must be
4662	* set. Returns zero on success and %-EINVAL or %-EBUSY otherwise.
4663	*/
4664	int kmsg_dump_register(struct kmsg_dumper *dumper)
4665	{
4666	unsigned long flags;
4667	int err = -EBUSY;
4668
4669	/* The dump callback needs to be set */
4670	if (!dumper->dump)
4671	return -EINVAL;
4672
4673	spin_lock_irqsave(&dump_list_lock, flags);
4674	/* Don't allow registering multiple times */
4675	if (!dumper->registered) {
4676	dumper->registered = 1;
4677	list_add_tail_rcu(new: &dumper->list, head: &dump_list);
4678	err = 0;
4679	}
4680	spin_unlock_irqrestore(lock: &dump_list_lock, flags);
4681
4682	return err;
4683	}
4684	EXPORT_SYMBOL_GPL(kmsg_dump_register);
4685
4686	/**
4687	* kmsg_dump_unregister - unregister a kmsg dumper.
4688	* @dumper: pointer to the kmsg_dumper structure
4689	*
4690	* Removes a dump device from the system. Returns zero on success and
4691	* %-EINVAL otherwise.
4692	*/
4693	int kmsg_dump_unregister(struct kmsg_dumper *dumper)
4694	{
4695	unsigned long flags;
4696	int err = -EINVAL;
4697
4698	spin_lock_irqsave(&dump_list_lock, flags);
4699	if (dumper->registered) {
4700	dumper->registered = 0;
4701	list_del_rcu(entry: &dumper->list);
4702	err = 0;
4703	}
4704	spin_unlock_irqrestore(lock: &dump_list_lock, flags);
4705	synchronize_rcu();
4706
4707	return err;
4708	}
4709	EXPORT_SYMBOL_GPL(kmsg_dump_unregister);
4710
4711	static bool always_kmsg_dump;
4712	module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR);
4713
4714	const char *kmsg_dump_reason_str(enum kmsg_dump_reason reason)
4715	{
4716	switch (reason) {
4717	case KMSG_DUMP_PANIC:
4718	return "Panic";
4719	case KMSG_DUMP_OOPS:
4720	return "Oops";
4721	case KMSG_DUMP_EMERG:
4722	return "Emergency";
4723	case KMSG_DUMP_SHUTDOWN:
4724	return "Shutdown";
4725	default:
4726	return "Unknown";
4727	}
4728	}
4729	EXPORT_SYMBOL_GPL(kmsg_dump_reason_str);
4730
4731	/**
4732	* kmsg_dump_desc - dump kernel log to kernel message dumpers.
4733	* @reason: the reason (oops, panic etc) for dumping
4734	* @desc: a short string to describe what caused the panic or oops. Can be NULL
4735	* if no additional description is available.
4736	*
4737	* Call each of the registered dumper's dump() callback, which can
4738	* retrieve the kmsg records with kmsg_dump_get_line() or
4739	* kmsg_dump_get_buffer().
4740	*/
4741	void kmsg_dump_desc(enum kmsg_dump_reason reason, const char *desc)
4742	{
4743	struct kmsg_dumper *dumper;
4744	struct kmsg_dump_detail detail = {
4745	.reason = reason,
4746	.description = desc};
4747
4748	rcu_read_lock();
4749	list_for_each_entry_rcu(dumper, &dump_list, list) {
4750	enum kmsg_dump_reason max_reason = dumper->max_reason;
4751
4752	/*
4753	* If client has not provided a specific max_reason, default
4754	* to KMSG_DUMP_OOPS, unless always_kmsg_dump was set.
4755	*/
4756	if (max_reason == KMSG_DUMP_UNDEF) {
4757	max_reason = always_kmsg_dump ? KMSG_DUMP_MAX :
4758	KMSG_DUMP_OOPS;
4759	}
4760	if (reason > max_reason)
4761	continue;
4762
4763	/* invoke dumper which will iterate over records */
4764	dumper->dump(dumper, &detail);
4765	}
4766	rcu_read_unlock();
4767	}
4768
4769	/**
4770	* kmsg_dump_get_line - retrieve one kmsg log line
4771	* @iter: kmsg dump iterator
4772	* @syslog: include the "<4>" prefixes
4773	* @line: buffer to copy the line to
4774	* @size: maximum size of the buffer
4775	* @len: length of line placed into buffer
4776	*
4777	* Start at the beginning of the kmsg buffer, with the oldest kmsg
4778	* record, and copy one record into the provided buffer.
4779	*
4780	* Consecutive calls will return the next available record moving
4781	* towards the end of the buffer with the youngest messages.
4782	*
4783	* A return value of FALSE indicates that there are no more records to
4784	* read.
4785	*/
4786	bool kmsg_dump_get_line(struct kmsg_dump_iter *iter, bool syslog,
4787	char *line, size_t size, size_t *len)
4788	{
4789	u64 min_seq = latched_seq_read_nolock(ls: &clear_seq);
4790	struct printk_info info;
4791	unsigned int line_count;
4792	struct printk_record r;
4793	size_t l = 0;
4794	bool ret = false;
4795
4796	if (iter->cur_seq < min_seq)
4797	iter->cur_seq = min_seq;
4798
4799	prb_rec_init_rd(r: &r, info: &info, text_buf: line, text_buf_size: size);
4800
4801	/* Read text or count text lines? */
4802	if (line) {
4803	if (!prb_read_valid(rb: prb, seq: iter->cur_seq, r: &r))
4804	goto out;
4805	l = record_print_text(r: &r, syslog, time: printk_time);
4806	} else {
4807	if (!prb_read_valid_info(rb: prb, seq: iter->cur_seq,
4808	info: &info, line_count: &line_count)) {
4809	goto out;
4810	}
4811	l = get_record_print_text_size(info: &info, line_count, syslog,
4812	time: printk_time);
4813
4814	}
4815
4816	iter->cur_seq = r.info->seq + 1;
4817	ret = true;
4818	out:
4819	if (len)
4820	*len = l;
4821	return ret;
4822	}
4823	EXPORT_SYMBOL_GPL(kmsg_dump_get_line);
4824
4825	/**
4826	* kmsg_dump_get_buffer - copy kmsg log lines
4827	* @iter: kmsg dump iterator
4828	* @syslog: include the "<4>" prefixes
4829	* @buf: buffer to copy the line to
4830	* @size: maximum size of the buffer
4831	* @len_out: length of line placed into buffer
4832	*
4833	* Start at the end of the kmsg buffer and fill the provided buffer
4834	* with as many of the *youngest* kmsg records that fit into it.
4835	* If the buffer is large enough, all available kmsg records will be
4836	* copied with a single call.
4837	*
4838	* Consecutive calls will fill the buffer with the next block of
4839	* available older records, not including the earlier retrieved ones.
4840	*
4841	* A return value of FALSE indicates that there are no more records to
4842	* read.
4843	*/
4844	bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog,
4845	char *buf, size_t size, size_t *len_out)
4846	{
4847	u64 min_seq = latched_seq_read_nolock(ls: &clear_seq);
4848	struct printk_info info;
4849	struct printk_record r;
4850	u64 seq;
4851	u64 next_seq;
4852	size_t len = 0;
4853	bool ret = false;
4854	bool time = printk_time;
4855
4856	if (!buf || !size)
4857	goto out;
4858
4859	if (iter->cur_seq < min_seq)
4860	iter->cur_seq = min_seq;
4861
4862	if (prb_read_valid_info(rb: prb, seq: iter->cur_seq, info: &info, NULL)) {
4863	if (info.seq != iter->cur_seq) {
4864	/* messages are gone, move to first available one */
4865	iter->cur_seq = info.seq;
4866	}
4867	}
4868
4869	/* last entry */
4870	if (iter->cur_seq >= iter->next_seq)
4871	goto out;
4872
4873	/*
4874	* Find first record that fits, including all following records,
4875	* into the user-provided buffer for this dump. Pass in size-1
4876	* because this function (by way of record_print_text()) will
4877	* not write more than size-1 bytes of text into @buf.
4878	*/
4879	seq = find_first_fitting_seq(start_seq: iter->cur_seq, max_seq: iter->next_seq,
4880	size: size - 1, syslog, time);
4881
4882	/*
4883	* Next kmsg_dump_get_buffer() invocation will dump block of
4884	* older records stored right before this one.
4885	*/
4886	next_seq = seq;
4887
4888	prb_rec_init_rd(r: &r, info: &info, text_buf: buf, text_buf_size: size);
4889
4890	prb_for_each_record(seq, prb, seq, &r) {
4891	if (r.info->seq >= iter->next_seq)
4892	break;
4893
4894	len += record_print_text(r: &r, syslog, time);
4895
4896	/* Adjust record to store to remaining buffer space. */
4897	prb_rec_init_rd(r: &r, info: &info, text_buf: buf + len, text_buf_size: size - len);
4898	}
4899
4900	iter->next_seq = next_seq;
4901	ret = true;
4902	out:
4903	if (len_out)
4904	*len_out = len;
4905	return ret;
4906	}
4907	EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer);
4908
4909	/**
4910	* kmsg_dump_rewind - reset the iterator
4911	* @iter: kmsg dump iterator
4912	*
4913	* Reset the dumper's iterator so that kmsg_dump_get_line() and
4914	* kmsg_dump_get_buffer() can be called again and used multiple
4915	* times within the same dumper.dump() callback.
4916	*/
4917	void kmsg_dump_rewind(struct kmsg_dump_iter *iter)
4918	{
4919	iter->cur_seq = latched_seq_read_nolock(ls: &clear_seq);
4920	iter->next_seq = prb_next_seq(rb: prb);
4921	}
4922	EXPORT_SYMBOL_GPL(kmsg_dump_rewind);
4923
4924	/**
4925	* console_try_replay_all - try to replay kernel log on consoles
4926	*
4927	* Try to obtain lock on console subsystem and replay all
4928	* available records in printk buffer on the consoles.
4929	* Does nothing if lock is not obtained.
4930	*
4931	* Context: Any, except for NMI.
4932	*/
4933	void console_try_replay_all(void)
4934	{
4935	struct console_flush_type ft;
4936
4937	printk_get_console_flush_type(ft: &ft);
4938	if (console_trylock()) {
4939	__console_rewind_all();
4940	if (ft.nbcon_atomic)
4941	nbcon_atomic_flush_pending();
4942	if (ft.nbcon_offload)
4943	nbcon_kthreads_wake();
4944	if (ft.legacy_offload)
4945	defer_console_output();
4946	/* Consoles are flushed as part of console_unlock(). */
4947	console_unlock();
4948	}
4949	}
4950	#endif
4951
4952	#ifdef CONFIG_SMP
4953	static atomic_t printk_cpu_sync_owner = ATOMIC_INIT(-1);
4954	static atomic_t printk_cpu_sync_nested = ATOMIC_INIT(0);
4955
4956	bool is_printk_cpu_sync_owner(void)
4957	{
4958	return (atomic_read(v: &printk_cpu_sync_owner) == raw_smp_processor_id());
4959	}
4960
4961	/**
4962	* __printk_cpu_sync_wait() - Busy wait until the printk cpu-reentrant
4963	* spinning lock is not owned by any CPU.
4964	*
4965	* Context: Any context.
4966	*/
4967	void __printk_cpu_sync_wait(void)
4968	{
4969	do {
4970	cpu_relax();
4971	} while (atomic_read(v: &printk_cpu_sync_owner) != -1);
4972	}
4973	EXPORT_SYMBOL(__printk_cpu_sync_wait);
4974
4975	/**
4976	* __printk_cpu_sync_try_get() - Try to acquire the printk cpu-reentrant
4977	* spinning lock.
4978	*
4979	* If no processor has the lock, the calling processor takes the lock and
4980	* becomes the owner. If the calling processor is already the owner of the
4981	* lock, this function succeeds immediately.
4982	*
4983	* Context: Any context. Expects interrupts to be disabled.
4984	* Return: 1 on success, otherwise 0.
4985	*/
4986	int __printk_cpu_sync_try_get(void)
4987	{
4988	int cpu;
4989	int old;
4990
4991	cpu = smp_processor_id();
4992
4993	/*
4994	* Guarantee loads and stores from this CPU when it is the lock owner
4995	* are _not_ visible to the previous lock owner. This pairs with
4996	* __printk_cpu_sync_put:B.
4997	*
4998	* Memory barrier involvement:
4999	*
5000	* If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
5001	* then __printk_cpu_sync_put:A can never read from
5002	* __printk_cpu_sync_try_get:B.
5003	*
5004	* Relies on:
5005	*
5006	* RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
5007	* of the previous CPU
5008	* matching
5009	* ACQUIRE from __printk_cpu_sync_try_get:A to
5010	* __printk_cpu_sync_try_get:B of this CPU
5011	*/
5012	old = atomic_cmpxchg_acquire(v: &printk_cpu_sync_owner, old: -1,
5013	new: cpu); /* LMM(__printk_cpu_sync_try_get:A) */
5014	if (old == -1) {
5015	/*
5016	* This CPU is now the owner and begins loading/storing
5017	* data: LMM(__printk_cpu_sync_try_get:B)
5018	*/
5019	return 1;
5020
5021	} else if (old == cpu) {
5022	/* This CPU is already the owner. */
5023	atomic_inc(v: &printk_cpu_sync_nested);
5024	return 1;
5025	}
5026
5027	return 0;
5028	}
5029	EXPORT_SYMBOL(__printk_cpu_sync_try_get);
5030
5031	/**
5032	* __printk_cpu_sync_put() - Release the printk cpu-reentrant spinning lock.
5033	*
5034	* The calling processor must be the owner of the lock.
5035	*
5036	* Context: Any context. Expects interrupts to be disabled.
5037	*/
5038	void __printk_cpu_sync_put(void)
5039	{
5040	if (atomic_read(v: &printk_cpu_sync_nested)) {
5041	atomic_dec(v: &printk_cpu_sync_nested);
5042	return;
5043	}
5044
5045	/*
5046	* This CPU is finished loading/storing data:
5047	* LMM(__printk_cpu_sync_put:A)
5048	*/
5049
5050	/*
5051	* Guarantee loads and stores from this CPU when it was the
5052	* lock owner are visible to the next lock owner. This pairs
5053	* with __printk_cpu_sync_try_get:A.
5054	*
5055	* Memory barrier involvement:
5056	*
5057	* If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
5058	* then __printk_cpu_sync_try_get:B reads from __printk_cpu_sync_put:A.
5059	*
5060	* Relies on:
5061	*
5062	* RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
5063	* of this CPU
5064	* matching
5065	* ACQUIRE from __printk_cpu_sync_try_get:A to
5066	* __printk_cpu_sync_try_get:B of the next CPU
5067	*/
5068	atomic_set_release(v: &printk_cpu_sync_owner,
5069	i: -1); /* LMM(__printk_cpu_sync_put:B) */
5070	}
5071	EXPORT_SYMBOL(__printk_cpu_sync_put);
5072	#endif /* CONFIG_SMP */
5073