1	// SPDX-License-Identifier: GPL-2.0
2
3	#include <linux/anon_inodes.h>
4	#include <linux/atomic.h>
5	#include <linux/bitmap.h>
6	#include <linux/build_bug.h>
7	#include <linux/cdev.h>
8	#include <linux/cleanup.h>
9	#include <linux/compat.h>
10	#include <linux/compiler.h>
11	#include <linux/device.h>
12	#include <linux/err.h>
13	#include <linux/file.h>
14	#include <linux/gpio.h>
15	#include <linux/gpio/driver.h>
16	#include <linux/hte.h>
17	#include <linux/interrupt.h>
18	#include <linux/irqreturn.h>
19	#include <linux/kernel.h>
20	#include <linux/kfifo.h>
21	#include <linux/module.h>
22	#include <linux/mutex.h>
23	#include <linux/overflow.h>
24	#include <linux/pinctrl/consumer.h>
25	#include <linux/poll.h>
26	#include <linux/rbtree.h>
27	#include <linux/seq_file.h>
28	#include <linux/spinlock.h>
29	#include <linux/timekeeping.h>
30	#include <linux/uaccess.h>
31	#include <linux/workqueue.h>
32
33	#include <uapi/linux/gpio.h>
34
35	#include "gpiolib.h"
36	#include "gpiolib-cdev.h"
37
38	/*
39	* Array sizes must ensure 64-bit alignment and not create holes in the
40	* struct packing.
41	*/
42	static_assert(IS_ALIGNED(GPIO_V2_LINES_MAX, 2));
43	static_assert(IS_ALIGNED(GPIO_MAX_NAME_SIZE, 8));
44
45	/*
46	* Check that uAPI structs are 64-bit aligned for 32/64-bit compatibility
47	*/
48	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_attribute), 8));
49	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_config_attribute), 8));
50	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_config), 8));
51	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_request), 8));
52	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_info), 8));
53	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_info_changed), 8));
54	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_event), 8));
55	static_assert(IS_ALIGNED(sizeof(struct gpio_v2_line_values), 8));
56
57	/* Character device interface to GPIO.
58	*
59	* The GPIO character device, /dev/gpiochipN, provides userspace an
60	* interface to gpiolib GPIOs via ioctl()s.
61	*/
62
63	/*
64	* GPIO line handle management
65	*/
66
67	#ifdef CONFIG_GPIO_CDEV_V1
68	/**
69	* struct linehandle_state - contains the state of a userspace handle
70	* @gdev: the GPIO device the handle pertains to
71	* @label: consumer label used to tag descriptors
72	* @descs: the GPIO descriptors held by this handle
73	* @num_descs: the number of descriptors held in the descs array
74	*/
75	struct linehandle_state {
76	struct gpio_device *gdev;
77	const char *label;
78	struct gpio_desc *descs[GPIOHANDLES_MAX];
79	u32 num_descs;
80	};
81
82	#define GPIOHANDLE_REQUEST_VALID_FLAGS \
83	(GPIOHANDLE_REQUEST_INPUT | \
84	GPIOHANDLE_REQUEST_OUTPUT | \
85	GPIOHANDLE_REQUEST_ACTIVE_LOW | \
86	GPIOHANDLE_REQUEST_BIAS_PULL_UP | \
87	GPIOHANDLE_REQUEST_BIAS_PULL_DOWN | \
88	GPIOHANDLE_REQUEST_BIAS_DISABLE | \
89	GPIOHANDLE_REQUEST_OPEN_DRAIN | \
90	GPIOHANDLE_REQUEST_OPEN_SOURCE)
91
92	static int linehandle_validate_flags(u32 flags)
93	{
94	/* Return an error if an unknown flag is set */
95	if (flags & ~GPIOHANDLE_REQUEST_VALID_FLAGS)
96	return -EINVAL;
97
98	/*
99	* Do not allow both INPUT & OUTPUT flags to be set as they are
100	* contradictory.
101	*/
102	if ((flags & GPIOHANDLE_REQUEST_INPUT) &&
103	(flags & GPIOHANDLE_REQUEST_OUTPUT))
104	return -EINVAL;
105
106	/*
107	* Do not allow OPEN_SOURCE & OPEN_DRAIN flags in a single request. If
108	* the hardware actually supports enabling both at the same time the
109	* electrical result would be disastrous.
110	*/
111	if ((flags & GPIOHANDLE_REQUEST_OPEN_DRAIN) &&
112	(flags & GPIOHANDLE_REQUEST_OPEN_SOURCE))
113	return -EINVAL;
114
115	/* OPEN_DRAIN and OPEN_SOURCE flags only make sense for output mode. */
116	if (!(flags & GPIOHANDLE_REQUEST_OUTPUT) &&
117	((flags & GPIOHANDLE_REQUEST_OPEN_DRAIN) ||
118	(flags & GPIOHANDLE_REQUEST_OPEN_SOURCE)))
119	return -EINVAL;
120
121	/* Bias flags only allowed for input or output mode. */
122	if (!((flags & GPIOHANDLE_REQUEST_INPUT) ||
123	(flags & GPIOHANDLE_REQUEST_OUTPUT)) &&
124	((flags & GPIOHANDLE_REQUEST_BIAS_DISABLE) ||
125	(flags & GPIOHANDLE_REQUEST_BIAS_PULL_UP) ||
126	(flags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN)))
127	return -EINVAL;
128
129	/* Only one bias flag can be set. */
130	if (((flags & GPIOHANDLE_REQUEST_BIAS_DISABLE) &&
131	(flags & (GPIOHANDLE_REQUEST_BIAS_PULL_DOWN |
132	GPIOHANDLE_REQUEST_BIAS_PULL_UP))) ||
133	((flags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN) &&
134	(flags & GPIOHANDLE_REQUEST_BIAS_PULL_UP)))
135	return -EINVAL;
136
137	return 0;
138	}
139
140	static void linehandle_flags_to_desc_flags(u32 lflags, unsigned long *flagsp)
141	{
142	assign_bit(FLAG_ACTIVE_LOW, addr: flagsp,
143	value: lflags & GPIOHANDLE_REQUEST_ACTIVE_LOW);
144	assign_bit(FLAG_OPEN_DRAIN, addr: flagsp,
145	value: lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN);
146	assign_bit(FLAG_OPEN_SOURCE, addr: flagsp,
147	value: lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE);
148	assign_bit(FLAG_PULL_UP, addr: flagsp,
149	value: lflags & GPIOHANDLE_REQUEST_BIAS_PULL_UP);
150	assign_bit(FLAG_PULL_DOWN, addr: flagsp,
151	value: lflags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN);
152	assign_bit(FLAG_BIAS_DISABLE, addr: flagsp,
153	value: lflags & GPIOHANDLE_REQUEST_BIAS_DISABLE);
154	}
155
156	static long linehandle_set_config(struct linehandle_state *lh,
157	void __user *ip)
158	{
159	struct gpiohandle_config gcnf;
160	struct gpio_desc *desc;
161	int i, ret;
162	u32 lflags;
163
164	if (copy_from_user(to: &gcnf, from: ip, n: sizeof(gcnf)))
165	return -EFAULT;
166
167	lflags = gcnf.flags;
168	ret = linehandle_validate_flags(flags: lflags);
169	if (ret)
170	return ret;
171
172	for (i = 0; i < lh->num_descs; i++) {
173	desc = lh->descs[i];
174	linehandle_flags_to_desc_flags(lflags: gcnf.flags, flagsp: &desc->flags);
175
176	/*
177	* Lines have to be requested explicitly for input
178	* or output, else the line will be treated "as is".
179	*/
180	if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
181	int val = !!gcnf.default_values[i];
182
183	ret = gpiod_direction_output(desc, value: val);
184	if (ret)
185	return ret;
186	} else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
187	ret = gpiod_direction_input(desc);
188	if (ret)
189	return ret;
190	}
191
192	gpiod_line_state_notify(desc, action: GPIO_V2_LINE_CHANGED_CONFIG);
193	}
194	return 0;
195	}
196
197	static long linehandle_ioctl(struct file *file, unsigned int cmd,
198	unsigned long arg)
199	{
200	struct linehandle_state *lh = file->private_data;
201	void __user *ip = (void __user *)arg;
202	struct gpiohandle_data ghd;
203	DECLARE_BITMAP(vals, GPIOHANDLES_MAX);
204	unsigned int i;
205	int ret;
206
207	guard(srcu)(l: &lh->gdev->srcu);
208
209	if (!rcu_access_pointer(lh->gdev->chip))
210	return -ENODEV;
211
212	switch (cmd) {
213	case GPIOHANDLE_GET_LINE_VALUES_IOCTL:
214	/* NOTE: It's okay to read values of output lines */
215	ret = gpiod_get_array_value_complex(raw: false, can_sleep: true,
216	array_size: lh->num_descs, desc_array: lh->descs,
217	NULL, value_bitmap: vals);
218	if (ret)
219	return ret;
220
221	memset(&ghd, 0, sizeof(ghd));
222	for (i = 0; i < lh->num_descs; i++)
223	ghd.values[i] = test_bit(i, vals);
224
225	if (copy_to_user(to: ip, from: &ghd, n: sizeof(ghd)))
226	return -EFAULT;
227
228	return 0;
229	case GPIOHANDLE_SET_LINE_VALUES_IOCTL:
230	/*
231	* All line descriptors were created at once with the same
232	* flags so just check if the first one is really output.
233	*/
234	if (!test_bit(FLAG_IS_OUT, &lh->descs[0]->flags))
235	return -EPERM;
236
237	if (copy_from_user(to: &ghd, from: ip, n: sizeof(ghd)))
238	return -EFAULT;
239
240	/* Clamp all values to [0,1] */
241	for (i = 0; i < lh->num_descs; i++)
242	__assign_bit(nr: i, addr: vals, value: ghd.values[i]);
243
244	/* Reuse the array setting function */
245	return gpiod_set_array_value_complex(raw: false,
246	can_sleep: true,
247	array_size: lh->num_descs,
248	desc_array: lh->descs,
249	NULL,
250	value_bitmap: vals);
251	case GPIOHANDLE_SET_CONFIG_IOCTL:
252	return linehandle_set_config(lh, ip);
253	default:
254	return -EINVAL;
255	}
256	}
257
258	#ifdef CONFIG_COMPAT
259	static long linehandle_ioctl_compat(struct file *file, unsigned int cmd,
260	unsigned long arg)
261	{
262	return linehandle_ioctl(file, cmd, arg: (unsigned long)compat_ptr(uptr: arg));
263	}
264	#endif
265
266	static void linehandle_free(struct linehandle_state *lh)
267	{
268	int i;
269
270	for (i = 0; i < lh->num_descs; i++)
271	if (lh->descs[i])
272	gpiod_free(desc: lh->descs[i]);
273	kfree(objp: lh->label);
274	gpio_device_put(gdev: lh->gdev);
275	kfree(objp: lh);
276	}
277
278	static int linehandle_release(struct inode *inode, struct file *file)
279	{
280	linehandle_free(lh: file->private_data);
281	return 0;
282	}
283
284	static const struct file_operations linehandle_fileops = {
285	.release = linehandle_release,
286	.owner = THIS_MODULE,
287	.llseek = noop_llseek,
288	.unlocked_ioctl = linehandle_ioctl,
289	#ifdef CONFIG_COMPAT
290	.compat_ioctl = linehandle_ioctl_compat,
291	#endif
292	};
293
294	static int linehandle_create(struct gpio_device *gdev, void __user *ip)
295	{
296	struct gpiohandle_request handlereq;
297	struct linehandle_state *lh;
298	struct file *file;
299	int fd, i, ret;
300	u32 lflags;
301
302	if (copy_from_user(to: &handlereq, from: ip, n: sizeof(handlereq)))
303	return -EFAULT;
304	if ((handlereq.lines == 0) || (handlereq.lines > GPIOHANDLES_MAX))
305	return -EINVAL;
306
307	lflags = handlereq.flags;
308
309	ret = linehandle_validate_flags(flags: lflags);
310	if (ret)
311	return ret;
312
313	lh = kzalloc(size: sizeof(*lh), GFP_KERNEL);
314	if (!lh)
315	return -ENOMEM;
316	lh->gdev = gpio_device_get(gdev);
317
318	if (handlereq.consumer_label[0] != '\0') {
319	/* label is only initialized if consumer_label is set */
320	lh->label = kstrndup(s: handlereq.consumer_label,
321	len: sizeof(handlereq.consumer_label) - 1,
322	GFP_KERNEL);
323	if (!lh->label) {
324	ret = -ENOMEM;
325	goto out_free_lh;
326	}
327	}
328
329	lh->num_descs = handlereq.lines;
330
331	/* Request each GPIO */
332	for (i = 0; i < handlereq.lines; i++) {
333	u32 offset = handlereq.lineoffsets[i];
334	struct gpio_desc *desc = gpio_device_get_desc(gdev, hwnum: offset);
335
336	if (IS_ERR(ptr: desc)) {
337	ret = PTR_ERR(ptr: desc);
338	goto out_free_lh;
339	}
340
341	ret = gpiod_request_user(desc, label: lh->label);
342	if (ret)
343	goto out_free_lh;
344	lh->descs[i] = desc;
345	linehandle_flags_to_desc_flags(lflags: handlereq.flags, flagsp: &desc->flags);
346
347	ret = gpiod_set_transitory(desc, transitory: false);
348	if (ret < 0)
349	goto out_free_lh;
350
351	/*
352	* Lines have to be requested explicitly for input
353	* or output, else the line will be treated "as is".
354	*/
355	if (lflags & GPIOHANDLE_REQUEST_OUTPUT) {
356	int val = !!handlereq.default_values[i];
357
358	ret = gpiod_direction_output(desc, value: val);
359	if (ret)
360	goto out_free_lh;
361	} else if (lflags & GPIOHANDLE_REQUEST_INPUT) {
362	ret = gpiod_direction_input(desc);
363	if (ret)
364	goto out_free_lh;
365	}
366
367	gpiod_line_state_notify(desc, action: GPIO_V2_LINE_CHANGED_REQUESTED);
368
369	dev_dbg(&gdev->dev, "registered chardev handle for line %d\n",
370	offset);
371	}
372
373	fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
374	if (fd < 0) {
375	ret = fd;
376	goto out_free_lh;
377	}
378
379	file = anon_inode_getfile(name: "gpio-linehandle",
380	fops: &linehandle_fileops,
381	priv: lh,
382	O_RDONLY | O_CLOEXEC);
383	if (IS_ERR(ptr: file)) {
384	ret = PTR_ERR(ptr: file);
385	goto out_put_unused_fd;
386	}
387
388	handlereq.fd = fd;
389	if (copy_to_user(to: ip, from: &handlereq, n: sizeof(handlereq))) {
390	/*
391	* fput() will trigger the release() callback, so do not go onto
392	* the regular error cleanup path here.
393	*/
394	fput(file);
395	put_unused_fd(fd);
396	return -EFAULT;
397	}
398
399	fd_install(fd, file);
400
401	dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
402	lh->num_descs);
403
404	return 0;
405
406	out_put_unused_fd:
407	put_unused_fd(fd);
408	out_free_lh:
409	linehandle_free(lh);
410	return ret;
411	}
412	#endif /* CONFIG_GPIO_CDEV_V1 */
413
414	/**
415	* struct line - contains the state of a requested line
416	* @node: to store the object in supinfo_tree if supplemental
417	* @desc: the GPIO descriptor for this line.
418	* @req: the corresponding line request
419	* @irq: the interrupt triggered in response to events on this GPIO
420	* @edflags: the edge flags, GPIO_V2_LINE_FLAG_EDGE_RISING and/or
421	* GPIO_V2_LINE_FLAG_EDGE_FALLING, indicating the edge detection applied
422	* @timestamp_ns: cache for the timestamp storing it between hardirq and
423	* IRQ thread, used to bring the timestamp close to the actual event
424	* @req_seqno: the seqno for the current edge event in the sequence of
425	* events for the corresponding line request. This is drawn from the @req.
426	* @line_seqno: the seqno for the current edge event in the sequence of
427	* events for this line.
428	* @work: the worker that implements software debouncing
429	* @debounce_period_us: the debounce period in microseconds
430	* @sw_debounced: flag indicating if the software debouncer is active
431	* @level: the current debounced physical level of the line
432	* @hdesc: the Hardware Timestamp Engine (HTE) descriptor
433	* @raw_level: the line level at the time of event
434	* @total_discard_seq: the running counter of the discarded events
435	* @last_seqno: the last sequence number before debounce period expires
436	*/
437	struct line {
438	struct rb_node node;
439	struct gpio_desc *desc;
440	/*
441	* -- edge detector specific fields --
442	*/
443	struct linereq *req;
444	unsigned int irq;
445	/*
446	* The flags for the active edge detector configuration.
447	*
448	* edflags is set by linereq_create(), linereq_free(), and
449	* linereq_set_config_unlocked(), which are themselves mutually
450	* exclusive, and is accessed by edge_irq_thread(),
451	* process_hw_ts_thread() and debounce_work_func(),
452	* which can all live with a slightly stale value.
453	*/
454	u64 edflags;
455	/*
456	* timestamp_ns and req_seqno are accessed only by
457	* edge_irq_handler() and edge_irq_thread(), which are themselves
458	* mutually exclusive, so no additional protection is necessary.
459	*/
460	u64 timestamp_ns;
461	u32 req_seqno;
462	/*
463	* line_seqno is accessed by either edge_irq_thread() or
464	* debounce_work_func(), which are themselves mutually exclusive,
465	* so no additional protection is necessary.
466	*/
467	u32 line_seqno;
468	/*
469	* -- debouncer specific fields --
470	*/
471	struct delayed_work work;
472	/*
473	* debounce_period_us is accessed by debounce_irq_handler() and
474	* process_hw_ts() which are disabled when modified by
475	* debounce_setup(), edge_detector_setup() or edge_detector_stop()
476	* or can live with a stale version when updated by
477	* edge_detector_update().
478	* The modifying functions are themselves mutually exclusive.
479	*/
480	unsigned int debounce_period_us;
481	/*
482	* sw_debounce is accessed by linereq_set_config(), which is the
483	* only setter, and linereq_get_values(), which can live with a
484	* slightly stale value.
485	*/
486	unsigned int sw_debounced;
487	/*
488	* level is accessed by debounce_work_func(), which is the only
489	* setter, and linereq_get_values() which can live with a slightly
490	* stale value.
491	*/
492	unsigned int level;
493	#ifdef CONFIG_HTE
494	struct hte_ts_desc hdesc;
495	/*
496	* HTE provider sets line level at the time of event. The valid
497	* value is 0 or 1 and negative value for an error.
498	*/
499	int raw_level;
500	/*
501	* when sw_debounce is set on HTE enabled line, this is running
502	* counter of the discarded events.
503	*/
504	u32 total_discard_seq;
505	/*
506	* when sw_debounce is set on HTE enabled line, this variable records
507	* last sequence number before debounce period expires.
508	*/
509	u32 last_seqno;
510	#endif /* CONFIG_HTE */
511	};
512
513	/*
514	* a rbtree of the struct lines containing supplemental info.
515	* Used to populate gpio_v2_line_info with cdev specific fields not contained
516	* in the struct gpio_desc.
517	* A line is determined to contain supplemental information by
518	* line_has_supinfo().
519	*/
520	static struct rb_root supinfo_tree = RB_ROOT;
521	/* covers supinfo_tree */
522	static DEFINE_SPINLOCK(supinfo_lock);
523
524	/**
525	* struct linereq - contains the state of a userspace line request
526	* @gdev: the GPIO device the line request pertains to
527	* @label: consumer label used to tag GPIO descriptors
528	* @num_lines: the number of lines in the lines array
529	* @wait: wait queue that handles blocking reads of events
530	* @device_unregistered_nb: notifier block for receiving gdev unregister events
531	* @event_buffer_size: the number of elements allocated in @events
532	* @events: KFIFO for the GPIO events
533	* @seqno: the sequence number for edge events generated on all lines in
534	* this line request. Note that this is not used when @num_lines is 1, as
535	* the line_seqno is then the same and is cheaper to calculate.
536	* @config_mutex: mutex for serializing ioctl() calls to ensure consistency
537	* of configuration, particularly multi-step accesses to desc flags and
538	* changes to supinfo status.
539	* @lines: the lines held by this line request, with @num_lines elements.
540	*/
541	struct linereq {
542	struct gpio_device *gdev;
543	const char *label;
544	u32 num_lines;
545	wait_queue_head_t wait;
546	struct notifier_block device_unregistered_nb;
547	u32 event_buffer_size;
548	DECLARE_KFIFO_PTR(events, struct gpio_v2_line_event);
549	atomic_t seqno;
550	struct mutex config_mutex;
551	struct line lines[] __counted_by(num_lines);
552	};
553
554	static void supinfo_insert(struct line *line)
555	{
556	struct rb_node **new = &(supinfo_tree.rb_node), *parent = NULL;
557	struct line *entry;
558
559	guard(spinlock)(l: &supinfo_lock);
560
561	while (*new) {
562	entry = container_of(*new, struct line, node);
563
564	parent = *new;
565	if (line->desc < entry->desc) {
566	new = &((*new)->rb_left);
567	} else if (line->desc > entry->desc) {
568	new = &((*new)->rb_right);
569	} else {
570	/* this should never happen */
571	WARN(1, "duplicate line inserted");
572	return;
573	}
574	}
575
576	rb_link_node(node: &line->node, parent, rb_link: new);
577	rb_insert_color(&line->node, &supinfo_tree);
578	}
579
580	static void supinfo_erase(struct line *line)
581	{
582	guard(spinlock)(l: &supinfo_lock);
583
584	rb_erase(&line->node, &supinfo_tree);
585	}
586
587	static struct line *supinfo_find(struct gpio_desc *desc)
588	{
589	struct rb_node *node = supinfo_tree.rb_node;
590	struct line *line;
591
592	while (node) {
593	line = container_of(node, struct line, node);
594	if (desc < line->desc)
595	node = node->rb_left;
596	else if (desc > line->desc)
597	node = node->rb_right;
598	else
599	return line;
600	}
601	return NULL;
602	}
603
604	static void supinfo_to_lineinfo(struct gpio_desc *desc,
605	struct gpio_v2_line_info *info)
606	{
607	struct gpio_v2_line_attribute *attr;
608	struct line *line;
609
610	guard(spinlock)(l: &supinfo_lock);
611
612	line = supinfo_find(desc);
613	if (!line)
614	return;
615
616	attr = &info->attrs[info->num_attrs];
617	attr->id = GPIO_V2_LINE_ATTR_ID_DEBOUNCE;
618	attr->debounce_period_us = READ_ONCE(line->debounce_period_us);
619	info->num_attrs++;
620	}
621
622	static inline bool line_has_supinfo(struct line *line)
623	{
624	return READ_ONCE(line->debounce_period_us);
625	}
626
627	/*
628	* Checks line_has_supinfo() before and after the change to avoid unnecessary
629	* supinfo_tree access.
630	* Called indirectly by linereq_create() or linereq_set_config() so line
631	* is already protected from concurrent changes.
632	*/
633	static void line_set_debounce_period(struct line *line,
634	unsigned int debounce_period_us)
635	{
636	bool was_suppl = line_has_supinfo(line);
637
638	WRITE_ONCE(line->debounce_period_us, debounce_period_us);
639
640	/* if supinfo status is unchanged then we're done */
641	if (line_has_supinfo(line) == was_suppl)
642	return;
643
644	/* supinfo status has changed, so update the tree */
645	if (was_suppl)
646	supinfo_erase(line);
647	else
648	supinfo_insert(line);
649	}
650
651	#define GPIO_V2_LINE_BIAS_FLAGS \
652	(GPIO_V2_LINE_FLAG_BIAS_PULL_UP | \
653	GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN | \
654	GPIO_V2_LINE_FLAG_BIAS_DISABLED)
655
656	#define GPIO_V2_LINE_DIRECTION_FLAGS \
657	(GPIO_V2_LINE_FLAG_INPUT | \
658	GPIO_V2_LINE_FLAG_OUTPUT)
659
660	#define GPIO_V2_LINE_DRIVE_FLAGS \
661	(GPIO_V2_LINE_FLAG_OPEN_DRAIN | \
662	GPIO_V2_LINE_FLAG_OPEN_SOURCE)
663
664	#define GPIO_V2_LINE_EDGE_FLAGS \
665	(GPIO_V2_LINE_FLAG_EDGE_RISING | \
666	GPIO_V2_LINE_FLAG_EDGE_FALLING)
667
668	#define GPIO_V2_LINE_FLAG_EDGE_BOTH GPIO_V2_LINE_EDGE_FLAGS
669
670	#define GPIO_V2_LINE_VALID_FLAGS \
671	(GPIO_V2_LINE_FLAG_ACTIVE_LOW | \
672	GPIO_V2_LINE_DIRECTION_FLAGS | \
673	GPIO_V2_LINE_DRIVE_FLAGS | \
674	GPIO_V2_LINE_EDGE_FLAGS | \
675	GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME | \
676	GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE | \
677	GPIO_V2_LINE_BIAS_FLAGS)
678
679	/* subset of flags relevant for edge detector configuration */
680	#define GPIO_V2_LINE_EDGE_DETECTOR_FLAGS \
681	(GPIO_V2_LINE_FLAG_ACTIVE_LOW | \
682	GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE | \
683	GPIO_V2_LINE_EDGE_FLAGS)
684
685	static int linereq_unregistered_notify(struct notifier_block *nb,
686	unsigned long action, void *data)
687	{
688	struct linereq *lr = container_of(nb, struct linereq,
689	device_unregistered_nb);
690
691	wake_up_poll(&lr->wait, EPOLLIN | EPOLLERR);
692
693	return NOTIFY_OK;
694	}
695
696	static void linereq_put_event(struct linereq *lr,
697	struct gpio_v2_line_event *le)
698	{
699	bool overflow = false;
700
701	scoped_guard(spinlock, &lr->wait.lock) {
702	if (kfifo_is_full(&lr->events)) {
703	overflow = true;
704	kfifo_skip(&lr->events);
705	}
706	kfifo_in(&lr->events, le, 1);
707	}
708	if (!overflow)
709	wake_up_poll(&lr->wait, EPOLLIN);
710	else
711	pr_debug_ratelimited("event FIFO is full - event dropped\n");
712	}
713
714	static u64 line_event_timestamp(struct line *line)
715	{
716	if (test_bit(FLAG_EVENT_CLOCK_REALTIME, &line->desc->flags))
717	return ktime_get_real_ns();
718	else if (IS_ENABLED(CONFIG_HTE) &&
719	test_bit(FLAG_EVENT_CLOCK_HTE, &line->desc->flags))
720	return line->timestamp_ns;
721
722	return ktime_get_ns();
723	}
724
725	static u32 line_event_id(int level)
726	{
727	return level ? GPIO_V2_LINE_EVENT_RISING_EDGE :
728	GPIO_V2_LINE_EVENT_FALLING_EDGE;
729	}
730
731	static inline char *make_irq_label(const char *orig)
732	{
733	char *new;
734
735	if (!orig)
736	return NULL;
737
738	new = kstrdup_and_replace(src: orig, old: '/', new: ':', GFP_KERNEL);
739	if (!new)
740	return ERR_PTR(error: -ENOMEM);
741
742	return new;
743	}
744
745	static inline void free_irq_label(const char *label)
746	{
747	kfree(objp: label);
748	}
749
750	#ifdef CONFIG_HTE
751
752	static enum hte_return process_hw_ts_thread(void *p)
753	{
754	struct line *line;
755	struct linereq *lr;
756	struct gpio_v2_line_event le;
757	u64 edflags;
758	int level;
759
760	if (!p)
761	return HTE_CB_HANDLED;
762
763	line = p;
764	lr = line->req;
765
766	memset(&le, 0, sizeof(le));
767
768	le.timestamp_ns = line->timestamp_ns;
769	edflags = READ_ONCE(line->edflags);
770
771	switch (edflags & GPIO_V2_LINE_EDGE_FLAGS) {
772	case GPIO_V2_LINE_FLAG_EDGE_BOTH:
773	level = (line->raw_level >= 0) ?
774	line->raw_level :
775	gpiod_get_raw_value_cansleep(desc: line->desc);
776
777	if (edflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
778	level = !level;
779
780	le.id = line_event_id(level);
781	break;
782	case GPIO_V2_LINE_FLAG_EDGE_RISING:
783	le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
784	break;
785	case GPIO_V2_LINE_FLAG_EDGE_FALLING:
786	le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
787	break;
788	default:
789	return HTE_CB_HANDLED;
790	}
791	le.line_seqno = line->line_seqno;
792	le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno;
793	le.offset = gpio_chip_hwgpio(desc: line->desc);
794
795	linereq_put_event(lr, le: &le);
796
797	return HTE_CB_HANDLED;
798	}
799
800	static enum hte_return process_hw_ts(struct hte_ts_data *ts, void *p)
801	{
802	struct line *line;
803	struct linereq *lr;
804	int diff_seqno = 0;
805
806	if (!ts || !p)
807	return HTE_CB_HANDLED;
808
809	line = p;
810	line->timestamp_ns = ts->tsc;
811	line->raw_level = ts->raw_level;
812	lr = line->req;
813
814	if (READ_ONCE(line->sw_debounced)) {
815	line->total_discard_seq++;
816	line->last_seqno = ts->seq;
817	mod_delayed_work(wq: system_wq, dwork: &line->work,
818	delay: usecs_to_jiffies(READ_ONCE(line->debounce_period_us)));
819	} else {
820	if (unlikely(ts->seq < line->line_seqno))
821	return HTE_CB_HANDLED;
822
823	diff_seqno = ts->seq - line->line_seqno;
824	line->line_seqno = ts->seq;
825	if (lr->num_lines != 1)
826	line->req_seqno = atomic_add_return(i: diff_seqno,
827	v: &lr->seqno);
828
829	return HTE_RUN_SECOND_CB;
830	}
831
832	return HTE_CB_HANDLED;
833	}
834
835	static int hte_edge_setup(struct line *line, u64 eflags)
836	{
837	int ret;
838	unsigned long flags = 0;
839	struct hte_ts_desc *hdesc = &line->hdesc;
840
841	if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING)
842	flags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ?
843	HTE_FALLING_EDGE_TS :
844	HTE_RISING_EDGE_TS;
845	if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING)
846	flags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ?
847	HTE_RISING_EDGE_TS :
848	HTE_FALLING_EDGE_TS;
849
850	line->total_discard_seq = 0;
851
852	hte_init_line_attr(desc: hdesc, line_id: desc_to_gpio(desc: line->desc), edge_flags: flags, NULL,
853	data: line->desc);
854
855	ret = hte_ts_get(NULL, desc: hdesc, index: 0);
856	if (ret)
857	return ret;
858
859	return hte_request_ts_ns(desc: hdesc, cb: process_hw_ts, tcb: process_hw_ts_thread,
860	data: line);
861	}
862
863	#else
864
865	static int hte_edge_setup(struct line *line, u64 eflags)
866	{
867	return 0;
868	}
869	#endif /* CONFIG_HTE */
870
871	static irqreturn_t edge_irq_thread(int irq, void *p)
872	{
873	struct line *line = p;
874	struct linereq *lr = line->req;
875	struct gpio_v2_line_event le;
876
877	/* Do not leak kernel stack to userspace */
878	memset(&le, 0, sizeof(le));
879
880	if (line->timestamp_ns) {
881	le.timestamp_ns = line->timestamp_ns;
882	} else {
883	/*
884	* We may be running from a nested threaded interrupt in
885	* which case we didn't get the timestamp from
886	* edge_irq_handler().
887	*/
888	le.timestamp_ns = line_event_timestamp(line);
889	if (lr->num_lines != 1)
890	line->req_seqno = atomic_inc_return(v: &lr->seqno);
891	}
892	line->timestamp_ns = 0;
893
894	switch (READ_ONCE(line->edflags) & GPIO_V2_LINE_EDGE_FLAGS) {
895	case GPIO_V2_LINE_FLAG_EDGE_BOTH:
896	le.id = line_event_id(level: gpiod_get_value_cansleep(desc: line->desc));
897	break;
898	case GPIO_V2_LINE_FLAG_EDGE_RISING:
899	le.id = GPIO_V2_LINE_EVENT_RISING_EDGE;
900	break;
901	case GPIO_V2_LINE_FLAG_EDGE_FALLING:
902	le.id = GPIO_V2_LINE_EVENT_FALLING_EDGE;
903	break;
904	default:
905	return IRQ_NONE;
906	}
907	line->line_seqno++;
908	le.line_seqno = line->line_seqno;
909	le.seqno = (lr->num_lines == 1) ? le.line_seqno : line->req_seqno;
910	le.offset = gpio_chip_hwgpio(desc: line->desc);
911
912	linereq_put_event(lr, le: &le);
913
914	return IRQ_HANDLED;
915	}
916
917	static irqreturn_t edge_irq_handler(int irq, void *p)
918	{
919	struct line *line = p;
920	struct linereq *lr = line->req;
921
922	/*
923	* Just store the timestamp in hardirq context so we get it as
924	* close in time as possible to the actual event.
925	*/
926	line->timestamp_ns = line_event_timestamp(line);
927
928	if (lr->num_lines != 1)
929	line->req_seqno = atomic_inc_return(v: &lr->seqno);
930
931	return IRQ_WAKE_THREAD;
932	}
933
934	/*
935	* returns the current debounced logical value.
936	*/
937	static bool debounced_value(struct line *line)
938	{
939	bool value;
940
941	/*
942	* minor race - debouncer may be stopped here, so edge_detector_stop()
943	* must leave the value unchanged so the following will read the level
944	* from when the debouncer was last running.
945	*/
946	value = READ_ONCE(line->level);
947
948	if (test_bit(FLAG_ACTIVE_LOW, &line->desc->flags))
949	value = !value;
950
951	return value;
952	}
953
954	static irqreturn_t debounce_irq_handler(int irq, void *p)
955	{
956	struct line *line = p;
957
958	mod_delayed_work(wq: system_wq, dwork: &line->work,
959	delay: usecs_to_jiffies(READ_ONCE(line->debounce_period_us)));
960
961	return IRQ_HANDLED;
962	}
963
964	static void debounce_work_func(struct work_struct *work)
965	{
966	struct gpio_v2_line_event le;
967	struct line *line = container_of(work, struct line, work.work);
968	struct linereq *lr;
969	u64 eflags, edflags = READ_ONCE(line->edflags);
970	int level = -1;
971	#ifdef CONFIG_HTE
972	int diff_seqno;
973
974	if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)
975	level = line->raw_level;
976	#endif
977	if (level < 0)
978	level = gpiod_get_raw_value_cansleep(desc: line->desc);
979	if (level < 0) {
980	pr_debug_ratelimited("debouncer failed to read line value\n");
981	return;
982	}
983
984	if (READ_ONCE(line->level) == level)
985	return;
986
987	WRITE_ONCE(line->level, level);
988
989	/* -- edge detection -- */
990	eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS;
991	if (!eflags)
992	return;
993
994	/* switch from physical level to logical - if they differ */
995	if (edflags & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
996	level = !level;
997
998	/* ignore edges that are not being monitored */
999	if (((eflags == GPIO_V2_LINE_FLAG_EDGE_RISING) && !level) ||
1000	((eflags == GPIO_V2_LINE_FLAG_EDGE_FALLING) && level))
1001	return;
1002
1003	/* Do not leak kernel stack to userspace */
1004	memset(&le, 0, sizeof(le));
1005
1006	lr = line->req;
1007	le.timestamp_ns = line_event_timestamp(line);
1008	le.offset = gpio_chip_hwgpio(desc: line->desc);
1009	#ifdef CONFIG_HTE
1010	if (edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) {
1011	/* discard events except the last one */
1012	line->total_discard_seq -= 1;
1013	diff_seqno = line->last_seqno - line->total_discard_seq -
1014	line->line_seqno;
1015	line->line_seqno = line->last_seqno - line->total_discard_seq;
1016	le.line_seqno = line->line_seqno;
1017	le.seqno = (lr->num_lines == 1) ?
1018	le.line_seqno : atomic_add_return(i: diff_seqno, v: &lr->seqno);
1019	} else
1020	#endif /* CONFIG_HTE */
1021	{
1022	line->line_seqno++;
1023	le.line_seqno = line->line_seqno;
1024	le.seqno = (lr->num_lines == 1) ?
1025	le.line_seqno : atomic_inc_return(v: &lr->seqno);
1026	}
1027
1028	le.id = line_event_id(level);
1029
1030	linereq_put_event(lr, le: &le);
1031	}
1032
1033	static int debounce_setup(struct line *line, unsigned int debounce_period_us)
1034	{
1035	unsigned long irqflags;
1036	int ret, level, irq;
1037	char *label;
1038
1039	/* try hardware */
1040	ret = gpiod_set_debounce(desc: line->desc, debounce: debounce_period_us);
1041	if (!ret) {
1042	line_set_debounce_period(line, debounce_period_us);
1043	return ret;
1044	}
1045	if (ret != -ENOTSUPP)
1046	return ret;
1047
1048	if (debounce_period_us) {
1049	/* setup software debounce */
1050	level = gpiod_get_raw_value_cansleep(desc: line->desc);
1051	if (level < 0)
1052	return level;
1053
1054	if (!(IS_ENABLED(CONFIG_HTE) &&
1055	test_bit(FLAG_EVENT_CLOCK_HTE, &line->desc->flags))) {
1056	irq = gpiod_to_irq(desc: line->desc);
1057	if (irq < 0)
1058	return -ENXIO;
1059
1060	label = make_irq_label(orig: line->req->label);
1061	if (IS_ERR(ptr: label))
1062	return -ENOMEM;
1063
1064	irqflags = IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING;
1065	ret = request_irq(irq, handler: debounce_irq_handler, flags: irqflags,
1066	name: label, dev: line);
1067	if (ret) {
1068	free_irq_label(label);
1069	return ret;
1070	}
1071	line->irq = irq;
1072	} else {
1073	ret = hte_edge_setup(line, GPIO_V2_LINE_FLAG_EDGE_BOTH);
1074	if (ret)
1075	return ret;
1076	}
1077
1078	WRITE_ONCE(line->level, level);
1079	WRITE_ONCE(line->sw_debounced, 1);
1080	}
1081	return 0;
1082	}
1083
1084	static bool gpio_v2_line_config_debounced(struct gpio_v2_line_config *lc,
1085	unsigned int line_idx)
1086	{
1087	unsigned int i;
1088	u64 mask = BIT_ULL(line_idx);
1089
1090	for (i = 0; i < lc->num_attrs; i++) {
1091	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE) &&
1092	(lc->attrs[i].mask & mask))
1093	return true;
1094	}
1095	return false;
1096	}
1097
1098	static u32 gpio_v2_line_config_debounce_period(struct gpio_v2_line_config *lc,
1099	unsigned int line_idx)
1100	{
1101	unsigned int i;
1102	u64 mask = BIT_ULL(line_idx);
1103
1104	for (i = 0; i < lc->num_attrs; i++) {
1105	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_DEBOUNCE) &&
1106	(lc->attrs[i].mask & mask))
1107	return lc->attrs[i].attr.debounce_period_us;
1108	}
1109	return 0;
1110	}
1111
1112	static void edge_detector_stop(struct line *line)
1113	{
1114	if (line->irq) {
1115	free_irq_label(label: free_irq(line->irq, line));
1116	line->irq = 0;
1117	}
1118
1119	#ifdef CONFIG_HTE
1120	if (READ_ONCE(line->edflags) & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE)
1121	hte_ts_put(desc: &line->hdesc);
1122	#endif
1123
1124	cancel_delayed_work_sync(dwork: &line->work);
1125	WRITE_ONCE(line->sw_debounced, 0);
1126	WRITE_ONCE(line->edflags, 0);
1127	line_set_debounce_period(line, debounce_period_us: 0);
1128	/* do not change line->level - see comment in debounced_value() */
1129	}
1130
1131	static int edge_detector_setup(struct line *line,
1132	struct gpio_v2_line_config *lc,
1133	unsigned int line_idx, u64 edflags)
1134	{
1135	u32 debounce_period_us;
1136	unsigned long irqflags = 0;
1137	u64 eflags;
1138	int irq, ret;
1139	char *label;
1140
1141	eflags = edflags & GPIO_V2_LINE_EDGE_FLAGS;
1142	if (eflags && !kfifo_initialized(&line->req->events)) {
1143	ret = kfifo_alloc(&line->req->events,
1144	line->req->event_buffer_size, GFP_KERNEL);
1145	if (ret)
1146	return ret;
1147	}
1148	if (gpio_v2_line_config_debounced(lc, line_idx)) {
1149	debounce_period_us = gpio_v2_line_config_debounce_period(lc, line_idx);
1150	ret = debounce_setup(line, debounce_period_us);
1151	if (ret)
1152	return ret;
1153	line_set_debounce_period(line, debounce_period_us);
1154	}
1155
1156	/* detection disabled or sw debouncer will provide edge detection */
1157	if (!eflags || READ_ONCE(line->sw_debounced))
1158	return 0;
1159
1160	if (IS_ENABLED(CONFIG_HTE) &&
1161	(edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE))
1162	return hte_edge_setup(line, eflags: edflags);
1163
1164	irq = gpiod_to_irq(desc: line->desc);
1165	if (irq < 0)
1166	return -ENXIO;
1167
1168	if (eflags & GPIO_V2_LINE_FLAG_EDGE_RISING)
1169	irqflags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ?
1170	IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
1171	if (eflags & GPIO_V2_LINE_FLAG_EDGE_FALLING)
1172	irqflags |= test_bit(FLAG_ACTIVE_LOW, &line->desc->flags) ?
1173	IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING;
1174	irqflags |= IRQF_ONESHOT;
1175
1176	label = make_irq_label(orig: line->req->label);
1177	if (IS_ERR(ptr: label))
1178	return PTR_ERR(ptr: label);
1179
1180	/* Request a thread to read the events */
1181	ret = request_threaded_irq(irq, handler: edge_irq_handler, thread_fn: edge_irq_thread,
1182	flags: irqflags, name: label, dev: line);
1183	if (ret) {
1184	free_irq_label(label);
1185	return ret;
1186	}
1187
1188	line->irq = irq;
1189	return 0;
1190	}
1191
1192	static int edge_detector_update(struct line *line,
1193	struct gpio_v2_line_config *lc,
1194	unsigned int line_idx, u64 edflags)
1195	{
1196	u64 active_edflags = READ_ONCE(line->edflags);
1197	unsigned int debounce_period_us =
1198	gpio_v2_line_config_debounce_period(lc, line_idx);
1199
1200	if ((active_edflags == edflags) &&
1201	(READ_ONCE(line->debounce_period_us) == debounce_period_us))
1202	return 0;
1203
1204	/* sw debounced and still will be...*/
1205	if (debounce_period_us && READ_ONCE(line->sw_debounced)) {
1206	line_set_debounce_period(line, debounce_period_us);
1207	return 0;
1208	}
1209
1210	/* reconfiguring edge detection or sw debounce being disabled */
1211	if ((line->irq && !READ_ONCE(line->sw_debounced)) ||
1212	(active_edflags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE) ||
1213	(!debounce_period_us && READ_ONCE(line->sw_debounced)))
1214	edge_detector_stop(line);
1215
1216	return edge_detector_setup(line, lc, line_idx, edflags);
1217	}
1218
1219	static u64 gpio_v2_line_config_flags(struct gpio_v2_line_config *lc,
1220	unsigned int line_idx)
1221	{
1222	unsigned int i;
1223	u64 mask = BIT_ULL(line_idx);
1224
1225	for (i = 0; i < lc->num_attrs; i++) {
1226	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_FLAGS) &&
1227	(lc->attrs[i].mask & mask))
1228	return lc->attrs[i].attr.flags;
1229	}
1230	return lc->flags;
1231	}
1232
1233	static int gpio_v2_line_config_output_value(struct gpio_v2_line_config *lc,
1234	unsigned int line_idx)
1235	{
1236	unsigned int i;
1237	u64 mask = BIT_ULL(line_idx);
1238
1239	for (i = 0; i < lc->num_attrs; i++) {
1240	if ((lc->attrs[i].attr.id == GPIO_V2_LINE_ATTR_ID_OUTPUT_VALUES) &&
1241	(lc->attrs[i].mask & mask))
1242	return !!(lc->attrs[i].attr.values & mask);
1243	}
1244	return 0;
1245	}
1246
1247	static int gpio_v2_line_flags_validate(u64 flags)
1248	{
1249	/* Return an error if an unknown flag is set */
1250	if (flags & ~GPIO_V2_LINE_VALID_FLAGS)
1251	return -EINVAL;
1252
1253	if (!IS_ENABLED(CONFIG_HTE) &&
1254	(flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE))
1255	return -EOPNOTSUPP;
1256
1257	/*
1258	* Do not allow both INPUT and OUTPUT flags to be set as they are
1259	* contradictory.
1260	*/
1261	if ((flags & GPIO_V2_LINE_FLAG_INPUT) &&
1262	(flags & GPIO_V2_LINE_FLAG_OUTPUT))
1263	return -EINVAL;
1264
1265	/* Only allow one event clock source */
1266	if (IS_ENABLED(CONFIG_HTE) &&
1267	(flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME) &&
1268	(flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE))
1269	return -EINVAL;
1270
1271	/* Edge detection requires explicit input. */
1272	if ((flags & GPIO_V2_LINE_EDGE_FLAGS) &&
1273	!(flags & GPIO_V2_LINE_FLAG_INPUT))
1274	return -EINVAL;
1275
1276	/*
1277	* Do not allow OPEN_SOURCE and OPEN_DRAIN flags in a single
1278	* request. If the hardware actually supports enabling both at the
1279	* same time the electrical result would be disastrous.
1280	*/
1281	if ((flags & GPIO_V2_LINE_FLAG_OPEN_DRAIN) &&
1282	(flags & GPIO_V2_LINE_FLAG_OPEN_SOURCE))
1283	return -EINVAL;
1284
1285	/* Drive requires explicit output direction. */
1286	if ((flags & GPIO_V2_LINE_DRIVE_FLAGS) &&
1287	!(flags & GPIO_V2_LINE_FLAG_OUTPUT))
1288	return -EINVAL;
1289
1290	/* Bias requires explicit direction. */
1291	if ((flags & GPIO_V2_LINE_BIAS_FLAGS) &&
1292	!(flags & GPIO_V2_LINE_DIRECTION_FLAGS))
1293	return -EINVAL;
1294
1295	/* Only one bias flag can be set. */
1296	if (((flags & GPIO_V2_LINE_FLAG_BIAS_DISABLED) &&
1297	(flags & (GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN |
1298	GPIO_V2_LINE_FLAG_BIAS_PULL_UP))) ||
1299	((flags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN) &&
1300	(flags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP)))
1301	return -EINVAL;
1302
1303	return 0;
1304	}
1305
1306	static int gpio_v2_line_config_validate(struct gpio_v2_line_config *lc,
1307	unsigned int num_lines)
1308	{
1309	unsigned int i;
1310	u64 flags;
1311	int ret;
1312
1313	if (lc->num_attrs > GPIO_V2_LINE_NUM_ATTRS_MAX)
1314	return -EINVAL;
1315
1316	if (memchr_inv(p: lc->padding, c: 0, size: sizeof(lc->padding)))
1317	return -EINVAL;
1318
1319	for (i = 0; i < num_lines; i++) {
1320	flags = gpio_v2_line_config_flags(lc, line_idx: i);
1321	ret = gpio_v2_line_flags_validate(flags);
1322	if (ret)
1323	return ret;
1324
1325	/* debounce requires explicit input */
1326	if (gpio_v2_line_config_debounced(lc, line_idx: i) &&
1327	!(flags & GPIO_V2_LINE_FLAG_INPUT))
1328	return -EINVAL;
1329	}
1330	return 0;
1331	}
1332
1333	static void gpio_v2_line_config_flags_to_desc_flags(u64 flags,
1334	unsigned long *flagsp)
1335	{
1336	assign_bit(FLAG_ACTIVE_LOW, addr: flagsp,
1337	value: flags & GPIO_V2_LINE_FLAG_ACTIVE_LOW);
1338
1339	if (flags & GPIO_V2_LINE_FLAG_OUTPUT)
1340	set_bit(FLAG_IS_OUT, addr: flagsp);
1341	else if (flags & GPIO_V2_LINE_FLAG_INPUT)
1342	clear_bit(FLAG_IS_OUT, addr: flagsp);
1343
1344	assign_bit(FLAG_EDGE_RISING, addr: flagsp,
1345	value: flags & GPIO_V2_LINE_FLAG_EDGE_RISING);
1346	assign_bit(FLAG_EDGE_FALLING, addr: flagsp,
1347	value: flags & GPIO_V2_LINE_FLAG_EDGE_FALLING);
1348
1349	assign_bit(FLAG_OPEN_DRAIN, addr: flagsp,
1350	value: flags & GPIO_V2_LINE_FLAG_OPEN_DRAIN);
1351	assign_bit(FLAG_OPEN_SOURCE, addr: flagsp,
1352	value: flags & GPIO_V2_LINE_FLAG_OPEN_SOURCE);
1353
1354	assign_bit(FLAG_PULL_UP, addr: flagsp,
1355	value: flags & GPIO_V2_LINE_FLAG_BIAS_PULL_UP);
1356	assign_bit(FLAG_PULL_DOWN, addr: flagsp,
1357	value: flags & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN);
1358	assign_bit(FLAG_BIAS_DISABLE, addr: flagsp,
1359	value: flags & GPIO_V2_LINE_FLAG_BIAS_DISABLED);
1360
1361	assign_bit(FLAG_EVENT_CLOCK_REALTIME, addr: flagsp,
1362	value: flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME);
1363	assign_bit(FLAG_EVENT_CLOCK_HTE, addr: flagsp,
1364	value: flags & GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE);
1365	}
1366
1367	static long linereq_get_values(struct linereq *lr, void __user *ip)
1368	{
1369	struct gpio_v2_line_values lv;
1370	DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX);
1371	struct gpio_desc **descs;
1372	unsigned int i, didx, num_get;
1373	bool val;
1374	int ret;
1375
1376	/* NOTE: It's ok to read values of output lines. */
1377	if (copy_from_user(to: &lv, from: ip, n: sizeof(lv)))
1378	return -EFAULT;
1379
1380	/*
1381	* gpiod_get_array_value_complex() requires compacted desc and val
1382	* arrays, rather than the sparse ones in lv.
1383	* Calculation of num_get and construction of the desc array is
1384	* optimized to avoid allocation for the desc array for the common
1385	* num_get == 1 case.
1386	*/
1387	/* scan requested lines to calculate the subset to get */
1388	for (num_get = 0, i = 0; i < lr->num_lines; i++) {
1389	if (lv.mask & BIT_ULL(i)) {
1390	num_get++;
1391	/* capture desc for the num_get == 1 case */
1392	descs = &lr->lines[i].desc;
1393	}
1394	}
1395
1396	if (num_get == 0)
1397	return -EINVAL;
1398
1399	if (num_get != 1) {
1400	/* build compacted desc array */
1401	descs = kmalloc_array(n: num_get, size: sizeof(*descs), GFP_KERNEL);
1402	if (!descs)
1403	return -ENOMEM;
1404	for (didx = 0, i = 0; i < lr->num_lines; i++) {
1405	if (lv.mask & BIT_ULL(i)) {
1406	descs[didx] = lr->lines[i].desc;
1407	didx++;
1408	}
1409	}
1410	}
1411	ret = gpiod_get_array_value_complex(raw: false, can_sleep: true, array_size: num_get,
1412	desc_array: descs, NULL, value_bitmap: vals);
1413
1414	if (num_get != 1)
1415	kfree(objp: descs);
1416	if (ret)
1417	return ret;
1418
1419	lv.bits = 0;
1420	for (didx = 0, i = 0; i < lr->num_lines; i++) {
1421	/* unpack compacted vals for the response */
1422	if (lv.mask & BIT_ULL(i)) {
1423	if (lr->lines[i].sw_debounced)
1424	val = debounced_value(line: &lr->lines[i]);
1425	else
1426	val = test_bit(didx, vals);
1427	if (val)
1428	lv.bits |= BIT_ULL(i);
1429	didx++;
1430	}
1431	}
1432
1433	if (copy_to_user(to: ip, from: &lv, n: sizeof(lv)))
1434	return -EFAULT;
1435
1436	return 0;
1437	}
1438
1439	static long linereq_set_values(struct linereq *lr, void __user *ip)
1440	{
1441	DECLARE_BITMAP(vals, GPIO_V2_LINES_MAX);
1442	struct gpio_v2_line_values lv;
1443	struct gpio_desc **descs;
1444	unsigned int i, didx, num_set;
1445	int ret;
1446
1447	if (copy_from_user(to: &lv, from: ip, n: sizeof(lv)))
1448	return -EFAULT;
1449
1450	guard(mutex)(T: &lr->config_mutex);
1451
1452	/*
1453	* gpiod_set_array_value_complex() requires compacted desc and val
1454	* arrays, rather than the sparse ones in lv.
1455	* Calculation of num_set and construction of the descs and vals arrays
1456	* is optimized to minimize scanning the lv->mask, and to avoid
1457	* allocation for the desc array for the common num_set == 1 case.
1458	*/
1459	bitmap_zero(dst: vals, GPIO_V2_LINES_MAX);
1460	/* scan requested lines to determine the subset to be set */
1461	for (num_set = 0, i = 0; i < lr->num_lines; i++) {
1462	if (lv.mask & BIT_ULL(i)) {
1463	/* setting inputs is not allowed */
1464	if (!test_bit(FLAG_IS_OUT, &lr->lines[i].desc->flags))
1465	return -EPERM;
1466	/* add to compacted values */
1467	if (lv.bits & BIT_ULL(i))
1468	__set_bit(num_set, vals);
1469	num_set++;
1470	/* capture desc for the num_set == 1 case */
1471	descs = &lr->lines[i].desc;
1472	}
1473	}
1474	if (num_set == 0)
1475	return -EINVAL;
1476
1477	if (num_set != 1) {
1478	/* build compacted desc array */
1479	descs = kmalloc_array(n: num_set, size: sizeof(*descs), GFP_KERNEL);
1480	if (!descs)
1481	return -ENOMEM;
1482	for (didx = 0, i = 0; i < lr->num_lines; i++) {
1483	if (lv.mask & BIT_ULL(i)) {
1484	descs[didx] = lr->lines[i].desc;
1485	didx++;
1486	}
1487	}
1488	}
1489	ret = gpiod_set_array_value_complex(raw: false, can_sleep: true, array_size: num_set,
1490	desc_array: descs, NULL, value_bitmap: vals);
1491
1492	if (num_set != 1)
1493	kfree(objp: descs);
1494	return ret;
1495	}
1496
1497	static long linereq_set_config(struct linereq *lr, void __user *ip)
1498	{
1499	struct gpio_v2_line_config lc;
1500	struct gpio_desc *desc;
1501	struct line *line;
1502	unsigned int i;
1503	u64 flags, edflags;
1504	int ret;
1505
1506	if (copy_from_user(to: &lc, from: ip, n: sizeof(lc)))
1507	return -EFAULT;
1508
1509	ret = gpio_v2_line_config_validate(lc: &lc, num_lines: lr->num_lines);
1510	if (ret)
1511	return ret;
1512
1513	guard(mutex)(T: &lr->config_mutex);
1514
1515	for (i = 0; i < lr->num_lines; i++) {
1516	line = &lr->lines[i];
1517	desc = lr->lines[i].desc;
1518	flags = gpio_v2_line_config_flags(lc: &lc, line_idx: i);
1519	gpio_v2_line_config_flags_to_desc_flags(flags, flagsp: &desc->flags);
1520	edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS;
1521	/*
1522	* Lines have to be requested explicitly for input
1523	* or output, else the line will be treated "as is".
1524	*/
1525	if (flags & GPIO_V2_LINE_FLAG_OUTPUT) {
1526	int val = gpio_v2_line_config_output_value(lc: &lc, line_idx: i);
1527
1528	edge_detector_stop(line);
1529	ret = gpiod_direction_output(desc, value: val);
1530	if (ret)
1531	return ret;
1532	} else if (flags & GPIO_V2_LINE_FLAG_INPUT) {
1533	ret = gpiod_direction_input(desc);
1534	if (ret)
1535	return ret;
1536
1537	ret = edge_detector_update(line, lc: &lc, line_idx: i, edflags);
1538	if (ret)
1539	return ret;
1540	}
1541
1542	WRITE_ONCE(line->edflags, edflags);
1543
1544	gpiod_line_state_notify(desc, action: GPIO_V2_LINE_CHANGED_CONFIG);
1545	}
1546	return 0;
1547	}
1548
1549	static long linereq_ioctl(struct file *file, unsigned int cmd,
1550	unsigned long arg)
1551	{
1552	struct linereq *lr = file->private_data;
1553	void __user *ip = (void __user *)arg;
1554
1555	guard(srcu)(l: &lr->gdev->srcu);
1556
1557	if (!rcu_access_pointer(lr->gdev->chip))
1558	return -ENODEV;
1559
1560	switch (cmd) {
1561	case GPIO_V2_LINE_GET_VALUES_IOCTL:
1562	return linereq_get_values(lr, ip);
1563	case GPIO_V2_LINE_SET_VALUES_IOCTL:
1564	return linereq_set_values(lr, ip);
1565	case GPIO_V2_LINE_SET_CONFIG_IOCTL:
1566	return linereq_set_config(lr, ip);
1567	default:
1568	return -EINVAL;
1569	}
1570	}
1571
1572	#ifdef CONFIG_COMPAT
1573	static long linereq_ioctl_compat(struct file *file, unsigned int cmd,
1574	unsigned long arg)
1575	{
1576	return linereq_ioctl(file, cmd, arg: (unsigned long)compat_ptr(uptr: arg));
1577	}
1578	#endif
1579
1580	static __poll_t linereq_poll(struct file *file,
1581	struct poll_table_struct *wait)
1582	{
1583	struct linereq *lr = file->private_data;
1584	__poll_t events = 0;
1585
1586	guard(srcu)(l: &lr->gdev->srcu);
1587
1588	if (!rcu_access_pointer(lr->gdev->chip))
1589	return EPOLLHUP | EPOLLERR;
1590
1591	poll_wait(filp: file, wait_address: &lr->wait, p: wait);
1592
1593	if (!kfifo_is_empty_spinlocked_noirqsave(&lr->events,
1594	&lr->wait.lock))
1595	events = EPOLLIN | EPOLLRDNORM;
1596
1597	return events;
1598	}
1599
1600	static ssize_t linereq_read(struct file *file, char __user *buf,
1601	size_t count, loff_t *f_ps)
1602	{
1603	struct linereq *lr = file->private_data;
1604	struct gpio_v2_line_event le;
1605	ssize_t bytes_read = 0;
1606	int ret;
1607
1608	guard(srcu)(l: &lr->gdev->srcu);
1609
1610	if (!rcu_access_pointer(lr->gdev->chip))
1611	return -ENODEV;
1612
1613	if (count < sizeof(le))
1614	return -EINVAL;
1615
1616	do {
1617	scoped_guard(spinlock, &lr->wait.lock) {
1618	if (kfifo_is_empty(&lr->events)) {
1619	if (bytes_read)
1620	return bytes_read;
1621
1622	if (file->f_flags & O_NONBLOCK)
1623	return -EAGAIN;
1624
1625	ret = wait_event_interruptible_locked(lr->wait,
1626	!kfifo_is_empty(&lr->events));
1627	if (ret)
1628	return ret;
1629	}
1630
1631	ret = kfifo_out(&lr->events, &le, 1);
1632	}
1633	if (ret != 1) {
1634	/*
1635	* This should never happen - we were holding the
1636	* lock from the moment we learned the fifo is no
1637	* longer empty until now.
1638	*/
1639	ret = -EIO;
1640	break;
1641	}
1642
1643	if (copy_to_user(to: buf + bytes_read, from: &le, n: sizeof(le)))
1644	return -EFAULT;
1645	bytes_read += sizeof(le);
1646	} while (count >= bytes_read + sizeof(le));
1647
1648	return bytes_read;
1649	}
1650
1651	static void linereq_free(struct linereq *lr)
1652	{
1653	struct line *line;
1654	unsigned int i;
1655
1656	if (lr->device_unregistered_nb.notifier_call)
1657	blocking_notifier_chain_unregister(nh: &lr->gdev->device_notifier,
1658	nb: &lr->device_unregistered_nb);
1659
1660	for (i = 0; i < lr->num_lines; i++) {
1661	line = &lr->lines[i];
1662	if (!line->desc)
1663	continue;
1664
1665	edge_detector_stop(line);
1666	if (line_has_supinfo(line))
1667	supinfo_erase(line);
1668	gpiod_free(desc: line->desc);
1669	}
1670	kfifo_free(&lr->events);
1671	kfree(objp: lr->label);
1672	gpio_device_put(gdev: lr->gdev);
1673	kvfree(addr: lr);
1674	}
1675
1676	static int linereq_release(struct inode *inode, struct file *file)
1677	{
1678	struct linereq *lr = file->private_data;
1679
1680	linereq_free(lr);
1681	return 0;
1682	}
1683
1684	#ifdef CONFIG_PROC_FS
1685	static void linereq_show_fdinfo(struct seq_file *out, struct file *file)
1686	{
1687	struct linereq *lr = file->private_data;
1688	struct device *dev = &lr->gdev->dev;
1689	u16 i;
1690
1691	seq_printf(m: out, fmt: "gpio-chip:\t%s\n", dev_name(dev));
1692
1693	for (i = 0; i < lr->num_lines; i++)
1694	seq_printf(m: out, fmt: "gpio-line:\t%d\n",
1695	gpio_chip_hwgpio(desc: lr->lines[i].desc));
1696	}
1697	#endif
1698
1699	static const struct file_operations line_fileops = {
1700	.release = linereq_release,
1701	.read = linereq_read,
1702	.poll = linereq_poll,
1703	.owner = THIS_MODULE,
1704	.llseek = noop_llseek,
1705	.unlocked_ioctl = linereq_ioctl,
1706	#ifdef CONFIG_COMPAT
1707	.compat_ioctl = linereq_ioctl_compat,
1708	#endif
1709	#ifdef CONFIG_PROC_FS
1710	.show_fdinfo = linereq_show_fdinfo,
1711	#endif
1712	};
1713
1714	static int linereq_create(struct gpio_device *gdev, void __user *ip)
1715	{
1716	struct gpio_v2_line_request ulr;
1717	struct gpio_v2_line_config *lc;
1718	struct linereq *lr;
1719	struct file *file;
1720	u64 flags, edflags;
1721	unsigned int i;
1722	int fd, ret;
1723
1724	if (copy_from_user(to: &ulr, from: ip, n: sizeof(ulr)))
1725	return -EFAULT;
1726
1727	if ((ulr.num_lines == 0) || (ulr.num_lines > GPIO_V2_LINES_MAX))
1728	return -EINVAL;
1729
1730	if (memchr_inv(p: ulr.padding, c: 0, size: sizeof(ulr.padding)))
1731	return -EINVAL;
1732
1733	lc = &ulr.config;
1734	ret = gpio_v2_line_config_validate(lc, num_lines: ulr.num_lines);
1735	if (ret)
1736	return ret;
1737
1738	lr = kvzalloc(struct_size(lr, lines, ulr.num_lines), GFP_KERNEL);
1739	if (!lr)
1740	return -ENOMEM;
1741	lr->num_lines = ulr.num_lines;
1742
1743	lr->gdev = gpio_device_get(gdev);
1744
1745	for (i = 0; i < ulr.num_lines; i++) {
1746	lr->lines[i].req = lr;
1747	WRITE_ONCE(lr->lines[i].sw_debounced, 0);
1748	INIT_DELAYED_WORK(&lr->lines[i].work, debounce_work_func);
1749	}
1750
1751	if (ulr.consumer[0] != '\0') {
1752	/* label is only initialized if consumer is set */
1753	lr->label = kstrndup(s: ulr.consumer, len: sizeof(ulr.consumer) - 1,
1754	GFP_KERNEL);
1755	if (!lr->label) {
1756	ret = -ENOMEM;
1757	goto out_free_linereq;
1758	}
1759	}
1760
1761	mutex_init(&lr->config_mutex);
1762	init_waitqueue_head(&lr->wait);
1763	lr->event_buffer_size = ulr.event_buffer_size;
1764	if (lr->event_buffer_size == 0)
1765	lr->event_buffer_size = ulr.num_lines * 16;
1766	else if (lr->event_buffer_size > GPIO_V2_LINES_MAX * 16)
1767	lr->event_buffer_size = GPIO_V2_LINES_MAX * 16;
1768
1769	atomic_set(v: &lr->seqno, i: 0);
1770
1771	/* Request each GPIO */
1772	for (i = 0; i < ulr.num_lines; i++) {
1773	u32 offset = ulr.offsets[i];
1774	struct gpio_desc *desc = gpio_device_get_desc(gdev, hwnum: offset);
1775
1776	if (IS_ERR(ptr: desc)) {
1777	ret = PTR_ERR(ptr: desc);
1778	goto out_free_linereq;
1779	}
1780
1781	ret = gpiod_request_user(desc, label: lr->label);
1782	if (ret)
1783	goto out_free_linereq;
1784
1785	lr->lines[i].desc = desc;
1786	flags = gpio_v2_line_config_flags(lc, line_idx: i);
1787	gpio_v2_line_config_flags_to_desc_flags(flags, flagsp: &desc->flags);
1788
1789	ret = gpiod_set_transitory(desc, transitory: false);
1790	if (ret < 0)
1791	goto out_free_linereq;
1792
1793	edflags = flags & GPIO_V2_LINE_EDGE_DETECTOR_FLAGS;
1794	/*
1795	* Lines have to be requested explicitly for input
1796	* or output, else the line will be treated "as is".
1797	*/
1798	if (flags & GPIO_V2_LINE_FLAG_OUTPUT) {
1799	int val = gpio_v2_line_config_output_value(lc, line_idx: i);
1800
1801	ret = gpiod_direction_output(desc, value: val);
1802	if (ret)
1803	goto out_free_linereq;
1804	} else if (flags & GPIO_V2_LINE_FLAG_INPUT) {
1805	ret = gpiod_direction_input(desc);
1806	if (ret)
1807	goto out_free_linereq;
1808
1809	ret = edge_detector_setup(line: &lr->lines[i], lc, line_idx: i,
1810	edflags);
1811	if (ret)
1812	goto out_free_linereq;
1813	}
1814
1815	lr->lines[i].edflags = edflags;
1816
1817	gpiod_line_state_notify(desc, action: GPIO_V2_LINE_CHANGED_REQUESTED);
1818
1819	dev_dbg(&gdev->dev, "registered chardev handle for line %d\n",
1820	offset);
1821	}
1822
1823	lr->device_unregistered_nb.notifier_call = linereq_unregistered_notify;
1824	ret = blocking_notifier_chain_register(nh: &gdev->device_notifier,
1825	nb: &lr->device_unregistered_nb);
1826	if (ret)
1827	goto out_free_linereq;
1828
1829	fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
1830	if (fd < 0) {
1831	ret = fd;
1832	goto out_free_linereq;
1833	}
1834
1835	file = anon_inode_getfile(name: "gpio-line", fops: &line_fileops, priv: lr,
1836	O_RDONLY | O_CLOEXEC);
1837	if (IS_ERR(ptr: file)) {
1838	ret = PTR_ERR(ptr: file);
1839	goto out_put_unused_fd;
1840	}
1841
1842	ulr.fd = fd;
1843	if (copy_to_user(to: ip, from: &ulr, n: sizeof(ulr))) {
1844	/*
1845	* fput() will trigger the release() callback, so do not go onto
1846	* the regular error cleanup path here.
1847	*/
1848	fput(file);
1849	put_unused_fd(fd);
1850	return -EFAULT;
1851	}
1852
1853	fd_install(fd, file);
1854
1855	dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n",
1856	lr->num_lines);
1857
1858	return 0;
1859
1860	out_put_unused_fd:
1861	put_unused_fd(fd);
1862	out_free_linereq:
1863	linereq_free(lr);
1864	return ret;
1865	}
1866
1867	#ifdef CONFIG_GPIO_CDEV_V1
1868
1869	/*
1870	* GPIO line event management
1871	*/
1872
1873	/**
1874	* struct lineevent_state - contains the state of a userspace event
1875	* @gdev: the GPIO device the event pertains to
1876	* @label: consumer label used to tag descriptors
1877	* @desc: the GPIO descriptor held by this event
1878	* @eflags: the event flags this line was requested with
1879	* @irq: the interrupt that trigger in response to events on this GPIO
1880	* @wait: wait queue that handles blocking reads of events
1881	* @device_unregistered_nb: notifier block for receiving gdev unregister events
1882	* @events: KFIFO for the GPIO events
1883	* @timestamp: cache for the timestamp storing it between hardirq
1884	* and IRQ thread, used to bring the timestamp close to the actual
1885	* event
1886	*/
1887	struct lineevent_state {
1888	struct gpio_device *gdev;
1889	const char *label;
1890	struct gpio_desc *desc;
1891	u32 eflags;
1892	int irq;
1893	wait_queue_head_t wait;
1894	struct notifier_block device_unregistered_nb;
1895	DECLARE_KFIFO(events, struct gpioevent_data, 16);
1896	u64 timestamp;
1897	};
1898
1899	#define GPIOEVENT_REQUEST_VALID_FLAGS \
1900	(GPIOEVENT_REQUEST_RISING_EDGE | \
1901	GPIOEVENT_REQUEST_FALLING_EDGE)
1902
1903	static __poll_t lineevent_poll(struct file *file,
1904	struct poll_table_struct *wait)
1905	{
1906	struct lineevent_state *le = file->private_data;
1907	__poll_t events = 0;
1908
1909	guard(srcu)(l: &le->gdev->srcu);
1910
1911	if (!rcu_access_pointer(le->gdev->chip))
1912	return EPOLLHUP | EPOLLERR;
1913
1914	poll_wait(filp: file, wait_address: &le->wait, p: wait);
1915
1916	if (!kfifo_is_empty_spinlocked_noirqsave(&le->events, &le->wait.lock))
1917	events = EPOLLIN | EPOLLRDNORM;
1918
1919	return events;
1920	}
1921
1922	static int lineevent_unregistered_notify(struct notifier_block *nb,
1923	unsigned long action, void *data)
1924	{
1925	struct lineevent_state *le = container_of(nb, struct lineevent_state,
1926	device_unregistered_nb);
1927
1928	wake_up_poll(&le->wait, EPOLLIN | EPOLLERR);
1929
1930	return NOTIFY_OK;
1931	}
1932
1933	struct compat_gpioeevent_data {
1934	compat_u64 timestamp;
1935	u32 id;
1936	};
1937
1938	static ssize_t lineevent_read(struct file *file, char __user *buf,
1939	size_t count, loff_t *f_ps)
1940	{
1941	struct lineevent_state *le = file->private_data;
1942	struct gpioevent_data ge;
1943	ssize_t bytes_read = 0;
1944	ssize_t ge_size;
1945	int ret;
1946
1947	guard(srcu)(l: &le->gdev->srcu);
1948
1949	if (!rcu_access_pointer(le->gdev->chip))
1950	return -ENODEV;
1951
1952	/*
1953	* When compatible system call is being used the struct gpioevent_data,
1954	* in case of at least ia32, has different size due to the alignment
1955	* differences. Because we have first member 64 bits followed by one of
1956	* 32 bits there is no gap between them. The only difference is the
1957	* padding at the end of the data structure. Hence, we calculate the
1958	* actual sizeof() and pass this as an argument to copy_to_user() to
1959	* drop unneeded bytes from the output.
1960	*/
1961	if (compat_need_64bit_alignment_fixup())
1962	ge_size = sizeof(struct compat_gpioeevent_data);
1963	else
1964	ge_size = sizeof(struct gpioevent_data);
1965	if (count < ge_size)
1966	return -EINVAL;
1967
1968	do {
1969	scoped_guard(spinlock, &le->wait.lock) {
1970	if (kfifo_is_empty(&le->events)) {
1971	if (bytes_read)
1972	return bytes_read;
1973
1974	if (file->f_flags & O_NONBLOCK)
1975	return -EAGAIN;
1976
1977	ret = wait_event_interruptible_locked(le->wait,
1978	!kfifo_is_empty(&le->events));
1979	if (ret)
1980	return ret;
1981	}
1982
1983	ret = kfifo_out(&le->events, &ge, 1);
1984	}
1985	if (ret != 1) {
1986	/*
1987	* This should never happen - we were holding the lock
1988	* from the moment we learned the fifo is no longer
1989	* empty until now.
1990	*/
1991	ret = -EIO;
1992	break;
1993	}
1994
1995	if (copy_to_user(to: buf + bytes_read, from: &ge, n: ge_size))
1996	return -EFAULT;
1997	bytes_read += ge_size;
1998	} while (count >= bytes_read + ge_size);
1999
2000	return bytes_read;
2001	}
2002
2003	static void lineevent_free(struct lineevent_state *le)
2004	{
2005	if (le->device_unregistered_nb.notifier_call)
2006	blocking_notifier_chain_unregister(nh: &le->gdev->device_notifier,
2007	nb: &le->device_unregistered_nb);
2008	if (le->irq)
2009	free_irq_label(label: free_irq(le->irq, le));
2010	if (le->desc)
2011	gpiod_free(desc: le->desc);
2012	kfree(objp: le->label);
2013	gpio_device_put(gdev: le->gdev);
2014	kfree(objp: le);
2015	}
2016
2017	static int lineevent_release(struct inode *inode, struct file *file)
2018	{
2019	lineevent_free(le: file->private_data);
2020	return 0;
2021	}
2022
2023	static long lineevent_ioctl(struct file *file, unsigned int cmd,
2024	unsigned long arg)
2025	{
2026	struct lineevent_state *le = file->private_data;
2027	void __user *ip = (void __user *)arg;
2028	struct gpiohandle_data ghd;
2029
2030	guard(srcu)(l: &le->gdev->srcu);
2031
2032	if (!rcu_access_pointer(le->gdev->chip))
2033	return -ENODEV;
2034
2035	/*
2036	* We can get the value for an event line but not set it,
2037	* because it is input by definition.
2038	*/
2039	if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) {
2040	int val;
2041
2042	memset(&ghd, 0, sizeof(ghd));
2043
2044	val = gpiod_get_value_cansleep(desc: le->desc);
2045	if (val < 0)
2046	return val;
2047	ghd.values[0] = val;
2048
2049	if (copy_to_user(to: ip, from: &ghd, n: sizeof(ghd)))
2050	return -EFAULT;
2051
2052	return 0;
2053	}
2054	return -EINVAL;
2055	}
2056
2057	#ifdef CONFIG_COMPAT
2058	static long lineevent_ioctl_compat(struct file *file, unsigned int cmd,
2059	unsigned long arg)
2060	{
2061	return lineevent_ioctl(file, cmd, arg: (unsigned long)compat_ptr(uptr: arg));
2062	}
2063	#endif
2064
2065	static const struct file_operations lineevent_fileops = {
2066	.release = lineevent_release,
2067	.read = lineevent_read,
2068	.poll = lineevent_poll,
2069	.owner = THIS_MODULE,
2070	.llseek = noop_llseek,
2071	.unlocked_ioctl = lineevent_ioctl,
2072	#ifdef CONFIG_COMPAT
2073	.compat_ioctl = lineevent_ioctl_compat,
2074	#endif
2075	};
2076
2077	static irqreturn_t lineevent_irq_thread(int irq, void *p)
2078	{
2079	struct lineevent_state *le = p;
2080	struct gpioevent_data ge;
2081	int ret;
2082
2083	/* Do not leak kernel stack to userspace */
2084	memset(&ge, 0, sizeof(ge));
2085
2086	/*
2087	* We may be running from a nested threaded interrupt in which case
2088	* we didn't get the timestamp from lineevent_irq_handler().
2089	*/
2090	if (!le->timestamp)
2091	ge.timestamp = ktime_get_ns();
2092	else
2093	ge.timestamp = le->timestamp;
2094
2095	if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE
2096	&& le->eflags & GPIOEVENT_REQUEST_FALLING_EDGE) {
2097	int level = gpiod_get_value_cansleep(desc: le->desc);
2098
2099	if (level)
2100	/* Emit low-to-high event */
2101	ge.id = GPIOEVENT_EVENT_RISING_EDGE;
2102	else
2103	/* Emit high-to-low event */
2104	ge.id = GPIOEVENT_EVENT_FALLING_EDGE;
2105	} else if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE) {
2106	/* Emit low-to-high event */
2107	ge.id = GPIOEVENT_EVENT_RISING_EDGE;
2108	} else if (le->eflags & GPIOEVENT_REQUEST_FALLING_EDGE) {
2109	/* Emit high-to-low event */
2110	ge.id = GPIOEVENT_EVENT_FALLING_EDGE;
2111	} else {
2112	return IRQ_NONE;
2113	}
2114
2115	ret = kfifo_in_spinlocked_noirqsave(&le->events, &ge,
2116	1, &le->wait.lock);
2117	if (ret)
2118	wake_up_poll(&le->wait, EPOLLIN);
2119	else
2120	pr_debug_ratelimited("event FIFO is full - event dropped\n");
2121
2122	return IRQ_HANDLED;
2123	}
2124
2125	static irqreturn_t lineevent_irq_handler(int irq, void *p)
2126	{
2127	struct lineevent_state *le = p;
2128
2129	/*
2130	* Just store the timestamp in hardirq context so we get it as
2131	* close in time as possible to the actual event.
2132	*/
2133	le->timestamp = ktime_get_ns();
2134
2135	return IRQ_WAKE_THREAD;
2136	}
2137
2138	static int lineevent_create(struct gpio_device *gdev, void __user *ip)
2139	{
2140	struct gpioevent_request eventreq;
2141	struct lineevent_state *le;
2142	struct gpio_desc *desc;
2143	struct file *file;
2144	u32 offset;
2145	u32 lflags;
2146	u32 eflags;
2147	int fd;
2148	int ret;
2149	int irq, irqflags = 0;
2150	char *label;
2151
2152	if (copy_from_user(to: &eventreq, from: ip, n: sizeof(eventreq)))
2153	return -EFAULT;
2154
2155	offset = eventreq.lineoffset;
2156	lflags = eventreq.handleflags;
2157	eflags = eventreq.eventflags;
2158
2159	desc = gpio_device_get_desc(gdev, hwnum: offset);
2160	if (IS_ERR(ptr: desc))
2161	return PTR_ERR(ptr: desc);
2162
2163	/* Return an error if a unknown flag is set */
2164	if ((lflags & ~GPIOHANDLE_REQUEST_VALID_FLAGS) ||
2165	(eflags & ~GPIOEVENT_REQUEST_VALID_FLAGS))
2166	return -EINVAL;
2167
2168	/* This is just wrong: we don't look for events on output lines */
2169	if ((lflags & GPIOHANDLE_REQUEST_OUTPUT) ||
2170	(lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) ||
2171	(lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE))
2172	return -EINVAL;
2173
2174	/* Only one bias flag can be set. */
2175	if (((lflags & GPIOHANDLE_REQUEST_BIAS_DISABLE) &&
2176	(lflags & (GPIOHANDLE_REQUEST_BIAS_PULL_DOWN |
2177	GPIOHANDLE_REQUEST_BIAS_PULL_UP))) ||
2178	((lflags & GPIOHANDLE_REQUEST_BIAS_PULL_DOWN) &&
2179	(lflags & GPIOHANDLE_REQUEST_BIAS_PULL_UP)))
2180	return -EINVAL;
2181
2182	le = kzalloc(size: sizeof(*le), GFP_KERNEL);
2183	if (!le)
2184	return -ENOMEM;
2185	le->gdev = gpio_device_get(gdev);
2186
2187	if (eventreq.consumer_label[0] != '\0') {
2188	/* label is only initialized if consumer_label is set */
2189	le->label = kstrndup(s: eventreq.consumer_label,
2190	len: sizeof(eventreq.consumer_label) - 1,
2191	GFP_KERNEL);
2192	if (!le->label) {
2193	ret = -ENOMEM;
2194	goto out_free_le;
2195	}
2196	}
2197
2198	ret = gpiod_request_user(desc, label: le->label);
2199	if (ret)
2200	goto out_free_le;
2201	le->desc = desc;
2202	le->eflags = eflags;
2203
2204	linehandle_flags_to_desc_flags(lflags, flagsp: &desc->flags);
2205
2206	ret = gpiod_direction_input(desc);
2207	if (ret)
2208	goto out_free_le;
2209
2210	gpiod_line_state_notify(desc, action: GPIO_V2_LINE_CHANGED_REQUESTED);
2211
2212	irq = gpiod_to_irq(desc);
2213	if (irq <= 0) {
2214	ret = -ENODEV;
2215	goto out_free_le;
2216	}
2217
2218	if (eflags & GPIOEVENT_REQUEST_RISING_EDGE)
2219	irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ?
2220	IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING;
2221	if (eflags & GPIOEVENT_REQUEST_FALLING_EDGE)
2222	irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ?
2223	IRQF_TRIGGER_RISING : IRQF_TRIGGER_FALLING;
2224	irqflags |= IRQF_ONESHOT;
2225
2226	INIT_KFIFO(le->events);
2227	init_waitqueue_head(&le->wait);
2228
2229	le->device_unregistered_nb.notifier_call = lineevent_unregistered_notify;
2230	ret = blocking_notifier_chain_register(nh: &gdev->device_notifier,
2231	nb: &le->device_unregistered_nb);
2232	if (ret)
2233	goto out_free_le;
2234
2235	label = make_irq_label(orig: le->label);
2236	if (IS_ERR(ptr: label)) {
2237	ret = PTR_ERR(ptr: label);
2238	goto out_free_le;
2239	}
2240
2241	/* Request a thread to read the events */
2242	ret = request_threaded_irq(irq,
2243	handler: lineevent_irq_handler,
2244	thread_fn: lineevent_irq_thread,
2245	flags: irqflags,
2246	name: label,
2247	dev: le);
2248	if (ret) {
2249	free_irq_label(label);
2250	goto out_free_le;
2251	}
2252
2253	le->irq = irq;
2254
2255	fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC);
2256	if (fd < 0) {
2257	ret = fd;
2258	goto out_free_le;
2259	}
2260
2261	file = anon_inode_getfile(name: "gpio-event",
2262	fops: &lineevent_fileops,
2263	priv: le,
2264	O_RDONLY | O_CLOEXEC);
2265	if (IS_ERR(ptr: file)) {
2266	ret = PTR_ERR(ptr: file);
2267	goto out_put_unused_fd;
2268	}
2269
2270	eventreq.fd = fd;
2271	if (copy_to_user(to: ip, from: &eventreq, n: sizeof(eventreq))) {
2272	/*
2273	* fput() will trigger the release() callback, so do not go onto
2274	* the regular error cleanup path here.
2275	*/
2276	fput(file);
2277	put_unused_fd(fd);
2278	return -EFAULT;
2279	}
2280
2281	fd_install(fd, file);
2282
2283	return 0;
2284
2285	out_put_unused_fd:
2286	put_unused_fd(fd);
2287	out_free_le:
2288	lineevent_free(le);
2289	return ret;
2290	}
2291
2292	static void gpio_v2_line_info_to_v1(struct gpio_v2_line_info *info_v2,
2293	struct gpioline_info *info_v1)
2294	{
2295	u64 flagsv2 = info_v2->flags;
2296
2297	memcpy(info_v1->name, info_v2->name, sizeof(info_v1->name));
2298	memcpy(info_v1->consumer, info_v2->consumer, sizeof(info_v1->consumer));
2299	info_v1->line_offset = info_v2->offset;
2300	info_v1->flags = 0;
2301
2302	if (flagsv2 & GPIO_V2_LINE_FLAG_USED)
2303	info_v1->flags |= GPIOLINE_FLAG_KERNEL;
2304
2305	if (flagsv2 & GPIO_V2_LINE_FLAG_OUTPUT)
2306	info_v1->flags |= GPIOLINE_FLAG_IS_OUT;
2307
2308	if (flagsv2 & GPIO_V2_LINE_FLAG_ACTIVE_LOW)
2309	info_v1->flags |= GPIOLINE_FLAG_ACTIVE_LOW;
2310
2311	if (flagsv2 & GPIO_V2_LINE_FLAG_OPEN_DRAIN)
2312	info_v1->flags |= GPIOLINE_FLAG_OPEN_DRAIN;
2313	if (flagsv2 & GPIO_V2_LINE_FLAG_OPEN_SOURCE)
2314	info_v1->flags |= GPIOLINE_FLAG_OPEN_SOURCE;
2315
2316	if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_PULL_UP)
2317	info_v1->flags |= GPIOLINE_FLAG_BIAS_PULL_UP;
2318	if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN)
2319	info_v1->flags |= GPIOLINE_FLAG_BIAS_PULL_DOWN;
2320	if (flagsv2 & GPIO_V2_LINE_FLAG_BIAS_DISABLED)
2321	info_v1->flags |= GPIOLINE_FLAG_BIAS_DISABLE;
2322	}
2323
2324	static void gpio_v2_line_info_changed_to_v1(
2325	struct gpio_v2_line_info_changed *lic_v2,
2326	struct gpioline_info_changed *lic_v1)
2327	{
2328	memset(lic_v1, 0, sizeof(*lic_v1));
2329	gpio_v2_line_info_to_v1(info_v2: &lic_v2->info, info_v1: &lic_v1->info);
2330	lic_v1->timestamp = lic_v2->timestamp_ns;
2331	lic_v1->event_type = lic_v2->event_type;
2332	}
2333
2334	#endif /* CONFIG_GPIO_CDEV_V1 */
2335
2336	static void gpio_desc_to_lineinfo(struct gpio_desc *desc,
2337	struct gpio_v2_line_info *info)
2338	{
2339	unsigned long dflags;
2340	const char *label;
2341
2342	CLASS(gpio_chip_guard, guard)(desc);
2343	if (!guard.gc)
2344	return;
2345
2346	memset(info, 0, sizeof(*info));
2347	info->offset = gpio_chip_hwgpio(desc);
2348
2349	if (desc->name)
2350	strscpy(info->name, desc->name, sizeof(info->name));
2351
2352	dflags = READ_ONCE(desc->flags);
2353
2354	scoped_guard(srcu, &desc->srcu) {
2355	label = gpiod_get_label(desc);
2356	if (label && test_bit(FLAG_REQUESTED, &dflags))
2357	strscpy(info->consumer, label,
2358	sizeof(info->consumer));
2359	}
2360
2361	/*
2362	* Userspace only need know that the kernel is using this GPIO so it
2363	* can't use it.
2364	* The calculation of the used flag is slightly racy, as it may read
2365	* desc, gc and pinctrl state without a lock covering all three at
2366	* once. Worst case if the line is in transition and the calculation
2367	* is inconsistent then it looks to the user like they performed the
2368	* read on the other side of the transition - but that can always
2369	* happen.
2370	* The definitive test that a line is available to userspace is to
2371	* request it.
2372	*/
2373	if (test_bit(FLAG_REQUESTED, &dflags) ||
2374	test_bit(FLAG_IS_HOGGED, &dflags) ||
2375	test_bit(FLAG_USED_AS_IRQ, &dflags) ||
2376	test_bit(FLAG_EXPORT, &dflags) ||
2377	test_bit(FLAG_SYSFS, &dflags) ||
2378	!gpiochip_line_is_valid(gc: guard.gc, offset: info->offset) ||
2379	!pinctrl_gpio_can_use_line(gc: guard.gc, offset: info->offset))
2380	info->flags |= GPIO_V2_LINE_FLAG_USED;
2381
2382	if (test_bit(FLAG_IS_OUT, &dflags))
2383	info->flags |= GPIO_V2_LINE_FLAG_OUTPUT;
2384	else
2385	info->flags |= GPIO_V2_LINE_FLAG_INPUT;
2386
2387	if (test_bit(FLAG_ACTIVE_LOW, &dflags))
2388	info->flags |= GPIO_V2_LINE_FLAG_ACTIVE_LOW;
2389
2390	if (test_bit(FLAG_OPEN_DRAIN, &dflags))
2391	info->flags |= GPIO_V2_LINE_FLAG_OPEN_DRAIN;
2392	if (test_bit(FLAG_OPEN_SOURCE, &dflags))
2393	info->flags |= GPIO_V2_LINE_FLAG_OPEN_SOURCE;
2394
2395	if (test_bit(FLAG_BIAS_DISABLE, &dflags))
2396	info->flags |= GPIO_V2_LINE_FLAG_BIAS_DISABLED;
2397	if (test_bit(FLAG_PULL_DOWN, &dflags))
2398	info->flags |= GPIO_V2_LINE_FLAG_BIAS_PULL_DOWN;
2399	if (test_bit(FLAG_PULL_UP, &dflags))
2400	info->flags |= GPIO_V2_LINE_FLAG_BIAS_PULL_UP;
2401
2402	if (test_bit(FLAG_EDGE_RISING, &dflags))
2403	info->flags |= GPIO_V2_LINE_FLAG_EDGE_RISING;
2404	if (test_bit(FLAG_EDGE_FALLING, &dflags))
2405	info->flags |= GPIO_V2_LINE_FLAG_EDGE_FALLING;
2406
2407	if (test_bit(FLAG_EVENT_CLOCK_REALTIME, &dflags))
2408	info->flags |= GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME;
2409	else if (test_bit(FLAG_EVENT_CLOCK_HTE, &dflags))
2410	info->flags |= GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE;
2411	}
2412
2413	struct gpio_chardev_data {
2414	struct gpio_device *gdev;
2415	wait_queue_head_t wait;
2416	DECLARE_KFIFO(events, struct gpio_v2_line_info_changed, 32);
2417	struct notifier_block lineinfo_changed_nb;
2418	struct notifier_block device_unregistered_nb;
2419	unsigned long *watched_lines;
2420	#ifdef CONFIG_GPIO_CDEV_V1
2421	atomic_t watch_abi_version;
2422	#endif
2423	};
2424
2425	static int chipinfo_get(struct gpio_chardev_data *cdev, void __user *ip)
2426	{
2427	struct gpio_device *gdev = cdev->gdev;
2428	struct gpiochip_info chipinfo;
2429
2430	memset(&chipinfo, 0, sizeof(chipinfo));
2431
2432	strscpy(chipinfo.name, dev_name(&gdev->dev), sizeof(chipinfo.name));
2433	strscpy(chipinfo.label, gdev->label, sizeof(chipinfo.label));
2434	chipinfo.lines = gdev->ngpio;
2435	if (copy_to_user(to: ip, from: &chipinfo, n: sizeof(chipinfo)))
2436	return -EFAULT;
2437	return 0;
2438	}
2439
2440	#ifdef CONFIG_GPIO_CDEV_V1
2441	/*
2442	* returns 0 if the versions match, else the previously selected ABI version
2443	*/
2444	static int lineinfo_ensure_abi_version(struct gpio_chardev_data *cdata,
2445	unsigned int version)
2446	{
2447	int abiv = atomic_cmpxchg(v: &cdata->watch_abi_version, old: 0, new: version);
2448
2449	if (abiv == version)
2450	return 0;
2451
2452	return abiv;
2453	}
2454
2455	static int lineinfo_get_v1(struct gpio_chardev_data *cdev, void __user *ip,
2456	bool watch)
2457	{
2458	struct gpio_desc *desc;
2459	struct gpioline_info lineinfo;
2460	struct gpio_v2_line_info lineinfo_v2;
2461
2462	if (copy_from_user(to: &lineinfo, from: ip, n: sizeof(lineinfo)))
2463	return -EFAULT;
2464
2465	/* this doubles as a range check on line_offset */
2466	desc = gpio_device_get_desc(gdev: cdev->gdev, hwnum: lineinfo.line_offset);
2467	if (IS_ERR(ptr: desc))
2468	return PTR_ERR(ptr: desc);
2469
2470	if (watch) {
2471	if (lineinfo_ensure_abi_version(cdata: cdev, version: 1))
2472	return -EPERM;
2473
2474	if (test_and_set_bit(nr: lineinfo.line_offset, addr: cdev->watched_lines))
2475	return -EBUSY;
2476	}
2477
2478	gpio_desc_to_lineinfo(desc, info: &lineinfo_v2);
2479	gpio_v2_line_info_to_v1(info_v2: &lineinfo_v2, info_v1: &lineinfo);
2480
2481	if (copy_to_user(to: ip, from: &lineinfo, n: sizeof(lineinfo))) {
2482	if (watch)
2483	clear_bit(nr: lineinfo.line_offset, addr: cdev->watched_lines);
2484	return -EFAULT;
2485	}
2486
2487	return 0;
2488	}
2489	#endif
2490
2491	static int lineinfo_get(struct gpio_chardev_data *cdev, void __user *ip,
2492	bool watch)
2493	{
2494	struct gpio_desc *desc;
2495	struct gpio_v2_line_info lineinfo;
2496
2497	if (copy_from_user(to: &lineinfo, from: ip, n: sizeof(lineinfo)))
2498	return -EFAULT;
2499
2500	if (memchr_inv(p: lineinfo.padding, c: 0, size: sizeof(lineinfo.padding)))
2501	return -EINVAL;
2502
2503	desc = gpio_device_get_desc(gdev: cdev->gdev, hwnum: lineinfo.offset);
2504	if (IS_ERR(ptr: desc))
2505	return PTR_ERR(ptr: desc);
2506
2507	if (watch) {
2508	#ifdef CONFIG_GPIO_CDEV_V1
2509	if (lineinfo_ensure_abi_version(cdata: cdev, version: 2))
2510	return -EPERM;
2511	#endif
2512	if (test_and_set_bit(nr: lineinfo.offset, addr: cdev->watched_lines))
2513	return -EBUSY;
2514	}
2515	gpio_desc_to_lineinfo(desc, info: &lineinfo);
2516	supinfo_to_lineinfo(desc, info: &lineinfo);
2517
2518	if (copy_to_user(to: ip, from: &lineinfo, n: sizeof(lineinfo))) {
2519	if (watch)
2520	clear_bit(nr: lineinfo.offset, addr: cdev->watched_lines);
2521	return -EFAULT;
2522	}
2523
2524	return 0;
2525	}
2526
2527	static int lineinfo_unwatch(struct gpio_chardev_data *cdev, void __user *ip)
2528	{
2529	__u32 offset;
2530
2531	if (copy_from_user(to: &offset, from: ip, n: sizeof(offset)))
2532	return -EFAULT;
2533
2534	if (offset >= cdev->gdev->ngpio)
2535	return -EINVAL;
2536
2537	if (!test_and_clear_bit(nr: offset, addr: cdev->watched_lines))
2538	return -EBUSY;
2539
2540	return 0;
2541	}
2542
2543	/*
2544	* gpio_ioctl() - ioctl handler for the GPIO chardev
2545	*/
2546	static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2547	{
2548	struct gpio_chardev_data *cdev = file->private_data;
2549	struct gpio_device *gdev = cdev->gdev;
2550	void __user *ip = (void __user *)arg;
2551
2552	guard(srcu)(l: &gdev->srcu);
2553
2554	/* We fail any subsequent ioctl():s when the chip is gone */
2555	if (!rcu_access_pointer(gdev->chip))
2556	return -ENODEV;
2557
2558	/* Fill in the struct and pass to userspace */
2559	switch (cmd) {
2560	case GPIO_GET_CHIPINFO_IOCTL:
2561	return chipinfo_get(cdev, ip);
2562	#ifdef CONFIG_GPIO_CDEV_V1
2563	case GPIO_GET_LINEHANDLE_IOCTL:
2564	return linehandle_create(gdev, ip);
2565	case GPIO_GET_LINEEVENT_IOCTL:
2566	return lineevent_create(gdev, ip);
2567	case GPIO_GET_LINEINFO_IOCTL:
2568	return lineinfo_get_v1(cdev, ip, watch: false);
2569	case GPIO_GET_LINEINFO_WATCH_IOCTL:
2570	return lineinfo_get_v1(cdev, ip, watch: true);
2571	#endif /* CONFIG_GPIO_CDEV_V1 */
2572	case GPIO_V2_GET_LINEINFO_IOCTL:
2573	return lineinfo_get(cdev, ip, watch: false);
2574	case GPIO_V2_GET_LINEINFO_WATCH_IOCTL:
2575	return lineinfo_get(cdev, ip, watch: true);
2576	case GPIO_V2_GET_LINE_IOCTL:
2577	return linereq_create(gdev, ip);
2578	case GPIO_GET_LINEINFO_UNWATCH_IOCTL:
2579	return lineinfo_unwatch(cdev, ip);
2580	default:
2581	return -EINVAL;
2582	}
2583	}
2584
2585	#ifdef CONFIG_COMPAT
2586	static long gpio_ioctl_compat(struct file *file, unsigned int cmd,
2587	unsigned long arg)
2588	{
2589	return gpio_ioctl(file, cmd, arg: (unsigned long)compat_ptr(uptr: arg));
2590	}
2591	#endif
2592
2593	static int lineinfo_changed_notify(struct notifier_block *nb,
2594	unsigned long action, void *data)
2595	{
2596	struct gpio_chardev_data *cdev =
2597	container_of(nb, struct gpio_chardev_data, lineinfo_changed_nb);
2598	struct gpio_v2_line_info_changed chg;
2599	struct gpio_desc *desc = data;
2600	int ret;
2601
2602	if (!test_bit(gpio_chip_hwgpio(desc), cdev->watched_lines))
2603	return NOTIFY_DONE;
2604
2605	memset(&chg, 0, sizeof(chg));
2606	chg.event_type = action;
2607	chg.timestamp_ns = ktime_get_ns();
2608	gpio_desc_to_lineinfo(desc, info: &chg.info);
2609	supinfo_to_lineinfo(desc, info: &chg.info);
2610
2611	ret = kfifo_in_spinlocked(&cdev->events, &chg, 1, &cdev->wait.lock);
2612	if (ret)
2613	wake_up_poll(&cdev->wait, EPOLLIN);
2614	else
2615	pr_debug_ratelimited("lineinfo event FIFO is full - event dropped\n");
2616
2617	return NOTIFY_OK;
2618	}
2619
2620	static int gpio_device_unregistered_notify(struct notifier_block *nb,
2621	unsigned long action, void *data)
2622	{
2623	struct gpio_chardev_data *cdev = container_of(nb,
2624	struct gpio_chardev_data,
2625	device_unregistered_nb);
2626
2627	wake_up_poll(&cdev->wait, EPOLLIN | EPOLLERR);
2628
2629	return NOTIFY_OK;
2630	}
2631
2632	static __poll_t lineinfo_watch_poll(struct file *file,
2633	struct poll_table_struct *pollt)
2634	{
2635	struct gpio_chardev_data *cdev = file->private_data;
2636	__poll_t events = 0;
2637
2638	guard(srcu)(l: &cdev->gdev->srcu);
2639
2640	if (!rcu_access_pointer(cdev->gdev->chip))
2641	return EPOLLHUP | EPOLLERR;
2642
2643	poll_wait(filp: file, wait_address: &cdev->wait, p: pollt);
2644
2645	if (!kfifo_is_empty_spinlocked_noirqsave(&cdev->events,
2646	&cdev->wait.lock))
2647	events = EPOLLIN | EPOLLRDNORM;
2648
2649	return events;
2650	}
2651
2652	static ssize_t lineinfo_watch_read(struct file *file, char __user *buf,
2653	size_t count, loff_t *off)
2654	{
2655	struct gpio_chardev_data *cdev = file->private_data;
2656	struct gpio_v2_line_info_changed event;
2657	ssize_t bytes_read = 0;
2658	int ret;
2659	size_t event_size;
2660
2661	guard(srcu)(l: &cdev->gdev->srcu);
2662
2663	if (!rcu_access_pointer(cdev->gdev->chip))
2664	return -ENODEV;
2665
2666	#ifndef CONFIG_GPIO_CDEV_V1
2667	event_size = sizeof(struct gpio_v2_line_info_changed);
2668	if (count < event_size)
2669	return -EINVAL;
2670	#endif
2671
2672	do {
2673	scoped_guard(spinlock, &cdev->wait.lock) {
2674	if (kfifo_is_empty(&cdev->events)) {
2675	if (bytes_read)
2676	return bytes_read;
2677
2678	if (file->f_flags & O_NONBLOCK)
2679	return -EAGAIN;
2680
2681	ret = wait_event_interruptible_locked(cdev->wait,
2682	!kfifo_is_empty(&cdev->events));
2683	if (ret)
2684	return ret;
2685	}
2686	#ifdef CONFIG_GPIO_CDEV_V1
2687	/* must be after kfifo check so watch_abi_version is set */
2688	if (atomic_read(v: &cdev->watch_abi_version) == 2)
2689	event_size = sizeof(struct gpio_v2_line_info_changed);
2690	else
2691	event_size = sizeof(struct gpioline_info_changed);
2692	if (count < event_size)
2693	return -EINVAL;
2694	#endif
2695	ret = kfifo_out(&cdev->events, &event, 1);
2696	}
2697	if (ret != 1) {
2698	ret = -EIO;
2699	break;
2700	/* We should never get here. See lineevent_read(). */
2701	}
2702
2703	#ifdef CONFIG_GPIO_CDEV_V1
2704	if (event_size == sizeof(struct gpio_v2_line_info_changed)) {
2705	if (copy_to_user(to: buf + bytes_read, from: &event, n: event_size))
2706	return -EFAULT;
2707	} else {
2708	struct gpioline_info_changed event_v1;
2709
2710	gpio_v2_line_info_changed_to_v1(lic_v2: &event, lic_v1: &event_v1);
2711	if (copy_to_user(to: buf + bytes_read, from: &event_v1,
2712	n: event_size))
2713	return -EFAULT;
2714	}
2715	#else
2716	if (copy_to_user(buf + bytes_read, &event, event_size))
2717	return -EFAULT;
2718	#endif
2719	bytes_read += event_size;
2720	} while (count >= bytes_read + sizeof(event));
2721
2722	return bytes_read;
2723	}
2724
2725	/**
2726	* gpio_chrdev_open() - open the chardev for ioctl operations
2727	* @inode: inode for this chardev
2728	* @file: file struct for storing private data
2729	* Returns 0 on success
2730	*/
2731	static int gpio_chrdev_open(struct inode *inode, struct file *file)
2732	{
2733	struct gpio_device *gdev = container_of(inode->i_cdev,
2734	struct gpio_device, chrdev);
2735	struct gpio_chardev_data *cdev;
2736	int ret = -ENOMEM;
2737
2738	guard(srcu)(l: &gdev->srcu);
2739
2740	/* Fail on open if the backing gpiochip is gone */
2741	if (!rcu_access_pointer(gdev->chip))
2742	return -ENODEV;
2743
2744	cdev = kzalloc(size: sizeof(*cdev), GFP_KERNEL);
2745	if (!cdev)
2746	return -ENODEV;
2747
2748	cdev->watched_lines = bitmap_zalloc(nbits: gdev->ngpio, GFP_KERNEL);
2749	if (!cdev->watched_lines)
2750	goto out_free_cdev;
2751
2752	init_waitqueue_head(&cdev->wait);
2753	INIT_KFIFO(cdev->events);
2754	cdev->gdev = gpio_device_get(gdev);
2755
2756	cdev->lineinfo_changed_nb.notifier_call = lineinfo_changed_notify;
2757	ret = blocking_notifier_chain_register(nh: &gdev->line_state_notifier,
2758	nb: &cdev->lineinfo_changed_nb);
2759	if (ret)
2760	goto out_free_bitmap;
2761
2762	cdev->device_unregistered_nb.notifier_call =
2763	gpio_device_unregistered_notify;
2764	ret = blocking_notifier_chain_register(nh: &gdev->device_notifier,
2765	nb: &cdev->device_unregistered_nb);
2766	if (ret)
2767	goto out_unregister_line_notifier;
2768
2769	file->private_data = cdev;
2770
2771	ret = nonseekable_open(inode, filp: file);
2772	if (ret)
2773	goto out_unregister_device_notifier;
2774
2775	return ret;
2776
2777	out_unregister_device_notifier:
2778	blocking_notifier_chain_unregister(nh: &gdev->device_notifier,
2779	nb: &cdev->device_unregistered_nb);
2780	out_unregister_line_notifier:
2781	blocking_notifier_chain_unregister(nh: &gdev->line_state_notifier,
2782	nb: &cdev->lineinfo_changed_nb);
2783	out_free_bitmap:
2784	gpio_device_put(gdev);
2785	bitmap_free(bitmap: cdev->watched_lines);
2786	out_free_cdev:
2787	kfree(objp: cdev);
2788	return ret;
2789	}
2790
2791	/**
2792	* gpio_chrdev_release() - close chardev after ioctl operations
2793	* @inode: inode for this chardev
2794	* @file: file struct for storing private data
2795	* Returns 0 on success
2796	*/
2797	static int gpio_chrdev_release(struct inode *inode, struct file *file)
2798	{
2799	struct gpio_chardev_data *cdev = file->private_data;
2800	struct gpio_device *gdev = cdev->gdev;
2801
2802	bitmap_free(bitmap: cdev->watched_lines);
2803	blocking_notifier_chain_unregister(nh: &gdev->device_notifier,
2804	nb: &cdev->device_unregistered_nb);
2805	blocking_notifier_chain_unregister(nh: &gdev->line_state_notifier,
2806	nb: &cdev->lineinfo_changed_nb);
2807	gpio_device_put(gdev);
2808	kfree(objp: cdev);
2809
2810	return 0;
2811	}
2812
2813	static const struct file_operations gpio_fileops = {
2814	.release = gpio_chrdev_release,
2815	.open = gpio_chrdev_open,
2816	.poll = lineinfo_watch_poll,
2817	.read = lineinfo_watch_read,
2818	.owner = THIS_MODULE,
2819	.llseek = no_llseek,
2820	.unlocked_ioctl = gpio_ioctl,
2821	#ifdef CONFIG_COMPAT
2822	.compat_ioctl = gpio_ioctl_compat,
2823	#endif
2824	};
2825
2826	int gpiolib_cdev_register(struct gpio_device *gdev, dev_t devt)
2827	{
2828	struct gpio_chip *gc;
2829	int ret;
2830
2831	cdev_init(&gdev->chrdev, &gpio_fileops);
2832	gdev->chrdev.owner = THIS_MODULE;
2833	gdev->dev.devt = MKDEV(MAJOR(devt), gdev->id);
2834
2835	ret = cdev_device_add(cdev: &gdev->chrdev, dev: &gdev->dev);
2836	if (ret)
2837	return ret;
2838
2839	guard(srcu)(l: &gdev->srcu);
2840	gc = srcu_dereference(gdev->chip, &gdev->srcu);
2841	if (!gc)
2842	return -ENODEV;
2843
2844	chip_dbg(gc, "added GPIO chardev (%d:%d)\n", MAJOR(devt), gdev->id);
2845
2846	return 0;
2847	}
2848
2849	void gpiolib_cdev_unregister(struct gpio_device *gdev)
2850	{
2851	cdev_device_del(cdev: &gdev->chrdev, dev: &gdev->dev);
2852	blocking_notifier_call_chain(nh: &gdev->device_notifier, val: 0, NULL);
2853	}
2854