1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Tty buffer allocation management |
4 | */ |
5 | |
6 | #include <linux/types.h> |
7 | #include <linux/errno.h> |
8 | #include <linux/minmax.h> |
9 | #include <linux/tty.h> |
10 | #include <linux/tty_buffer.h> |
11 | #include <linux/tty_driver.h> |
12 | #include <linux/tty_flip.h> |
13 | #include <linux/timer.h> |
14 | #include <linux/string.h> |
15 | #include <linux/slab.h> |
16 | #include <linux/sched.h> |
17 | #include <linux/wait.h> |
18 | #include <linux/bitops.h> |
19 | #include <linux/delay.h> |
20 | #include <linux/module.h> |
21 | #include <linux/ratelimit.h> |
22 | #include "tty.h" |
23 | |
24 | #define MIN_TTYB_SIZE 256 |
25 | #define TTYB_ALIGN_MASK 0xff |
26 | |
27 | /* |
28 | * Byte threshold to limit memory consumption for flip buffers. |
29 | * The actual memory limit is > 2x this amount. |
30 | */ |
31 | #define TTYB_DEFAULT_MEM_LIMIT (640 * 1024UL) |
32 | |
33 | /* |
34 | * We default to dicing tty buffer allocations to this many characters |
35 | * in order to avoid multiple page allocations. We know the size of |
36 | * tty_buffer itself but it must also be taken into account that the |
37 | * buffer is 256 byte aligned. See tty_buffer_find for the allocation |
38 | * logic this must match. |
39 | */ |
40 | |
41 | #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~TTYB_ALIGN_MASK) |
42 | |
43 | /** |
44 | * tty_buffer_lock_exclusive - gain exclusive access to buffer |
45 | * @port: tty port owning the flip buffer |
46 | * |
47 | * Guarantees safe use of the &tty_ldisc_ops.receive_buf() method by excluding |
48 | * the buffer work and any pending flush from using the flip buffer. Data can |
49 | * continue to be added concurrently to the flip buffer from the driver side. |
50 | * |
51 | * See also tty_buffer_unlock_exclusive(). |
52 | */ |
53 | void tty_buffer_lock_exclusive(struct tty_port *port) |
54 | { |
55 | struct tty_bufhead *buf = &port->buf; |
56 | |
57 | atomic_inc(v: &buf->priority); |
58 | mutex_lock(&buf->lock); |
59 | } |
60 | EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive); |
61 | |
62 | /** |
63 | * tty_buffer_unlock_exclusive - release exclusive access |
64 | * @port: tty port owning the flip buffer |
65 | * |
66 | * The buffer work is restarted if there is data in the flip buffer. |
67 | * |
68 | * See also tty_buffer_lock_exclusive(). |
69 | */ |
70 | void tty_buffer_unlock_exclusive(struct tty_port *port) |
71 | { |
72 | struct tty_bufhead *buf = &port->buf; |
73 | bool restart = buf->head->commit != buf->head->read; |
74 | |
75 | atomic_dec(v: &buf->priority); |
76 | mutex_unlock(lock: &buf->lock); |
77 | |
78 | if (restart) |
79 | queue_work(wq: system_unbound_wq, work: &buf->work); |
80 | } |
81 | EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive); |
82 | |
83 | /** |
84 | * tty_buffer_space_avail - return unused buffer space |
85 | * @port: tty port owning the flip buffer |
86 | * |
87 | * Returns: the # of bytes which can be written by the driver without reaching |
88 | * the buffer limit. |
89 | * |
90 | * Note: this does not guarantee that memory is available to write the returned |
91 | * # of bytes (use tty_prepare_flip_string() to pre-allocate if memory |
92 | * guarantee is required). |
93 | */ |
94 | unsigned int tty_buffer_space_avail(struct tty_port *port) |
95 | { |
96 | int space = port->buf.mem_limit - atomic_read(v: &port->buf.mem_used); |
97 | |
98 | return max(space, 0); |
99 | } |
100 | EXPORT_SYMBOL_GPL(tty_buffer_space_avail); |
101 | |
102 | static void tty_buffer_reset(struct tty_buffer *p, size_t size) |
103 | { |
104 | p->used = 0; |
105 | p->size = size; |
106 | p->next = NULL; |
107 | p->commit = 0; |
108 | p->lookahead = 0; |
109 | p->read = 0; |
110 | p->flags = true; |
111 | } |
112 | |
113 | /** |
114 | * tty_buffer_free_all - free buffers used by a tty |
115 | * @port: tty port to free from |
116 | * |
117 | * Remove all the buffers pending on a tty whether queued with data or in the |
118 | * free ring. Must be called when the tty is no longer in use. |
119 | */ |
120 | void tty_buffer_free_all(struct tty_port *port) |
121 | { |
122 | struct tty_bufhead *buf = &port->buf; |
123 | struct tty_buffer *p, *next; |
124 | struct llist_node *llist; |
125 | unsigned int freed = 0; |
126 | int still_used; |
127 | |
128 | while ((p = buf->head) != NULL) { |
129 | buf->head = p->next; |
130 | freed += p->size; |
131 | if (p->size > 0) |
132 | kfree(objp: p); |
133 | } |
134 | llist = llist_del_all(head: &buf->free); |
135 | llist_for_each_entry_safe(p, next, llist, free) |
136 | kfree(objp: p); |
137 | |
138 | tty_buffer_reset(p: &buf->sentinel, size: 0); |
139 | buf->head = &buf->sentinel; |
140 | buf->tail = &buf->sentinel; |
141 | |
142 | still_used = atomic_xchg(v: &buf->mem_used, new: 0); |
143 | WARN(still_used != freed, "we still have not freed %d bytes!" , |
144 | still_used - freed); |
145 | } |
146 | |
147 | /** |
148 | * tty_buffer_alloc - allocate a tty buffer |
149 | * @port: tty port |
150 | * @size: desired size (characters) |
151 | * |
152 | * Allocate a new tty buffer to hold the desired number of characters. We |
153 | * round our buffers off in 256 character chunks to get better allocation |
154 | * behaviour. |
155 | * |
156 | * Returns: %NULL if out of memory or the allocation would exceed the per |
157 | * device queue. |
158 | */ |
159 | static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) |
160 | { |
161 | struct llist_node *free; |
162 | struct tty_buffer *p; |
163 | |
164 | /* Round the buffer size out */ |
165 | size = __ALIGN_MASK(size, TTYB_ALIGN_MASK); |
166 | |
167 | if (size <= MIN_TTYB_SIZE) { |
168 | free = llist_del_first(head: &port->buf.free); |
169 | if (free) { |
170 | p = llist_entry(free, struct tty_buffer, free); |
171 | goto found; |
172 | } |
173 | } |
174 | |
175 | /* Should possibly check if this fails for the largest buffer we |
176 | * have queued and recycle that ? |
177 | */ |
178 | if (atomic_read(v: &port->buf.mem_used) > port->buf.mem_limit) |
179 | return NULL; |
180 | p = kmalloc(struct_size(p, data, 2 * size), GFP_ATOMIC | __GFP_NOWARN); |
181 | if (p == NULL) |
182 | return NULL; |
183 | |
184 | found: |
185 | tty_buffer_reset(p, size); |
186 | atomic_add(i: size, v: &port->buf.mem_used); |
187 | return p; |
188 | } |
189 | |
190 | /** |
191 | * tty_buffer_free - free a tty buffer |
192 | * @port: tty port owning the buffer |
193 | * @b: the buffer to free |
194 | * |
195 | * Free a tty buffer, or add it to the free list according to our internal |
196 | * strategy. |
197 | */ |
198 | static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b) |
199 | { |
200 | struct tty_bufhead *buf = &port->buf; |
201 | |
202 | /* Dumb strategy for now - should keep some stats */ |
203 | WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0); |
204 | |
205 | if (b->size > MIN_TTYB_SIZE) |
206 | kfree(objp: b); |
207 | else if (b->size > 0) |
208 | llist_add(new: &b->free, head: &buf->free); |
209 | } |
210 | |
211 | /** |
212 | * tty_buffer_flush - flush full tty buffers |
213 | * @tty: tty to flush |
214 | * @ld: optional ldisc ptr (must be referenced) |
215 | * |
216 | * Flush all the buffers containing receive data. If @ld != %NULL, flush the |
217 | * ldisc input buffer. |
218 | * |
219 | * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. |
220 | */ |
221 | void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) |
222 | { |
223 | struct tty_port *port = tty->port; |
224 | struct tty_bufhead *buf = &port->buf; |
225 | struct tty_buffer *next; |
226 | |
227 | atomic_inc(v: &buf->priority); |
228 | |
229 | mutex_lock(&buf->lock); |
230 | /* paired w/ release in __tty_buffer_request_room; ensures there are |
231 | * no pending memory accesses to the freed buffer |
232 | */ |
233 | while ((next = smp_load_acquire(&buf->head->next)) != NULL) { |
234 | tty_buffer_free(port, b: buf->head); |
235 | buf->head = next; |
236 | } |
237 | buf->head->read = buf->head->commit; |
238 | buf->head->lookahead = buf->head->read; |
239 | |
240 | if (ld && ld->ops->flush_buffer) |
241 | ld->ops->flush_buffer(tty); |
242 | |
243 | atomic_dec(v: &buf->priority); |
244 | mutex_unlock(lock: &buf->lock); |
245 | } |
246 | |
247 | /** |
248 | * __tty_buffer_request_room - grow tty buffer if needed |
249 | * @port: tty port |
250 | * @size: size desired |
251 | * @flags: buffer has to store flags along character data |
252 | * |
253 | * Make at least @size bytes of linear space available for the tty buffer. |
254 | * |
255 | * Will change over to a new buffer if the current buffer is encoded as |
256 | * %TTY_NORMAL (so has no flags buffer) and the new buffer requires a flags |
257 | * buffer. |
258 | * |
259 | * Returns: the size we managed to find. |
260 | */ |
261 | static int __tty_buffer_request_room(struct tty_port *port, size_t size, |
262 | bool flags) |
263 | { |
264 | struct tty_bufhead *buf = &port->buf; |
265 | struct tty_buffer *n, *b = buf->tail; |
266 | size_t left = (b->flags ? 1 : 2) * b->size - b->used; |
267 | bool change = !b->flags && flags; |
268 | |
269 | if (!change && left >= size) |
270 | return size; |
271 | |
272 | /* This is the slow path - looking for new buffers to use */ |
273 | n = tty_buffer_alloc(port, size); |
274 | if (n == NULL) |
275 | return change ? 0 : left; |
276 | |
277 | n->flags = flags; |
278 | buf->tail = n; |
279 | /* |
280 | * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() |
281 | * ensures they see all buffer data. |
282 | */ |
283 | smp_store_release(&b->commit, b->used); |
284 | /* |
285 | * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() |
286 | * ensures the latest commit value can be read before the head |
287 | * is advanced to the next buffer. |
288 | */ |
289 | smp_store_release(&b->next, n); |
290 | |
291 | return size; |
292 | } |
293 | |
294 | int tty_buffer_request_room(struct tty_port *port, size_t size) |
295 | { |
296 | return __tty_buffer_request_room(port, size, flags: true); |
297 | } |
298 | EXPORT_SYMBOL_GPL(tty_buffer_request_room); |
299 | |
300 | size_t __tty_insert_flip_string_flags(struct tty_port *port, const u8 *chars, |
301 | const u8 *flags, bool mutable_flags, |
302 | size_t size) |
303 | { |
304 | bool need_flags = mutable_flags || flags[0] != TTY_NORMAL; |
305 | size_t copied = 0; |
306 | |
307 | do { |
308 | size_t goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); |
309 | size_t space = __tty_buffer_request_room(port, size: goal, flags: need_flags); |
310 | struct tty_buffer *tb = port->buf.tail; |
311 | |
312 | if (unlikely(space == 0)) |
313 | break; |
314 | |
315 | memcpy(char_buf_ptr(tb, tb->used), chars, space); |
316 | |
317 | if (mutable_flags) { |
318 | memcpy(flag_buf_ptr(tb, tb->used), flags, space); |
319 | flags += space; |
320 | } else if (tb->flags) { |
321 | memset(flag_buf_ptr(tb, tb->used), flags[0], space); |
322 | } else { |
323 | /* tb->flags should be available once requested */ |
324 | WARN_ON_ONCE(need_flags); |
325 | } |
326 | |
327 | tb->used += space; |
328 | copied += space; |
329 | chars += space; |
330 | |
331 | /* There is a small chance that we need to split the data over |
332 | * several buffers. If this is the case we must loop. |
333 | */ |
334 | } while (unlikely(size > copied)); |
335 | |
336 | return copied; |
337 | } |
338 | EXPORT_SYMBOL(__tty_insert_flip_string_flags); |
339 | |
340 | /** |
341 | * tty_prepare_flip_string - make room for characters |
342 | * @port: tty port |
343 | * @chars: return pointer for character write area |
344 | * @size: desired size |
345 | * |
346 | * Prepare a block of space in the buffer for data. |
347 | * |
348 | * This is used for drivers that need their own block copy routines into the |
349 | * buffer. There is no guarantee the buffer is a DMA target! |
350 | * |
351 | * Returns: the length available and buffer pointer (@chars) to the space which |
352 | * is now allocated and accounted for as ready for normal characters. |
353 | */ |
354 | size_t tty_prepare_flip_string(struct tty_port *port, u8 **chars, size_t size) |
355 | { |
356 | size_t space = __tty_buffer_request_room(port, size, flags: false); |
357 | |
358 | if (likely(space)) { |
359 | struct tty_buffer *tb = port->buf.tail; |
360 | |
361 | *chars = char_buf_ptr(b: tb, ofs: tb->used); |
362 | if (tb->flags) |
363 | memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space); |
364 | tb->used += space; |
365 | } |
366 | |
367 | return space; |
368 | } |
369 | EXPORT_SYMBOL_GPL(tty_prepare_flip_string); |
370 | |
371 | /** |
372 | * tty_ldisc_receive_buf - forward data to line discipline |
373 | * @ld: line discipline to process input |
374 | * @p: char buffer |
375 | * @f: %TTY_NORMAL, %TTY_BREAK, etc. flags buffer |
376 | * @count: number of bytes to process |
377 | * |
378 | * Callers other than flush_to_ldisc() need to exclude the kworker from |
379 | * concurrent use of the line discipline, see paste_selection(). |
380 | * |
381 | * Returns: the number of bytes processed. |
382 | */ |
383 | size_t tty_ldisc_receive_buf(struct tty_ldisc *ld, const u8 *p, const u8 *f, |
384 | size_t count) |
385 | { |
386 | if (ld->ops->receive_buf2) |
387 | count = ld->ops->receive_buf2(ld->tty, p, f, count); |
388 | else { |
389 | count = min_t(size_t, count, ld->tty->receive_room); |
390 | if (count && ld->ops->receive_buf) |
391 | ld->ops->receive_buf(ld->tty, p, f, count); |
392 | } |
393 | return count; |
394 | } |
395 | EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf); |
396 | |
397 | static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) |
398 | { |
399 | head->lookahead = max(head->lookahead, head->read); |
400 | |
401 | while (head) { |
402 | struct tty_buffer *next; |
403 | unsigned int count; |
404 | |
405 | /* |
406 | * Paired w/ release in __tty_buffer_request_room(); |
407 | * ensures commit value read is not stale if the head |
408 | * is advancing to the next buffer. |
409 | */ |
410 | next = smp_load_acquire(&head->next); |
411 | /* |
412 | * Paired w/ release in __tty_buffer_request_room() or in |
413 | * tty_buffer_flush(); ensures we see the committed buffer data. |
414 | */ |
415 | count = smp_load_acquire(&head->commit) - head->lookahead; |
416 | if (!count) { |
417 | head = next; |
418 | continue; |
419 | } |
420 | |
421 | if (port->client_ops->lookahead_buf) { |
422 | u8 *p, *f = NULL; |
423 | |
424 | p = char_buf_ptr(b: head, ofs: head->lookahead); |
425 | if (head->flags) |
426 | f = flag_buf_ptr(b: head, ofs: head->lookahead); |
427 | |
428 | port->client_ops->lookahead_buf(port, p, f, count); |
429 | } |
430 | |
431 | head->lookahead += count; |
432 | } |
433 | } |
434 | |
435 | static size_t |
436 | receive_buf(struct tty_port *port, struct tty_buffer *head, size_t count) |
437 | { |
438 | u8 *p = char_buf_ptr(b: head, ofs: head->read); |
439 | const u8 *f = NULL; |
440 | size_t n; |
441 | |
442 | if (head->flags) |
443 | f = flag_buf_ptr(b: head, ofs: head->read); |
444 | |
445 | n = port->client_ops->receive_buf(port, p, f, count); |
446 | if (n > 0) |
447 | memset(p, 0, n); |
448 | return n; |
449 | } |
450 | |
451 | /** |
452 | * flush_to_ldisc - flush data from buffer to ldisc |
453 | * @work: tty structure passed from work queue. |
454 | * |
455 | * This routine is called out of the software interrupt to flush data from the |
456 | * buffer chain to the line discipline. |
457 | * |
458 | * The receive_buf() method is single threaded for each tty instance. |
459 | * |
460 | * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. |
461 | */ |
462 | static void flush_to_ldisc(struct work_struct *work) |
463 | { |
464 | struct tty_port *port = container_of(work, struct tty_port, buf.work); |
465 | struct tty_bufhead *buf = &port->buf; |
466 | |
467 | mutex_lock(&buf->lock); |
468 | |
469 | while (1) { |
470 | struct tty_buffer *head = buf->head; |
471 | struct tty_buffer *next; |
472 | size_t count, rcvd; |
473 | |
474 | /* Ldisc or user is trying to gain exclusive access */ |
475 | if (atomic_read(v: &buf->priority)) |
476 | break; |
477 | |
478 | /* paired w/ release in __tty_buffer_request_room(); |
479 | * ensures commit value read is not stale if the head |
480 | * is advancing to the next buffer |
481 | */ |
482 | next = smp_load_acquire(&head->next); |
483 | /* paired w/ release in __tty_buffer_request_room() or in |
484 | * tty_buffer_flush(); ensures we see the committed buffer data |
485 | */ |
486 | count = smp_load_acquire(&head->commit) - head->read; |
487 | if (!count) { |
488 | if (next == NULL) |
489 | break; |
490 | buf->head = next; |
491 | tty_buffer_free(port, b: head); |
492 | continue; |
493 | } |
494 | |
495 | rcvd = receive_buf(port, head, count); |
496 | head->read += rcvd; |
497 | if (rcvd < count) |
498 | lookahead_bufs(port, head); |
499 | if (!rcvd) |
500 | break; |
501 | |
502 | if (need_resched()) |
503 | cond_resched(); |
504 | } |
505 | |
506 | mutex_unlock(lock: &buf->lock); |
507 | |
508 | } |
509 | |
510 | static inline void tty_flip_buffer_commit(struct tty_buffer *tail) |
511 | { |
512 | /* |
513 | * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees |
514 | * buffer data. |
515 | */ |
516 | smp_store_release(&tail->commit, tail->used); |
517 | } |
518 | |
519 | /** |
520 | * tty_flip_buffer_push - push terminal buffers |
521 | * @port: tty port to push |
522 | * |
523 | * Queue a push of the terminal flip buffers to the line discipline. Can be |
524 | * called from IRQ/atomic context. |
525 | * |
526 | * In the event of the queue being busy for flipping the work will be held off |
527 | * and retried later. |
528 | */ |
529 | void tty_flip_buffer_push(struct tty_port *port) |
530 | { |
531 | struct tty_bufhead *buf = &port->buf; |
532 | |
533 | tty_flip_buffer_commit(tail: buf->tail); |
534 | queue_work(wq: system_unbound_wq, work: &buf->work); |
535 | } |
536 | EXPORT_SYMBOL(tty_flip_buffer_push); |
537 | |
538 | /** |
539 | * tty_insert_flip_string_and_push_buffer - add characters to the tty buffer and |
540 | * push |
541 | * @port: tty port |
542 | * @chars: characters |
543 | * @size: size |
544 | * |
545 | * The function combines tty_insert_flip_string() and tty_flip_buffer_push() |
546 | * with the exception of properly holding the @port->lock. |
547 | * |
548 | * To be used only internally (by pty currently). |
549 | * |
550 | * Returns: the number added. |
551 | */ |
552 | int tty_insert_flip_string_and_push_buffer(struct tty_port *port, |
553 | const u8 *chars, size_t size) |
554 | { |
555 | struct tty_bufhead *buf = &port->buf; |
556 | unsigned long flags; |
557 | |
558 | spin_lock_irqsave(&port->lock, flags); |
559 | size = tty_insert_flip_string(port, chars, size); |
560 | if (size) |
561 | tty_flip_buffer_commit(tail: buf->tail); |
562 | spin_unlock_irqrestore(lock: &port->lock, flags); |
563 | |
564 | queue_work(wq: system_unbound_wq, work: &buf->work); |
565 | |
566 | return size; |
567 | } |
568 | |
569 | /** |
570 | * tty_buffer_init - prepare a tty buffer structure |
571 | * @port: tty port to initialise |
572 | * |
573 | * Set up the initial state of the buffer management for a tty device. Must be |
574 | * called before the other tty buffer functions are used. |
575 | */ |
576 | void tty_buffer_init(struct tty_port *port) |
577 | { |
578 | struct tty_bufhead *buf = &port->buf; |
579 | |
580 | mutex_init(&buf->lock); |
581 | tty_buffer_reset(p: &buf->sentinel, size: 0); |
582 | buf->head = &buf->sentinel; |
583 | buf->tail = &buf->sentinel; |
584 | init_llist_head(list: &buf->free); |
585 | atomic_set(v: &buf->mem_used, i: 0); |
586 | atomic_set(v: &buf->priority, i: 0); |
587 | INIT_WORK(&buf->work, flush_to_ldisc); |
588 | buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT; |
589 | } |
590 | |
591 | /** |
592 | * tty_buffer_set_limit - change the tty buffer memory limit |
593 | * @port: tty port to change |
594 | * @limit: memory limit to set |
595 | * |
596 | * Change the tty buffer memory limit. |
597 | * |
598 | * Must be called before the other tty buffer functions are used. |
599 | */ |
600 | int tty_buffer_set_limit(struct tty_port *port, int limit) |
601 | { |
602 | if (limit < MIN_TTYB_SIZE) |
603 | return -EINVAL; |
604 | port->buf.mem_limit = limit; |
605 | return 0; |
606 | } |
607 | EXPORT_SYMBOL_GPL(tty_buffer_set_limit); |
608 | |
609 | /* slave ptys can claim nested buffer lock when handling BRK and INTR */ |
610 | void tty_buffer_set_lock_subclass(struct tty_port *port) |
611 | { |
612 | lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE); |
613 | } |
614 | |
615 | bool tty_buffer_restart_work(struct tty_port *port) |
616 | { |
617 | return queue_work(wq: system_unbound_wq, work: &port->buf.work); |
618 | } |
619 | |
620 | bool tty_buffer_cancel_work(struct tty_port *port) |
621 | { |
622 | return cancel_work_sync(work: &port->buf.work); |
623 | } |
624 | |
625 | void tty_buffer_flush_work(struct tty_port *port) |
626 | { |
627 | flush_work(work: &port->buf.work); |
628 | } |
629 | |