1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright 2014-2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | */ |
23 | |
24 | #include <linux/mm_types.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/types.h> |
27 | #include <linux/sched/signal.h> |
28 | #include <linux/sched/mm.h> |
29 | #include <linux/uaccess.h> |
30 | #include <linux/mman.h> |
31 | #include <linux/memory.h> |
32 | #include "kfd_priv.h" |
33 | #include "kfd_events.h" |
34 | #include <linux/device.h> |
35 | |
36 | /* |
37 | * Wrapper around wait_queue_entry_t |
38 | */ |
39 | struct kfd_event_waiter { |
40 | wait_queue_entry_t wait; |
41 | struct kfd_event *event; /* Event to wait for */ |
42 | bool activated; /* Becomes true when event is signaled */ |
43 | bool event_age_enabled; /* set to true when last_event_age is non-zero */ |
44 | }; |
45 | |
46 | /* |
47 | * Each signal event needs a 64-bit signal slot where the signaler will write |
48 | * a 1 before sending an interrupt. (This is needed because some interrupts |
49 | * do not contain enough spare data bits to identify an event.) |
50 | * We get whole pages and map them to the process VA. |
51 | * Individual signal events use their event_id as slot index. |
52 | */ |
53 | struct kfd_signal_page { |
54 | uint64_t *kernel_address; |
55 | uint64_t __user *user_address; |
56 | bool need_to_free_pages; |
57 | }; |
58 | |
59 | static uint64_t *page_slots(struct kfd_signal_page *page) |
60 | { |
61 | return page->kernel_address; |
62 | } |
63 | |
64 | static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p) |
65 | { |
66 | void *backing_store; |
67 | struct kfd_signal_page *page; |
68 | |
69 | page = kzalloc(size: sizeof(*page), GFP_KERNEL); |
70 | if (!page) |
71 | return NULL; |
72 | |
73 | backing_store = (void *) __get_free_pages(GFP_KERNEL, |
74 | order: get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); |
75 | if (!backing_store) |
76 | goto fail_alloc_signal_store; |
77 | |
78 | /* Initialize all events to unsignaled */ |
79 | memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, |
80 | KFD_SIGNAL_EVENT_LIMIT * 8); |
81 | |
82 | page->kernel_address = backing_store; |
83 | page->need_to_free_pages = true; |
84 | pr_debug("Allocated new event signal page at %p, for process %p\n" , |
85 | page, p); |
86 | |
87 | return page; |
88 | |
89 | fail_alloc_signal_store: |
90 | kfree(objp: page); |
91 | return NULL; |
92 | } |
93 | |
94 | static int allocate_event_notification_slot(struct kfd_process *p, |
95 | struct kfd_event *ev, |
96 | const int *restore_id) |
97 | { |
98 | int id; |
99 | |
100 | if (!p->signal_page) { |
101 | p->signal_page = allocate_signal_page(p); |
102 | if (!p->signal_page) |
103 | return -ENOMEM; |
104 | /* Oldest user mode expects 256 event slots */ |
105 | p->signal_mapped_size = 256*8; |
106 | } |
107 | |
108 | if (restore_id) { |
109 | id = idr_alloc(&p->event_idr, ptr: ev, start: *restore_id, end: *restore_id + 1, |
110 | GFP_KERNEL); |
111 | } else { |
112 | /* |
113 | * Compatibility with old user mode: Only use signal slots |
114 | * user mode has mapped, may be less than |
115 | * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase |
116 | * of the event limit without breaking user mode. |
117 | */ |
118 | id = idr_alloc(&p->event_idr, ptr: ev, start: 0, end: p->signal_mapped_size / 8, |
119 | GFP_KERNEL); |
120 | } |
121 | if (id < 0) |
122 | return id; |
123 | |
124 | ev->event_id = id; |
125 | page_slots(page: p->signal_page)[id] = UNSIGNALED_EVENT_SLOT; |
126 | |
127 | return 0; |
128 | } |
129 | |
130 | /* |
131 | * Assumes that p->event_mutex or rcu_readlock is held and of course that p is |
132 | * not going away. |
133 | */ |
134 | static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id) |
135 | { |
136 | return idr_find(&p->event_idr, id); |
137 | } |
138 | |
139 | /** |
140 | * lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID |
141 | * @p: Pointer to struct kfd_process |
142 | * @id: ID to look up |
143 | * @bits: Number of valid bits in @id |
144 | * |
145 | * Finds the first signaled event with a matching partial ID. If no |
146 | * matching signaled event is found, returns NULL. In that case the |
147 | * caller should assume that the partial ID is invalid and do an |
148 | * exhaustive search of all siglaned events. |
149 | * |
150 | * If multiple events with the same partial ID signal at the same |
151 | * time, they will be found one interrupt at a time, not necessarily |
152 | * in the same order the interrupts occurred. As long as the number of |
153 | * interrupts is correct, all signaled events will be seen by the |
154 | * driver. |
155 | */ |
156 | static struct kfd_event *lookup_signaled_event_by_partial_id( |
157 | struct kfd_process *p, uint32_t id, uint32_t bits) |
158 | { |
159 | struct kfd_event *ev; |
160 | |
161 | if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT) |
162 | return NULL; |
163 | |
164 | /* Fast path for the common case that @id is not a partial ID |
165 | * and we only need a single lookup. |
166 | */ |
167 | if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) { |
168 | if (page_slots(page: p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) |
169 | return NULL; |
170 | |
171 | return idr_find(&p->event_idr, id); |
172 | } |
173 | |
174 | /* General case for partial IDs: Iterate over all matching IDs |
175 | * and find the first one that has signaled. |
176 | */ |
177 | for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) { |
178 | if (page_slots(page: p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) |
179 | continue; |
180 | |
181 | ev = idr_find(&p->event_idr, id); |
182 | } |
183 | |
184 | return ev; |
185 | } |
186 | |
187 | static int create_signal_event(struct file *devkfd, struct kfd_process *p, |
188 | struct kfd_event *ev, const int *restore_id) |
189 | { |
190 | int ret; |
191 | |
192 | if (p->signal_mapped_size && |
193 | p->signal_event_count == p->signal_mapped_size / 8) { |
194 | if (!p->signal_event_limit_reached) { |
195 | pr_debug("Signal event wasn't created because limit was reached\n" ); |
196 | p->signal_event_limit_reached = true; |
197 | } |
198 | return -ENOSPC; |
199 | } |
200 | |
201 | ret = allocate_event_notification_slot(p, ev, restore_id); |
202 | if (ret) { |
203 | pr_warn("Signal event wasn't created because out of kernel memory\n" ); |
204 | return ret; |
205 | } |
206 | |
207 | p->signal_event_count++; |
208 | |
209 | ev->user_signal_address = &p->signal_page->user_address[ev->event_id]; |
210 | pr_debug("Signal event number %zu created with id %d, address %p\n" , |
211 | p->signal_event_count, ev->event_id, |
212 | ev->user_signal_address); |
213 | |
214 | return 0; |
215 | } |
216 | |
217 | static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id) |
218 | { |
219 | int id; |
220 | |
221 | if (restore_id) |
222 | id = idr_alloc(&p->event_idr, ptr: ev, start: *restore_id, end: *restore_id + 1, |
223 | GFP_KERNEL); |
224 | else |
225 | /* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an |
226 | * intentional integer overflow to -1 without a compiler |
227 | * warning. idr_alloc treats a negative value as "maximum |
228 | * signed integer". |
229 | */ |
230 | id = idr_alloc(&p->event_idr, ptr: ev, KFD_FIRST_NONSIGNAL_EVENT_ID, |
231 | end: (uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1, |
232 | GFP_KERNEL); |
233 | |
234 | if (id < 0) |
235 | return id; |
236 | ev->event_id = id; |
237 | |
238 | return 0; |
239 | } |
240 | |
241 | int kfd_event_init_process(struct kfd_process *p) |
242 | { |
243 | int id; |
244 | |
245 | mutex_init(&p->event_mutex); |
246 | idr_init(idr: &p->event_idr); |
247 | p->signal_page = NULL; |
248 | p->signal_event_count = 1; |
249 | /* Allocate event ID 0. It is used for a fast path to ignore bogus events |
250 | * that are sent by the CP without a context ID |
251 | */ |
252 | id = idr_alloc(&p->event_idr, NULL, start: 0, end: 1, GFP_KERNEL); |
253 | if (id < 0) { |
254 | idr_destroy(&p->event_idr); |
255 | mutex_destroy(lock: &p->event_mutex); |
256 | return id; |
257 | } |
258 | return 0; |
259 | } |
260 | |
261 | static void destroy_event(struct kfd_process *p, struct kfd_event *ev) |
262 | { |
263 | struct kfd_event_waiter *waiter; |
264 | |
265 | /* Wake up pending waiters. They will return failure */ |
266 | spin_lock(lock: &ev->lock); |
267 | list_for_each_entry(waiter, &ev->wq.head, wait.entry) |
268 | WRITE_ONCE(waiter->event, NULL); |
269 | wake_up_all(&ev->wq); |
270 | spin_unlock(lock: &ev->lock); |
271 | |
272 | if (ev->type == KFD_EVENT_TYPE_SIGNAL || |
273 | ev->type == KFD_EVENT_TYPE_DEBUG) |
274 | p->signal_event_count--; |
275 | |
276 | idr_remove(&p->event_idr, id: ev->event_id); |
277 | kfree_rcu(ev, rcu); |
278 | } |
279 | |
280 | static void destroy_events(struct kfd_process *p) |
281 | { |
282 | struct kfd_event *ev; |
283 | uint32_t id; |
284 | |
285 | idr_for_each_entry(&p->event_idr, ev, id) |
286 | if (ev) |
287 | destroy_event(p, ev); |
288 | idr_destroy(&p->event_idr); |
289 | mutex_destroy(lock: &p->event_mutex); |
290 | } |
291 | |
292 | /* |
293 | * We assume that the process is being destroyed and there is no need to |
294 | * unmap the pages or keep bookkeeping data in order. |
295 | */ |
296 | static void shutdown_signal_page(struct kfd_process *p) |
297 | { |
298 | struct kfd_signal_page *page = p->signal_page; |
299 | |
300 | if (page) { |
301 | if (page->need_to_free_pages) |
302 | free_pages(addr: (unsigned long)page->kernel_address, |
303 | order: get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); |
304 | kfree(objp: page); |
305 | } |
306 | } |
307 | |
308 | void kfd_event_free_process(struct kfd_process *p) |
309 | { |
310 | destroy_events(p); |
311 | shutdown_signal_page(p); |
312 | } |
313 | |
314 | static bool event_can_be_gpu_signaled(const struct kfd_event *ev) |
315 | { |
316 | return ev->type == KFD_EVENT_TYPE_SIGNAL || |
317 | ev->type == KFD_EVENT_TYPE_DEBUG; |
318 | } |
319 | |
320 | static bool event_can_be_cpu_signaled(const struct kfd_event *ev) |
321 | { |
322 | return ev->type == KFD_EVENT_TYPE_SIGNAL; |
323 | } |
324 | |
325 | static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, |
326 | uint64_t size, uint64_t user_handle) |
327 | { |
328 | struct kfd_signal_page *page; |
329 | |
330 | if (p->signal_page) |
331 | return -EBUSY; |
332 | |
333 | page = kzalloc(size: sizeof(*page), GFP_KERNEL); |
334 | if (!page) |
335 | return -ENOMEM; |
336 | |
337 | /* Initialize all events to unsignaled */ |
338 | memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT, |
339 | KFD_SIGNAL_EVENT_LIMIT * 8); |
340 | |
341 | page->kernel_address = kernel_address; |
342 | |
343 | p->signal_page = page; |
344 | p->signal_mapped_size = size; |
345 | p->signal_handle = user_handle; |
346 | return 0; |
347 | } |
348 | |
349 | int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset) |
350 | { |
351 | struct kfd_node *kfd; |
352 | struct kfd_process_device *pdd; |
353 | void *mem, *kern_addr; |
354 | uint64_t size; |
355 | int err = 0; |
356 | |
357 | if (p->signal_page) { |
358 | pr_err("Event page is already set\n" ); |
359 | return -EINVAL; |
360 | } |
361 | |
362 | pdd = kfd_process_device_data_by_id(process: p, GET_GPU_ID(event_page_offset)); |
363 | if (!pdd) { |
364 | pr_err("Getting device by id failed in %s\n" , __func__); |
365 | return -EINVAL; |
366 | } |
367 | kfd = pdd->dev; |
368 | |
369 | pdd = kfd_bind_process_to_device(dev: kfd, p); |
370 | if (IS_ERR(ptr: pdd)) |
371 | return PTR_ERR(ptr: pdd); |
372 | |
373 | mem = kfd_process_device_translate_handle(p: pdd, |
374 | GET_IDR_HANDLE(event_page_offset)); |
375 | if (!mem) { |
376 | pr_err("Can't find BO, offset is 0x%llx\n" , event_page_offset); |
377 | return -EINVAL; |
378 | } |
379 | |
380 | err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(mem, kptr: &kern_addr, size: &size); |
381 | if (err) { |
382 | pr_err("Failed to map event page to kernel\n" ); |
383 | return err; |
384 | } |
385 | |
386 | err = kfd_event_page_set(p, kernel_address: kern_addr, size, user_handle: event_page_offset); |
387 | if (err) { |
388 | pr_err("Failed to set event page\n" ); |
389 | amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); |
390 | return err; |
391 | } |
392 | return err; |
393 | } |
394 | |
395 | int kfd_event_create(struct file *devkfd, struct kfd_process *p, |
396 | uint32_t event_type, bool auto_reset, uint32_t node_id, |
397 | uint32_t *event_id, uint32_t *event_trigger_data, |
398 | uint64_t *event_page_offset, uint32_t *event_slot_index) |
399 | { |
400 | int ret = 0; |
401 | struct kfd_event *ev = kzalloc(size: sizeof(*ev), GFP_KERNEL); |
402 | |
403 | if (!ev) |
404 | return -ENOMEM; |
405 | |
406 | ev->type = event_type; |
407 | ev->auto_reset = auto_reset; |
408 | ev->signaled = false; |
409 | |
410 | spin_lock_init(&ev->lock); |
411 | init_waitqueue_head(&ev->wq); |
412 | |
413 | *event_page_offset = 0; |
414 | |
415 | mutex_lock(&p->event_mutex); |
416 | |
417 | switch (event_type) { |
418 | case KFD_EVENT_TYPE_SIGNAL: |
419 | case KFD_EVENT_TYPE_DEBUG: |
420 | ret = create_signal_event(devkfd, p, ev, NULL); |
421 | if (!ret) { |
422 | *event_page_offset = KFD_MMAP_TYPE_EVENTS; |
423 | *event_slot_index = ev->event_id; |
424 | } |
425 | break; |
426 | default: |
427 | ret = create_other_event(p, ev, NULL); |
428 | break; |
429 | } |
430 | |
431 | if (!ret) { |
432 | *event_id = ev->event_id; |
433 | *event_trigger_data = ev->event_id; |
434 | ev->event_age = 1; |
435 | } else { |
436 | kfree(objp: ev); |
437 | } |
438 | |
439 | mutex_unlock(lock: &p->event_mutex); |
440 | |
441 | return ret; |
442 | } |
443 | |
444 | int kfd_criu_restore_event(struct file *devkfd, |
445 | struct kfd_process *p, |
446 | uint8_t __user *user_priv_ptr, |
447 | uint64_t *priv_data_offset, |
448 | uint64_t max_priv_data_size) |
449 | { |
450 | struct kfd_criu_event_priv_data *ev_priv; |
451 | struct kfd_event *ev = NULL; |
452 | int ret = 0; |
453 | |
454 | ev_priv = kmalloc(size: sizeof(*ev_priv), GFP_KERNEL); |
455 | if (!ev_priv) |
456 | return -ENOMEM; |
457 | |
458 | ev = kzalloc(size: sizeof(*ev), GFP_KERNEL); |
459 | if (!ev) { |
460 | ret = -ENOMEM; |
461 | goto exit; |
462 | } |
463 | |
464 | if (*priv_data_offset + sizeof(*ev_priv) > max_priv_data_size) { |
465 | ret = -EINVAL; |
466 | goto exit; |
467 | } |
468 | |
469 | ret = copy_from_user(to: ev_priv, from: user_priv_ptr + *priv_data_offset, n: sizeof(*ev_priv)); |
470 | if (ret) { |
471 | ret = -EFAULT; |
472 | goto exit; |
473 | } |
474 | *priv_data_offset += sizeof(*ev_priv); |
475 | |
476 | if (ev_priv->user_handle) { |
477 | ret = kfd_kmap_event_page(p, event_page_offset: ev_priv->user_handle); |
478 | if (ret) |
479 | goto exit; |
480 | } |
481 | |
482 | ev->type = ev_priv->type; |
483 | ev->auto_reset = ev_priv->auto_reset; |
484 | ev->signaled = ev_priv->signaled; |
485 | |
486 | spin_lock_init(&ev->lock); |
487 | init_waitqueue_head(&ev->wq); |
488 | |
489 | mutex_lock(&p->event_mutex); |
490 | switch (ev->type) { |
491 | case KFD_EVENT_TYPE_SIGNAL: |
492 | case KFD_EVENT_TYPE_DEBUG: |
493 | ret = create_signal_event(devkfd, p, ev, restore_id: &ev_priv->event_id); |
494 | break; |
495 | case KFD_EVENT_TYPE_MEMORY: |
496 | memcpy(&ev->memory_exception_data, |
497 | &ev_priv->memory_exception_data, |
498 | sizeof(struct kfd_hsa_memory_exception_data)); |
499 | |
500 | ret = create_other_event(p, ev, restore_id: &ev_priv->event_id); |
501 | break; |
502 | case KFD_EVENT_TYPE_HW_EXCEPTION: |
503 | memcpy(&ev->hw_exception_data, |
504 | &ev_priv->hw_exception_data, |
505 | sizeof(struct kfd_hsa_hw_exception_data)); |
506 | |
507 | ret = create_other_event(p, ev, restore_id: &ev_priv->event_id); |
508 | break; |
509 | } |
510 | mutex_unlock(lock: &p->event_mutex); |
511 | |
512 | exit: |
513 | if (ret) |
514 | kfree(objp: ev); |
515 | |
516 | kfree(objp: ev_priv); |
517 | |
518 | return ret; |
519 | } |
520 | |
521 | int kfd_criu_checkpoint_events(struct kfd_process *p, |
522 | uint8_t __user *user_priv_data, |
523 | uint64_t *priv_data_offset) |
524 | { |
525 | struct kfd_criu_event_priv_data *ev_privs; |
526 | int i = 0; |
527 | int ret = 0; |
528 | struct kfd_event *ev; |
529 | uint32_t ev_id; |
530 | |
531 | uint32_t num_events = kfd_get_num_events(p); |
532 | |
533 | if (!num_events) |
534 | return 0; |
535 | |
536 | ev_privs = kvzalloc(size: num_events * sizeof(*ev_privs), GFP_KERNEL); |
537 | if (!ev_privs) |
538 | return -ENOMEM; |
539 | |
540 | |
541 | idr_for_each_entry(&p->event_idr, ev, ev_id) { |
542 | struct kfd_criu_event_priv_data *ev_priv; |
543 | |
544 | /* |
545 | * Currently, all events have same size of private_data, but the current ioctl's |
546 | * and CRIU plugin supports private_data of variable sizes |
547 | */ |
548 | ev_priv = &ev_privs[i]; |
549 | |
550 | ev_priv->object_type = KFD_CRIU_OBJECT_TYPE_EVENT; |
551 | |
552 | /* We store the user_handle with the first event */ |
553 | if (i == 0 && p->signal_page) |
554 | ev_priv->user_handle = p->signal_handle; |
555 | |
556 | ev_priv->event_id = ev->event_id; |
557 | ev_priv->auto_reset = ev->auto_reset; |
558 | ev_priv->type = ev->type; |
559 | ev_priv->signaled = ev->signaled; |
560 | |
561 | if (ev_priv->type == KFD_EVENT_TYPE_MEMORY) |
562 | memcpy(&ev_priv->memory_exception_data, |
563 | &ev->memory_exception_data, |
564 | sizeof(struct kfd_hsa_memory_exception_data)); |
565 | else if (ev_priv->type == KFD_EVENT_TYPE_HW_EXCEPTION) |
566 | memcpy(&ev_priv->hw_exception_data, |
567 | &ev->hw_exception_data, |
568 | sizeof(struct kfd_hsa_hw_exception_data)); |
569 | |
570 | pr_debug("Checkpointed event[%d] id = 0x%08x auto_reset = %x type = %x signaled = %x\n" , |
571 | i, |
572 | ev_priv->event_id, |
573 | ev_priv->auto_reset, |
574 | ev_priv->type, |
575 | ev_priv->signaled); |
576 | i++; |
577 | } |
578 | |
579 | ret = copy_to_user(to: user_priv_data + *priv_data_offset, |
580 | from: ev_privs, n: num_events * sizeof(*ev_privs)); |
581 | if (ret) { |
582 | pr_err("Failed to copy events priv to user\n" ); |
583 | ret = -EFAULT; |
584 | } |
585 | |
586 | *priv_data_offset += num_events * sizeof(*ev_privs); |
587 | |
588 | kvfree(addr: ev_privs); |
589 | return ret; |
590 | } |
591 | |
592 | int kfd_get_num_events(struct kfd_process *p) |
593 | { |
594 | struct kfd_event *ev; |
595 | uint32_t id; |
596 | u32 num_events = 0; |
597 | |
598 | idr_for_each_entry(&p->event_idr, ev, id) |
599 | num_events++; |
600 | |
601 | return num_events; |
602 | } |
603 | |
604 | /* Assumes that p is current. */ |
605 | int kfd_event_destroy(struct kfd_process *p, uint32_t event_id) |
606 | { |
607 | struct kfd_event *ev; |
608 | int ret = 0; |
609 | |
610 | mutex_lock(&p->event_mutex); |
611 | |
612 | ev = lookup_event_by_id(p, id: event_id); |
613 | |
614 | if (ev) |
615 | destroy_event(p, ev); |
616 | else |
617 | ret = -EINVAL; |
618 | |
619 | mutex_unlock(lock: &p->event_mutex); |
620 | return ret; |
621 | } |
622 | |
623 | static void set_event(struct kfd_event *ev) |
624 | { |
625 | struct kfd_event_waiter *waiter; |
626 | |
627 | /* Auto reset if the list is non-empty and we're waking |
628 | * someone. waitqueue_active is safe here because we're |
629 | * protected by the ev->lock, which is also held when |
630 | * updating the wait queues in kfd_wait_on_events. |
631 | */ |
632 | ev->signaled = !ev->auto_reset || !waitqueue_active(wq_head: &ev->wq); |
633 | if (!(++ev->event_age)) { |
634 | /* Never wrap back to reserved/default event age 0/1 */ |
635 | ev->event_age = 2; |
636 | WARN_ONCE(1, "event_age wrap back!" ); |
637 | } |
638 | |
639 | list_for_each_entry(waiter, &ev->wq.head, wait.entry) |
640 | WRITE_ONCE(waiter->activated, true); |
641 | |
642 | wake_up_all(&ev->wq); |
643 | } |
644 | |
645 | /* Assumes that p is current. */ |
646 | int kfd_set_event(struct kfd_process *p, uint32_t event_id) |
647 | { |
648 | int ret = 0; |
649 | struct kfd_event *ev; |
650 | |
651 | rcu_read_lock(); |
652 | |
653 | ev = lookup_event_by_id(p, id: event_id); |
654 | if (!ev) { |
655 | ret = -EINVAL; |
656 | goto unlock_rcu; |
657 | } |
658 | spin_lock(lock: &ev->lock); |
659 | |
660 | if (event_can_be_cpu_signaled(ev)) |
661 | set_event(ev); |
662 | else |
663 | ret = -EINVAL; |
664 | |
665 | spin_unlock(lock: &ev->lock); |
666 | unlock_rcu: |
667 | rcu_read_unlock(); |
668 | return ret; |
669 | } |
670 | |
671 | static void reset_event(struct kfd_event *ev) |
672 | { |
673 | ev->signaled = false; |
674 | } |
675 | |
676 | /* Assumes that p is current. */ |
677 | int kfd_reset_event(struct kfd_process *p, uint32_t event_id) |
678 | { |
679 | int ret = 0; |
680 | struct kfd_event *ev; |
681 | |
682 | rcu_read_lock(); |
683 | |
684 | ev = lookup_event_by_id(p, id: event_id); |
685 | if (!ev) { |
686 | ret = -EINVAL; |
687 | goto unlock_rcu; |
688 | } |
689 | spin_lock(lock: &ev->lock); |
690 | |
691 | if (event_can_be_cpu_signaled(ev)) |
692 | reset_event(ev); |
693 | else |
694 | ret = -EINVAL; |
695 | |
696 | spin_unlock(lock: &ev->lock); |
697 | unlock_rcu: |
698 | rcu_read_unlock(); |
699 | return ret; |
700 | |
701 | } |
702 | |
703 | static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) |
704 | { |
705 | WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT); |
706 | } |
707 | |
708 | static void set_event_from_interrupt(struct kfd_process *p, |
709 | struct kfd_event *ev) |
710 | { |
711 | if (ev && event_can_be_gpu_signaled(ev)) { |
712 | acknowledge_signal(p, ev); |
713 | spin_lock(lock: &ev->lock); |
714 | set_event(ev); |
715 | spin_unlock(lock: &ev->lock); |
716 | } |
717 | } |
718 | |
719 | void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, |
720 | uint32_t valid_id_bits) |
721 | { |
722 | struct kfd_event *ev = NULL; |
723 | |
724 | /* |
725 | * Because we are called from arbitrary context (workqueue) as opposed |
726 | * to process context, kfd_process could attempt to exit while we are |
727 | * running so the lookup function increments the process ref count. |
728 | */ |
729 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); |
730 | |
731 | if (!p) |
732 | return; /* Presumably process exited. */ |
733 | |
734 | rcu_read_lock(); |
735 | |
736 | if (valid_id_bits) |
737 | ev = lookup_signaled_event_by_partial_id(p, id: partial_id, |
738 | bits: valid_id_bits); |
739 | if (ev) { |
740 | set_event_from_interrupt(p, ev); |
741 | } else if (p->signal_page) { |
742 | /* |
743 | * Partial ID lookup failed. Assume that the event ID |
744 | * in the interrupt payload was invalid and do an |
745 | * exhaustive search of signaled events. |
746 | */ |
747 | uint64_t *slots = page_slots(page: p->signal_page); |
748 | uint32_t id; |
749 | |
750 | if (valid_id_bits) |
751 | pr_debug_ratelimited("Partial ID invalid: %u (%u valid bits)\n" , |
752 | partial_id, valid_id_bits); |
753 | |
754 | if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT / 64) { |
755 | /* With relatively few events, it's faster to |
756 | * iterate over the event IDR |
757 | */ |
758 | idr_for_each_entry(&p->event_idr, ev, id) { |
759 | if (id >= KFD_SIGNAL_EVENT_LIMIT) |
760 | break; |
761 | |
762 | if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT) |
763 | set_event_from_interrupt(p, ev); |
764 | } |
765 | } else { |
766 | /* With relatively many events, it's faster to |
767 | * iterate over the signal slots and lookup |
768 | * only signaled events from the IDR. |
769 | */ |
770 | for (id = 1; id < KFD_SIGNAL_EVENT_LIMIT; id++) |
771 | if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT) { |
772 | ev = lookup_event_by_id(p, id); |
773 | set_event_from_interrupt(p, ev); |
774 | } |
775 | } |
776 | } |
777 | |
778 | rcu_read_unlock(); |
779 | kfd_unref_process(p); |
780 | } |
781 | |
782 | static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events) |
783 | { |
784 | struct kfd_event_waiter *event_waiters; |
785 | uint32_t i; |
786 | |
787 | event_waiters = kcalloc(n: num_events, size: sizeof(struct kfd_event_waiter), |
788 | GFP_KERNEL); |
789 | if (!event_waiters) |
790 | return NULL; |
791 | |
792 | for (i = 0; i < num_events; i++) |
793 | init_wait(&event_waiters[i].wait); |
794 | |
795 | return event_waiters; |
796 | } |
797 | |
798 | static int init_event_waiter(struct kfd_process *p, |
799 | struct kfd_event_waiter *waiter, |
800 | struct kfd_event_data *event_data) |
801 | { |
802 | struct kfd_event *ev = lookup_event_by_id(p, id: event_data->event_id); |
803 | |
804 | if (!ev) |
805 | return -EINVAL; |
806 | |
807 | spin_lock(lock: &ev->lock); |
808 | waiter->event = ev; |
809 | waiter->activated = ev->signaled; |
810 | ev->signaled = ev->signaled && !ev->auto_reset; |
811 | |
812 | /* last_event_age = 0 reserved for backward compatible */ |
813 | if (waiter->event->type == KFD_EVENT_TYPE_SIGNAL && |
814 | event_data->signal_event_data.last_event_age) { |
815 | waiter->event_age_enabled = true; |
816 | if (ev->event_age != event_data->signal_event_data.last_event_age) |
817 | waiter->activated = true; |
818 | } |
819 | |
820 | if (!waiter->activated) |
821 | add_wait_queue(wq_head: &ev->wq, wq_entry: &waiter->wait); |
822 | spin_unlock(lock: &ev->lock); |
823 | |
824 | return 0; |
825 | } |
826 | |
827 | /* test_event_condition - Test condition of events being waited for |
828 | * @all: Return completion only if all events have signaled |
829 | * @num_events: Number of events to wait for |
830 | * @event_waiters: Array of event waiters, one per event |
831 | * |
832 | * Returns KFD_IOC_WAIT_RESULT_COMPLETE if all (or one) event(s) have |
833 | * signaled. Returns KFD_IOC_WAIT_RESULT_TIMEOUT if no (or not all) |
834 | * events have signaled. Returns KFD_IOC_WAIT_RESULT_FAIL if any of |
835 | * the events have been destroyed. |
836 | */ |
837 | static uint32_t test_event_condition(bool all, uint32_t num_events, |
838 | struct kfd_event_waiter *event_waiters) |
839 | { |
840 | uint32_t i; |
841 | uint32_t activated_count = 0; |
842 | |
843 | for (i = 0; i < num_events; i++) { |
844 | if (!READ_ONCE(event_waiters[i].event)) |
845 | return KFD_IOC_WAIT_RESULT_FAIL; |
846 | |
847 | if (READ_ONCE(event_waiters[i].activated)) { |
848 | if (!all) |
849 | return KFD_IOC_WAIT_RESULT_COMPLETE; |
850 | |
851 | activated_count++; |
852 | } |
853 | } |
854 | |
855 | return activated_count == num_events ? |
856 | KFD_IOC_WAIT_RESULT_COMPLETE : KFD_IOC_WAIT_RESULT_TIMEOUT; |
857 | } |
858 | |
859 | /* |
860 | * Copy event specific data, if defined. |
861 | * Currently only memory exception events have additional data to copy to user |
862 | */ |
863 | static int copy_signaled_event_data(uint32_t num_events, |
864 | struct kfd_event_waiter *event_waiters, |
865 | struct kfd_event_data __user *data) |
866 | { |
867 | void *src; |
868 | void __user *dst; |
869 | struct kfd_event_waiter *waiter; |
870 | struct kfd_event *event; |
871 | uint32_t i, size = 0; |
872 | |
873 | for (i = 0; i < num_events; i++) { |
874 | waiter = &event_waiters[i]; |
875 | event = waiter->event; |
876 | if (!event) |
877 | return -EINVAL; /* event was destroyed */ |
878 | if (waiter->activated) { |
879 | if (event->type == KFD_EVENT_TYPE_MEMORY) { |
880 | dst = &data[i].memory_exception_data; |
881 | src = &event->memory_exception_data; |
882 | size = sizeof(struct kfd_hsa_memory_exception_data); |
883 | } else if (event->type == KFD_EVENT_TYPE_HW_EXCEPTION) { |
884 | dst = &data[i].memory_exception_data; |
885 | src = &event->hw_exception_data; |
886 | size = sizeof(struct kfd_hsa_hw_exception_data); |
887 | } else if (event->type == KFD_EVENT_TYPE_SIGNAL && |
888 | waiter->event_age_enabled) { |
889 | dst = &data[i].signal_event_data.last_event_age; |
890 | src = &event->event_age; |
891 | size = sizeof(u64); |
892 | } |
893 | if (size && copy_to_user(to: dst, from: src, n: size)) |
894 | return -EFAULT; |
895 | } |
896 | } |
897 | |
898 | return 0; |
899 | } |
900 | |
901 | static long user_timeout_to_jiffies(uint32_t user_timeout_ms) |
902 | { |
903 | if (user_timeout_ms == KFD_EVENT_TIMEOUT_IMMEDIATE) |
904 | return 0; |
905 | |
906 | if (user_timeout_ms == KFD_EVENT_TIMEOUT_INFINITE) |
907 | return MAX_SCHEDULE_TIMEOUT; |
908 | |
909 | /* |
910 | * msecs_to_jiffies interprets all values above 2^31-1 as infinite, |
911 | * but we consider them finite. |
912 | * This hack is wrong, but nobody is likely to notice. |
913 | */ |
914 | user_timeout_ms = min_t(uint32_t, user_timeout_ms, 0x7FFFFFFF); |
915 | |
916 | return msecs_to_jiffies(m: user_timeout_ms) + 1; |
917 | } |
918 | |
919 | static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters, |
920 | bool undo_auto_reset) |
921 | { |
922 | uint32_t i; |
923 | |
924 | for (i = 0; i < num_events; i++) |
925 | if (waiters[i].event) { |
926 | spin_lock(lock: &waiters[i].event->lock); |
927 | remove_wait_queue(wq_head: &waiters[i].event->wq, |
928 | wq_entry: &waiters[i].wait); |
929 | if (undo_auto_reset && waiters[i].activated && |
930 | waiters[i].event && waiters[i].event->auto_reset) |
931 | set_event(waiters[i].event); |
932 | spin_unlock(lock: &waiters[i].event->lock); |
933 | } |
934 | |
935 | kfree(objp: waiters); |
936 | } |
937 | |
938 | int kfd_wait_on_events(struct kfd_process *p, |
939 | uint32_t num_events, void __user *data, |
940 | bool all, uint32_t *user_timeout_ms, |
941 | uint32_t *wait_result) |
942 | { |
943 | struct kfd_event_data __user *events = |
944 | (struct kfd_event_data __user *) data; |
945 | uint32_t i; |
946 | int ret = 0; |
947 | |
948 | struct kfd_event_waiter *event_waiters = NULL; |
949 | long timeout = user_timeout_to_jiffies(user_timeout_ms: *user_timeout_ms); |
950 | |
951 | event_waiters = alloc_event_waiters(num_events); |
952 | if (!event_waiters) { |
953 | ret = -ENOMEM; |
954 | goto out; |
955 | } |
956 | |
957 | /* Use p->event_mutex here to protect against concurrent creation and |
958 | * destruction of events while we initialize event_waiters. |
959 | */ |
960 | mutex_lock(&p->event_mutex); |
961 | |
962 | for (i = 0; i < num_events; i++) { |
963 | struct kfd_event_data event_data; |
964 | |
965 | if (copy_from_user(to: &event_data, from: &events[i], |
966 | n: sizeof(struct kfd_event_data))) { |
967 | ret = -EFAULT; |
968 | goto out_unlock; |
969 | } |
970 | |
971 | ret = init_event_waiter(p, waiter: &event_waiters[i], event_data: &event_data); |
972 | if (ret) |
973 | goto out_unlock; |
974 | } |
975 | |
976 | /* Check condition once. */ |
977 | *wait_result = test_event_condition(all, num_events, event_waiters); |
978 | if (*wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) { |
979 | ret = copy_signaled_event_data(num_events, |
980 | event_waiters, data: events); |
981 | goto out_unlock; |
982 | } else if (WARN_ON(*wait_result == KFD_IOC_WAIT_RESULT_FAIL)) { |
983 | /* This should not happen. Events shouldn't be |
984 | * destroyed while we're holding the event_mutex |
985 | */ |
986 | goto out_unlock; |
987 | } |
988 | |
989 | mutex_unlock(lock: &p->event_mutex); |
990 | |
991 | while (true) { |
992 | if (fatal_signal_pending(current)) { |
993 | ret = -EINTR; |
994 | break; |
995 | } |
996 | |
997 | if (signal_pending(current)) { |
998 | ret = -ERESTARTSYS; |
999 | if (*user_timeout_ms != KFD_EVENT_TIMEOUT_IMMEDIATE && |
1000 | *user_timeout_ms != KFD_EVENT_TIMEOUT_INFINITE) |
1001 | *user_timeout_ms = jiffies_to_msecs( |
1002 | max(0l, timeout-1)); |
1003 | break; |
1004 | } |
1005 | |
1006 | /* Set task state to interruptible sleep before |
1007 | * checking wake-up conditions. A concurrent wake-up |
1008 | * will put the task back into runnable state. In that |
1009 | * case schedule_timeout will not put the task to |
1010 | * sleep and we'll get a chance to re-check the |
1011 | * updated conditions almost immediately. Otherwise, |
1012 | * this race condition would lead to a soft hang or a |
1013 | * very long sleep. |
1014 | */ |
1015 | set_current_state(TASK_INTERRUPTIBLE); |
1016 | |
1017 | *wait_result = test_event_condition(all, num_events, |
1018 | event_waiters); |
1019 | if (*wait_result != KFD_IOC_WAIT_RESULT_TIMEOUT) |
1020 | break; |
1021 | |
1022 | if (timeout <= 0) |
1023 | break; |
1024 | |
1025 | timeout = schedule_timeout(timeout); |
1026 | } |
1027 | __set_current_state(TASK_RUNNING); |
1028 | |
1029 | mutex_lock(&p->event_mutex); |
1030 | /* copy_signaled_event_data may sleep. So this has to happen |
1031 | * after the task state is set back to RUNNING. |
1032 | * |
1033 | * The event may also have been destroyed after signaling. So |
1034 | * copy_signaled_event_data also must confirm that the event |
1035 | * still exists. Therefore this must be under the p->event_mutex |
1036 | * which is also held when events are destroyed. |
1037 | */ |
1038 | if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) |
1039 | ret = copy_signaled_event_data(num_events, |
1040 | event_waiters, data: events); |
1041 | |
1042 | out_unlock: |
1043 | free_waiters(num_events, waiters: event_waiters, undo_auto_reset: ret == -ERESTARTSYS); |
1044 | mutex_unlock(lock: &p->event_mutex); |
1045 | out: |
1046 | if (ret) |
1047 | *wait_result = KFD_IOC_WAIT_RESULT_FAIL; |
1048 | else if (*wait_result == KFD_IOC_WAIT_RESULT_FAIL) |
1049 | ret = -EIO; |
1050 | |
1051 | return ret; |
1052 | } |
1053 | |
1054 | int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) |
1055 | { |
1056 | unsigned long pfn; |
1057 | struct kfd_signal_page *page; |
1058 | int ret; |
1059 | |
1060 | /* check required size doesn't exceed the allocated size */ |
1061 | if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) < |
1062 | get_order(size: vma->vm_end - vma->vm_start)) { |
1063 | pr_err("Event page mmap requested illegal size\n" ); |
1064 | return -EINVAL; |
1065 | } |
1066 | |
1067 | page = p->signal_page; |
1068 | if (!page) { |
1069 | /* Probably KFD bug, but mmap is user-accessible. */ |
1070 | pr_debug("Signal page could not be found\n" ); |
1071 | return -EINVAL; |
1072 | } |
1073 | |
1074 | pfn = __pa(page->kernel_address); |
1075 | pfn >>= PAGE_SHIFT; |
1076 | |
1077 | vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
1078 | | VM_DONTDUMP | VM_PFNMAP); |
1079 | |
1080 | pr_debug("Mapping signal page\n" ); |
1081 | pr_debug(" start user address == 0x%08lx\n" , vma->vm_start); |
1082 | pr_debug(" end user address == 0x%08lx\n" , vma->vm_end); |
1083 | pr_debug(" pfn == 0x%016lX\n" , pfn); |
1084 | pr_debug(" vm_flags == 0x%08lX\n" , vma->vm_flags); |
1085 | pr_debug(" size == 0x%08lX\n" , |
1086 | vma->vm_end - vma->vm_start); |
1087 | |
1088 | page->user_address = (uint64_t __user *)vma->vm_start; |
1089 | |
1090 | /* mapping the page to user process */ |
1091 | ret = remap_pfn_range(vma, addr: vma->vm_start, pfn, |
1092 | size: vma->vm_end - vma->vm_start, vma->vm_page_prot); |
1093 | if (!ret) |
1094 | p->signal_mapped_size = vma->vm_end - vma->vm_start; |
1095 | |
1096 | return ret; |
1097 | } |
1098 | |
1099 | /* |
1100 | * Assumes that p is not going away. |
1101 | */ |
1102 | static void lookup_events_by_type_and_signal(struct kfd_process *p, |
1103 | int type, void *event_data) |
1104 | { |
1105 | struct kfd_hsa_memory_exception_data *ev_data; |
1106 | struct kfd_event *ev; |
1107 | uint32_t id; |
1108 | bool send_signal = true; |
1109 | |
1110 | ev_data = (struct kfd_hsa_memory_exception_data *) event_data; |
1111 | |
1112 | rcu_read_lock(); |
1113 | |
1114 | id = KFD_FIRST_NONSIGNAL_EVENT_ID; |
1115 | idr_for_each_entry_continue(&p->event_idr, ev, id) |
1116 | if (ev->type == type) { |
1117 | send_signal = false; |
1118 | dev_dbg(kfd_device, |
1119 | "Event found: id %X type %d" , |
1120 | ev->event_id, ev->type); |
1121 | spin_lock(lock: &ev->lock); |
1122 | set_event(ev); |
1123 | if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data) |
1124 | ev->memory_exception_data = *ev_data; |
1125 | spin_unlock(lock: &ev->lock); |
1126 | } |
1127 | |
1128 | if (type == KFD_EVENT_TYPE_MEMORY) { |
1129 | dev_warn(kfd_device, |
1130 | "Sending SIGSEGV to process %d (pasid 0x%x)" , |
1131 | p->lead_thread->pid, p->pasid); |
1132 | send_sig(SIGSEGV, p->lead_thread, 0); |
1133 | } |
1134 | |
1135 | /* Send SIGTERM no event of type "type" has been found*/ |
1136 | if (send_signal) { |
1137 | if (send_sigterm) { |
1138 | dev_warn(kfd_device, |
1139 | "Sending SIGTERM to process %d (pasid 0x%x)" , |
1140 | p->lead_thread->pid, p->pasid); |
1141 | send_sig(SIGTERM, p->lead_thread, 0); |
1142 | } else { |
1143 | dev_err(kfd_device, |
1144 | "Process %d (pasid 0x%x) got unhandled exception" , |
1145 | p->lead_thread->pid, p->pasid); |
1146 | } |
1147 | } |
1148 | |
1149 | rcu_read_unlock(); |
1150 | } |
1151 | |
1152 | void kfd_signal_hw_exception_event(u32 pasid) |
1153 | { |
1154 | /* |
1155 | * Because we are called from arbitrary context (workqueue) as opposed |
1156 | * to process context, kfd_process could attempt to exit while we are |
1157 | * running so the lookup function increments the process ref count. |
1158 | */ |
1159 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); |
1160 | |
1161 | if (!p) |
1162 | return; /* Presumably process exited. */ |
1163 | |
1164 | lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL); |
1165 | kfd_unref_process(p); |
1166 | } |
1167 | |
1168 | void kfd_signal_vm_fault_event(struct kfd_node *dev, u32 pasid, |
1169 | struct kfd_vm_fault_info *info, |
1170 | struct kfd_hsa_memory_exception_data *data) |
1171 | { |
1172 | struct kfd_event *ev; |
1173 | uint32_t id; |
1174 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); |
1175 | struct kfd_hsa_memory_exception_data memory_exception_data; |
1176 | int user_gpu_id; |
1177 | |
1178 | if (!p) |
1179 | return; /* Presumably process exited. */ |
1180 | |
1181 | user_gpu_id = kfd_process_get_user_gpu_id(p, actual_gpu_id: dev->id); |
1182 | if (unlikely(user_gpu_id == -EINVAL)) { |
1183 | WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n" , dev->id); |
1184 | return; |
1185 | } |
1186 | |
1187 | /* SoC15 chips and onwards will pass in data from now on. */ |
1188 | if (!data) { |
1189 | memset(&memory_exception_data, 0, sizeof(memory_exception_data)); |
1190 | memory_exception_data.gpu_id = user_gpu_id; |
1191 | memory_exception_data.failure.imprecise = true; |
1192 | |
1193 | /* Set failure reason */ |
1194 | if (info) { |
1195 | memory_exception_data.va = (info->page_addr) << |
1196 | PAGE_SHIFT; |
1197 | memory_exception_data.failure.NotPresent = |
1198 | info->prot_valid ? 1 : 0; |
1199 | memory_exception_data.failure.NoExecute = |
1200 | info->prot_exec ? 1 : 0; |
1201 | memory_exception_data.failure.ReadOnly = |
1202 | info->prot_write ? 1 : 0; |
1203 | memory_exception_data.failure.imprecise = 0; |
1204 | } |
1205 | } |
1206 | |
1207 | rcu_read_lock(); |
1208 | |
1209 | id = KFD_FIRST_NONSIGNAL_EVENT_ID; |
1210 | idr_for_each_entry_continue(&p->event_idr, ev, id) |
1211 | if (ev->type == KFD_EVENT_TYPE_MEMORY) { |
1212 | spin_lock(lock: &ev->lock); |
1213 | ev->memory_exception_data = data ? *data : |
1214 | memory_exception_data; |
1215 | set_event(ev); |
1216 | spin_unlock(lock: &ev->lock); |
1217 | } |
1218 | |
1219 | rcu_read_unlock(); |
1220 | kfd_unref_process(p); |
1221 | } |
1222 | |
1223 | void kfd_signal_reset_event(struct kfd_node *dev) |
1224 | { |
1225 | struct kfd_hsa_hw_exception_data hw_exception_data; |
1226 | struct kfd_hsa_memory_exception_data memory_exception_data; |
1227 | struct kfd_process *p; |
1228 | struct kfd_event *ev; |
1229 | unsigned int temp; |
1230 | uint32_t id, idx; |
1231 | int reset_cause = atomic_read(v: &dev->sram_ecc_flag) ? |
1232 | KFD_HW_EXCEPTION_ECC : |
1233 | KFD_HW_EXCEPTION_GPU_HANG; |
1234 | |
1235 | /* Whole gpu reset caused by GPU hang and memory is lost */ |
1236 | memset(&hw_exception_data, 0, sizeof(hw_exception_data)); |
1237 | hw_exception_data.memory_lost = 1; |
1238 | hw_exception_data.reset_cause = reset_cause; |
1239 | |
1240 | memset(&memory_exception_data, 0, sizeof(memory_exception_data)); |
1241 | memory_exception_data.ErrorType = KFD_MEM_ERR_SRAM_ECC; |
1242 | memory_exception_data.failure.imprecise = true; |
1243 | |
1244 | idx = srcu_read_lock(ssp: &kfd_processes_srcu); |
1245 | hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { |
1246 | int user_gpu_id = kfd_process_get_user_gpu_id(p, actual_gpu_id: dev->id); |
1247 | |
1248 | if (unlikely(user_gpu_id == -EINVAL)) { |
1249 | WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n" , dev->id); |
1250 | continue; |
1251 | } |
1252 | |
1253 | rcu_read_lock(); |
1254 | |
1255 | id = KFD_FIRST_NONSIGNAL_EVENT_ID; |
1256 | idr_for_each_entry_continue(&p->event_idr, ev, id) { |
1257 | if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) { |
1258 | spin_lock(lock: &ev->lock); |
1259 | ev->hw_exception_data = hw_exception_data; |
1260 | ev->hw_exception_data.gpu_id = user_gpu_id; |
1261 | set_event(ev); |
1262 | spin_unlock(lock: &ev->lock); |
1263 | } |
1264 | if (ev->type == KFD_EVENT_TYPE_MEMORY && |
1265 | reset_cause == KFD_HW_EXCEPTION_ECC) { |
1266 | spin_lock(lock: &ev->lock); |
1267 | ev->memory_exception_data = memory_exception_data; |
1268 | ev->memory_exception_data.gpu_id = user_gpu_id; |
1269 | set_event(ev); |
1270 | spin_unlock(lock: &ev->lock); |
1271 | } |
1272 | } |
1273 | |
1274 | rcu_read_unlock(); |
1275 | } |
1276 | srcu_read_unlock(ssp: &kfd_processes_srcu, idx); |
1277 | } |
1278 | |
1279 | void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid) |
1280 | { |
1281 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); |
1282 | struct kfd_hsa_memory_exception_data memory_exception_data; |
1283 | struct kfd_hsa_hw_exception_data hw_exception_data; |
1284 | struct kfd_event *ev; |
1285 | uint32_t id = KFD_FIRST_NONSIGNAL_EVENT_ID; |
1286 | int user_gpu_id; |
1287 | |
1288 | if (!p) { |
1289 | dev_warn(dev->adev->dev, "Not find process with pasid:%d\n" , pasid); |
1290 | return; /* Presumably process exited. */ |
1291 | } |
1292 | |
1293 | user_gpu_id = kfd_process_get_user_gpu_id(p, actual_gpu_id: dev->id); |
1294 | if (unlikely(user_gpu_id == -EINVAL)) { |
1295 | WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n" , dev->id); |
1296 | return; |
1297 | } |
1298 | |
1299 | memset(&hw_exception_data, 0, sizeof(hw_exception_data)); |
1300 | hw_exception_data.gpu_id = user_gpu_id; |
1301 | hw_exception_data.memory_lost = 1; |
1302 | hw_exception_data.reset_cause = KFD_HW_EXCEPTION_ECC; |
1303 | |
1304 | memset(&memory_exception_data, 0, sizeof(memory_exception_data)); |
1305 | memory_exception_data.ErrorType = KFD_MEM_ERR_POISON_CONSUMED; |
1306 | memory_exception_data.gpu_id = user_gpu_id; |
1307 | memory_exception_data.failure.imprecise = true; |
1308 | |
1309 | rcu_read_lock(); |
1310 | |
1311 | idr_for_each_entry_continue(&p->event_idr, ev, id) { |
1312 | if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) { |
1313 | spin_lock(lock: &ev->lock); |
1314 | ev->hw_exception_data = hw_exception_data; |
1315 | set_event(ev); |
1316 | spin_unlock(lock: &ev->lock); |
1317 | } |
1318 | |
1319 | if (ev->type == KFD_EVENT_TYPE_MEMORY) { |
1320 | spin_lock(lock: &ev->lock); |
1321 | ev->memory_exception_data = memory_exception_data; |
1322 | set_event(ev); |
1323 | spin_unlock(lock: &ev->lock); |
1324 | } |
1325 | } |
1326 | |
1327 | dev_warn(dev->adev->dev, "Send SIGBUS to process %s(pasid:%d)\n" , |
1328 | p->lead_thread->comm, pasid); |
1329 | rcu_read_unlock(); |
1330 | |
1331 | /* user application will handle SIGBUS signal */ |
1332 | send_sig(SIGBUS, p->lead_thread, 0); |
1333 | |
1334 | kfd_unref_process(p); |
1335 | } |
1336 | |