1 | /* SPDX-License-Identifier: MIT */ |
2 | /* |
3 | * Copyright © 2014-2019 Intel Corporation |
4 | */ |
5 | |
6 | #ifndef _INTEL_GUC_H_ |
7 | #define _INTEL_GUC_H_ |
8 | |
9 | #include <linux/delay.h> |
10 | #include <linux/iosys-map.h> |
11 | #include <linux/xarray.h> |
12 | |
13 | #include "intel_guc_ct.h" |
14 | #include "intel_guc_fw.h" |
15 | #include "intel_guc_fwif.h" |
16 | #include "intel_guc_log.h" |
17 | #include "intel_guc_reg.h" |
18 | #include "intel_guc_slpc_types.h" |
19 | #include "intel_uc_fw.h" |
20 | #include "intel_uncore.h" |
21 | #include "i915_utils.h" |
22 | #include "i915_vma.h" |
23 | |
24 | struct __guc_ads_blob; |
25 | struct intel_guc_state_capture; |
26 | |
27 | /** |
28 | * struct intel_guc - Top level structure of GuC. |
29 | * |
30 | * It handles firmware loading and manages client pool. intel_guc owns an |
31 | * i915_sched_engine for submission. |
32 | */ |
33 | struct intel_guc { |
34 | /** @fw: the GuC firmware */ |
35 | struct intel_uc_fw fw; |
36 | /** @log: sub-structure containing GuC log related data and objects */ |
37 | struct intel_guc_log log; |
38 | /** @ct: the command transport communication channel */ |
39 | struct intel_guc_ct ct; |
40 | /** @slpc: sub-structure containing SLPC related data and objects */ |
41 | struct intel_guc_slpc slpc; |
42 | /** @capture: the error-state-capture module's data and objects */ |
43 | struct intel_guc_state_capture *capture; |
44 | |
45 | /** @dbgfs_node: debugfs node */ |
46 | struct dentry *dbgfs_node; |
47 | |
48 | /** @sched_engine: Global engine used to submit requests to GuC */ |
49 | struct i915_sched_engine *sched_engine; |
50 | /** |
51 | * @stalled_request: if GuC can't process a request for any reason, we |
52 | * save it until GuC restarts processing. No other request can be |
53 | * submitted until the stalled request is processed. |
54 | */ |
55 | struct i915_request *stalled_request; |
56 | /** |
57 | * @submission_stall_reason: reason why submission is stalled |
58 | */ |
59 | enum { |
60 | STALL_NONE, |
61 | STALL_REGISTER_CONTEXT, |
62 | STALL_MOVE_LRC_TAIL, |
63 | STALL_ADD_REQUEST, |
64 | } submission_stall_reason; |
65 | |
66 | /* intel_guc_recv interrupt related state */ |
67 | /** @irq_lock: protects GuC irq state */ |
68 | spinlock_t irq_lock; |
69 | /** |
70 | * @msg_enabled_mask: mask of events that are processed when receiving |
71 | * an INTEL_GUC_ACTION_DEFAULT G2H message. |
72 | */ |
73 | unsigned int msg_enabled_mask; |
74 | |
75 | /** |
76 | * @outstanding_submission_g2h: number of outstanding GuC to Host |
77 | * responses related to GuC submission, used to determine if the GT is |
78 | * idle |
79 | */ |
80 | atomic_t outstanding_submission_g2h; |
81 | |
82 | /** @tlb_lookup: xarray to store all pending TLB invalidation requests */ |
83 | struct xarray tlb_lookup; |
84 | |
85 | /** |
86 | * @serial_slot: id to the initial waiter created in tlb_lookup, |
87 | * which is used only when failed to allocate new waiter. |
88 | */ |
89 | u32 serial_slot; |
90 | |
91 | /** @next_seqno: the next id (sequence number) to allocate. */ |
92 | u32 next_seqno; |
93 | |
94 | /** @interrupts: pointers to GuC interrupt-managing functions. */ |
95 | struct { |
96 | bool enabled; |
97 | void (*reset)(struct intel_guc *guc); |
98 | void (*enable)(struct intel_guc *guc); |
99 | void (*disable)(struct intel_guc *guc); |
100 | } interrupts; |
101 | |
102 | /** |
103 | * @submission_state: sub-structure for submission state protected by |
104 | * single lock |
105 | */ |
106 | struct { |
107 | /** |
108 | * @submission_state.lock: protects everything in |
109 | * submission_state, ce->guc_id.id, and ce->guc_id.ref |
110 | * when transitioning in and out of zero |
111 | */ |
112 | spinlock_t lock; |
113 | /** |
114 | * @submission_state.guc_ids: used to allocate new |
115 | * guc_ids, single-lrc |
116 | */ |
117 | struct ida guc_ids; |
118 | /** |
119 | * @submission_state.num_guc_ids: Number of guc_ids, selftest |
120 | * feature to be able to reduce this number while testing. |
121 | */ |
122 | int num_guc_ids; |
123 | /** |
124 | * @submission_state.guc_ids_bitmap: used to allocate |
125 | * new guc_ids, multi-lrc |
126 | */ |
127 | unsigned long *guc_ids_bitmap; |
128 | /** |
129 | * @submission_state.guc_id_list: list of intel_context |
130 | * with valid guc_ids but no refs |
131 | */ |
132 | struct list_head guc_id_list; |
133 | /** |
134 | * @submission_state.guc_ids_in_use: Number single-lrc |
135 | * guc_ids in use |
136 | */ |
137 | unsigned int guc_ids_in_use; |
138 | /** |
139 | * @submission_state.destroyed_contexts: list of contexts |
140 | * waiting to be destroyed (deregistered with the GuC) |
141 | */ |
142 | struct list_head destroyed_contexts; |
143 | /** |
144 | * @submission_state.destroyed_worker: worker to deregister |
145 | * contexts, need as we need to take a GT PM reference and |
146 | * can't from destroy function as it might be in an atomic |
147 | * context (no sleeping) |
148 | */ |
149 | struct work_struct destroyed_worker; |
150 | /** |
151 | * @submission_state.reset_fail_worker: worker to trigger |
152 | * a GT reset after an engine reset fails |
153 | */ |
154 | struct work_struct reset_fail_worker; |
155 | /** |
156 | * @submission_state.reset_fail_mask: mask of engines that |
157 | * failed to reset |
158 | */ |
159 | intel_engine_mask_t reset_fail_mask; |
160 | /** |
161 | * @submission_state.sched_disable_delay_ms: schedule |
162 | * disable delay, in ms, for contexts |
163 | */ |
164 | unsigned int sched_disable_delay_ms; |
165 | /** |
166 | * @submission_state.sched_disable_gucid_threshold: |
167 | * threshold of min remaining available guc_ids before |
168 | * we start bypassing the schedule disable delay |
169 | */ |
170 | unsigned int sched_disable_gucid_threshold; |
171 | } submission_state; |
172 | |
173 | /** |
174 | * @submission_supported: tracks whether we support GuC submission on |
175 | * the current platform |
176 | */ |
177 | bool submission_supported; |
178 | /** @submission_selected: tracks whether the user enabled GuC submission */ |
179 | bool submission_selected; |
180 | /** @submission_initialized: tracks whether GuC submission has been initialised */ |
181 | bool submission_initialized; |
182 | /** @submission_version: Submission API version of the currently loaded firmware */ |
183 | struct intel_uc_fw_ver submission_version; |
184 | |
185 | /** |
186 | * @rc_supported: tracks whether we support GuC rc on the current platform |
187 | */ |
188 | bool rc_supported; |
189 | /** @rc_selected: tracks whether the user enabled GuC rc */ |
190 | bool rc_selected; |
191 | |
192 | /** @ads_vma: object allocated to hold the GuC ADS */ |
193 | struct i915_vma *ads_vma; |
194 | /** @ads_map: contents of the GuC ADS */ |
195 | struct iosys_map ads_map; |
196 | /** @ads_regset_size: size of the save/restore regsets in the ADS */ |
197 | u32 ads_regset_size; |
198 | /** |
199 | * @ads_regset_count: number of save/restore registers in the ADS for |
200 | * each engine |
201 | */ |
202 | u32 ads_regset_count[I915_NUM_ENGINES]; |
203 | /** @ads_regset: save/restore regsets in the ADS */ |
204 | struct guc_mmio_reg *ads_regset; |
205 | /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */ |
206 | u32 ads_golden_ctxt_size; |
207 | /** @ads_capture_size: size of register lists in the ADS used for error capture */ |
208 | u32 ads_capture_size; |
209 | |
210 | /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */ |
211 | struct i915_vma *lrc_desc_pool_v69; |
212 | /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */ |
213 | void *lrc_desc_pool_vaddr_v69; |
214 | |
215 | /** |
216 | * @context_lookup: used to resolve intel_context from guc_id, if a |
217 | * context is present in this structure it is registered with the GuC |
218 | */ |
219 | struct xarray context_lookup; |
220 | |
221 | /** @params: Control params for fw initialization */ |
222 | u32 params[GUC_CTL_MAX_DWORDS]; |
223 | |
224 | /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */ |
225 | struct { |
226 | u32 base; |
227 | unsigned int count; |
228 | enum forcewake_domains fw_domains; |
229 | } send_regs; |
230 | |
231 | /** @notify_reg: register used to send interrupts to the GuC FW */ |
232 | i915_reg_t notify_reg; |
233 | |
234 | /** |
235 | * @mmio_msg: notification bitmask that the GuC writes in one of its |
236 | * registers when the CT channel is disabled, to be processed when the |
237 | * channel is back up. |
238 | */ |
239 | u32 mmio_msg; |
240 | |
241 | /** @send_mutex: used to serialize the intel_guc_send actions */ |
242 | struct mutex send_mutex; |
243 | |
244 | /** |
245 | * @timestamp: GT timestamp object that stores a copy of the timestamp |
246 | * and adjusts it for overflow using a worker. |
247 | */ |
248 | struct { |
249 | /** |
250 | * @timestamp.lock: Lock protecting the below fields and |
251 | * the engine stats. |
252 | */ |
253 | spinlock_t lock; |
254 | |
255 | /** |
256 | * @timestamp.gt_stamp: 64-bit extended value of the GT |
257 | * timestamp. |
258 | */ |
259 | u64 gt_stamp; |
260 | |
261 | /** |
262 | * @timestamp.ping_delay: Period for polling the GT |
263 | * timestamp for overflow. |
264 | */ |
265 | unsigned long ping_delay; |
266 | |
267 | /** |
268 | * @timestamp.work: Periodic work to adjust GT timestamp, |
269 | * engine and context usage for overflows. |
270 | */ |
271 | struct delayed_work work; |
272 | |
273 | /** |
274 | * @timestamp.shift: Right shift value for the gpm timestamp |
275 | */ |
276 | u32 shift; |
277 | |
278 | /** |
279 | * @timestamp.last_stat_jiffies: jiffies at last actual |
280 | * stats collection time. We use this timestamp to ensure |
281 | * we don't oversample the stats because runtime power |
282 | * management events can trigger stats collection at much |
283 | * higher rates than required. |
284 | */ |
285 | unsigned long last_stat_jiffies; |
286 | } timestamp; |
287 | |
288 | /** |
289 | * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset. |
290 | * Specifically used when the G2H handler wants to issue a reset. Resets |
291 | * require flushing the G2H queue. So, the G2H processing itself must not |
292 | * trigger a reset directly. Instead, go via this worker. |
293 | */ |
294 | struct work_struct dead_guc_worker; |
295 | /** |
296 | * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance |
297 | * used to prevent a fundamentally broken system from continuously |
298 | * reloading the GuC. |
299 | */ |
300 | unsigned long last_dead_guc_jiffies; |
301 | |
302 | #ifdef CONFIG_DRM_I915_SELFTEST |
303 | /** |
304 | * @number_guc_id_stolen: The number of guc_ids that have been stolen |
305 | */ |
306 | int number_guc_id_stolen; |
307 | /** |
308 | * @fast_response_selftest: Backdoor to CT handler for fast response selftest |
309 | */ |
310 | u32 fast_response_selftest; |
311 | #endif |
312 | }; |
313 | |
314 | struct intel_guc_tlb_wait { |
315 | struct wait_queue_head wq; |
316 | bool busy; |
317 | }; |
318 | |
319 | /* |
320 | * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8 |
321 | * integer works. |
322 | */ |
323 | #define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat)) |
324 | #define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch) |
325 | #define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version) |
326 | #define GUC_FIRMWARE_VER(guc) MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver) |
327 | |
328 | static inline struct intel_guc *log_to_guc(struct intel_guc_log *log) |
329 | { |
330 | return container_of(log, struct intel_guc, log); |
331 | } |
332 | |
333 | static |
334 | inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len) |
335 | { |
336 | return intel_guc_ct_send(ct: &guc->ct, action, len, NULL, response_buf_size: 0, flags: 0); |
337 | } |
338 | |
339 | static |
340 | inline int intel_guc_send_nb(struct intel_guc *guc, const u32 *action, u32 len, |
341 | u32 g2h_len_dw) |
342 | { |
343 | return intel_guc_ct_send(ct: &guc->ct, action, len, NULL, response_buf_size: 0, |
344 | MAKE_SEND_FLAGS(g2h_len_dw)); |
345 | } |
346 | |
347 | static inline int |
348 | intel_guc_send_and_receive(struct intel_guc *guc, const u32 *action, u32 len, |
349 | u32 *response_buf, u32 response_buf_size) |
350 | { |
351 | return intel_guc_ct_send(ct: &guc->ct, action, len, |
352 | response_buf, response_buf_size, flags: 0); |
353 | } |
354 | |
355 | static inline int intel_guc_send_busy_loop(struct intel_guc *guc, |
356 | const u32 *action, |
357 | u32 len, |
358 | u32 g2h_len_dw, |
359 | bool loop) |
360 | { |
361 | int err; |
362 | unsigned int sleep_period_ms = 1; |
363 | bool not_atomic = !in_atomic() && !irqs_disabled(); |
364 | |
365 | /* |
366 | * FIXME: Have caller pass in if we are in an atomic context to avoid |
367 | * using in_atomic(). It is likely safe here as we check for irqs |
368 | * disabled which basically all the spin locks in the i915 do but |
369 | * regardless this should be cleaned up. |
370 | */ |
371 | |
372 | /* No sleeping with spin locks, just busy loop */ |
373 | might_sleep_if(loop && not_atomic); |
374 | |
375 | retry: |
376 | err = intel_guc_send_nb(guc, action, len, g2h_len_dw); |
377 | if (unlikely(err == -EBUSY && loop)) { |
378 | if (likely(not_atomic)) { |
379 | if (msleep_interruptible(msecs: sleep_period_ms)) |
380 | return -EINTR; |
381 | sleep_period_ms = sleep_period_ms << 1; |
382 | } else { |
383 | cpu_relax(); |
384 | } |
385 | goto retry; |
386 | } |
387 | |
388 | return err; |
389 | } |
390 | |
391 | /* Only call this from the interrupt handler code */ |
392 | static inline void intel_guc_to_host_event_handler(struct intel_guc *guc) |
393 | { |
394 | if (guc->interrupts.enabled) |
395 | intel_guc_ct_event_handler(ct: &guc->ct); |
396 | } |
397 | |
398 | /* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */ |
399 | #define GUC_GGTT_TOP 0xFEE00000 |
400 | |
401 | /** |
402 | * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma |
403 | * @guc: intel_guc structure. |
404 | * @vma: i915 graphics virtual memory area. |
405 | * |
406 | * GuC does not allow any gfx GGTT address that falls into range |
407 | * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM. |
408 | * Currently, in order to exclude [0, ggtt.pin_bias) address space from |
409 | * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma() |
410 | * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias. |
411 | * |
412 | * Return: GGTT offset of the @vma. |
413 | */ |
414 | static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc, |
415 | struct i915_vma *vma) |
416 | { |
417 | u32 offset = i915_ggtt_offset(vma); |
418 | |
419 | GEM_BUG_ON(offset < i915_ggtt_pin_bias(vma)); |
420 | GEM_BUG_ON(range_overflows_t(u64, offset, vma->size, GUC_GGTT_TOP)); |
421 | |
422 | return offset; |
423 | } |
424 | |
425 | void intel_guc_init_early(struct intel_guc *guc); |
426 | void intel_guc_init_late(struct intel_guc *guc); |
427 | void intel_guc_init_send_regs(struct intel_guc *guc); |
428 | void intel_guc_write_params(struct intel_guc *guc); |
429 | int intel_guc_init(struct intel_guc *guc); |
430 | void intel_guc_fini(struct intel_guc *guc); |
431 | void intel_guc_notify(struct intel_guc *guc); |
432 | int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len, |
433 | u32 *response_buf, u32 response_buf_size); |
434 | int intel_guc_to_host_process_recv_msg(struct intel_guc *guc, |
435 | const u32 *payload, u32 len); |
436 | int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset); |
437 | int intel_guc_suspend(struct intel_guc *guc); |
438 | int intel_guc_resume(struct intel_guc *guc); |
439 | struct i915_vma *intel_guc_allocate_vma(struct intel_guc *guc, u32 size); |
440 | int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, |
441 | struct i915_vma **out_vma, void **out_vaddr); |
442 | int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value); |
443 | int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value); |
444 | |
445 | static inline bool intel_guc_is_supported(struct intel_guc *guc) |
446 | { |
447 | return intel_uc_fw_is_supported(uc_fw: &guc->fw); |
448 | } |
449 | |
450 | static inline bool intel_guc_is_wanted(struct intel_guc *guc) |
451 | { |
452 | return intel_uc_fw_is_enabled(uc_fw: &guc->fw); |
453 | } |
454 | |
455 | static inline bool intel_guc_is_used(struct intel_guc *guc) |
456 | { |
457 | GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED); |
458 | return intel_uc_fw_is_available(uc_fw: &guc->fw); |
459 | } |
460 | |
461 | static inline bool intel_guc_is_fw_running(struct intel_guc *guc) |
462 | { |
463 | return intel_uc_fw_is_running(uc_fw: &guc->fw); |
464 | } |
465 | |
466 | static inline bool intel_guc_is_ready(struct intel_guc *guc) |
467 | { |
468 | return intel_guc_is_fw_running(guc) && intel_guc_ct_enabled(ct: &guc->ct); |
469 | } |
470 | |
471 | static inline void intel_guc_reset_interrupts(struct intel_guc *guc) |
472 | { |
473 | guc->interrupts.reset(guc); |
474 | } |
475 | |
476 | static inline void intel_guc_enable_interrupts(struct intel_guc *guc) |
477 | { |
478 | guc->interrupts.enable(guc); |
479 | } |
480 | |
481 | static inline void intel_guc_disable_interrupts(struct intel_guc *guc) |
482 | { |
483 | guc->interrupts.disable(guc); |
484 | } |
485 | |
486 | static inline int intel_guc_sanitize(struct intel_guc *guc) |
487 | { |
488 | intel_uc_fw_sanitize(uc_fw: &guc->fw); |
489 | intel_guc_disable_interrupts(guc); |
490 | intel_guc_ct_sanitize(ct: &guc->ct); |
491 | guc->mmio_msg = 0; |
492 | |
493 | return 0; |
494 | } |
495 | |
496 | static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask) |
497 | { |
498 | spin_lock_irq(lock: &guc->irq_lock); |
499 | guc->msg_enabled_mask |= mask; |
500 | spin_unlock_irq(lock: &guc->irq_lock); |
501 | } |
502 | |
503 | static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask) |
504 | { |
505 | spin_lock_irq(lock: &guc->irq_lock); |
506 | guc->msg_enabled_mask &= ~mask; |
507 | spin_unlock_irq(lock: &guc->irq_lock); |
508 | } |
509 | |
510 | int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout); |
511 | |
512 | int intel_guc_deregister_done_process_msg(struct intel_guc *guc, |
513 | const u32 *msg, u32 len); |
514 | int intel_guc_sched_done_process_msg(struct intel_guc *guc, |
515 | const u32 *msg, u32 len); |
516 | int intel_guc_context_reset_process_msg(struct intel_guc *guc, |
517 | const u32 *msg, u32 len); |
518 | int intel_guc_engine_failure_process_msg(struct intel_guc *guc, |
519 | const u32 *msg, u32 len); |
520 | int intel_guc_error_capture_process_msg(struct intel_guc *guc, |
521 | const u32 *msg, u32 len); |
522 | int intel_guc_crash_process_msg(struct intel_guc *guc, u32 action); |
523 | |
524 | struct intel_engine_cs * |
525 | intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance); |
526 | |
527 | void intel_guc_find_hung_context(struct intel_engine_cs *engine); |
528 | |
529 | int intel_guc_global_policies_update(struct intel_guc *guc); |
530 | |
531 | void intel_guc_context_ban(struct intel_context *ce, struct i915_request *rq); |
532 | |
533 | void intel_guc_submission_reset_prepare(struct intel_guc *guc); |
534 | void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled); |
535 | void intel_guc_submission_reset_finish(struct intel_guc *guc); |
536 | void intel_guc_submission_cancel_requests(struct intel_guc *guc); |
537 | |
538 | void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p); |
539 | |
540 | void intel_guc_write_barrier(struct intel_guc *guc); |
541 | |
542 | void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p); |
543 | |
544 | int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc); |
545 | |
546 | bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc); |
547 | int intel_guc_invalidate_tlb_engines(struct intel_guc *guc); |
548 | int intel_guc_invalidate_tlb_guc(struct intel_guc *guc); |
549 | int intel_guc_tlb_invalidation_done(struct intel_guc *guc, |
550 | const u32 *payload, u32 len); |
551 | void wake_up_all_tlb_invalidate(struct intel_guc *guc); |
552 | #endif |
553 | |