1 | /* SPDX-License-Identifier: GPL-2.0 |
2 | * |
3 | * Copyright 2016-2023 HabanaLabs, Ltd. |
4 | * All Rights Reserved. |
5 | * |
6 | */ |
7 | |
8 | #ifndef HABANALABSP_H_ |
9 | #define HABANALABSP_H_ |
10 | |
11 | #include <linux/habanalabs/cpucp_if.h> |
12 | #include "../include/common/qman_if.h" |
13 | #include "../include/hw_ip/mmu/mmu_general.h" |
14 | #include <uapi/drm/habanalabs_accel.h> |
15 | |
16 | #include <linux/cdev.h> |
17 | #include <linux/iopoll.h> |
18 | #include <linux/irqreturn.h> |
19 | #include <linux/dma-direction.h> |
20 | #include <linux/scatterlist.h> |
21 | #include <linux/hashtable.h> |
22 | #include <linux/debugfs.h> |
23 | #include <linux/rwsem.h> |
24 | #include <linux/eventfd.h> |
25 | #include <linux/bitfield.h> |
26 | #include <linux/genalloc.h> |
27 | #include <linux/sched/signal.h> |
28 | #include <linux/io-64-nonatomic-lo-hi.h> |
29 | #include <linux/coresight.h> |
30 | #include <linux/dma-buf.h> |
31 | |
32 | #include <drm/drm_device.h> |
33 | #include <drm/drm_file.h> |
34 | |
35 | #include "security.h" |
36 | |
37 | #define HL_NAME "habanalabs" |
38 | |
39 | struct hl_device; |
40 | struct hl_fpriv; |
41 | |
42 | #define PCI_VENDOR_ID_HABANALABS 0x1da3 |
43 | |
44 | /* Use upper bits of mmap offset to store habana driver specific information. |
45 | * bits[63:59] - Encode mmap type |
46 | * bits[45:0] - mmap offset value |
47 | * |
48 | * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these |
49 | * defines are w.r.t to PAGE_SIZE |
50 | */ |
51 | #define HL_MMAP_TYPE_SHIFT (59 - PAGE_SHIFT) |
52 | #define HL_MMAP_TYPE_MASK (0x1full << HL_MMAP_TYPE_SHIFT) |
53 | #define HL_MMAP_TYPE_TS_BUFF (0x10ull << HL_MMAP_TYPE_SHIFT) |
54 | #define HL_MMAP_TYPE_BLOCK (0x4ull << HL_MMAP_TYPE_SHIFT) |
55 | #define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT) |
56 | |
57 | #define HL_MMAP_OFFSET_VALUE_MASK (0x1FFFFFFFFFFFull >> PAGE_SHIFT) |
58 | #define HL_MMAP_OFFSET_VALUE_GET(off) (off & HL_MMAP_OFFSET_VALUE_MASK) |
59 | |
60 | #define HL_PENDING_RESET_PER_SEC 10 |
61 | #define HL_PENDING_RESET_MAX_TRIALS 60 /* 10 minutes */ |
62 | #define HL_PENDING_RESET_LONG_SEC 60 |
63 | /* |
64 | * In device fini, wait 10 minutes for user processes to be terminated after we kill them. |
65 | * This is needed to prevent situation of clearing resources while user processes are still alive. |
66 | */ |
67 | #define HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI 600 |
68 | |
69 | #define HL_HARD_RESET_MAX_TIMEOUT 120 |
70 | #define HL_PLDM_HARD_RESET_MAX_TIMEOUT (HL_HARD_RESET_MAX_TIMEOUT * 3) |
71 | |
72 | #define HL_DEVICE_TIMEOUT_USEC 1000000 /* 1 s */ |
73 | |
74 | #define HL_HEARTBEAT_PER_USEC 5000000 /* 5 s */ |
75 | |
76 | #define HL_PLL_LOW_JOB_FREQ_USEC 5000000 /* 5 s */ |
77 | |
78 | #define HL_CPUCP_INFO_TIMEOUT_USEC 10000000 /* 10s */ |
79 | #define HL_CPUCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */ |
80 | #define HL_CPUCP_MON_DUMP_TIMEOUT_USEC 10000000 /* 10s */ |
81 | #define HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC 10000000 /* 10s */ |
82 | |
83 | #define HL_FW_STATUS_POLL_INTERVAL_USEC 10000 /* 10ms */ |
84 | #define HL_FW_COMMS_STATUS_PLDM_POLL_INTERVAL_USEC 1000000 /* 1s */ |
85 | |
86 | #define HL_PCI_ELBI_TIMEOUT_MSEC 10 /* 10ms */ |
87 | |
88 | #define HL_INVALID_QUEUE UINT_MAX |
89 | |
90 | #define HL_COMMON_USER_CQ_INTERRUPT_ID 0xFFF |
91 | #define HL_COMMON_DEC_INTERRUPT_ID 0xFFE |
92 | |
93 | #define HL_STATE_DUMP_HIST_LEN 5 |
94 | |
95 | /* Default value for device reset trigger , an invalid value */ |
96 | #define HL_RESET_TRIGGER_DEFAULT 0xFF |
97 | |
98 | #define OBJ_NAMES_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ |
99 | #define SYNC_TO_ENGINE_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ |
100 | |
101 | /* Memory */ |
102 | #define MEM_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ |
103 | |
104 | /* MMU */ |
105 | #define MMU_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ |
106 | |
107 | #define TIMESTAMP_FREE_NODES_NUM 512 |
108 | |
109 | /** |
110 | * enum hl_mmu_page_table_location - mmu page table location |
111 | * @MMU_DR_PGT: page-table is located on device DRAM. |
112 | * @MMU_HR_PGT: page-table is located on host memory. |
113 | * @MMU_NUM_PGT_LOCATIONS: number of page-table locations currently supported. |
114 | */ |
115 | enum hl_mmu_page_table_location { |
116 | MMU_DR_PGT = 0, /* device-dram-resident MMU PGT */ |
117 | MMU_HR_PGT, /* host resident MMU PGT */ |
118 | MMU_NUM_PGT_LOCATIONS /* num of PGT locations */ |
119 | }; |
120 | |
121 | /* |
122 | * HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream |
123 | * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream |
124 | */ |
125 | #define HL_RSVD_SOBS 2 |
126 | #define HL_RSVD_MONS 1 |
127 | |
128 | /* |
129 | * HL_COLLECTIVE_RSVD_MSTR_MONS 'collective' reserved monitors per QMAN stream |
130 | */ |
131 | #define HL_COLLECTIVE_RSVD_MSTR_MONS 2 |
132 | |
133 | #define HL_MAX_SOB_VAL (1 << 15) |
134 | |
135 | #define IS_POWER_OF_2(n) (n != 0 && ((n & (n - 1)) == 0)) |
136 | #define IS_MAX_PENDING_CS_VALID(n) (IS_POWER_OF_2(n) && (n > 1)) |
137 | |
138 | #define HL_PCI_NUM_BARS 6 |
139 | |
140 | /* Completion queue entry relates to completed job */ |
141 | #define HL_COMPLETION_MODE_JOB 0 |
142 | /* Completion queue entry relates to completed command submission */ |
143 | #define HL_COMPLETION_MODE_CS 1 |
144 | |
145 | #define HL_MAX_DCORES 8 |
146 | |
147 | /* DMA alloc/free wrappers */ |
148 | #define hl_asic_dma_alloc_coherent(hdev, size, dma_handle, flags) \ |
149 | hl_asic_dma_alloc_coherent_caller(hdev, size, dma_handle, flags, __func__) |
150 | |
151 | #define hl_asic_dma_pool_zalloc(hdev, size, mem_flags, dma_handle) \ |
152 | hl_asic_dma_pool_zalloc_caller(hdev, size, mem_flags, dma_handle, __func__) |
153 | |
154 | #define hl_asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle) \ |
155 | hl_asic_dma_free_coherent_caller(hdev, size, cpu_addr, dma_handle, __func__) |
156 | |
157 | #define hl_asic_dma_pool_free(hdev, vaddr, dma_addr) \ |
158 | hl_asic_dma_pool_free_caller(hdev, vaddr, dma_addr, __func__) |
159 | |
160 | #define hl_dma_map_sgtable(hdev, sgt, dir) \ |
161 | hl_dma_map_sgtable_caller(hdev, sgt, dir, __func__) |
162 | #define hl_dma_unmap_sgtable(hdev, sgt, dir) \ |
163 | hl_dma_unmap_sgtable_caller(hdev, sgt, dir, __func__) |
164 | |
165 | /* |
166 | * Reset Flags |
167 | * |
168 | * - HL_DRV_RESET_HARD |
169 | * If set do hard reset to all engines. If not set reset just |
170 | * compute/DMA engines. |
171 | * |
172 | * - HL_DRV_RESET_FROM_RESET_THR |
173 | * Set if the caller is the hard-reset thread |
174 | * |
175 | * - HL_DRV_RESET_HEARTBEAT |
176 | * Set if reset is due to heartbeat |
177 | * |
178 | * - HL_DRV_RESET_TDR |
179 | * Set if reset is due to TDR |
180 | * |
181 | * - HL_DRV_RESET_DEV_RELEASE |
182 | * Set if reset is due to device release |
183 | * |
184 | * - HL_DRV_RESET_BYPASS_REQ_TO_FW |
185 | * F/W will perform the reset. No need to ask it to reset the device. This is relevant |
186 | * only when running with secured f/w |
187 | * |
188 | * - HL_DRV_RESET_FW_FATAL_ERR |
189 | * Set if reset is due to a fatal error from FW |
190 | * |
191 | * - HL_DRV_RESET_DELAY |
192 | * Set if a delay should be added before the reset |
193 | * |
194 | * - HL_DRV_RESET_FROM_WD_THR |
195 | * Set if the caller is the device release watchdog thread |
196 | */ |
197 | |
198 | #define HL_DRV_RESET_HARD (1 << 0) |
199 | #define HL_DRV_RESET_FROM_RESET_THR (1 << 1) |
200 | #define HL_DRV_RESET_HEARTBEAT (1 << 2) |
201 | #define HL_DRV_RESET_TDR (1 << 3) |
202 | #define HL_DRV_RESET_DEV_RELEASE (1 << 4) |
203 | #define HL_DRV_RESET_BYPASS_REQ_TO_FW (1 << 5) |
204 | #define HL_DRV_RESET_FW_FATAL_ERR (1 << 6) |
205 | #define HL_DRV_RESET_DELAY (1 << 7) |
206 | #define HL_DRV_RESET_FROM_WD_THR (1 << 8) |
207 | |
208 | /* |
209 | * Security |
210 | */ |
211 | |
212 | #define HL_PB_SHARED 1 |
213 | #define HL_PB_NA 0 |
214 | #define HL_PB_SINGLE_INSTANCE 1 |
215 | #define HL_BLOCK_SIZE 0x1000 |
216 | #define HL_BLOCK_GLBL_ERR_MASK 0xF40 |
217 | #define HL_BLOCK_GLBL_ERR_ADDR 0xF44 |
218 | #define HL_BLOCK_GLBL_ERR_CAUSE 0xF48 |
219 | #define HL_BLOCK_GLBL_SEC_OFFS 0xF80 |
220 | #define HL_BLOCK_GLBL_SEC_SIZE (HL_BLOCK_SIZE - HL_BLOCK_GLBL_SEC_OFFS) |
221 | #define HL_BLOCK_GLBL_SEC_LEN (HL_BLOCK_GLBL_SEC_SIZE / sizeof(u32)) |
222 | #define UNSET_GLBL_SEC_BIT(array, b) ((array)[((b) / 32)] |= (1 << ((b) % 32))) |
223 | |
224 | enum hl_protection_levels { |
225 | SECURED_LVL, |
226 | PRIVILEGED_LVL, |
227 | NON_SECURED_LVL |
228 | }; |
229 | |
230 | /** |
231 | * struct iterate_module_ctx - HW module iterator |
232 | * @fn: function to apply to each HW module instance |
233 | * @data: optional internal data to the function iterator |
234 | * @rc: return code for optional use of iterator/iterator-caller |
235 | */ |
236 | struct iterate_module_ctx { |
237 | /* |
238 | * callback for the HW module iterator |
239 | * @hdev: pointer to the habanalabs device structure |
240 | * @block: block (ASIC specific definition can be dcore/hdcore) |
241 | * @inst: HW module instance within the block |
242 | * @offset: current HW module instance offset from the 1-st HW module instance |
243 | * in the 1-st block |
244 | * @ctx: the iterator context. |
245 | */ |
246 | void (*fn)(struct hl_device *hdev, int block, int inst, u32 offset, |
247 | struct iterate_module_ctx *ctx); |
248 | void *data; |
249 | int rc; |
250 | }; |
251 | |
252 | struct hl_block_glbl_sec { |
253 | u32 sec_array[HL_BLOCK_GLBL_SEC_LEN]; |
254 | }; |
255 | |
256 | #define HL_MAX_SOBS_PER_MONITOR 8 |
257 | |
258 | /** |
259 | * struct hl_gen_wait_properties - properties for generating a wait CB |
260 | * @data: command buffer |
261 | * @q_idx: queue id is used to extract fence register address |
262 | * @size: offset in command buffer |
263 | * @sob_base: SOB base to use in this wait CB |
264 | * @sob_val: SOB value to wait for |
265 | * @mon_id: monitor to use in this wait CB |
266 | * @sob_mask: each bit represents a SOB offset from sob_base to be used |
267 | */ |
268 | struct hl_gen_wait_properties { |
269 | void *data; |
270 | u32 q_idx; |
271 | u32 size; |
272 | u16 sob_base; |
273 | u16 sob_val; |
274 | u16 mon_id; |
275 | u8 sob_mask; |
276 | }; |
277 | |
278 | /** |
279 | * struct pgt_info - MMU hop page info. |
280 | * @node: hash linked-list node for the pgts on host (shadow pgts for device resident MMU and |
281 | * actual pgts for host resident MMU). |
282 | * @phys_addr: physical address of the pgt. |
283 | * @virt_addr: host virtual address of the pgt (see above device/host resident). |
284 | * @shadow_addr: shadow hop in the host for device resident MMU. |
285 | * @ctx: pointer to the owner ctx. |
286 | * @num_of_ptes: indicates how many ptes are used in the pgt. used only for dynamically |
287 | * allocated HOPs (all HOPs but HOP0) |
288 | * |
289 | * The MMU page tables hierarchy can be placed either on the device's DRAM (in which case shadow |
290 | * pgts will be stored on host memory) or on host memory (in which case no shadow is required). |
291 | * |
292 | * When a new level (hop) is needed during mapping this structure will be used to describe |
293 | * the newly allocated hop as well as to track number of PTEs in it. |
294 | * During unmapping, if no valid PTEs remained in the page of a newly allocated hop, it is |
295 | * freed with its pgt_info structure. |
296 | */ |
297 | struct pgt_info { |
298 | struct hlist_node node; |
299 | u64 phys_addr; |
300 | u64 virt_addr; |
301 | u64 shadow_addr; |
302 | struct hl_ctx *ctx; |
303 | int num_of_ptes; |
304 | }; |
305 | |
306 | /** |
307 | * enum hl_pci_match_mode - pci match mode per region |
308 | * @PCI_ADDRESS_MATCH_MODE: address match mode |
309 | * @PCI_BAR_MATCH_MODE: bar match mode |
310 | */ |
311 | enum hl_pci_match_mode { |
312 | PCI_ADDRESS_MATCH_MODE, |
313 | PCI_BAR_MATCH_MODE |
314 | }; |
315 | |
316 | /** |
317 | * enum hl_fw_component - F/W components to read version through registers. |
318 | * @FW_COMP_BOOT_FIT: boot fit. |
319 | * @FW_COMP_PREBOOT: preboot. |
320 | * @FW_COMP_LINUX: linux. |
321 | */ |
322 | enum hl_fw_component { |
323 | FW_COMP_BOOT_FIT, |
324 | FW_COMP_PREBOOT, |
325 | FW_COMP_LINUX, |
326 | }; |
327 | |
328 | /** |
329 | * enum hl_fw_types - F/W types present in the system |
330 | * @FW_TYPE_NONE: no FW component indication |
331 | * @FW_TYPE_LINUX: Linux image for device CPU |
332 | * @FW_TYPE_BOOT_CPU: Boot image for device CPU |
333 | * @FW_TYPE_PREBOOT_CPU: Indicates pre-loaded CPUs are present in the system |
334 | * (preboot, ppboot etc...) |
335 | * @FW_TYPE_ALL_TYPES: Mask for all types |
336 | */ |
337 | enum hl_fw_types { |
338 | FW_TYPE_NONE = 0x0, |
339 | FW_TYPE_LINUX = 0x1, |
340 | FW_TYPE_BOOT_CPU = 0x2, |
341 | FW_TYPE_PREBOOT_CPU = 0x4, |
342 | FW_TYPE_ALL_TYPES = |
343 | (FW_TYPE_LINUX | FW_TYPE_BOOT_CPU | FW_TYPE_PREBOOT_CPU) |
344 | }; |
345 | |
346 | /** |
347 | * enum hl_queue_type - Supported QUEUE types. |
348 | * @QUEUE_TYPE_NA: queue is not available. |
349 | * @QUEUE_TYPE_EXT: external queue which is a DMA channel that may access the |
350 | * host. |
351 | * @QUEUE_TYPE_INT: internal queue that performs DMA inside the device's |
352 | * memories and/or operates the compute engines. |
353 | * @QUEUE_TYPE_CPU: S/W queue for communication with the device's CPU. |
354 | * @QUEUE_TYPE_HW: queue of DMA and compute engines jobs, for which completion |
355 | * notifications are sent by H/W. |
356 | */ |
357 | enum hl_queue_type { |
358 | QUEUE_TYPE_NA, |
359 | QUEUE_TYPE_EXT, |
360 | QUEUE_TYPE_INT, |
361 | QUEUE_TYPE_CPU, |
362 | QUEUE_TYPE_HW |
363 | }; |
364 | |
365 | enum hl_cs_type { |
366 | CS_TYPE_DEFAULT, |
367 | CS_TYPE_SIGNAL, |
368 | CS_TYPE_WAIT, |
369 | CS_TYPE_COLLECTIVE_WAIT, |
370 | CS_RESERVE_SIGNALS, |
371 | CS_UNRESERVE_SIGNALS, |
372 | CS_TYPE_ENGINE_CORE, |
373 | CS_TYPE_ENGINES, |
374 | CS_TYPE_FLUSH_PCI_HBW_WRITES, |
375 | }; |
376 | |
377 | /* |
378 | * struct hl_inbound_pci_region - inbound region descriptor |
379 | * @mode: pci match mode for this region |
380 | * @addr: region target address |
381 | * @size: region size in bytes |
382 | * @offset_in_bar: offset within bar (address match mode) |
383 | * @bar: bar id |
384 | */ |
385 | struct hl_inbound_pci_region { |
386 | enum hl_pci_match_mode mode; |
387 | u64 addr; |
388 | u64 size; |
389 | u64 offset_in_bar; |
390 | u8 bar; |
391 | }; |
392 | |
393 | /* |
394 | * struct hl_outbound_pci_region - outbound region descriptor |
395 | * @addr: region target address |
396 | * @size: region size in bytes |
397 | */ |
398 | struct hl_outbound_pci_region { |
399 | u64 addr; |
400 | u64 size; |
401 | }; |
402 | |
403 | /* |
404 | * enum queue_cb_alloc_flags - Indicates queue support for CBs that |
405 | * allocated by Kernel or by User |
406 | * @CB_ALLOC_KERNEL: support only CBs that allocated by Kernel |
407 | * @CB_ALLOC_USER: support only CBs that allocated by User |
408 | */ |
409 | enum queue_cb_alloc_flags { |
410 | CB_ALLOC_KERNEL = 0x1, |
411 | CB_ALLOC_USER = 0x2 |
412 | }; |
413 | |
414 | /* |
415 | * struct hl_hw_sob - H/W SOB info. |
416 | * @hdev: habanalabs device structure. |
417 | * @kref: refcount of this SOB. The SOB will reset once the refcount is zero. |
418 | * @sob_id: id of this SOB. |
419 | * @sob_addr: the sob offset from the base address. |
420 | * @q_idx: the H/W queue that uses this SOB. |
421 | * @need_reset: reset indication set when switching to the other sob. |
422 | */ |
423 | struct hl_hw_sob { |
424 | struct hl_device *hdev; |
425 | struct kref kref; |
426 | u32 sob_id; |
427 | u32 sob_addr; |
428 | u32 q_idx; |
429 | bool need_reset; |
430 | }; |
431 | |
432 | enum hl_collective_mode { |
433 | HL_COLLECTIVE_NOT_SUPPORTED = 0x0, |
434 | HL_COLLECTIVE_MASTER = 0x1, |
435 | HL_COLLECTIVE_SLAVE = 0x2 |
436 | }; |
437 | |
438 | /** |
439 | * struct hw_queue_properties - queue information. |
440 | * @type: queue type. |
441 | * @cb_alloc_flags: bitmap which indicates if the hw queue supports CB |
442 | * that allocated by the Kernel driver and therefore, |
443 | * a CB handle can be provided for jobs on this queue. |
444 | * Otherwise, a CB address must be provided. |
445 | * @collective_mode: collective mode of current queue |
446 | * @q_dram_bd_address: PQ dram address, used when PQ need to reside in DRAM. |
447 | * @driver_only: true if only the driver is allowed to send a job to this queue, |
448 | * false otherwise. |
449 | * @binned: True if the queue is binned out and should not be used |
450 | * @supports_sync_stream: True if queue supports sync stream |
451 | * @dram_bd: True if the bd should be copied to dram, needed for PQ which has been allocated on dram |
452 | */ |
453 | struct hw_queue_properties { |
454 | enum hl_queue_type type; |
455 | enum queue_cb_alloc_flags cb_alloc_flags; |
456 | enum hl_collective_mode collective_mode; |
457 | u64 q_dram_bd_address; |
458 | u8 driver_only; |
459 | u8 binned; |
460 | u8 supports_sync_stream; |
461 | u8 dram_bd; |
462 | }; |
463 | |
464 | /** |
465 | * enum vm_type - virtual memory mapping request information. |
466 | * @VM_TYPE_USERPTR: mapping of user memory to device virtual address. |
467 | * @VM_TYPE_PHYS_PACK: mapping of DRAM memory to device virtual address. |
468 | */ |
469 | enum vm_type { |
470 | VM_TYPE_USERPTR = 0x1, |
471 | VM_TYPE_PHYS_PACK = 0x2 |
472 | }; |
473 | |
474 | /** |
475 | * enum mmu_op_flags - mmu operation relevant information. |
476 | * @MMU_OP_USERPTR: operation on user memory (host resident). |
477 | * @MMU_OP_PHYS_PACK: operation on DRAM (device resident). |
478 | * @MMU_OP_CLEAR_MEMCACHE: operation has to clear memcache. |
479 | * @MMU_OP_SKIP_LOW_CACHE_INV: operation is allowed to skip parts of cache invalidation. |
480 | */ |
481 | enum mmu_op_flags { |
482 | MMU_OP_USERPTR = 0x1, |
483 | MMU_OP_PHYS_PACK = 0x2, |
484 | MMU_OP_CLEAR_MEMCACHE = 0x4, |
485 | MMU_OP_SKIP_LOW_CACHE_INV = 0x8, |
486 | }; |
487 | |
488 | |
489 | /** |
490 | * enum hl_device_hw_state - H/W device state. use this to understand whether |
491 | * to do reset before hw_init or not |
492 | * @HL_DEVICE_HW_STATE_CLEAN: H/W state is clean. i.e. after hard reset |
493 | * @HL_DEVICE_HW_STATE_DIRTY: H/W state is dirty. i.e. we started to execute |
494 | * hw_init |
495 | */ |
496 | enum hl_device_hw_state { |
497 | HL_DEVICE_HW_STATE_CLEAN = 0, |
498 | HL_DEVICE_HW_STATE_DIRTY |
499 | }; |
500 | |
501 | #define HL_MMU_VA_ALIGNMENT_NOT_NEEDED 0 |
502 | |
503 | /** |
504 | * struct hl_mmu_properties - ASIC specific MMU address translation properties. |
505 | * @start_addr: virtual start address of the memory region. |
506 | * @end_addr: virtual end address of the memory region. |
507 | * @hop_shifts: array holds HOPs shifts. |
508 | * @hop_masks: array holds HOPs masks. |
509 | * @last_mask: mask to get the bit indicating this is the last hop. |
510 | * @pgt_size: size for page tables. |
511 | * @supported_pages_mask: bitmask for supported page size (relevant only for MMUs |
512 | * supporting multiple page size). |
513 | * @page_size: default page size used to allocate memory. |
514 | * @num_hops: The amount of hops supported by the translation table. |
515 | * @hop_table_size: HOP table size. |
516 | * @hop0_tables_total_size: total size for all HOP0 tables. |
517 | * @host_resident: Should the MMU page table reside in host memory or in the |
518 | * device DRAM. |
519 | */ |
520 | struct hl_mmu_properties { |
521 | u64 start_addr; |
522 | u64 end_addr; |
523 | u64 hop_shifts[MMU_HOP_MAX]; |
524 | u64 hop_masks[MMU_HOP_MAX]; |
525 | u64 last_mask; |
526 | u64 pgt_size; |
527 | u64 supported_pages_mask; |
528 | u32 page_size; |
529 | u32 num_hops; |
530 | u32 hop_table_size; |
531 | u32 hop0_tables_total_size; |
532 | u8 host_resident; |
533 | }; |
534 | |
535 | /** |
536 | * struct hl_hints_range - hint addresses reserved va range. |
537 | * @start_addr: start address of the va range. |
538 | * @end_addr: end address of the va range. |
539 | */ |
540 | struct hl_hints_range { |
541 | u64 start_addr; |
542 | u64 end_addr; |
543 | }; |
544 | |
545 | /** |
546 | * struct asic_fixed_properties - ASIC specific immutable properties. |
547 | * @hw_queues_props: H/W queues properties. |
548 | * @special_blocks: points to an array containing special blocks info. |
549 | * @skip_special_blocks_cfg: special blocks skip configs. |
550 | * @cpucp_info: received various information from CPU-CP regarding the H/W, e.g. |
551 | * available sensors. |
552 | * @uboot_ver: F/W U-boot version. |
553 | * @preboot_ver: F/W Preboot version. |
554 | * @dmmu: DRAM MMU address translation properties. |
555 | * @pmmu: PCI (host) MMU address translation properties. |
556 | * @pmmu_huge: PCI (host) MMU address translation properties for memory |
557 | * allocated with huge pages. |
558 | * @hints_dram_reserved_va_range: dram hint addresses reserved range. |
559 | * @hints_host_reserved_va_range: host hint addresses reserved range. |
560 | * @hints_host_hpage_reserved_va_range: host huge page hint addresses reserved range. |
561 | * @sram_base_address: SRAM physical start address. |
562 | * @sram_end_address: SRAM physical end address. |
563 | * @sram_user_base_address - SRAM physical start address for user access. |
564 | * @dram_base_address: DRAM physical start address. |
565 | * @dram_end_address: DRAM physical end address. |
566 | * @dram_user_base_address: DRAM physical start address for user access. |
567 | * @dram_size: DRAM total size. |
568 | * @dram_pci_bar_size: size of PCI bar towards DRAM. |
569 | * @max_power_default: max power of the device after reset. |
570 | * @dc_power_default: power consumed by the device in mode idle. |
571 | * @dram_size_for_default_page_mapping: DRAM size needed to map to avoid page |
572 | * fault. |
573 | * @pcie_dbi_base_address: Base address of the PCIE_DBI block. |
574 | * @pcie_aux_dbi_reg_addr: Address of the PCIE_AUX DBI register. |
575 | * @mmu_pgt_addr: base physical address in DRAM of MMU page tables. |
576 | * @mmu_dram_default_page_addr: DRAM default page physical address. |
577 | * @tpc_enabled_mask: which TPCs are enabled. |
578 | * @tpc_binning_mask: which TPCs are binned. 0 means usable and 1 means binned. |
579 | * @dram_enabled_mask: which DRAMs are enabled. |
580 | * @dram_binning_mask: which DRAMs are binned. 0 means usable, 1 means binned. |
581 | * @dram_hints_align_mask: dram va hint addresses alignment mask which is used |
582 | * for hints validity check. |
583 | * @cfg_base_address: config space base address. |
584 | * @mmu_cache_mng_addr: address of the MMU cache. |
585 | * @mmu_cache_mng_size: size of the MMU cache. |
586 | * @device_dma_offset_for_host_access: the offset to add to host DMA addresses |
587 | * to enable the device to access them. |
588 | * @host_base_address: host physical start address for host DMA from device |
589 | * @host_end_address: host physical end address for host DMA from device |
590 | * @max_freq_value: current max clk frequency. |
591 | * @engine_core_interrupt_reg_addr: interrupt register address for engine core to use |
592 | * in order to raise events toward FW. |
593 | * @clk_pll_index: clock PLL index that specify which PLL determines the clock |
594 | * we display to the user |
595 | * @mmu_pgt_size: MMU page tables total size. |
596 | * @mmu_pte_size: PTE size in MMU page tables. |
597 | * @dram_page_size: The DRAM physical page size. |
598 | * @cfg_size: configuration space size on SRAM. |
599 | * @sram_size: total size of SRAM. |
600 | * @max_asid: maximum number of open contexts (ASIDs). |
601 | * @num_of_events: number of possible internal H/W IRQs. |
602 | * @psoc_pci_pll_nr: PCI PLL NR value. |
603 | * @psoc_pci_pll_nf: PCI PLL NF value. |
604 | * @psoc_pci_pll_od: PCI PLL OD value. |
605 | * @psoc_pci_pll_div_factor: PCI PLL DIV FACTOR 1 value. |
606 | * @psoc_timestamp_frequency: frequency of the psoc timestamp clock. |
607 | * @high_pll: high PLL frequency used by the device. |
608 | * @cb_pool_cb_cnt: number of CBs in the CB pool. |
609 | * @cb_pool_cb_size: size of each CB in the CB pool. |
610 | * @decoder_enabled_mask: which decoders are enabled. |
611 | * @decoder_binning_mask: which decoders are binned, 0 means usable and 1 means binned. |
612 | * @rotator_enabled_mask: which rotators are enabled. |
613 | * @edma_enabled_mask: which EDMAs are enabled. |
614 | * @edma_binning_mask: which EDMAs are binned, 0 means usable and 1 means |
615 | * binned (at most one binned DMA). |
616 | * @max_pending_cs: maximum of concurrent pending command submissions |
617 | * @max_queues: maximum amount of queues in the system |
618 | * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu |
619 | * capabilities reported by FW, bit description |
620 | * can be found in CPU_BOOT_DEV_STS0 |
621 | * @fw_preboot_cpu_boot_dev_sts1: bitmap representation of preboot cpu |
622 | * capabilities reported by FW, bit description |
623 | * can be found in CPU_BOOT_DEV_STS1 |
624 | * @fw_bootfit_cpu_boot_dev_sts0: bitmap representation of boot cpu security |
625 | * status reported by FW, bit description can be |
626 | * found in CPU_BOOT_DEV_STS0 |
627 | * @fw_bootfit_cpu_boot_dev_sts1: bitmap representation of boot cpu security |
628 | * status reported by FW, bit description can be |
629 | * found in CPU_BOOT_DEV_STS1 |
630 | * @fw_app_cpu_boot_dev_sts0: bitmap representation of application security |
631 | * status reported by FW, bit description can be |
632 | * found in CPU_BOOT_DEV_STS0 |
633 | * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security |
634 | * status reported by FW, bit description can be |
635 | * found in CPU_BOOT_DEV_STS1 |
636 | * @max_dec: maximum number of decoders |
637 | * @hmmu_hif_enabled_mask: mask of HMMUs/HIFs that are not isolated (enabled) |
638 | * 1- enabled, 0- isolated. |
639 | * @faulty_dram_cluster_map: mask of faulty DRAM cluster. |
640 | * 1- faulty cluster, 0- good cluster. |
641 | * @xbar_edge_enabled_mask: mask of XBAR_EDGEs that are not isolated (enabled) |
642 | * 1- enabled, 0- isolated. |
643 | * @device_mem_alloc_default_page_size: may be different than dram_page_size only for ASICs for |
644 | * which the property supports_user_set_page_size is true |
645 | * (i.e. the DRAM supports multiple page sizes), otherwise |
646 | * it will shall be equal to dram_page_size. |
647 | * @num_engine_cores: number of engine cpu cores. |
648 | * @max_num_of_engines: maximum number of all engines in the ASIC. |
649 | * @num_of_special_blocks: special_blocks array size. |
650 | * @glbl_err_max_cause_num: global err max cause number. |
651 | * @hbw_flush_reg: register to read to generate HBW flush. value of 0 means HBW flush is |
652 | * not supported. |
653 | * @reserved_fw_mem_size: size of dram memory reserved for FW. |
654 | * @collective_first_sob: first sync object available for collective use |
655 | * @collective_first_mon: first monitor available for collective use |
656 | * @sync_stream_first_sob: first sync object available for sync stream use |
657 | * @sync_stream_first_mon: first monitor available for sync stream use |
658 | * @first_available_user_sob: first sob available for the user |
659 | * @first_available_user_mon: first monitor available for the user |
660 | * @first_available_user_interrupt: first available interrupt reserved for the user |
661 | * @first_available_cq: first available CQ for the user. |
662 | * @user_interrupt_count: number of user interrupts. |
663 | * @user_dec_intr_count: number of decoder interrupts exposed to user. |
664 | * @tpc_interrupt_id: interrupt id for TPC to use in order to raise events towards the host. |
665 | * @eq_interrupt_id: interrupt id for EQ, uses to synchronize EQ interrupts in hard-reset. |
666 | * @cache_line_size: device cache line size. |
667 | * @server_type: Server type that the ASIC is currently installed in. |
668 | * The value is according to enum hl_server_type in uapi file. |
669 | * @completion_queues_count: number of completion queues. |
670 | * @completion_mode: 0 - job based completion, 1 - cs based completion |
671 | * @mme_master_slave_mode: 0 - Each MME works independently, 1 - MME works |
672 | * in Master/Slave mode |
673 | * @fw_security_enabled: true if security measures are enabled in firmware, |
674 | * false otherwise |
675 | * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from |
676 | * BOOT_DEV_STS0 |
677 | * @fw_cpu_boot_dev_sts1_valid: status bits are valid and can be fetched from |
678 | * BOOT_DEV_STS1 |
679 | * @dram_supports_virtual_memory: is there an MMU towards the DRAM |
680 | * @hard_reset_done_by_fw: true if firmware is handling hard reset flow |
681 | * @num_functional_hbms: number of functional HBMs in each DCORE. |
682 | * @hints_range_reservation: device support hint addresses range reservation. |
683 | * @iatu_done_by_fw: true if iATU configuration is being done by FW. |
684 | * @dynamic_fw_load: is dynamic FW load is supported. |
685 | * @gic_interrupts_enable: true if FW is not blocking GIC controller, |
686 | * false otherwise. |
687 | * @use_get_power_for_reset_history: To support backward compatibility for Goya |
688 | * and Gaudi |
689 | * @supports_compute_reset: is a reset which is not a hard-reset supported by this asic. |
690 | * @allow_inference_soft_reset: true if the ASIC supports soft reset that is |
691 | * initiated by user or TDR. This is only true |
692 | * in inference ASICs, as there is no real-world |
693 | * use-case of doing soft-reset in training (due |
694 | * to the fact that training runs on multiple |
695 | * devices) |
696 | * @configurable_stop_on_err: is stop-on-error option configurable via debugfs. |
697 | * @set_max_power_on_device_init: true if need to set max power in F/W on device init. |
698 | * @supports_user_set_page_size: true if user can set the allocation page size. |
699 | * @dma_mask: the dma mask to be set for this device. |
700 | * @supports_advanced_cpucp_rc: true if new cpucp opcodes are supported. |
701 | * @supports_engine_modes: true if changing engines/engine_cores modes is supported. |
702 | * @support_dynamic_resereved_fw_size: true if we support dynamic reserved size for fw. |
703 | */ |
704 | struct asic_fixed_properties { |
705 | struct hw_queue_properties *hw_queues_props; |
706 | struct hl_special_block_info *special_blocks; |
707 | struct hl_skip_blocks_cfg skip_special_blocks_cfg; |
708 | struct cpucp_info cpucp_info; |
709 | char uboot_ver[VERSION_MAX_LEN]; |
710 | char preboot_ver[VERSION_MAX_LEN]; |
711 | struct hl_mmu_properties dmmu; |
712 | struct hl_mmu_properties pmmu; |
713 | struct hl_mmu_properties pmmu_huge; |
714 | struct hl_hints_range hints_dram_reserved_va_range; |
715 | struct hl_hints_range hints_host_reserved_va_range; |
716 | struct hl_hints_range hints_host_hpage_reserved_va_range; |
717 | u64 sram_base_address; |
718 | u64 sram_end_address; |
719 | u64 sram_user_base_address; |
720 | u64 dram_base_address; |
721 | u64 dram_end_address; |
722 | u64 dram_user_base_address; |
723 | u64 dram_size; |
724 | u64 dram_pci_bar_size; |
725 | u64 max_power_default; |
726 | u64 dc_power_default; |
727 | u64 dram_size_for_default_page_mapping; |
728 | u64 pcie_dbi_base_address; |
729 | u64 pcie_aux_dbi_reg_addr; |
730 | u64 mmu_pgt_addr; |
731 | u64 mmu_dram_default_page_addr; |
732 | u64 tpc_enabled_mask; |
733 | u64 tpc_binning_mask; |
734 | u64 dram_enabled_mask; |
735 | u64 dram_binning_mask; |
736 | u64 dram_hints_align_mask; |
737 | u64 cfg_base_address; |
738 | u64 mmu_cache_mng_addr; |
739 | u64 mmu_cache_mng_size; |
740 | u64 device_dma_offset_for_host_access; |
741 | u64 host_base_address; |
742 | u64 host_end_address; |
743 | u64 max_freq_value; |
744 | u64 engine_core_interrupt_reg_addr; |
745 | u32 clk_pll_index; |
746 | u32 mmu_pgt_size; |
747 | u32 mmu_pte_size; |
748 | u32 dram_page_size; |
749 | u32 cfg_size; |
750 | u32 sram_size; |
751 | u32 max_asid; |
752 | u32 num_of_events; |
753 | u32 psoc_pci_pll_nr; |
754 | u32 psoc_pci_pll_nf; |
755 | u32 psoc_pci_pll_od; |
756 | u32 psoc_pci_pll_div_factor; |
757 | u32 psoc_timestamp_frequency; |
758 | u32 high_pll; |
759 | u32 cb_pool_cb_cnt; |
760 | u32 cb_pool_cb_size; |
761 | u32 decoder_enabled_mask; |
762 | u32 decoder_binning_mask; |
763 | u32 rotator_enabled_mask; |
764 | u32 edma_enabled_mask; |
765 | u32 edma_binning_mask; |
766 | u32 max_pending_cs; |
767 | u32 max_queues; |
768 | u32 fw_preboot_cpu_boot_dev_sts0; |
769 | u32 fw_preboot_cpu_boot_dev_sts1; |
770 | u32 fw_bootfit_cpu_boot_dev_sts0; |
771 | u32 fw_bootfit_cpu_boot_dev_sts1; |
772 | u32 fw_app_cpu_boot_dev_sts0; |
773 | u32 fw_app_cpu_boot_dev_sts1; |
774 | u32 max_dec; |
775 | u32 hmmu_hif_enabled_mask; |
776 | u32 faulty_dram_cluster_map; |
777 | u32 xbar_edge_enabled_mask; |
778 | u32 device_mem_alloc_default_page_size; |
779 | u32 num_engine_cores; |
780 | u32 max_num_of_engines; |
781 | u32 num_of_special_blocks; |
782 | u32 glbl_err_max_cause_num; |
783 | u32 hbw_flush_reg; |
784 | u32 reserved_fw_mem_size; |
785 | u16 collective_first_sob; |
786 | u16 collective_first_mon; |
787 | u16 sync_stream_first_sob; |
788 | u16 sync_stream_first_mon; |
789 | u16 first_available_user_sob[HL_MAX_DCORES]; |
790 | u16 first_available_user_mon[HL_MAX_DCORES]; |
791 | u16 first_available_user_interrupt; |
792 | u16 first_available_cq[HL_MAX_DCORES]; |
793 | u16 user_interrupt_count; |
794 | u16 user_dec_intr_count; |
795 | u16 tpc_interrupt_id; |
796 | u16 eq_interrupt_id; |
797 | u16 cache_line_size; |
798 | u16 server_type; |
799 | u8 completion_queues_count; |
800 | u8 completion_mode; |
801 | u8 mme_master_slave_mode; |
802 | u8 fw_security_enabled; |
803 | u8 fw_cpu_boot_dev_sts0_valid; |
804 | u8 fw_cpu_boot_dev_sts1_valid; |
805 | u8 dram_supports_virtual_memory; |
806 | u8 hard_reset_done_by_fw; |
807 | u8 num_functional_hbms; |
808 | u8 hints_range_reservation; |
809 | u8 iatu_done_by_fw; |
810 | u8 dynamic_fw_load; |
811 | u8 gic_interrupts_enable; |
812 | u8 use_get_power_for_reset_history; |
813 | u8 supports_compute_reset; |
814 | u8 allow_inference_soft_reset; |
815 | u8 configurable_stop_on_err; |
816 | u8 set_max_power_on_device_init; |
817 | u8 supports_user_set_page_size; |
818 | u8 dma_mask; |
819 | u8 supports_advanced_cpucp_rc; |
820 | u8 supports_engine_modes; |
821 | u8 support_dynamic_resereved_fw_size; |
822 | }; |
823 | |
824 | /** |
825 | * struct hl_fence - software synchronization primitive |
826 | * @completion: fence is implemented using completion |
827 | * @refcount: refcount for this fence |
828 | * @cs_sequence: sequence of the corresponding command submission |
829 | * @stream_master_qid_map: streams masters QID bitmap to represent all streams |
830 | * masters QIDs that multi cs is waiting on |
831 | * @error: mark this fence with error |
832 | * @timestamp: timestamp upon completion |
833 | * @mcs_handling_done: indicates that corresponding command submission has |
834 | * finished msc handling, this does not mean it was part |
835 | * of the mcs |
836 | */ |
837 | struct hl_fence { |
838 | struct completion completion; |
839 | struct kref refcount; |
840 | u64 cs_sequence; |
841 | u32 stream_master_qid_map; |
842 | int error; |
843 | ktime_t timestamp; |
844 | u8 mcs_handling_done; |
845 | }; |
846 | |
847 | /** |
848 | * struct hl_cs_compl - command submission completion object. |
849 | * @base_fence: hl fence object. |
850 | * @lock: spinlock to protect fence. |
851 | * @hdev: habanalabs device structure. |
852 | * @hw_sob: the H/W SOB used in this signal/wait CS. |
853 | * @encaps_sig_hdl: encaps signals handler. |
854 | * @cs_seq: command submission sequence number. |
855 | * @type: type of the CS - signal/wait. |
856 | * @sob_val: the SOB value that is used in this signal/wait CS. |
857 | * @sob_group: the SOB group that is used in this collective wait CS. |
858 | * @encaps_signals: indication whether it's a completion object of cs with |
859 | * encaps signals or not. |
860 | */ |
861 | struct hl_cs_compl { |
862 | struct hl_fence base_fence; |
863 | spinlock_t lock; |
864 | struct hl_device *hdev; |
865 | struct hl_hw_sob *hw_sob; |
866 | struct hl_cs_encaps_sig_handle *encaps_sig_hdl; |
867 | u64 cs_seq; |
868 | enum hl_cs_type type; |
869 | u16 sob_val; |
870 | u16 sob_group; |
871 | bool encaps_signals; |
872 | }; |
873 | |
874 | /* |
875 | * Command Buffers |
876 | */ |
877 | |
878 | /** |
879 | * struct hl_ts_buff - describes a timestamp buffer. |
880 | * @kernel_buff_address: Holds the internal buffer's kernel virtual address. |
881 | * @user_buff_address: Holds the user buffer's kernel virtual address. |
882 | * @kernel_buff_size: Holds the internal kernel buffer size. |
883 | */ |
884 | struct hl_ts_buff { |
885 | void *kernel_buff_address; |
886 | void *user_buff_address; |
887 | u32 kernel_buff_size; |
888 | }; |
889 | |
890 | struct hl_mmap_mem_buf; |
891 | |
892 | /** |
893 | * struct hl_mem_mgr - describes unified memory manager for mappable memory chunks. |
894 | * @dev: back pointer to the owning device |
895 | * @lock: protects handles |
896 | * @handles: an idr holding all active handles to the memory buffers in the system. |
897 | */ |
898 | struct hl_mem_mgr { |
899 | struct device *dev; |
900 | spinlock_t lock; |
901 | struct idr handles; |
902 | }; |
903 | |
904 | /** |
905 | * struct hl_mmap_mem_buf_behavior - describes unified memory manager buffer behavior |
906 | * @topic: string identifier used for logging |
907 | * @mem_id: memory type identifier, embedded in the handle and used to identify |
908 | * the memory type by handle. |
909 | * @alloc: callback executed on buffer allocation, shall allocate the memory, |
910 | * set it under buffer private, and set mappable size. |
911 | * @mmap: callback executed on mmap, must map the buffer to vma |
912 | * @release: callback executed on release, must free the resources used by the buffer |
913 | */ |
914 | struct hl_mmap_mem_buf_behavior { |
915 | const char *topic; |
916 | u64 mem_id; |
917 | |
918 | int (*alloc)(struct hl_mmap_mem_buf *buf, gfp_t gfp, void *args); |
919 | int (*mmap)(struct hl_mmap_mem_buf *buf, struct vm_area_struct *vma, void *args); |
920 | void (*release)(struct hl_mmap_mem_buf *buf); |
921 | }; |
922 | |
923 | /** |
924 | * struct hl_mmap_mem_buf - describes a single unified memory buffer |
925 | * @behavior: buffer behavior |
926 | * @mmg: back pointer to the unified memory manager |
927 | * @refcount: reference counter for buffer users |
928 | * @private: pointer to buffer behavior private data |
929 | * @mmap: atomic boolean indicating whether or not the buffer is mapped right now |
930 | * @real_mapped_size: the actual size of buffer mapped, after part of it may be released, |
931 | * may change at runtime. |
932 | * @mappable_size: the original mappable size of the buffer, does not change after |
933 | * the allocation. |
934 | * @handle: the buffer id in mmg handles store |
935 | */ |
936 | struct hl_mmap_mem_buf { |
937 | struct hl_mmap_mem_buf_behavior *behavior; |
938 | struct hl_mem_mgr *mmg; |
939 | struct kref refcount; |
940 | void *private; |
941 | atomic_t mmap; |
942 | u64 real_mapped_size; |
943 | u64 mappable_size; |
944 | u64 handle; |
945 | }; |
946 | |
947 | /** |
948 | * struct hl_cb - describes a Command Buffer. |
949 | * @hdev: pointer to device this CB belongs to. |
950 | * @ctx: pointer to the CB owner's context. |
951 | * @buf: back pointer to the parent mappable memory buffer |
952 | * @debugfs_list: node in debugfs list of command buffers. |
953 | * @pool_list: node in pool list of command buffers. |
954 | * @kernel_address: Holds the CB's kernel virtual address. |
955 | * @virtual_addr: Holds the CB's virtual address. |
956 | * @bus_address: Holds the CB's DMA address. |
957 | * @size: holds the CB's size. |
958 | * @roundup_size: holds the cb size after roundup to page size. |
959 | * @cs_cnt: holds number of CS that this CB participates in. |
960 | * @is_handle_destroyed: atomic boolean indicating whether or not the CB handle was destroyed. |
961 | * @is_pool: true if CB was acquired from the pool, false otherwise. |
962 | * @is_internal: internally allocated |
963 | * @is_mmu_mapped: true if the CB is mapped to the device's MMU. |
964 | */ |
965 | struct hl_cb { |
966 | struct hl_device *hdev; |
967 | struct hl_ctx *ctx; |
968 | struct hl_mmap_mem_buf *buf; |
969 | struct list_head debugfs_list; |
970 | struct list_head pool_list; |
971 | void *kernel_address; |
972 | u64 virtual_addr; |
973 | dma_addr_t bus_address; |
974 | u32 size; |
975 | u32 roundup_size; |
976 | atomic_t cs_cnt; |
977 | atomic_t is_handle_destroyed; |
978 | u8 is_pool; |
979 | u8 is_internal; |
980 | u8 is_mmu_mapped; |
981 | }; |
982 | |
983 | |
984 | /* |
985 | * QUEUES |
986 | */ |
987 | |
988 | struct hl_cs_job; |
989 | |
990 | /* Queue length of external and HW queues */ |
991 | #define HL_QUEUE_LENGTH 4096 |
992 | #define HL_QUEUE_SIZE_IN_BYTES (HL_QUEUE_LENGTH * HL_BD_SIZE) |
993 | |
994 | #if (HL_MAX_JOBS_PER_CS > HL_QUEUE_LENGTH) |
995 | #error "HL_QUEUE_LENGTH must be greater than HL_MAX_JOBS_PER_CS" |
996 | #endif |
997 | |
998 | /* HL_CQ_LENGTH is in units of struct hl_cq_entry */ |
999 | #define HL_CQ_LENGTH HL_QUEUE_LENGTH |
1000 | #define HL_CQ_SIZE_IN_BYTES (HL_CQ_LENGTH * HL_CQ_ENTRY_SIZE) |
1001 | |
1002 | /* Must be power of 2 */ |
1003 | #define HL_EQ_LENGTH 64 |
1004 | #define HL_EQ_SIZE_IN_BYTES (HL_EQ_LENGTH * HL_EQ_ENTRY_SIZE) |
1005 | |
1006 | /* Host <-> CPU-CP shared memory size */ |
1007 | #define HL_CPU_ACCESSIBLE_MEM_SIZE SZ_2M |
1008 | |
1009 | /** |
1010 | * struct hl_sync_stream_properties - |
1011 | * describes a H/W queue sync stream properties |
1012 | * @hw_sob: array of the used H/W SOBs by this H/W queue. |
1013 | * @next_sob_val: the next value to use for the currently used SOB. |
1014 | * @base_sob_id: the base SOB id of the SOBs used by this queue. |
1015 | * @base_mon_id: the base MON id of the MONs used by this queue. |
1016 | * @collective_mstr_mon_id: the MON ids of the MONs used by this master queue |
1017 | * in order to sync with all slave queues. |
1018 | * @collective_slave_mon_id: the MON id used by this slave queue in order to |
1019 | * sync with its master queue. |
1020 | * @collective_sob_id: current SOB id used by this collective slave queue |
1021 | * to signal its collective master queue upon completion. |
1022 | * @curr_sob_offset: the id offset to the currently used SOB from the |
1023 | * HL_RSVD_SOBS that are being used by this queue. |
1024 | */ |
1025 | struct hl_sync_stream_properties { |
1026 | struct hl_hw_sob hw_sob[HL_RSVD_SOBS]; |
1027 | u16 next_sob_val; |
1028 | u16 base_sob_id; |
1029 | u16 base_mon_id; |
1030 | u16 collective_mstr_mon_id[HL_COLLECTIVE_RSVD_MSTR_MONS]; |
1031 | u16 collective_slave_mon_id; |
1032 | u16 collective_sob_id; |
1033 | u8 curr_sob_offset; |
1034 | }; |
1035 | |
1036 | /** |
1037 | * struct hl_encaps_signals_mgr - describes sync stream encapsulated signals |
1038 | * handlers manager |
1039 | * @lock: protects handles. |
1040 | * @handles: an idr to hold all encapsulated signals handles. |
1041 | */ |
1042 | struct hl_encaps_signals_mgr { |
1043 | spinlock_t lock; |
1044 | struct idr handles; |
1045 | }; |
1046 | |
1047 | /** |
1048 | * struct hl_hw_queue - describes a H/W transport queue. |
1049 | * @shadow_queue: pointer to a shadow queue that holds pointers to jobs. |
1050 | * @sync_stream_prop: sync stream queue properties |
1051 | * @queue_type: type of queue. |
1052 | * @collective_mode: collective mode of current queue |
1053 | * @kernel_address: holds the queue's kernel virtual address. |
1054 | * @bus_address: holds the queue's DMA address. |
1055 | * @pq_dram_address: hold the dram address when the PQ is allocated, used when dram_bd is true in |
1056 | * queue properites. |
1057 | * @pi: holds the queue's pi value. |
1058 | * @ci: holds the queue's ci value, AS CALCULATED BY THE DRIVER (not real ci). |
1059 | * @hw_queue_id: the id of the H/W queue. |
1060 | * @cq_id: the id for the corresponding CQ for this H/W queue. |
1061 | * @msi_vec: the IRQ number of the H/W queue. |
1062 | * @int_queue_len: length of internal queue (number of entries). |
1063 | * @valid: is the queue valid (we have array of 32 queues, not all of them |
1064 | * exist). |
1065 | * @supports_sync_stream: True if queue supports sync stream |
1066 | * @dram_bd: True if the bd should be copied to dram, needed for PQ which has been allocated on dram |
1067 | */ |
1068 | struct hl_hw_queue { |
1069 | struct hl_cs_job **shadow_queue; |
1070 | struct hl_sync_stream_properties sync_stream_prop; |
1071 | enum hl_queue_type queue_type; |
1072 | enum hl_collective_mode collective_mode; |
1073 | void *kernel_address; |
1074 | dma_addr_t bus_address; |
1075 | u64 pq_dram_address; |
1076 | u32 pi; |
1077 | atomic_t ci; |
1078 | u32 hw_queue_id; |
1079 | u32 cq_id; |
1080 | u32 msi_vec; |
1081 | u16 int_queue_len; |
1082 | u8 valid; |
1083 | u8 supports_sync_stream; |
1084 | u8 dram_bd; |
1085 | }; |
1086 | |
1087 | /** |
1088 | * struct hl_cq - describes a completion queue |
1089 | * @hdev: pointer to the device structure |
1090 | * @kernel_address: holds the queue's kernel virtual address |
1091 | * @bus_address: holds the queue's DMA address |
1092 | * @cq_idx: completion queue index in array |
1093 | * @hw_queue_id: the id of the matching H/W queue |
1094 | * @ci: ci inside the queue |
1095 | * @pi: pi inside the queue |
1096 | * @free_slots_cnt: counter of free slots in queue |
1097 | */ |
1098 | struct hl_cq { |
1099 | struct hl_device *hdev; |
1100 | void *kernel_address; |
1101 | dma_addr_t bus_address; |
1102 | u32 cq_idx; |
1103 | u32 hw_queue_id; |
1104 | u32 ci; |
1105 | u32 pi; |
1106 | atomic_t free_slots_cnt; |
1107 | }; |
1108 | |
1109 | enum hl_user_interrupt_type { |
1110 | HL_USR_INTERRUPT_CQ = 0, |
1111 | HL_USR_INTERRUPT_DECODER, |
1112 | HL_USR_INTERRUPT_TPC, |
1113 | HL_USR_INTERRUPT_UNEXPECTED |
1114 | }; |
1115 | |
1116 | /** |
1117 | * struct hl_ts_free_jobs - holds user interrupt ts free nodes related data |
1118 | * @free_nodes_pool: pool of nodes to be used for free timestamp jobs |
1119 | * @free_nodes_length: number of nodes in free_nodes_pool |
1120 | * @next_avail_free_node_idx: index of the next free node in the pool |
1121 | * |
1122 | * the free nodes pool must be protected by the user interrupt lock |
1123 | * to avoid race between different interrupts which are using the same |
1124 | * ts buffer with different offsets. |
1125 | */ |
1126 | struct hl_ts_free_jobs { |
1127 | struct timestamp_reg_free_node *free_nodes_pool; |
1128 | u32 free_nodes_length; |
1129 | u32 next_avail_free_node_idx; |
1130 | }; |
1131 | |
1132 | /** |
1133 | * struct hl_user_interrupt - holds user interrupt information |
1134 | * @hdev: pointer to the device structure |
1135 | * @ts_free_jobs_data: timestamp free jobs related data |
1136 | * @type: user interrupt type |
1137 | * @wait_list_head: head to the list of user threads pending on this interrupt |
1138 | * @ts_list_head: head to the list of timestamp records |
1139 | * @wait_list_lock: protects wait_list_head |
1140 | * @ts_list_lock: protects ts_list_head |
1141 | * @timestamp: last timestamp taken upon interrupt |
1142 | * @interrupt_id: msix interrupt id |
1143 | */ |
1144 | struct hl_user_interrupt { |
1145 | struct hl_device *hdev; |
1146 | struct hl_ts_free_jobs ts_free_jobs_data; |
1147 | enum hl_user_interrupt_type type; |
1148 | struct list_head wait_list_head; |
1149 | struct list_head ts_list_head; |
1150 | spinlock_t wait_list_lock; |
1151 | spinlock_t ts_list_lock; |
1152 | ktime_t timestamp; |
1153 | u32 interrupt_id; |
1154 | }; |
1155 | |
1156 | /** |
1157 | * struct timestamp_reg_free_node - holds the timestamp registration free objects node |
1158 | * @free_objects_node: node in the list free_obj_jobs |
1159 | * @cq_cb: pointer to cq command buffer to be freed |
1160 | * @buf: pointer to timestamp buffer to be freed |
1161 | * @in_use: indicates whether the node still in use in workqueue thread. |
1162 | * @dynamic_alloc: indicates whether the node was allocated dynamically in the interrupt handler |
1163 | */ |
1164 | struct timestamp_reg_free_node { |
1165 | struct list_head free_objects_node; |
1166 | struct hl_cb *cq_cb; |
1167 | struct hl_mmap_mem_buf *buf; |
1168 | atomic_t in_use; |
1169 | u8 dynamic_alloc; |
1170 | }; |
1171 | |
1172 | /* struct timestamp_reg_work_obj - holds the timestamp registration free objects job |
1173 | * the job will be to pass over the free_obj_jobs list and put refcount to objects |
1174 | * in each node of the list |
1175 | * @free_obj: workqueue object to free timestamp registration node objects |
1176 | * @hdev: pointer to the device structure |
1177 | * @free_obj_head: list of free jobs nodes (node type timestamp_reg_free_node) |
1178 | * @dynamic_alloc_free_obj_head: list of free jobs nodes which were dynamically allocated in the |
1179 | * interrupt handler. |
1180 | */ |
1181 | struct timestamp_reg_work_obj { |
1182 | struct work_struct free_obj; |
1183 | struct hl_device *hdev; |
1184 | struct list_head *free_obj_head; |
1185 | struct list_head *dynamic_alloc_free_obj_head; |
1186 | }; |
1187 | |
1188 | /* struct timestamp_reg_info - holds the timestamp registration related data. |
1189 | * @buf: pointer to the timestamp buffer which include both user/kernel buffers. |
1190 | * relevant only when doing timestamps records registration. |
1191 | * @cq_cb: pointer to CQ counter CB. |
1192 | * @interrupt: interrupt that the node hanged on it's wait list. |
1193 | * @timestamp_kernel_addr: timestamp handle address, where to set timestamp |
1194 | * relevant only when doing timestamps records |
1195 | * registration. |
1196 | * @in_use: indicates if the node already in use. relevant only when doing |
1197 | * timestamps records registration, since in this case the driver |
1198 | * will have it's own buffer which serve as a records pool instead of |
1199 | * allocating records dynamically. |
1200 | */ |
1201 | struct timestamp_reg_info { |
1202 | struct hl_mmap_mem_buf *buf; |
1203 | struct hl_cb *cq_cb; |
1204 | struct hl_user_interrupt *interrupt; |
1205 | u64 *timestamp_kernel_addr; |
1206 | bool in_use; |
1207 | }; |
1208 | |
1209 | /** |
1210 | * struct hl_user_pending_interrupt - holds a context to a user thread |
1211 | * pending on an interrupt |
1212 | * @ts_reg_info: holds the timestamps registration nodes info |
1213 | * @list_node: node in the list of user threads pending on an interrupt or timestamp |
1214 | * @fence: hl fence object for interrupt completion |
1215 | * @cq_target_value: CQ target value |
1216 | * @cq_kernel_addr: CQ kernel address, to be used in the cq interrupt |
1217 | * handler for target value comparison |
1218 | */ |
1219 | struct hl_user_pending_interrupt { |
1220 | struct timestamp_reg_info ts_reg_info; |
1221 | struct list_head list_node; |
1222 | struct hl_fence fence; |
1223 | u64 cq_target_value; |
1224 | u64 *cq_kernel_addr; |
1225 | }; |
1226 | |
1227 | /** |
1228 | * struct hl_eq - describes the event queue (single one per device) |
1229 | * @hdev: pointer to the device structure |
1230 | * @kernel_address: holds the queue's kernel virtual address |
1231 | * @bus_address: holds the queue's DMA address |
1232 | * @ci: ci inside the queue |
1233 | * @prev_eqe_index: the index of the previous event queue entry. The index of |
1234 | * the current entry's index must be +1 of the previous one. |
1235 | * @check_eqe_index: do we need to check the index of the current entry vs. the |
1236 | * previous one. This is for backward compatibility with older |
1237 | * firmwares |
1238 | */ |
1239 | struct hl_eq { |
1240 | struct hl_device *hdev; |
1241 | void *kernel_address; |
1242 | dma_addr_t bus_address; |
1243 | u32 ci; |
1244 | u32 prev_eqe_index; |
1245 | bool check_eqe_index; |
1246 | }; |
1247 | |
1248 | /** |
1249 | * struct hl_dec - describes a decoder sw instance. |
1250 | * @hdev: pointer to the device structure. |
1251 | * @abnrm_intr_work: workqueue work item to run when decoder generates an error interrupt. |
1252 | * @core_id: ID of the decoder. |
1253 | * @base_addr: base address of the decoder. |
1254 | */ |
1255 | struct hl_dec { |
1256 | struct hl_device *hdev; |
1257 | struct work_struct abnrm_intr_work; |
1258 | u32 core_id; |
1259 | u32 base_addr; |
1260 | }; |
1261 | |
1262 | /** |
1263 | * enum hl_asic_type - supported ASIC types. |
1264 | * @ASIC_INVALID: Invalid ASIC type. |
1265 | * @ASIC_GOYA: Goya device (HL-1000). |
1266 | * @ASIC_GAUDI: Gaudi device (HL-2000). |
1267 | * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000). |
1268 | * @ASIC_GAUDI2: Gaudi2 device. |
1269 | * @ASIC_GAUDI2B: Gaudi2B device. |
1270 | * @ASIC_GAUDI2C: Gaudi2C device. |
1271 | */ |
1272 | enum hl_asic_type { |
1273 | ASIC_INVALID, |
1274 | ASIC_GOYA, |
1275 | ASIC_GAUDI, |
1276 | ASIC_GAUDI_SEC, |
1277 | ASIC_GAUDI2, |
1278 | ASIC_GAUDI2B, |
1279 | ASIC_GAUDI2C, |
1280 | }; |
1281 | |
1282 | struct hl_cs_parser; |
1283 | |
1284 | /** |
1285 | * enum hl_pm_mng_profile - power management profile. |
1286 | * @PM_AUTO: internal clock is set by the Linux driver. |
1287 | * @PM_MANUAL: internal clock is set by the user. |
1288 | * @PM_LAST: last power management type. |
1289 | */ |
1290 | enum hl_pm_mng_profile { |
1291 | PM_AUTO = 1, |
1292 | PM_MANUAL, |
1293 | PM_LAST |
1294 | }; |
1295 | |
1296 | /** |
1297 | * enum hl_pll_frequency - PLL frequency. |
1298 | * @PLL_HIGH: high frequency. |
1299 | * @PLL_LOW: low frequency. |
1300 | * @PLL_LAST: last frequency values that were configured by the user. |
1301 | */ |
1302 | enum hl_pll_frequency { |
1303 | PLL_HIGH = 1, |
1304 | PLL_LOW, |
1305 | PLL_LAST |
1306 | }; |
1307 | |
1308 | #define PLL_REF_CLK 50 |
1309 | |
1310 | enum div_select_defs { |
1311 | DIV_SEL_REF_CLK = 0, |
1312 | DIV_SEL_PLL_CLK = 1, |
1313 | DIV_SEL_DIVIDED_REF = 2, |
1314 | DIV_SEL_DIVIDED_PLL = 3, |
1315 | }; |
1316 | |
1317 | enum debugfs_access_type { |
1318 | DEBUGFS_READ8, |
1319 | DEBUGFS_WRITE8, |
1320 | DEBUGFS_READ32, |
1321 | DEBUGFS_WRITE32, |
1322 | DEBUGFS_READ64, |
1323 | DEBUGFS_WRITE64, |
1324 | }; |
1325 | |
1326 | enum pci_region { |
1327 | PCI_REGION_CFG, |
1328 | PCI_REGION_SRAM, |
1329 | PCI_REGION_DRAM, |
1330 | PCI_REGION_SP_SRAM, |
1331 | PCI_REGION_NUMBER, |
1332 | }; |
1333 | |
1334 | /** |
1335 | * struct pci_mem_region - describe memory region in a PCI bar |
1336 | * @region_base: region base address |
1337 | * @region_size: region size |
1338 | * @bar_size: size of the BAR |
1339 | * @offset_in_bar: region offset into the bar |
1340 | * @bar_id: bar ID of the region |
1341 | * @used: if used 1, otherwise 0 |
1342 | */ |
1343 | struct pci_mem_region { |
1344 | u64 region_base; |
1345 | u64 region_size; |
1346 | u64 bar_size; |
1347 | u64 offset_in_bar; |
1348 | u8 bar_id; |
1349 | u8 used; |
1350 | }; |
1351 | |
1352 | /** |
1353 | * struct static_fw_load_mgr - static FW load manager |
1354 | * @preboot_version_max_off: max offset to preboot version |
1355 | * @boot_fit_version_max_off: max offset to boot fit version |
1356 | * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages |
1357 | * @cpu_cmd_status_to_host_reg: register address for CPU command status response |
1358 | * @cpu_boot_status_reg: boot status register |
1359 | * @cpu_boot_dev_status0_reg: boot device status register 0 |
1360 | * @cpu_boot_dev_status1_reg: boot device status register 1 |
1361 | * @boot_err0_reg: boot error register 0 |
1362 | * @boot_err1_reg: boot error register 1 |
1363 | * @preboot_version_offset_reg: SRAM offset to preboot version register |
1364 | * @boot_fit_version_offset_reg: SRAM offset to boot fit version register |
1365 | * @sram_offset_mask: mask for getting offset into the SRAM |
1366 | * @cpu_reset_wait_msec: used when setting WFE via kmd_msg_to_cpu_reg |
1367 | */ |
1368 | struct static_fw_load_mgr { |
1369 | u64 preboot_version_max_off; |
1370 | u64 boot_fit_version_max_off; |
1371 | u32 kmd_msg_to_cpu_reg; |
1372 | u32 cpu_cmd_status_to_host_reg; |
1373 | u32 cpu_boot_status_reg; |
1374 | u32 cpu_boot_dev_status0_reg; |
1375 | u32 cpu_boot_dev_status1_reg; |
1376 | u32 boot_err0_reg; |
1377 | u32 boot_err1_reg; |
1378 | u32 preboot_version_offset_reg; |
1379 | u32 boot_fit_version_offset_reg; |
1380 | u32 sram_offset_mask; |
1381 | u32 cpu_reset_wait_msec; |
1382 | }; |
1383 | |
1384 | /** |
1385 | * struct fw_response - FW response to LKD command |
1386 | * @ram_offset: descriptor offset into the RAM |
1387 | * @ram_type: RAM type containing the descriptor (SRAM/DRAM) |
1388 | * @status: command status |
1389 | */ |
1390 | struct fw_response { |
1391 | u32 ram_offset; |
1392 | u8 ram_type; |
1393 | u8 status; |
1394 | }; |
1395 | |
1396 | /** |
1397 | * struct dynamic_fw_load_mgr - dynamic FW load manager |
1398 | * @response: FW to LKD response |
1399 | * @comm_desc: the communication descriptor with FW |
1400 | * @image_region: region to copy the FW image to |
1401 | * @fw_image_size: size of FW image to load |
1402 | * @wait_for_bl_timeout: timeout for waiting for boot loader to respond |
1403 | * @fw_desc_valid: true if FW descriptor has been validated and hence the data can be used |
1404 | */ |
1405 | struct dynamic_fw_load_mgr { |
1406 | struct fw_response response; |
1407 | struct lkd_fw_comms_desc comm_desc; |
1408 | struct pci_mem_region *image_region; |
1409 | size_t fw_image_size; |
1410 | u32 wait_for_bl_timeout; |
1411 | bool fw_desc_valid; |
1412 | }; |
1413 | |
1414 | /** |
1415 | * struct pre_fw_load_props - needed properties for pre-FW load |
1416 | * @cpu_boot_status_reg: cpu_boot_status register address |
1417 | * @sts_boot_dev_sts0_reg: sts_boot_dev_sts0 register address |
1418 | * @sts_boot_dev_sts1_reg: sts_boot_dev_sts1 register address |
1419 | * @boot_err0_reg: boot_err0 register address |
1420 | * @boot_err1_reg: boot_err1 register address |
1421 | * @wait_for_preboot_timeout: timeout to poll for preboot ready |
1422 | * @wait_for_preboot_extended_timeout: timeout to pull for preboot ready in case where we know |
1423 | * preboot needs longer time. |
1424 | */ |
1425 | struct pre_fw_load_props { |
1426 | u32 cpu_boot_status_reg; |
1427 | u32 sts_boot_dev_sts0_reg; |
1428 | u32 sts_boot_dev_sts1_reg; |
1429 | u32 boot_err0_reg; |
1430 | u32 boot_err1_reg; |
1431 | u32 wait_for_preboot_timeout; |
1432 | u32 wait_for_preboot_extended_timeout; |
1433 | }; |
1434 | |
1435 | /** |
1436 | * struct fw_image_props - properties of FW image |
1437 | * @image_name: name of the image |
1438 | * @src_off: offset in src FW to copy from |
1439 | * @copy_size: amount of bytes to copy (0 to copy the whole binary) |
1440 | */ |
1441 | struct fw_image_props { |
1442 | char *image_name; |
1443 | u32 src_off; |
1444 | u32 copy_size; |
1445 | }; |
1446 | |
1447 | /** |
1448 | * struct fw_load_mgr - manager FW loading process |
1449 | * @dynamic_loader: specific structure for dynamic load |
1450 | * @static_loader: specific structure for static load |
1451 | * @pre_fw_load_props: parameter for pre FW load |
1452 | * @boot_fit_img: boot fit image properties |
1453 | * @linux_img: linux image properties |
1454 | * @cpu_timeout: CPU response timeout in usec |
1455 | * @boot_fit_timeout: Boot fit load timeout in usec |
1456 | * @skip_bmc: should BMC be skipped |
1457 | * @sram_bar_id: SRAM bar ID |
1458 | * @dram_bar_id: DRAM bar ID |
1459 | * @fw_comp_loaded: bitmask of loaded FW components. set bit meaning loaded |
1460 | * component. values are set according to enum hl_fw_types. |
1461 | */ |
1462 | struct fw_load_mgr { |
1463 | union { |
1464 | struct dynamic_fw_load_mgr dynamic_loader; |
1465 | struct static_fw_load_mgr static_loader; |
1466 | }; |
1467 | struct pre_fw_load_props pre_fw_load; |
1468 | struct fw_image_props boot_fit_img; |
1469 | struct fw_image_props linux_img; |
1470 | u32 cpu_timeout; |
1471 | u32 boot_fit_timeout; |
1472 | u8 skip_bmc; |
1473 | u8 sram_bar_id; |
1474 | u8 dram_bar_id; |
1475 | u8 fw_comp_loaded; |
1476 | }; |
1477 | |
1478 | struct hl_cs; |
1479 | |
1480 | /** |
1481 | * struct engines_data - asic engines data |
1482 | * @buf: buffer for engines data in ascii |
1483 | * @actual_size: actual size of data that was written by the driver to the allocated buffer |
1484 | * @allocated_buf_size: total size of allocated buffer |
1485 | */ |
1486 | struct engines_data { |
1487 | char *buf; |
1488 | int actual_size; |
1489 | u32 allocated_buf_size; |
1490 | }; |
1491 | |
1492 | /** |
1493 | * struct hl_asic_funcs - ASIC specific functions that are can be called from |
1494 | * common code. |
1495 | * @early_init: sets up early driver state (pre sw_init), doesn't configure H/W. |
1496 | * @early_fini: tears down what was done in early_init. |
1497 | * @late_init: sets up late driver/hw state (post hw_init) - Optional. |
1498 | * @late_fini: tears down what was done in late_init (pre hw_fini) - Optional. |
1499 | * @sw_init: sets up driver state, does not configure H/W. |
1500 | * @sw_fini: tears down driver state, does not configure H/W. |
1501 | * @hw_init: sets up the H/W state. |
1502 | * @hw_fini: tears down the H/W state. |
1503 | * @halt_engines: halt engines, needed for reset sequence. This also disables |
1504 | * interrupts from the device. Should be called before |
1505 | * hw_fini and before CS rollback. |
1506 | * @suspend: handles IP specific H/W or SW changes for suspend. |
1507 | * @resume: handles IP specific H/W or SW changes for resume. |
1508 | * @mmap: maps a memory. |
1509 | * @ring_doorbell: increment PI on a given QMAN. |
1510 | * @pqe_write: Write the PQ entry to the PQ. This is ASIC-specific |
1511 | * function because the PQs are located in different memory areas |
1512 | * per ASIC (SRAM, DRAM, Host memory) and therefore, the method of |
1513 | * writing the PQE must match the destination memory area |
1514 | * properties. |
1515 | * @asic_dma_alloc_coherent: Allocate coherent DMA memory by calling |
1516 | * dma_alloc_coherent(). This is ASIC function because |
1517 | * its implementation is not trivial when the driver |
1518 | * is loaded in simulation mode (not upstreamed). |
1519 | * @asic_dma_free_coherent: Free coherent DMA memory by calling |
1520 | * dma_free_coherent(). This is ASIC function because |
1521 | * its implementation is not trivial when the driver |
1522 | * is loaded in simulation mode (not upstreamed). |
1523 | * @scrub_device_mem: Scrub the entire SRAM and DRAM. |
1524 | * @scrub_device_dram: Scrub the dram memory of the device. |
1525 | * @get_int_queue_base: get the internal queue base address. |
1526 | * @test_queues: run simple test on all queues for sanity check. |
1527 | * @asic_dma_pool_zalloc: small DMA allocation of coherent memory from DMA pool. |
1528 | * size of allocation is HL_DMA_POOL_BLK_SIZE. |
1529 | * @asic_dma_pool_free: free small DMA allocation from pool. |
1530 | * @cpu_accessible_dma_pool_alloc: allocate CPU PQ packet from DMA pool. |
1531 | * @cpu_accessible_dma_pool_free: free CPU PQ packet from DMA pool. |
1532 | * @dma_unmap_sgtable: DMA unmap scatter-gather table. |
1533 | * @dma_map_sgtable: DMA map scatter-gather table. |
1534 | * @cs_parser: parse Command Submission. |
1535 | * @add_end_of_cb_packets: Add packets to the end of CB, if device requires it. |
1536 | * @update_eq_ci: update event queue CI. |
1537 | * @context_switch: called upon ASID context switch. |
1538 | * @restore_phase_topology: clear all SOBs amd MONs. |
1539 | * @debugfs_read_dma: debug interface for reading up to 2MB from the device's |
1540 | * internal memory via DMA engine. |
1541 | * @add_device_attr: add ASIC specific device attributes. |
1542 | * @handle_eqe: handle event queue entry (IRQ) from CPU-CP. |
1543 | * @get_events_stat: retrieve event queue entries histogram. |
1544 | * @read_pte: read MMU page table entry from DRAM. |
1545 | * @write_pte: write MMU page table entry to DRAM. |
1546 | * @mmu_invalidate_cache: flush MMU STLB host/DRAM cache, either with soft |
1547 | * (L1 only) or hard (L0 & L1) flush. |
1548 | * @mmu_invalidate_cache_range: flush specific MMU STLB cache lines with ASID-VA-size mask. |
1549 | * @mmu_prefetch_cache_range: pre-fetch specific MMU STLB cache lines with ASID-VA-size mask. |
1550 | * @send_heartbeat: send is-alive packet to CPU-CP and verify response. |
1551 | * @debug_coresight: perform certain actions on Coresight for debugging. |
1552 | * @is_device_idle: return true if device is idle, false otherwise. |
1553 | * @compute_reset_late_init: perform certain actions needed after a compute reset |
1554 | * @hw_queues_lock: acquire H/W queues lock. |
1555 | * @hw_queues_unlock: release H/W queues lock. |
1556 | * @get_pci_id: retrieve PCI ID. |
1557 | * @get_eeprom_data: retrieve EEPROM data from F/W. |
1558 | * @get_monitor_dump: retrieve monitor registers dump from F/W. |
1559 | * @send_cpu_message: send message to F/W. If the message is timedout, the |
1560 | * driver will eventually reset the device. The timeout can |
1561 | * be determined by the calling function or it can be 0 and |
1562 | * then the timeout is the default timeout for the specific |
1563 | * ASIC |
1564 | * @get_hw_state: retrieve the H/W state |
1565 | * @pci_bars_map: Map PCI BARs. |
1566 | * @init_iatu: Initialize the iATU unit inside the PCI controller. |
1567 | * @rreg: Read a register. Needed for simulator support. |
1568 | * @wreg: Write a register. Needed for simulator support. |
1569 | * @halt_coresight: stop the ETF and ETR traces. |
1570 | * @ctx_init: context dependent initialization. |
1571 | * @ctx_fini: context dependent cleanup. |
1572 | * @pre_schedule_cs: Perform pre-CS-scheduling operations. |
1573 | * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. |
1574 | * @load_firmware_to_device: load the firmware to the device's memory |
1575 | * @load_boot_fit_to_device: load boot fit to device's memory |
1576 | * @get_signal_cb_size: Get signal CB size. |
1577 | * @get_wait_cb_size: Get wait CB size. |
1578 | * @gen_signal_cb: Generate a signal CB. |
1579 | * @gen_wait_cb: Generate a wait CB. |
1580 | * @reset_sob: Reset a SOB. |
1581 | * @reset_sob_group: Reset SOB group |
1582 | * @get_device_time: Get the device time. |
1583 | * @pb_print_security_errors: print security errors according block and cause |
1584 | * @collective_wait_init_cs: Generate collective master/slave packets |
1585 | * and place them in the relevant cs jobs |
1586 | * @collective_wait_create_jobs: allocate collective wait cs jobs |
1587 | * @get_dec_base_addr: get the base address of a given decoder. |
1588 | * @scramble_addr: Routine to scramble the address prior of mapping it |
1589 | * in the MMU. |
1590 | * @descramble_addr: Routine to de-scramble the address prior of |
1591 | * showing it to users. |
1592 | * @ack_protection_bits_errors: ack and dump all security violations |
1593 | * @get_hw_block_id: retrieve a HW block id to be used by the user to mmap it. |
1594 | * also returns the size of the block if caller supplies |
1595 | * a valid pointer for it |
1596 | * @hw_block_mmap: mmap a HW block with a given id. |
1597 | * @enable_events_from_fw: send interrupt to firmware to notify them the |
1598 | * driver is ready to receive asynchronous events. This |
1599 | * function should be called during the first init and |
1600 | * after every hard-reset of the device |
1601 | * @ack_mmu_errors: check and ack mmu errors, page fault, access violation. |
1602 | * @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event |
1603 | * @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to |
1604 | * generic f/w compatible PLL Indexes |
1605 | * @init_firmware_preload_params: initialize pre FW-load parameters. |
1606 | * @init_firmware_loader: initialize data for FW loader. |
1607 | * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling |
1608 | * @state_dump_init: initialize constants required for state dump |
1609 | * @get_sob_addr: get SOB base address offset. |
1610 | * @set_pci_memory_regions: setting properties of PCI memory regions |
1611 | * @get_stream_master_qid_arr: get pointer to stream masters QID array |
1612 | * @check_if_razwi_happened: check if there was a razwi due to RR violation. |
1613 | * @access_dev_mem: access device memory |
1614 | * @set_dram_bar_base: set the base of the DRAM BAR |
1615 | * @set_engine_cores: set a config command to engine cores |
1616 | * @set_engines: set a config command to user engines |
1617 | * @send_device_activity: indication to FW about device availability |
1618 | * @set_dram_properties: set DRAM related properties. |
1619 | * @set_binning_masks: set binning/enable masks for all relevant components. |
1620 | */ |
1621 | struct hl_asic_funcs { |
1622 | int (*early_init)(struct hl_device *hdev); |
1623 | int (*early_fini)(struct hl_device *hdev); |
1624 | int (*late_init)(struct hl_device *hdev); |
1625 | void (*late_fini)(struct hl_device *hdev); |
1626 | int (*sw_init)(struct hl_device *hdev); |
1627 | int (*sw_fini)(struct hl_device *hdev); |
1628 | int (*hw_init)(struct hl_device *hdev); |
1629 | int (*hw_fini)(struct hl_device *hdev, bool hard_reset, bool fw_reset); |
1630 | void (*halt_engines)(struct hl_device *hdev, bool hard_reset, bool fw_reset); |
1631 | int (*suspend)(struct hl_device *hdev); |
1632 | int (*resume)(struct hl_device *hdev); |
1633 | int (*mmap)(struct hl_device *hdev, struct vm_area_struct *vma, |
1634 | void *cpu_addr, dma_addr_t dma_addr, size_t size); |
1635 | void (*ring_doorbell)(struct hl_device *hdev, u32 hw_queue_id, u32 pi); |
1636 | void (*pqe_write)(struct hl_device *hdev, __le64 *pqe, |
1637 | struct hl_bd *bd); |
1638 | void* (*asic_dma_alloc_coherent)(struct hl_device *hdev, size_t size, |
1639 | dma_addr_t *dma_handle, gfp_t flag); |
1640 | void (*asic_dma_free_coherent)(struct hl_device *hdev, size_t size, |
1641 | void *cpu_addr, dma_addr_t dma_handle); |
1642 | int (*scrub_device_mem)(struct hl_device *hdev); |
1643 | int (*scrub_device_dram)(struct hl_device *hdev, u64 val); |
1644 | void* (*get_int_queue_base)(struct hl_device *hdev, u32 queue_id, |
1645 | dma_addr_t *dma_handle, u16 *queue_len); |
1646 | int (*test_queues)(struct hl_device *hdev); |
1647 | void* (*asic_dma_pool_zalloc)(struct hl_device *hdev, size_t size, |
1648 | gfp_t mem_flags, dma_addr_t *dma_handle); |
1649 | void (*asic_dma_pool_free)(struct hl_device *hdev, void *vaddr, |
1650 | dma_addr_t dma_addr); |
1651 | void* (*cpu_accessible_dma_pool_alloc)(struct hl_device *hdev, |
1652 | size_t size, dma_addr_t *dma_handle); |
1653 | void (*cpu_accessible_dma_pool_free)(struct hl_device *hdev, |
1654 | size_t size, void *vaddr); |
1655 | void (*dma_unmap_sgtable)(struct hl_device *hdev, struct sg_table *sgt, |
1656 | enum dma_data_direction dir); |
1657 | int (*dma_map_sgtable)(struct hl_device *hdev, struct sg_table *sgt, |
1658 | enum dma_data_direction dir); |
1659 | int (*cs_parser)(struct hl_device *hdev, struct hl_cs_parser *parser); |
1660 | void (*add_end_of_cb_packets)(struct hl_device *hdev, |
1661 | void *kernel_address, u32 len, |
1662 | u32 original_len, |
1663 | u64 cq_addr, u32 cq_val, u32 msix_num, |
1664 | bool eb); |
1665 | void (*update_eq_ci)(struct hl_device *hdev, u32 val); |
1666 | int (*context_switch)(struct hl_device *hdev, u32 asid); |
1667 | void (*restore_phase_topology)(struct hl_device *hdev); |
1668 | int (*debugfs_read_dma)(struct hl_device *hdev, u64 addr, u32 size, |
1669 | void *blob_addr); |
1670 | void (*add_device_attr)(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp, |
1671 | struct attribute_group *dev_vrm_attr_grp); |
1672 | void (*handle_eqe)(struct hl_device *hdev, |
1673 | struct hl_eq_entry *eq_entry); |
1674 | void* (*get_events_stat)(struct hl_device *hdev, bool aggregate, |
1675 | u32 *size); |
1676 | u64 (*read_pte)(struct hl_device *hdev, u64 addr); |
1677 | void (*write_pte)(struct hl_device *hdev, u64 addr, u64 val); |
1678 | int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard, |
1679 | u32 flags); |
1680 | int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard, |
1681 | u32 flags, u32 asid, u64 va, u64 size); |
1682 | int (*mmu_prefetch_cache_range)(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); |
1683 | int (*send_heartbeat)(struct hl_device *hdev); |
1684 | int (*debug_coresight)(struct hl_device *hdev, struct hl_ctx *ctx, void *data); |
1685 | bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, |
1686 | struct engines_data *e); |
1687 | int (*compute_reset_late_init)(struct hl_device *hdev); |
1688 | void (*hw_queues_lock)(struct hl_device *hdev); |
1689 | void (*hw_queues_unlock)(struct hl_device *hdev); |
1690 | u32 (*get_pci_id)(struct hl_device *hdev); |
1691 | int (*get_eeprom_data)(struct hl_device *hdev, void *data, size_t max_size); |
1692 | int (*get_monitor_dump)(struct hl_device *hdev, void *data); |
1693 | int (*send_cpu_message)(struct hl_device *hdev, u32 *msg, |
1694 | u16 len, u32 timeout, u64 *result); |
1695 | int (*pci_bars_map)(struct hl_device *hdev); |
1696 | int (*init_iatu)(struct hl_device *hdev); |
1697 | u32 (*rreg)(struct hl_device *hdev, u32 reg); |
1698 | void (*wreg)(struct hl_device *hdev, u32 reg, u32 val); |
1699 | void (*halt_coresight)(struct hl_device *hdev, struct hl_ctx *ctx); |
1700 | int (*ctx_init)(struct hl_ctx *ctx); |
1701 | void (*ctx_fini)(struct hl_ctx *ctx); |
1702 | int (*pre_schedule_cs)(struct hl_cs *cs); |
1703 | u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx); |
1704 | int (*load_firmware_to_device)(struct hl_device *hdev); |
1705 | int (*load_boot_fit_to_device)(struct hl_device *hdev); |
1706 | u32 (*get_signal_cb_size)(struct hl_device *hdev); |
1707 | u32 (*get_wait_cb_size)(struct hl_device *hdev); |
1708 | u32 (*gen_signal_cb)(struct hl_device *hdev, void *data, u16 sob_id, |
1709 | u32 size, bool eb); |
1710 | u32 (*gen_wait_cb)(struct hl_device *hdev, |
1711 | struct hl_gen_wait_properties *prop); |
1712 | void (*reset_sob)(struct hl_device *hdev, void *data); |
1713 | void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group); |
1714 | u64 (*get_device_time)(struct hl_device *hdev); |
1715 | void (*pb_print_security_errors)(struct hl_device *hdev, |
1716 | u32 block_addr, u32 cause, u32 offended_addr); |
1717 | int (*collective_wait_init_cs)(struct hl_cs *cs); |
1718 | int (*collective_wait_create_jobs)(struct hl_device *hdev, |
1719 | struct hl_ctx *ctx, struct hl_cs *cs, |
1720 | u32 wait_queue_id, u32 collective_engine_id, |
1721 | u32 encaps_signal_offset); |
1722 | u32 (*get_dec_base_addr)(struct hl_device *hdev, u32 core_id); |
1723 | u64 (*scramble_addr)(struct hl_device *hdev, u64 addr); |
1724 | u64 (*descramble_addr)(struct hl_device *hdev, u64 addr); |
1725 | void (*ack_protection_bits_errors)(struct hl_device *hdev); |
1726 | int (*get_hw_block_id)(struct hl_device *hdev, u64 block_addr, |
1727 | u32 *block_size, u32 *block_id); |
1728 | int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma, |
1729 | u32 block_id, u32 block_size); |
1730 | void (*enable_events_from_fw)(struct hl_device *hdev); |
1731 | int (*ack_mmu_errors)(struct hl_device *hdev, u64 mmu_cap_mask); |
1732 | void (*get_msi_info)(__le32 *table); |
1733 | int (*map_pll_idx_to_fw_idx)(u32 pll_idx); |
1734 | void (*init_firmware_preload_params)(struct hl_device *hdev); |
1735 | void (*init_firmware_loader)(struct hl_device *hdev); |
1736 | void (*init_cpu_scrambler_dram)(struct hl_device *hdev); |
1737 | void (*state_dump_init)(struct hl_device *hdev); |
1738 | u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id); |
1739 | void (*set_pci_memory_regions)(struct hl_device *hdev); |
1740 | u32* (*get_stream_master_qid_arr)(void); |
1741 | void (*check_if_razwi_happened)(struct hl_device *hdev); |
1742 | int (*mmu_get_real_page_size)(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, |
1743 | u32 page_size, u32 *real_page_size, bool is_dram_addr); |
1744 | int (*access_dev_mem)(struct hl_device *hdev, enum pci_region region_type, |
1745 | u64 addr, u64 *val, enum debugfs_access_type acc_type); |
1746 | u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr); |
1747 | int (*set_engine_cores)(struct hl_device *hdev, u32 *core_ids, |
1748 | u32 num_cores, u32 core_command); |
1749 | int (*set_engines)(struct hl_device *hdev, u32 *engine_ids, |
1750 | u32 num_engines, u32 engine_command); |
1751 | int (*send_device_activity)(struct hl_device *hdev, bool open); |
1752 | int (*set_dram_properties)(struct hl_device *hdev); |
1753 | int (*set_binning_masks)(struct hl_device *hdev); |
1754 | }; |
1755 | |
1756 | |
1757 | /* |
1758 | * CONTEXTS |
1759 | */ |
1760 | |
1761 | #define HL_KERNEL_ASID_ID 0 |
1762 | |
1763 | /** |
1764 | * enum hl_va_range_type - virtual address range type. |
1765 | * @HL_VA_RANGE_TYPE_HOST: range type of host pages |
1766 | * @HL_VA_RANGE_TYPE_HOST_HUGE: range type of host huge pages |
1767 | * @HL_VA_RANGE_TYPE_DRAM: range type of dram pages |
1768 | */ |
1769 | enum hl_va_range_type { |
1770 | HL_VA_RANGE_TYPE_HOST, |
1771 | HL_VA_RANGE_TYPE_HOST_HUGE, |
1772 | HL_VA_RANGE_TYPE_DRAM, |
1773 | HL_VA_RANGE_TYPE_MAX |
1774 | }; |
1775 | |
1776 | /** |
1777 | * struct hl_va_range - virtual addresses range. |
1778 | * @lock: protects the virtual addresses list. |
1779 | * @list: list of virtual addresses blocks available for mappings. |
1780 | * @start_addr: range start address. |
1781 | * @end_addr: range end address. |
1782 | * @page_size: page size of this va range. |
1783 | */ |
1784 | struct hl_va_range { |
1785 | struct mutex lock; |
1786 | struct list_head list; |
1787 | u64 start_addr; |
1788 | u64 end_addr; |
1789 | u32 page_size; |
1790 | }; |
1791 | |
1792 | /** |
1793 | * struct hl_cs_counters_atomic - command submission counters |
1794 | * @out_of_mem_drop_cnt: dropped due to memory allocation issue |
1795 | * @parsing_drop_cnt: dropped due to error in packet parsing |
1796 | * @queue_full_drop_cnt: dropped due to queue full |
1797 | * @device_in_reset_drop_cnt: dropped due to device in reset |
1798 | * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight |
1799 | * @validation_drop_cnt: dropped due to error in validation |
1800 | */ |
1801 | struct hl_cs_counters_atomic { |
1802 | atomic64_t out_of_mem_drop_cnt; |
1803 | atomic64_t parsing_drop_cnt; |
1804 | atomic64_t queue_full_drop_cnt; |
1805 | atomic64_t device_in_reset_drop_cnt; |
1806 | atomic64_t max_cs_in_flight_drop_cnt; |
1807 | atomic64_t validation_drop_cnt; |
1808 | }; |
1809 | |
1810 | /** |
1811 | * struct hl_dmabuf_priv - a dma-buf private object. |
1812 | * @dmabuf: pointer to dma-buf object. |
1813 | * @ctx: pointer to the dma-buf owner's context. |
1814 | * @phys_pg_pack: pointer to physical page pack if the dma-buf was exported |
1815 | * where virtual memory is supported. |
1816 | * @memhash_hnode: pointer to the memhash node. this object holds the export count. |
1817 | * @offset: the offset into the buffer from which the memory is exported. |
1818 | * Relevant only if virtual memory is supported and phys_pg_pack is being used. |
1819 | * device_phys_addr: physical address of the device's memory. Relevant only |
1820 | * if phys_pg_pack is NULL (dma-buf was exported from address). |
1821 | * The total size can be taken from the dmabuf object. |
1822 | */ |
1823 | struct hl_dmabuf_priv { |
1824 | struct dma_buf *dmabuf; |
1825 | struct hl_ctx *ctx; |
1826 | struct hl_vm_phys_pg_pack *phys_pg_pack; |
1827 | struct hl_vm_hash_node *memhash_hnode; |
1828 | u64 offset; |
1829 | u64 device_phys_addr; |
1830 | }; |
1831 | |
1832 | #define HL_CS_OUTCOME_HISTORY_LEN 256 |
1833 | |
1834 | /** |
1835 | * struct hl_cs_outcome - represents a single completed CS outcome |
1836 | * @list_link: link to either container's used list or free list |
1837 | * @map_link: list to the container hash map |
1838 | * @ts: completion ts |
1839 | * @seq: the original cs sequence |
1840 | * @error: error code cs completed with, if any |
1841 | */ |
1842 | struct hl_cs_outcome { |
1843 | struct list_head list_link; |
1844 | struct hlist_node map_link; |
1845 | ktime_t ts; |
1846 | u64 seq; |
1847 | int error; |
1848 | }; |
1849 | |
1850 | /** |
1851 | * struct hl_cs_outcome_store - represents a limited store of completed CS outcomes |
1852 | * @outcome_map: index of completed CS searchable by sequence number |
1853 | * @used_list: list of outcome objects currently in use |
1854 | * @free_list: list of outcome objects currently not in use |
1855 | * @nodes_pool: a static pool of pre-allocated outcome objects |
1856 | * @db_lock: any operation on the store must take this lock |
1857 | */ |
1858 | struct hl_cs_outcome_store { |
1859 | DECLARE_HASHTABLE(outcome_map, 8); |
1860 | struct list_head used_list; |
1861 | struct list_head free_list; |
1862 | struct hl_cs_outcome nodes_pool[HL_CS_OUTCOME_HISTORY_LEN]; |
1863 | spinlock_t db_lock; |
1864 | }; |
1865 | |
1866 | /** |
1867 | * struct hl_ctx - user/kernel context. |
1868 | * @mem_hash: holds mapping from virtual address to virtual memory area |
1869 | * descriptor (hl_vm_phys_pg_list or hl_userptr). |
1870 | * @mmu_shadow_hash: holds a mapping from shadow address to pgt_info structure. |
1871 | * @hr_mmu_phys_hash: if host-resident MMU is used, holds a mapping from |
1872 | * MMU-hop-page physical address to its host-resident |
1873 | * pgt_info structure. |
1874 | * @hpriv: pointer to the private (Kernel Driver) data of the process (fd). |
1875 | * @hdev: pointer to the device structure. |
1876 | * @refcount: reference counter for the context. Context is released only when |
1877 | * this hits 0. It is incremented on CS and CS_WAIT. |
1878 | * @cs_pending: array of hl fence objects representing pending CS. |
1879 | * @outcome_store: storage data structure used to remember outcomes of completed |
1880 | * command submissions for a long time after CS id wraparound. |
1881 | * @va_range: holds available virtual addresses for host and dram mappings. |
1882 | * @mem_hash_lock: protects the mem_hash. |
1883 | * @hw_block_list_lock: protects the HW block memory list. |
1884 | * @ts_reg_lock: timestamp registration ioctls lock. |
1885 | * @debugfs_list: node in debugfs list of contexts. |
1886 | * @hw_block_mem_list: list of HW block virtual mapped addresses. |
1887 | * @cs_counters: context command submission counters. |
1888 | * @cb_va_pool: device VA pool for command buffers which are mapped to the |
1889 | * device's MMU. |
1890 | * @sig_mgr: encaps signals handle manager. |
1891 | * @cb_va_pool_base: the base address for the device VA pool |
1892 | * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed |
1893 | * to user so user could inquire about CS. It is used as |
1894 | * index to cs_pending array. |
1895 | * @dram_default_hops: array that holds all hops addresses needed for default |
1896 | * DRAM mapping. |
1897 | * @cs_lock: spinlock to protect cs_sequence. |
1898 | * @dram_phys_mem: amount of used physical DRAM memory by this context. |
1899 | * @thread_ctx_switch_token: token to prevent multiple threads of the same |
1900 | * context from running the context switch phase. |
1901 | * Only a single thread should run it. |
1902 | * @thread_ctx_switch_wait_token: token to prevent the threads that didn't run |
1903 | * the context switch phase from moving to their |
1904 | * execution phase before the context switch phase |
1905 | * has finished. |
1906 | * @asid: context's unique address space ID in the device's MMU. |
1907 | * @handle: context's opaque handle for user |
1908 | */ |
1909 | struct hl_ctx { |
1910 | DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS); |
1911 | DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS); |
1912 | DECLARE_HASHTABLE(hr_mmu_phys_hash, MMU_HASH_TABLE_BITS); |
1913 | struct hl_fpriv *hpriv; |
1914 | struct hl_device *hdev; |
1915 | struct kref refcount; |
1916 | struct hl_fence **cs_pending; |
1917 | struct hl_cs_outcome_store outcome_store; |
1918 | struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX]; |
1919 | struct mutex mem_hash_lock; |
1920 | struct mutex hw_block_list_lock; |
1921 | struct mutex ts_reg_lock; |
1922 | struct list_head debugfs_list; |
1923 | struct list_head hw_block_mem_list; |
1924 | struct hl_cs_counters_atomic cs_counters; |
1925 | struct gen_pool *cb_va_pool; |
1926 | struct hl_encaps_signals_mgr sig_mgr; |
1927 | u64 cb_va_pool_base; |
1928 | u64 cs_sequence; |
1929 | u64 *dram_default_hops; |
1930 | spinlock_t cs_lock; |
1931 | atomic64_t dram_phys_mem; |
1932 | atomic_t thread_ctx_switch_token; |
1933 | u32 thread_ctx_switch_wait_token; |
1934 | u32 asid; |
1935 | u32 handle; |
1936 | }; |
1937 | |
1938 | /** |
1939 | * struct hl_ctx_mgr - for handling multiple contexts. |
1940 | * @lock: protects ctx_handles. |
1941 | * @handles: idr to hold all ctx handles. |
1942 | */ |
1943 | struct hl_ctx_mgr { |
1944 | struct mutex lock; |
1945 | struct idr handles; |
1946 | }; |
1947 | |
1948 | |
1949 | /* |
1950 | * COMMAND SUBMISSIONS |
1951 | */ |
1952 | |
1953 | /** |
1954 | * struct hl_userptr - memory mapping chunk information |
1955 | * @vm_type: type of the VM. |
1956 | * @job_node: linked-list node for hanging the object on the Job's list. |
1957 | * @pages: pointer to struct page array |
1958 | * @npages: size of @pages array |
1959 | * @sgt: pointer to the scatter-gather table that holds the pages. |
1960 | * @dir: for DMA unmapping, the direction must be supplied, so save it. |
1961 | * @debugfs_list: node in debugfs list of command submissions. |
1962 | * @pid: the pid of the user process owning the memory |
1963 | * @addr: user-space virtual address of the start of the memory area. |
1964 | * @size: size of the memory area to pin & map. |
1965 | * @dma_mapped: true if the SG was mapped to DMA addresses, false otherwise. |
1966 | */ |
1967 | struct hl_userptr { |
1968 | enum vm_type vm_type; /* must be first */ |
1969 | struct list_head job_node; |
1970 | struct page **pages; |
1971 | unsigned int npages; |
1972 | struct sg_table *sgt; |
1973 | enum dma_data_direction dir; |
1974 | struct list_head debugfs_list; |
1975 | pid_t pid; |
1976 | u64 addr; |
1977 | u64 size; |
1978 | u8 dma_mapped; |
1979 | }; |
1980 | |
1981 | /** |
1982 | * struct hl_cs - command submission. |
1983 | * @jobs_in_queue_cnt: per each queue, maintain counter of submitted jobs. |
1984 | * @ctx: the context this CS belongs to. |
1985 | * @job_list: list of the CS's jobs in the various queues. |
1986 | * @job_lock: spinlock for the CS's jobs list. Needed for free_job. |
1987 | * @refcount: reference counter for usage of the CS. |
1988 | * @fence: pointer to the fence object of this CS. |
1989 | * @signal_fence: pointer to the fence object of the signal CS (used by wait |
1990 | * CS only). |
1991 | * @finish_work: workqueue object to run when CS is completed by H/W. |
1992 | * @work_tdr: delayed work node for TDR. |
1993 | * @mirror_node : node in device mirror list of command submissions. |
1994 | * @staged_cs_node: node in the staged cs list. |
1995 | * @debugfs_list: node in debugfs list of command submissions. |
1996 | * @encaps_sig_hdl: holds the encaps signals handle. |
1997 | * @sequence: the sequence number of this CS. |
1998 | * @staged_sequence: the sequence of the staged submission this CS is part of, |
1999 | * relevant only if staged_cs is set. |
2000 | * @timeout_jiffies: cs timeout in jiffies. |
2001 | * @submission_time_jiffies: submission time of the cs |
2002 | * @type: CS_TYPE_*. |
2003 | * @jobs_cnt: counter of submitted jobs on all queues. |
2004 | * @encaps_sig_hdl_id: encaps signals handle id, set for the first staged cs. |
2005 | * @completion_timestamp: timestamp of the last completed cs job. |
2006 | * @sob_addr_offset: sob offset from the configuration base address. |
2007 | * @initial_sob_count: count of completed signals in SOB before current submission of signal or |
2008 | * cs with encaps signals. |
2009 | * @submitted: true if CS was submitted to H/W. |
2010 | * @completed: true if CS was completed by device. |
2011 | * @timedout : true if CS was timedout. |
2012 | * @tdr_active: true if TDR was activated for this CS (to prevent |
2013 | * double TDR activation). |
2014 | * @aborted: true if CS was aborted due to some device error. |
2015 | * @timestamp: true if a timestamp must be captured upon completion. |
2016 | * @staged_last: true if this is the last staged CS and needs completion. |
2017 | * @staged_first: true if this is the first staged CS and we need to receive |
2018 | * timeout for this CS. |
2019 | * @staged_cs: true if this CS is part of a staged submission. |
2020 | * @skip_reset_on_timeout: true if we shall not reset the device in case |
2021 | * timeout occurs (debug scenario). |
2022 | * @encaps_signals: true if this CS has encaps reserved signals. |
2023 | */ |
2024 | struct hl_cs { |
2025 | u16 *jobs_in_queue_cnt; |
2026 | struct hl_ctx *ctx; |
2027 | struct list_head job_list; |
2028 | spinlock_t job_lock; |
2029 | struct kref refcount; |
2030 | struct hl_fence *fence; |
2031 | struct hl_fence *signal_fence; |
2032 | struct work_struct finish_work; |
2033 | struct delayed_work work_tdr; |
2034 | struct list_head mirror_node; |
2035 | struct list_head staged_cs_node; |
2036 | struct list_head debugfs_list; |
2037 | struct hl_cs_encaps_sig_handle *encaps_sig_hdl; |
2038 | ktime_t completion_timestamp; |
2039 | u64 sequence; |
2040 | u64 staged_sequence; |
2041 | u64 timeout_jiffies; |
2042 | u64 submission_time_jiffies; |
2043 | enum hl_cs_type type; |
2044 | u32 jobs_cnt; |
2045 | u32 encaps_sig_hdl_id; |
2046 | u32 sob_addr_offset; |
2047 | u16 initial_sob_count; |
2048 | u8 submitted; |
2049 | u8 completed; |
2050 | u8 timedout; |
2051 | u8 tdr_active; |
2052 | u8 aborted; |
2053 | u8 timestamp; |
2054 | u8 staged_last; |
2055 | u8 staged_first; |
2056 | u8 staged_cs; |
2057 | u8 skip_reset_on_timeout; |
2058 | u8 encaps_signals; |
2059 | }; |
2060 | |
2061 | /** |
2062 | * struct hl_cs_job - command submission job. |
2063 | * @cs_node: the node to hang on the CS jobs list. |
2064 | * @cs: the CS this job belongs to. |
2065 | * @user_cb: the CB we got from the user. |
2066 | * @patched_cb: in case of patching, this is internal CB which is submitted on |
2067 | * the queue instead of the CB we got from the IOCTL. |
2068 | * @finish_work: workqueue object to run when job is completed. |
2069 | * @userptr_list: linked-list of userptr mappings that belong to this job and |
2070 | * wait for completion. |
2071 | * @debugfs_list: node in debugfs list of command submission jobs. |
2072 | * @refcount: reference counter for usage of the CS job. |
2073 | * @queue_type: the type of the H/W queue this job is submitted to. |
2074 | * @timestamp: timestamp upon job completion |
2075 | * @id: the id of this job inside a CS. |
2076 | * @hw_queue_id: the id of the H/W queue this job is submitted to. |
2077 | * @user_cb_size: the actual size of the CB we got from the user. |
2078 | * @job_cb_size: the actual size of the CB that we put on the queue. |
2079 | * @encaps_sig_wait_offset: encapsulated signals offset, which allow user |
2080 | * to wait on part of the reserved signals. |
2081 | * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a |
2082 | * handle to a kernel-allocated CB object, false |
2083 | * otherwise (SRAM/DRAM/host address). |
2084 | * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This |
2085 | * info is needed later, when adding the 2xMSG_PROT at the |
2086 | * end of the JOB, to know which barriers to put in the |
2087 | * MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't |
2088 | * have streams so the engine can't be busy by another |
2089 | * stream. |
2090 | */ |
2091 | struct hl_cs_job { |
2092 | struct list_head cs_node; |
2093 | struct hl_cs *cs; |
2094 | struct hl_cb *user_cb; |
2095 | struct hl_cb *patched_cb; |
2096 | struct work_struct finish_work; |
2097 | struct list_head userptr_list; |
2098 | struct list_head debugfs_list; |
2099 | struct kref refcount; |
2100 | enum hl_queue_type queue_type; |
2101 | ktime_t timestamp; |
2102 | u32 id; |
2103 | u32 hw_queue_id; |
2104 | u32 user_cb_size; |
2105 | u32 job_cb_size; |
2106 | u32 encaps_sig_wait_offset; |
2107 | u8 is_kernel_allocated_cb; |
2108 | u8 contains_dma_pkt; |
2109 | }; |
2110 | |
2111 | /** |
2112 | * struct hl_cs_parser - command submission parser properties. |
2113 | * @user_cb: the CB we got from the user. |
2114 | * @patched_cb: in case of patching, this is internal CB which is submitted on |
2115 | * the queue instead of the CB we got from the IOCTL. |
2116 | * @job_userptr_list: linked-list of userptr mappings that belong to the related |
2117 | * job and wait for completion. |
2118 | * @cs_sequence: the sequence number of the related CS. |
2119 | * @queue_type: the type of the H/W queue this job is submitted to. |
2120 | * @ctx_id: the ID of the context the related CS belongs to. |
2121 | * @hw_queue_id: the id of the H/W queue this job is submitted to. |
2122 | * @user_cb_size: the actual size of the CB we got from the user. |
2123 | * @patched_cb_size: the size of the CB after parsing. |
2124 | * @job_id: the id of the related job inside the related CS. |
2125 | * @is_kernel_allocated_cb: true if the CB handle we got from the user holds a |
2126 | * handle to a kernel-allocated CB object, false |
2127 | * otherwise (SRAM/DRAM/host address). |
2128 | * @contains_dma_pkt: whether the JOB contains at least one DMA packet. This |
2129 | * info is needed later, when adding the 2xMSG_PROT at the |
2130 | * end of the JOB, to know which barriers to put in the |
2131 | * MSG_PROT packets. Relevant only for GAUDI as GOYA doesn't |
2132 | * have streams so the engine can't be busy by another |
2133 | * stream. |
2134 | * @completion: true if we need completion for this CS. |
2135 | */ |
2136 | struct hl_cs_parser { |
2137 | struct hl_cb *user_cb; |
2138 | struct hl_cb *patched_cb; |
2139 | struct list_head *job_userptr_list; |
2140 | u64 cs_sequence; |
2141 | enum hl_queue_type queue_type; |
2142 | u32 ctx_id; |
2143 | u32 hw_queue_id; |
2144 | u32 user_cb_size; |
2145 | u32 patched_cb_size; |
2146 | u8 job_id; |
2147 | u8 is_kernel_allocated_cb; |
2148 | u8 contains_dma_pkt; |
2149 | u8 completion; |
2150 | }; |
2151 | |
2152 | /* |
2153 | * MEMORY STRUCTURE |
2154 | */ |
2155 | |
2156 | /** |
2157 | * struct hl_vm_hash_node - hash element from virtual address to virtual |
2158 | * memory area descriptor (hl_vm_phys_pg_list or |
2159 | * hl_userptr). |
2160 | * @node: node to hang on the hash table in context object. |
2161 | * @vaddr: key virtual address. |
2162 | * @handle: memory handle for device memory allocation. |
2163 | * @ptr: value pointer (hl_vm_phys_pg_list or hl_userptr). |
2164 | * @export_cnt: number of exports from within the VA block. |
2165 | */ |
2166 | struct hl_vm_hash_node { |
2167 | struct hlist_node node; |
2168 | u64 vaddr; |
2169 | u64 handle; |
2170 | void *ptr; |
2171 | int export_cnt; |
2172 | }; |
2173 | |
2174 | /** |
2175 | * struct hl_vm_hw_block_list_node - list element from user virtual address to |
2176 | * HW block id. |
2177 | * @node: node to hang on the list in context object. |
2178 | * @ctx: the context this node belongs to. |
2179 | * @vaddr: virtual address of the HW block. |
2180 | * @block_size: size of the block. |
2181 | * @mapped_size: size of the block which is mapped. May change if partial un-mappings are done. |
2182 | * @id: HW block id (handle). |
2183 | */ |
2184 | struct hl_vm_hw_block_list_node { |
2185 | struct list_head node; |
2186 | struct hl_ctx *ctx; |
2187 | unsigned long vaddr; |
2188 | u32 block_size; |
2189 | u32 mapped_size; |
2190 | u32 id; |
2191 | }; |
2192 | |
2193 | /** |
2194 | * struct hl_vm_phys_pg_pack - physical page pack. |
2195 | * @vm_type: describes the type of the virtual area descriptor. |
2196 | * @pages: the physical page array. |
2197 | * @npages: num physical pages in the pack. |
2198 | * @total_size: total size of all the pages in this list. |
2199 | * @node: used to attach to deletion list that is used when all the allocations are cleared |
2200 | * at the teardown of the context. |
2201 | * @mapping_cnt: number of shared mappings. |
2202 | * @asid: the context related to this list. |
2203 | * @page_size: size of each page in the pack. |
2204 | * @flags: HL_MEM_* flags related to this list. |
2205 | * @handle: the provided handle related to this list. |
2206 | * @offset: offset from the first page. |
2207 | * @contiguous: is contiguous physical memory. |
2208 | * @created_from_userptr: is product of host virtual address. |
2209 | */ |
2210 | struct hl_vm_phys_pg_pack { |
2211 | enum vm_type vm_type; /* must be first */ |
2212 | u64 *pages; |
2213 | u64 npages; |
2214 | u64 total_size; |
2215 | struct list_head node; |
2216 | atomic_t mapping_cnt; |
2217 | u32 asid; |
2218 | u32 page_size; |
2219 | u32 flags; |
2220 | u32 handle; |
2221 | u32 offset; |
2222 | u8 contiguous; |
2223 | u8 created_from_userptr; |
2224 | }; |
2225 | |
2226 | /** |
2227 | * struct hl_vm_va_block - virtual range block information. |
2228 | * @node: node to hang on the virtual range list in context object. |
2229 | * @start: virtual range start address. |
2230 | * @end: virtual range end address. |
2231 | * @size: virtual range size. |
2232 | */ |
2233 | struct hl_vm_va_block { |
2234 | struct list_head node; |
2235 | u64 start; |
2236 | u64 end; |
2237 | u64 size; |
2238 | }; |
2239 | |
2240 | /** |
2241 | * struct hl_vm - virtual memory manager for MMU. |
2242 | * @dram_pg_pool: pool for DRAM physical pages of 2MB. |
2243 | * @dram_pg_pool_refcount: reference counter for the pool usage. |
2244 | * @idr_lock: protects the phys_pg_list_handles. |
2245 | * @phys_pg_pack_handles: idr to hold all device allocations handles. |
2246 | * @init_done: whether initialization was done. We need this because VM |
2247 | * initialization might be skipped during device initialization. |
2248 | */ |
2249 | struct hl_vm { |
2250 | struct gen_pool *dram_pg_pool; |
2251 | struct kref dram_pg_pool_refcount; |
2252 | spinlock_t idr_lock; |
2253 | struct idr phys_pg_pack_handles; |
2254 | u8 init_done; |
2255 | }; |
2256 | |
2257 | |
2258 | /* |
2259 | * DEBUG, PROFILING STRUCTURE |
2260 | */ |
2261 | |
2262 | /** |
2263 | * struct hl_debug_params - Coresight debug parameters. |
2264 | * @input: pointer to component specific input parameters. |
2265 | * @output: pointer to component specific output parameters. |
2266 | * @output_size: size of output buffer. |
2267 | * @reg_idx: relevant register ID. |
2268 | * @op: component operation to execute. |
2269 | * @enable: true if to enable component debugging, false otherwise. |
2270 | */ |
2271 | struct hl_debug_params { |
2272 | void *input; |
2273 | void *output; |
2274 | u32 output_size; |
2275 | u32 reg_idx; |
2276 | u32 op; |
2277 | bool enable; |
2278 | }; |
2279 | |
2280 | /** |
2281 | * struct hl_notifier_event - holds the notifier data structure |
2282 | * @eventfd: the event file descriptor to raise the notifications |
2283 | * @lock: mutex lock to protect the notifier data flows |
2284 | * @events_mask: indicates the bitmap events |
2285 | */ |
2286 | struct hl_notifier_event { |
2287 | struct eventfd_ctx *eventfd; |
2288 | struct mutex lock; |
2289 | u64 events_mask; |
2290 | }; |
2291 | |
2292 | /* |
2293 | * FILE PRIVATE STRUCTURE |
2294 | */ |
2295 | |
2296 | /** |
2297 | * struct hl_fpriv - process information stored in FD private data. |
2298 | * @hdev: habanalabs device structure. |
2299 | * @file_priv: pointer to the DRM file private data structure. |
2300 | * @taskpid: current process ID. |
2301 | * @ctx: current executing context. TODO: remove for multiple ctx per process |
2302 | * @ctx_mgr: context manager to handle multiple context for this FD. |
2303 | * @mem_mgr: manager descriptor for memory exportable via mmap |
2304 | * @notifier_event: notifier eventfd towards user process |
2305 | * @debugfs_list: list of relevant ASIC debugfs. |
2306 | * @dev_node: node in the device list of file private data |
2307 | * @refcount: number of related contexts. |
2308 | * @restore_phase_mutex: lock for context switch and restore phase. |
2309 | * @ctx_lock: protects the pointer to current executing context pointer. TODO: remove for multiple |
2310 | * ctx per process. |
2311 | */ |
2312 | struct hl_fpriv { |
2313 | struct hl_device *hdev; |
2314 | struct drm_file *file_priv; |
2315 | struct pid *taskpid; |
2316 | struct hl_ctx *ctx; |
2317 | struct hl_ctx_mgr ctx_mgr; |
2318 | struct hl_mem_mgr mem_mgr; |
2319 | struct hl_notifier_event notifier_event; |
2320 | struct list_head debugfs_list; |
2321 | struct list_head dev_node; |
2322 | struct kref refcount; |
2323 | struct mutex restore_phase_mutex; |
2324 | struct mutex ctx_lock; |
2325 | }; |
2326 | |
2327 | |
2328 | /* |
2329 | * DebugFS |
2330 | */ |
2331 | |
2332 | /** |
2333 | * struct hl_info_list - debugfs file ops. |
2334 | * @name: file name. |
2335 | * @show: function to output information. |
2336 | * @write: function to write to the file. |
2337 | */ |
2338 | struct hl_info_list { |
2339 | const char *name; |
2340 | int (*show)(struct seq_file *s, void *data); |
2341 | ssize_t (*write)(struct file *file, const char __user *buf, |
2342 | size_t count, loff_t *f_pos); |
2343 | }; |
2344 | |
2345 | /** |
2346 | * struct hl_debugfs_entry - debugfs dentry wrapper. |
2347 | * @info_ent: dentry related ops. |
2348 | * @dev_entry: ASIC specific debugfs manager. |
2349 | */ |
2350 | struct hl_debugfs_entry { |
2351 | const struct hl_info_list *info_ent; |
2352 | struct hl_dbg_device_entry *dev_entry; |
2353 | }; |
2354 | |
2355 | /** |
2356 | * struct hl_dbg_device_entry - ASIC specific debugfs manager. |
2357 | * @root: root dentry. |
2358 | * @hdev: habanalabs device structure. |
2359 | * @entry_arr: array of available hl_debugfs_entry. |
2360 | * @file_list: list of available debugfs files. |
2361 | * @file_mutex: protects file_list. |
2362 | * @cb_list: list of available CBs. |
2363 | * @cb_spinlock: protects cb_list. |
2364 | * @cs_list: list of available CSs. |
2365 | * @cs_spinlock: protects cs_list. |
2366 | * @cs_job_list: list of available CB jobs. |
2367 | * @cs_job_spinlock: protects cs_job_list. |
2368 | * @userptr_list: list of available userptrs (virtual memory chunk descriptor). |
2369 | * @userptr_spinlock: protects userptr_list. |
2370 | * @ctx_mem_hash_list: list of available contexts with MMU mappings. |
2371 | * @ctx_mem_hash_mutex: protects list of available contexts with MMU mappings. |
2372 | * @data_dma_blob_desc: data DMA descriptor of blob. |
2373 | * @mon_dump_blob_desc: monitor dump descriptor of blob. |
2374 | * @state_dump: data of the system states in case of a bad cs. |
2375 | * @state_dump_sem: protects state_dump. |
2376 | * @addr: next address to read/write from/to in read/write32. |
2377 | * @mmu_addr: next virtual address to translate to physical address in mmu_show. |
2378 | * @mmu_cap_mask: mmu hw capability mask, to be used in mmu_ack_error. |
2379 | * @userptr_lookup: the target user ptr to look up for on demand. |
2380 | * @mmu_asid: ASID to use while translating in mmu_show. |
2381 | * @state_dump_head: index of the latest state dump |
2382 | * @i2c_bus: generic u8 debugfs file for bus value to use in i2c_data_read. |
2383 | * @i2c_addr: generic u8 debugfs file for address value to use in i2c_data_read. |
2384 | * @i2c_reg: generic u8 debugfs file for register value to use in i2c_data_read. |
2385 | * @i2c_len: generic u8 debugfs file for length value to use in i2c_data_read. |
2386 | */ |
2387 | struct hl_dbg_device_entry { |
2388 | struct dentry *root; |
2389 | struct hl_device *hdev; |
2390 | struct hl_debugfs_entry *entry_arr; |
2391 | struct list_head file_list; |
2392 | struct mutex file_mutex; |
2393 | struct list_head cb_list; |
2394 | spinlock_t cb_spinlock; |
2395 | struct list_head cs_list; |
2396 | spinlock_t cs_spinlock; |
2397 | struct list_head cs_job_list; |
2398 | spinlock_t cs_job_spinlock; |
2399 | struct list_head userptr_list; |
2400 | spinlock_t userptr_spinlock; |
2401 | struct list_head ctx_mem_hash_list; |
2402 | struct mutex ctx_mem_hash_mutex; |
2403 | struct debugfs_blob_wrapper data_dma_blob_desc; |
2404 | struct debugfs_blob_wrapper mon_dump_blob_desc; |
2405 | char *state_dump[HL_STATE_DUMP_HIST_LEN]; |
2406 | struct rw_semaphore state_dump_sem; |
2407 | u64 addr; |
2408 | u64 mmu_addr; |
2409 | u64 mmu_cap_mask; |
2410 | u64 userptr_lookup; |
2411 | u32 mmu_asid; |
2412 | u32 state_dump_head; |
2413 | u8 i2c_bus; |
2414 | u8 i2c_addr; |
2415 | u8 i2c_reg; |
2416 | u8 i2c_len; |
2417 | }; |
2418 | |
2419 | /** |
2420 | * struct hl_hw_obj_name_entry - single hw object name, member of |
2421 | * hl_state_dump_specs |
2422 | * @node: link to the containing hash table |
2423 | * @name: hw object name |
2424 | * @id: object identifier |
2425 | */ |
2426 | struct hl_hw_obj_name_entry { |
2427 | struct hlist_node node; |
2428 | const char *name; |
2429 | u32 id; |
2430 | }; |
2431 | |
2432 | enum hl_state_dump_specs_props { |
2433 | SP_SYNC_OBJ_BASE_ADDR, |
2434 | SP_NEXT_SYNC_OBJ_ADDR, |
2435 | SP_SYNC_OBJ_AMOUNT, |
2436 | SP_MON_OBJ_WR_ADDR_LOW, |
2437 | SP_MON_OBJ_WR_ADDR_HIGH, |
2438 | SP_MON_OBJ_WR_DATA, |
2439 | SP_MON_OBJ_ARM_DATA, |
2440 | SP_MON_OBJ_STATUS, |
2441 | SP_MONITORS_AMOUNT, |
2442 | SP_TPC0_CMDQ, |
2443 | SP_TPC0_CFG_SO, |
2444 | SP_NEXT_TPC, |
2445 | SP_MME_CMDQ, |
2446 | SP_MME_CFG_SO, |
2447 | SP_NEXT_MME, |
2448 | SP_DMA_CMDQ, |
2449 | SP_DMA_CFG_SO, |
2450 | SP_DMA_QUEUES_OFFSET, |
2451 | SP_NUM_OF_MME_ENGINES, |
2452 | SP_SUB_MME_ENG_NUM, |
2453 | SP_NUM_OF_DMA_ENGINES, |
2454 | SP_NUM_OF_TPC_ENGINES, |
2455 | SP_ENGINE_NUM_OF_QUEUES, |
2456 | SP_ENGINE_NUM_OF_STREAMS, |
2457 | SP_ENGINE_NUM_OF_FENCES, |
2458 | SP_FENCE0_CNT_OFFSET, |
2459 | SP_FENCE0_RDATA_OFFSET, |
2460 | SP_CP_STS_OFFSET, |
2461 | SP_NUM_CORES, |
2462 | |
2463 | SP_MAX |
2464 | }; |
2465 | |
2466 | enum hl_sync_engine_type { |
2467 | ENGINE_TPC, |
2468 | ENGINE_DMA, |
2469 | ENGINE_MME, |
2470 | }; |
2471 | |
2472 | /** |
2473 | * struct hl_mon_state_dump - represents a state dump of a single monitor |
2474 | * @id: monitor id |
2475 | * @wr_addr_low: address monitor will write to, low bits |
2476 | * @wr_addr_high: address monitor will write to, high bits |
2477 | * @wr_data: data monitor will write |
2478 | * @arm_data: register value containing monitor configuration |
2479 | * @status: monitor status |
2480 | */ |
2481 | struct hl_mon_state_dump { |
2482 | u32 id; |
2483 | u32 wr_addr_low; |
2484 | u32 wr_addr_high; |
2485 | u32 wr_data; |
2486 | u32 arm_data; |
2487 | u32 status; |
2488 | }; |
2489 | |
2490 | /** |
2491 | * struct hl_sync_to_engine_map_entry - sync object id to engine mapping entry |
2492 | * @engine_type: type of the engine |
2493 | * @engine_id: id of the engine |
2494 | * @sync_id: id of the sync object |
2495 | */ |
2496 | struct hl_sync_to_engine_map_entry { |
2497 | struct hlist_node node; |
2498 | enum hl_sync_engine_type engine_type; |
2499 | u32 engine_id; |
2500 | u32 sync_id; |
2501 | }; |
2502 | |
2503 | /** |
2504 | * struct hl_sync_to_engine_map - maps sync object id to associated engine id |
2505 | * @tb: hash table containing the mapping, each element is of type |
2506 | * struct hl_sync_to_engine_map_entry |
2507 | */ |
2508 | struct hl_sync_to_engine_map { |
2509 | DECLARE_HASHTABLE(tb, SYNC_TO_ENGINE_HASH_TABLE_BITS); |
2510 | }; |
2511 | |
2512 | /** |
2513 | * struct hl_state_dump_specs_funcs - virtual functions used by the state dump |
2514 | * @gen_sync_to_engine_map: generate a hash map from sync obj id to its engine |
2515 | * @print_single_monitor: format monitor data as string |
2516 | * @monitor_valid: return true if given monitor dump is valid |
2517 | * @print_fences_single_engine: format fences data as string |
2518 | */ |
2519 | struct hl_state_dump_specs_funcs { |
2520 | int (*gen_sync_to_engine_map)(struct hl_device *hdev, |
2521 | struct hl_sync_to_engine_map *map); |
2522 | int (*print_single_monitor)(char **buf, size_t *size, size_t *offset, |
2523 | struct hl_device *hdev, |
2524 | struct hl_mon_state_dump *mon); |
2525 | int (*monitor_valid)(struct hl_mon_state_dump *mon); |
2526 | int (*print_fences_single_engine)(struct hl_device *hdev, |
2527 | u64 base_offset, |
2528 | u64 status_base_offset, |
2529 | enum hl_sync_engine_type engine_type, |
2530 | u32 engine_id, char **buf, |
2531 | size_t *size, size_t *offset); |
2532 | }; |
2533 | |
2534 | /** |
2535 | * struct hl_state_dump_specs - defines ASIC known hw objects names |
2536 | * @so_id_to_str_tb: sync objects names index table |
2537 | * @monitor_id_to_str_tb: monitors names index table |
2538 | * @funcs: virtual functions used for state dump |
2539 | * @sync_namager_names: readable names for sync manager if available (ex: N_E) |
2540 | * @props: pointer to a per asic const props array required for state dump |
2541 | */ |
2542 | struct hl_state_dump_specs { |
2543 | DECLARE_HASHTABLE(so_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS); |
2544 | DECLARE_HASHTABLE(monitor_id_to_str_tb, OBJ_NAMES_HASH_TABLE_BITS); |
2545 | struct hl_state_dump_specs_funcs funcs; |
2546 | const char * const *sync_namager_names; |
2547 | s64 *props; |
2548 | }; |
2549 | |
2550 | |
2551 | /* |
2552 | * DEVICES |
2553 | */ |
2554 | |
2555 | #define HL_STR_MAX 64 |
2556 | |
2557 | #define HL_DEV_STS_MAX (HL_DEVICE_STATUS_LAST + 1) |
2558 | |
2559 | /* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe |
2560 | * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards. |
2561 | */ |
2562 | #define HL_MAX_MINORS 256 |
2563 | |
2564 | /* |
2565 | * Registers read & write functions. |
2566 | */ |
2567 | |
2568 | u32 hl_rreg(struct hl_device *hdev, u32 reg); |
2569 | void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); |
2570 | |
2571 | #define RREG32(reg) hdev->asic_funcs->rreg(hdev, (reg)) |
2572 | #define WREG32(reg, v) hdev->asic_funcs->wreg(hdev, (reg), (v)) |
2573 | #define DREG32(reg) pr_info("REGISTER: " #reg " : 0x%08X\n", \ |
2574 | hdev->asic_funcs->rreg(hdev, (reg))) |
2575 | |
2576 | #define WREG32_P(reg, val, mask) \ |
2577 | do { \ |
2578 | u32 tmp_ = RREG32(reg); \ |
2579 | tmp_ &= (mask); \ |
2580 | tmp_ |= ((val) & ~(mask)); \ |
2581 | WREG32(reg, tmp_); \ |
2582 | } while (0) |
2583 | #define WREG32_AND(reg, and) WREG32_P(reg, 0, and) |
2584 | #define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or)) |
2585 | |
2586 | #define RMWREG32_SHIFTED(reg, val, mask) WREG32_P(reg, val, ~(mask)) |
2587 | |
2588 | #define RMWREG32(reg, val, mask) RMWREG32_SHIFTED(reg, (val) << __ffs(mask), mask) |
2589 | |
2590 | #define RREG32_MASK(reg, mask) ((RREG32(reg) & mask) >> __ffs(mask)) |
2591 | |
2592 | #define REG_FIELD_SHIFT(reg, field) reg##_##field##_SHIFT |
2593 | #define REG_FIELD_MASK(reg, field) reg##_##field##_MASK |
2594 | #define WREG32_FIELD(reg, offset, field, val) \ |
2595 | WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & \ |
2596 | ~REG_FIELD_MASK(reg, field)) | \ |
2597 | (val) << REG_FIELD_SHIFT(reg, field)) |
2598 | |
2599 | /* Timeout should be longer when working with simulator but cap the |
2600 | * increased timeout to some maximum |
2601 | */ |
2602 | #define hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, elbi) \ |
2603 | ({ \ |
2604 | ktime_t __timeout; \ |
2605 | u32 __elbi_read; \ |
2606 | int __rc = 0; \ |
2607 | __timeout = ktime_add_us(ktime_get(), timeout_us); \ |
2608 | might_sleep_if(sleep_us); \ |
2609 | for (;;) { \ |
2610 | if (elbi) { \ |
2611 | __rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \ |
2612 | if (__rc) \ |
2613 | break; \ |
2614 | (val) = __elbi_read; \ |
2615 | } else {\ |
2616 | (val) = RREG32(lower_32_bits(addr)); \ |
2617 | } \ |
2618 | if (cond) \ |
2619 | break; \ |
2620 | if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ |
2621 | if (elbi) { \ |
2622 | __rc = hl_pci_elbi_read(hdev, addr, &__elbi_read); \ |
2623 | if (__rc) \ |
2624 | break; \ |
2625 | (val) = __elbi_read; \ |
2626 | } else {\ |
2627 | (val) = RREG32(lower_32_bits(addr)); \ |
2628 | } \ |
2629 | break; \ |
2630 | } \ |
2631 | if (sleep_us) \ |
2632 | usleep_range((sleep_us >> 2) + 1, sleep_us); \ |
2633 | } \ |
2634 | __rc ? __rc : ((cond) ? 0 : -ETIMEDOUT); \ |
2635 | }) |
2636 | |
2637 | #define hl_poll_timeout(hdev, addr, val, cond, sleep_us, timeout_us) \ |
2638 | hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, false) |
2639 | |
2640 | #define hl_poll_timeout_elbi(hdev, addr, val, cond, sleep_us, timeout_us) \ |
2641 | hl_poll_timeout_common(hdev, addr, val, cond, sleep_us, timeout_us, true) |
2642 | |
2643 | /* |
2644 | * poll array of register addresses. |
2645 | * condition is satisfied if all registers values match the expected value. |
2646 | * once some register in the array satisfies the condition it will not be polled again, |
2647 | * this is done both for efficiency and due to some registers are "clear on read". |
2648 | * TODO: use read from PCI bar in other places in the code (SW-91406) |
2649 | */ |
2650 | #define hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ |
2651 | timeout_us, elbi) \ |
2652 | ({ \ |
2653 | ktime_t __timeout; \ |
2654 | u64 __elem_bitmask; \ |
2655 | u32 __read_val; \ |
2656 | u8 __arr_idx; \ |
2657 | int __rc = 0; \ |
2658 | \ |
2659 | __timeout = ktime_add_us(ktime_get(), timeout_us); \ |
2660 | might_sleep_if(sleep_us); \ |
2661 | if (arr_size >= 64) \ |
2662 | __rc = -EINVAL; \ |
2663 | else \ |
2664 | __elem_bitmask = BIT_ULL(arr_size) - 1; \ |
2665 | for (;;) { \ |
2666 | if (__rc) \ |
2667 | break; \ |
2668 | for (__arr_idx = 0; __arr_idx < (arr_size); __arr_idx++) { \ |
2669 | if (!(__elem_bitmask & BIT_ULL(__arr_idx))) \ |
2670 | continue; \ |
2671 | if (elbi) { \ |
2672 | __rc = hl_pci_elbi_read(hdev, (addr_arr)[__arr_idx], &__read_val); \ |
2673 | if (__rc) \ |
2674 | break; \ |
2675 | } else { \ |
2676 | __read_val = RREG32(lower_32_bits(addr_arr[__arr_idx])); \ |
2677 | } \ |
2678 | if (__read_val == (expected_val)) \ |
2679 | __elem_bitmask &= ~BIT_ULL(__arr_idx); \ |
2680 | } \ |
2681 | if (__rc || (__elem_bitmask == 0)) \ |
2682 | break; \ |
2683 | if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) \ |
2684 | break; \ |
2685 | if (sleep_us) \ |
2686 | usleep_range((sleep_us >> 2) + 1, sleep_us); \ |
2687 | } \ |
2688 | __rc ? __rc : ((__elem_bitmask == 0) ? 0 : -ETIMEDOUT); \ |
2689 | }) |
2690 | |
2691 | #define hl_poll_reg_array_timeout(hdev, addr_arr, arr_size, expected_val, sleep_us, \ |
2692 | timeout_us) \ |
2693 | hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ |
2694 | timeout_us, false) |
2695 | |
2696 | #define hl_poll_reg_array_timeout_elbi(hdev, addr_arr, arr_size, expected_val, sleep_us, \ |
2697 | timeout_us) \ |
2698 | hl_poll_reg_array_timeout_common(hdev, addr_arr, arr_size, expected_val, sleep_us, \ |
2699 | timeout_us, true) |
2700 | |
2701 | /* |
2702 | * address in this macro points always to a memory location in the |
2703 | * host's (server's) memory. That location is updated asynchronously |
2704 | * either by the direct access of the device or by another core. |
2705 | * |
2706 | * To work both in LE and BE architectures, we need to distinguish between the |
2707 | * two states (device or another core updates the memory location). Therefore, |
2708 | * if mem_written_by_device is true, the host memory being polled will be |
2709 | * updated directly by the device. If false, the host memory being polled will |
2710 | * be updated by host CPU. Required so host knows whether or not the memory |
2711 | * might need to be byte-swapped before returning value to caller. |
2712 | */ |
2713 | #define hl_poll_timeout_memory(hdev, addr, val, cond, sleep_us, timeout_us, \ |
2714 | mem_written_by_device) \ |
2715 | ({ \ |
2716 | ktime_t __timeout; \ |
2717 | \ |
2718 | __timeout = ktime_add_us(ktime_get(), timeout_us); \ |
2719 | might_sleep_if(sleep_us); \ |
2720 | for (;;) { \ |
2721 | /* Verify we read updates done by other cores or by device */ \ |
2722 | mb(); \ |
2723 | (val) = *((u32 *)(addr)); \ |
2724 | if (mem_written_by_device) \ |
2725 | (val) = le32_to_cpu(*(__le32 *) &(val)); \ |
2726 | if (cond) \ |
2727 | break; \ |
2728 | if (timeout_us && ktime_compare(ktime_get(), __timeout) > 0) { \ |
2729 | (val) = *((u32 *)(addr)); \ |
2730 | if (mem_written_by_device) \ |
2731 | (val) = le32_to_cpu(*(__le32 *) &(val)); \ |
2732 | break; \ |
2733 | } \ |
2734 | if (sleep_us) \ |
2735 | usleep_range((sleep_us >> 2) + 1, sleep_us); \ |
2736 | } \ |
2737 | (cond) ? 0 : -ETIMEDOUT; \ |
2738 | }) |
2739 | |
2740 | #define HL_USR_MAPPED_BLK_INIT(blk, base, sz) \ |
2741 | ({ \ |
2742 | struct user_mapped_block *p = blk; \ |
2743 | \ |
2744 | p->address = base; \ |
2745 | p->size = sz; \ |
2746 | }) |
2747 | |
2748 | #define HL_USR_INTR_STRUCT_INIT(usr_intr, hdev, intr_id, intr_type) \ |
2749 | ({ \ |
2750 | usr_intr.hdev = hdev; \ |
2751 | usr_intr.interrupt_id = intr_id; \ |
2752 | usr_intr.type = intr_type; \ |
2753 | INIT_LIST_HEAD(&usr_intr.wait_list_head); \ |
2754 | spin_lock_init(&usr_intr.wait_list_lock); \ |
2755 | INIT_LIST_HEAD(&usr_intr.ts_list_head); \ |
2756 | spin_lock_init(&usr_intr.ts_list_lock); \ |
2757 | }) |
2758 | |
2759 | struct hwmon_chip_info; |
2760 | |
2761 | /** |
2762 | * struct hl_device_reset_work - reset work wrapper. |
2763 | * @reset_work: reset work to be done. |
2764 | * @hdev: habanalabs device structure. |
2765 | * @flags: reset flags. |
2766 | */ |
2767 | struct hl_device_reset_work { |
2768 | struct delayed_work reset_work; |
2769 | struct hl_device *hdev; |
2770 | u32 flags; |
2771 | }; |
2772 | |
2773 | /** |
2774 | * struct hl_mmu_hr_pgt_priv - used for holding per-device mmu host-resident |
2775 | * page-table internal information. |
2776 | * @mmu_pgt_pool: pool of page tables used by a host-resident MMU for |
2777 | * allocating hops. |
2778 | * @mmu_asid_hop0: per-ASID array of host-resident hop0 tables. |
2779 | */ |
2780 | struct hl_mmu_hr_priv { |
2781 | struct gen_pool *mmu_pgt_pool; |
2782 | struct pgt_info *mmu_asid_hop0; |
2783 | }; |
2784 | |
2785 | /** |
2786 | * struct hl_mmu_dr_pgt_priv - used for holding per-device mmu device-resident |
2787 | * page-table internal information. |
2788 | * @mmu_pgt_pool: pool of page tables used by MMU for allocating hops. |
2789 | * @mmu_shadow_hop0: shadow array of hop0 tables. |
2790 | */ |
2791 | struct hl_mmu_dr_priv { |
2792 | struct gen_pool *mmu_pgt_pool; |
2793 | void *mmu_shadow_hop0; |
2794 | }; |
2795 | |
2796 | /** |
2797 | * struct hl_mmu_priv - used for holding per-device mmu internal information. |
2798 | * @dr: information on the device-resident MMU, when exists. |
2799 | * @hr: information on the host-resident MMU, when exists. |
2800 | */ |
2801 | struct hl_mmu_priv { |
2802 | struct hl_mmu_dr_priv dr; |
2803 | struct hl_mmu_hr_priv hr; |
2804 | }; |
2805 | |
2806 | /** |
2807 | * struct hl_mmu_per_hop_info - A structure describing one TLB HOP and its entry |
2808 | * that was created in order to translate a virtual address to a |
2809 | * physical one. |
2810 | * @hop_addr: The address of the hop. |
2811 | * @hop_pte_addr: The address of the hop entry. |
2812 | * @hop_pte_val: The value in the hop entry. |
2813 | */ |
2814 | struct hl_mmu_per_hop_info { |
2815 | u64 hop_addr; |
2816 | u64 hop_pte_addr; |
2817 | u64 hop_pte_val; |
2818 | }; |
2819 | |
2820 | /** |
2821 | * struct hl_mmu_hop_info - A structure describing the TLB hops and their |
2822 | * hop-entries that were created in order to translate a virtual address to a |
2823 | * physical one. |
2824 | * @scrambled_vaddr: The value of the virtual address after scrambling. This |
2825 | * address replaces the original virtual-address when mapped |
2826 | * in the MMU tables. |
2827 | * @unscrambled_paddr: The un-scrambled physical address. |
2828 | * @hop_info: Array holding the per-hop information used for the translation. |
2829 | * @used_hops: The number of hops used for the translation. |
2830 | * @range_type: virtual address range type. |
2831 | */ |
2832 | struct hl_mmu_hop_info { |
2833 | u64 scrambled_vaddr; |
2834 | u64 unscrambled_paddr; |
2835 | struct hl_mmu_per_hop_info hop_info[MMU_ARCH_6_HOPS]; |
2836 | u32 used_hops; |
2837 | enum hl_va_range_type range_type; |
2838 | }; |
2839 | |
2840 | /** |
2841 | * struct hl_hr_mmu_funcs - Device related host resident MMU functions. |
2842 | * @get_hop0_pgt_info: get page table info structure for HOP0. |
2843 | * @get_pgt_info: get page table info structure for HOP other than HOP0. |
2844 | * @add_pgt_info: add page table info structure to hash. |
2845 | * @get_tlb_mapping_params: get mapping parameters needed for getting TLB info for specific mapping. |
2846 | */ |
2847 | struct hl_hr_mmu_funcs { |
2848 | struct pgt_info *(*get_hop0_pgt_info)(struct hl_ctx *ctx); |
2849 | struct pgt_info *(*get_pgt_info)(struct hl_ctx *ctx, u64 phys_hop_addr); |
2850 | void (*add_pgt_info)(struct hl_ctx *ctx, struct pgt_info *pgt_info, dma_addr_t phys_addr); |
2851 | int (*get_tlb_mapping_params)(struct hl_device *hdev, struct hl_mmu_properties **mmu_prop, |
2852 | struct hl_mmu_hop_info *hops, |
2853 | u64 virt_addr, bool *is_huge); |
2854 | }; |
2855 | |
2856 | /** |
2857 | * struct hl_mmu_funcs - Device related MMU functions. |
2858 | * @init: initialize the MMU module. |
2859 | * @fini: release the MMU module. |
2860 | * @ctx_init: Initialize a context for using the MMU module. |
2861 | * @ctx_fini: disable a ctx from using the mmu module. |
2862 | * @map: maps a virtual address to physical address for a context. |
2863 | * @unmap: unmap a virtual address of a context. |
2864 | * @flush: flush all writes from all cores to reach device MMU. |
2865 | * @swap_out: marks all mapping of the given context as swapped out. |
2866 | * @swap_in: marks all mapping of the given context as swapped in. |
2867 | * @get_tlb_info: returns the list of hops and hop-entries used that were |
2868 | * created in order to translate the giver virtual address to a |
2869 | * physical one. |
2870 | * @hr_funcs: functions specific to host resident MMU. |
2871 | */ |
2872 | struct hl_mmu_funcs { |
2873 | int (*init)(struct hl_device *hdev); |
2874 | void (*fini)(struct hl_device *hdev); |
2875 | int (*ctx_init)(struct hl_ctx *ctx); |
2876 | void (*ctx_fini)(struct hl_ctx *ctx); |
2877 | int (*map)(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size, |
2878 | bool is_dram_addr); |
2879 | int (*unmap)(struct hl_ctx *ctx, u64 virt_addr, bool is_dram_addr); |
2880 | void (*flush)(struct hl_ctx *ctx); |
2881 | void (*swap_out)(struct hl_ctx *ctx); |
2882 | void (*swap_in)(struct hl_ctx *ctx); |
2883 | int (*get_tlb_info)(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops); |
2884 | struct hl_hr_mmu_funcs hr_funcs; |
2885 | }; |
2886 | |
2887 | /** |
2888 | * struct hl_prefetch_work - prefetch work structure handler |
2889 | * @prefetch_work: actual work struct. |
2890 | * @ctx: compute context. |
2891 | * @va: virtual address to pre-fetch. |
2892 | * @size: pre-fetch size. |
2893 | * @flags: operation flags. |
2894 | * @asid: ASID for maintenance operation. |
2895 | */ |
2896 | struct hl_prefetch_work { |
2897 | struct work_struct prefetch_work; |
2898 | struct hl_ctx *ctx; |
2899 | u64 va; |
2900 | u64 size; |
2901 | u32 flags; |
2902 | u32 asid; |
2903 | }; |
2904 | |
2905 | /* |
2906 | * number of user contexts allowed to call wait_for_multi_cs ioctl in |
2907 | * parallel |
2908 | */ |
2909 | #define MULTI_CS_MAX_USER_CTX 2 |
2910 | |
2911 | /** |
2912 | * struct multi_cs_completion - multi CS wait completion. |
2913 | * @completion: completion of any of the CS in the list |
2914 | * @lock: spinlock for the completion structure |
2915 | * @timestamp: timestamp for the multi-CS completion |
2916 | * @stream_master_qid_map: bitmap of all stream masters on which the multi-CS |
2917 | * is waiting |
2918 | * @used: 1 if in use, otherwise 0 |
2919 | */ |
2920 | struct multi_cs_completion { |
2921 | struct completion completion; |
2922 | spinlock_t lock; |
2923 | s64 timestamp; |
2924 | u32 stream_master_qid_map; |
2925 | u8 used; |
2926 | }; |
2927 | |
2928 | /** |
2929 | * struct multi_cs_data - internal data for multi CS call |
2930 | * @ctx: pointer to the context structure |
2931 | * @fence_arr: array of fences of all CSs |
2932 | * @seq_arr: array of CS sequence numbers |
2933 | * @timeout_jiffies: timeout in jiffies for waiting for CS to complete |
2934 | * @timestamp: timestamp of first completed CS |
2935 | * @wait_status: wait for CS status |
2936 | * @completion_bitmap: bitmap of completed CSs (1- completed, otherwise 0) |
2937 | * @arr_len: fence_arr and seq_arr array length |
2938 | * @gone_cs: indication of gone CS (1- there was gone CS, otherwise 0) |
2939 | * @update_ts: update timestamp. 1- update the timestamp, otherwise 0. |
2940 | */ |
2941 | struct multi_cs_data { |
2942 | struct hl_ctx *ctx; |
2943 | struct hl_fence **fence_arr; |
2944 | u64 *seq_arr; |
2945 | s64 timeout_jiffies; |
2946 | s64 timestamp; |
2947 | long wait_status; |
2948 | u32 completion_bitmap; |
2949 | u8 arr_len; |
2950 | u8 gone_cs; |
2951 | u8 update_ts; |
2952 | }; |
2953 | |
2954 | /** |
2955 | * struct hl_clk_throttle_timestamp - current/last clock throttling timestamp |
2956 | * @start: timestamp taken when 'start' event is received in driver |
2957 | * @end: timestamp taken when 'end' event is received in driver |
2958 | */ |
2959 | struct hl_clk_throttle_timestamp { |
2960 | ktime_t start; |
2961 | ktime_t end; |
2962 | }; |
2963 | |
2964 | /** |
2965 | * struct hl_clk_throttle - keeps current/last clock throttling timestamps |
2966 | * @timestamp: timestamp taken by driver and firmware, index 0 refers to POWER |
2967 | * index 1 refers to THERMAL |
2968 | * @lock: protects this structure as it can be accessed from both event queue |
2969 | * context and info_ioctl context |
2970 | * @current_reason: bitmask represents the current clk throttling reasons |
2971 | * @aggregated_reason: bitmask represents aggregated clk throttling reasons since driver load |
2972 | */ |
2973 | struct hl_clk_throttle { |
2974 | struct hl_clk_throttle_timestamp timestamp[HL_CLK_THROTTLE_TYPE_MAX]; |
2975 | struct mutex lock; |
2976 | u32 current_reason; |
2977 | u32 aggregated_reason; |
2978 | }; |
2979 | |
2980 | /** |
2981 | * struct user_mapped_block - describes a hw block allowed to be mmapped by user |
2982 | * @address: physical HW block address |
2983 | * @size: allowed size for mmap |
2984 | */ |
2985 | struct user_mapped_block { |
2986 | u32 address; |
2987 | u32 size; |
2988 | }; |
2989 | |
2990 | /** |
2991 | * struct cs_timeout_info - info of last CS timeout occurred. |
2992 | * @timestamp: CS timeout timestamp. |
2993 | * @write_enable: if set writing to CS parameters in the structure is enabled. otherwise - disabled, |
2994 | * so the first (root cause) CS timeout will not be overwritten. |
2995 | * @seq: CS timeout sequence number. |
2996 | */ |
2997 | struct cs_timeout_info { |
2998 | ktime_t timestamp; |
2999 | atomic_t write_enable; |
3000 | u64 seq; |
3001 | }; |
3002 | |
3003 | #define MAX_QMAN_STREAMS_INFO 4 |
3004 | #define OPCODE_INFO_MAX_ADDR_SIZE 8 |
3005 | /** |
3006 | * struct undefined_opcode_info - info about last undefined opcode error |
3007 | * @timestamp: timestamp of the undefined opcode error |
3008 | * @cb_addr_streams: CB addresses (per stream) that are currently exists in the PQ |
3009 | * entries. In case all streams array entries are |
3010 | * filled with values, it means the execution was in Lower-CP. |
3011 | * @cq_addr: the address of the current handled command buffer |
3012 | * @cq_size: the size of the current handled command buffer |
3013 | * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array. |
3014 | * should be equal to 1 in case of undefined opcode |
3015 | * in Upper-CP (specific stream) and equal to 4 in case |
3016 | * of undefined opcode in Lower-CP. |
3017 | * @engine_id: engine-id that the error occurred on |
3018 | * @stream_id: the stream id the error occurred on. In case the stream equals to |
3019 | * MAX_QMAN_STREAMS_INFO it means the error occurred on a Lower-CP. |
3020 | * @write_enable: if set, writing to undefined opcode parameters in the structure |
3021 | * is enable so the first (root cause) undefined opcode will not be |
3022 | * overwritten. |
3023 | */ |
3024 | struct undefined_opcode_info { |
3025 | ktime_t timestamp; |
3026 | u64 cb_addr_streams[MAX_QMAN_STREAMS_INFO][OPCODE_INFO_MAX_ADDR_SIZE]; |
3027 | u64 cq_addr; |
3028 | u32 cq_size; |
3029 | u32 cb_addr_streams_len; |
3030 | u32 engine_id; |
3031 | u32 stream_id; |
3032 | bool write_enable; |
3033 | }; |
3034 | |
3035 | /** |
3036 | * struct page_fault_info - page fault information. |
3037 | * @page_fault: holds information collected during a page fault. |
3038 | * @user_mappings: buffer containing user mappings. |
3039 | * @num_of_user_mappings: number of user mappings. |
3040 | * @page_fault_detected: if set as 1, then a page-fault was discovered for the |
3041 | * first time after the driver has finished booting-up. |
3042 | * Since we're looking for the page-fault's root cause, |
3043 | * we don't care of the others that might follow it- |
3044 | * so once changed to 1, it will remain that way. |
3045 | * @page_fault_info_available: indicates that a page fault info is now available. |
3046 | */ |
3047 | struct page_fault_info { |
3048 | struct hl_page_fault_info page_fault; |
3049 | struct hl_user_mapping *user_mappings; |
3050 | u64 num_of_user_mappings; |
3051 | atomic_t page_fault_detected; |
3052 | bool page_fault_info_available; |
3053 | }; |
3054 | |
3055 | /** |
3056 | * struct razwi_info - RAZWI information. |
3057 | * @razwi: holds information collected during a RAZWI |
3058 | * @razwi_detected: if set as 1, then a RAZWI was discovered for the |
3059 | * first time after the driver has finished booting-up. |
3060 | * Since we're looking for the RAZWI's root cause, |
3061 | * we don't care of the others that might follow it- |
3062 | * so once changed to 1, it will remain that way. |
3063 | * @razwi_info_available: indicates that a RAZWI info is now available. |
3064 | */ |
3065 | struct razwi_info { |
3066 | struct hl_info_razwi_event razwi; |
3067 | atomic_t razwi_detected; |
3068 | bool razwi_info_available; |
3069 | }; |
3070 | |
3071 | /** |
3072 | * struct hw_err_info - HW error information. |
3073 | * @event: holds information on the event. |
3074 | * @event_detected: if set as 1, then a HW event was discovered for the |
3075 | * first time after the driver has finished booting-up. |
3076 | * currently we assume that only fatal events (that require hard-reset) are |
3077 | * reported so we don't care of the others that might follow it. |
3078 | * so once changed to 1, it will remain that way. |
3079 | * TODO: support multiple events. |
3080 | * @event_info_available: indicates that a HW event info is now available. |
3081 | */ |
3082 | struct hw_err_info { |
3083 | struct hl_info_hw_err_event event; |
3084 | atomic_t event_detected; |
3085 | bool event_info_available; |
3086 | }; |
3087 | |
3088 | /** |
3089 | * struct fw_err_info - FW error information. |
3090 | * @event: holds information on the event. |
3091 | * @event_detected: if set as 1, then a FW event was discovered for the |
3092 | * first time after the driver has finished booting-up. |
3093 | * currently we assume that only fatal events (that require hard-reset) are |
3094 | * reported so we don't care of the others that might follow it. |
3095 | * so once changed to 1, it will remain that way. |
3096 | * TODO: support multiple events. |
3097 | * @event_info_available: indicates that a HW event info is now available. |
3098 | */ |
3099 | struct fw_err_info { |
3100 | struct hl_info_fw_err_event event; |
3101 | atomic_t event_detected; |
3102 | bool event_info_available; |
3103 | }; |
3104 | |
3105 | /** |
3106 | * struct engine_err_info - engine error information. |
3107 | * @event: holds information on the event. |
3108 | * @event_detected: if set as 1, then an engine event was discovered for the |
3109 | * first time after the driver has finished booting-up. |
3110 | * @event_info_available: indicates that an engine event info is now available. |
3111 | */ |
3112 | struct engine_err_info { |
3113 | struct hl_info_engine_err_event event; |
3114 | atomic_t event_detected; |
3115 | bool event_info_available; |
3116 | }; |
3117 | |
3118 | |
3119 | /** |
3120 | * struct hl_error_info - holds information collected during an error. |
3121 | * @cs_timeout: CS timeout error information. |
3122 | * @razwi_info: RAZWI information. |
3123 | * @undef_opcode: undefined opcode information. |
3124 | * @page_fault_info: page fault information. |
3125 | * @hw_err: (fatal) hardware error information. |
3126 | * @fw_err: firmware error information. |
3127 | * @engine_err: engine error information. |
3128 | */ |
3129 | struct hl_error_info { |
3130 | struct cs_timeout_info cs_timeout; |
3131 | struct razwi_info razwi_info; |
3132 | struct undefined_opcode_info undef_opcode; |
3133 | struct page_fault_info page_fault_info; |
3134 | struct hw_err_info hw_err; |
3135 | struct fw_err_info fw_err; |
3136 | struct engine_err_info engine_err; |
3137 | }; |
3138 | |
3139 | /** |
3140 | * struct hl_reset_info - holds current device reset information. |
3141 | * @lock: lock to protect critical reset flows. |
3142 | * @compute_reset_cnt: number of compute resets since the driver was loaded. |
3143 | * @hard_reset_cnt: number of hard resets since the driver was loaded. |
3144 | * @hard_reset_schedule_flags: hard reset is scheduled to after current compute reset, |
3145 | * here we hold the hard reset flags. |
3146 | * @in_reset: is device in reset flow. |
3147 | * @in_compute_reset: Device is currently in reset but not in hard-reset. |
3148 | * @needs_reset: true if reset_on_lockup is false and device should be reset |
3149 | * due to lockup. |
3150 | * @hard_reset_pending: is there a hard reset work pending. |
3151 | * @curr_reset_cause: saves an enumerated reset cause when a hard reset is |
3152 | * triggered, and cleared after it is shared with preboot. |
3153 | * @prev_reset_trigger: saves the previous trigger which caused a reset, overridden |
3154 | * with a new value on next reset |
3155 | * @reset_trigger_repeated: set if device reset is triggered more than once with |
3156 | * same cause. |
3157 | * @skip_reset_on_timeout: Skip device reset if CS has timed out, wait for it to |
3158 | * complete instead. |
3159 | * @watchdog_active: true if a device release watchdog work is scheduled. |
3160 | */ |
3161 | struct hl_reset_info { |
3162 | spinlock_t lock; |
3163 | u32 compute_reset_cnt; |
3164 | u32 hard_reset_cnt; |
3165 | u32 hard_reset_schedule_flags; |
3166 | u8 in_reset; |
3167 | u8 in_compute_reset; |
3168 | u8 needs_reset; |
3169 | u8 hard_reset_pending; |
3170 | u8 curr_reset_cause; |
3171 | u8 prev_reset_trigger; |
3172 | u8 reset_trigger_repeated; |
3173 | u8 skip_reset_on_timeout; |
3174 | u8 watchdog_active; |
3175 | }; |
3176 | |
3177 | /** |
3178 | * struct hl_device - habanalabs device structure. |
3179 | * @pdev: pointer to PCI device, can be NULL in case of simulator device. |
3180 | * @pcie_bar_phys: array of available PCIe bars physical addresses. |
3181 | * (required only for PCI address match mode) |
3182 | * @pcie_bar: array of available PCIe bars virtual addresses. |
3183 | * @rmmio: configuration area address on SRAM. |
3184 | * @drm: related DRM device. |
3185 | * @cdev_ctrl: char device for control operations only (INFO IOCTL) |
3186 | * @dev: related kernel basic device structure. |
3187 | * @dev_ctrl: related kernel device structure for the control device |
3188 | * @work_heartbeat: delayed work for CPU-CP is-alive check. |
3189 | * @device_reset_work: delayed work which performs hard reset |
3190 | * @device_release_watchdog_work: watchdog work that performs hard reset if user doesn't release |
3191 | * device upon certain error cases. |
3192 | * @asic_name: ASIC specific name. |
3193 | * @asic_type: ASIC specific type. |
3194 | * @completion_queue: array of hl_cq. |
3195 | * @user_interrupt: array of hl_user_interrupt. upon the corresponding user |
3196 | * interrupt, driver will monitor the list of fences |
3197 | * registered to this interrupt. |
3198 | * @tpc_interrupt: single TPC interrupt for all TPCs. |
3199 | * @unexpected_error_interrupt: single interrupt for unexpected user error indication. |
3200 | * @common_user_cq_interrupt: common user CQ interrupt for all user CQ interrupts. |
3201 | * upon any user CQ interrupt, driver will monitor the |
3202 | * list of fences registered to this common structure. |
3203 | * @common_decoder_interrupt: common decoder interrupt for all user decoder interrupts. |
3204 | * @shadow_cs_queue: pointer to a shadow queue that holds pointers to |
3205 | * outstanding command submissions. |
3206 | * @cq_wq: work queues of completion queues for executing work in process |
3207 | * context. |
3208 | * @eq_wq: work queue of event queue for executing work in process context. |
3209 | * @cs_cmplt_wq: work queue of CS completions for executing work in process |
3210 | * context. |
3211 | * @ts_free_obj_wq: work queue for timestamp registration objects release. |
3212 | * @prefetch_wq: work queue for MMU pre-fetch operations. |
3213 | * @reset_wq: work queue for device reset procedure. |
3214 | * @kernel_ctx: Kernel driver context structure. |
3215 | * @kernel_queues: array of hl_hw_queue. |
3216 | * @cs_mirror_list: CS mirror list for TDR. |
3217 | * @cs_mirror_lock: protects cs_mirror_list. |
3218 | * @kernel_mem_mgr: memory manager for memory buffers with lifespan of driver. |
3219 | * @event_queue: event queue for IRQ from CPU-CP. |
3220 | * @dma_pool: DMA pool for small allocations. |
3221 | * @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address. |
3222 | * @cpu_accessible_dma_address: Host <-> CPU-CP shared memory DMA address. |
3223 | * @cpu_accessible_dma_pool: Host <-> CPU-CP shared memory pool. |
3224 | * @asid_bitmap: holds used/available ASIDs. |
3225 | * @asid_mutex: protects asid_bitmap. |
3226 | * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue. |
3227 | * @debug_lock: protects critical section of setting debug mode for device |
3228 | * @mmu_lock: protects the MMU page tables and invalidation h/w. Although the |
3229 | * page tables are per context, the invalidation h/w is per MMU. |
3230 | * Therefore, we can't allow multiple contexts (we only have two, |
3231 | * user and kernel) to access the invalidation h/w at the same time. |
3232 | * In addition, any change to the PGT, modifying the MMU hash or |
3233 | * walking the PGT requires talking this lock. |
3234 | * @asic_prop: ASIC specific immutable properties. |
3235 | * @asic_funcs: ASIC specific functions. |
3236 | * @asic_specific: ASIC specific information to use only from ASIC files. |
3237 | * @vm: virtual memory manager for MMU. |
3238 | * @hwmon_dev: H/W monitor device. |
3239 | * @hl_chip_info: ASIC's sensors information. |
3240 | * @device_status_description: device status description. |
3241 | * @hl_debugfs: device's debugfs manager. |
3242 | * @cb_pool: list of pre allocated CBs. |
3243 | * @cb_pool_lock: protects the CB pool. |
3244 | * @internal_cb_pool_virt_addr: internal command buffer pool virtual address. |
3245 | * @internal_cb_pool_dma_addr: internal command buffer pool dma address. |
3246 | * @internal_cb_pool: internal command buffer memory pool. |
3247 | * @internal_cb_va_base: internal cb pool mmu virtual address base |
3248 | * @fpriv_list: list of file private data structures. Each structure is created |
3249 | * when a user opens the device |
3250 | * @fpriv_ctrl_list: list of file private data structures. Each structure is created |
3251 | * when a user opens the control device |
3252 | * @fpriv_list_lock: protects the fpriv_list |
3253 | * @fpriv_ctrl_list_lock: protects the fpriv_ctrl_list |
3254 | * @aggregated_cs_counters: aggregated cs counters among all contexts |
3255 | * @mmu_priv: device-specific MMU data. |
3256 | * @mmu_func: device-related MMU functions. |
3257 | * @dec: list of decoder sw instance |
3258 | * @fw_loader: FW loader manager. |
3259 | * @pci_mem_region: array of memory regions in the PCI |
3260 | * @state_dump_specs: constants and dictionaries needed to dump system state. |
3261 | * @multi_cs_completion: array of multi-CS completion. |
3262 | * @clk_throttling: holds information about current/previous clock throttling events |
3263 | * @captured_err_info: holds information about errors. |
3264 | * @reset_info: holds current device reset information. |
3265 | * @irq_affinity_mask: mask of available CPU cores for user and decoder interrupt handling. |
3266 | * @stream_master_qid_arr: pointer to array with QIDs of master streams. |
3267 | * @fw_inner_major_ver: the major of current loaded preboot inner version. |
3268 | * @fw_inner_minor_ver: the minor of current loaded preboot inner version. |
3269 | * @fw_sw_major_ver: the major of current loaded preboot SW version. |
3270 | * @fw_sw_minor_ver: the minor of current loaded preboot SW version. |
3271 | * @fw_sw_sub_minor_ver: the sub-minor of current loaded preboot SW version. |
3272 | * @dram_used_mem: current DRAM memory consumption. |
3273 | * @memory_scrub_val: the value to which the dram will be scrubbed to using cb scrub_device_dram |
3274 | * @timeout_jiffies: device CS timeout value. |
3275 | * @max_power: the max power of the device, as configured by the sysadmin. This |
3276 | * value is saved so in case of hard-reset, the driver will restore |
3277 | * this value and update the F/W after the re-initialization |
3278 | * @boot_error_status_mask: contains a mask of the device boot error status. |
3279 | * Each bit represents a different error, according to |
3280 | * the defines in hl_boot_if.h. If the bit is cleared, |
3281 | * the error will be ignored by the driver during |
3282 | * device initialization. Mainly used to debug and |
3283 | * workaround firmware bugs |
3284 | * @dram_pci_bar_start: start bus address of PCIe bar towards DRAM. |
3285 | * @last_successful_open_ktime: timestamp (ktime) of the last successful device open. |
3286 | * @last_successful_open_jif: timestamp (jiffies) of the last successful |
3287 | * device open. |
3288 | * @last_open_session_duration_jif: duration (jiffies) of the last device open |
3289 | * session. |
3290 | * @open_counter: number of successful device open operations. |
3291 | * @fw_poll_interval_usec: FW status poll interval in usec. |
3292 | * used for CPU boot status |
3293 | * @fw_comms_poll_interval_usec: FW comms/protocol poll interval in usec. |
3294 | * used for COMMs protocols cmds(COMMS_STS_*) |
3295 | * @dram_binning: contains mask of drams that is received from the f/w which indicates which |
3296 | * drams are binned-out |
3297 | * @tpc_binning: contains mask of tpc engines that is received from the f/w which indicates which |
3298 | * tpc engines are binned-out |
3299 | * @dmabuf_export_cnt: number of dma-buf exporting. |
3300 | * @card_type: Various ASICs have several card types. This indicates the card |
3301 | * type of the current device. |
3302 | * @major: habanalabs kernel driver major. |
3303 | * @high_pll: high PLL profile frequency. |
3304 | * @decoder_binning: contains mask of decoder engines that is received from the f/w which |
3305 | * indicates which decoder engines are binned-out |
3306 | * @edma_binning: contains mask of edma engines that is received from the f/w which |
3307 | * indicates which edma engines are binned-out |
3308 | * @device_release_watchdog_timeout_sec: device release watchdog timeout value in seconds. |
3309 | * @rotator_binning: contains mask of rotators engines that is received from the f/w |
3310 | * which indicates which rotator engines are binned-out(Gaudi3 and above). |
3311 | * @id: device minor. |
3312 | * @cdev_idx: char device index. |
3313 | * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit |
3314 | * addresses. |
3315 | * @is_in_dram_scrub: true if dram scrub operation is on going. |
3316 | * @disabled: is device disabled. |
3317 | * @late_init_done: is late init stage was done during initialization. |
3318 | * @hwmon_initialized: is H/W monitor sensors was initialized. |
3319 | * @reset_on_lockup: true if a reset should be done in case of stuck CS, false |
3320 | * otherwise. |
3321 | * @dram_default_page_mapping: is DRAM default page mapping enabled. |
3322 | * @memory_scrub: true to perform device memory scrub in various locations, |
3323 | * such as context-switch, context close, page free, etc. |
3324 | * @pmmu_huge_range: is a different virtual addresses range used for PMMU with |
3325 | * huge pages. |
3326 | * @init_done: is the initialization of the device done. |
3327 | * @device_cpu_disabled: is the device CPU disabled (due to timeouts) |
3328 | * @in_debug: whether the device is in a state where the profiling/tracing infrastructure |
3329 | * can be used. This indication is needed because in some ASICs we need to do |
3330 | * specific operations to enable that infrastructure. |
3331 | * @cdev_sysfs_debugfs_created: were char devices and sysfs/debugfs files created. |
3332 | * @stop_on_err: true if engines should stop on error. |
3333 | * @supports_sync_stream: is sync stream supported. |
3334 | * @sync_stream_queue_idx: helper index for sync stream queues initialization. |
3335 | * @collective_mon_idx: helper index for collective initialization |
3336 | * @supports_coresight: is CoreSight supported. |
3337 | * @supports_cb_mapping: is mapping a CB to the device's MMU supported. |
3338 | * @process_kill_trial_cnt: number of trials reset thread tried killing |
3339 | * user processes |
3340 | * @device_fini_pending: true if device_fini was called and might be |
3341 | * waiting for the reset thread to finish |
3342 | * @supports_staged_submission: true if staged submissions are supported |
3343 | * @device_cpu_is_halted: Flag to indicate whether the device CPU was already |
3344 | * halted. We can't halt it again because the COMMS |
3345 | * protocol will throw an error. Relevant only for |
3346 | * cases where Linux was not loaded to device CPU |
3347 | * @supports_wait_for_multi_cs: true if wait for multi CS is supported |
3348 | * @is_compute_ctx_active: Whether there is an active compute context executing. |
3349 | * @compute_ctx_in_release: true if the current compute context is being released. |
3350 | * @supports_mmu_prefetch: true if prefetch is supported, otherwise false. |
3351 | * @reset_upon_device_release: reset the device when the user closes the file descriptor of the |
3352 | * device. |
3353 | * @supports_ctx_switch: true if a ctx switch is required upon first submission. |
3354 | * @support_preboot_binning: true if we support read binning info from preboot. |
3355 | * @eq_heartbeat_received: indication that eq heartbeat event has received from FW. |
3356 | * @nic_ports_mask: Controls which NIC ports are enabled. Used only for testing. |
3357 | * @fw_components: Controls which f/w components to load to the device. There are multiple f/w |
3358 | * stages and sometimes we want to stop at a certain stage. Used only for testing. |
3359 | * @mmu_disable: Disable the device MMU(s). Used only for testing. |
3360 | * @cpu_queues_enable: Whether to enable queues communication vs. the f/w. Used only for testing. |
3361 | * @pldm: Whether we are running in Palladium environment. Used only for testing. |
3362 | * @hard_reset_on_fw_events: Whether to do device hard-reset when a fatal event is received from |
3363 | * the f/w. Used only for testing. |
3364 | * @bmc_enable: Whether we are running in a box with BMC. Used only for testing. |
3365 | * @reset_on_preboot_fail: Whether to reset the device if preboot f/w fails to load. |
3366 | * Used only for testing. |
3367 | * @heartbeat: Controls if we want to enable the heartbeat mechanism vs. the f/w, which verifies |
3368 | * that the f/w is always alive. Used only for testing. |
3369 | */ |
3370 | struct hl_device { |
3371 | struct pci_dev *pdev; |
3372 | u64 pcie_bar_phys[HL_PCI_NUM_BARS]; |
3373 | void __iomem *pcie_bar[HL_PCI_NUM_BARS]; |
3374 | void __iomem *rmmio; |
3375 | struct drm_device drm; |
3376 | struct cdev cdev_ctrl; |
3377 | struct device *dev; |
3378 | struct device *dev_ctrl; |
3379 | struct delayed_work work_heartbeat; |
3380 | struct hl_device_reset_work device_reset_work; |
3381 | struct hl_device_reset_work device_release_watchdog_work; |
3382 | char asic_name[HL_STR_MAX]; |
3383 | char status[HL_DEV_STS_MAX][HL_STR_MAX]; |
3384 | enum hl_asic_type asic_type; |
3385 | struct hl_cq *completion_queue; |
3386 | struct hl_user_interrupt *user_interrupt; |
3387 | struct hl_user_interrupt tpc_interrupt; |
3388 | struct hl_user_interrupt unexpected_error_interrupt; |
3389 | struct hl_user_interrupt common_user_cq_interrupt; |
3390 | struct hl_user_interrupt common_decoder_interrupt; |
3391 | struct hl_cs **shadow_cs_queue; |
3392 | struct workqueue_struct **cq_wq; |
3393 | struct workqueue_struct *eq_wq; |
3394 | struct workqueue_struct *cs_cmplt_wq; |
3395 | struct workqueue_struct *ts_free_obj_wq; |
3396 | struct workqueue_struct *prefetch_wq; |
3397 | struct workqueue_struct *reset_wq; |
3398 | struct hl_ctx *kernel_ctx; |
3399 | struct hl_hw_queue *kernel_queues; |
3400 | struct list_head cs_mirror_list; |
3401 | spinlock_t cs_mirror_lock; |
3402 | struct hl_mem_mgr kernel_mem_mgr; |
3403 | struct hl_eq event_queue; |
3404 | struct dma_pool *dma_pool; |
3405 | void *cpu_accessible_dma_mem; |
3406 | dma_addr_t cpu_accessible_dma_address; |
3407 | struct gen_pool *cpu_accessible_dma_pool; |
3408 | unsigned long *asid_bitmap; |
3409 | struct mutex asid_mutex; |
3410 | struct mutex send_cpu_message_lock; |
3411 | struct mutex debug_lock; |
3412 | struct mutex mmu_lock; |
3413 | struct asic_fixed_properties asic_prop; |
3414 | const struct hl_asic_funcs *asic_funcs; |
3415 | void *asic_specific; |
3416 | struct hl_vm vm; |
3417 | struct device *hwmon_dev; |
3418 | struct hwmon_chip_info *hl_chip_info; |
3419 | |
3420 | struct hl_dbg_device_entry hl_debugfs; |
3421 | |
3422 | struct list_head cb_pool; |
3423 | spinlock_t cb_pool_lock; |
3424 | |
3425 | void *internal_cb_pool_virt_addr; |
3426 | dma_addr_t internal_cb_pool_dma_addr; |
3427 | struct gen_pool *internal_cb_pool; |
3428 | u64 internal_cb_va_base; |
3429 | |
3430 | struct list_head fpriv_list; |
3431 | struct list_head fpriv_ctrl_list; |
3432 | struct mutex fpriv_list_lock; |
3433 | struct mutex fpriv_ctrl_list_lock; |
3434 | |
3435 | struct hl_cs_counters_atomic aggregated_cs_counters; |
3436 | |
3437 | struct hl_mmu_priv mmu_priv; |
3438 | struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS]; |
3439 | |
3440 | struct hl_dec *dec; |
3441 | |
3442 | struct fw_load_mgr fw_loader; |
3443 | |
3444 | struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER]; |
3445 | |
3446 | struct hl_state_dump_specs state_dump_specs; |
3447 | |
3448 | struct multi_cs_completion multi_cs_completion[ |
3449 | MULTI_CS_MAX_USER_CTX]; |
3450 | struct hl_clk_throttle clk_throttling; |
3451 | struct hl_error_info captured_err_info; |
3452 | |
3453 | struct hl_reset_info reset_info; |
3454 | |
3455 | cpumask_t irq_affinity_mask; |
3456 | |
3457 | u32 *stream_master_qid_arr; |
3458 | u32 fw_inner_major_ver; |
3459 | u32 fw_inner_minor_ver; |
3460 | u32 fw_sw_major_ver; |
3461 | u32 fw_sw_minor_ver; |
3462 | u32 fw_sw_sub_minor_ver; |
3463 | atomic64_t dram_used_mem; |
3464 | u64 memory_scrub_val; |
3465 | u64 timeout_jiffies; |
3466 | u64 max_power; |
3467 | u64 boot_error_status_mask; |
3468 | u64 dram_pci_bar_start; |
3469 | u64 last_successful_open_jif; |
3470 | u64 last_open_session_duration_jif; |
3471 | u64 open_counter; |
3472 | u64 fw_poll_interval_usec; |
3473 | ktime_t last_successful_open_ktime; |
3474 | u64 fw_comms_poll_interval_usec; |
3475 | u64 dram_binning; |
3476 | u64 tpc_binning; |
3477 | atomic_t dmabuf_export_cnt; |
3478 | enum cpucp_card_types card_type; |
3479 | u32 major; |
3480 | u32 high_pll; |
3481 | u32 decoder_binning; |
3482 | u32 edma_binning; |
3483 | u32 device_release_watchdog_timeout_sec; |
3484 | u32 rotator_binning; |
3485 | u16 id; |
3486 | u16 cdev_idx; |
3487 | u16 cpu_pci_msb_addr; |
3488 | u8 is_in_dram_scrub; |
3489 | u8 disabled; |
3490 | u8 late_init_done; |
3491 | u8 hwmon_initialized; |
3492 | u8 reset_on_lockup; |
3493 | u8 dram_default_page_mapping; |
3494 | u8 memory_scrub; |
3495 | u8 pmmu_huge_range; |
3496 | u8 init_done; |
3497 | u8 device_cpu_disabled; |
3498 | u8 in_debug; |
3499 | u8 cdev_sysfs_debugfs_created; |
3500 | u8 stop_on_err; |
3501 | u8 supports_sync_stream; |
3502 | u8 sync_stream_queue_idx; |
3503 | u8 collective_mon_idx; |
3504 | u8 supports_coresight; |
3505 | u8 supports_cb_mapping; |
3506 | u8 process_kill_trial_cnt; |
3507 | u8 device_fini_pending; |
3508 | u8 supports_staged_submission; |
3509 | u8 device_cpu_is_halted; |
3510 | u8 supports_wait_for_multi_cs; |
3511 | u8 stream_master_qid_arr_size; |
3512 | u8 is_compute_ctx_active; |
3513 | u8 compute_ctx_in_release; |
3514 | u8 supports_mmu_prefetch; |
3515 | u8 reset_upon_device_release; |
3516 | u8 supports_ctx_switch; |
3517 | u8 support_preboot_binning; |
3518 | u8 eq_heartbeat_received; |
3519 | |
3520 | /* Parameters for bring-up to be upstreamed */ |
3521 | u64 nic_ports_mask; |
3522 | u64 fw_components; |
3523 | u8 mmu_disable; |
3524 | u8 cpu_queues_enable; |
3525 | u8 pldm; |
3526 | u8 hard_reset_on_fw_events; |
3527 | u8 bmc_enable; |
3528 | u8 reset_on_preboot_fail; |
3529 | u8 heartbeat; |
3530 | }; |
3531 | |
3532 | /* Retrieve PCI device name in case of a PCI device or dev name in simulator */ |
3533 | #define HL_DEV_NAME(hdev) \ |
3534 | ((hdev)->pdev ? dev_name(&(hdev)->pdev->dev) : "NA-DEVICE") |
3535 | |
3536 | /** |
3537 | * struct hl_cs_encaps_sig_handle - encapsulated signals handle structure |
3538 | * @refcount: refcount used to protect removing this id when several |
3539 | * wait cs are used to wait of the reserved encaps signals. |
3540 | * @hdev: pointer to habanalabs device structure. |
3541 | * @hw_sob: pointer to H/W SOB used in the reservation. |
3542 | * @ctx: pointer to the user's context data structure |
3543 | * @cs_seq: staged cs sequence which contains encapsulated signals |
3544 | * @id: idr handler id to be used to fetch the handler info |
3545 | * @q_idx: stream queue index |
3546 | * @pre_sob_val: current SOB value before reservation |
3547 | * @count: signals number |
3548 | */ |
3549 | struct hl_cs_encaps_sig_handle { |
3550 | struct kref refcount; |
3551 | struct hl_device *hdev; |
3552 | struct hl_hw_sob *hw_sob; |
3553 | struct hl_ctx *ctx; |
3554 | u64 cs_seq; |
3555 | u32 id; |
3556 | u32 q_idx; |
3557 | u32 pre_sob_val; |
3558 | u32 count; |
3559 | }; |
3560 | |
3561 | /** |
3562 | * struct hl_info_fw_err_info - firmware error information structure |
3563 | * @err_type: The type of error detected (or reported). |
3564 | * @event_mask: Pointer to the event mask to be modified with the detected error flag |
3565 | * (can be NULL) |
3566 | * @event_id: The id of the event that reported the error |
3567 | * (applicable when err_type is HL_INFO_FW_REPORTED_ERR). |
3568 | */ |
3569 | struct hl_info_fw_err_info { |
3570 | enum hl_info_fw_err_type err_type; |
3571 | u64 *event_mask; |
3572 | u16 event_id; |
3573 | }; |
3574 | |
3575 | /* |
3576 | * IOCTLs |
3577 | */ |
3578 | |
3579 | /** |
3580 | * typedef hl_ioctl_t - typedef for ioctl function in the driver |
3581 | * @hpriv: pointer to the FD's private data, which contains state of |
3582 | * user process |
3583 | * @data: pointer to the input/output arguments structure of the IOCTL |
3584 | * |
3585 | * Return: 0 for success, negative value for error |
3586 | */ |
3587 | typedef int hl_ioctl_t(struct hl_fpriv *hpriv, void *data); |
3588 | |
3589 | /** |
3590 | * struct hl_ioctl_desc - describes an IOCTL entry of the driver. |
3591 | * @cmd: the IOCTL code as created by the kernel macros. |
3592 | * @func: pointer to the driver's function that should be called for this IOCTL. |
3593 | */ |
3594 | struct hl_ioctl_desc { |
3595 | unsigned int cmd; |
3596 | hl_ioctl_t *func; |
3597 | }; |
3598 | |
3599 | static inline bool hl_is_fw_sw_ver_below(struct hl_device *hdev, u32 fw_sw_major, u32 fw_sw_minor) |
3600 | { |
3601 | if (hdev->fw_sw_major_ver < fw_sw_major) |
3602 | return true; |
3603 | if (hdev->fw_sw_major_ver > fw_sw_major) |
3604 | return false; |
3605 | if (hdev->fw_sw_minor_ver < fw_sw_minor) |
3606 | return true; |
3607 | return false; |
3608 | } |
3609 | |
3610 | static inline bool hl_is_fw_sw_ver_equal_or_greater(struct hl_device *hdev, u32 fw_sw_major, |
3611 | u32 fw_sw_minor) |
3612 | { |
3613 | return (hdev->fw_sw_major_ver > fw_sw_major || |
3614 | (hdev->fw_sw_major_ver == fw_sw_major && |
3615 | hdev->fw_sw_minor_ver >= fw_sw_minor)); |
3616 | } |
3617 | |
3618 | /* |
3619 | * Kernel module functions that can be accessed by entire module |
3620 | */ |
3621 | |
3622 | /** |
3623 | * hl_get_sg_info() - get number of pages and the DMA address from SG list. |
3624 | * @sg: the SG list. |
3625 | * @dma_addr: pointer to DMA address to return. |
3626 | * |
3627 | * Calculate the number of consecutive pages described by the SG list. Take the |
3628 | * offset of the address in the first page, add to it the length and round it up |
3629 | * to the number of needed pages. |
3630 | */ |
3631 | static inline u32 hl_get_sg_info(struct scatterlist *sg, dma_addr_t *dma_addr) |
3632 | { |
3633 | *dma_addr = sg_dma_address(sg); |
3634 | |
3635 | return ((((*dma_addr) & (PAGE_SIZE - 1)) + sg_dma_len(sg)) + |
3636 | (PAGE_SIZE - 1)) >> PAGE_SHIFT; |
3637 | } |
3638 | |
3639 | /** |
3640 | * hl_mem_area_inside_range() - Checks whether address+size are inside a range. |
3641 | * @address: The start address of the area we want to validate. |
3642 | * @size: The size in bytes of the area we want to validate. |
3643 | * @range_start_address: The start address of the valid range. |
3644 | * @range_end_address: The end address of the valid range. |
3645 | * |
3646 | * Return: true if the area is inside the valid range, false otherwise. |
3647 | */ |
3648 | static inline bool hl_mem_area_inside_range(u64 address, u64 size, |
3649 | u64 range_start_address, u64 range_end_address) |
3650 | { |
3651 | u64 end_address = address + size; |
3652 | |
3653 | if ((address >= range_start_address) && |
3654 | (end_address <= range_end_address) && |
3655 | (end_address > address)) |
3656 | return true; |
3657 | |
3658 | return false; |
3659 | } |
3660 | |
3661 | static inline struct hl_device *to_hl_device(struct drm_device *ddev) |
3662 | { |
3663 | return container_of(ddev, struct hl_device, drm); |
3664 | } |
3665 | |
3666 | /** |
3667 | * hl_mem_area_crosses_range() - Checks whether address+size crossing a range. |
3668 | * @address: The start address of the area we want to validate. |
3669 | * @size: The size in bytes of the area we want to validate. |
3670 | * @range_start_address: The start address of the valid range. |
3671 | * @range_end_address: The end address of the valid range. |
3672 | * |
3673 | * Return: true if the area overlaps part or all of the valid range, |
3674 | * false otherwise. |
3675 | */ |
3676 | static inline bool hl_mem_area_crosses_range(u64 address, u32 size, |
3677 | u64 range_start_address, u64 range_end_address) |
3678 | { |
3679 | u64 end_address = address + size - 1; |
3680 | |
3681 | return ((address <= range_end_address) && (range_start_address <= end_address)); |
3682 | } |
3683 | |
3684 | uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr); |
3685 | void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle); |
3686 | void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); |
3687 | void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, |
3688 | gfp_t flag, const char *caller); |
3689 | void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, |
3690 | dma_addr_t dma_handle, const char *caller); |
3691 | void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, |
3692 | dma_addr_t *dma_handle, const char *caller); |
3693 | void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, |
3694 | const char *caller); |
3695 | int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt, |
3696 | enum dma_data_direction dir, const char *caller); |
3697 | void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt, |
3698 | enum dma_data_direction dir, const char *caller); |
3699 | int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, |
3700 | enum dma_data_direction dir); |
3701 | void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, |
3702 | enum dma_data_direction dir); |
3703 | int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, |
3704 | enum debugfs_access_type acc_type, enum pci_region region_type, bool set_dram_bar); |
3705 | int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val, |
3706 | enum debugfs_access_type acc_type); |
3707 | int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, |
3708 | u64 addr, u64 *val, enum debugfs_access_type acc_type); |
3709 | |
3710 | int hl_mmap(struct file *filp, struct vm_area_struct *vma); |
3711 | |
3712 | int hl_device_open(struct drm_device *drm, struct drm_file *file_priv); |
3713 | void hl_device_release(struct drm_device *ddev, struct drm_file *file_priv); |
3714 | |
3715 | int hl_device_open_ctrl(struct inode *inode, struct file *filp); |
3716 | bool hl_device_operational(struct hl_device *hdev, |
3717 | enum hl_device_status *status); |
3718 | bool hl_ctrl_device_operational(struct hl_device *hdev, |
3719 | enum hl_device_status *status); |
3720 | enum hl_device_status hl_device_status(struct hl_device *hdev); |
3721 | int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable); |
3722 | int hl_hw_queues_create(struct hl_device *hdev); |
3723 | void hl_hw_queues_destroy(struct hl_device *hdev); |
3724 | int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id, |
3725 | u32 cb_size, u64 cb_ptr); |
3726 | void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q, |
3727 | u32 ctl, u32 len, u64 ptr); |
3728 | int hl_hw_queue_schedule_cs(struct hl_cs *cs); |
3729 | u32 hl_hw_queue_add_ptr(u32 ptr, u16 val); |
3730 | void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id); |
3731 | void hl_hw_queue_update_ci(struct hl_cs *cs); |
3732 | void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset); |
3733 | |
3734 | #define hl_queue_inc_ptr(p) hl_hw_queue_add_ptr(p, 1) |
3735 | #define hl_pi_2_offset(pi) ((pi) & (HL_QUEUE_LENGTH - 1)) |
3736 | |
3737 | int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id); |
3738 | void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q); |
3739 | int hl_eq_init(struct hl_device *hdev, struct hl_eq *q); |
3740 | void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q); |
3741 | void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q); |
3742 | void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q); |
3743 | irqreturn_t hl_irq_handler_cq(int irq, void *arg); |
3744 | irqreturn_t hl_irq_handler_eq(int irq, void *arg); |
3745 | irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg); |
3746 | irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg); |
3747 | irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg); |
3748 | irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg); |
3749 | u32 hl_cq_inc_ptr(u32 ptr); |
3750 | |
3751 | int hl_asid_init(struct hl_device *hdev); |
3752 | void hl_asid_fini(struct hl_device *hdev); |
3753 | unsigned long hl_asid_alloc(struct hl_device *hdev); |
3754 | void hl_asid_free(struct hl_device *hdev, unsigned long asid); |
3755 | |
3756 | int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv); |
3757 | void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx); |
3758 | int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx); |
3759 | void hl_ctx_do_release(struct kref *ref); |
3760 | void hl_ctx_get(struct hl_ctx *ctx); |
3761 | int hl_ctx_put(struct hl_ctx *ctx); |
3762 | struct hl_ctx *hl_get_compute_ctx(struct hl_device *hdev); |
3763 | struct hl_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq); |
3764 | int hl_ctx_get_fences(struct hl_ctx *ctx, u64 *seq_arr, |
3765 | struct hl_fence **fence, u32 arr_len); |
3766 | void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr); |
3767 | void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr); |
3768 | |
3769 | int hl_device_init(struct hl_device *hdev); |
3770 | void hl_device_fini(struct hl_device *hdev); |
3771 | int hl_device_suspend(struct hl_device *hdev); |
3772 | int hl_device_resume(struct hl_device *hdev); |
3773 | int hl_device_reset(struct hl_device *hdev, u32 flags); |
3774 | int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask); |
3775 | void hl_hpriv_get(struct hl_fpriv *hpriv); |
3776 | int hl_hpriv_put(struct hl_fpriv *hpriv); |
3777 | int hl_device_utilization(struct hl_device *hdev, u32 *utilization); |
3778 | |
3779 | int hl_build_hwmon_channel_info(struct hl_device *hdev, |
3780 | struct cpucp_sensor *sensors_arr); |
3781 | |
3782 | void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask); |
3783 | |
3784 | int hl_sysfs_init(struct hl_device *hdev); |
3785 | void hl_sysfs_fini(struct hl_device *hdev); |
3786 | |
3787 | int hl_hwmon_init(struct hl_device *hdev); |
3788 | void hl_hwmon_fini(struct hl_device *hdev); |
3789 | void hl_hwmon_release_resources(struct hl_device *hdev); |
3790 | |
3791 | int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, |
3792 | struct hl_ctx *ctx, u32 cb_size, bool internal_cb, |
3793 | bool map_cb, u64 *handle); |
3794 | int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle); |
3795 | int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); |
3796 | struct hl_cb *hl_cb_get(struct hl_mem_mgr *mmg, u64 handle); |
3797 | void hl_cb_put(struct hl_cb *cb); |
3798 | struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size, |
3799 | bool internal_cb); |
3800 | int hl_cb_pool_init(struct hl_device *hdev); |
3801 | int hl_cb_pool_fini(struct hl_device *hdev); |
3802 | int hl_cb_va_pool_init(struct hl_ctx *ctx); |
3803 | void hl_cb_va_pool_fini(struct hl_ctx *ctx); |
3804 | |
3805 | void hl_cs_rollback_all(struct hl_device *hdev, bool skip_wq_flush); |
3806 | struct hl_cs_job *hl_cs_allocate_job(struct hl_device *hdev, |
3807 | enum hl_queue_type queue_type, bool is_kernel_allocated_cb); |
3808 | void hl_sob_reset_error(struct kref *ref); |
3809 | int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask); |
3810 | void hl_fence_put(struct hl_fence *fence); |
3811 | void hl_fences_put(struct hl_fence **fence, int len); |
3812 | void hl_fence_get(struct hl_fence *fence); |
3813 | void cs_get(struct hl_cs *cs); |
3814 | bool cs_needs_completion(struct hl_cs *cs); |
3815 | bool cs_needs_timeout(struct hl_cs *cs); |
3816 | bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs); |
3817 | struct hl_cs *hl_staged_cs_find_first(struct hl_device *hdev, u64 cs_seq); |
3818 | void hl_multi_cs_completion_init(struct hl_device *hdev); |
3819 | u32 hl_get_active_cs_num(struct hl_device *hdev); |
3820 | |
3821 | void goya_set_asic_funcs(struct hl_device *hdev); |
3822 | void gaudi_set_asic_funcs(struct hl_device *hdev); |
3823 | void gaudi2_set_asic_funcs(struct hl_device *hdev); |
3824 | |
3825 | int hl_vm_ctx_init(struct hl_ctx *ctx); |
3826 | void hl_vm_ctx_fini(struct hl_ctx *ctx); |
3827 | |
3828 | int hl_vm_init(struct hl_device *hdev); |
3829 | void hl_vm_fini(struct hl_device *hdev); |
3830 | |
3831 | void hl_hw_block_mem_init(struct hl_ctx *ctx); |
3832 | void hl_hw_block_mem_fini(struct hl_ctx *ctx); |
3833 | |
3834 | u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, |
3835 | enum hl_va_range_type type, u64 size, u32 alignment); |
3836 | int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, |
3837 | u64 start_addr, u64 size); |
3838 | int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, |
3839 | struct hl_userptr *userptr); |
3840 | void hl_unpin_host_memory(struct hl_device *hdev, struct hl_userptr *userptr); |
3841 | void hl_userptr_delete_list(struct hl_device *hdev, |
3842 | struct list_head *userptr_list); |
3843 | bool hl_userptr_is_pinned(struct hl_device *hdev, u64 addr, u32 size, |
3844 | struct list_head *userptr_list, |
3845 | struct hl_userptr **userptr); |
3846 | |
3847 | int hl_mmu_init(struct hl_device *hdev); |
3848 | void hl_mmu_fini(struct hl_device *hdev); |
3849 | int hl_mmu_ctx_init(struct hl_ctx *ctx); |
3850 | void hl_mmu_ctx_fini(struct hl_ctx *ctx); |
3851 | int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, |
3852 | u32 page_size, bool flush_pte); |
3853 | int hl_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, |
3854 | u32 page_size, u32 *real_page_size, bool is_dram_addr); |
3855 | int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, |
3856 | bool flush_pte); |
3857 | int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr, |
3858 | u64 phys_addr, u32 size); |
3859 | int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size); |
3860 | int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags); |
3861 | int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard, |
3862 | u32 flags, u32 asid, u64 va, u64 size); |
3863 | int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); |
3864 | u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte); |
3865 | u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop, |
3866 | u8 hop_idx, u64 hop_addr, u64 virt_addr); |
3867 | void hl_mmu_hr_flush(struct hl_ctx *ctx); |
3868 | int hl_mmu_hr_init(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size, |
3869 | u64 pgt_size); |
3870 | void hl_mmu_hr_fini(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size); |
3871 | void hl_mmu_hr_free_hop_remove_pgt(struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv, |
3872 | u32 hop_table_size); |
3873 | u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, u64 phys_pte_addr, |
3874 | u32 hop_table_size); |
3875 | void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, |
3876 | u64 val, u32 hop_table_size); |
3877 | void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, |
3878 | u32 hop_table_size); |
3879 | int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv, |
3880 | u32 hop_table_size); |
3881 | void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr); |
3882 | struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx, |
3883 | struct hl_hr_mmu_funcs *hr_func, |
3884 | u64 curr_pte); |
3885 | struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv, |
3886 | struct hl_hr_mmu_funcs *hr_func, |
3887 | struct hl_mmu_properties *mmu_prop); |
3888 | struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx, |
3889 | struct hl_mmu_hr_priv *hr_priv, |
3890 | struct hl_hr_mmu_funcs *hr_func, |
3891 | struct hl_mmu_properties *mmu_prop, |
3892 | u64 curr_pte, bool *is_new_hop); |
3893 | int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops, |
3894 | struct hl_hr_mmu_funcs *hr_func); |
3895 | int hl_mmu_if_set_funcs(struct hl_device *hdev); |
3896 | void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); |
3897 | void hl_mmu_v2_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); |
3898 | void hl_mmu_v2_hr_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu); |
3899 | int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr); |
3900 | int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, |
3901 | struct hl_mmu_hop_info *hops); |
3902 | u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr); |
3903 | u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr); |
3904 | bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr); |
3905 | struct pgt_info *hl_mmu_dr_get_pgt_info(struct hl_ctx *ctx, u64 hop_addr); |
3906 | void hl_mmu_dr_free_hop(struct hl_ctx *ctx, u64 hop_addr); |
3907 | void hl_mmu_dr_free_pgt_node(struct hl_ctx *ctx, struct pgt_info *pgt_info); |
3908 | u64 hl_mmu_dr_get_phys_hop0_addr(struct hl_ctx *ctx); |
3909 | u64 hl_mmu_dr_get_hop0_addr(struct hl_ctx *ctx); |
3910 | void hl_mmu_dr_write_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val); |
3911 | void hl_mmu_dr_write_final_pte(struct hl_ctx *ctx, u64 shadow_pte_addr, u64 val); |
3912 | void hl_mmu_dr_clear_pte(struct hl_ctx *ctx, u64 pte_addr); |
3913 | u64 hl_mmu_dr_get_phys_addr(struct hl_ctx *ctx, u64 shadow_addr); |
3914 | void hl_mmu_dr_get_pte(struct hl_ctx *ctx, u64 hop_addr); |
3915 | int hl_mmu_dr_put_pte(struct hl_ctx *ctx, u64 hop_addr); |
3916 | u64 hl_mmu_dr_get_alloc_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte, bool *is_new_hop); |
3917 | u64 hl_mmu_dr_alloc_hop(struct hl_ctx *ctx); |
3918 | void hl_mmu_dr_flush(struct hl_ctx *ctx); |
3919 | int hl_mmu_dr_init(struct hl_device *hdev); |
3920 | void hl_mmu_dr_fini(struct hl_device *hdev); |
3921 | |
3922 | int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name, |
3923 | void __iomem *dst, u32 src_offset, u32 size); |
3924 | int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value); |
3925 | int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, |
3926 | u16 len, u32 timeout, u64 *result); |
3927 | int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type); |
3928 | int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr, |
3929 | size_t irq_arr_size); |
3930 | int hl_fw_test_cpu_queue(struct hl_device *hdev); |
3931 | void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, |
3932 | dma_addr_t *dma_handle); |
3933 | void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, |
3934 | void *vaddr); |
3935 | int hl_fw_send_heartbeat(struct hl_device *hdev); |
3936 | int hl_fw_cpucp_info_get(struct hl_device *hdev, |
3937 | u32 sts_boot_dev_sts0_reg, |
3938 | u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, |
3939 | u32 boot_err1_reg); |
3940 | int hl_fw_cpucp_handshake(struct hl_device *hdev, |
3941 | u32 sts_boot_dev_sts0_reg, |
3942 | u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg, |
3943 | u32 boot_err1_reg); |
3944 | int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); |
3945 | int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data); |
3946 | int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, |
3947 | struct hl_info_pci_counters *counters); |
3948 | int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, |
3949 | u64 *total_energy); |
3950 | int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index, |
3951 | enum pll_index *pll_index); |
3952 | int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index, |
3953 | u16 *pll_freq_arr); |
3954 | int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power); |
3955 | void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev); |
3956 | void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev); |
3957 | int hl_fw_init_cpu(struct hl_device *hdev); |
3958 | int hl_fw_wait_preboot_ready(struct hl_device *hdev); |
3959 | int hl_fw_read_preboot_status(struct hl_device *hdev); |
3960 | int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, |
3961 | struct fw_load_mgr *fw_loader, |
3962 | enum comms_cmd cmd, unsigned int size, |
3963 | bool wait_ok, u32 timeout); |
3964 | int hl_fw_dram_replaced_row_get(struct hl_device *hdev, |
3965 | struct cpucp_hbm_row_info *info); |
3966 | int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num); |
3967 | int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid); |
3968 | int hl_fw_send_device_activity(struct hl_device *hdev, bool open); |
3969 | int hl_fw_send_soft_reset(struct hl_device *hdev); |
3970 | int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3], |
3971 | bool is_wc[3]); |
3972 | int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data); |
3973 | int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data); |
3974 | int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region, |
3975 | struct hl_inbound_pci_region *pci_region); |
3976 | int hl_pci_set_outbound_region(struct hl_device *hdev, |
3977 | struct hl_outbound_pci_region *pci_region); |
3978 | enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr); |
3979 | int hl_pci_init(struct hl_device *hdev); |
3980 | void hl_pci_fini(struct hl_device *hdev); |
3981 | |
3982 | long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr); |
3983 | void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq); |
3984 | int hl_get_temperature(struct hl_device *hdev, int sensor_index, u32 attr, long *value); |
3985 | int hl_set_temperature(struct hl_device *hdev, int sensor_index, u32 attr, long value); |
3986 | int hl_get_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long *value); |
3987 | int hl_get_current(struct hl_device *hdev, int sensor_index, u32 attr, long *value); |
3988 | int hl_get_fan_speed(struct hl_device *hdev, int sensor_index, u32 attr, long *value); |
3989 | int hl_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long *value); |
3990 | void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long value); |
3991 | long hl_fw_get_max_power(struct hl_device *hdev); |
3992 | void hl_fw_set_max_power(struct hl_device *hdev); |
3993 | int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info, |
3994 | u32 nonce); |
3995 | int hl_fw_get_dev_info_signed(struct hl_device *hdev, |
3996 | struct cpucp_dev_info_signed *dev_info_signed, u32 nonce); |
3997 | int hl_set_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long value); |
3998 | int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value); |
3999 | int hl_set_power(struct hl_device *hdev, int sensor_index, u32 attr, long value); |
4000 | int hl_get_power(struct hl_device *hdev, int sensor_index, u32 attr, long *value); |
4001 | int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk); |
4002 | void hl_fw_set_pll_profile(struct hl_device *hdev); |
4003 | void hl_sysfs_add_dev_clk_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp); |
4004 | void hl_sysfs_add_dev_vrm_attr(struct hl_device *hdev, struct attribute_group *dev_vrm_attr_grp); |
4005 | int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode, |
4006 | dma_addr_t buff, u32 *size); |
4007 | |
4008 | void hw_sob_get(struct hl_hw_sob *hw_sob); |
4009 | void hw_sob_put(struct hl_hw_sob *hw_sob); |
4010 | void hl_encaps_release_handle_and_put_ctx(struct kref *ref); |
4011 | void hl_encaps_release_handle_and_put_sob_ctx(struct kref *ref); |
4012 | void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev, |
4013 | struct hl_cs *cs, struct hl_cs_job *job, |
4014 | struct hl_cs_compl *cs_cmpl); |
4015 | |
4016 | int hl_dec_init(struct hl_device *hdev); |
4017 | void hl_dec_fini(struct hl_device *hdev); |
4018 | void hl_dec_ctx_fini(struct hl_ctx *ctx); |
4019 | |
4020 | void hl_release_pending_user_interrupts(struct hl_device *hdev); |
4021 | void hl_abort_waiting_for_cs_completions(struct hl_device *hdev); |
4022 | int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx, |
4023 | struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig); |
4024 | |
4025 | int hl_state_dump(struct hl_device *hdev); |
4026 | const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id); |
4027 | const char *hl_state_dump_get_monitor_name(struct hl_device *hdev, |
4028 | struct hl_mon_state_dump *mon); |
4029 | void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map); |
4030 | __printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset, |
4031 | const char *format, ...); |
4032 | char *hl_format_as_binary(char *buf, size_t buf_len, u32 n); |
4033 | const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type); |
4034 | |
4035 | void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg); |
4036 | void hl_mem_mgr_fini(struct hl_mem_mgr *mmg); |
4037 | void hl_mem_mgr_idr_destroy(struct hl_mem_mgr *mmg); |
4038 | int hl_mem_mgr_mmap(struct hl_mem_mgr *mmg, struct vm_area_struct *vma, |
4039 | void *args); |
4040 | struct hl_mmap_mem_buf *hl_mmap_mem_buf_get(struct hl_mem_mgr *mmg, |
4041 | u64 handle); |
4042 | int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle); |
4043 | int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf); |
4044 | struct hl_mmap_mem_buf * |
4045 | hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg, |
4046 | struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp, |
4047 | void *args); |
4048 | __printf(2, 3) void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...); |
4049 | void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, |
4050 | u8 flags); |
4051 | void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines, |
4052 | u8 flags, u64 *event_mask); |
4053 | void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu); |
4054 | void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu, |
4055 | u64 *event_mask); |
4056 | void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask); |
4057 | void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info); |
4058 | void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count); |
4059 | void hl_enable_err_info_capture(struct hl_error_info *captured_err_info); |
4060 | void hl_init_cpu_for_irq(struct hl_device *hdev); |
4061 | void hl_set_irq_affinity(struct hl_device *hdev, int irq); |
4062 | |
4063 | #ifdef CONFIG_DEBUG_FS |
4064 | |
4065 | int hl_debugfs_device_init(struct hl_device *hdev); |
4066 | void hl_debugfs_device_fini(struct hl_device *hdev); |
4067 | void hl_debugfs_add_device(struct hl_device *hdev); |
4068 | void hl_debugfs_add_file(struct hl_fpriv *hpriv); |
4069 | void hl_debugfs_remove_file(struct hl_fpriv *hpriv); |
4070 | void hl_debugfs_add_cb(struct hl_cb *cb); |
4071 | void hl_debugfs_remove_cb(struct hl_cb *cb); |
4072 | void hl_debugfs_add_cs(struct hl_cs *cs); |
4073 | void hl_debugfs_remove_cs(struct hl_cs *cs); |
4074 | void hl_debugfs_add_job(struct hl_device *hdev, struct hl_cs_job *job); |
4075 | void hl_debugfs_remove_job(struct hl_device *hdev, struct hl_cs_job *job); |
4076 | void hl_debugfs_add_userptr(struct hl_device *hdev, struct hl_userptr *userptr); |
4077 | void hl_debugfs_remove_userptr(struct hl_device *hdev, |
4078 | struct hl_userptr *userptr); |
4079 | void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx); |
4080 | void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, struct hl_ctx *ctx); |
4081 | void hl_debugfs_set_state_dump(struct hl_device *hdev, char *data, |
4082 | unsigned long length); |
4083 | |
4084 | #else |
4085 | |
4086 | static inline int hl_debugfs_device_init(struct hl_device *hdev) |
4087 | { |
4088 | return 0; |
4089 | } |
4090 | |
4091 | static inline void hl_debugfs_device_fini(struct hl_device *hdev) |
4092 | { |
4093 | } |
4094 | |
4095 | static inline void hl_debugfs_add_device(struct hl_device *hdev) |
4096 | { |
4097 | } |
4098 | |
4099 | static inline void hl_debugfs_add_file(struct hl_fpriv *hpriv) |
4100 | { |
4101 | } |
4102 | |
4103 | static inline void hl_debugfs_remove_file(struct hl_fpriv *hpriv) |
4104 | { |
4105 | } |
4106 | |
4107 | static inline void hl_debugfs_add_cb(struct hl_cb *cb) |
4108 | { |
4109 | } |
4110 | |
4111 | static inline void hl_debugfs_remove_cb(struct hl_cb *cb) |
4112 | { |
4113 | } |
4114 | |
4115 | static inline void hl_debugfs_add_cs(struct hl_cs *cs) |
4116 | { |
4117 | } |
4118 | |
4119 | static inline void hl_debugfs_remove_cs(struct hl_cs *cs) |
4120 | { |
4121 | } |
4122 | |
4123 | static inline void hl_debugfs_add_job(struct hl_device *hdev, |
4124 | struct hl_cs_job *job) |
4125 | { |
4126 | } |
4127 | |
4128 | static inline void hl_debugfs_remove_job(struct hl_device *hdev, |
4129 | struct hl_cs_job *job) |
4130 | { |
4131 | } |
4132 | |
4133 | static inline void hl_debugfs_add_userptr(struct hl_device *hdev, |
4134 | struct hl_userptr *userptr) |
4135 | { |
4136 | } |
4137 | |
4138 | static inline void hl_debugfs_remove_userptr(struct hl_device *hdev, |
4139 | struct hl_userptr *userptr) |
4140 | { |
4141 | } |
4142 | |
4143 | static inline void hl_debugfs_add_ctx_mem_hash(struct hl_device *hdev, |
4144 | struct hl_ctx *ctx) |
4145 | { |
4146 | } |
4147 | |
4148 | static inline void hl_debugfs_remove_ctx_mem_hash(struct hl_device *hdev, |
4149 | struct hl_ctx *ctx) |
4150 | { |
4151 | } |
4152 | |
4153 | static inline void hl_debugfs_set_state_dump(struct hl_device *hdev, |
4154 | char *data, unsigned long length) |
4155 | { |
4156 | } |
4157 | |
4158 | #endif |
4159 | |
4160 | /* Security */ |
4161 | int hl_unsecure_register(struct hl_device *hdev, u32 mm_reg_addr, int offset, |
4162 | const u32 pb_blocks[], struct hl_block_glbl_sec sgs_array[], |
4163 | int array_size); |
4164 | int hl_unsecure_registers(struct hl_device *hdev, const u32 mm_reg_array[], |
4165 | int mm_array_size, int offset, const u32 pb_blocks[], |
4166 | struct hl_block_glbl_sec sgs_array[], int blocks_array_size); |
4167 | void hl_config_glbl_sec(struct hl_device *hdev, const u32 pb_blocks[], |
4168 | struct hl_block_glbl_sec sgs_array[], u32 block_offset, |
4169 | int array_size); |
4170 | void hl_secure_block(struct hl_device *hdev, |
4171 | struct hl_block_glbl_sec sgs_array[], int array_size); |
4172 | int hl_init_pb_with_mask(struct hl_device *hdev, u32 num_dcores, |
4173 | u32 dcore_offset, u32 num_instances, u32 instance_offset, |
4174 | const u32 pb_blocks[], u32 blocks_array_size, |
4175 | const u32 *regs_array, u32 regs_array_size, u64 mask); |
4176 | int hl_init_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset, |
4177 | u32 num_instances, u32 instance_offset, |
4178 | const u32 pb_blocks[], u32 blocks_array_size, |
4179 | const u32 *regs_array, u32 regs_array_size); |
4180 | int hl_init_pb_ranges_with_mask(struct hl_device *hdev, u32 num_dcores, |
4181 | u32 dcore_offset, u32 num_instances, u32 instance_offset, |
4182 | const u32 pb_blocks[], u32 blocks_array_size, |
4183 | const struct range *regs_range_array, u32 regs_range_array_size, |
4184 | u64 mask); |
4185 | int hl_init_pb_ranges(struct hl_device *hdev, u32 num_dcores, |
4186 | u32 dcore_offset, u32 num_instances, u32 instance_offset, |
4187 | const u32 pb_blocks[], u32 blocks_array_size, |
4188 | const struct range *regs_range_array, |
4189 | u32 regs_range_array_size); |
4190 | int hl_init_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset, |
4191 | u32 num_instances, u32 instance_offset, |
4192 | const u32 pb_blocks[], u32 blocks_array_size, |
4193 | const u32 *regs_array, u32 regs_array_size); |
4194 | int hl_init_pb_ranges_single_dcore(struct hl_device *hdev, u32 dcore_offset, |
4195 | u32 num_instances, u32 instance_offset, |
4196 | const u32 pb_blocks[], u32 blocks_array_size, |
4197 | const struct range *regs_range_array, |
4198 | u32 regs_range_array_size); |
4199 | void hl_ack_pb(struct hl_device *hdev, u32 num_dcores, u32 dcore_offset, |
4200 | u32 num_instances, u32 instance_offset, |
4201 | const u32 pb_blocks[], u32 blocks_array_size); |
4202 | void hl_ack_pb_with_mask(struct hl_device *hdev, u32 num_dcores, |
4203 | u32 dcore_offset, u32 num_instances, u32 instance_offset, |
4204 | const u32 pb_blocks[], u32 blocks_array_size, u64 mask); |
4205 | void hl_ack_pb_single_dcore(struct hl_device *hdev, u32 dcore_offset, |
4206 | u32 num_instances, u32 instance_offset, |
4207 | const u32 pb_blocks[], u32 blocks_array_size); |
4208 | |
4209 | /* IOCTLs */ |
4210 | long hl_ioctl_control(struct file *filep, unsigned int cmd, unsigned long arg); |
4211 | int hl_info_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); |
4212 | int hl_cb_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); |
4213 | int hl_cs_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); |
4214 | int hl_wait_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); |
4215 | int hl_mem_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); |
4216 | int hl_debug_ioctl(struct drm_device *ddev, void *data, struct drm_file *file_priv); |
4217 | |
4218 | #endif /* HABANALABSP_H_ */ |
4219 | |