1 | // SPDX-License-Identifier: MIT |
2 | /* |
3 | * Copyright 2022 Advanced Micro Devices, Inc. |
4 | * |
5 | * Permission is hereby granted, free of charge, to any person obtaining a |
6 | * copy of this software and associated documentation files (the "Software"), |
7 | * to deal in the Software without restriction, including without limitation |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
9 | * and/or sell copies of the Software, and to permit persons to whom the |
10 | * Software is furnished to do so, subject to the following conditions: |
11 | * |
12 | * The above copyright notice and this permission notice shall be included in |
13 | * all copies or substantial portions of the Software. |
14 | * |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
21 | * OTHER DEALINGS IN THE SOFTWARE. |
22 | * |
23 | */ |
24 | |
25 | #include "amdgpu.h" |
26 | |
27 | /** |
28 | * amdgpu_mm_rdoorbell - read a doorbell dword |
29 | * |
30 | * @adev: amdgpu_device pointer |
31 | * @index: doorbell index |
32 | * |
33 | * Returns the value in the doorbell aperture at the |
34 | * requested doorbell index (CIK). |
35 | */ |
36 | u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) |
37 | { |
38 | if (amdgpu_device_skip_hw_access(adev)) |
39 | return 0; |
40 | |
41 | if (index < adev->doorbell.num_kernel_doorbells) |
42 | return readl(addr: adev->doorbell.cpu_addr + index); |
43 | |
44 | DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n" , index); |
45 | return 0; |
46 | } |
47 | |
48 | /** |
49 | * amdgpu_mm_wdoorbell - write a doorbell dword |
50 | * |
51 | * @adev: amdgpu_device pointer |
52 | * @index: doorbell index |
53 | * @v: value to write |
54 | * |
55 | * Writes @v to the doorbell aperture at the |
56 | * requested doorbell index (CIK). |
57 | */ |
58 | void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) |
59 | { |
60 | if (amdgpu_device_skip_hw_access(adev)) |
61 | return; |
62 | |
63 | if (index < adev->doorbell.num_kernel_doorbells) |
64 | writel(val: v, addr: adev->doorbell.cpu_addr + index); |
65 | else |
66 | DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n" , index); |
67 | } |
68 | |
69 | /** |
70 | * amdgpu_mm_rdoorbell64 - read a doorbell Qword |
71 | * |
72 | * @adev: amdgpu_device pointer |
73 | * @index: doorbell index |
74 | * |
75 | * Returns the value in the doorbell aperture at the |
76 | * requested doorbell index (VEGA10+). |
77 | */ |
78 | u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) |
79 | { |
80 | if (amdgpu_device_skip_hw_access(adev)) |
81 | return 0; |
82 | |
83 | if (index < adev->doorbell.num_kernel_doorbells) |
84 | return atomic64_read(v: (atomic64_t *)(adev->doorbell.cpu_addr + index)); |
85 | |
86 | DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n" , index); |
87 | return 0; |
88 | } |
89 | |
90 | /** |
91 | * amdgpu_mm_wdoorbell64 - write a doorbell Qword |
92 | * |
93 | * @adev: amdgpu_device pointer |
94 | * @index: doorbell index |
95 | * @v: value to write |
96 | * |
97 | * Writes @v to the doorbell aperture at the |
98 | * requested doorbell index (VEGA10+). |
99 | */ |
100 | void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) |
101 | { |
102 | if (amdgpu_device_skip_hw_access(adev)) |
103 | return; |
104 | |
105 | if (index < adev->doorbell.num_kernel_doorbells) |
106 | atomic64_set(v: (atomic64_t *)(adev->doorbell.cpu_addr + index), i: v); |
107 | else |
108 | DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n" , index); |
109 | } |
110 | |
111 | /** |
112 | * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR |
113 | * |
114 | * @adev: amdgpu_device pointer |
115 | * @db_bo: doorbell object's bo |
116 | * @doorbell_index: doorbell relative index in this doorbell object |
117 | * @db_size: doorbell size is in byte |
118 | * |
119 | * returns doorbell's absolute index in BAR |
120 | */ |
121 | uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev, |
122 | struct amdgpu_bo *db_bo, |
123 | uint32_t doorbell_index, |
124 | uint32_t db_size) |
125 | { |
126 | int db_bo_offset; |
127 | |
128 | db_bo_offset = amdgpu_bo_gpu_offset_no_check(bo: db_bo); |
129 | |
130 | /* doorbell index is 32 bit but doorbell's size can be 32 bit |
131 | * or 64 bit, so *db_size(in byte)/4 for alignment. |
132 | */ |
133 | return db_bo_offset / sizeof(u32) + doorbell_index * |
134 | DIV_ROUND_UP(db_size, 4); |
135 | } |
136 | |
137 | /** |
138 | * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics |
139 | * |
140 | * @adev: amdgpu_device pointer |
141 | * |
142 | * Creates doorbells for graphics driver usages. |
143 | * returns 0 on success, error otherwise. |
144 | */ |
145 | int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev) |
146 | { |
147 | int r; |
148 | int size; |
149 | |
150 | /* SI HW does not have doorbells, skip allocation */ |
151 | if (adev->doorbell.num_kernel_doorbells == 0) |
152 | return 0; |
153 | |
154 | /* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */ |
155 | size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE); |
156 | |
157 | /* Allocate an extra page for MES kernel usages (ring test) */ |
158 | adev->mes.db_start_dw_offset = size / sizeof(u32); |
159 | size += PAGE_SIZE; |
160 | |
161 | r = amdgpu_bo_create_kernel(adev, |
162 | size, |
163 | PAGE_SIZE, |
164 | AMDGPU_GEM_DOMAIN_DOORBELL, |
165 | bo_ptr: &adev->doorbell.kernel_doorbells, |
166 | NULL, |
167 | cpu_addr: (void **)&adev->doorbell.cpu_addr); |
168 | if (r) { |
169 | DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n" , r); |
170 | return r; |
171 | } |
172 | |
173 | adev->doorbell.num_kernel_doorbells = size / sizeof(u32); |
174 | return 0; |
175 | } |
176 | |
177 | /* |
178 | * GPU doorbell aperture helpers function. |
179 | */ |
180 | /** |
181 | * amdgpu_doorbell_init - Init doorbell driver information. |
182 | * |
183 | * @adev: amdgpu_device pointer |
184 | * |
185 | * Init doorbell driver information (CIK) |
186 | * Returns 0 on success, error on failure. |
187 | */ |
188 | int amdgpu_doorbell_init(struct amdgpu_device *adev) |
189 | { |
190 | |
191 | /* No doorbell on SI hardware generation */ |
192 | if (adev->asic_type < CHIP_BONAIRE) { |
193 | adev->doorbell.base = 0; |
194 | adev->doorbell.size = 0; |
195 | adev->doorbell.num_kernel_doorbells = 0; |
196 | return 0; |
197 | } |
198 | |
199 | if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) |
200 | return -EINVAL; |
201 | |
202 | amdgpu_asic_init_doorbell_index(adev); |
203 | |
204 | /* doorbell bar mapping */ |
205 | adev->doorbell.base = pci_resource_start(adev->pdev, 2); |
206 | adev->doorbell.size = pci_resource_len(adev->pdev, 2); |
207 | |
208 | adev->doorbell.num_kernel_doorbells = |
209 | min_t(u32, adev->doorbell.size / sizeof(u32), |
210 | adev->doorbell_index.max_assignment + 1); |
211 | if (adev->doorbell.num_kernel_doorbells == 0) |
212 | return -EINVAL; |
213 | |
214 | /* |
215 | * For Vega, reserve and map two pages on doorbell BAR since SDMA |
216 | * paging queue doorbell use the second page. The |
217 | * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the |
218 | * doorbells are in the first page. So with paging queue enabled, |
219 | * the max num_kernel_doorbells should + 1 page (0x400 in dword) |
220 | */ |
221 | if (adev->asic_type >= CHIP_VEGA10) |
222 | adev->doorbell.num_kernel_doorbells += 0x400; |
223 | |
224 | return 0; |
225 | } |
226 | |
227 | /** |
228 | * amdgpu_doorbell_fini - Tear down doorbell driver information. |
229 | * |
230 | * @adev: amdgpu_device pointer |
231 | * |
232 | * Tear down doorbell driver information (CIK) |
233 | */ |
234 | void amdgpu_doorbell_fini(struct amdgpu_device *adev) |
235 | { |
236 | amdgpu_bo_free_kernel(bo: &adev->doorbell.kernel_doorbells, |
237 | NULL, |
238 | cpu_addr: (void **)&adev->doorbell.cpu_addr); |
239 | } |
240 | |