1	/*
2	* Copyright 2016 Advanced Micro Devices, Inc.
3	*
4	* Permission is hereby granted, free of charge, to any person obtaining a
5	* copy of this software and associated documentation files (the "Software"),
6	* to deal in the Software without restriction, including without limitation
7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8	* and/or sell copies of the Software, and to permit persons to whom the
9	* Software is furnished to do so, subject to the following conditions:
10	*
11	* The above copyright notice and this permission notice shall be included in
12	* all copies or substantial portions of the Software.
13	*
14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20	* OTHER DEALINGS IN THE SOFTWARE.
21	*
22	*/
23
24	#include <linux/module.h>
25
26	#ifdef CONFIG_X86
27	#include <asm/hypervisor.h>
28	#endif
29
30	#include <drm/drm_drv.h>
31	#include <xen/xen.h>
32
33	#include "amdgpu.h"
34	#include "amdgpu_ras.h"
35	#include "amdgpu_reset.h"
36	#include "amdgpu_dpm.h"
37	#include "vi.h"
38	#include "soc15.h"
39	#include "nv.h"
40
41	#define POPULATE_UCODE_INFO(vf2pf_info, ucode, ver) \
42	do { \
43	vf2pf_info->ucode_info[ucode].id = ucode; \
44	vf2pf_info->ucode_info[ucode].version = ver; \
45	} while (0)
46
47	bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev)
48	{
49	/* By now all MMIO pages except mailbox are blocked */
50	/* if blocking is enabled in hypervisor. Choose the */
51	/* SCRATCH_REG0 to test. */
52	return RREG32_NO_KIQ(0xc040) == 0xffffffff;
53	}
54
55	void amdgpu_virt_init_setting(struct amdgpu_device *adev)
56	{
57	struct drm_device *ddev = adev_to_drm(adev);
58
59	/* enable virtual display */
60	if (adev->asic_type != CHIP_ALDEBARAN &&
61	adev->asic_type != CHIP_ARCTURUS &&
62	((adev->pdev->class >> 8) != PCI_CLASS_ACCELERATOR_PROCESSING)) {
63	if (adev->mode_info.num_crtc == 0)
64	adev->mode_info.num_crtc = 1;
65	adev->enable_virtual_display = true;
66	}
67	ddev->driver_features &= ~DRIVER_ATOMIC;
68	adev->cg_flags = 0;
69	adev->pg_flags = 0;
70
71	/* Reduce kcq number to 2 to reduce latency */
72	if (amdgpu_num_kcq == -1)
73	amdgpu_num_kcq = 2;
74	}
75
76	/**
77	* amdgpu_virt_request_full_gpu() - request full gpu access
78	* @adev: amdgpu device.
79	* @init: is driver init time.
80	* When start to init/fini driver, first need to request full gpu access.
81	* Return: Zero if request success, otherwise will return error.
82	*/
83	int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init)
84	{
85	struct amdgpu_virt *virt = &adev->virt;
86	int r;
87
88	if (virt->ops && virt->ops->req_full_gpu) {
89	r = virt->ops->req_full_gpu(adev, init);
90	if (r) {
91	adev->no_hw_access = true;
92	return r;
93	}
94
95	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
96	}
97
98	return 0;
99	}
100
101	/**
102	* amdgpu_virt_release_full_gpu() - release full gpu access
103	* @adev: amdgpu device.
104	* @init: is driver init time.
105	* When finishing driver init/fini, need to release full gpu access.
106	* Return: Zero if release success, otherwise will returen error.
107	*/
108	int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init)
109	{
110	struct amdgpu_virt *virt = &adev->virt;
111	int r;
112
113	if (virt->ops && virt->ops->rel_full_gpu) {
114	r = virt->ops->rel_full_gpu(adev, init);
115	if (r)
116	return r;
117
118	adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
119	}
120	return 0;
121	}
122
123	/**
124	* amdgpu_virt_reset_gpu() - reset gpu
125	* @adev: amdgpu device.
126	* Send reset command to GPU hypervisor to reset GPU that VM is using
127	* Return: Zero if reset success, otherwise will return error.
128	*/
129	int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
130	{
131	struct amdgpu_virt *virt = &adev->virt;
132	int r;
133
134	if (virt->ops && virt->ops->reset_gpu) {
135	r = virt->ops->reset_gpu(adev);
136	if (r)
137	return r;
138
139	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
140	}
141
142	return 0;
143	}
144
145	void amdgpu_virt_request_init_data(struct amdgpu_device *adev)
146	{
147	struct amdgpu_virt *virt = &adev->virt;
148
149	if (virt->ops && virt->ops->req_init_data)
150	virt->ops->req_init_data(adev);
151
152	if (adev->virt.req_init_data_ver > 0)
153	DRM_INFO("host supports REQ_INIT_DATA handshake\n");
154	else
155	DRM_WARN("host doesn't support REQ_INIT_DATA handshake\n");
156	}
157
158	/**
159	* amdgpu_virt_ready_to_reset() - send ready to reset to host
160	* @adev: amdgpu device.
161	* Send ready to reset message to GPU hypervisor to signal we have stopped GPU
162	* activity and is ready for host FLR
163	*/
164	void amdgpu_virt_ready_to_reset(struct amdgpu_device *adev)
165	{
166	struct amdgpu_virt *virt = &adev->virt;
167
168	if (virt->ops && virt->ops->reset_gpu)
169	virt->ops->ready_to_reset(adev);
170	}
171
172	/**
173	* amdgpu_virt_wait_reset() - wait for reset gpu completed
174	* @adev: amdgpu device.
175	* Wait for GPU reset completed.
176	* Return: Zero if reset success, otherwise will return error.
177	*/
178	int amdgpu_virt_wait_reset(struct amdgpu_device *adev)
179	{
180	struct amdgpu_virt *virt = &adev->virt;
181
182	if (!virt->ops || !virt->ops->wait_reset)
183	return -EINVAL;
184
185	return virt->ops->wait_reset(adev);
186	}
187
188	/**
189	* amdgpu_virt_alloc_mm_table() - alloc memory for mm table
190	* @adev: amdgpu device.
191	* MM table is used by UVD and VCE for its initialization
192	* Return: Zero if allocate success.
193	*/
194	int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev)
195	{
196	int r;
197
198	if (!amdgpu_sriov_vf(adev) || adev->virt.mm_table.gpu_addr)
199	return 0;
200
201	r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
202	AMDGPU_GEM_DOMAIN_VRAM |
203	AMDGPU_GEM_DOMAIN_GTT,
204	bo_ptr: &adev->virt.mm_table.bo,
205	gpu_addr: &adev->virt.mm_table.gpu_addr,
206	cpu_addr: (void *)&adev->virt.mm_table.cpu_addr);
207	if (r) {
208	DRM_ERROR("failed to alloc mm table and error = %d.\n", r);
209	return r;
210	}
211
212	memset((void *)adev->virt.mm_table.cpu_addr, 0, PAGE_SIZE);
213	DRM_INFO("MM table gpu addr = 0x%llx, cpu addr = %p.\n",
214	adev->virt.mm_table.gpu_addr,
215	adev->virt.mm_table.cpu_addr);
216	return 0;
217	}
218
219	/**
220	* amdgpu_virt_free_mm_table() - free mm table memory
221	* @adev: amdgpu device.
222	* Free MM table memory
223	*/
224	void amdgpu_virt_free_mm_table(struct amdgpu_device *adev)
225	{
226	if (!amdgpu_sriov_vf(adev) || !adev->virt.mm_table.gpu_addr)
227	return;
228
229	amdgpu_bo_free_kernel(bo: &adev->virt.mm_table.bo,
230	gpu_addr: &adev->virt.mm_table.gpu_addr,
231	cpu_addr: (void *)&adev->virt.mm_table.cpu_addr);
232	adev->virt.mm_table.gpu_addr = 0;
233	}
234
235	/**
236	* amdgpu_virt_rcvd_ras_interrupt() - receive ras interrupt
237	* @adev: amdgpu device.
238	* Check whether host sent RAS error message
239	* Return: true if found, otherwise false
240	*/
241	bool amdgpu_virt_rcvd_ras_interrupt(struct amdgpu_device *adev)
242	{
243	struct amdgpu_virt *virt = &adev->virt;
244
245	if (!virt->ops || !virt->ops->rcvd_ras_intr)
246	return false;
247
248	return virt->ops->rcvd_ras_intr(adev);
249	}
250
251
252	unsigned int amd_sriov_msg_checksum(void *obj,
253	unsigned long obj_size,
254	unsigned int key,
255	unsigned int checksum)
256	{
257	unsigned int ret = key;
258	unsigned long i = 0;
259	unsigned char *pos;
260
261	pos = (char *)obj;
262	/* calculate checksum */
263	for (i = 0; i < obj_size; ++i)
264	ret += *(pos + i);
265	/* minus the checksum itself */
266	pos = (char *)&checksum;
267	for (i = 0; i < sizeof(checksum); ++i)
268	ret -= *(pos + i);
269	return ret;
270	}
271
272	static int amdgpu_virt_init_ras_err_handler_data(struct amdgpu_device *adev)
273	{
274	struct amdgpu_virt *virt = &adev->virt;
275	struct amdgpu_virt_ras_err_handler_data **data = &virt->virt_eh_data;
276	/* GPU will be marked bad on host if bp count more then 10,
277	* so alloc 512 is enough.
278	*/
279	unsigned int align_space = 512;
280	void *bps = NULL;
281	struct amdgpu_bo **bps_bo = NULL;
282
283	*data = kmalloc(sizeof(struct amdgpu_virt_ras_err_handler_data), GFP_KERNEL);
284	if (!*data)
285	goto data_failure;
286
287	bps = kmalloc_array(align_space, sizeof(*(*data)->bps), GFP_KERNEL);
288	if (!bps)
289	goto bps_failure;
290
291	bps_bo = kmalloc_array(align_space, sizeof(*(*data)->bps_bo), GFP_KERNEL);
292	if (!bps_bo)
293	goto bps_bo_failure;
294
295	(*data)->bps = bps;
296	(*data)->bps_bo = bps_bo;
297	(*data)->count = 0;
298	(*data)->last_reserved = 0;
299
300	virt->ras_init_done = true;
301
302	return 0;
303
304	bps_bo_failure:
305	kfree(objp: bps);
306	bps_failure:
307	kfree(objp: *data);
308	data_failure:
309	return -ENOMEM;
310	}
311
312	static void amdgpu_virt_ras_release_bp(struct amdgpu_device *adev)
313	{
314	struct amdgpu_virt *virt = &adev->virt;
315	struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
316	struct amdgpu_bo *bo;
317	int i;
318
319	if (!data)
320	return;
321
322	for (i = data->last_reserved - 1; i >= 0; i--) {
323	bo = data->bps_bo[i];
324	if (bo) {
325	amdgpu_bo_free_kernel(bo: &bo, NULL, NULL);
326	data->bps_bo[i] = bo;
327	}
328	data->last_reserved = i;
329	}
330	}
331
332	void amdgpu_virt_release_ras_err_handler_data(struct amdgpu_device *adev)
333	{
334	struct amdgpu_virt *virt = &adev->virt;
335	struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
336
337	virt->ras_init_done = false;
338
339	if (!data)
340	return;
341
342	amdgpu_virt_ras_release_bp(adev);
343
344	kfree(objp: data->bps);
345	kfree(objp: data->bps_bo);
346	kfree(objp: data);
347	virt->virt_eh_data = NULL;
348	}
349
350	static void amdgpu_virt_ras_add_bps(struct amdgpu_device *adev,
351	struct eeprom_table_record *bps, int pages)
352	{
353	struct amdgpu_virt *virt = &adev->virt;
354	struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
355
356	if (!data)
357	return;
358
359	memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps));
360	data->count += pages;
361	}
362
363	static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev)
364	{
365	struct amdgpu_virt *virt = &adev->virt;
366	struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
367	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
368	struct ttm_resource_manager *man = &mgr->manager;
369	struct amdgpu_bo *bo = NULL;
370	uint64_t bp;
371	int i;
372
373	if (!data)
374	return;
375
376	for (i = data->last_reserved; i < data->count; i++) {
377	bp = data->bps[i].retired_page;
378
379	/* There are two cases of reserve error should be ignored:
380	* 1) a ras bad page has been allocated (used by someone);
381	* 2) a ras bad page has been reserved (duplicate error injection
382	* for one page);
383	*/
384	if (ttm_resource_manager_used(man)) {
385	amdgpu_vram_mgr_reserve_range(mgr: &adev->mman.vram_mgr,
386	start: bp << AMDGPU_GPU_PAGE_SHIFT,
387	AMDGPU_GPU_PAGE_SIZE);
388	data->bps_bo[i] = NULL;
389	} else {
390	if (amdgpu_bo_create_kernel_at(adev, offset: bp << AMDGPU_GPU_PAGE_SHIFT,
391	AMDGPU_GPU_PAGE_SIZE,
392	bo_ptr: &bo, NULL))
393	DRM_DEBUG("RAS WARN: reserve vram for retired page %llx fail\n", bp);
394	data->bps_bo[i] = bo;
395	}
396	data->last_reserved = i + 1;
397	bo = NULL;
398	}
399	}
400
401	static bool amdgpu_virt_ras_check_bad_page(struct amdgpu_device *adev,
402	uint64_t retired_page)
403	{
404	struct amdgpu_virt *virt = &adev->virt;
405	struct amdgpu_virt_ras_err_handler_data *data = virt->virt_eh_data;
406	int i;
407
408	if (!data)
409	return true;
410
411	for (i = 0; i < data->count; i++)
412	if (retired_page == data->bps[i].retired_page)
413	return true;
414
415	return false;
416	}
417
418	static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev,
419	uint64_t bp_block_offset, uint32_t bp_block_size)
420	{
421	struct eeprom_table_record bp;
422	uint64_t retired_page;
423	uint32_t bp_idx, bp_cnt;
424	void *vram_usage_va = NULL;
425
426	if (adev->mman.fw_vram_usage_va)
427	vram_usage_va = adev->mman.fw_vram_usage_va;
428	else
429	vram_usage_va = adev->mman.drv_vram_usage_va;
430
431	memset(&bp, 0, sizeof(bp));
432
433	if (bp_block_size) {
434	bp_cnt = bp_block_size / sizeof(uint64_t);
435	for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) {
436	retired_page = *(uint64_t *)(vram_usage_va +
437	bp_block_offset + bp_idx * sizeof(uint64_t));
438	bp.retired_page = retired_page;
439
440	if (amdgpu_virt_ras_check_bad_page(adev, retired_page))
441	continue;
442
443	amdgpu_virt_ras_add_bps(adev, bps: &bp, pages: 1);
444
445	amdgpu_virt_ras_reserve_bps(adev);
446	}
447	}
448	}
449
450	static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
451	{
452	struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf;
453	uint32_t checksum;
454	uint32_t checkval;
455
456	uint32_t i;
457	uint32_t tmp;
458
459	if (adev->virt.fw_reserve.p_pf2vf == NULL)
460	return -EINVAL;
461
462	if (pf2vf_info->size > 1024) {
463	dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size);
464	return -EINVAL;
465	}
466
467	switch (pf2vf_info->version) {
468	case 1:
469	checksum = ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->checksum;
470	checkval = amd_sriov_msg_checksum(
471	obj: adev->virt.fw_reserve.p_pf2vf, obj_size: pf2vf_info->size,
472	key: adev->virt.fw_reserve.checksum_key, checksum);
473	if (checksum != checkval) {
474	dev_err(adev->dev,
475	"invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
476	checksum, checkval);
477	return -EINVAL;
478	}
479
480	adev->virt.gim_feature =
481	((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->feature_flags;
482	break;
483	case 2:
484	/* TODO: missing key, need to add it later */
485	checksum = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->checksum;
486	checkval = amd_sriov_msg_checksum(
487	obj: adev->virt.fw_reserve.p_pf2vf, obj_size: pf2vf_info->size,
488	key: 0, checksum);
489	if (checksum != checkval) {
490	dev_err(adev->dev,
491	"invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
492	checksum, checkval);
493	return -EINVAL;
494	}
495
496	adev->virt.vf2pf_update_interval_ms =
497	((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms;
498	adev->virt.gim_feature =
499	((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all;
500	adev->virt.reg_access =
501	((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all;
502
503	adev->virt.decode_max_dimension_pixels = 0;
504	adev->virt.decode_max_frame_pixels = 0;
505	adev->virt.encode_max_dimension_pixels = 0;
506	adev->virt.encode_max_frame_pixels = 0;
507	adev->virt.is_mm_bw_enabled = false;
508	for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) {
509	tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_dimension_pixels;
510	adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels);
511
512	tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_frame_pixels;
513	adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels);
514
515	tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_dimension_pixels;
516	adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels);
517
518	tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_frame_pixels;
519	adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels);
520	}
521	if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0))
522	adev->virt.is_mm_bw_enabled = true;
523
524	adev->unique_id =
525	((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid;
526	adev->virt.ras_en_caps.all = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_en_caps.all;
527	adev->virt.ras_telemetry_en_caps.all =
528	((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_telemetry_en_caps.all;
529	break;
530	default:
531	dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version);
532	return -EINVAL;
533	}
534
535	/* correct too large or too little interval value */
536	if (adev->virt.vf2pf_update_interval_ms < 200 || adev->virt.vf2pf_update_interval_ms > 10000)
537	adev->virt.vf2pf_update_interval_ms = 2000;
538
539	return 0;
540	}
541
542	static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev)
543	{
544	struct amd_sriov_msg_vf2pf_info *vf2pf_info;
545	vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf;
546
547	if (adev->virt.fw_reserve.p_vf2pf == NULL)
548	return;
549
550	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCE, adev->vce.fw_version);
551	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_UVD, adev->uvd.fw_version);
552	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MC, adev->gmc.fw_version);
553	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ME, adev->gfx.me_fw_version);
554	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_PFP, adev->gfx.pfp_fw_version);
555	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_CE, adev->gfx.ce_fw_version);
556	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC, adev->gfx.rlc_fw_version);
557	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLC, adev->gfx.rlc_srlc_fw_version);
558	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLG, adev->gfx.rlc_srlg_fw_version);
559	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version);
560	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version);
561	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version);
562	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version);
563	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD,
564	adev->psp.asd_context.bin_desc.fw_version);
565	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_RAS,
566	adev->psp.ras_context.context.bin_desc.fw_version);
567	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_TA_XGMI,
568	adev->psp.xgmi_context.context.bin_desc.fw_version);
569	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SMC, adev->pm.fw_version);
570	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA, adev->sdma.instance[0].fw_version);
571	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SDMA2, adev->sdma.instance[1].fw_version);
572	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_VCN, adev->vcn.fw_version);
573	POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_DMCU, adev->dm.dmcu_fw_version);
574	}
575
576	static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev)
577	{
578	struct amd_sriov_msg_vf2pf_info *vf2pf_info;
579
580	vf2pf_info = (struct amd_sriov_msg_vf2pf_info *) adev->virt.fw_reserve.p_vf2pf;
581
582	if (adev->virt.fw_reserve.p_vf2pf == NULL)
583	return -EINVAL;
584
585	memset(vf2pf_info, 0, sizeof(struct amd_sriov_msg_vf2pf_info));
586
587	vf2pf_info->header.size = sizeof(struct amd_sriov_msg_vf2pf_info);
588	vf2pf_info->header.version = AMD_SRIOV_MSG_FW_VRAM_VF2PF_VER;
589
590	#ifdef MODULE
591	if (THIS_MODULE->version != NULL)
592	strcpy(vf2pf_info->driver_version, THIS_MODULE->version);
593	else
594	#endif
595	strcpy(p: vf2pf_info->driver_version, q: "N/A");
596
597	vf2pf_info->pf2vf_version_required = 0; // no requirement, guest understands all
598	vf2pf_info->driver_cert = 0;
599	vf2pf_info->os_info.all = 0;
600
601	vf2pf_info->fb_usage =
602	ttm_resource_manager_usage(man: &adev->mman.vram_mgr.manager) >> 20;
603	vf2pf_info->fb_vis_usage =
604	amdgpu_vram_mgr_vis_usage(mgr: &adev->mman.vram_mgr) >> 20;
605	vf2pf_info->fb_size = adev->gmc.real_vram_size >> 20;
606	vf2pf_info->fb_vis_size = adev->gmc.visible_vram_size >> 20;
607
608	amdgpu_virt_populate_vf2pf_ucode_info(adev);
609
610	/* TODO: read dynamic info */
611	vf2pf_info->gfx_usage = 0;
612	vf2pf_info->compute_usage = 0;
613	vf2pf_info->encode_usage = 0;
614	vf2pf_info->decode_usage = 0;
615
616	vf2pf_info->dummy_page_addr = (uint64_t)adev->dummy_page_addr;
617	if (amdgpu_sriov_is_mes_info_enable(adev)) {
618	vf2pf_info->mes_info_addr =
619	(uint64_t)(adev->mes.resource_1_gpu_addr[0] + AMDGPU_GPU_PAGE_SIZE);
620	vf2pf_info->mes_info_size =
621	adev->mes.resource_1[0]->tbo.base.size - AMDGPU_GPU_PAGE_SIZE;
622	}
623	vf2pf_info->checksum =
624	amd_sriov_msg_checksum(
625	obj: vf2pf_info, obj_size: sizeof(*vf2pf_info), key: 0, checksum: 0);
626
627	return 0;
628	}
629
630	static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work)
631	{
632	struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work);
633	int ret;
634
635	ret = amdgpu_virt_read_pf2vf_data(adev);
636	if (ret) {
637	adev->virt.vf2pf_update_retry_cnt++;
638
639	if ((amdgpu_virt_rcvd_ras_interrupt(adev) ||
640	adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) &&
641	amdgpu_sriov_runtime(adev)) {
642
643	amdgpu_ras_set_fed(adev, status: true);
644	if (amdgpu_reset_domain_schedule(domain: adev->reset_domain,
645	work: &adev->kfd.reset_work))
646	return;
647	else
648	dev_err(adev->dev, "Failed to queue work! at %s", __func__);
649	}
650
651	goto out;
652	}
653
654	adev->virt.vf2pf_update_retry_cnt = 0;
655	amdgpu_virt_write_vf2pf_data(adev);
656
657	out:
658	schedule_delayed_work(dwork: &(adev->virt.vf2pf_work), delay: adev->virt.vf2pf_update_interval_ms);
659	}
660
661	void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev)
662	{
663	if (adev->virt.vf2pf_update_interval_ms != 0) {
664	DRM_INFO("clean up the vf2pf work item\n");
665	cancel_delayed_work_sync(dwork: &adev->virt.vf2pf_work);
666	adev->virt.vf2pf_update_interval_ms = 0;
667	}
668	}
669
670	void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
671	{
672	adev->virt.fw_reserve.p_pf2vf = NULL;
673	adev->virt.fw_reserve.p_vf2pf = NULL;
674	adev->virt.vf2pf_update_interval_ms = 0;
675	adev->virt.vf2pf_update_retry_cnt = 0;
676
677	if (adev->mman.fw_vram_usage_va && adev->mman.drv_vram_usage_va) {
678	DRM_WARN("Currently fw_vram and drv_vram should not have values at the same time!");
679	} else if (adev->mman.fw_vram_usage_va || adev->mman.drv_vram_usage_va) {
680	/* go through this logic in ip_init and reset to init workqueue*/
681	amdgpu_virt_exchange_data(adev);
682
683	INIT_DELAYED_WORK(&adev->virt.vf2pf_work, amdgpu_virt_update_vf2pf_work_item);
684	schedule_delayed_work(dwork: &(adev->virt.vf2pf_work), delay: msecs_to_jiffies(m: adev->virt.vf2pf_update_interval_ms));
685	} else if (adev->bios != NULL) {
686	/* got through this logic in early init stage to get necessary flags, e.g. rlcg_acc related*/
687	adev->virt.fw_reserve.p_pf2vf =
688	(struct amd_sriov_msg_pf2vf_info_header *)
689	(adev->bios + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
690
691	amdgpu_virt_read_pf2vf_data(adev);
692	}
693	}
694
695
696	void amdgpu_virt_exchange_data(struct amdgpu_device *adev)
697	{
698	uint64_t bp_block_offset = 0;
699	uint32_t bp_block_size = 0;
700	struct amd_sriov_msg_pf2vf_info *pf2vf_v2 = NULL;
701
702	if (adev->mman.fw_vram_usage_va || adev->mman.drv_vram_usage_va) {
703	if (adev->mman.fw_vram_usage_va) {
704	adev->virt.fw_reserve.p_pf2vf =
705	(struct amd_sriov_msg_pf2vf_info_header *)
706	(adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
707	adev->virt.fw_reserve.p_vf2pf =
708	(struct amd_sriov_msg_vf2pf_info_header *)
709	(adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB << 10));
710	adev->virt.fw_reserve.ras_telemetry =
711	(adev->mman.fw_vram_usage_va + (AMD_SRIOV_MSG_RAS_TELEMETRY_OFFSET_KB << 10));
712	} else if (adev->mman.drv_vram_usage_va) {
713	adev->virt.fw_reserve.p_pf2vf =
714	(struct amd_sriov_msg_pf2vf_info_header *)
715	(adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_PF2VF_OFFSET_KB << 10));
716	adev->virt.fw_reserve.p_vf2pf =
717	(struct amd_sriov_msg_vf2pf_info_header *)
718	(adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_VF2PF_OFFSET_KB << 10));
719	adev->virt.fw_reserve.ras_telemetry =
720	(adev->mman.drv_vram_usage_va + (AMD_SRIOV_MSG_RAS_TELEMETRY_OFFSET_KB << 10));
721	}
722
723	amdgpu_virt_read_pf2vf_data(adev);
724	amdgpu_virt_write_vf2pf_data(adev);
725
726	/* bad page handling for version 2 */
727	if (adev->virt.fw_reserve.p_pf2vf->version == 2) {
728	pf2vf_v2 = (struct amd_sriov_msg_pf2vf_info *)adev->virt.fw_reserve.p_pf2vf;
729
730	bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_low & 0xFFFFFFFF) |
731	((((uint64_t)pf2vf_v2->bp_block_offset_high) << 32) & 0xFFFFFFFF00000000);
732	bp_block_size = pf2vf_v2->bp_block_size;
733
734	if (bp_block_size && !adev->virt.ras_init_done)
735	amdgpu_virt_init_ras_err_handler_data(adev);
736
737	if (adev->virt.ras_init_done)
738	amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size);
739	}
740	}
741	}
742
743	static u32 amdgpu_virt_init_detect_asic(struct amdgpu_device *adev)
744	{
745	uint32_t reg;
746
747	switch (adev->asic_type) {
748	case CHIP_TONGA:
749	case CHIP_FIJI:
750	reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER);
751	break;
752	case CHIP_VEGA10:
753	case CHIP_VEGA20:
754	case CHIP_NAVI10:
755	case CHIP_NAVI12:
756	case CHIP_SIENNA_CICHLID:
757	case CHIP_ARCTURUS:
758	case CHIP_ALDEBARAN:
759	case CHIP_IP_DISCOVERY:
760	reg = RREG32(mmRCC_IOV_FUNC_IDENTIFIER);
761	break;
762	default: /* other chip doesn't support SRIOV */
763	reg = 0;
764	break;
765	}
766
767	if (reg & 1)
768	adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF;
769
770	if (reg & 0x80000000)
771	adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
772
773	if (!reg) {
774	/* passthrough mode exclus sriov mod */
775	if (is_virtual_machine() && !xen_initial_domain())
776	adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
777	}
778
779	return reg;
780	}
781
782	static bool amdgpu_virt_init_req_data(struct amdgpu_device *adev, u32 reg)
783	{
784	bool is_sriov = false;
785
786	/* we have the ability to check now */
787	if (amdgpu_sriov_vf(adev)) {
788	is_sriov = true;
789
790	switch (adev->asic_type) {
791	case CHIP_TONGA:
792	case CHIP_FIJI:
793	vi_set_virt_ops(adev);
794	break;
795	case CHIP_VEGA10:
796	soc15_set_virt_ops(adev);
797	#ifdef CONFIG_X86
798	/* not send GPU_INIT_DATA with MS_HYPERV*/
799	if (!hypervisor_is_type(type: X86_HYPER_MS_HYPERV))
800	#endif
801	/* send a dummy GPU_INIT_DATA request to host on vega10 */
802	amdgpu_virt_request_init_data(adev);
803	break;
804	case CHIP_VEGA20:
805	case CHIP_ARCTURUS:
806	case CHIP_ALDEBARAN:
807	soc15_set_virt_ops(adev);
808	break;
809	case CHIP_NAVI10:
810	case CHIP_NAVI12:
811	case CHIP_SIENNA_CICHLID:
812	case CHIP_IP_DISCOVERY:
813	nv_set_virt_ops(adev);
814	/* try send GPU_INIT_DATA request to host */
815	amdgpu_virt_request_init_data(adev);
816	break;
817	default: /* other chip doesn't support SRIOV */
818	is_sriov = false;
819	DRM_ERROR("Unknown asic type: %d!\n", adev->asic_type);
820	break;
821	}
822	}
823
824	return is_sriov;
825	}
826
827	static void amdgpu_virt_init_ras(struct amdgpu_device *adev)
828	{
829	ratelimit_state_init(rs: &adev->virt.ras.ras_error_cnt_rs, interval: 5 * HZ, burst: 1);
830	ratelimit_state_init(rs: &adev->virt.ras.ras_cper_dump_rs, interval: 5 * HZ, burst: 1);
831
832	ratelimit_set_flags(rs: &adev->virt.ras.ras_error_cnt_rs,
833	RATELIMIT_MSG_ON_RELEASE);
834	ratelimit_set_flags(rs: &adev->virt.ras.ras_cper_dump_rs,
835	RATELIMIT_MSG_ON_RELEASE);
836
837	mutex_init(&adev->virt.ras.ras_telemetry_mutex);
838
839	adev->virt.ras.cper_rptr = 0;
840	}
841
842	void amdgpu_virt_init(struct amdgpu_device *adev)
843	{
844	bool is_sriov = false;
845	uint32_t reg = amdgpu_virt_init_detect_asic(adev);
846
847	is_sriov = amdgpu_virt_init_req_data(adev, reg);
848
849	if (is_sriov)
850	amdgpu_virt_init_ras(adev);
851	}
852
853	static bool amdgpu_virt_access_debugfs_is_mmio(struct amdgpu_device *adev)
854	{
855	return amdgpu_sriov_is_debug(adev) ? true : false;
856	}
857
858	static bool amdgpu_virt_access_debugfs_is_kiq(struct amdgpu_device *adev)
859	{
860	return amdgpu_sriov_is_normal(adev) ? true : false;
861	}
862
863	int amdgpu_virt_enable_access_debugfs(struct amdgpu_device *adev)
864	{
865	if (!amdgpu_sriov_vf(adev) ||
866	amdgpu_virt_access_debugfs_is_kiq(adev))
867	return 0;
868
869	if (amdgpu_virt_access_debugfs_is_mmio(adev))
870	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
871	else
872	return -EPERM;
873
874	return 0;
875	}
876
877	void amdgpu_virt_disable_access_debugfs(struct amdgpu_device *adev)
878	{
879	if (amdgpu_sriov_vf(adev))
880	adev->virt.caps |= AMDGPU_SRIOV_CAPS_RUNTIME;
881	}
882
883	enum amdgpu_sriov_vf_mode amdgpu_virt_get_sriov_vf_mode(struct amdgpu_device *adev)
884	{
885	enum amdgpu_sriov_vf_mode mode;
886
887	if (amdgpu_sriov_vf(adev)) {
888	if (amdgpu_sriov_is_pp_one_vf(adev))
889	mode = SRIOV_VF_MODE_ONE_VF;
890	else
891	mode = SRIOV_VF_MODE_MULTI_VF;
892	} else {
893	mode = SRIOV_VF_MODE_BARE_METAL;
894	}
895
896	return mode;
897	}
898
899	void amdgpu_virt_pre_reset(struct amdgpu_device *adev)
900	{
901	/* stop the data exchange thread */
902	amdgpu_virt_fini_data_exchange(adev);
903	amdgpu_dpm_set_mp1_state(adev, mp1_state: PP_MP1_STATE_FLR);
904	}
905
906	void amdgpu_virt_post_reset(struct amdgpu_device *adev)
907	{
908	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(11, 0, 3)) {
909	/* force set to GFXOFF state after reset,
910	* to avoid some invalid operation before GC enable
911	*/
912	adev->gfx.is_poweron = false;
913	}
914
915	adev->mes.ring[0].sched.ready = false;
916	}
917
918	bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_id)
919	{
920	switch (amdgpu_ip_version(adev, ip: MP0_HWIP, inst: 0)) {
921	case IP_VERSION(13, 0, 0):
922	/* no vf autoload, white list */
923	if (ucode_id == AMDGPU_UCODE_ID_VCN1 ||
924	ucode_id == AMDGPU_UCODE_ID_VCN)
925	return false;
926	else
927	return true;
928	case IP_VERSION(11, 0, 9):
929	case IP_VERSION(11, 0, 7):
930	/* black list for CHIP_NAVI12 and CHIP_SIENNA_CICHLID */
931	if (ucode_id == AMDGPU_UCODE_ID_RLC_G
932	|| ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
933	|| ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
934	|| ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
935	|| ucode_id == AMDGPU_UCODE_ID_SMC)
936	return true;
937	else
938	return false;
939	case IP_VERSION(13, 0, 10):
940	/* white list */
941	if (ucode_id == AMDGPU_UCODE_ID_CAP
942	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP
943	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME
944	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC
945	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK
946	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK
947	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK
948	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK
949	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK
950	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK
951	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK
952	|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK
953	|| ucode_id == AMDGPU_UCODE_ID_CP_MES
954	|| ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA
955	|| ucode_id == AMDGPU_UCODE_ID_CP_MES1
956	|| ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA
957	|| ucode_id == AMDGPU_UCODE_ID_VCN1
958	|| ucode_id == AMDGPU_UCODE_ID_VCN)
959	return false;
960	else
961	return true;
962	default:
963	/* lagacy black list */
964	if (ucode_id == AMDGPU_UCODE_ID_SDMA0
965	|| ucode_id == AMDGPU_UCODE_ID_SDMA1
966	|| ucode_id == AMDGPU_UCODE_ID_SDMA2
967	|| ucode_id == AMDGPU_UCODE_ID_SDMA3
968	|| ucode_id == AMDGPU_UCODE_ID_SDMA4
969	|| ucode_id == AMDGPU_UCODE_ID_SDMA5
970	|| ucode_id == AMDGPU_UCODE_ID_SDMA6
971	|| ucode_id == AMDGPU_UCODE_ID_SDMA7
972	|| ucode_id == AMDGPU_UCODE_ID_RLC_G
973	|| ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
974	|| ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
975	|| ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
976	|| ucode_id == AMDGPU_UCODE_ID_SMC)
977	return true;
978	else
979	return false;
980	}
981	}
982
983	void amdgpu_virt_update_sriov_video_codec(struct amdgpu_device *adev,
984	struct amdgpu_video_codec_info *encode, uint32_t encode_array_size,
985	struct amdgpu_video_codec_info *decode, uint32_t decode_array_size)
986	{
987	uint32_t i;
988
989	if (!adev->virt.is_mm_bw_enabled)
990	return;
991
992	if (encode) {
993	for (i = 0; i < encode_array_size; i++) {
994	encode[i].max_width = adev->virt.encode_max_dimension_pixels;
995	encode[i].max_pixels_per_frame = adev->virt.encode_max_frame_pixels;
996	if (encode[i].max_width > 0)
997	encode[i].max_height = encode[i].max_pixels_per_frame / encode[i].max_width;
998	else
999	encode[i].max_height = 0;
1000	}
1001	}
1002
1003	if (decode) {
1004	for (i = 0; i < decode_array_size; i++) {
1005	decode[i].max_width = adev->virt.decode_max_dimension_pixels;
1006	decode[i].max_pixels_per_frame = adev->virt.decode_max_frame_pixels;
1007	if (decode[i].max_width > 0)
1008	decode[i].max_height = decode[i].max_pixels_per_frame / decode[i].max_width;
1009	else
1010	decode[i].max_height = 0;
1011	}
1012	}
1013	}
1014
1015	bool amdgpu_virt_get_rlcg_reg_access_flag(struct amdgpu_device *adev,
1016	u32 acc_flags, u32 hwip,
1017	bool write, u32 *rlcg_flag)
1018	{
1019	bool ret = false;
1020
1021	switch (hwip) {
1022	case GC_HWIP:
1023	if (amdgpu_sriov_reg_indirect_gc(adev)) {
1024	*rlcg_flag =
1025	write ? AMDGPU_RLCG_GC_WRITE : AMDGPU_RLCG_GC_READ;
1026	ret = true;
1027	/* only in new version, AMDGPU_REGS_NO_KIQ and
1028	* AMDGPU_REGS_RLC are enabled simultaneously */
1029	} else if ((acc_flags & AMDGPU_REGS_RLC) &&
1030	!(acc_flags & AMDGPU_REGS_NO_KIQ) && write) {
1031	*rlcg_flag = AMDGPU_RLCG_GC_WRITE_LEGACY;
1032	ret = true;
1033	}
1034	break;
1035	case MMHUB_HWIP:
1036	if (amdgpu_sriov_reg_indirect_mmhub(adev) &&
1037	(acc_flags & AMDGPU_REGS_RLC) && write) {
1038	*rlcg_flag = AMDGPU_RLCG_MMHUB_WRITE;
1039	ret = true;
1040	}
1041	break;
1042	default:
1043	break;
1044	}
1045	return ret;
1046	}
1047
1048	u32 amdgpu_virt_rlcg_reg_rw(struct amdgpu_device *adev, u32 offset, u32 v, u32 flag, u32 xcc_id)
1049	{
1050	struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl;
1051	uint32_t timeout = 50000;
1052	uint32_t i, tmp;
1053	uint32_t ret = 0;
1054	void *scratch_reg0;
1055	void *scratch_reg1;
1056	void *scratch_reg2;
1057	void *scratch_reg3;
1058	void *spare_int;
1059	unsigned long flags;
1060
1061	if (!adev->gfx.rlc.rlcg_reg_access_supported) {
1062	dev_err(adev->dev,
1063	"indirect registers access through rlcg is not available\n");
1064	return 0;
1065	}
1066
1067	if (adev->gfx.xcc_mask && (((1 << xcc_id) & adev->gfx.xcc_mask) == 0)) {
1068	dev_err(adev->dev, "invalid xcc\n");
1069	return 0;
1070	}
1071
1072	if (amdgpu_device_skip_hw_access(adev))
1073	return 0;
1074
1075	reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[xcc_id];
1076	scratch_reg0 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg0;
1077	scratch_reg1 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg1;
1078	scratch_reg2 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg2;
1079	scratch_reg3 = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->scratch_reg3;
1080
1081	spin_lock_irqsave(&adev->virt.rlcg_reg_lock, flags);
1082
1083	if (reg_access_ctrl->spare_int)
1084	spare_int = (void __iomem *)adev->rmmio + 4 * reg_access_ctrl->spare_int;
1085
1086	if (offset == reg_access_ctrl->grbm_cntl) {
1087	/* if the target reg offset is grbm_cntl, write to scratch_reg2 */
1088	writel(val: v, addr: scratch_reg2);
1089	if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY)
1090	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (offset * 4));
1091	} else if (offset == reg_access_ctrl->grbm_idx) {
1092	/* if the target reg offset is grbm_idx, write to scratch_reg3 */
1093	writel(val: v, addr: scratch_reg3);
1094	if (flag == AMDGPU_RLCG_GC_WRITE_LEGACY)
1095	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (offset * 4));
1096	} else {
1097	/*
1098	* SCRATCH_REG0 = read/write value
1099	* SCRATCH_REG1[30:28] = command
1100	* SCRATCH_REG1[19:0] = address in dword
1101	* SCRATCH_REG1[27:24] = Error reporting
1102	*/
1103	writel(val: v, addr: scratch_reg0);
1104	writel(val: (offset | flag), addr: scratch_reg1);
1105	if (reg_access_ctrl->spare_int)
1106	writel(val: 1, addr: spare_int);
1107
1108	for (i = 0; i < timeout; i++) {
1109	tmp = readl(addr: scratch_reg1);
1110	if (!(tmp & AMDGPU_RLCG_SCRATCH1_ADDRESS_MASK))
1111	break;
1112	udelay(usec: 10);
1113	}
1114
1115	tmp = readl(addr: scratch_reg1);
1116	if (i >= timeout || (tmp & AMDGPU_RLCG_SCRATCH1_ERROR_MASK) != 0) {
1117	if (amdgpu_sriov_rlcg_error_report_enabled(adev)) {
1118	if (tmp & AMDGPU_RLCG_VFGATE_DISABLED) {
1119	dev_err(adev->dev,
1120	"vfgate is disabled, rlcg failed to program reg: 0x%05x\n", offset);
1121	} else if (tmp & AMDGPU_RLCG_WRONG_OPERATION_TYPE) {
1122	dev_err(adev->dev,
1123	"wrong operation type, rlcg failed to program reg: 0x%05x\n", offset);
1124	} else if (tmp & AMDGPU_RLCG_REG_NOT_IN_RANGE) {
1125	dev_err(adev->dev,
1126	"register is not in range, rlcg failed to program reg: 0x%05x\n", offset);
1127	} else {
1128	dev_err(adev->dev,
1129	"unknown error type, rlcg failed to program reg: 0x%05x\n", offset);
1130	}
1131	} else {
1132	dev_err(adev->dev,
1133	"timeout: rlcg faled to program reg: 0x%05x\n", offset);
1134	}
1135	}
1136	}
1137
1138	ret = readl(addr: scratch_reg0);
1139
1140	spin_unlock_irqrestore(lock: &adev->virt.rlcg_reg_lock, flags);
1141
1142	return ret;
1143	}
1144
1145	void amdgpu_sriov_wreg(struct amdgpu_device *adev,
1146	u32 offset, u32 value,
1147	u32 acc_flags, u32 hwip, u32 xcc_id)
1148	{
1149	u32 rlcg_flag;
1150
1151	if (amdgpu_device_skip_hw_access(adev))
1152	return;
1153
1154	if (!amdgpu_sriov_runtime(adev) &&
1155	amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, write: true, rlcg_flag: &rlcg_flag)) {
1156	amdgpu_virt_rlcg_reg_rw(adev, offset, v: value, flag: rlcg_flag, xcc_id);
1157	return;
1158	}
1159
1160	if (acc_flags & AMDGPU_REGS_NO_KIQ)
1161	WREG32_NO_KIQ(offset, value);
1162	else
1163	WREG32(offset, value);
1164	}
1165
1166	u32 amdgpu_sriov_rreg(struct amdgpu_device *adev,
1167	u32 offset, u32 acc_flags, u32 hwip, u32 xcc_id)
1168	{
1169	u32 rlcg_flag;
1170
1171	if (amdgpu_device_skip_hw_access(adev))
1172	return 0;
1173
1174	if (!amdgpu_sriov_runtime(adev) &&
1175	amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags, hwip, write: false, rlcg_flag: &rlcg_flag))
1176	return amdgpu_virt_rlcg_reg_rw(adev, offset, v: 0, flag: rlcg_flag, xcc_id);
1177
1178	if (acc_flags & AMDGPU_REGS_NO_KIQ)
1179	return RREG32_NO_KIQ(offset);
1180	else
1181	return RREG32(offset);
1182	}
1183
1184	bool amdgpu_sriov_xnack_support(struct amdgpu_device *adev)
1185	{
1186	bool xnack_mode = true;
1187
1188	if (amdgpu_sriov_vf(adev) &&
1189	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 2))
1190	xnack_mode = false;
1191
1192	return xnack_mode;
1193	}
1194
1195	bool amdgpu_virt_get_ras_capability(struct amdgpu_device *adev)
1196	{
1197	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1198
1199	if (!amdgpu_sriov_ras_caps_en(adev))
1200	return false;
1201
1202	if (adev->virt.ras_en_caps.bits.block_umc)
1203	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__UMC);
1204	if (adev->virt.ras_en_caps.bits.block_sdma)
1205	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SDMA);
1206	if (adev->virt.ras_en_caps.bits.block_gfx)
1207	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__GFX);
1208	if (adev->virt.ras_en_caps.bits.block_mmhub)
1209	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MMHUB);
1210	if (adev->virt.ras_en_caps.bits.block_athub)
1211	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__ATHUB);
1212	if (adev->virt.ras_en_caps.bits.block_pcie_bif)
1213	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__PCIE_BIF);
1214	if (adev->virt.ras_en_caps.bits.block_hdp)
1215	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__HDP);
1216	if (adev->virt.ras_en_caps.bits.block_xgmi_wafl)
1217	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__XGMI_WAFL);
1218	if (adev->virt.ras_en_caps.bits.block_df)
1219	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__DF);
1220	if (adev->virt.ras_en_caps.bits.block_smn)
1221	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SMN);
1222	if (adev->virt.ras_en_caps.bits.block_sem)
1223	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__SEM);
1224	if (adev->virt.ras_en_caps.bits.block_mp0)
1225	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MP0);
1226	if (adev->virt.ras_en_caps.bits.block_mp1)
1227	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MP1);
1228	if (adev->virt.ras_en_caps.bits.block_fuse)
1229	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__FUSE);
1230	if (adev->virt.ras_en_caps.bits.block_mca)
1231	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MCA);
1232	if (adev->virt.ras_en_caps.bits.block_vcn)
1233	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__VCN);
1234	if (adev->virt.ras_en_caps.bits.block_jpeg)
1235	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__JPEG);
1236	if (adev->virt.ras_en_caps.bits.block_ih)
1237	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__IH);
1238	if (adev->virt.ras_en_caps.bits.block_mpio)
1239	adev->ras_hw_enabled |= BIT(AMDGPU_RAS_BLOCK__MPIO);
1240
1241	if (adev->virt.ras_en_caps.bits.poison_propogation_mode)
1242	con->poison_supported = true; /* Poison is handled by host */
1243
1244	return true;
1245	}
1246
1247	static inline enum amd_sriov_ras_telemetry_gpu_block
1248	amdgpu_ras_block_to_sriov(struct amdgpu_device *adev, enum amdgpu_ras_block block) {
1249	switch (block) {
1250	case AMDGPU_RAS_BLOCK__UMC:
1251	return RAS_TELEMETRY_GPU_BLOCK_UMC;
1252	case AMDGPU_RAS_BLOCK__SDMA:
1253	return RAS_TELEMETRY_GPU_BLOCK_SDMA;
1254	case AMDGPU_RAS_BLOCK__GFX:
1255	return RAS_TELEMETRY_GPU_BLOCK_GFX;
1256	case AMDGPU_RAS_BLOCK__MMHUB:
1257	return RAS_TELEMETRY_GPU_BLOCK_MMHUB;
1258	case AMDGPU_RAS_BLOCK__ATHUB:
1259	return RAS_TELEMETRY_GPU_BLOCK_ATHUB;
1260	case AMDGPU_RAS_BLOCK__PCIE_BIF:
1261	return RAS_TELEMETRY_GPU_BLOCK_PCIE_BIF;
1262	case AMDGPU_RAS_BLOCK__HDP:
1263	return RAS_TELEMETRY_GPU_BLOCK_HDP;
1264	case AMDGPU_RAS_BLOCK__XGMI_WAFL:
1265	return RAS_TELEMETRY_GPU_BLOCK_XGMI_WAFL;
1266	case AMDGPU_RAS_BLOCK__DF:
1267	return RAS_TELEMETRY_GPU_BLOCK_DF;
1268	case AMDGPU_RAS_BLOCK__SMN:
1269	return RAS_TELEMETRY_GPU_BLOCK_SMN;
1270	case AMDGPU_RAS_BLOCK__SEM:
1271	return RAS_TELEMETRY_GPU_BLOCK_SEM;
1272	case AMDGPU_RAS_BLOCK__MP0:
1273	return RAS_TELEMETRY_GPU_BLOCK_MP0;
1274	case AMDGPU_RAS_BLOCK__MP1:
1275	return RAS_TELEMETRY_GPU_BLOCK_MP1;
1276	case AMDGPU_RAS_BLOCK__FUSE:
1277	return RAS_TELEMETRY_GPU_BLOCK_FUSE;
1278	case AMDGPU_RAS_BLOCK__MCA:
1279	return RAS_TELEMETRY_GPU_BLOCK_MCA;
1280	case AMDGPU_RAS_BLOCK__VCN:
1281	return RAS_TELEMETRY_GPU_BLOCK_VCN;
1282	case AMDGPU_RAS_BLOCK__JPEG:
1283	return RAS_TELEMETRY_GPU_BLOCK_JPEG;
1284	case AMDGPU_RAS_BLOCK__IH:
1285	return RAS_TELEMETRY_GPU_BLOCK_IH;
1286	case AMDGPU_RAS_BLOCK__MPIO:
1287	return RAS_TELEMETRY_GPU_BLOCK_MPIO;
1288	default:
1289	DRM_WARN_ONCE("Unsupported SRIOV RAS telemetry block 0x%x\n",
1290	block);
1291	return RAS_TELEMETRY_GPU_BLOCK_COUNT;
1292	}
1293	}
1294
1295	static int amdgpu_virt_cache_host_error_counts(struct amdgpu_device *adev,
1296	struct amdsriov_ras_telemetry *host_telemetry)
1297	{
1298	struct amd_sriov_ras_telemetry_error_count *tmp = NULL;
1299	uint32_t checksum, used_size;
1300
1301	checksum = host_telemetry->header.checksum;
1302	used_size = host_telemetry->header.used_size;
1303
1304	if (used_size > (AMD_SRIOV_RAS_TELEMETRY_SIZE_KB << 10))
1305	return 0;
1306
1307	tmp = kmemdup(&host_telemetry->body.error_count, used_size, GFP_KERNEL);
1308	if (!tmp)
1309	return -ENOMEM;
1310
1311	if (checksum != amd_sriov_msg_checksum(obj: tmp, obj_size: used_size, key: 0, checksum: 0))
1312	goto out;
1313
1314	memcpy(&adev->virt.count_cache, tmp,
1315	min(used_size, sizeof(adev->virt.count_cache)));
1316	out:
1317	kfree(objp: tmp);
1318
1319	return 0;
1320	}
1321
1322	static int amdgpu_virt_req_ras_err_count_internal(struct amdgpu_device *adev, bool force_update)
1323	{
1324	struct amdgpu_virt *virt = &adev->virt;
1325
1326	if (!virt->ops || !virt->ops->req_ras_err_count)
1327	return -EOPNOTSUPP;
1328
1329	/* Host allows 15 ras telemetry requests per 60 seconds. Afterwhich, the Host
1330	* will ignore incoming guest messages. Ratelimit the guest messages to
1331	* prevent guest self DOS.
1332	*/
1333	if (__ratelimit(&virt->ras.ras_error_cnt_rs) || force_update) {
1334	mutex_lock(&virt->ras.ras_telemetry_mutex);
1335	if (!virt->ops->req_ras_err_count(adev))
1336	amdgpu_virt_cache_host_error_counts(adev,
1337	host_telemetry: virt->fw_reserve.ras_telemetry);
1338	mutex_unlock(lock: &virt->ras.ras_telemetry_mutex);
1339	}
1340
1341	return 0;
1342	}
1343
1344	/* Bypass ACA interface and query ECC counts directly from host */
1345	int amdgpu_virt_req_ras_err_count(struct amdgpu_device *adev, enum amdgpu_ras_block block,
1346	struct ras_err_data *err_data)
1347	{
1348	enum amd_sriov_ras_telemetry_gpu_block sriov_block;
1349
1350	sriov_block = amdgpu_ras_block_to_sriov(adev, block);
1351
1352	if (sriov_block >= RAS_TELEMETRY_GPU_BLOCK_COUNT ||
1353	!amdgpu_sriov_ras_telemetry_block_en(adev, sriov_block))
1354	return -EOPNOTSUPP;
1355
1356	/* Host Access may be lost during reset, just return last cached data. */
1357	if (down_read_trylock(sem: &adev->reset_domain->sem)) {
1358	amdgpu_virt_req_ras_err_count_internal(adev, force_update: false);
1359	up_read(sem: &adev->reset_domain->sem);
1360	}
1361
1362	err_data->ue_count = adev->virt.count_cache.block[sriov_block].ue_count;
1363	err_data->ce_count = adev->virt.count_cache.block[sriov_block].ce_count;
1364	err_data->de_count = adev->virt.count_cache.block[sriov_block].de_count;
1365
1366	return 0;
1367	}
1368
1369	static int
1370	amdgpu_virt_write_cpers_to_ring(struct amdgpu_device *adev,
1371	struct amdsriov_ras_telemetry *host_telemetry,
1372	u32 *more)
1373	{
1374	struct amd_sriov_ras_cper_dump *cper_dump = NULL;
1375	struct cper_hdr *entry = NULL;
1376	struct amdgpu_ring *ring = &adev->cper.ring_buf;
1377	uint32_t checksum, used_size, i;
1378	int ret = 0;
1379
1380	checksum = host_telemetry->header.checksum;
1381	used_size = host_telemetry->header.used_size;
1382
1383	if (used_size > (AMD_SRIOV_RAS_TELEMETRY_SIZE_KB << 10))
1384	return -EINVAL;
1385
1386	cper_dump = kmemdup(&host_telemetry->body.cper_dump, used_size, GFP_KERNEL);
1387	if (!cper_dump)
1388	return -ENOMEM;
1389
1390	if (checksum != amd_sriov_msg_checksum(obj: cper_dump, obj_size: used_size, key: 0, checksum: 0)) {
1391	ret = -EINVAL;
1392	goto out;
1393	}
1394
1395	*more = cper_dump->more;
1396
1397	if (cper_dump->wptr < adev->virt.ras.cper_rptr) {
1398	dev_warn(
1399	adev->dev,
1400	"guest specified rptr that was too high! guest rptr: 0x%llx, host rptr: 0x%llx\n",
1401	adev->virt.ras.cper_rptr, cper_dump->wptr);
1402
1403	adev->virt.ras.cper_rptr = cper_dump->wptr;
1404	goto out;
1405	}
1406
1407	entry = (struct cper_hdr *)&cper_dump->buf[0];
1408
1409	for (i = 0; i < cper_dump->count; i++) {
1410	amdgpu_cper_ring_write(ring, src: entry, count: entry->record_length);
1411	entry = (struct cper_hdr *)((char *)entry +
1412	entry->record_length);
1413	}
1414
1415	if (cper_dump->overflow_count)
1416	dev_warn(adev->dev,
1417	"host reported CPER overflow of 0x%llx entries!\n",
1418	cper_dump->overflow_count);
1419
1420	adev->virt.ras.cper_rptr = cper_dump->wptr;
1421	out:
1422	kfree(objp: cper_dump);
1423
1424	return ret;
1425	}
1426
1427	static int amdgpu_virt_req_ras_cper_dump_internal(struct amdgpu_device *adev)
1428	{
1429	struct amdgpu_virt *virt = &adev->virt;
1430	int ret = 0;
1431	uint32_t more = 0;
1432
1433	if (!virt->ops || !virt->ops->req_ras_cper_dump)
1434	return -EOPNOTSUPP;
1435
1436	do {
1437	if (!virt->ops->req_ras_cper_dump(adev, virt->ras.cper_rptr))
1438	ret = amdgpu_virt_write_cpers_to_ring(
1439	adev, host_telemetry: virt->fw_reserve.ras_telemetry, more: &more);
1440	else
1441	ret = 0;
1442	} while (more && !ret);
1443
1444	return ret;
1445	}
1446
1447	int amdgpu_virt_req_ras_cper_dump(struct amdgpu_device *adev, bool force_update)
1448	{
1449	struct amdgpu_virt *virt = &adev->virt;
1450	int ret = 0;
1451
1452	if (!amdgpu_sriov_ras_cper_en(adev))
1453	return -EOPNOTSUPP;
1454
1455	if ((__ratelimit(&virt->ras.ras_cper_dump_rs) || force_update) &&
1456	down_read_trylock(sem: &adev->reset_domain->sem)) {
1457	mutex_lock(&virt->ras.ras_telemetry_mutex);
1458	ret = amdgpu_virt_req_ras_cper_dump_internal(adev);
1459	mutex_unlock(lock: &virt->ras.ras_telemetry_mutex);
1460	up_read(sem: &adev->reset_domain->sem);
1461	}
1462
1463	return ret;
1464	}
1465
1466	int amdgpu_virt_ras_telemetry_post_reset(struct amdgpu_device *adev)
1467	{
1468	unsigned long ue_count, ce_count;
1469
1470	if (amdgpu_sriov_ras_telemetry_en(adev)) {
1471	amdgpu_virt_req_ras_err_count_internal(adev, force_update: true);
1472	amdgpu_ras_query_error_count(adev, ce_count: &ce_count, ue_count: &ue_count, NULL);
1473	}
1474
1475	return 0;
1476	}
1477
1478	bool amdgpu_virt_ras_telemetry_block_en(struct amdgpu_device *adev,
1479	enum amdgpu_ras_block block)
1480	{
1481	enum amd_sriov_ras_telemetry_gpu_block sriov_block;
1482
1483	sriov_block = amdgpu_ras_block_to_sriov(adev, block);
1484
1485	if (sriov_block >= RAS_TELEMETRY_GPU_BLOCK_COUNT ||
1486	!amdgpu_sriov_ras_telemetry_block_en(adev, sriov_block))
1487	return false;
1488
1489	return true;
1490	}
1491
1492	/*
1493	* amdgpu_virt_request_bad_pages() - request bad pages
1494	* @adev: amdgpu device.
1495	* Send command to GPU hypervisor to write new bad pages into the shared PF2VF region
1496	*/
1497	void amdgpu_virt_request_bad_pages(struct amdgpu_device *adev)
1498	{
1499	struct amdgpu_virt *virt = &adev->virt;
1500
1501	if (virt->ops && virt->ops->req_bad_pages)
1502	virt->ops->req_bad_pages(adev);
1503	}
1504