1	/*
2	* Copyright 2008 Advanced Micro Devices, Inc.
3	* Copyright 2008 Red Hat Inc.
4	* Copyright 2009 Jerome Glisse.
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a
7	* copy of this software and associated documentation files (the "Software"),
8	* to deal in the Software without restriction, including without limitation
9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
10	* and/or sell copies of the Software, and to permit persons to whom the
11	* Software is furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22	* OTHER DEALINGS IN THE SOFTWARE.
23	*
24	* Authors: Dave Airlie
25	* Alex Deucher
26	* Jerome Glisse
27	*/
28	#include <linux/power_supply.h>
29	#include <linux/kthread.h>
30	#include <linux/module.h>
31	#include <linux/console.h>
32	#include <linux/slab.h>
33	#include <linux/iommu.h>
34	#include <linux/pci.h>
35	#include <linux/pci-p2pdma.h>
36	#include <linux/apple-gmux.h>
37
38	#include <drm/drm_aperture.h>
39	#include <drm/drm_atomic_helper.h>
40	#include <drm/drm_crtc_helper.h>
41	#include <drm/drm_fb_helper.h>
42	#include <drm/drm_probe_helper.h>
43	#include <drm/amdgpu_drm.h>
44	#include <linux/device.h>
45	#include <linux/vgaarb.h>
46	#include <linux/vga_switcheroo.h>
47	#include <linux/efi.h>
48	#include "amdgpu.h"
49	#include "amdgpu_trace.h"
50	#include "amdgpu_i2c.h"
51	#include "atom.h"
52	#include "amdgpu_atombios.h"
53	#include "amdgpu_atomfirmware.h"
54	#include "amd_pcie.h"
55	#ifdef CONFIG_DRM_AMDGPU_SI
56	#include "si.h"
57	#endif
58	#ifdef CONFIG_DRM_AMDGPU_CIK
59	#include "cik.h"
60	#endif
61	#include "vi.h"
62	#include "soc15.h"
63	#include "nv.h"
64	#include "bif/bif_4_1_d.h"
65	#include <linux/firmware.h>
66	#include "amdgpu_vf_error.h"
67
68	#include "amdgpu_amdkfd.h"
69	#include "amdgpu_pm.h"
70
71	#include "amdgpu_xgmi.h"
72	#include "amdgpu_ras.h"
73	#include "amdgpu_pmu.h"
74	#include "amdgpu_fru_eeprom.h"
75	#include "amdgpu_reset.h"
76	#include "amdgpu_virt.h"
77
78	#include <linux/suspend.h>
79	#include <drm/task_barrier.h>
80	#include <linux/pm_runtime.h>
81
82	#include <drm/drm_drv.h>
83
84	#if IS_ENABLED(CONFIG_X86)
85	#include <asm/intel-family.h>
86	#endif
87
88	MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
89	MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
90	MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
91	MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
92	MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
93	MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
94	MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
95
96	#define AMDGPU_RESUME_MS 2000
97	#define AMDGPU_MAX_RETRY_LIMIT 2
98	#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
99	#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
100	#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
101	#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
102
103	static const struct drm_driver amdgpu_kms_driver;
104
105	const char *amdgpu_asic_name[] = {
106	"TAHITI",
107	"PITCAIRN",
108	"VERDE",
109	"OLAND",
110	"HAINAN",
111	"BONAIRE",
112	"KAVERI",
113	"KABINI",
114	"HAWAII",
115	"MULLINS",
116	"TOPAZ",
117	"TONGA",
118	"FIJI",
119	"CARRIZO",
120	"STONEY",
121	"POLARIS10",
122	"POLARIS11",
123	"POLARIS12",
124	"VEGAM",
125	"VEGA10",
126	"VEGA12",
127	"VEGA20",
128	"RAVEN",
129	"ARCTURUS",
130	"RENOIR",
131	"ALDEBARAN",
132	"NAVI10",
133	"CYAN_SKILLFISH",
134	"NAVI14",
135	"NAVI12",
136	"SIENNA_CICHLID",
137	"NAVY_FLOUNDER",
138	"VANGOGH",
139	"DIMGREY_CAVEFISH",
140	"BEIGE_GOBY",
141	"YELLOW_CARP",
142	"IP DISCOVERY",
143	"LAST",
144	};
145
146	/**
147	* DOC: pcie_replay_count
148	*
149	* The amdgpu driver provides a sysfs API for reporting the total number
150	* of PCIe replays (NAKs)
151	* The file pcie_replay_count is used for this and returns the total
152	* number of replays as a sum of the NAKs generated and NAKs received
153	*/
154
155	static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
156	struct device_attribute *attr, char *buf)
157	{
158	struct drm_device *ddev = dev_get_drvdata(dev);
159	struct amdgpu_device *adev = drm_to_adev(ddev);
160	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
161
162	return sysfs_emit(buf, fmt: "%llu\n", cnt);
163	}
164
165	static DEVICE_ATTR(pcie_replay_count, 0444,
166	amdgpu_device_get_pcie_replay_count, NULL);
167
168	static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
169	struct bin_attribute *attr, char *buf,
170	loff_t ppos, size_t count)
171	{
172	struct device *dev = kobj_to_dev(kobj);
173	struct drm_device *ddev = dev_get_drvdata(dev);
174	struct amdgpu_device *adev = drm_to_adev(ddev);
175	ssize_t bytes_read;
176
177	switch (ppos) {
178	case AMDGPU_SYS_REG_STATE_XGMI:
179	bytes_read = amdgpu_asic_get_reg_state(
180	adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
181	break;
182	case AMDGPU_SYS_REG_STATE_WAFL:
183	bytes_read = amdgpu_asic_get_reg_state(
184	adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
185	break;
186	case AMDGPU_SYS_REG_STATE_PCIE:
187	bytes_read = amdgpu_asic_get_reg_state(
188	adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
189	break;
190	case AMDGPU_SYS_REG_STATE_USR:
191	bytes_read = amdgpu_asic_get_reg_state(
192	adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
193	break;
194	case AMDGPU_SYS_REG_STATE_USR_1:
195	bytes_read = amdgpu_asic_get_reg_state(
196	adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
197	break;
198	default:
199	return -EINVAL;
200	}
201
202	return bytes_read;
203	}
204
205	BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
206	AMDGPU_SYS_REG_STATE_END);
207
208	int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
209	{
210	int ret;
211
212	if (!amdgpu_asic_get_reg_state_supported(adev))
213	return 0;
214
215	ret = sysfs_create_bin_file(kobj: &adev->dev->kobj, attr: &bin_attr_reg_state);
216
217	return ret;
218	}
219
220	void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
221	{
222	if (!amdgpu_asic_get_reg_state_supported(adev))
223	return;
224	sysfs_remove_bin_file(kobj: &adev->dev->kobj, attr: &bin_attr_reg_state);
225	}
226
227	/**
228	* DOC: board_info
229	*
230	* The amdgpu driver provides a sysfs API for giving board related information.
231	* It provides the form factor information in the format
232	*
233	* type : form factor
234	*
235	* Possible form factor values
236	*
237	* - "cem" - PCIE CEM card
238	* - "oam" - Open Compute Accelerator Module
239	* - "unknown" - Not known
240	*
241	*/
242
243	static ssize_t amdgpu_device_get_board_info(struct device *dev,
244	struct device_attribute *attr,
245	char *buf)
246	{
247	struct drm_device *ddev = dev_get_drvdata(dev);
248	struct amdgpu_device *adev = drm_to_adev(ddev);
249	enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
250	const char *pkg;
251
252	if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
253	pkg_type = adev->smuio.funcs->get_pkg_type(adev);
254
255	switch (pkg_type) {
256	case AMDGPU_PKG_TYPE_CEM:
257	pkg = "cem";
258	break;
259	case AMDGPU_PKG_TYPE_OAM:
260	pkg = "oam";
261	break;
262	default:
263	pkg = "unknown";
264	break;
265	}
266
267	return sysfs_emit(buf, fmt: "%s : %s\n", "type", pkg);
268	}
269
270	static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
271
272	static struct attribute *amdgpu_board_attrs[] = {
273	&dev_attr_board_info.attr,
274	NULL,
275	};
276
277	static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
278	struct attribute *attr, int n)
279	{
280	struct device *dev = kobj_to_dev(kobj);
281	struct drm_device *ddev = dev_get_drvdata(dev);
282	struct amdgpu_device *adev = drm_to_adev(ddev);
283
284	if (adev->flags & AMD_IS_APU)
285	return 0;
286
287	return attr->mode;
288	}
289
290	static const struct attribute_group amdgpu_board_attrs_group = {
291	.attrs = amdgpu_board_attrs,
292	.is_visible = amdgpu_board_attrs_is_visible
293	};
294
295	static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
296
297
298	/**
299	* amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
300	*
301	* @dev: drm_device pointer
302	*
303	* Returns true if the device is a dGPU with ATPX power control,
304	* otherwise return false.
305	*/
306	bool amdgpu_device_supports_px(struct drm_device *dev)
307	{
308	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
309
310	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
311	return true;
312	return false;
313	}
314
315	/**
316	* amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
317	*
318	* @dev: drm_device pointer
319	*
320	* Returns true if the device is a dGPU with ACPI power control,
321	* otherwise return false.
322	*/
323	bool amdgpu_device_supports_boco(struct drm_device *dev)
324	{
325	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
326
327	if (adev->has_pr3 ||
328	((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
329	return true;
330	return false;
331	}
332
333	/**
334	* amdgpu_device_supports_baco - Does the device support BACO
335	*
336	* @dev: drm_device pointer
337	*
338	* Returns true if the device supporte BACO,
339	* otherwise return false.
340	*/
341	bool amdgpu_device_supports_baco(struct drm_device *dev)
342	{
343	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
344
345	return amdgpu_asic_supports_baco(adev);
346	}
347
348	/**
349	* amdgpu_device_supports_smart_shift - Is the device dGPU with
350	* smart shift support
351	*
352	* @dev: drm_device pointer
353	*
354	* Returns true if the device is a dGPU with Smart Shift support,
355	* otherwise returns false.
356	*/
357	bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
358	{
359	return (amdgpu_device_supports_boco(dev) &&
360	amdgpu_acpi_is_power_shift_control_supported());
361	}
362
363	/*
364	* VRAM access helper functions
365	*/
366
367	/**
368	* amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
369	*
370	* @adev: amdgpu_device pointer
371	* @pos: offset of the buffer in vram
372	* @buf: virtual address of the buffer in system memory
373	* @size: read/write size, sizeof(@buf) must > @size
374	* @write: true - write to vram, otherwise - read from vram
375	*/
376	void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
377	void *buf, size_t size, bool write)
378	{
379	unsigned long flags;
380	uint32_t hi = ~0, tmp = 0;
381	uint32_t *data = buf;
382	uint64_t last;
383	int idx;
384
385	if (!drm_dev_enter(dev: adev_to_drm(adev), idx: &idx))
386	return;
387
388	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
389
390	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
391	for (last = pos + size; pos < last; pos += 4) {
392	tmp = pos >> 31;
393
394	WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
395	if (tmp != hi) {
396	WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
397	hi = tmp;
398	}
399	if (write)
400	WREG32_NO_KIQ(mmMM_DATA, *data++);
401	else
402	*data++ = RREG32_NO_KIQ(mmMM_DATA);
403	}
404
405	spin_unlock_irqrestore(lock: &adev->mmio_idx_lock, flags);
406	drm_dev_exit(idx);
407	}
408
409	/**
410	* amdgpu_device_aper_access - access vram by vram aperature
411	*
412	* @adev: amdgpu_device pointer
413	* @pos: offset of the buffer in vram
414	* @buf: virtual address of the buffer in system memory
415	* @size: read/write size, sizeof(@buf) must > @size
416	* @write: true - write to vram, otherwise - read from vram
417	*
418	* The return value means how many bytes have been transferred.
419	*/
420	size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
421	void *buf, size_t size, bool write)
422	{
423	#ifdef CONFIG_64BIT
424	void __iomem *addr;
425	size_t count = 0;
426	uint64_t last;
427
428	if (!adev->mman.aper_base_kaddr)
429	return 0;
430
431	last = min(pos + size, adev->gmc.visible_vram_size);
432	if (last > pos) {
433	addr = adev->mman.aper_base_kaddr + pos;
434	count = last - pos;
435
436	if (write) {
437	memcpy_toio(addr, buf, count);
438	/* Make sure HDP write cache flush happens without any reordering
439	* after the system memory contents are sent over PCIe device
440	*/
441	mb();
442	amdgpu_device_flush_hdp(adev, NULL);
443	} else {
444	amdgpu_device_invalidate_hdp(adev, NULL);
445	/* Make sure HDP read cache is invalidated before issuing a read
446	* to the PCIe device
447	*/
448	mb();
449	memcpy_fromio(buf, addr, count);
450	}
451
452	}
453
454	return count;
455	#else
456	return 0;
457	#endif
458	}
459
460	/**
461	* amdgpu_device_vram_access - read/write a buffer in vram
462	*
463	* @adev: amdgpu_device pointer
464	* @pos: offset of the buffer in vram
465	* @buf: virtual address of the buffer in system memory
466	* @size: read/write size, sizeof(@buf) must > @size
467	* @write: true - write to vram, otherwise - read from vram
468	*/
469	void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
470	void *buf, size_t size, bool write)
471	{
472	size_t count;
473
474	/* try to using vram apreature to access vram first */
475	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
476	size -= count;
477	if (size) {
478	/* using MM to access rest vram */
479	pos += count;
480	buf += count;
481	amdgpu_device_mm_access(adev, pos, buf, size, write);
482	}
483	}
484
485	/*
486	* register access helper functions.
487	*/
488
489	/* Check if hw access should be skipped because of hotplug or device error */
490	bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
491	{
492	if (adev->no_hw_access)
493	return true;
494
495	#ifdef CONFIG_LOCKDEP
496	/*
497	* This is a bit complicated to understand, so worth a comment. What we assert
498	* here is that the GPU reset is not running on another thread in parallel.
499	*
500	* For this we trylock the read side of the reset semaphore, if that succeeds
501	* we know that the reset is not running in paralell.
502	*
503	* If the trylock fails we assert that we are either already holding the read
504	* side of the lock or are the reset thread itself and hold the write side of
505	* the lock.
506	*/
507	if (in_task()) {
508	if (down_read_trylock(sem: &adev->reset_domain->sem))
509	up_read(sem: &adev->reset_domain->sem);
510	else
511	lockdep_assert_held(&adev->reset_domain->sem);
512	}
513	#endif
514	return false;
515	}
516
517	/**
518	* amdgpu_device_rreg - read a memory mapped IO or indirect register
519	*
520	* @adev: amdgpu_device pointer
521	* @reg: dword aligned register offset
522	* @acc_flags: access flags which require special behavior
523	*
524	* Returns the 32 bit value from the offset specified.
525	*/
526	uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
527	uint32_t reg, uint32_t acc_flags)
528	{
529	uint32_t ret;
530
531	if (amdgpu_device_skip_hw_access(adev))
532	return 0;
533
534	if ((reg * 4) < adev->rmmio_size) {
535	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
536	amdgpu_sriov_runtime(adev) &&
537	down_read_trylock(sem: &adev->reset_domain->sem)) {
538	ret = amdgpu_kiq_rreg(adev, reg, xcc_id: 0);
539	up_read(sem: &adev->reset_domain->sem);
540	} else {
541	ret = readl(addr: ((void __iomem *)adev->rmmio) + (reg * 4));
542	}
543	} else {
544	ret = adev->pcie_rreg(adev, reg * 4);
545	}
546
547	trace_amdgpu_device_rreg(did: adev->pdev->device, reg, value: ret);
548
549	return ret;
550	}
551
552	/*
553	* MMIO register read with bytes helper functions
554	* @offset:bytes offset from MMIO start
555	*/
556
557	/**
558	* amdgpu_mm_rreg8 - read a memory mapped IO register
559	*
560	* @adev: amdgpu_device pointer
561	* @offset: byte aligned register offset
562	*
563	* Returns the 8 bit value from the offset specified.
564	*/
565	uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
566	{
567	if (amdgpu_device_skip_hw_access(adev))
568	return 0;
569
570	if (offset < adev->rmmio_size)
571	return (readb(addr: adev->rmmio + offset));
572	BUG();
573	}
574
575
576	/**
577	* amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
578	*
579	* @adev: amdgpu_device pointer
580	* @reg: dword aligned register offset
581	* @acc_flags: access flags which require special behavior
582	* @xcc_id: xcc accelerated compute core id
583	*
584	* Returns the 32 bit value from the offset specified.
585	*/
586	uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
587	uint32_t reg, uint32_t acc_flags,
588	uint32_t xcc_id)
589	{
590	uint32_t ret, rlcg_flag;
591
592	if (amdgpu_device_skip_hw_access(adev))
593	return 0;
594
595	if ((reg * 4) < adev->rmmio_size) {
596	if (amdgpu_sriov_vf(adev) &&
597	!amdgpu_sriov_runtime(adev) &&
598	adev->gfx.rlc.rlcg_reg_access_supported &&
599	amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
600	hwip: GC_HWIP, write: false,
601	rlcg_flag: &rlcg_flag)) {
602	ret = amdgpu_virt_rlcg_reg_rw(adev, offset: reg, v: 0, flag: rlcg_flag, xcc_id);
603	} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
604	amdgpu_sriov_runtime(adev) &&
605	down_read_trylock(sem: &adev->reset_domain->sem)) {
606	ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
607	up_read(sem: &adev->reset_domain->sem);
608	} else {
609	ret = readl(addr: ((void __iomem *)adev->rmmio) + (reg * 4));
610	}
611	} else {
612	ret = adev->pcie_rreg(adev, reg * 4);
613	}
614
615	return ret;
616	}
617
618	/*
619	* MMIO register write with bytes helper functions
620	* @offset:bytes offset from MMIO start
621	* @value: the value want to be written to the register
622	*/
623
624	/**
625	* amdgpu_mm_wreg8 - read a memory mapped IO register
626	*
627	* @adev: amdgpu_device pointer
628	* @offset: byte aligned register offset
629	* @value: 8 bit value to write
630	*
631	* Writes the value specified to the offset specified.
632	*/
633	void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
634	{
635	if (amdgpu_device_skip_hw_access(adev))
636	return;
637
638	if (offset < adev->rmmio_size)
639	writeb(val: value, addr: adev->rmmio + offset);
640	else
641	BUG();
642	}
643
644	/**
645	* amdgpu_device_wreg - write to a memory mapped IO or indirect register
646	*
647	* @adev: amdgpu_device pointer
648	* @reg: dword aligned register offset
649	* @v: 32 bit value to write to the register
650	* @acc_flags: access flags which require special behavior
651	*
652	* Writes the value specified to the offset specified.
653	*/
654	void amdgpu_device_wreg(struct amdgpu_device *adev,
655	uint32_t reg, uint32_t v,
656	uint32_t acc_flags)
657	{
658	if (amdgpu_device_skip_hw_access(adev))
659	return;
660
661	if ((reg * 4) < adev->rmmio_size) {
662	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
663	amdgpu_sriov_runtime(adev) &&
664	down_read_trylock(sem: &adev->reset_domain->sem)) {
665	amdgpu_kiq_wreg(adev, reg, v, xcc_id: 0);
666	up_read(sem: &adev->reset_domain->sem);
667	} else {
668	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (reg * 4));
669	}
670	} else {
671	adev->pcie_wreg(adev, reg * 4, v);
672	}
673
674	trace_amdgpu_device_wreg(did: adev->pdev->device, reg, value: v);
675	}
676
677	/**
678	* amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range
679	*
680	* @adev: amdgpu_device pointer
681	* @reg: mmio/rlc register
682	* @v: value to write
683	* @xcc_id: xcc accelerated compute core id
684	*
685	* this function is invoked only for the debugfs register access
686	*/
687	void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
688	uint32_t reg, uint32_t v,
689	uint32_t xcc_id)
690	{
691	if (amdgpu_device_skip_hw_access(adev))
692	return;
693
694	if (amdgpu_sriov_fullaccess(adev) &&
695	adev->gfx.rlc.funcs &&
696	adev->gfx.rlc.funcs->is_rlcg_access_range) {
697	if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
698	return amdgpu_sriov_wreg(adev, offset: reg, value: v, acc_flags: 0, hwip: 0, xcc_id);
699	} else if ((reg * 4) >= adev->rmmio_size) {
700	adev->pcie_wreg(adev, reg * 4, v);
701	} else {
702	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (reg * 4));
703	}
704	}
705
706	/**
707	* amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
708	*
709	* @adev: amdgpu_device pointer
710	* @reg: dword aligned register offset
711	* @v: 32 bit value to write to the register
712	* @acc_flags: access flags which require special behavior
713	* @xcc_id: xcc accelerated compute core id
714	*
715	* Writes the value specified to the offset specified.
716	*/
717	void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
718	uint32_t reg, uint32_t v,
719	uint32_t acc_flags, uint32_t xcc_id)
720	{
721	uint32_t rlcg_flag;
722
723	if (amdgpu_device_skip_hw_access(adev))
724	return;
725
726	if ((reg * 4) < adev->rmmio_size) {
727	if (amdgpu_sriov_vf(adev) &&
728	!amdgpu_sriov_runtime(adev) &&
729	adev->gfx.rlc.rlcg_reg_access_supported &&
730	amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
731	hwip: GC_HWIP, write: true,
732	rlcg_flag: &rlcg_flag)) {
733	amdgpu_virt_rlcg_reg_rw(adev, offset: reg, v, flag: rlcg_flag, xcc_id);
734	} else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
735	amdgpu_sriov_runtime(adev) &&
736	down_read_trylock(sem: &adev->reset_domain->sem)) {
737	amdgpu_kiq_wreg(adev, reg, v, xcc_id);
738	up_read(sem: &adev->reset_domain->sem);
739	} else {
740	writel(val: v, addr: ((void __iomem *)adev->rmmio) + (reg * 4));
741	}
742	} else {
743	adev->pcie_wreg(adev, reg * 4, v);
744	}
745	}
746
747	/**
748	* amdgpu_device_indirect_rreg - read an indirect register
749	*
750	* @adev: amdgpu_device pointer
751	* @reg_addr: indirect register address to read from
752	*
753	* Returns the value of indirect register @reg_addr
754	*/
755	u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
756	u32 reg_addr)
757	{
758	unsigned long flags, pcie_index, pcie_data;
759	void __iomem *pcie_index_offset;
760	void __iomem *pcie_data_offset;
761	u32 r;
762
763	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
764	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
765
766	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
767	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
768	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
769
770	writel(val: reg_addr, addr: pcie_index_offset);
771	readl(addr: pcie_index_offset);
772	r = readl(addr: pcie_data_offset);
773	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
774
775	return r;
776	}
777
778	u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
779	u64 reg_addr)
780	{
781	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
782	u32 r;
783	void __iomem *pcie_index_offset;
784	void __iomem *pcie_index_hi_offset;
785	void __iomem *pcie_data_offset;
786
787	if (unlikely(!adev->nbio.funcs)) {
788	pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
789	pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
790	} else {
791	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
792	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
793	}
794
795	if (reg_addr >> 32) {
796	if (unlikely(!adev->nbio.funcs))
797	pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
798	else
799	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
800	} else {
801	pcie_index_hi = 0;
802	}
803
804	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
805	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
806	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
807	if (pcie_index_hi != 0)
808	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
809	pcie_index_hi * 4;
810
811	writel(val: reg_addr, addr: pcie_index_offset);
812	readl(addr: pcie_index_offset);
813	if (pcie_index_hi != 0) {
814	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
815	readl(addr: pcie_index_hi_offset);
816	}
817	r = readl(addr: pcie_data_offset);
818
819	/* clear the high bits */
820	if (pcie_index_hi != 0) {
821	writel(val: 0, addr: pcie_index_hi_offset);
822	readl(addr: pcie_index_hi_offset);
823	}
824
825	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
826
827	return r;
828	}
829
830	/**
831	* amdgpu_device_indirect_rreg64 - read a 64bits indirect register
832	*
833	* @adev: amdgpu_device pointer
834	* @reg_addr: indirect register address to read from
835	*
836	* Returns the value of indirect register @reg_addr
837	*/
838	u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
839	u32 reg_addr)
840	{
841	unsigned long flags, pcie_index, pcie_data;
842	void __iomem *pcie_index_offset;
843	void __iomem *pcie_data_offset;
844	u64 r;
845
846	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
847	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
848
849	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
850	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
851	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
852
853	/* read low 32 bits */
854	writel(val: reg_addr, addr: pcie_index_offset);
855	readl(addr: pcie_index_offset);
856	r = readl(addr: pcie_data_offset);
857	/* read high 32 bits */
858	writel(val: reg_addr + 4, addr: pcie_index_offset);
859	readl(addr: pcie_index_offset);
860	r |= ((u64)readl(addr: pcie_data_offset) << 32);
861	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
862
863	return r;
864	}
865
866	u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
867	u64 reg_addr)
868	{
869	unsigned long flags, pcie_index, pcie_data;
870	unsigned long pcie_index_hi = 0;
871	void __iomem *pcie_index_offset;
872	void __iomem *pcie_index_hi_offset;
873	void __iomem *pcie_data_offset;
874	u64 r;
875
876	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
877	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
878	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
879	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
880
881	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
882	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
883	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
884	if (pcie_index_hi != 0)
885	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
886	pcie_index_hi * 4;
887
888	/* read low 32 bits */
889	writel(val: reg_addr, addr: pcie_index_offset);
890	readl(addr: pcie_index_offset);
891	if (pcie_index_hi != 0) {
892	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
893	readl(addr: pcie_index_hi_offset);
894	}
895	r = readl(addr: pcie_data_offset);
896	/* read high 32 bits */
897	writel(val: reg_addr + 4, addr: pcie_index_offset);
898	readl(addr: pcie_index_offset);
899	if (pcie_index_hi != 0) {
900	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
901	readl(addr: pcie_index_hi_offset);
902	}
903	r |= ((u64)readl(addr: pcie_data_offset) << 32);
904
905	/* clear the high bits */
906	if (pcie_index_hi != 0) {
907	writel(val: 0, addr: pcie_index_hi_offset);
908	readl(addr: pcie_index_hi_offset);
909	}
910
911	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
912
913	return r;
914	}
915
916	/**
917	* amdgpu_device_indirect_wreg - write an indirect register address
918	*
919	* @adev: amdgpu_device pointer
920	* @reg_addr: indirect register offset
921	* @reg_data: indirect register data
922	*
923	*/
924	void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
925	u32 reg_addr, u32 reg_data)
926	{
927	unsigned long flags, pcie_index, pcie_data;
928	void __iomem *pcie_index_offset;
929	void __iomem *pcie_data_offset;
930
931	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
932	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
933
934	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
935	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
936	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
937
938	writel(val: reg_addr, addr: pcie_index_offset);
939	readl(addr: pcie_index_offset);
940	writel(val: reg_data, addr: pcie_data_offset);
941	readl(addr: pcie_data_offset);
942	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
943	}
944
945	void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
946	u64 reg_addr, u32 reg_data)
947	{
948	unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
949	void __iomem *pcie_index_offset;
950	void __iomem *pcie_index_hi_offset;
951	void __iomem *pcie_data_offset;
952
953	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
954	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
955	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
956	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
957	else
958	pcie_index_hi = 0;
959
960	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
961	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
962	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
963	if (pcie_index_hi != 0)
964	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
965	pcie_index_hi * 4;
966
967	writel(val: reg_addr, addr: pcie_index_offset);
968	readl(addr: pcie_index_offset);
969	if (pcie_index_hi != 0) {
970	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
971	readl(addr: pcie_index_hi_offset);
972	}
973	writel(val: reg_data, addr: pcie_data_offset);
974	readl(addr: pcie_data_offset);
975
976	/* clear the high bits */
977	if (pcie_index_hi != 0) {
978	writel(val: 0, addr: pcie_index_hi_offset);
979	readl(addr: pcie_index_hi_offset);
980	}
981
982	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
983	}
984
985	/**
986	* amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
987	*
988	* @adev: amdgpu_device pointer
989	* @reg_addr: indirect register offset
990	* @reg_data: indirect register data
991	*
992	*/
993	void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
994	u32 reg_addr, u64 reg_data)
995	{
996	unsigned long flags, pcie_index, pcie_data;
997	void __iomem *pcie_index_offset;
998	void __iomem *pcie_data_offset;
999
1000	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1001	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1002
1003	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1004	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1005	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1006
1007	/* write low 32 bits */
1008	writel(val: reg_addr, addr: pcie_index_offset);
1009	readl(addr: pcie_index_offset);
1010	writel(val: (u32)(reg_data & 0xffffffffULL), addr: pcie_data_offset);
1011	readl(addr: pcie_data_offset);
1012	/* write high 32 bits */
1013	writel(val: reg_addr + 4, addr: pcie_index_offset);
1014	readl(addr: pcie_index_offset);
1015	writel(val: (u32)(reg_data >> 32), addr: pcie_data_offset);
1016	readl(addr: pcie_data_offset);
1017	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
1018	}
1019
1020	void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1021	u64 reg_addr, u64 reg_data)
1022	{
1023	unsigned long flags, pcie_index, pcie_data;
1024	unsigned long pcie_index_hi = 0;
1025	void __iomem *pcie_index_offset;
1026	void __iomem *pcie_index_hi_offset;
1027	void __iomem *pcie_data_offset;
1028
1029	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1030	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1031	if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1032	pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1033
1034	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1035	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1036	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1037	if (pcie_index_hi != 0)
1038	pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1039	pcie_index_hi * 4;
1040
1041	/* write low 32 bits */
1042	writel(val: reg_addr, addr: pcie_index_offset);
1043	readl(addr: pcie_index_offset);
1044	if (pcie_index_hi != 0) {
1045	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
1046	readl(addr: pcie_index_hi_offset);
1047	}
1048	writel(val: (u32)(reg_data & 0xffffffffULL), addr: pcie_data_offset);
1049	readl(addr: pcie_data_offset);
1050	/* write high 32 bits */
1051	writel(val: reg_addr + 4, addr: pcie_index_offset);
1052	readl(addr: pcie_index_offset);
1053	if (pcie_index_hi != 0) {
1054	writel(val: (reg_addr >> 32) & 0xff, addr: pcie_index_hi_offset);
1055	readl(addr: pcie_index_hi_offset);
1056	}
1057	writel(val: (u32)(reg_data >> 32), addr: pcie_data_offset);
1058	readl(addr: pcie_data_offset);
1059
1060	/* clear the high bits */
1061	if (pcie_index_hi != 0) {
1062	writel(val: 0, addr: pcie_index_hi_offset);
1063	readl(addr: pcie_index_hi_offset);
1064	}
1065
1066	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
1067	}
1068
1069	/**
1070	* amdgpu_device_get_rev_id - query device rev_id
1071	*
1072	* @adev: amdgpu_device pointer
1073	*
1074	* Return device rev_id
1075	*/
1076	u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1077	{
1078	return adev->nbio.funcs->get_rev_id(adev);
1079	}
1080
1081	/**
1082	* amdgpu_invalid_rreg - dummy reg read function
1083	*
1084	* @adev: amdgpu_device pointer
1085	* @reg: offset of register
1086	*
1087	* Dummy register read function. Used for register blocks
1088	* that certain asics don't have (all asics).
1089	* Returns the value in the register.
1090	*/
1091	static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1092	{
1093	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1094	BUG();
1095	return 0;
1096	}
1097
1098	static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1099	{
1100	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1101	BUG();
1102	return 0;
1103	}
1104
1105	/**
1106	* amdgpu_invalid_wreg - dummy reg write function
1107	*
1108	* @adev: amdgpu_device pointer
1109	* @reg: offset of register
1110	* @v: value to write to the register
1111	*
1112	* Dummy register read function. Used for register blocks
1113	* that certain asics don't have (all asics).
1114	*/
1115	static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1116	{
1117	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1118	reg, v);
1119	BUG();
1120	}
1121
1122	static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1123	{
1124	DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1125	reg, v);
1126	BUG();
1127	}
1128
1129	/**
1130	* amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1131	*
1132	* @adev: amdgpu_device pointer
1133	* @reg: offset of register
1134	*
1135	* Dummy register read function. Used for register blocks
1136	* that certain asics don't have (all asics).
1137	* Returns the value in the register.
1138	*/
1139	static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1140	{
1141	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1142	BUG();
1143	return 0;
1144	}
1145
1146	static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1147	{
1148	DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1149	BUG();
1150	return 0;
1151	}
1152
1153	/**
1154	* amdgpu_invalid_wreg64 - dummy reg write function
1155	*
1156	* @adev: amdgpu_device pointer
1157	* @reg: offset of register
1158	* @v: value to write to the register
1159	*
1160	* Dummy register read function. Used for register blocks
1161	* that certain asics don't have (all asics).
1162	*/
1163	static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1164	{
1165	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1166	reg, v);
1167	BUG();
1168	}
1169
1170	static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1171	{
1172	DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1173	reg, v);
1174	BUG();
1175	}
1176
1177	/**
1178	* amdgpu_block_invalid_rreg - dummy reg read function
1179	*
1180	* @adev: amdgpu_device pointer
1181	* @block: offset of instance
1182	* @reg: offset of register
1183	*
1184	* Dummy register read function. Used for register blocks
1185	* that certain asics don't have (all asics).
1186	* Returns the value in the register.
1187	*/
1188	static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1189	uint32_t block, uint32_t reg)
1190	{
1191	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1192	reg, block);
1193	BUG();
1194	return 0;
1195	}
1196
1197	/**
1198	* amdgpu_block_invalid_wreg - dummy reg write function
1199	*
1200	* @adev: amdgpu_device pointer
1201	* @block: offset of instance
1202	* @reg: offset of register
1203	* @v: value to write to the register
1204	*
1205	* Dummy register read function. Used for register blocks
1206	* that certain asics don't have (all asics).
1207	*/
1208	static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1209	uint32_t block,
1210	uint32_t reg, uint32_t v)
1211	{
1212	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1213	reg, block, v);
1214	BUG();
1215	}
1216
1217	/**
1218	* amdgpu_device_asic_init - Wrapper for atom asic_init
1219	*
1220	* @adev: amdgpu_device pointer
1221	*
1222	* Does any asic specific work and then calls atom asic init.
1223	*/
1224	static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1225	{
1226	int ret;
1227
1228	amdgpu_asic_pre_asic_init(adev);
1229
1230	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3) ||
1231	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >= IP_VERSION(11, 0, 0)) {
1232	amdgpu_psp_wait_for_bootloader(adev);
1233	ret = amdgpu_atomfirmware_asic_init(adev, fb_reset: true);
1234	return ret;
1235	} else {
1236	return amdgpu_atom_asic_init(ctx: adev->mode_info.atom_context);
1237	}
1238
1239	return 0;
1240	}
1241
1242	/**
1243	* amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1244	*
1245	* @adev: amdgpu_device pointer
1246	*
1247	* Allocates a scratch page of VRAM for use by various things in the
1248	* driver.
1249	*/
1250	static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1251	{
1252	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1253	AMDGPU_GEM_DOMAIN_VRAM |
1254	AMDGPU_GEM_DOMAIN_GTT,
1255	bo_ptr: &adev->mem_scratch.robj,
1256	gpu_addr: &adev->mem_scratch.gpu_addr,
1257	cpu_addr: (void **)&adev->mem_scratch.ptr);
1258	}
1259
1260	/**
1261	* amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1262	*
1263	* @adev: amdgpu_device pointer
1264	*
1265	* Frees the VRAM scratch page.
1266	*/
1267	static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1268	{
1269	amdgpu_bo_free_kernel(bo: &adev->mem_scratch.robj, NULL, NULL);
1270	}
1271
1272	/**
1273	* amdgpu_device_program_register_sequence - program an array of registers.
1274	*
1275	* @adev: amdgpu_device pointer
1276	* @registers: pointer to the register array
1277	* @array_size: size of the register array
1278	*
1279	* Programs an array or registers with and or masks.
1280	* This is a helper for setting golden registers.
1281	*/
1282	void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1283	const u32 *registers,
1284	const u32 array_size)
1285	{
1286	u32 tmp, reg, and_mask, or_mask;
1287	int i;
1288
1289	if (array_size % 3)
1290	return;
1291
1292	for (i = 0; i < array_size; i += 3) {
1293	reg = registers[i + 0];
1294	and_mask = registers[i + 1];
1295	or_mask = registers[i + 2];
1296
1297	if (and_mask == 0xffffffff) {
1298	tmp = or_mask;
1299	} else {
1300	tmp = RREG32(reg);
1301	tmp &= ~and_mask;
1302	if (adev->family >= AMDGPU_FAMILY_AI)
1303	tmp |= (or_mask & and_mask);
1304	else
1305	tmp |= or_mask;
1306	}
1307	WREG32(reg, tmp);
1308	}
1309	}
1310
1311	/**
1312	* amdgpu_device_pci_config_reset - reset the GPU
1313	*
1314	* @adev: amdgpu_device pointer
1315	*
1316	* Resets the GPU using the pci config reset sequence.
1317	* Only applicable to asics prior to vega10.
1318	*/
1319	void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1320	{
1321	pci_write_config_dword(dev: adev->pdev, where: 0x7c, AMDGPU_ASIC_RESET_DATA);
1322	}
1323
1324	/**
1325	* amdgpu_device_pci_reset - reset the GPU using generic PCI means
1326	*
1327	* @adev: amdgpu_device pointer
1328	*
1329	* Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1330	*/
1331	int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1332	{
1333	return pci_reset_function(dev: adev->pdev);
1334	}
1335
1336	/*
1337	* amdgpu_device_wb_*()
1338	* Writeback is the method by which the GPU updates special pages in memory
1339	* with the status of certain GPU events (fences, ring pointers,etc.).
1340	*/
1341
1342	/**
1343	* amdgpu_device_wb_fini - Disable Writeback and free memory
1344	*
1345	* @adev: amdgpu_device pointer
1346	*
1347	* Disables Writeback and frees the Writeback memory (all asics).
1348	* Used at driver shutdown.
1349	*/
1350	static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1351	{
1352	if (adev->wb.wb_obj) {
1353	amdgpu_bo_free_kernel(bo: &adev->wb.wb_obj,
1354	gpu_addr: &adev->wb.gpu_addr,
1355	cpu_addr: (void **)&adev->wb.wb);
1356	adev->wb.wb_obj = NULL;
1357	}
1358	}
1359
1360	/**
1361	* amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1362	*
1363	* @adev: amdgpu_device pointer
1364	*
1365	* Initializes writeback and allocates writeback memory (all asics).
1366	* Used at driver startup.
1367	* Returns 0 on success or an -error on failure.
1368	*/
1369	static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1370	{
1371	int r;
1372
1373	if (adev->wb.wb_obj == NULL) {
1374	/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1375	r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1376	PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1377	bo_ptr: &adev->wb.wb_obj, gpu_addr: &adev->wb.gpu_addr,
1378	cpu_addr: (void **)&adev->wb.wb);
1379	if (r) {
1380	dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1381	return r;
1382	}
1383
1384	adev->wb.num_wb = AMDGPU_MAX_WB;
1385	memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1386
1387	/* clear wb memory */
1388	memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1389	}
1390
1391	return 0;
1392	}
1393
1394	/**
1395	* amdgpu_device_wb_get - Allocate a wb entry
1396	*
1397	* @adev: amdgpu_device pointer
1398	* @wb: wb index
1399	*
1400	* Allocate a wb slot for use by the driver (all asics).
1401	* Returns 0 on success or -EINVAL on failure.
1402	*/
1403	int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1404	{
1405	unsigned long offset = find_first_zero_bit(addr: adev->wb.used, size: adev->wb.num_wb);
1406
1407	if (offset < adev->wb.num_wb) {
1408	__set_bit(offset, adev->wb.used);
1409	*wb = offset << 3; /* convert to dw offset */
1410	return 0;
1411	} else {
1412	return -EINVAL;
1413	}
1414	}
1415
1416	/**
1417	* amdgpu_device_wb_free - Free a wb entry
1418	*
1419	* @adev: amdgpu_device pointer
1420	* @wb: wb index
1421	*
1422	* Free a wb slot allocated for use by the driver (all asics)
1423	*/
1424	void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1425	{
1426	wb >>= 3;
1427	if (wb < adev->wb.num_wb)
1428	__clear_bit(wb, adev->wb.used);
1429	}
1430
1431	/**
1432	* amdgpu_device_resize_fb_bar - try to resize FB BAR
1433	*
1434	* @adev: amdgpu_device pointer
1435	*
1436	* Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1437	* to fail, but if any of the BARs is not accessible after the size we abort
1438	* driver loading by returning -ENODEV.
1439	*/
1440	int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1441	{
1442	int rbar_size = pci_rebar_bytes_to_size(bytes: adev->gmc.real_vram_size);
1443	struct pci_bus *root;
1444	struct resource *res;
1445	unsigned int i;
1446	u16 cmd;
1447	int r;
1448
1449	if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1450	return 0;
1451
1452	/* Bypass for VF */
1453	if (amdgpu_sriov_vf(adev))
1454	return 0;
1455
1456	/* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1457	if (!pci_find_ext_capability(dev: adev->pdev, PCI_EXT_CAP_ID_VNDR))
1458	DRM_WARN("System can't access extended configuration space,please check!!\n");
1459
1460	/* skip if the bios has already enabled large BAR */
1461	if (adev->gmc.real_vram_size &&
1462	(pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1463	return 0;
1464
1465	/* Check if the root BUS has 64bit memory resources */
1466	root = adev->pdev->bus;
1467	while (root->parent)
1468	root = root->parent;
1469
1470	pci_bus_for_each_resource(root, res, i) {
1471	if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1472	res->start > 0x100000000ull)
1473	break;
1474	}
1475
1476	/* Trying to resize is pointless without a root hub window above 4GB */
1477	if (!res)
1478	return 0;
1479
1480	/* Limit the BAR size to what is available */
1481	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1482	rbar_size);
1483
1484	/* Disable memory decoding while we change the BAR addresses and size */
1485	pci_read_config_word(dev: adev->pdev, PCI_COMMAND, val: &cmd);
1486	pci_write_config_word(dev: adev->pdev, PCI_COMMAND,
1487	val: cmd & ~PCI_COMMAND_MEMORY);
1488
1489	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1490	amdgpu_doorbell_fini(adev);
1491	if (adev->asic_type >= CHIP_BONAIRE)
1492	pci_release_resource(dev: adev->pdev, resno: 2);
1493
1494	pci_release_resource(dev: adev->pdev, resno: 0);
1495
1496	r = pci_resize_resource(dev: adev->pdev, i: 0, size: rbar_size);
1497	if (r == -ENOSPC)
1498	DRM_INFO("Not enough PCI address space for a large BAR.");
1499	else if (r && r != -ENOTSUPP)
1500	DRM_ERROR("Problem resizing BAR0 (%d).", r);
1501
1502	pci_assign_unassigned_bus_resources(bus: adev->pdev->bus);
1503
1504	/* When the doorbell or fb BAR isn't available we have no chance of
1505	* using the device.
1506	*/
1507	r = amdgpu_doorbell_init(adev);
1508	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1509	return -ENODEV;
1510
1511	pci_write_config_word(dev: adev->pdev, PCI_COMMAND, val: cmd);
1512
1513	return 0;
1514	}
1515
1516	static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1517	{
1518	if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1519	return false;
1520
1521	return true;
1522	}
1523
1524	/*
1525	* GPU helpers function.
1526	*/
1527	/**
1528	* amdgpu_device_need_post - check if the hw need post or not
1529	*
1530	* @adev: amdgpu_device pointer
1531	*
1532	* Check if the asic has been initialized (all asics) at driver startup
1533	* or post is needed if hw reset is performed.
1534	* Returns true if need or false if not.
1535	*/
1536	bool amdgpu_device_need_post(struct amdgpu_device *adev)
1537	{
1538	uint32_t reg;
1539
1540	if (amdgpu_sriov_vf(adev))
1541	return false;
1542
1543	if (!amdgpu_device_read_bios(adev))
1544	return false;
1545
1546	if (amdgpu_passthrough(adev)) {
1547	/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1548	* some old smc fw still need driver do vPost otherwise gpu hang, while
1549	* those smc fw version above 22.15 doesn't have this flaw, so we force
1550	* vpost executed for smc version below 22.15
1551	*/
1552	if (adev->asic_type == CHIP_FIJI) {
1553	int err;
1554	uint32_t fw_ver;
1555
1556	err = request_firmware(fw: &adev->pm.fw, name: "amdgpu/fiji_smc.bin", device: adev->dev);
1557	/* force vPost if error occured */
1558	if (err)
1559	return true;
1560
1561	fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1562	release_firmware(fw: adev->pm.fw);
1563	if (fw_ver < 0x00160e00)
1564	return true;
1565	}
1566	}
1567
1568	/* Don't post if we need to reset whole hive on init */
1569	if (adev->gmc.xgmi.pending_reset)
1570	return false;
1571
1572	if (adev->has_hw_reset) {
1573	adev->has_hw_reset = false;
1574	return true;
1575	}
1576
1577	/* bios scratch used on CIK+ */
1578	if (adev->asic_type >= CHIP_BONAIRE)
1579	return amdgpu_atombios_scratch_need_asic_init(adev);
1580
1581	/* check MEM_SIZE for older asics */
1582	reg = amdgpu_asic_get_config_memsize(adev);
1583
1584	if ((reg != 0) && (reg != 0xffffffff))
1585	return false;
1586
1587	return true;
1588	}
1589
1590	/*
1591	* Check whether seamless boot is supported.
1592	*
1593	* So far we only support seamless boot on DCE 3.0 or later.
1594	* If users report that it works on older ASICS as well, we may
1595	* loosen this.
1596	*/
1597	bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1598	{
1599	switch (amdgpu_seamless) {
1600	case -1:
1601	break;
1602	case 1:
1603	return true;
1604	case 0:
1605	return false;
1606	default:
1607	DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1608	amdgpu_seamless);
1609	return false;
1610	}
1611
1612	if (!(adev->flags & AMD_IS_APU))
1613	return false;
1614
1615	if (adev->mman.keep_stolen_vga_memory)
1616	return false;
1617
1618	return amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) >= IP_VERSION(3, 0, 0);
1619	}
1620
1621	/*
1622	* Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1623	* don't support dynamic speed switching. Until we have confirmation from Intel
1624	* that a specific host supports it, it's safer that we keep it disabled for all.
1625	*
1626	* https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1627	* https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1628	*/
1629	static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1630	{
1631	#if IS_ENABLED(CONFIG_X86)
1632	struct cpuinfo_x86 *c = &cpu_data(0);
1633
1634	/* eGPU change speeds based on USB4 fabric conditions */
1635	if (dev_is_removable(dev: adev->dev))
1636	return true;
1637
1638	if (c->x86_vendor == X86_VENDOR_INTEL)
1639	return false;
1640	#endif
1641	return true;
1642	}
1643
1644	/**
1645	* amdgpu_device_should_use_aspm - check if the device should program ASPM
1646	*
1647	* @adev: amdgpu_device pointer
1648	*
1649	* Confirm whether the module parameter and pcie bridge agree that ASPM should
1650	* be set for this device.
1651	*
1652	* Returns true if it should be used or false if not.
1653	*/
1654	bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1655	{
1656	switch (amdgpu_aspm) {
1657	case -1:
1658	break;
1659	case 0:
1660	return false;
1661	case 1:
1662	return true;
1663	default:
1664	return false;
1665	}
1666	if (adev->flags & AMD_IS_APU)
1667	return false;
1668	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1669	return false;
1670	return pcie_aspm_enabled(pdev: adev->pdev);
1671	}
1672
1673	/* if we get transitioned to only one device, take VGA back */
1674	/**
1675	* amdgpu_device_vga_set_decode - enable/disable vga decode
1676	*
1677	* @pdev: PCI device pointer
1678	* @state: enable/disable vga decode
1679	*
1680	* Enable/disable vga decode (all asics).
1681	* Returns VGA resource flags.
1682	*/
1683	static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1684	bool state)
1685	{
1686	struct amdgpu_device *adev = drm_to_adev(ddev: pci_get_drvdata(pdev));
1687
1688	amdgpu_asic_set_vga_state(adev, state);
1689	if (state)
1690	return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1691	VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1692	else
1693	return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1694	}
1695
1696	/**
1697	* amdgpu_device_check_block_size - validate the vm block size
1698	*
1699	* @adev: amdgpu_device pointer
1700	*
1701	* Validates the vm block size specified via module parameter.
1702	* The vm block size defines number of bits in page table versus page directory,
1703	* a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1704	* page table and the remaining bits are in the page directory.
1705	*/
1706	static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1707	{
1708	/* defines number of bits in page table versus page directory,
1709	* a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1710	* page table and the remaining bits are in the page directory
1711	*/
1712	if (amdgpu_vm_block_size == -1)
1713	return;
1714
1715	if (amdgpu_vm_block_size < 9) {
1716	dev_warn(adev->dev, "VM page table size (%d) too small\n",
1717	amdgpu_vm_block_size);
1718	amdgpu_vm_block_size = -1;
1719	}
1720	}
1721
1722	/**
1723	* amdgpu_device_check_vm_size - validate the vm size
1724	*
1725	* @adev: amdgpu_device pointer
1726	*
1727	* Validates the vm size in GB specified via module parameter.
1728	* The VM size is the size of the GPU virtual memory space in GB.
1729	*/
1730	static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1731	{
1732	/* no need to check the default value */
1733	if (amdgpu_vm_size == -1)
1734	return;
1735
1736	if (amdgpu_vm_size < 1) {
1737	dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1738	amdgpu_vm_size);
1739	amdgpu_vm_size = -1;
1740	}
1741	}
1742
1743	static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1744	{
1745	struct sysinfo si;
1746	bool is_os_64 = (sizeof(void *) == 8);
1747	uint64_t total_memory;
1748	uint64_t dram_size_seven_GB = 0x1B8000000;
1749	uint64_t dram_size_three_GB = 0xB8000000;
1750
1751	if (amdgpu_smu_memory_pool_size == 0)
1752	return;
1753
1754	if (!is_os_64) {
1755	DRM_WARN("Not 64-bit OS, feature not supported\n");
1756	goto def_value;
1757	}
1758	si_meminfo(val: &si);
1759	total_memory = (uint64_t)si.totalram * si.mem_unit;
1760
1761	if ((amdgpu_smu_memory_pool_size == 1) ||
1762	(amdgpu_smu_memory_pool_size == 2)) {
1763	if (total_memory < dram_size_three_GB)
1764	goto def_value1;
1765	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1766	(amdgpu_smu_memory_pool_size == 8)) {
1767	if (total_memory < dram_size_seven_GB)
1768	goto def_value1;
1769	} else {
1770	DRM_WARN("Smu memory pool size not supported\n");
1771	goto def_value;
1772	}
1773	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1774
1775	return;
1776
1777	def_value1:
1778	DRM_WARN("No enough system memory\n");
1779	def_value:
1780	adev->pm.smu_prv_buffer_size = 0;
1781	}
1782
1783	static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1784	{
1785	if (!(adev->flags & AMD_IS_APU) ||
1786	adev->asic_type < CHIP_RAVEN)
1787	return 0;
1788
1789	switch (adev->asic_type) {
1790	case CHIP_RAVEN:
1791	if (adev->pdev->device == 0x15dd)
1792	adev->apu_flags |= AMD_APU_IS_RAVEN;
1793	if (adev->pdev->device == 0x15d8)
1794	adev->apu_flags |= AMD_APU_IS_PICASSO;
1795	break;
1796	case CHIP_RENOIR:
1797	if ((adev->pdev->device == 0x1636) ||
1798	(adev->pdev->device == 0x164c))
1799	adev->apu_flags |= AMD_APU_IS_RENOIR;
1800	else
1801	adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1802	break;
1803	case CHIP_VANGOGH:
1804	adev->apu_flags |= AMD_APU_IS_VANGOGH;
1805	break;
1806	case CHIP_YELLOW_CARP:
1807	break;
1808	case CHIP_CYAN_SKILLFISH:
1809	if ((adev->pdev->device == 0x13FE) ||
1810	(adev->pdev->device == 0x143F))
1811	adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1812	break;
1813	default:
1814	break;
1815	}
1816
1817	return 0;
1818	}
1819
1820	/**
1821	* amdgpu_device_check_arguments - validate module params
1822	*
1823	* @adev: amdgpu_device pointer
1824	*
1825	* Validates certain module parameters and updates
1826	* the associated values used by the driver (all asics).
1827	*/
1828	static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1829	{
1830	if (amdgpu_sched_jobs < 4) {
1831	dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1832	amdgpu_sched_jobs);
1833	amdgpu_sched_jobs = 4;
1834	} else if (!is_power_of_2(n: amdgpu_sched_jobs)) {
1835	dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1836	amdgpu_sched_jobs);
1837	amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1838	}
1839
1840	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1841	/* gart size must be greater or equal to 32M */
1842	dev_warn(adev->dev, "gart size (%d) too small\n",
1843	amdgpu_gart_size);
1844	amdgpu_gart_size = -1;
1845	}
1846
1847	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1848	/* gtt size must be greater or equal to 32M */
1849	dev_warn(adev->dev, "gtt size (%d) too small\n",
1850	amdgpu_gtt_size);
1851	amdgpu_gtt_size = -1;
1852	}
1853
1854	/* valid range is between 4 and 9 inclusive */
1855	if (amdgpu_vm_fragment_size != -1 &&
1856	(amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1857	dev_warn(adev->dev, "valid range is between 4 and 9\n");
1858	amdgpu_vm_fragment_size = -1;
1859	}
1860
1861	if (amdgpu_sched_hw_submission < 2) {
1862	dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1863	amdgpu_sched_hw_submission);
1864	amdgpu_sched_hw_submission = 2;
1865	} else if (!is_power_of_2(n: amdgpu_sched_hw_submission)) {
1866	dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1867	amdgpu_sched_hw_submission);
1868	amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1869	}
1870
1871	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1872	dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1873	amdgpu_reset_method = -1;
1874	}
1875
1876	amdgpu_device_check_smu_prv_buffer_size(adev);
1877
1878	amdgpu_device_check_vm_size(adev);
1879
1880	amdgpu_device_check_block_size(adev);
1881
1882	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, load_type: amdgpu_fw_load_type);
1883
1884	return 0;
1885	}
1886
1887	/**
1888	* amdgpu_switcheroo_set_state - set switcheroo state
1889	*
1890	* @pdev: pci dev pointer
1891	* @state: vga_switcheroo state
1892	*
1893	* Callback for the switcheroo driver. Suspends or resumes
1894	* the asics before or after it is powered up using ACPI methods.
1895	*/
1896	static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1897	enum vga_switcheroo_state state)
1898	{
1899	struct drm_device *dev = pci_get_drvdata(pdev);
1900	int r;
1901
1902	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1903	return;
1904
1905	if (state == VGA_SWITCHEROO_ON) {
1906	pr_info("switched on\n");
1907	/* don't suspend or resume card normally */
1908	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1909
1910	pci_set_power_state(dev: pdev, PCI_D0);
1911	amdgpu_device_load_pci_state(pdev);
1912	r = pci_enable_device(dev: pdev);
1913	if (r)
1914	DRM_WARN("pci_enable_device failed (%d)\n", r);
1915	amdgpu_device_resume(dev, fbcon: true);
1916
1917	dev->switch_power_state = DRM_SWITCH_POWER_ON;
1918	} else {
1919	pr_info("switched off\n");
1920	dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1921	amdgpu_device_prepare(dev);
1922	amdgpu_device_suspend(dev, fbcon: true);
1923	amdgpu_device_cache_pci_state(pdev);
1924	/* Shut down the device */
1925	pci_disable_device(dev: pdev);
1926	pci_set_power_state(dev: pdev, PCI_D3cold);
1927	dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1928	}
1929	}
1930
1931	/**
1932	* amdgpu_switcheroo_can_switch - see if switcheroo state can change
1933	*
1934	* @pdev: pci dev pointer
1935	*
1936	* Callback for the switcheroo driver. Check of the switcheroo
1937	* state can be changed.
1938	* Returns true if the state can be changed, false if not.
1939	*/
1940	static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1941	{
1942	struct drm_device *dev = pci_get_drvdata(pdev);
1943
1944	/*
1945	* FIXME: open_count is protected by drm_global_mutex but that would lead to
1946	* locking inversion with the driver load path. And the access here is
1947	* completely racy anyway. So don't bother with locking for now.
1948	*/
1949	return atomic_read(v: &dev->open_count) == 0;
1950	}
1951
1952	static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1953	.set_gpu_state = amdgpu_switcheroo_set_state,
1954	.reprobe = NULL,
1955	.can_switch = amdgpu_switcheroo_can_switch,
1956	};
1957
1958	/**
1959	* amdgpu_device_ip_set_clockgating_state - set the CG state
1960	*
1961	* @dev: amdgpu_device pointer
1962	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1963	* @state: clockgating state (gate or ungate)
1964	*
1965	* Sets the requested clockgating state for all instances of
1966	* the hardware IP specified.
1967	* Returns the error code from the last instance.
1968	*/
1969	int amdgpu_device_ip_set_clockgating_state(void *dev,
1970	enum amd_ip_block_type block_type,
1971	enum amd_clockgating_state state)
1972	{
1973	struct amdgpu_device *adev = dev;
1974	int i, r = 0;
1975
1976	for (i = 0; i < adev->num_ip_blocks; i++) {
1977	if (!adev->ip_blocks[i].status.valid)
1978	continue;
1979	if (adev->ip_blocks[i].version->type != block_type)
1980	continue;
1981	if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1982	continue;
1983	r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1984	(void *)adev, state);
1985	if (r)
1986	DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1987	adev->ip_blocks[i].version->funcs->name, r);
1988	}
1989	return r;
1990	}
1991
1992	/**
1993	* amdgpu_device_ip_set_powergating_state - set the PG state
1994	*
1995	* @dev: amdgpu_device pointer
1996	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1997	* @state: powergating state (gate or ungate)
1998	*
1999	* Sets the requested powergating state for all instances of
2000	* the hardware IP specified.
2001	* Returns the error code from the last instance.
2002	*/
2003	int amdgpu_device_ip_set_powergating_state(void *dev,
2004	enum amd_ip_block_type block_type,
2005	enum amd_powergating_state state)
2006	{
2007	struct amdgpu_device *adev = dev;
2008	int i, r = 0;
2009
2010	for (i = 0; i < adev->num_ip_blocks; i++) {
2011	if (!adev->ip_blocks[i].status.valid)
2012	continue;
2013	if (adev->ip_blocks[i].version->type != block_type)
2014	continue;
2015	if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2016	continue;
2017	r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2018	(void *)adev, state);
2019	if (r)
2020	DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2021	adev->ip_blocks[i].version->funcs->name, r);
2022	}
2023	return r;
2024	}
2025
2026	/**
2027	* amdgpu_device_ip_get_clockgating_state - get the CG state
2028	*
2029	* @adev: amdgpu_device pointer
2030	* @flags: clockgating feature flags
2031	*
2032	* Walks the list of IPs on the device and updates the clockgating
2033	* flags for each IP.
2034	* Updates @flags with the feature flags for each hardware IP where
2035	* clockgating is enabled.
2036	*/
2037	void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2038	u64 *flags)
2039	{
2040	int i;
2041
2042	for (i = 0; i < adev->num_ip_blocks; i++) {
2043	if (!adev->ip_blocks[i].status.valid)
2044	continue;
2045	if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2046	adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2047	}
2048	}
2049
2050	/**
2051	* amdgpu_device_ip_wait_for_idle - wait for idle
2052	*
2053	* @adev: amdgpu_device pointer
2054	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2055	*
2056	* Waits for the request hardware IP to be idle.
2057	* Returns 0 for success or a negative error code on failure.
2058	*/
2059	int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2060	enum amd_ip_block_type block_type)
2061	{
2062	int i, r;
2063
2064	for (i = 0; i < adev->num_ip_blocks; i++) {
2065	if (!adev->ip_blocks[i].status.valid)
2066	continue;
2067	if (adev->ip_blocks[i].version->type == block_type) {
2068	r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
2069	if (r)
2070	return r;
2071	break;
2072	}
2073	}
2074	return 0;
2075
2076	}
2077
2078	/**
2079	* amdgpu_device_ip_is_idle - is the hardware IP idle
2080	*
2081	* @adev: amdgpu_device pointer
2082	* @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2083	*
2084	* Check if the hardware IP is idle or not.
2085	* Returns true if it the IP is idle, false if not.
2086	*/
2087	bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
2088	enum amd_ip_block_type block_type)
2089	{
2090	int i;
2091
2092	for (i = 0; i < adev->num_ip_blocks; i++) {
2093	if (!adev->ip_blocks[i].status.valid)
2094	continue;
2095	if (adev->ip_blocks[i].version->type == block_type)
2096	return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
2097	}
2098	return true;
2099
2100	}
2101
2102	/**
2103	* amdgpu_device_ip_get_ip_block - get a hw IP pointer
2104	*
2105	* @adev: amdgpu_device pointer
2106	* @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2107	*
2108	* Returns a pointer to the hardware IP block structure
2109	* if it exists for the asic, otherwise NULL.
2110	*/
2111	struct amdgpu_ip_block *
2112	amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2113	enum amd_ip_block_type type)
2114	{
2115	int i;
2116
2117	for (i = 0; i < adev->num_ip_blocks; i++)
2118	if (adev->ip_blocks[i].version->type == type)
2119	return &adev->ip_blocks[i];
2120
2121	return NULL;
2122	}
2123
2124	/**
2125	* amdgpu_device_ip_block_version_cmp
2126	*
2127	* @adev: amdgpu_device pointer
2128	* @type: enum amd_ip_block_type
2129	* @major: major version
2130	* @minor: minor version
2131	*
2132	* return 0 if equal or greater
2133	* return 1 if smaller or the ip_block doesn't exist
2134	*/
2135	int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2136	enum amd_ip_block_type type,
2137	u32 major, u32 minor)
2138	{
2139	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2140
2141	if (ip_block && ((ip_block->version->major > major) ||
2142	((ip_block->version->major == major) &&
2143	(ip_block->version->minor >= minor))))
2144	return 0;
2145
2146	return 1;
2147	}
2148
2149	/**
2150	* amdgpu_device_ip_block_add
2151	*
2152	* @adev: amdgpu_device pointer
2153	* @ip_block_version: pointer to the IP to add
2154	*
2155	* Adds the IP block driver information to the collection of IPs
2156	* on the asic.
2157	*/
2158	int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2159	const struct amdgpu_ip_block_version *ip_block_version)
2160	{
2161	if (!ip_block_version)
2162	return -EINVAL;
2163
2164	switch (ip_block_version->type) {
2165	case AMD_IP_BLOCK_TYPE_VCN:
2166	if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2167	return 0;
2168	break;
2169	case AMD_IP_BLOCK_TYPE_JPEG:
2170	if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2171	return 0;
2172	break;
2173	default:
2174	break;
2175	}
2176
2177	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2178	ip_block_version->funcs->name);
2179
2180	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2181
2182	return 0;
2183	}
2184
2185	/**
2186	* amdgpu_device_enable_virtual_display - enable virtual display feature
2187	*
2188	* @adev: amdgpu_device pointer
2189	*
2190	* Enabled the virtual display feature if the user has enabled it via
2191	* the module parameter virtual_display. This feature provides a virtual
2192	* display hardware on headless boards or in virtualized environments.
2193	* This function parses and validates the configuration string specified by
2194	* the user and configues the virtual display configuration (number of
2195	* virtual connectors, crtcs, etc.) specified.
2196	*/
2197	static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2198	{
2199	adev->enable_virtual_display = false;
2200
2201	if (amdgpu_virtual_display) {
2202	const char *pci_address_name = pci_name(pdev: adev->pdev);
2203	char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2204
2205	pciaddstr = kstrdup(s: amdgpu_virtual_display, GFP_KERNEL);
2206	pciaddstr_tmp = pciaddstr;
2207	while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2208	pciaddname = strsep(&pciaddname_tmp, ",");
2209	if (!strcmp("all", pciaddname)
2210	|| !strcmp(pci_address_name, pciaddname)) {
2211	long num_crtc;
2212	int res = -1;
2213
2214	adev->enable_virtual_display = true;
2215
2216	if (pciaddname_tmp)
2217	res = kstrtol(s: pciaddname_tmp, base: 10,
2218	res: &num_crtc);
2219
2220	if (!res) {
2221	if (num_crtc < 1)
2222	num_crtc = 1;
2223	if (num_crtc > 6)
2224	num_crtc = 6;
2225	adev->mode_info.num_crtc = num_crtc;
2226	} else {
2227	adev->mode_info.num_crtc = 1;
2228	}
2229	break;
2230	}
2231	}
2232
2233	DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2234	amdgpu_virtual_display, pci_address_name,
2235	adev->enable_virtual_display, adev->mode_info.num_crtc);
2236
2237	kfree(objp: pciaddstr);
2238	}
2239	}
2240
2241	void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2242	{
2243	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2244	adev->mode_info.num_crtc = 1;
2245	adev->enable_virtual_display = true;
2246	DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2247	adev->enable_virtual_display, adev->mode_info.num_crtc);
2248	}
2249	}
2250
2251	/**
2252	* amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2253	*
2254	* @adev: amdgpu_device pointer
2255	*
2256	* Parses the asic configuration parameters specified in the gpu info
2257	* firmware and makes them availale to the driver for use in configuring
2258	* the asic.
2259	* Returns 0 on success, -EINVAL on failure.
2260	*/
2261	static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2262	{
2263	const char *chip_name;
2264	char fw_name[40];
2265	int err;
2266	const struct gpu_info_firmware_header_v1_0 *hdr;
2267
2268	adev->firmware.gpu_info_fw = NULL;
2269
2270	if (adev->mman.discovery_bin)
2271	return 0;
2272
2273	switch (adev->asic_type) {
2274	default:
2275	return 0;
2276	case CHIP_VEGA10:
2277	chip_name = "vega10";
2278	break;
2279	case CHIP_VEGA12:
2280	chip_name = "vega12";
2281	break;
2282	case CHIP_RAVEN:
2283	if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2284	chip_name = "raven2";
2285	else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2286	chip_name = "picasso";
2287	else
2288	chip_name = "raven";
2289	break;
2290	case CHIP_ARCTURUS:
2291	chip_name = "arcturus";
2292	break;
2293	case CHIP_NAVI12:
2294	chip_name = "navi12";
2295	break;
2296	}
2297
2298	snprintf(buf: fw_name, size: sizeof(fw_name), fmt: "amdgpu/%s_gpu_info.bin", chip_name);
2299	err = amdgpu_ucode_request(adev, fw: &adev->firmware.gpu_info_fw, fw_name);
2300	if (err) {
2301	dev_err(adev->dev,
2302	"Failed to get gpu_info firmware \"%s\"\n",
2303	fw_name);
2304	goto out;
2305	}
2306
2307	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2308	amdgpu_ucode_print_gpu_info_hdr(hdr: &hdr->header);
2309
2310	switch (hdr->version_major) {
2311	case 1:
2312	{
2313	const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2314	(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2315	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2316
2317	/*
2318	* Should be droped when DAL no longer needs it.
2319	*/
2320	if (adev->asic_type == CHIP_NAVI12)
2321	goto parse_soc_bounding_box;
2322
2323	adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2324	adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2325	adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2326	adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2327	adev->gfx.config.max_texture_channel_caches =
2328	le32_to_cpu(gpu_info_fw->gc_num_tccs);
2329	adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2330	adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2331	adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2332	adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2333	adev->gfx.config.double_offchip_lds_buf =
2334	le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2335	adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2336	adev->gfx.cu_info.max_waves_per_simd =
2337	le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2338	adev->gfx.cu_info.max_scratch_slots_per_cu =
2339	le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2340	adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2341	if (hdr->version_minor >= 1) {
2342	const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2343	(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2344	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2345	adev->gfx.config.num_sc_per_sh =
2346	le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2347	adev->gfx.config.num_packer_per_sc =
2348	le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2349	}
2350
2351	parse_soc_bounding_box:
2352	/*
2353	* soc bounding box info is not integrated in disocovery table,
2354	* we always need to parse it from gpu info firmware if needed.
2355	*/
2356	if (hdr->version_minor == 2) {
2357	const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2358	(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2359	le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2360	adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2361	}
2362	break;
2363	}
2364	default:
2365	dev_err(adev->dev,
2366	"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2367	err = -EINVAL;
2368	goto out;
2369	}
2370	out:
2371	return err;
2372	}
2373
2374	/**
2375	* amdgpu_device_ip_early_init - run early init for hardware IPs
2376	*
2377	* @adev: amdgpu_device pointer
2378	*
2379	* Early initialization pass for hardware IPs. The hardware IPs that make
2380	* up each asic are discovered each IP's early_init callback is run. This
2381	* is the first stage in initializing the asic.
2382	* Returns 0 on success, negative error code on failure.
2383	*/
2384	static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2385	{
2386	struct pci_dev *parent;
2387	int i, r;
2388	bool total;
2389
2390	amdgpu_device_enable_virtual_display(adev);
2391
2392	if (amdgpu_sriov_vf(adev)) {
2393	r = amdgpu_virt_request_full_gpu(adev, init: true);
2394	if (r)
2395	return r;
2396	}
2397
2398	switch (adev->asic_type) {
2399	#ifdef CONFIG_DRM_AMDGPU_SI
2400	case CHIP_VERDE:
2401	case CHIP_TAHITI:
2402	case CHIP_PITCAIRN:
2403	case CHIP_OLAND:
2404	case CHIP_HAINAN:
2405	adev->family = AMDGPU_FAMILY_SI;
2406	r = si_set_ip_blocks(adev);
2407	if (r)
2408	return r;
2409	break;
2410	#endif
2411	#ifdef CONFIG_DRM_AMDGPU_CIK
2412	case CHIP_BONAIRE:
2413	case CHIP_HAWAII:
2414	case CHIP_KAVERI:
2415	case CHIP_KABINI:
2416	case CHIP_MULLINS:
2417	if (adev->flags & AMD_IS_APU)
2418	adev->family = AMDGPU_FAMILY_KV;
2419	else
2420	adev->family = AMDGPU_FAMILY_CI;
2421
2422	r = cik_set_ip_blocks(adev);
2423	if (r)
2424	return r;
2425	break;
2426	#endif
2427	case CHIP_TOPAZ:
2428	case CHIP_TONGA:
2429	case CHIP_FIJI:
2430	case CHIP_POLARIS10:
2431	case CHIP_POLARIS11:
2432	case CHIP_POLARIS12:
2433	case CHIP_VEGAM:
2434	case CHIP_CARRIZO:
2435	case CHIP_STONEY:
2436	if (adev->flags & AMD_IS_APU)
2437	adev->family = AMDGPU_FAMILY_CZ;
2438	else
2439	adev->family = AMDGPU_FAMILY_VI;
2440
2441	r = vi_set_ip_blocks(adev);
2442	if (r)
2443	return r;
2444	break;
2445	default:
2446	r = amdgpu_discovery_set_ip_blocks(adev);
2447	if (r)
2448	return r;
2449	break;
2450	}
2451
2452	if (amdgpu_has_atpx() &&
2453	(amdgpu_is_atpx_hybrid() ||
2454	amdgpu_has_atpx_dgpu_power_cntl()) &&
2455	((adev->flags & AMD_IS_APU) == 0) &&
2456	!dev_is_removable(dev: &adev->pdev->dev))
2457	adev->flags |= AMD_IS_PX;
2458
2459	if (!(adev->flags & AMD_IS_APU)) {
2460	parent = pcie_find_root_port(dev: adev->pdev);
2461	adev->has_pr3 = parent ? pci_pr3_present(pdev: parent) : false;
2462	}
2463
2464
2465	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2466	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2467	adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2468	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2469	adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2470	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2471	adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2472
2473	total = true;
2474	for (i = 0; i < adev->num_ip_blocks; i++) {
2475	if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2476	DRM_WARN("disabled ip block: %d <%s>\n",
2477	i, adev->ip_blocks[i].version->funcs->name);
2478	adev->ip_blocks[i].status.valid = false;
2479	} else {
2480	if (adev->ip_blocks[i].version->funcs->early_init) {
2481	r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2482	if (r == -ENOENT) {
2483	adev->ip_blocks[i].status.valid = false;
2484	} else if (r) {
2485	DRM_ERROR("early_init of IP block <%s> failed %d\n",
2486	adev->ip_blocks[i].version->funcs->name, r);
2487	total = false;
2488	} else {
2489	adev->ip_blocks[i].status.valid = true;
2490	}
2491	} else {
2492	adev->ip_blocks[i].status.valid = true;
2493	}
2494	}
2495	/* get the vbios after the asic_funcs are set up */
2496	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2497	r = amdgpu_device_parse_gpu_info_fw(adev);
2498	if (r)
2499	return r;
2500
2501	/* Read BIOS */
2502	if (amdgpu_device_read_bios(adev)) {
2503	if (!amdgpu_get_bios(adev))
2504	return -EINVAL;
2505
2506	r = amdgpu_atombios_init(adev);
2507	if (r) {
2508	dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2509	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, error_flags: 0, error_data: 0);
2510	return r;
2511	}
2512	}
2513
2514	/*get pf2vf msg info at it's earliest time*/
2515	if (amdgpu_sriov_vf(adev))
2516	amdgpu_virt_init_data_exchange(adev);
2517
2518	}
2519	}
2520	if (!total)
2521	return -ENODEV;
2522
2523	amdgpu_amdkfd_device_probe(adev);
2524	adev->cg_flags &= amdgpu_cg_mask;
2525	adev->pg_flags &= amdgpu_pg_mask;
2526
2527	return 0;
2528	}
2529
2530	static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2531	{
2532	int i, r;
2533
2534	for (i = 0; i < adev->num_ip_blocks; i++) {
2535	if (!adev->ip_blocks[i].status.sw)
2536	continue;
2537	if (adev->ip_blocks[i].status.hw)
2538	continue;
2539	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2540	(amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2541	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2542	r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2543	if (r) {
2544	DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2545	adev->ip_blocks[i].version->funcs->name, r);
2546	return r;
2547	}
2548	adev->ip_blocks[i].status.hw = true;
2549	}
2550	}
2551
2552	return 0;
2553	}
2554
2555	static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2556	{
2557	int i, r;
2558
2559	for (i = 0; i < adev->num_ip_blocks; i++) {
2560	if (!adev->ip_blocks[i].status.sw)
2561	continue;
2562	if (adev->ip_blocks[i].status.hw)
2563	continue;
2564	r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2565	if (r) {
2566	DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2567	adev->ip_blocks[i].version->funcs->name, r);
2568	return r;
2569	}
2570	adev->ip_blocks[i].status.hw = true;
2571	}
2572
2573	return 0;
2574	}
2575
2576	static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2577	{
2578	int r = 0;
2579	int i;
2580	uint32_t smu_version;
2581
2582	if (adev->asic_type >= CHIP_VEGA10) {
2583	for (i = 0; i < adev->num_ip_blocks; i++) {
2584	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2585	continue;
2586
2587	if (!adev->ip_blocks[i].status.sw)
2588	continue;
2589
2590	/* no need to do the fw loading again if already done*/
2591	if (adev->ip_blocks[i].status.hw == true)
2592	break;
2593
2594	if (amdgpu_in_reset(adev) || adev->in_suspend) {
2595	r = adev->ip_blocks[i].version->funcs->resume(adev);
2596	if (r) {
2597	DRM_ERROR("resume of IP block <%s> failed %d\n",
2598	adev->ip_blocks[i].version->funcs->name, r);
2599	return r;
2600	}
2601	} else {
2602	r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2603	if (r) {
2604	DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2605	adev->ip_blocks[i].version->funcs->name, r);
2606	return r;
2607	}
2608	}
2609
2610	adev->ip_blocks[i].status.hw = true;
2611	break;
2612	}
2613	}
2614
2615	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2616	r = amdgpu_pm_load_smu_firmware(adev, smu_version: &smu_version);
2617
2618	return r;
2619	}
2620
2621	static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2622	{
2623	long timeout;
2624	int r, i;
2625
2626	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2627	struct amdgpu_ring *ring = adev->rings[i];
2628
2629	/* No need to setup the GPU scheduler for rings that don't need it */
2630	if (!ring || ring->no_scheduler)
2631	continue;
2632
2633	switch (ring->funcs->type) {
2634	case AMDGPU_RING_TYPE_GFX:
2635	timeout = adev->gfx_timeout;
2636	break;
2637	case AMDGPU_RING_TYPE_COMPUTE:
2638	timeout = adev->compute_timeout;
2639	break;
2640	case AMDGPU_RING_TYPE_SDMA:
2641	timeout = adev->sdma_timeout;
2642	break;
2643	default:
2644	timeout = adev->video_timeout;
2645	break;
2646	}
2647
2648	r = drm_sched_init(sched: &ring->sched, ops: &amdgpu_sched_ops, NULL,
2649	num_rqs: DRM_SCHED_PRIORITY_COUNT,
2650	credit_limit: ring->num_hw_submission, hang_limit: 0,
2651	timeout, timeout_wq: adev->reset_domain->wq,
2652	score: ring->sched_score, name: ring->name,
2653	dev: adev->dev);
2654	if (r) {
2655	DRM_ERROR("Failed to create scheduler on ring %s.\n",
2656	ring->name);
2657	return r;
2658	}
2659	r = amdgpu_uvd_entity_init(adev, ring);
2660	if (r) {
2661	DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2662	ring->name);
2663	return r;
2664	}
2665	r = amdgpu_vce_entity_init(adev, ring);
2666	if (r) {
2667	DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2668	ring->name);
2669	return r;
2670	}
2671	}
2672
2673	amdgpu_xcp_update_partition_sched_list(adev);
2674
2675	return 0;
2676	}
2677
2678
2679	/**
2680	* amdgpu_device_ip_init - run init for hardware IPs
2681	*
2682	* @adev: amdgpu_device pointer
2683	*
2684	* Main initialization pass for hardware IPs. The list of all the hardware
2685	* IPs that make up the asic is walked and the sw_init and hw_init callbacks
2686	* are run. sw_init initializes the software state associated with each IP
2687	* and hw_init initializes the hardware associated with each IP.
2688	* Returns 0 on success, negative error code on failure.
2689	*/
2690	static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2691	{
2692	int i, r;
2693
2694	r = amdgpu_ras_init(adev);
2695	if (r)
2696	return r;
2697
2698	for (i = 0; i < adev->num_ip_blocks; i++) {
2699	if (!adev->ip_blocks[i].status.valid)
2700	continue;
2701	r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2702	if (r) {
2703	DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2704	adev->ip_blocks[i].version->funcs->name, r);
2705	goto init_failed;
2706	}
2707	adev->ip_blocks[i].status.sw = true;
2708
2709	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2710	/* need to do common hw init early so everything is set up for gmc */
2711	r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2712	if (r) {
2713	DRM_ERROR("hw_init %d failed %d\n", i, r);
2714	goto init_failed;
2715	}
2716	adev->ip_blocks[i].status.hw = true;
2717	} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2718	/* need to do gmc hw init early so we can allocate gpu mem */
2719	/* Try to reserve bad pages early */
2720	if (amdgpu_sriov_vf(adev))
2721	amdgpu_virt_exchange_data(adev);
2722
2723	r = amdgpu_device_mem_scratch_init(adev);
2724	if (r) {
2725	DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2726	goto init_failed;
2727	}
2728	r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2729	if (r) {
2730	DRM_ERROR("hw_init %d failed %d\n", i, r);
2731	goto init_failed;
2732	}
2733	r = amdgpu_device_wb_init(adev);
2734	if (r) {
2735	DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2736	goto init_failed;
2737	}
2738	adev->ip_blocks[i].status.hw = true;
2739
2740	/* right after GMC hw init, we create CSA */
2741	if (adev->gfx.mcbp) {
2742	r = amdgpu_allocate_static_csa(adev, bo: &adev->virt.csa_obj,
2743	AMDGPU_GEM_DOMAIN_VRAM |
2744	AMDGPU_GEM_DOMAIN_GTT,
2745	AMDGPU_CSA_SIZE);
2746	if (r) {
2747	DRM_ERROR("allocate CSA failed %d\n", r);
2748	goto init_failed;
2749	}
2750	}
2751
2752	r = amdgpu_seq64_init(adev);
2753	if (r) {
2754	DRM_ERROR("allocate seq64 failed %d\n", r);
2755	goto init_failed;
2756	}
2757	}
2758	}
2759
2760	if (amdgpu_sriov_vf(adev))
2761	amdgpu_virt_init_data_exchange(adev);
2762
2763	r = amdgpu_ib_pool_init(adev);
2764	if (r) {
2765	dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2766	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_IB_INIT_FAIL, error_flags: 0, error_data: r);
2767	goto init_failed;
2768	}
2769
2770	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2771	if (r)
2772	goto init_failed;
2773
2774	r = amdgpu_device_ip_hw_init_phase1(adev);
2775	if (r)
2776	goto init_failed;
2777
2778	r = amdgpu_device_fw_loading(adev);
2779	if (r)
2780	goto init_failed;
2781
2782	r = amdgpu_device_ip_hw_init_phase2(adev);
2783	if (r)
2784	goto init_failed;
2785
2786	/*
2787	* retired pages will be loaded from eeprom and reserved here,
2788	* it should be called after amdgpu_device_ip_hw_init_phase2 since
2789	* for some ASICs the RAS EEPROM code relies on SMU fully functioning
2790	* for I2C communication which only true at this point.
2791	*
2792	* amdgpu_ras_recovery_init may fail, but the upper only cares the
2793	* failure from bad gpu situation and stop amdgpu init process
2794	* accordingly. For other failed cases, it will still release all
2795	* the resource and print error message, rather than returning one
2796	* negative value to upper level.
2797	*
2798	* Note: theoretically, this should be called before all vram allocations
2799	* to protect retired page from abusing
2800	*/
2801	r = amdgpu_ras_recovery_init(adev);
2802	if (r)
2803	goto init_failed;
2804
2805	/**
2806	* In case of XGMI grab extra reference for reset domain for this device
2807	*/
2808	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2809	if (amdgpu_xgmi_add_device(adev) == 0) {
2810	if (!amdgpu_sriov_vf(adev)) {
2811	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2812
2813	if (WARN_ON(!hive)) {
2814	r = -ENOENT;
2815	goto init_failed;
2816	}
2817
2818	if (!hive->reset_domain ||
2819	!amdgpu_reset_get_reset_domain(domain: hive->reset_domain)) {
2820	r = -ENOENT;
2821	amdgpu_put_xgmi_hive(hive);
2822	goto init_failed;
2823	}
2824
2825	/* Drop the early temporary reset domain we created for device */
2826	amdgpu_reset_put_reset_domain(domain: adev->reset_domain);
2827	adev->reset_domain = hive->reset_domain;
2828	amdgpu_put_xgmi_hive(hive);
2829	}
2830	}
2831	}
2832
2833	r = amdgpu_device_init_schedulers(adev);
2834	if (r)
2835	goto init_failed;
2836
2837	if (adev->mman.buffer_funcs_ring->sched.ready)
2838	amdgpu_ttm_set_buffer_funcs_status(adev, enable: true);
2839
2840	/* Don't init kfd if whole hive need to be reset during init */
2841	if (!adev->gmc.xgmi.pending_reset) {
2842	kgd2kfd_init_zone_device(adev);
2843	amdgpu_amdkfd_device_init(adev);
2844	}
2845
2846	amdgpu_fru_get_product_info(adev);
2847
2848	init_failed:
2849
2850	return r;
2851	}
2852
2853	/**
2854	* amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2855	*
2856	* @adev: amdgpu_device pointer
2857	*
2858	* Writes a reset magic value to the gart pointer in VRAM. The driver calls
2859	* this function before a GPU reset. If the value is retained after a
2860	* GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2861	*/
2862	static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2863	{
2864	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2865	}
2866
2867	/**
2868	* amdgpu_device_check_vram_lost - check if vram is valid
2869	*
2870	* @adev: amdgpu_device pointer
2871	*
2872	* Checks the reset magic value written to the gart pointer in VRAM.
2873	* The driver calls this after a GPU reset to see if the contents of
2874	* VRAM is lost or now.
2875	* returns true if vram is lost, false if not.
2876	*/
2877	static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2878	{
2879	if (memcmp(p: adev->gart.ptr, q: adev->reset_magic,
2880	AMDGPU_RESET_MAGIC_NUM))
2881	return true;
2882
2883	if (!amdgpu_in_reset(adev))
2884	return false;
2885
2886	/*
2887	* For all ASICs with baco/mode1 reset, the VRAM is
2888	* always assumed to be lost.
2889	*/
2890	switch (amdgpu_asic_reset_method(adev)) {
2891	case AMD_RESET_METHOD_BACO:
2892	case AMD_RESET_METHOD_MODE1:
2893	return true;
2894	default:
2895	return false;
2896	}
2897	}
2898
2899	/**
2900	* amdgpu_device_set_cg_state - set clockgating for amdgpu device
2901	*
2902	* @adev: amdgpu_device pointer
2903	* @state: clockgating state (gate or ungate)
2904	*
2905	* The list of all the hardware IPs that make up the asic is walked and the
2906	* set_clockgating_state callbacks are run.
2907	* Late initialization pass enabling clockgating for hardware IPs.
2908	* Fini or suspend, pass disabling clockgating for hardware IPs.
2909	* Returns 0 on success, negative error code on failure.
2910	*/
2911
2912	int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2913	enum amd_clockgating_state state)
2914	{
2915	int i, j, r;
2916
2917	if (amdgpu_emu_mode == 1)
2918	return 0;
2919
2920	for (j = 0; j < adev->num_ip_blocks; j++) {
2921	i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2922	if (!adev->ip_blocks[i].status.late_initialized)
2923	continue;
2924	/* skip CG for GFX, SDMA on S0ix */
2925	if (adev->in_s0ix &&
2926	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2927	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2928	continue;
2929	/* skip CG for VCE/UVD, it's handled specially */
2930	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2931	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2932	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2933	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2934	adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2935	/* enable clockgating to save power */
2936	r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2937	state);
2938	if (r) {
2939	DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2940	adev->ip_blocks[i].version->funcs->name, r);
2941	return r;
2942	}
2943	}
2944	}
2945
2946	return 0;
2947	}
2948
2949	int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2950	enum amd_powergating_state state)
2951	{
2952	int i, j, r;
2953
2954	if (amdgpu_emu_mode == 1)
2955	return 0;
2956
2957	for (j = 0; j < adev->num_ip_blocks; j++) {
2958	i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2959	if (!adev->ip_blocks[i].status.late_initialized)
2960	continue;
2961	/* skip PG for GFX, SDMA on S0ix */
2962	if (adev->in_s0ix &&
2963	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2964	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2965	continue;
2966	/* skip CG for VCE/UVD, it's handled specially */
2967	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2968	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2969	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2970	adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2971	adev->ip_blocks[i].version->funcs->set_powergating_state) {
2972	/* enable powergating to save power */
2973	r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2974	state);
2975	if (r) {
2976	DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2977	adev->ip_blocks[i].version->funcs->name, r);
2978	return r;
2979	}
2980	}
2981	}
2982	return 0;
2983	}
2984
2985	static int amdgpu_device_enable_mgpu_fan_boost(void)
2986	{
2987	struct amdgpu_gpu_instance *gpu_ins;
2988	struct amdgpu_device *adev;
2989	int i, ret = 0;
2990
2991	mutex_lock(&mgpu_info.mutex);
2992
2993	/*
2994	* MGPU fan boost feature should be enabled
2995	* only when there are two or more dGPUs in
2996	* the system
2997	*/
2998	if (mgpu_info.num_dgpu < 2)
2999	goto out;
3000
3001	for (i = 0; i < mgpu_info.num_dgpu; i++) {
3002	gpu_ins = &(mgpu_info.gpu_ins[i]);
3003	adev = gpu_ins->adev;
3004	if (!(adev->flags & AMD_IS_APU) &&
3005	!gpu_ins->mgpu_fan_enabled) {
3006	ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3007	if (ret)
3008	break;
3009
3010	gpu_ins->mgpu_fan_enabled = 1;
3011	}
3012	}
3013
3014	out:
3015	mutex_unlock(lock: &mgpu_info.mutex);
3016
3017	return ret;
3018	}
3019
3020	/**
3021	* amdgpu_device_ip_late_init - run late init for hardware IPs
3022	*
3023	* @adev: amdgpu_device pointer
3024	*
3025	* Late initialization pass for hardware IPs. The list of all the hardware
3026	* IPs that make up the asic is walked and the late_init callbacks are run.
3027	* late_init covers any special initialization that an IP requires
3028	* after all of the have been initialized or something that needs to happen
3029	* late in the init process.
3030	* Returns 0 on success, negative error code on failure.
3031	*/
3032	static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3033	{
3034	struct amdgpu_gpu_instance *gpu_instance;
3035	int i = 0, r;
3036
3037	for (i = 0; i < adev->num_ip_blocks; i++) {
3038	if (!adev->ip_blocks[i].status.hw)
3039	continue;
3040	if (adev->ip_blocks[i].version->funcs->late_init) {
3041	r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
3042	if (r) {
3043	DRM_ERROR("late_init of IP block <%s> failed %d\n",
3044	adev->ip_blocks[i].version->funcs->name, r);
3045	return r;
3046	}
3047	}
3048	adev->ip_blocks[i].status.late_initialized = true;
3049	}
3050
3051	r = amdgpu_ras_late_init(adev);
3052	if (r) {
3053	DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3054	return r;
3055	}
3056
3057	amdgpu_ras_set_error_query_ready(adev, ready: true);
3058
3059	amdgpu_device_set_cg_state(adev, state: AMD_CG_STATE_GATE);
3060	amdgpu_device_set_pg_state(adev, state: AMD_PG_STATE_GATE);
3061
3062	amdgpu_device_fill_reset_magic(adev);
3063
3064	r = amdgpu_device_enable_mgpu_fan_boost();
3065	if (r)
3066	DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3067
3068	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3069	if (amdgpu_passthrough(adev) &&
3070	((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3071	adev->asic_type == CHIP_ALDEBARAN))
3072	amdgpu_dpm_handle_passthrough_sbr(adev, enable: true);
3073
3074	if (adev->gmc.xgmi.num_physical_nodes > 1) {
3075	mutex_lock(&mgpu_info.mutex);
3076
3077	/*
3078	* Reset device p-state to low as this was booted with high.
3079	*
3080	* This should be performed only after all devices from the same
3081	* hive get initialized.
3082	*
3083	* However, it's unknown how many device in the hive in advance.
3084	* As this is counted one by one during devices initializations.
3085	*
3086	* So, we wait for all XGMI interlinked devices initialized.
3087	* This may bring some delays as those devices may come from
3088	* different hives. But that should be OK.
3089	*/
3090	if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3091	for (i = 0; i < mgpu_info.num_gpu; i++) {
3092	gpu_instance = &(mgpu_info.gpu_ins[i]);
3093	if (gpu_instance->adev->flags & AMD_IS_APU)
3094	continue;
3095
3096	r = amdgpu_xgmi_set_pstate(adev: gpu_instance->adev,
3097	pstate: AMDGPU_XGMI_PSTATE_MIN);
3098	if (r) {
3099	DRM_ERROR("pstate setting failed (%d).\n", r);
3100	break;
3101	}
3102	}
3103	}
3104
3105	mutex_unlock(lock: &mgpu_info.mutex);
3106	}
3107
3108	return 0;
3109	}
3110
3111	/**
3112	* amdgpu_device_smu_fini_early - smu hw_fini wrapper
3113	*
3114	* @adev: amdgpu_device pointer
3115	*
3116	* For ASICs need to disable SMC first
3117	*/
3118	static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3119	{
3120	int i, r;
3121
3122	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) > IP_VERSION(9, 0, 0))
3123	return;
3124
3125	for (i = 0; i < adev->num_ip_blocks; i++) {
3126	if (!adev->ip_blocks[i].status.hw)
3127	continue;
3128	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3129	r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3130	/* XXX handle errors */
3131	if (r) {
3132	DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3133	adev->ip_blocks[i].version->funcs->name, r);
3134	}
3135	adev->ip_blocks[i].status.hw = false;
3136	break;
3137	}
3138	}
3139	}
3140
3141	static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3142	{
3143	int i, r;
3144
3145	for (i = 0; i < adev->num_ip_blocks; i++) {
3146	if (!adev->ip_blocks[i].version->funcs->early_fini)
3147	continue;
3148
3149	r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
3150	if (r) {
3151	DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3152	adev->ip_blocks[i].version->funcs->name, r);
3153	}
3154	}
3155
3156	amdgpu_device_set_pg_state(adev, state: AMD_PG_STATE_UNGATE);
3157	amdgpu_device_set_cg_state(adev, state: AMD_CG_STATE_UNGATE);
3158
3159	amdgpu_amdkfd_suspend(adev, run_pm: false);
3160
3161	/* Workaroud for ASICs need to disable SMC first */
3162	amdgpu_device_smu_fini_early(adev);
3163
3164	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3165	if (!adev->ip_blocks[i].status.hw)
3166	continue;
3167
3168	r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3169	/* XXX handle errors */
3170	if (r) {
3171	DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3172	adev->ip_blocks[i].version->funcs->name, r);
3173	}
3174
3175	adev->ip_blocks[i].status.hw = false;
3176	}
3177
3178	if (amdgpu_sriov_vf(adev)) {
3179	if (amdgpu_virt_release_full_gpu(adev, init: false))
3180	DRM_ERROR("failed to release exclusive mode on fini\n");
3181	}
3182
3183	return 0;
3184	}
3185
3186	/**
3187	* amdgpu_device_ip_fini - run fini for hardware IPs
3188	*
3189	* @adev: amdgpu_device pointer
3190	*
3191	* Main teardown pass for hardware IPs. The list of all the hardware
3192	* IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3193	* are run. hw_fini tears down the hardware associated with each IP
3194	* and sw_fini tears down any software state associated with each IP.
3195	* Returns 0 on success, negative error code on failure.
3196	*/
3197	static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3198	{
3199	int i, r;
3200
3201	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3202	amdgpu_virt_release_ras_err_handler_data(adev);
3203
3204	if (adev->gmc.xgmi.num_physical_nodes > 1)
3205	amdgpu_xgmi_remove_device(adev);
3206
3207	amdgpu_amdkfd_device_fini_sw(adev);
3208
3209	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3210	if (!adev->ip_blocks[i].status.sw)
3211	continue;
3212
3213	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3214	amdgpu_ucode_free_bo(adev);
3215	amdgpu_free_static_csa(bo: &adev->virt.csa_obj);
3216	amdgpu_device_wb_fini(adev);
3217	amdgpu_device_mem_scratch_fini(adev);
3218	amdgpu_ib_pool_fini(adev);
3219	amdgpu_seq64_fini(adev);
3220	}
3221
3222	r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3223	/* XXX handle errors */
3224	if (r) {
3225	DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3226	adev->ip_blocks[i].version->funcs->name, r);
3227	}
3228	adev->ip_blocks[i].status.sw = false;
3229	adev->ip_blocks[i].status.valid = false;
3230	}
3231
3232	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3233	if (!adev->ip_blocks[i].status.late_initialized)
3234	continue;
3235	if (adev->ip_blocks[i].version->funcs->late_fini)
3236	adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3237	adev->ip_blocks[i].status.late_initialized = false;
3238	}
3239
3240	amdgpu_ras_fini(adev);
3241
3242	return 0;
3243	}
3244
3245	/**
3246	* amdgpu_device_delayed_init_work_handler - work handler for IB tests
3247	*
3248	* @work: work_struct.
3249	*/
3250	static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3251	{
3252	struct amdgpu_device *adev =
3253	container_of(work, struct amdgpu_device, delayed_init_work.work);
3254	int r;
3255
3256	r = amdgpu_ib_ring_tests(adev);
3257	if (r)
3258	DRM_ERROR("ib ring test failed (%d).\n", r);
3259	}
3260
3261	static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3262	{
3263	struct amdgpu_device *adev =
3264	container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3265
3266	WARN_ON_ONCE(adev->gfx.gfx_off_state);
3267	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3268
3269	if (!amdgpu_dpm_set_powergating_by_smu(adev, block_type: AMD_IP_BLOCK_TYPE_GFX, gate: true))
3270	adev->gfx.gfx_off_state = true;
3271	}
3272
3273	/**
3274	* amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3275	*
3276	* @adev: amdgpu_device pointer
3277	*
3278	* Main suspend function for hardware IPs. The list of all the hardware
3279	* IPs that make up the asic is walked, clockgating is disabled and the
3280	* suspend callbacks are run. suspend puts the hardware and software state
3281	* in each IP into a state suitable for suspend.
3282	* Returns 0 on success, negative error code on failure.
3283	*/
3284	static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3285	{
3286	int i, r;
3287
3288	amdgpu_device_set_pg_state(adev, state: AMD_PG_STATE_UNGATE);
3289	amdgpu_device_set_cg_state(adev, state: AMD_CG_STATE_UNGATE);
3290
3291	/*
3292	* Per PMFW team's suggestion, driver needs to handle gfxoff
3293	* and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3294	* scenario. Add the missing df cstate disablement here.
3295	*/
3296	if (amdgpu_dpm_set_df_cstate(adev, cstate: DF_CSTATE_DISALLOW))
3297	dev_warn(adev->dev, "Failed to disallow df cstate");
3298
3299	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3300	if (!adev->ip_blocks[i].status.valid)
3301	continue;
3302
3303	/* displays are handled separately */
3304	if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3305	continue;
3306
3307	/* XXX handle errors */
3308	r = adev->ip_blocks[i].version->funcs->suspend(adev);
3309	/* XXX handle errors */
3310	if (r) {
3311	DRM_ERROR("suspend of IP block <%s> failed %d\n",
3312	adev->ip_blocks[i].version->funcs->name, r);
3313	return r;
3314	}
3315
3316	adev->ip_blocks[i].status.hw = false;
3317	}
3318
3319	return 0;
3320	}
3321
3322	/**
3323	* amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3324	*
3325	* @adev: amdgpu_device pointer
3326	*
3327	* Main suspend function for hardware IPs. The list of all the hardware
3328	* IPs that make up the asic is walked, clockgating is disabled and the
3329	* suspend callbacks are run. suspend puts the hardware and software state
3330	* in each IP into a state suitable for suspend.
3331	* Returns 0 on success, negative error code on failure.
3332	*/
3333	static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3334	{
3335	int i, r;
3336
3337	if (adev->in_s0ix)
3338	amdgpu_dpm_gfx_state_change(adev, state: sGpuChangeState_D3Entry);
3339
3340	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3341	if (!adev->ip_blocks[i].status.valid)
3342	continue;
3343	/* displays are handled in phase1 */
3344	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3345	continue;
3346	/* PSP lost connection when err_event_athub occurs */
3347	if (amdgpu_ras_intr_triggered() &&
3348	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3349	adev->ip_blocks[i].status.hw = false;
3350	continue;
3351	}
3352
3353	/* skip unnecessary suspend if we do not initialize them yet */
3354	if (adev->gmc.xgmi.pending_reset &&
3355	!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3356	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3357	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3358	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3359	adev->ip_blocks[i].status.hw = false;
3360	continue;
3361	}
3362
3363	/* skip suspend of gfx/mes and psp for S0ix
3364	* gfx is in gfxoff state, so on resume it will exit gfxoff just
3365	* like at runtime. PSP is also part of the always on hardware
3366	* so no need to suspend it.
3367	*/
3368	if (adev->in_s0ix &&
3369	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3370	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3371	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3372	continue;
3373
3374	/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3375	if (adev->in_s0ix &&
3376	(amdgpu_ip_version(adev, ip: SDMA0_HWIP, inst: 0) >=
3377	IP_VERSION(5, 0, 0)) &&
3378	(adev->ip_blocks[i].version->type ==
3379	AMD_IP_BLOCK_TYPE_SDMA))
3380	continue;
3381
3382	/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3383	* These are in TMR, hence are expected to be reused by PSP-TOS to reload
3384	* from this location and RLC Autoload automatically also gets loaded
3385	* from here based on PMFW -> PSP message during re-init sequence.
3386	* Therefore, the psp suspend & resume should be skipped to avoid destroy
3387	* the TMR and reload FWs again for IMU enabled APU ASICs.
3388	*/
3389	if (amdgpu_in_reset(adev) &&
3390	(adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3391	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3392	continue;
3393
3394	/* XXX handle errors */
3395	r = adev->ip_blocks[i].version->funcs->suspend(adev);
3396	/* XXX handle errors */
3397	if (r) {
3398	DRM_ERROR("suspend of IP block <%s> failed %d\n",
3399	adev->ip_blocks[i].version->funcs->name, r);
3400	}
3401	adev->ip_blocks[i].status.hw = false;
3402	/* handle putting the SMC in the appropriate state */
3403	if (!amdgpu_sriov_vf(adev)) {
3404	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3405	r = amdgpu_dpm_set_mp1_state(adev, mp1_state: adev->mp1_state);
3406	if (r) {
3407	DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3408	adev->mp1_state, r);
3409	return r;
3410	}
3411	}
3412	}
3413	}
3414
3415	return 0;
3416	}
3417
3418	/**
3419	* amdgpu_device_ip_suspend - run suspend for hardware IPs
3420	*
3421	* @adev: amdgpu_device pointer
3422	*
3423	* Main suspend function for hardware IPs. The list of all the hardware
3424	* IPs that make up the asic is walked, clockgating is disabled and the
3425	* suspend callbacks are run. suspend puts the hardware and software state
3426	* in each IP into a state suitable for suspend.
3427	* Returns 0 on success, negative error code on failure.
3428	*/
3429	int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3430	{
3431	int r;
3432
3433	if (amdgpu_sriov_vf(adev)) {
3434	amdgpu_virt_fini_data_exchange(adev);
3435	amdgpu_virt_request_full_gpu(adev, init: false);
3436	}
3437
3438	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
3439
3440	r = amdgpu_device_ip_suspend_phase1(adev);
3441	if (r)
3442	return r;
3443	r = amdgpu_device_ip_suspend_phase2(adev);
3444
3445	if (amdgpu_sriov_vf(adev))
3446	amdgpu_virt_release_full_gpu(adev, init: false);
3447
3448	return r;
3449	}
3450
3451	static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3452	{
3453	int i, r;
3454
3455	static enum amd_ip_block_type ip_order[] = {
3456	AMD_IP_BLOCK_TYPE_COMMON,
3457	AMD_IP_BLOCK_TYPE_GMC,
3458	AMD_IP_BLOCK_TYPE_PSP,
3459	AMD_IP_BLOCK_TYPE_IH,
3460	};
3461
3462	for (i = 0; i < adev->num_ip_blocks; i++) {
3463	int j;
3464	struct amdgpu_ip_block *block;
3465
3466	block = &adev->ip_blocks[i];
3467	block->status.hw = false;
3468
3469	for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3470
3471	if (block->version->type != ip_order[j] ||
3472	!block->status.valid)
3473	continue;
3474
3475	r = block->version->funcs->hw_init(adev);
3476	DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3477	if (r)
3478	return r;
3479	block->status.hw = true;
3480	}
3481	}
3482
3483	return 0;
3484	}
3485
3486	static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3487	{
3488	int i, r;
3489
3490	static enum amd_ip_block_type ip_order[] = {
3491	AMD_IP_BLOCK_TYPE_SMC,
3492	AMD_IP_BLOCK_TYPE_DCE,
3493	AMD_IP_BLOCK_TYPE_GFX,
3494	AMD_IP_BLOCK_TYPE_SDMA,
3495	AMD_IP_BLOCK_TYPE_MES,
3496	AMD_IP_BLOCK_TYPE_UVD,
3497	AMD_IP_BLOCK_TYPE_VCE,
3498	AMD_IP_BLOCK_TYPE_VCN,
3499	AMD_IP_BLOCK_TYPE_JPEG
3500	};
3501
3502	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3503	int j;
3504	struct amdgpu_ip_block *block;
3505
3506	for (j = 0; j < adev->num_ip_blocks; j++) {
3507	block = &adev->ip_blocks[j];
3508
3509	if (block->version->type != ip_order[i] ||
3510	!block->status.valid ||
3511	block->status.hw)
3512	continue;
3513
3514	if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3515	r = block->version->funcs->resume(adev);
3516	else
3517	r = block->version->funcs->hw_init(adev);
3518
3519	DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3520	if (r)
3521	return r;
3522	block->status.hw = true;
3523	}
3524	}
3525
3526	return 0;
3527	}
3528
3529	/**
3530	* amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3531	*
3532	* @adev: amdgpu_device pointer
3533	*
3534	* First resume function for hardware IPs. The list of all the hardware
3535	* IPs that make up the asic is walked and the resume callbacks are run for
3536	* COMMON, GMC, and IH. resume puts the hardware into a functional state
3537	* after a suspend and updates the software state as necessary. This
3538	* function is also used for restoring the GPU after a GPU reset.
3539	* Returns 0 on success, negative error code on failure.
3540	*/
3541	static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3542	{
3543	int i, r;
3544
3545	for (i = 0; i < adev->num_ip_blocks; i++) {
3546	if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3547	continue;
3548	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3549	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3550	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3551	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3552
3553	r = adev->ip_blocks[i].version->funcs->resume(adev);
3554	if (r) {
3555	DRM_ERROR("resume of IP block <%s> failed %d\n",
3556	adev->ip_blocks[i].version->funcs->name, r);
3557	return r;
3558	}
3559	adev->ip_blocks[i].status.hw = true;
3560	}
3561	}
3562
3563	return 0;
3564	}
3565
3566	/**
3567	* amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3568	*
3569	* @adev: amdgpu_device pointer
3570	*
3571	* First resume function for hardware IPs. The list of all the hardware
3572	* IPs that make up the asic is walked and the resume callbacks are run for
3573	* all blocks except COMMON, GMC, and IH. resume puts the hardware into a
3574	* functional state after a suspend and updates the software state as
3575	* necessary. This function is also used for restoring the GPU after a GPU
3576	* reset.
3577	* Returns 0 on success, negative error code on failure.
3578	*/
3579	static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3580	{
3581	int i, r;
3582
3583	for (i = 0; i < adev->num_ip_blocks; i++) {
3584	if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3585	continue;
3586	if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3587	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3588	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3589	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3590	continue;
3591	r = adev->ip_blocks[i].version->funcs->resume(adev);
3592	if (r) {
3593	DRM_ERROR("resume of IP block <%s> failed %d\n",
3594	adev->ip_blocks[i].version->funcs->name, r);
3595	return r;
3596	}
3597	adev->ip_blocks[i].status.hw = true;
3598	}
3599
3600	return 0;
3601	}
3602
3603	/**
3604	* amdgpu_device_ip_resume - run resume for hardware IPs
3605	*
3606	* @adev: amdgpu_device pointer
3607	*
3608	* Main resume function for hardware IPs. The hardware IPs
3609	* are split into two resume functions because they are
3610	* also used in recovering from a GPU reset and some additional
3611	* steps need to be take between them. In this case (S3/S4) they are
3612	* run sequentially.
3613	* Returns 0 on success, negative error code on failure.
3614	*/
3615	static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3616	{
3617	int r;
3618
3619	r = amdgpu_device_ip_resume_phase1(adev);
3620	if (r)
3621	return r;
3622
3623	r = amdgpu_device_fw_loading(adev);
3624	if (r)
3625	return r;
3626
3627	r = amdgpu_device_ip_resume_phase2(adev);
3628
3629	if (adev->mman.buffer_funcs_ring->sched.ready)
3630	amdgpu_ttm_set_buffer_funcs_status(adev, enable: true);
3631
3632	return r;
3633	}
3634
3635	/**
3636	* amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3637	*
3638	* @adev: amdgpu_device pointer
3639	*
3640	* Query the VBIOS data tables to determine if the board supports SR-IOV.
3641	*/
3642	static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3643	{
3644	if (amdgpu_sriov_vf(adev)) {
3645	if (adev->is_atom_fw) {
3646	if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3647	adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3648	} else {
3649	if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3650	adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3651	}
3652
3653	if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3654	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_NO_VBIOS, error_flags: 0, error_data: 0);
3655	}
3656	}
3657
3658	/**
3659	* amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3660	*
3661	* @asic_type: AMD asic type
3662	*
3663	* Check if there is DC (new modesetting infrastructre) support for an asic.
3664	* returns true if DC has support, false if not.
3665	*/
3666	bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3667	{
3668	switch (asic_type) {
3669	#ifdef CONFIG_DRM_AMDGPU_SI
3670	case CHIP_HAINAN:
3671	#endif
3672	case CHIP_TOPAZ:
3673	/* chips with no display hardware */
3674	return false;
3675	#if defined(CONFIG_DRM_AMD_DC)
3676	case CHIP_TAHITI:
3677	case CHIP_PITCAIRN:
3678	case CHIP_VERDE:
3679	case CHIP_OLAND:
3680	/*
3681	* We have systems in the wild with these ASICs that require
3682	* LVDS and VGA support which is not supported with DC.
3683	*
3684	* Fallback to the non-DC driver here by default so as not to
3685	* cause regressions.
3686	*/
3687	#if defined(CONFIG_DRM_AMD_DC_SI)
3688	return amdgpu_dc > 0;
3689	#else
3690	return false;
3691	#endif
3692	case CHIP_BONAIRE:
3693	case CHIP_KAVERI:
3694	case CHIP_KABINI:
3695	case CHIP_MULLINS:
3696	/*
3697	* We have systems in the wild with these ASICs that require
3698	* VGA support which is not supported with DC.
3699	*
3700	* Fallback to the non-DC driver here by default so as not to
3701	* cause regressions.
3702	*/
3703	return amdgpu_dc > 0;
3704	default:
3705	return amdgpu_dc != 0;
3706	#else
3707	default:
3708	if (amdgpu_dc > 0)
3709	DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3710	return false;
3711	#endif
3712	}
3713	}
3714
3715	/**
3716	* amdgpu_device_has_dc_support - check if dc is supported
3717	*
3718	* @adev: amdgpu_device pointer
3719	*
3720	* Returns true for supported, false for not supported
3721	*/
3722	bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3723	{
3724	if (adev->enable_virtual_display ||
3725	(adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3726	return false;
3727
3728	return amdgpu_device_asic_has_dc_support(asic_type: adev->asic_type);
3729	}
3730
3731	static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3732	{
3733	struct amdgpu_device *adev =
3734	container_of(__work, struct amdgpu_device, xgmi_reset_work);
3735	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3736
3737	/* It's a bug to not have a hive within this function */
3738	if (WARN_ON(!hive))
3739	return;
3740
3741	/*
3742	* Use task barrier to synchronize all xgmi reset works across the
3743	* hive. task_barrier_enter and task_barrier_exit will block
3744	* until all the threads running the xgmi reset works reach
3745	* those points. task_barrier_full will do both blocks.
3746	*/
3747	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3748
3749	task_barrier_enter(tb: &hive->tb);
3750	adev->asic_reset_res = amdgpu_device_baco_enter(dev: adev_to_drm(adev));
3751
3752	if (adev->asic_reset_res)
3753	goto fail;
3754
3755	task_barrier_exit(tb: &hive->tb);
3756	adev->asic_reset_res = amdgpu_device_baco_exit(dev: adev_to_drm(adev));
3757
3758	if (adev->asic_reset_res)
3759	goto fail;
3760
3761	amdgpu_ras_reset_error_count(adev, block: AMDGPU_RAS_BLOCK__MMHUB);
3762	} else {
3763
3764	task_barrier_full(tb: &hive->tb);
3765	adev->asic_reset_res = amdgpu_asic_reset(adev);
3766	}
3767
3768	fail:
3769	if (adev->asic_reset_res)
3770	DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3771	adev->asic_reset_res, adev_to_drm(adev)->unique);
3772	amdgpu_put_xgmi_hive(hive);
3773	}
3774
3775	static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3776	{
3777	char *input = amdgpu_lockup_timeout;
3778	char *timeout_setting = NULL;
3779	int index = 0;
3780	long timeout;
3781	int ret = 0;
3782
3783	/*
3784	* By default timeout for non compute jobs is 10000
3785	* and 60000 for compute jobs.
3786	* In SR-IOV or passthrough mode, timeout for compute
3787	* jobs are 60000 by default.
3788	*/
3789	adev->gfx_timeout = msecs_to_jiffies(m: 10000);
3790	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3791	if (amdgpu_sriov_vf(adev))
3792	adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3793	msecs_to_jiffies(m: 60000) : msecs_to_jiffies(m: 10000);
3794	else
3795	adev->compute_timeout = msecs_to_jiffies(m: 60000);
3796
3797	if (strnlen(p: input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3798	while ((timeout_setting = strsep(&input, ",")) &&
3799	strnlen(p: timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3800	ret = kstrtol(s: timeout_setting, base: 0, res: &timeout);
3801	if (ret)
3802	return ret;
3803
3804	if (timeout == 0) {
3805	index++;
3806	continue;
3807	} else if (timeout < 0) {
3808	timeout = MAX_SCHEDULE_TIMEOUT;
3809	dev_warn(adev->dev, "lockup timeout disabled");
3810	add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3811	} else {
3812	timeout = msecs_to_jiffies(m: timeout);
3813	}
3814
3815	switch (index++) {
3816	case 0:
3817	adev->gfx_timeout = timeout;
3818	break;
3819	case 1:
3820	adev->compute_timeout = timeout;
3821	break;
3822	case 2:
3823	adev->sdma_timeout = timeout;
3824	break;
3825	case 3:
3826	adev->video_timeout = timeout;
3827	break;
3828	default:
3829	break;
3830	}
3831	}
3832	/*
3833	* There is only one value specified and
3834	* it should apply to all non-compute jobs.
3835	*/
3836	if (index == 1) {
3837	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3838	if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3839	adev->compute_timeout = adev->gfx_timeout;
3840	}
3841	}
3842
3843	return ret;
3844	}
3845
3846	/**
3847	* amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3848	*
3849	* @adev: amdgpu_device pointer
3850	*
3851	* RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3852	*/
3853	static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3854	{
3855	struct iommu_domain *domain;
3856
3857	domain = iommu_get_domain_for_dev(dev: adev->dev);
3858	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3859	adev->ram_is_direct_mapped = true;
3860	}
3861
3862	static const struct attribute *amdgpu_dev_attributes[] = {
3863	&dev_attr_pcie_replay_count.attr,
3864	NULL
3865	};
3866
3867	static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3868	{
3869	if (amdgpu_mcbp == 1)
3870	adev->gfx.mcbp = true;
3871	else if (amdgpu_mcbp == 0)
3872	adev->gfx.mcbp = false;
3873
3874	if (amdgpu_sriov_vf(adev))
3875	adev->gfx.mcbp = true;
3876
3877	if (adev->gfx.mcbp)
3878	DRM_INFO("MCBP is enabled\n");
3879	}
3880
3881	/**
3882	* amdgpu_device_init - initialize the driver
3883	*
3884	* @adev: amdgpu_device pointer
3885	* @flags: driver flags
3886	*
3887	* Initializes the driver info and hw (all asics).
3888	* Returns 0 for success or an error on failure.
3889	* Called at driver startup.
3890	*/
3891	int amdgpu_device_init(struct amdgpu_device *adev,
3892	uint32_t flags)
3893	{
3894	struct drm_device *ddev = adev_to_drm(adev);
3895	struct pci_dev *pdev = adev->pdev;
3896	int r, i;
3897	bool px = false;
3898	u32 max_MBps;
3899	int tmp;
3900
3901	adev->shutdown = false;
3902	adev->flags = flags;
3903
3904	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3905	adev->asic_type = amdgpu_force_asic_type;
3906	else
3907	adev->asic_type = flags & AMD_ASIC_MASK;
3908
3909	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3910	if (amdgpu_emu_mode == 1)
3911	adev->usec_timeout *= 10;
3912	adev->gmc.gart_size = 512 * 1024 * 1024;
3913	adev->accel_working = false;
3914	adev->num_rings = 0;
3915	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3916	adev->mman.buffer_funcs = NULL;
3917	adev->mman.buffer_funcs_ring = NULL;
3918	adev->vm_manager.vm_pte_funcs = NULL;
3919	adev->vm_manager.vm_pte_num_scheds = 0;
3920	adev->gmc.gmc_funcs = NULL;
3921	adev->harvest_ip_mask = 0x0;
3922	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3923	bitmap_zero(dst: adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3924
3925	adev->smc_rreg = &amdgpu_invalid_rreg;
3926	adev->smc_wreg = &amdgpu_invalid_wreg;
3927	adev->pcie_rreg = &amdgpu_invalid_rreg;
3928	adev->pcie_wreg = &amdgpu_invalid_wreg;
3929	adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3930	adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3931	adev->pciep_rreg = &amdgpu_invalid_rreg;
3932	adev->pciep_wreg = &amdgpu_invalid_wreg;
3933	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3934	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3935	adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3936	adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3937	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3938	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3939	adev->didt_rreg = &amdgpu_invalid_rreg;
3940	adev->didt_wreg = &amdgpu_invalid_wreg;
3941	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3942	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3943	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3944	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3945
3946	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3947	amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3948	pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3949
3950	/* mutex initialization are all done here so we
3951	* can recall function without having locking issues
3952	*/
3953	mutex_init(&adev->firmware.mutex);
3954	mutex_init(&adev->pm.mutex);
3955	mutex_init(&adev->gfx.gpu_clock_mutex);
3956	mutex_init(&adev->srbm_mutex);
3957	mutex_init(&adev->gfx.pipe_reserve_mutex);
3958	mutex_init(&adev->gfx.gfx_off_mutex);
3959	mutex_init(&adev->gfx.partition_mutex);
3960	mutex_init(&adev->grbm_idx_mutex);
3961	mutex_init(&adev->mn_lock);
3962	mutex_init(&adev->virt.vf_errors.lock);
3963	hash_init(adev->mn_hash);
3964	mutex_init(&adev->psp.mutex);
3965	mutex_init(&adev->notifier_lock);
3966	mutex_init(&adev->pm.stable_pstate_ctx_lock);
3967	mutex_init(&adev->benchmark_mutex);
3968
3969	amdgpu_device_init_apu_flags(adev);
3970
3971	r = amdgpu_device_check_arguments(adev);
3972	if (r)
3973	return r;
3974
3975	spin_lock_init(&adev->mmio_idx_lock);
3976	spin_lock_init(&adev->smc_idx_lock);
3977	spin_lock_init(&adev->pcie_idx_lock);
3978	spin_lock_init(&adev->uvd_ctx_idx_lock);
3979	spin_lock_init(&adev->didt_idx_lock);
3980	spin_lock_init(&adev->gc_cac_idx_lock);
3981	spin_lock_init(&adev->se_cac_idx_lock);
3982	spin_lock_init(&adev->audio_endpt_idx_lock);
3983	spin_lock_init(&adev->mm_stats.lock);
3984
3985	INIT_LIST_HEAD(list: &adev->shadow_list);
3986	mutex_init(&adev->shadow_list_lock);
3987
3988	INIT_LIST_HEAD(list: &adev->reset_list);
3989
3990	INIT_LIST_HEAD(list: &adev->ras_list);
3991
3992	INIT_LIST_HEAD(list: &adev->pm.od_kobj_list);
3993
3994	INIT_DELAYED_WORK(&adev->delayed_init_work,
3995	amdgpu_device_delayed_init_work_handler);
3996	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3997	amdgpu_device_delay_enable_gfx_off);
3998
3999	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
4000
4001	adev->gfx.gfx_off_req_count = 1;
4002	adev->gfx.gfx_off_residency = 0;
4003	adev->gfx.gfx_off_entrycount = 0;
4004	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4005
4006	atomic_set(v: &adev->throttling_logging_enabled, i: 1);
4007	/*
4008	* If throttling continues, logging will be performed every minute
4009	* to avoid log flooding. "-1" is subtracted since the thermal
4010	* throttling interrupt comes every second. Thus, the total logging
4011	* interval is 59 seconds(retelimited printk interval) + 1(waiting
4012	* for throttling interrupt) = 60 seconds.
4013	*/
4014	ratelimit_state_init(rs: &adev->throttling_logging_rs, interval: (60 - 1) * HZ, burst: 1);
4015	ratelimit_set_flags(rs: &adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4016
4017	/* Registers mapping */
4018	/* TODO: block userspace mapping of io register */
4019	if (adev->asic_type >= CHIP_BONAIRE) {
4020	adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4021	adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4022	} else {
4023	adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4024	adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4025	}
4026
4027	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4028	atomic_set(v: &adev->pm.pwr_state[i], i: POWER_STATE_UNKNOWN);
4029
4030	adev->rmmio = ioremap(offset: adev->rmmio_base, size: adev->rmmio_size);
4031	if (!adev->rmmio)
4032	return -ENOMEM;
4033
4034	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4035	DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4036
4037	/*
4038	* Reset domain needs to be present early, before XGMI hive discovered
4039	* (if any) and intitialized to use reset sem and in_gpu reset flag
4040	* early on during init and before calling to RREG32.
4041	*/
4042	adev->reset_domain = amdgpu_reset_create_reset_domain(type: SINGLE_DEVICE, wq_name: "amdgpu-reset-dev");
4043	if (!adev->reset_domain)
4044	return -ENOMEM;
4045
4046	/* detect hw virtualization here */
4047	amdgpu_detect_virtualization(adev);
4048
4049	amdgpu_device_get_pcie_info(adev);
4050
4051	r = amdgpu_device_get_job_timeout_settings(adev);
4052	if (r) {
4053	dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4054	return r;
4055	}
4056
4057	amdgpu_device_set_mcbp(adev);
4058
4059	/* early init functions */
4060	r = amdgpu_device_ip_early_init(adev);
4061	if (r)
4062	return r;
4063
4064	/* Get rid of things like offb */
4065	r = drm_aperture_remove_conflicting_pci_framebuffers(pdev: adev->pdev, req_driver: &amdgpu_kms_driver);
4066	if (r)
4067	return r;
4068
4069	/* Enable TMZ based on IP_VERSION */
4070	amdgpu_gmc_tmz_set(adev);
4071
4072	amdgpu_gmc_noretry_set(adev);
4073	/* Need to get xgmi info early to decide the reset behavior*/
4074	if (adev->gmc.xgmi.supported) {
4075	r = adev->gfxhub.funcs->get_xgmi_info(adev);
4076	if (r)
4077	return r;
4078	}
4079
4080	/* enable PCIE atomic ops */
4081	if (amdgpu_sriov_vf(adev)) {
4082	if (adev->virt.fw_reserve.p_pf2vf)
4083	adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4084	adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4085	(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4086	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4087	* internal path natively support atomics, set have_atomics_support to true.
4088	*/
4089	} else if ((adev->flags & AMD_IS_APU) &&
4090	(amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) >
4091	IP_VERSION(9, 0, 0))) {
4092	adev->have_atomics_support = true;
4093	} else {
4094	adev->have_atomics_support =
4095	!pci_enable_atomic_ops_to_root(dev: adev->pdev,
4096	PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4097	PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4098	}
4099
4100	if (!adev->have_atomics_support)
4101	dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4102
4103	/* doorbell bar mapping and doorbell index init*/
4104	amdgpu_doorbell_init(adev);
4105
4106	if (amdgpu_emu_mode == 1) {
4107	/* post the asic on emulation mode */
4108	emu_soc_asic_init(adev);
4109	goto fence_driver_init;
4110	}
4111
4112	amdgpu_reset_init(adev);
4113
4114	/* detect if we are with an SRIOV vbios */
4115	if (adev->bios)
4116	amdgpu_device_detect_sriov_bios(adev);
4117
4118	/* check if we need to reset the asic
4119	* E.g., driver was not cleanly unloaded previously, etc.
4120	*/
4121	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4122	if (adev->gmc.xgmi.num_physical_nodes) {
4123	dev_info(adev->dev, "Pending hive reset.\n");
4124	adev->gmc.xgmi.pending_reset = true;
4125	/* Only need to init necessary block for SMU to handle the reset */
4126	for (i = 0; i < adev->num_ip_blocks; i++) {
4127	if (!adev->ip_blocks[i].status.valid)
4128	continue;
4129	if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
4130	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
4131	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
4132	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
4133	DRM_DEBUG("IP %s disabled for hw_init.\n",
4134	adev->ip_blocks[i].version->funcs->name);
4135	adev->ip_blocks[i].status.hw = true;
4136	}
4137	}
4138	} else if (amdgpu_ip_version(adev, ip: MP1_HWIP, inst: 0) == IP_VERSION(13, 0, 10) &&
4139	!amdgpu_device_has_display_hardware(adev)) {
4140	r = psp_gpu_reset(adev);
4141	} else {
4142	tmp = amdgpu_reset_method;
4143	/* It should do a default reset when loading or reloading the driver,
4144	* regardless of the module parameter reset_method.
4145	*/
4146	amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4147	r = amdgpu_asic_reset(adev);
4148	amdgpu_reset_method = tmp;
4149	}
4150
4151	if (r) {
4152	dev_err(adev->dev, "asic reset on init failed\n");
4153	goto failed;
4154	}
4155	}
4156
4157	/* Post card if necessary */
4158	if (amdgpu_device_need_post(adev)) {
4159	if (!adev->bios) {
4160	dev_err(adev->dev, "no vBIOS found\n");
4161	r = -EINVAL;
4162	goto failed;
4163	}
4164	DRM_INFO("GPU posting now...\n");
4165	r = amdgpu_device_asic_init(adev);
4166	if (r) {
4167	dev_err(adev->dev, "gpu post error!\n");
4168	goto failed;
4169	}
4170	}
4171
4172	if (adev->bios) {
4173	if (adev->is_atom_fw) {
4174	/* Initialize clocks */
4175	r = amdgpu_atomfirmware_get_clock_info(adev);
4176	if (r) {
4177	dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4178	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, error_flags: 0, error_data: 0);
4179	goto failed;
4180	}
4181	} else {
4182	/* Initialize clocks */
4183	r = amdgpu_atombios_get_clock_info(adev);
4184	if (r) {
4185	dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4186	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, error_flags: 0, error_data: 0);
4187	goto failed;
4188	}
4189	/* init i2c buses */
4190	if (!amdgpu_device_has_dc_support(adev))
4191	amdgpu_atombios_i2c_init(adev);
4192	}
4193	}
4194
4195	fence_driver_init:
4196	/* Fence driver */
4197	r = amdgpu_fence_driver_sw_init(adev);
4198	if (r) {
4199	dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4200	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_FENCE_INIT_FAIL, error_flags: 0, error_data: 0);
4201	goto failed;
4202	}
4203
4204	/* init the mode config */
4205	drm_mode_config_init(dev: adev_to_drm(adev));
4206
4207	r = amdgpu_device_ip_init(adev);
4208	if (r) {
4209	dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4210	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, error_flags: 0, error_data: 0);
4211	goto release_ras_con;
4212	}
4213
4214	amdgpu_fence_driver_hw_init(adev);
4215
4216	dev_info(adev->dev,
4217	"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4218	adev->gfx.config.max_shader_engines,
4219	adev->gfx.config.max_sh_per_se,
4220	adev->gfx.config.max_cu_per_sh,
4221	adev->gfx.cu_info.number);
4222
4223	adev->accel_working = true;
4224
4225	amdgpu_vm_check_compute_bug(adev);
4226
4227	/* Initialize the buffer migration limit. */
4228	if (amdgpu_moverate >= 0)
4229	max_MBps = amdgpu_moverate;
4230	else
4231	max_MBps = 8; /* Allow 8 MB/s. */
4232	/* Get a log2 for easy divisions. */
4233	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4234
4235	/*
4236	* Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4237	* Otherwise the mgpu fan boost feature will be skipped due to the
4238	* gpu instance is counted less.
4239	*/
4240	amdgpu_register_gpu_instance(adev);
4241
4242	/* enable clockgating, etc. after ib tests, etc. since some blocks require
4243	* explicit gating rather than handling it automatically.
4244	*/
4245	if (!adev->gmc.xgmi.pending_reset) {
4246	r = amdgpu_device_ip_late_init(adev);
4247	if (r) {
4248	dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4249	amdgpu_vf_error_put(adev, sub_error_code: AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, error_flags: 0, error_data: r);
4250	goto release_ras_con;
4251	}
4252	/* must succeed. */
4253	amdgpu_ras_resume(adev);
4254	queue_delayed_work(wq: system_wq, dwork: &adev->delayed_init_work,
4255	delay: msecs_to_jiffies(AMDGPU_RESUME_MS));
4256	}
4257
4258	if (amdgpu_sriov_vf(adev)) {
4259	amdgpu_virt_release_full_gpu(adev, init: true);
4260	flush_delayed_work(dwork: &adev->delayed_init_work);
4261	}
4262
4263	/*
4264	* Place those sysfs registering after `late_init`. As some of those
4265	* operations performed in `late_init` might affect the sysfs
4266	* interfaces creating.
4267	*/
4268	r = amdgpu_atombios_sysfs_init(adev);
4269	if (r)
4270	drm_err(&adev->ddev,
4271	"registering atombios sysfs failed (%d).\n", r);
4272
4273	r = amdgpu_pm_sysfs_init(adev);
4274	if (r)
4275	DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4276
4277	r = amdgpu_ucode_sysfs_init(adev);
4278	if (r) {
4279	adev->ucode_sysfs_en = false;
4280	DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4281	} else
4282	adev->ucode_sysfs_en = true;
4283
4284	r = sysfs_create_files(kobj: &adev->dev->kobj, attr: amdgpu_dev_attributes);
4285	if (r)
4286	dev_err(adev->dev, "Could not create amdgpu device attr\n");
4287
4288	r = devm_device_add_group(dev: adev->dev, grp: &amdgpu_board_attrs_group);
4289	if (r)
4290	dev_err(adev->dev,
4291	"Could not create amdgpu board attributes\n");
4292
4293	amdgpu_fru_sysfs_init(adev);
4294	amdgpu_reg_state_sysfs_init(adev);
4295
4296	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4297	r = amdgpu_pmu_init(adev);
4298	if (r)
4299	dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4300
4301	/* Have stored pci confspace at hand for restore in sudden PCI error */
4302	if (amdgpu_device_cache_pci_state(pdev: adev->pdev))
4303	pci_restore_state(dev: pdev);
4304
4305	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4306	/* this will fail for cards that aren't VGA class devices, just
4307	* ignore it
4308	*/
4309	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4310	vga_client_register(pdev: adev->pdev, set_decode: amdgpu_device_vga_set_decode);
4311
4312	px = amdgpu_device_supports_px(dev: ddev);
4313
4314	if (px || (!dev_is_removable(dev: &adev->pdev->dev) &&
4315	apple_gmux_detect(NULL, NULL)))
4316	vga_switcheroo_register_client(dev: adev->pdev,
4317	ops: &amdgpu_switcheroo_ops, driver_power_control: px);
4318
4319	if (px)
4320	vga_switcheroo_init_domain_pm_ops(dev: adev->dev, domain: &adev->vga_pm_domain);
4321
4322	if (adev->gmc.xgmi.pending_reset)
4323	queue_delayed_work(wq: system_wq, dwork: &mgpu_info.delayed_reset_work,
4324	delay: msecs_to_jiffies(AMDGPU_RESUME_MS));
4325
4326	amdgpu_device_check_iommu_direct_map(adev);
4327
4328	return 0;
4329
4330	release_ras_con:
4331	if (amdgpu_sriov_vf(adev))
4332	amdgpu_virt_release_full_gpu(adev, init: true);
4333
4334	/* failed in exclusive mode due to timeout */
4335	if (amdgpu_sriov_vf(adev) &&
4336	!amdgpu_sriov_runtime(adev) &&
4337	amdgpu_virt_mmio_blocked(adev) &&
4338	!amdgpu_virt_wait_reset(adev)) {
4339	dev_err(adev->dev, "VF exclusive mode timeout\n");
4340	/* Don't send request since VF is inactive. */
4341	adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4342	adev->virt.ops = NULL;
4343	r = -EAGAIN;
4344	}
4345	amdgpu_release_ras_context(adev);
4346
4347	failed:
4348	amdgpu_vf_error_trans_all(adev);
4349
4350	return r;
4351	}
4352
4353	static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4354	{
4355
4356	/* Clear all CPU mappings pointing to this device */
4357	unmap_mapping_range(mapping: adev->ddev.anon_inode->i_mapping, holebegin: 0, holelen: 0, even_cows: 1);
4358
4359	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4360	amdgpu_doorbell_fini(adev);
4361
4362	iounmap(addr: adev->rmmio);
4363	adev->rmmio = NULL;
4364	if (adev->mman.aper_base_kaddr)
4365	iounmap(addr: adev->mman.aper_base_kaddr);
4366	adev->mman.aper_base_kaddr = NULL;
4367
4368	/* Memory manager related */
4369	if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4370	arch_phys_wc_del(handle: adev->gmc.vram_mtrr);
4371	arch_io_free_memtype_wc(start: adev->gmc.aper_base, size: adev->gmc.aper_size);
4372	}
4373	}
4374
4375	/**
4376	* amdgpu_device_fini_hw - tear down the driver
4377	*
4378	* @adev: amdgpu_device pointer
4379	*
4380	* Tear down the driver info (all asics).
4381	* Called at driver shutdown.
4382	*/
4383	void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4384	{
4385	dev_info(adev->dev, "amdgpu: finishing device.\n");
4386	flush_delayed_work(dwork: &adev->delayed_init_work);
4387	adev->shutdown = true;
4388
4389	/* make sure IB test finished before entering exclusive mode
4390	* to avoid preemption on IB test
4391	*/
4392	if (amdgpu_sriov_vf(adev)) {
4393	amdgpu_virt_request_full_gpu(adev, init: false);
4394	amdgpu_virt_fini_data_exchange(adev);
4395	}
4396
4397	/* disable all interrupts */
4398	amdgpu_irq_disable_all(adev);
4399	if (adev->mode_info.mode_config_initialized) {
4400	if (!drm_drv_uses_atomic_modeset(dev: adev_to_drm(adev)))
4401	drm_helper_force_disable_all(dev: adev_to_drm(adev));
4402	else
4403	drm_atomic_helper_shutdown(dev: adev_to_drm(adev));
4404	}
4405	amdgpu_fence_driver_hw_fini(adev);
4406
4407	if (adev->mman.initialized)
4408	drain_workqueue(wq: adev->mman.bdev.wq);
4409
4410	if (adev->pm.sysfs_initialized)
4411	amdgpu_pm_sysfs_fini(adev);
4412	if (adev->ucode_sysfs_en)
4413	amdgpu_ucode_sysfs_fini(adev);
4414	sysfs_remove_files(kobj: &adev->dev->kobj, attr: amdgpu_dev_attributes);
4415	amdgpu_fru_sysfs_fini(adev);
4416
4417	amdgpu_reg_state_sysfs_fini(adev);
4418
4419	/* disable ras feature must before hw fini */
4420	amdgpu_ras_pre_fini(adev);
4421
4422	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
4423
4424	amdgpu_device_ip_fini_early(adev);
4425
4426	amdgpu_irq_fini_hw(adev);
4427
4428	if (adev->mman.initialized)
4429	ttm_device_clear_dma_mappings(bdev: &adev->mman.bdev);
4430
4431	amdgpu_gart_dummy_page_fini(adev);
4432
4433	if (drm_dev_is_unplugged(dev: adev_to_drm(adev)))
4434	amdgpu_device_unmap_mmio(adev);
4435
4436	}
4437
4438	void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4439	{
4440	int idx;
4441	bool px;
4442
4443	amdgpu_fence_driver_sw_fini(adev);
4444	amdgpu_device_ip_fini(adev);
4445	amdgpu_ucode_release(fw: &adev->firmware.gpu_info_fw);
4446	adev->accel_working = false;
4447	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4448
4449	amdgpu_reset_fini(adev);
4450
4451	/* free i2c buses */
4452	if (!amdgpu_device_has_dc_support(adev))
4453	amdgpu_i2c_fini(adev);
4454
4455	if (amdgpu_emu_mode != 1)
4456	amdgpu_atombios_fini(adev);
4457
4458	kfree(objp: adev->bios);
4459	adev->bios = NULL;
4460
4461	kfree(objp: adev->fru_info);
4462	adev->fru_info = NULL;
4463
4464	px = amdgpu_device_supports_px(dev: adev_to_drm(adev));
4465
4466	if (px || (!dev_is_removable(dev: &adev->pdev->dev) &&
4467	apple_gmux_detect(NULL, NULL)))
4468	vga_switcheroo_unregister_client(dev: adev->pdev);
4469
4470	if (px)
4471	vga_switcheroo_fini_domain_pm_ops(dev: adev->dev);
4472
4473	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4474	vga_client_unregister(pdev: adev->pdev);
4475
4476	if (drm_dev_enter(dev: adev_to_drm(adev), idx: &idx)) {
4477
4478	iounmap(addr: adev->rmmio);
4479	adev->rmmio = NULL;
4480	amdgpu_doorbell_fini(adev);
4481	drm_dev_exit(idx);
4482	}
4483
4484	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4485	amdgpu_pmu_fini(adev);
4486	if (adev->mman.discovery_bin)
4487	amdgpu_discovery_fini(adev);
4488
4489	amdgpu_reset_put_reset_domain(domain: adev->reset_domain);
4490	adev->reset_domain = NULL;
4491
4492	kfree(objp: adev->pci_state);
4493
4494	}
4495
4496	/**
4497	* amdgpu_device_evict_resources - evict device resources
4498	* @adev: amdgpu device object
4499	*
4500	* Evicts all ttm device resources(vram BOs, gart table) from the lru list
4501	* of the vram memory type. Mainly used for evicting device resources
4502	* at suspend time.
4503	*
4504	*/
4505	static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4506	{
4507	int ret;
4508
4509	/* No need to evict vram on APUs for suspend to ram or s2idle */
4510	if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4511	return 0;
4512
4513	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4514	if (ret)
4515	DRM_WARN("evicting device resources failed\n");
4516	return ret;
4517	}
4518
4519	/*
4520	* Suspend & resume.
4521	*/
4522	/**
4523	* amdgpu_device_prepare - prepare for device suspend
4524	*
4525	* @dev: drm dev pointer
4526	*
4527	* Prepare to put the hw in the suspend state (all asics).
4528	* Returns 0 for success or an error on failure.
4529	* Called at driver suspend.
4530	*/
4531	int amdgpu_device_prepare(struct drm_device *dev)
4532	{
4533	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
4534	int i, r;
4535
4536	amdgpu_choose_low_power_state(adev);
4537
4538	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4539	return 0;
4540
4541	/* Evict the majority of BOs before starting suspend sequence */
4542	r = amdgpu_device_evict_resources(adev);
4543	if (r)
4544	goto unprepare;
4545
4546	flush_delayed_work(dwork: &adev->gfx.gfx_off_delay_work);
4547
4548	for (i = 0; i < adev->num_ip_blocks; i++) {
4549	if (!adev->ip_blocks[i].status.valid)
4550	continue;
4551	if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4552	continue;
4553	r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4554	if (r)
4555	goto unprepare;
4556	}
4557
4558	return 0;
4559
4560	unprepare:
4561	adev->in_s0ix = adev->in_s3 = false;
4562
4563	return r;
4564	}
4565
4566	/**
4567	* amdgpu_device_suspend - initiate device suspend
4568	*
4569	* @dev: drm dev pointer
4570	* @fbcon : notify the fbdev of suspend
4571	*
4572	* Puts the hw in the suspend state (all asics).
4573	* Returns 0 for success or an error on failure.
4574	* Called at driver suspend.
4575	*/
4576	int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4577	{
4578	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
4579	int r = 0;
4580
4581	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4582	return 0;
4583
4584	adev->in_suspend = true;
4585
4586	if (amdgpu_sriov_vf(adev)) {
4587	amdgpu_virt_fini_data_exchange(adev);
4588	r = amdgpu_virt_request_full_gpu(adev, init: false);
4589	if (r)
4590	return r;
4591	}
4592
4593	if (amdgpu_acpi_smart_shift_update(dev, ss_state: AMDGPU_SS_DEV_D3))
4594	DRM_WARN("smart shift update failed\n");
4595
4596	if (fbcon)
4597	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev)->fb_helper, suspend: true);
4598
4599	cancel_delayed_work_sync(dwork: &adev->delayed_init_work);
4600
4601	amdgpu_ras_suspend(adev);
4602
4603	amdgpu_device_ip_suspend_phase1(adev);
4604
4605	if (!adev->in_s0ix)
4606	amdgpu_amdkfd_suspend(adev, run_pm: adev->in_runpm);
4607
4608	r = amdgpu_device_evict_resources(adev);
4609	if (r)
4610	return r;
4611
4612	amdgpu_ttm_set_buffer_funcs_status(adev, enable: false);
4613
4614	amdgpu_fence_driver_hw_fini(adev);
4615
4616	amdgpu_device_ip_suspend_phase2(adev);
4617
4618	if (amdgpu_sriov_vf(adev))
4619	amdgpu_virt_release_full_gpu(adev, init: false);
4620
4621	r = amdgpu_dpm_notify_rlc_state(adev, en: false);
4622	if (r)
4623	return r;
4624
4625	return 0;
4626	}
4627
4628	/**
4629	* amdgpu_device_resume - initiate device resume
4630	*
4631	* @dev: drm dev pointer
4632	* @fbcon : notify the fbdev of resume
4633	*
4634	* Bring the hw back to operating state (all asics).
4635	* Returns 0 for success or an error on failure.
4636	* Called at driver resume.
4637	*/
4638	int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4639	{
4640	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
4641	int r = 0;
4642
4643	if (amdgpu_sriov_vf(adev)) {
4644	r = amdgpu_virt_request_full_gpu(adev, init: true);
4645	if (r)
4646	return r;
4647	}
4648
4649	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4650	return 0;
4651
4652	if (adev->in_s0ix)
4653	amdgpu_dpm_gfx_state_change(adev, state: sGpuChangeState_D0Entry);
4654
4655	/* post card */
4656	if (amdgpu_device_need_post(adev)) {
4657	r = amdgpu_device_asic_init(adev);
4658	if (r)
4659	dev_err(adev->dev, "amdgpu asic init failed\n");
4660	}
4661
4662	r = amdgpu_device_ip_resume(adev);
4663
4664	if (r) {
4665	dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4666	goto exit;
4667	}
4668	amdgpu_fence_driver_hw_init(adev);
4669
4670	if (!adev->in_s0ix) {
4671	r = amdgpu_amdkfd_resume(adev, run_pm: adev->in_runpm);
4672	if (r)
4673	goto exit;
4674	}
4675
4676	r = amdgpu_device_ip_late_init(adev);
4677	if (r)
4678	goto exit;
4679
4680	queue_delayed_work(wq: system_wq, dwork: &adev->delayed_init_work,
4681	delay: msecs_to_jiffies(AMDGPU_RESUME_MS));
4682	exit:
4683	if (amdgpu_sriov_vf(adev)) {
4684	amdgpu_virt_init_data_exchange(adev);
4685	amdgpu_virt_release_full_gpu(adev, init: true);
4686	}
4687
4688	if (r)
4689	return r;
4690
4691	/* Make sure IB tests flushed */
4692	flush_delayed_work(dwork: &adev->delayed_init_work);
4693
4694	if (fbcon)
4695	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev)->fb_helper, suspend: false);
4696
4697	amdgpu_ras_resume(adev);
4698
4699	if (adev->mode_info.num_crtc) {
4700	/*
4701	* Most of the connector probing functions try to acquire runtime pm
4702	* refs to ensure that the GPU is powered on when connector polling is
4703	* performed. Since we're calling this from a runtime PM callback,
4704	* trying to acquire rpm refs will cause us to deadlock.
4705	*
4706	* Since we're guaranteed to be holding the rpm lock, it's safe to
4707	* temporarily disable the rpm helpers so this doesn't deadlock us.
4708	*/
4709	#ifdef CONFIG_PM
4710	dev->dev->power.disable_depth++;
4711	#endif
4712	if (!adev->dc_enabled)
4713	drm_helper_hpd_irq_event(dev);
4714	else
4715	drm_kms_helper_hotplug_event(dev);
4716	#ifdef CONFIG_PM
4717	dev->dev->power.disable_depth--;
4718	#endif
4719	}
4720	adev->in_suspend = false;
4721
4722	if (adev->enable_mes)
4723	amdgpu_mes_self_test(adev);
4724
4725	if (amdgpu_acpi_smart_shift_update(dev, ss_state: AMDGPU_SS_DEV_D0))
4726	DRM_WARN("smart shift update failed\n");
4727
4728	return 0;
4729	}
4730
4731	/**
4732	* amdgpu_device_ip_check_soft_reset - did soft reset succeed
4733	*
4734	* @adev: amdgpu_device pointer
4735	*
4736	* The list of all the hardware IPs that make up the asic is walked and
4737	* the check_soft_reset callbacks are run. check_soft_reset determines
4738	* if the asic is still hung or not.
4739	* Returns true if any of the IPs are still in a hung state, false if not.
4740	*/
4741	static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4742	{
4743	int i;
4744	bool asic_hang = false;
4745
4746	if (amdgpu_sriov_vf(adev))
4747	return true;
4748
4749	if (amdgpu_asic_need_full_reset(adev))
4750	return true;
4751
4752	for (i = 0; i < adev->num_ip_blocks; i++) {
4753	if (!adev->ip_blocks[i].status.valid)
4754	continue;
4755	if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4756	adev->ip_blocks[i].status.hang =
4757	adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4758	if (adev->ip_blocks[i].status.hang) {
4759	dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4760	asic_hang = true;
4761	}
4762	}
4763	return asic_hang;
4764	}
4765
4766	/**
4767	* amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4768	*
4769	* @adev: amdgpu_device pointer
4770	*
4771	* The list of all the hardware IPs that make up the asic is walked and the
4772	* pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
4773	* handles any IP specific hardware or software state changes that are
4774	* necessary for a soft reset to succeed.
4775	* Returns 0 on success, negative error code on failure.
4776	*/
4777	static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4778	{
4779	int i, r = 0;
4780
4781	for (i = 0; i < adev->num_ip_blocks; i++) {
4782	if (!adev->ip_blocks[i].status.valid)
4783	continue;
4784	if (adev->ip_blocks[i].status.hang &&
4785	adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4786	r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4787	if (r)
4788	return r;
4789	}
4790	}
4791
4792	return 0;
4793	}
4794
4795	/**
4796	* amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4797	*
4798	* @adev: amdgpu_device pointer
4799	*
4800	* Some hardware IPs cannot be soft reset. If they are hung, a full gpu
4801	* reset is necessary to recover.
4802	* Returns true if a full asic reset is required, false if not.
4803	*/
4804	static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4805	{
4806	int i;
4807
4808	if (amdgpu_asic_need_full_reset(adev))
4809	return true;
4810
4811	for (i = 0; i < adev->num_ip_blocks; i++) {
4812	if (!adev->ip_blocks[i].status.valid)
4813	continue;
4814	if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4815	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4816	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4817	(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4818	adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4819	if (adev->ip_blocks[i].status.hang) {
4820	dev_info(adev->dev, "Some block need full reset!\n");
4821	return true;
4822	}
4823	}
4824	}
4825	return false;
4826	}
4827
4828	/**
4829	* amdgpu_device_ip_soft_reset - do a soft reset
4830	*
4831	* @adev: amdgpu_device pointer
4832	*
4833	* The list of all the hardware IPs that make up the asic is walked and the
4834	* soft_reset callbacks are run if the block is hung. soft_reset handles any
4835	* IP specific hardware or software state changes that are necessary to soft
4836	* reset the IP.
4837	* Returns 0 on success, negative error code on failure.
4838	*/
4839	static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4840	{
4841	int i, r = 0;
4842
4843	for (i = 0; i < adev->num_ip_blocks; i++) {
4844	if (!adev->ip_blocks[i].status.valid)
4845	continue;
4846	if (adev->ip_blocks[i].status.hang &&
4847	adev->ip_blocks[i].version->funcs->soft_reset) {
4848	r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4849	if (r)
4850	return r;
4851	}
4852	}
4853
4854	return 0;
4855	}
4856
4857	/**
4858	* amdgpu_device_ip_post_soft_reset - clean up from soft reset
4859	*
4860	* @adev: amdgpu_device pointer
4861	*
4862	* The list of all the hardware IPs that make up the asic is walked and the
4863	* post_soft_reset callbacks are run if the asic was hung. post_soft_reset
4864	* handles any IP specific hardware or software state changes that are
4865	* necessary after the IP has been soft reset.
4866	* Returns 0 on success, negative error code on failure.
4867	*/
4868	static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4869	{
4870	int i, r = 0;
4871
4872	for (i = 0; i < adev->num_ip_blocks; i++) {
4873	if (!adev->ip_blocks[i].status.valid)
4874	continue;
4875	if (adev->ip_blocks[i].status.hang &&
4876	adev->ip_blocks[i].version->funcs->post_soft_reset)
4877	r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4878	if (r)
4879	return r;
4880	}
4881
4882	return 0;
4883	}
4884
4885	/**
4886	* amdgpu_device_recover_vram - Recover some VRAM contents
4887	*
4888	* @adev: amdgpu_device pointer
4889	*
4890	* Restores the contents of VRAM buffers from the shadows in GTT. Used to
4891	* restore things like GPUVM page tables after a GPU reset where
4892	* the contents of VRAM might be lost.
4893	*
4894	* Returns:
4895	* 0 on success, negative error code on failure.
4896	*/
4897	static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4898	{
4899	struct dma_fence *fence = NULL, *next = NULL;
4900	struct amdgpu_bo *shadow;
4901	struct amdgpu_bo_vm *vmbo;
4902	long r = 1, tmo;
4903
4904	if (amdgpu_sriov_runtime(adev))
4905	tmo = msecs_to_jiffies(m: 8000);
4906	else
4907	tmo = msecs_to_jiffies(m: 100);
4908
4909	dev_info(adev->dev, "recover vram bo from shadow start\n");
4910	mutex_lock(&adev->shadow_list_lock);
4911	list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4912	/* If vm is compute context or adev is APU, shadow will be NULL */
4913	if (!vmbo->shadow)
4914	continue;
4915	shadow = vmbo->shadow;
4916
4917	/* No need to recover an evicted BO */
4918	if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4919	shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4920	shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4921	continue;
4922
4923	r = amdgpu_bo_restore_shadow(shadow, fence: &next);
4924	if (r)
4925	break;
4926
4927	if (fence) {
4928	tmo = dma_fence_wait_timeout(fence, intr: false, timeout: tmo);
4929	dma_fence_put(fence);
4930	fence = next;
4931	if (tmo == 0) {
4932	r = -ETIMEDOUT;
4933	break;
4934	} else if (tmo < 0) {
4935	r = tmo;
4936	break;
4937	}
4938	} else {
4939	fence = next;
4940	}
4941	}
4942	mutex_unlock(lock: &adev->shadow_list_lock);
4943
4944	if (fence)
4945	tmo = dma_fence_wait_timeout(fence, intr: false, timeout: tmo);
4946	dma_fence_put(fence);
4947
4948	if (r < 0 || tmo <= 0) {
4949	dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4950	return -EIO;
4951	}
4952
4953	dev_info(adev->dev, "recover vram bo from shadow done\n");
4954	return 0;
4955	}
4956
4957
4958	/**
4959	* amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4960	*
4961	* @adev: amdgpu_device pointer
4962	* @from_hypervisor: request from hypervisor
4963	*
4964	* do VF FLR and reinitialize Asic
4965	* return 0 means succeeded otherwise failed
4966	*/
4967	static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4968	bool from_hypervisor)
4969	{
4970	int r;
4971	struct amdgpu_hive_info *hive = NULL;
4972	int retry_limit = 0;
4973
4974	retry:
4975	amdgpu_amdkfd_pre_reset(adev);
4976
4977	if (from_hypervisor)
4978	r = amdgpu_virt_request_full_gpu(adev, init: true);
4979	else
4980	r = amdgpu_virt_reset_gpu(adev);
4981	if (r)
4982	return r;
4983	amdgpu_irq_gpu_reset_resume_helper(adev);
4984
4985	/* some sw clean up VF needs to do before recover */
4986	amdgpu_virt_post_reset(adev);
4987
4988	/* Resume IP prior to SMC */
4989	r = amdgpu_device_ip_reinit_early_sriov(adev);
4990	if (r)
4991	goto error;
4992
4993	amdgpu_virt_init_data_exchange(adev);
4994
4995	r = amdgpu_device_fw_loading(adev);
4996	if (r)
4997	return r;
4998
4999	/* now we are okay to resume SMC/CP/SDMA */
5000	r = amdgpu_device_ip_reinit_late_sriov(adev);
5001	if (r)
5002	goto error;
5003
5004	hive = amdgpu_get_xgmi_hive(adev);
5005	/* Update PSP FW topology after reset */
5006	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5007	r = amdgpu_xgmi_update_topology(hive, adev);
5008
5009	if (hive)
5010	amdgpu_put_xgmi_hive(hive);
5011
5012	if (!r) {
5013	r = amdgpu_ib_ring_tests(adev);
5014
5015	amdgpu_amdkfd_post_reset(adev);
5016	}
5017
5018	error:
5019	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
5020	amdgpu_inc_vram_lost(adev);
5021	r = amdgpu_device_recover_vram(adev);
5022	}
5023	amdgpu_virt_release_full_gpu(adev, init: true);
5024
5025	if (AMDGPU_RETRY_SRIOV_RESET(r)) {
5026	if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
5027	retry_limit++;
5028	goto retry;
5029	} else
5030	DRM_ERROR("GPU reset retry is beyond the retry limit\n");
5031	}
5032
5033	return r;
5034	}
5035
5036	/**
5037	* amdgpu_device_has_job_running - check if there is any job in mirror list
5038	*
5039	* @adev: amdgpu_device pointer
5040	*
5041	* check if there is any job in mirror list
5042	*/
5043	bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5044	{
5045	int i;
5046	struct drm_sched_job *job;
5047
5048	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5049	struct amdgpu_ring *ring = adev->rings[i];
5050
5051	if (!amdgpu_ring_sched_ready(ring))
5052	continue;
5053
5054	spin_lock(lock: &ring->sched.job_list_lock);
5055	job = list_first_entry_or_null(&ring->sched.pending_list,
5056	struct drm_sched_job, list);
5057	spin_unlock(lock: &ring->sched.job_list_lock);
5058	if (job)
5059	return true;
5060	}
5061	return false;
5062	}
5063
5064	/**
5065	* amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5066	*
5067	* @adev: amdgpu_device pointer
5068	*
5069	* Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5070	* a hung GPU.
5071	*/
5072	bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5073	{
5074
5075	if (amdgpu_gpu_recovery == 0)
5076	goto disabled;
5077
5078	/* Skip soft reset check in fatal error mode */
5079	if (!amdgpu_ras_is_poison_mode_supported(adev))
5080	return true;
5081
5082	if (amdgpu_sriov_vf(adev))
5083	return true;
5084
5085	if (amdgpu_gpu_recovery == -1) {
5086	switch (adev->asic_type) {
5087	#ifdef CONFIG_DRM_AMDGPU_SI
5088	case CHIP_VERDE:
5089	case CHIP_TAHITI:
5090	case CHIP_PITCAIRN:
5091	case CHIP_OLAND:
5092	case CHIP_HAINAN:
5093	#endif
5094	#ifdef CONFIG_DRM_AMDGPU_CIK
5095	case CHIP_KAVERI:
5096	case CHIP_KABINI:
5097	case CHIP_MULLINS:
5098	#endif
5099	case CHIP_CARRIZO:
5100	case CHIP_STONEY:
5101	case CHIP_CYAN_SKILLFISH:
5102	goto disabled;
5103	default:
5104	break;
5105	}
5106	}
5107
5108	return true;
5109
5110	disabled:
5111	dev_info(adev->dev, "GPU recovery disabled.\n");
5112	return false;
5113	}
5114
5115	int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5116	{
5117	u32 i;
5118	int ret = 0;
5119
5120	amdgpu_atombios_scratch_regs_engine_hung(adev, hung: true);
5121
5122	dev_info(adev->dev, "GPU mode1 reset\n");
5123
5124	/* disable BM */
5125	pci_clear_master(dev: adev->pdev);
5126
5127	amdgpu_device_cache_pci_state(pdev: adev->pdev);
5128
5129	if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5130	dev_info(adev->dev, "GPU smu mode1 reset\n");
5131	ret = amdgpu_dpm_mode1_reset(adev);
5132	} else {
5133	dev_info(adev->dev, "GPU psp mode1 reset\n");
5134	ret = psp_gpu_reset(adev);
5135	}
5136
5137	if (ret)
5138	goto mode1_reset_failed;
5139
5140	amdgpu_device_load_pci_state(pdev: adev->pdev);
5141	ret = amdgpu_psp_wait_for_bootloader(adev);
5142	if (ret)
5143	goto mode1_reset_failed;
5144
5145	/* wait for asic to come out of reset */
5146	for (i = 0; i < adev->usec_timeout; i++) {
5147	u32 memsize = adev->nbio.funcs->get_memsize(adev);
5148
5149	if (memsize != 0xffffffff)
5150	break;
5151	udelay(1);
5152	}
5153
5154	if (i >= adev->usec_timeout) {
5155	ret = -ETIMEDOUT;
5156	goto mode1_reset_failed;
5157	}
5158
5159	amdgpu_atombios_scratch_regs_engine_hung(adev, hung: false);
5160
5161	return 0;
5162
5163	mode1_reset_failed:
5164	dev_err(adev->dev, "GPU mode1 reset failed\n");
5165	return ret;
5166	}
5167
5168	int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5169	struct amdgpu_reset_context *reset_context)
5170	{
5171	int i, r = 0;
5172	struct amdgpu_job *job = NULL;
5173	bool need_full_reset =
5174	test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5175
5176	if (reset_context->reset_req_dev == adev)
5177	job = reset_context->job;
5178
5179	if (amdgpu_sriov_vf(adev)) {
5180	/* stop the data exchange thread */
5181	amdgpu_virt_fini_data_exchange(adev);
5182	}
5183
5184	amdgpu_fence_driver_isr_toggle(adev, stop: true);
5185
5186	/* block all schedulers and reset given job's ring */
5187	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5188	struct amdgpu_ring *ring = adev->rings[i];
5189
5190	if (!amdgpu_ring_sched_ready(ring))
5191	continue;
5192
5193	/* Clear job fence from fence drv to avoid force_completion
5194	* leave NULL and vm flush fence in fence drv
5195	*/
5196	amdgpu_fence_driver_clear_job_fences(ring);
5197
5198	/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5199	amdgpu_fence_driver_force_completion(ring);
5200	}
5201
5202	amdgpu_fence_driver_isr_toggle(adev, stop: false);
5203
5204	if (job && job->vm)
5205	drm_sched_increase_karma(bad: &job->base);
5206
5207	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5208	/* If reset handler not implemented, continue; otherwise return */
5209	if (r == -EOPNOTSUPP)
5210	r = 0;
5211	else
5212	return r;
5213
5214	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5215	if (!amdgpu_sriov_vf(adev)) {
5216
5217	if (!need_full_reset)
5218	need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5219
5220	if (!need_full_reset && amdgpu_gpu_recovery &&
5221	amdgpu_device_ip_check_soft_reset(adev)) {
5222	amdgpu_device_ip_pre_soft_reset(adev);
5223	r = amdgpu_device_ip_soft_reset(adev);
5224	amdgpu_device_ip_post_soft_reset(adev);
5225	if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5226	dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5227	need_full_reset = true;
5228	}
5229	}
5230
5231	if (need_full_reset)
5232	r = amdgpu_device_ip_suspend(adev);
5233	if (need_full_reset)
5234	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context->flags);
5235	else
5236	clear_bit(nr: AMDGPU_NEED_FULL_RESET,
5237	addr: &reset_context->flags);
5238	}
5239
5240	return r;
5241	}
5242
5243	static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5244	{
5245	int i;
5246
5247	lockdep_assert_held(&adev->reset_domain->sem);
5248
5249	for (i = 0; i < adev->reset_info.num_regs; i++) {
5250	adev->reset_info.reset_dump_reg_value[i] =
5251	RREG32(adev->reset_info.reset_dump_reg_list[i]);
5252
5253	trace_amdgpu_reset_reg_dumps(address: adev->reset_info.reset_dump_reg_list[i],
5254	value: adev->reset_info.reset_dump_reg_value[i]);
5255	}
5256
5257	return 0;
5258	}
5259
5260	int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5261	struct amdgpu_reset_context *reset_context)
5262	{
5263	struct amdgpu_device *tmp_adev = NULL;
5264	bool need_full_reset, skip_hw_reset, vram_lost = false;
5265	int r = 0;
5266
5267	/* Try reset handler method first */
5268	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5269	reset_list);
5270	amdgpu_reset_reg_dumps(adev: tmp_adev);
5271
5272	reset_context->reset_device_list = device_list_handle;
5273	r = amdgpu_reset_perform_reset(adev: tmp_adev, reset_context);
5274	/* If reset handler not implemented, continue; otherwise return */
5275	if (r == -EOPNOTSUPP)
5276	r = 0;
5277	else
5278	return r;
5279
5280	/* Reset handler not implemented, use the default method */
5281	need_full_reset =
5282	test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5283	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5284
5285	/*
5286	* ASIC reset has to be done on all XGMI hive nodes ASAP
5287	* to allow proper links negotiation in FW (within 1 sec)
5288	*/
5289	if (!skip_hw_reset && need_full_reset) {
5290	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5291	/* For XGMI run all resets in parallel to speed up the process */
5292	if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5293	tmp_adev->gmc.xgmi.pending_reset = false;
5294	if (!queue_work(wq: system_unbound_wq, work: &tmp_adev->xgmi_reset_work))
5295	r = -EALREADY;
5296	} else
5297	r = amdgpu_asic_reset(tmp_adev);
5298
5299	if (r) {
5300	dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5301	r, adev_to_drm(tmp_adev)->unique);
5302	goto out;
5303	}
5304	}
5305
5306	/* For XGMI wait for all resets to complete before proceed */
5307	if (!r) {
5308	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5309	if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5310	flush_work(work: &tmp_adev->xgmi_reset_work);
5311	r = tmp_adev->asic_reset_res;
5312	if (r)
5313	break;
5314	}
5315	}
5316	}
5317	}
5318
5319	if (!r && amdgpu_ras_intr_triggered()) {
5320	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5321	amdgpu_ras_reset_error_count(adev: tmp_adev, block: AMDGPU_RAS_BLOCK__MMHUB);
5322	}
5323
5324	amdgpu_ras_intr_cleared();
5325	}
5326
5327	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5328	if (need_full_reset) {
5329	/* post card */
5330	amdgpu_ras_set_fed(adev: tmp_adev, status: false);
5331	r = amdgpu_device_asic_init(adev: tmp_adev);
5332	if (r) {
5333	dev_warn(tmp_adev->dev, "asic atom init failed!");
5334	} else {
5335	dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5336
5337	r = amdgpu_device_ip_resume_phase1(adev: tmp_adev);
5338	if (r)
5339	goto out;
5340
5341	vram_lost = amdgpu_device_check_vram_lost(adev: tmp_adev);
5342
5343	amdgpu_coredump(adev: tmp_adev, vram_lost, reset_context);
5344
5345	if (vram_lost) {
5346	DRM_INFO("VRAM is lost due to GPU reset!\n");
5347	amdgpu_inc_vram_lost(tmp_adev);
5348	}
5349
5350	r = amdgpu_device_fw_loading(adev: tmp_adev);
5351	if (r)
5352	return r;
5353
5354	r = amdgpu_xcp_restore_partition_mode(
5355	xcp_mgr: tmp_adev->xcp_mgr);
5356	if (r)
5357	goto out;
5358
5359	r = amdgpu_device_ip_resume_phase2(adev: tmp_adev);
5360	if (r)
5361	goto out;
5362
5363	if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5364	amdgpu_ttm_set_buffer_funcs_status(adev: tmp_adev, enable: true);
5365
5366	if (vram_lost)
5367	amdgpu_device_fill_reset_magic(adev: tmp_adev);
5368
5369	/*
5370	* Add this ASIC as tracked as reset was already
5371	* complete successfully.
5372	*/
5373	amdgpu_register_gpu_instance(adev: tmp_adev);
5374
5375	if (!reset_context->hive &&
5376	tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5377	amdgpu_xgmi_add_device(adev: tmp_adev);
5378
5379	r = amdgpu_device_ip_late_init(adev: tmp_adev);
5380	if (r)
5381	goto out;
5382
5383	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev: tmp_adev)->fb_helper, suspend: false);
5384
5385	/*
5386	* The GPU enters bad state once faulty pages
5387	* by ECC has reached the threshold, and ras
5388	* recovery is scheduled next. So add one check
5389	* here to break recovery if it indeed exceeds
5390	* bad page threshold, and remind user to
5391	* retire this GPU or setting one bigger
5392	* bad_page_threshold value to fix this once
5393	* probing driver again.
5394	*/
5395	if (!amdgpu_ras_eeprom_check_err_threshold(adev: tmp_adev)) {
5396	/* must succeed. */
5397	amdgpu_ras_resume(adev: tmp_adev);
5398	} else {
5399	r = -EINVAL;
5400	goto out;
5401	}
5402
5403	/* Update PSP FW topology after reset */
5404	if (reset_context->hive &&
5405	tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5406	r = amdgpu_xgmi_update_topology(
5407	hive: reset_context->hive, adev: tmp_adev);
5408	}
5409	}
5410
5411	out:
5412	if (!r) {
5413	amdgpu_irq_gpu_reset_resume_helper(adev: tmp_adev);
5414	r = amdgpu_ib_ring_tests(adev: tmp_adev);
5415	if (r) {
5416	dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5417	need_full_reset = true;
5418	r = -EAGAIN;
5419	goto end;
5420	}
5421	}
5422
5423	if (!r)
5424	r = amdgpu_device_recover_vram(adev: tmp_adev);
5425	else
5426	tmp_adev->asic_reset_res = r;
5427	}
5428
5429	end:
5430	if (need_full_reset)
5431	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context->flags);
5432	else
5433	clear_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context->flags);
5434	return r;
5435	}
5436
5437	static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5438	{
5439
5440	switch (amdgpu_asic_reset_method(adev)) {
5441	case AMD_RESET_METHOD_MODE1:
5442	adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5443	break;
5444	case AMD_RESET_METHOD_MODE2:
5445	adev->mp1_state = PP_MP1_STATE_RESET;
5446	break;
5447	default:
5448	adev->mp1_state = PP_MP1_STATE_NONE;
5449	break;
5450	}
5451	}
5452
5453	static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5454	{
5455	amdgpu_vf_error_trans_all(adev);
5456	adev->mp1_state = PP_MP1_STATE_NONE;
5457	}
5458
5459	static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5460	{
5461	struct pci_dev *p = NULL;
5462
5463	p = pci_get_domain_bus_and_slot(domain: pci_domain_nr(bus: adev->pdev->bus),
5464	bus: adev->pdev->bus->number, devfn: 1);
5465	if (p) {
5466	pm_runtime_enable(dev: &(p->dev));
5467	pm_runtime_resume(dev: &(p->dev));
5468	}
5469
5470	pci_dev_put(dev: p);
5471	}
5472
5473	static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5474	{
5475	enum amd_reset_method reset_method;
5476	struct pci_dev *p = NULL;
5477	u64 expires;
5478
5479	/*
5480	* For now, only BACO and mode1 reset are confirmed
5481	* to suffer the audio issue without proper suspended.
5482	*/
5483	reset_method = amdgpu_asic_reset_method(adev);
5484	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5485	(reset_method != AMD_RESET_METHOD_MODE1))
5486	return -EINVAL;
5487
5488	p = pci_get_domain_bus_and_slot(domain: pci_domain_nr(bus: adev->pdev->bus),
5489	bus: adev->pdev->bus->number, devfn: 1);
5490	if (!p)
5491	return -ENODEV;
5492
5493	expires = pm_runtime_autosuspend_expiration(dev: &(p->dev));
5494	if (!expires)
5495	/*
5496	* If we cannot get the audio device autosuspend delay,
5497	* a fixed 4S interval will be used. Considering 3S is
5498	* the audio controller default autosuspend delay setting.
5499	* 4S used here is guaranteed to cover that.
5500	*/
5501	expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5502
5503	while (!pm_runtime_status_suspended(dev: &(p->dev))) {
5504	if (!pm_runtime_suspend(dev: &(p->dev)))
5505	break;
5506
5507	if (expires < ktime_get_mono_fast_ns()) {
5508	dev_warn(adev->dev, "failed to suspend display audio\n");
5509	pci_dev_put(dev: p);
5510	/* TODO: abort the succeeding gpu reset? */
5511	return -ETIMEDOUT;
5512	}
5513	}
5514
5515	pm_runtime_disable(dev: &(p->dev));
5516
5517	pci_dev_put(dev: p);
5518	return 0;
5519	}
5520
5521	static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5522	{
5523	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5524
5525	#if defined(CONFIG_DEBUG_FS)
5526	if (!amdgpu_sriov_vf(adev))
5527	cancel_work(work: &adev->reset_work);
5528	#endif
5529
5530	if (adev->kfd.dev)
5531	cancel_work(work: &adev->kfd.reset_work);
5532
5533	if (amdgpu_sriov_vf(adev))
5534	cancel_work(work: &adev->virt.flr_work);
5535
5536	if (con && adev->ras_enabled)
5537	cancel_work(work: &con->recovery_work);
5538
5539	}
5540
5541	/**
5542	* amdgpu_device_gpu_recover - reset the asic and recover scheduler
5543	*
5544	* @adev: amdgpu_device pointer
5545	* @job: which job trigger hang
5546	* @reset_context: amdgpu reset context pointer
5547	*
5548	* Attempt to reset the GPU if it has hung (all asics).
5549	* Attempt to do soft-reset or full-reset and reinitialize Asic
5550	* Returns 0 for success or an error on failure.
5551	*/
5552
5553	int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5554	struct amdgpu_job *job,
5555	struct amdgpu_reset_context *reset_context)
5556	{
5557	struct list_head device_list, *device_list_handle = NULL;
5558	bool job_signaled = false;
5559	struct amdgpu_hive_info *hive = NULL;
5560	struct amdgpu_device *tmp_adev = NULL;
5561	int i, r = 0;
5562	bool need_emergency_restart = false;
5563	bool audio_suspended = false;
5564
5565	/*
5566	* Special case: RAS triggered and full reset isn't supported
5567	*/
5568	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5569
5570	/*
5571	* Flush RAM to disk so that after reboot
5572	* the user can read log and see why the system rebooted.
5573	*/
5574	if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5575	amdgpu_ras_get_context(adev)->reboot) {
5576	DRM_WARN("Emergency reboot.");
5577
5578	ksys_sync_helper();
5579	emergency_restart();
5580	}
5581
5582	dev_info(adev->dev, "GPU %s begin!\n",
5583	need_emergency_restart ? "jobs stop":"reset");
5584
5585	if (!amdgpu_sriov_vf(adev))
5586	hive = amdgpu_get_xgmi_hive(adev);
5587	if (hive)
5588	mutex_lock(&hive->hive_lock);
5589
5590	reset_context->job = job;
5591	reset_context->hive = hive;
5592	/*
5593	* Build list of devices to reset.
5594	* In case we are in XGMI hive mode, resort the device list
5595	* to put adev in the 1st position.
5596	*/
5597	INIT_LIST_HEAD(list: &device_list);
5598	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5599	list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5600	list_add_tail(new: &tmp_adev->reset_list, head: &device_list);
5601	if (adev->shutdown)
5602	tmp_adev->shutdown = true;
5603	}
5604	if (!list_is_first(list: &adev->reset_list, head: &device_list))
5605	list_rotate_to_front(list: &adev->reset_list, head: &device_list);
5606	device_list_handle = &device_list;
5607	} else {
5608	list_add_tail(new: &adev->reset_list, head: &device_list);
5609	device_list_handle = &device_list;
5610	}
5611
5612	/* We need to lock reset domain only once both for XGMI and single device */
5613	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5614	reset_list);
5615	amdgpu_device_lock_reset_domain(reset_domain: tmp_adev->reset_domain);
5616
5617	/* block all schedulers and reset given job's ring */
5618	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5619
5620	amdgpu_device_set_mp1_state(adev: tmp_adev);
5621
5622	/*
5623	* Try to put the audio codec into suspend state
5624	* before gpu reset started.
5625	*
5626	* Due to the power domain of the graphics device
5627	* is shared with AZ power domain. Without this,
5628	* we may change the audio hardware from behind
5629	* the audio driver's back. That will trigger
5630	* some audio codec errors.
5631	*/
5632	if (!amdgpu_device_suspend_display_audio(adev: tmp_adev))
5633	audio_suspended = true;
5634
5635	amdgpu_ras_set_error_query_ready(adev: tmp_adev, ready: false);
5636
5637	cancel_delayed_work_sync(dwork: &tmp_adev->delayed_init_work);
5638
5639	if (!amdgpu_sriov_vf(tmp_adev))
5640	amdgpu_amdkfd_pre_reset(adev: tmp_adev);
5641
5642	/*
5643	* Mark these ASICs to be reseted as untracked first
5644	* And add them back after reset completed
5645	*/
5646	amdgpu_unregister_gpu_instance(adev: tmp_adev);
5647
5648	drm_fb_helper_set_suspend_unlocked(fb_helper: adev_to_drm(adev: tmp_adev)->fb_helper, suspend: true);
5649
5650	/* disable ras on ALL IPs */
5651	if (!need_emergency_restart &&
5652	amdgpu_device_ip_need_full_reset(adev: tmp_adev))
5653	amdgpu_ras_suspend(adev: tmp_adev);
5654
5655	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5656	struct amdgpu_ring *ring = tmp_adev->rings[i];
5657
5658	if (!amdgpu_ring_sched_ready(ring))
5659	continue;
5660
5661	drm_sched_stop(sched: &ring->sched, bad: job ? &job->base : NULL);
5662
5663	if (need_emergency_restart)
5664	amdgpu_job_stop_all_jobs_on_sched(sched: &ring->sched);
5665	}
5666	atomic_inc(v: &tmp_adev->gpu_reset_counter);
5667	}
5668
5669	if (need_emergency_restart)
5670	goto skip_sched_resume;
5671
5672	/*
5673	* Must check guilty signal here since after this point all old
5674	* HW fences are force signaled.
5675	*
5676	* job->base holds a reference to parent fence
5677	*/
5678	if (job && dma_fence_is_signaled(fence: &job->hw_fence)) {
5679	job_signaled = true;
5680	dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5681	goto skip_hw_reset;
5682	}
5683
5684	retry: /* Rest of adevs pre asic reset from XGMI hive. */
5685	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5686	r = amdgpu_device_pre_asic_reset(adev: tmp_adev, reset_context);
5687	/*TODO Should we stop ?*/
5688	if (r) {
5689	dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5690	r, adev_to_drm(tmp_adev)->unique);
5691	tmp_adev->asic_reset_res = r;
5692	}
5693
5694	/*
5695	* Drop all pending non scheduler resets. Scheduler resets
5696	* were already dropped during drm_sched_stop
5697	*/
5698	amdgpu_device_stop_pending_resets(adev: tmp_adev);
5699	}
5700
5701	/* Actual ASIC resets if needed.*/
5702	/* Host driver will handle XGMI hive reset for SRIOV */
5703	if (amdgpu_sriov_vf(adev)) {
5704	r = amdgpu_device_reset_sriov(adev, from_hypervisor: job ? false : true);
5705	if (r)
5706	adev->asic_reset_res = r;
5707
5708	/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5709	if (amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) ==
5710	IP_VERSION(9, 4, 2) ||
5711	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3) ||
5712	amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(11, 0, 3))
5713	amdgpu_ras_resume(adev);
5714	} else {
5715	r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5716	if (r && r == -EAGAIN)
5717	goto retry;
5718	}
5719
5720	skip_hw_reset:
5721
5722	/* Post ASIC reset for all devs .*/
5723	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5724
5725	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5726	struct amdgpu_ring *ring = tmp_adev->rings[i];
5727
5728	if (!amdgpu_ring_sched_ready(ring))
5729	continue;
5730
5731	drm_sched_start(sched: &ring->sched, full_recovery: true);
5732	}
5733
5734	if (!drm_drv_uses_atomic_modeset(dev: adev_to_drm(adev: tmp_adev)) && !job_signaled)
5735	drm_helper_resume_force_mode(dev: adev_to_drm(adev: tmp_adev));
5736
5737	if (tmp_adev->asic_reset_res)
5738	r = tmp_adev->asic_reset_res;
5739
5740	tmp_adev->asic_reset_res = 0;
5741
5742	if (r) {
5743	/* bad news, how to tell it to userspace ? */
5744	dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5745	amdgpu_vf_error_put(adev: tmp_adev, sub_error_code: AMDGIM_ERROR_VF_GPU_RESET_FAIL, error_flags: 0, error_data: r);
5746	} else {
5747	dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5748	if (amdgpu_acpi_smart_shift_update(dev: adev_to_drm(adev: tmp_adev), ss_state: AMDGPU_SS_DEV_D0))
5749	DRM_WARN("smart shift update failed\n");
5750	}
5751	}
5752
5753	skip_sched_resume:
5754	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5755	/* unlock kfd: SRIOV would do it separately */
5756	if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5757	amdgpu_amdkfd_post_reset(adev: tmp_adev);
5758
5759	/* kfd_post_reset will do nothing if kfd device is not initialized,
5760	* need to bring up kfd here if it's not be initialized before
5761	*/
5762	if (!adev->kfd.init_complete)
5763	amdgpu_amdkfd_device_init(adev);
5764
5765	if (audio_suspended)
5766	amdgpu_device_resume_display_audio(adev: tmp_adev);
5767
5768	amdgpu_device_unset_mp1_state(adev: tmp_adev);
5769
5770	amdgpu_ras_set_error_query_ready(adev: tmp_adev, ready: true);
5771	}
5772
5773	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5774	reset_list);
5775	amdgpu_device_unlock_reset_domain(reset_domain: tmp_adev->reset_domain);
5776
5777	if (hive) {
5778	mutex_unlock(lock: &hive->hive_lock);
5779	amdgpu_put_xgmi_hive(hive);
5780	}
5781
5782	if (r)
5783	dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5784
5785	atomic_set(v: &adev->reset_domain->reset_res, i: r);
5786	return r;
5787	}
5788
5789	/**
5790	* amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5791	*
5792	* @adev: amdgpu_device pointer
5793	* @speed: pointer to the speed of the link
5794	* @width: pointer to the width of the link
5795	*
5796	* Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5797	* first physical partner to an AMD dGPU.
5798	* This will exclude any virtual switches and links.
5799	*/
5800	static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
5801	enum pci_bus_speed *speed,
5802	enum pcie_link_width *width)
5803	{
5804	struct pci_dev *parent = adev->pdev;
5805
5806	if (!speed || !width)
5807	return;
5808
5809	*speed = PCI_SPEED_UNKNOWN;
5810	*width = PCIE_LNK_WIDTH_UNKNOWN;
5811
5812	while ((parent = pci_upstream_bridge(dev: parent))) {
5813	/* skip upstream/downstream switches internal to dGPU*/
5814	if (parent->vendor == PCI_VENDOR_ID_ATI)
5815	continue;
5816	*speed = pcie_get_speed_cap(dev: parent);
5817	*width = pcie_get_width_cap(dev: parent);
5818	break;
5819	}
5820	}
5821
5822	/**
5823	* amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5824	*
5825	* @adev: amdgpu_device pointer
5826	*
5827	* Fetchs and stores in the driver the PCIE capabilities (gen speed
5828	* and lanes) of the slot the device is in. Handles APUs and
5829	* virtualized environments where PCIE config space may not be available.
5830	*/
5831	static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5832	{
5833	struct pci_dev *pdev;
5834	enum pci_bus_speed speed_cap, platform_speed_cap;
5835	enum pcie_link_width platform_link_width;
5836
5837	if (amdgpu_pcie_gen_cap)
5838	adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5839
5840	if (amdgpu_pcie_lane_cap)
5841	adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5842
5843	/* covers APUs as well */
5844	if (pci_is_root_bus(pbus: adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5845	if (adev->pm.pcie_gen_mask == 0)
5846	adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5847	if (adev->pm.pcie_mlw_mask == 0)
5848	adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5849	return;
5850	}
5851
5852	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5853	return;
5854
5855	amdgpu_device_partner_bandwidth(adev, speed: &platform_speed_cap,
5856	width: &platform_link_width);
5857
5858	if (adev->pm.pcie_gen_mask == 0) {
5859	/* asic caps */
5860	pdev = adev->pdev;
5861	speed_cap = pcie_get_speed_cap(dev: pdev);
5862	if (speed_cap == PCI_SPEED_UNKNOWN) {
5863	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5864	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5865	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5866	} else {
5867	if (speed_cap == PCIE_SPEED_32_0GT)
5868	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5869	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5870	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5871	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5872	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5873	else if (speed_cap == PCIE_SPEED_16_0GT)
5874	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5875	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5876	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5877	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5878	else if (speed_cap == PCIE_SPEED_8_0GT)
5879	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5880	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5881	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5882	else if (speed_cap == PCIE_SPEED_5_0GT)
5883	adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5884	CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5885	else
5886	adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5887	}
5888	/* platform caps */
5889	if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5890	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5891	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5892	} else {
5893	if (platform_speed_cap == PCIE_SPEED_32_0GT)
5894	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5895	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5896	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5897	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5898	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5899	else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5900	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5901	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5902	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5903	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5904	else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5905	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5906	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5907	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5908	else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5909	adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5910	CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5911	else
5912	adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5913
5914	}
5915	}
5916	if (adev->pm.pcie_mlw_mask == 0) {
5917	if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5918	adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5919	} else {
5920	switch (platform_link_width) {
5921	case PCIE_LNK_X32:
5922	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5923	CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5924	CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5925	CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5926	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5927	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5928	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5929	break;
5930	case PCIE_LNK_X16:
5931	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5932	CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5933	CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5934	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5935	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5936	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5937	break;
5938	case PCIE_LNK_X12:
5939	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5940	CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5941	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5942	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5943	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5944	break;
5945	case PCIE_LNK_X8:
5946	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5947	CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5948	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5949	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5950	break;
5951	case PCIE_LNK_X4:
5952	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5953	CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5954	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5955	break;
5956	case PCIE_LNK_X2:
5957	adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5958	CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5959	break;
5960	case PCIE_LNK_X1:
5961	adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5962	break;
5963	default:
5964	break;
5965	}
5966	}
5967	}
5968	}
5969
5970	/**
5971	* amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5972	*
5973	* @adev: amdgpu_device pointer
5974	* @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5975	*
5976	* Return true if @peer_adev can access (DMA) @adev through the PCIe
5977	* BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5978	* @peer_adev.
5979	*/
5980	bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5981	struct amdgpu_device *peer_adev)
5982	{
5983	#ifdef CONFIG_HSA_AMD_P2P
5984	uint64_t address_mask = peer_adev->dev->dma_mask ?
5985	~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5986	resource_size_t aper_limit =
5987	adev->gmc.aper_base + adev->gmc.aper_size - 1;
5988	bool p2p_access =
5989	!adev->gmc.xgmi.connected_to_cpu &&
5990	!(pci_p2pdma_distance(provider: adev->pdev, client: peer_adev->dev, verbose: false) < 0);
5991
5992	return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5993	adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5994	!(adev->gmc.aper_base & address_mask ||
5995	aper_limit & address_mask));
5996	#else
5997	return false;
5998	#endif
5999	}
6000
6001	int amdgpu_device_baco_enter(struct drm_device *dev)
6002	{
6003	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6004	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6005
6006	if (!amdgpu_device_supports_baco(dev))
6007	return -ENOTSUPP;
6008
6009	if (ras && adev->ras_enabled &&
6010	adev->nbio.funcs->enable_doorbell_interrupt)
6011	adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6012
6013	return amdgpu_dpm_baco_enter(adev);
6014	}
6015
6016	int amdgpu_device_baco_exit(struct drm_device *dev)
6017	{
6018	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6019	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6020	int ret = 0;
6021
6022	if (!amdgpu_device_supports_baco(dev))
6023	return -ENOTSUPP;
6024
6025	ret = amdgpu_dpm_baco_exit(adev);
6026	if (ret)
6027	return ret;
6028
6029	if (ras && adev->ras_enabled &&
6030	adev->nbio.funcs->enable_doorbell_interrupt)
6031	adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6032
6033	if (amdgpu_passthrough(adev) &&
6034	adev->nbio.funcs->clear_doorbell_interrupt)
6035	adev->nbio.funcs->clear_doorbell_interrupt(adev);
6036
6037	return 0;
6038	}
6039
6040	/**
6041	* amdgpu_pci_error_detected - Called when a PCI error is detected.
6042	* @pdev: PCI device struct
6043	* @state: PCI channel state
6044	*
6045	* Description: Called when a PCI error is detected.
6046	*
6047	* Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6048	*/
6049	pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6050	{
6051	struct drm_device *dev = pci_get_drvdata(pdev);
6052	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6053	int i;
6054
6055	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6056
6057	if (adev->gmc.xgmi.num_physical_nodes > 1) {
6058	DRM_WARN("No support for XGMI hive yet...");
6059	return PCI_ERS_RESULT_DISCONNECT;
6060	}
6061
6062	adev->pci_channel_state = state;
6063
6064	switch (state) {
6065	case pci_channel_io_normal:
6066	return PCI_ERS_RESULT_CAN_RECOVER;
6067	/* Fatal error, prepare for slot reset */
6068	case pci_channel_io_frozen:
6069	/*
6070	* Locking adev->reset_domain->sem will prevent any external access
6071	* to GPU during PCI error recovery
6072	*/
6073	amdgpu_device_lock_reset_domain(reset_domain: adev->reset_domain);
6074	amdgpu_device_set_mp1_state(adev);
6075
6076	/*
6077	* Block any work scheduling as we do for regular GPU reset
6078	* for the duration of the recovery
6079	*/
6080	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6081	struct amdgpu_ring *ring = adev->rings[i];
6082
6083	if (!amdgpu_ring_sched_ready(ring))
6084	continue;
6085
6086	drm_sched_stop(sched: &ring->sched, NULL);
6087	}
6088	atomic_inc(v: &adev->gpu_reset_counter);
6089	return PCI_ERS_RESULT_NEED_RESET;
6090	case pci_channel_io_perm_failure:
6091	/* Permanent error, prepare for device removal */
6092	return PCI_ERS_RESULT_DISCONNECT;
6093	}
6094
6095	return PCI_ERS_RESULT_NEED_RESET;
6096	}
6097
6098	/**
6099	* amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6100	* @pdev: pointer to PCI device
6101	*/
6102	pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6103	{
6104
6105	DRM_INFO("PCI error: mmio enabled callback!!\n");
6106
6107	/* TODO - dump whatever for debugging purposes */
6108
6109	/* This called only if amdgpu_pci_error_detected returns
6110	* PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6111	* works, no need to reset slot.
6112	*/
6113
6114	return PCI_ERS_RESULT_RECOVERED;
6115	}
6116
6117	/**
6118	* amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6119	* @pdev: PCI device struct
6120	*
6121	* Description: This routine is called by the pci error recovery
6122	* code after the PCI slot has been reset, just before we
6123	* should resume normal operations.
6124	*/
6125	pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6126	{
6127	struct drm_device *dev = pci_get_drvdata(pdev);
6128	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6129	int r, i;
6130	struct amdgpu_reset_context reset_context;
6131	u32 memsize;
6132	struct list_head device_list;
6133	struct amdgpu_hive_info *hive;
6134	int hive_ras_recovery = 0;
6135	struct amdgpu_ras *ras;
6136
6137	/* PCI error slot reset should be skipped During RAS recovery */
6138	hive = amdgpu_get_xgmi_hive(adev);
6139	if (hive) {
6140	hive_ras_recovery = atomic_read(v: &hive->ras_recovery);
6141	amdgpu_put_xgmi_hive(hive);
6142	}
6143	ras = amdgpu_ras_get_context(adev);
6144	if ((amdgpu_ip_version(adev, ip: GC_HWIP, inst: 0) == IP_VERSION(9, 4, 3)) &&
6145	ras && (atomic_read(v: &ras->in_recovery) || hive_ras_recovery))
6146	return PCI_ERS_RESULT_RECOVERED;
6147
6148	DRM_INFO("PCI error: slot reset callback!!\n");
6149
6150	memset(&reset_context, 0, sizeof(reset_context));
6151
6152	INIT_LIST_HEAD(list: &device_list);
6153	list_add_tail(new: &adev->reset_list, head: &device_list);
6154
6155	/* wait for asic to come out of reset */
6156	msleep(msecs: 500);
6157
6158	/* Restore PCI confspace */
6159	amdgpu_device_load_pci_state(pdev);
6160
6161	/* confirm ASIC came out of reset */
6162	for (i = 0; i < adev->usec_timeout; i++) {
6163	memsize = amdgpu_asic_get_config_memsize(adev);
6164
6165	if (memsize != 0xffffffff)
6166	break;
6167	udelay(1);
6168	}
6169	if (memsize == 0xffffffff) {
6170	r = -ETIME;
6171	goto out;
6172	}
6173
6174	reset_context.method = AMD_RESET_METHOD_NONE;
6175	reset_context.reset_req_dev = adev;
6176	set_bit(nr: AMDGPU_NEED_FULL_RESET, addr: &reset_context.flags);
6177	set_bit(nr: AMDGPU_SKIP_HW_RESET, addr: &reset_context.flags);
6178
6179	adev->no_hw_access = true;
6180	r = amdgpu_device_pre_asic_reset(adev, reset_context: &reset_context);
6181	adev->no_hw_access = false;
6182	if (r)
6183	goto out;
6184
6185	r = amdgpu_do_asic_reset(device_list_handle: &device_list, reset_context: &reset_context);
6186
6187	out:
6188	if (!r) {
6189	if (amdgpu_device_cache_pci_state(pdev: adev->pdev))
6190	pci_restore_state(dev: adev->pdev);
6191
6192	DRM_INFO("PCIe error recovery succeeded\n");
6193	} else {
6194	DRM_ERROR("PCIe error recovery failed, err:%d", r);
6195	amdgpu_device_unset_mp1_state(adev);
6196	amdgpu_device_unlock_reset_domain(reset_domain: adev->reset_domain);
6197	}
6198
6199	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6200	}
6201
6202	/**
6203	* amdgpu_pci_resume() - resume normal ops after PCI reset
6204	* @pdev: pointer to PCI device
6205	*
6206	* Called when the error recovery driver tells us that its
6207	* OK to resume normal operation.
6208	*/
6209	void amdgpu_pci_resume(struct pci_dev *pdev)
6210	{
6211	struct drm_device *dev = pci_get_drvdata(pdev);
6212	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6213	int i;
6214
6215
6216	DRM_INFO("PCI error: resume callback!!\n");
6217
6218	/* Only continue execution for the case of pci_channel_io_frozen */
6219	if (adev->pci_channel_state != pci_channel_io_frozen)
6220	return;
6221
6222	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6223	struct amdgpu_ring *ring = adev->rings[i];
6224
6225	if (!amdgpu_ring_sched_ready(ring))
6226	continue;
6227
6228	drm_sched_start(sched: &ring->sched, full_recovery: true);
6229	}
6230
6231	amdgpu_device_unset_mp1_state(adev);
6232	amdgpu_device_unlock_reset_domain(reset_domain: adev->reset_domain);
6233	}
6234
6235	bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6236	{
6237	struct drm_device *dev = pci_get_drvdata(pdev);
6238	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6239	int r;
6240
6241	r = pci_save_state(dev: pdev);
6242	if (!r) {
6243	kfree(objp: adev->pci_state);
6244
6245	adev->pci_state = pci_store_saved_state(dev: pdev);
6246
6247	if (!adev->pci_state) {
6248	DRM_ERROR("Failed to store PCI saved state");
6249	return false;
6250	}
6251	} else {
6252	DRM_WARN("Failed to save PCI state, err:%d\n", r);
6253	return false;
6254	}
6255
6256	return true;
6257	}
6258
6259	bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6260	{
6261	struct drm_device *dev = pci_get_drvdata(pdev);
6262	struct amdgpu_device *adev = drm_to_adev(ddev: dev);
6263	int r;
6264
6265	if (!adev->pci_state)
6266	return false;
6267
6268	r = pci_load_saved_state(dev: pdev, state: adev->pci_state);
6269
6270	if (!r) {
6271	pci_restore_state(dev: pdev);
6272	} else {
6273	DRM_WARN("Failed to load PCI state, err:%d\n", r);
6274	return false;
6275	}
6276
6277	return true;
6278	}
6279
6280	void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6281	struct amdgpu_ring *ring)
6282	{
6283	#ifdef CONFIG_X86_64
6284	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6285	return;
6286	#endif
6287	if (adev->gmc.xgmi.connected_to_cpu)
6288	return;
6289
6290	if (ring && ring->funcs->emit_hdp_flush)
6291	amdgpu_ring_emit_hdp_flush(ring);
6292	else
6293	amdgpu_asic_flush_hdp(adev, ring);
6294	}
6295
6296	void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6297	struct amdgpu_ring *ring)
6298	{
6299	#ifdef CONFIG_X86_64
6300	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6301	return;
6302	#endif
6303	if (adev->gmc.xgmi.connected_to_cpu)
6304	return;
6305
6306	amdgpu_asic_invalidate_hdp(adev, ring);
6307	}
6308
6309	int amdgpu_in_reset(struct amdgpu_device *adev)
6310	{
6311	return atomic_read(v: &adev->reset_domain->in_gpu_reset);
6312	}
6313
6314	/**
6315	* amdgpu_device_halt() - bring hardware to some kind of halt state
6316	*
6317	* @adev: amdgpu_device pointer
6318	*
6319	* Bring hardware to some kind of halt state so that no one can touch it
6320	* any more. It will help to maintain error context when error occurred.
6321	* Compare to a simple hang, the system will keep stable at least for SSH
6322	* access. Then it should be trivial to inspect the hardware state and
6323	* see what's going on. Implemented as following:
6324	*
6325	* 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6326	* clears all CPU mappings to device, disallows remappings through page faults
6327	* 2. amdgpu_irq_disable_all() disables all interrupts
6328	* 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6329	* 4. set adev->no_hw_access to avoid potential crashes after setp 5
6330	* 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6331	* 6. pci_disable_device() and pci_wait_for_pending_transaction()
6332	* flush any in flight DMA operations
6333	*/
6334	void amdgpu_device_halt(struct amdgpu_device *adev)
6335	{
6336	struct pci_dev *pdev = adev->pdev;
6337	struct drm_device *ddev = adev_to_drm(adev);
6338
6339	amdgpu_xcp_dev_unplug(adev);
6340	drm_dev_unplug(dev: ddev);
6341
6342	amdgpu_irq_disable_all(adev);
6343
6344	amdgpu_fence_driver_hw_fini(adev);
6345
6346	adev->no_hw_access = true;
6347
6348	amdgpu_device_unmap_mmio(adev);
6349
6350	pci_disable_device(dev: pdev);
6351	pci_wait_for_pending_transaction(dev: pdev);
6352	}
6353
6354	u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6355	u32 reg)
6356	{
6357	unsigned long flags, address, data;
6358	u32 r;
6359
6360	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6361	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6362
6363	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6364	WREG32(address, reg * 4);
6365	(void)RREG32(address);
6366	r = RREG32(data);
6367	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
6368	return r;
6369	}
6370
6371	void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6372	u32 reg, u32 v)
6373	{
6374	unsigned long flags, address, data;
6375
6376	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6377	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6378
6379	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6380	WREG32(address, reg * 4);
6381	(void)RREG32(address);
6382	WREG32(data, v);
6383	(void)RREG32(data);
6384	spin_unlock_irqrestore(lock: &adev->pcie_idx_lock, flags);
6385	}
6386
6387	/**
6388	* amdgpu_device_switch_gang - switch to a new gang
6389	* @adev: amdgpu_device pointer
6390	* @gang: the gang to switch to
6391	*
6392	* Try to switch to a new gang.
6393	* Returns: NULL if we switched to the new gang or a reference to the current
6394	* gang leader.
6395	*/
6396	struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6397	struct dma_fence *gang)
6398	{
6399	struct dma_fence *old = NULL;
6400
6401	do {
6402	dma_fence_put(fence: old);
6403	rcu_read_lock();
6404	old = dma_fence_get_rcu_safe(fencep: &adev->gang_submit);
6405	rcu_read_unlock();
6406
6407	if (old == gang)
6408	break;
6409
6410	if (!dma_fence_is_signaled(fence: old))
6411	return old;
6412
6413	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6414	old, gang) != old);
6415
6416	dma_fence_put(fence: old);
6417	return NULL;
6418	}
6419
6420	bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6421	{
6422	switch (adev->asic_type) {
6423	#ifdef CONFIG_DRM_AMDGPU_SI
6424	case CHIP_HAINAN:
6425	#endif
6426	case CHIP_TOPAZ:
6427	/* chips with no display hardware */
6428	return false;
6429	#ifdef CONFIG_DRM_AMDGPU_SI
6430	case CHIP_TAHITI:
6431	case CHIP_PITCAIRN:
6432	case CHIP_VERDE:
6433	case CHIP_OLAND:
6434	#endif
6435	#ifdef CONFIG_DRM_AMDGPU_CIK
6436	case CHIP_BONAIRE:
6437	case CHIP_HAWAII:
6438	case CHIP_KAVERI:
6439	case CHIP_KABINI:
6440	case CHIP_MULLINS:
6441	#endif
6442	case CHIP_TONGA:
6443	case CHIP_FIJI:
6444	case CHIP_POLARIS10:
6445	case CHIP_POLARIS11:
6446	case CHIP_POLARIS12:
6447	case CHIP_VEGAM:
6448	case CHIP_CARRIZO:
6449	case CHIP_STONEY:
6450	/* chips with display hardware */
6451	return true;
6452	default:
6453	/* IP discovery */
6454	if (!amdgpu_ip_version(adev, ip: DCE_HWIP, inst: 0) ||
6455	(adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6456	return false;
6457	return true;
6458	}
6459	}
6460
6461	uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6462	uint32_t inst, uint32_t reg_addr, char reg_name[],
6463	uint32_t expected_value, uint32_t mask)
6464	{
6465	uint32_t ret = 0;
6466	uint32_t old_ = 0;
6467	uint32_t tmp_ = RREG32(reg_addr);
6468	uint32_t loop = adev->usec_timeout;
6469
6470	while ((tmp_ & (mask)) != (expected_value)) {
6471	if (old_ != tmp_) {
6472	loop = adev->usec_timeout;
6473	old_ = tmp_;
6474	} else
6475	udelay(1);
6476	tmp_ = RREG32(reg_addr);
6477	loop--;
6478	if (!loop) {
6479	DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6480	inst, reg_name, (uint32_t)expected_value,
6481	(uint32_t)(tmp_ & (mask)));
6482	ret = -ETIMEDOUT;
6483	break;
6484	}
6485	}
6486	return ret;
6487	}
6488