1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#ifndef __AMDGPU_H__
29#define __AMDGPU_H__
30
31#ifdef pr_fmt
32#undef pr_fmt
33#endif
34
35#define pr_fmt(fmt) "amdgpu: " fmt
36
37#ifdef dev_fmt
38#undef dev_fmt
39#endif
40
41#define dev_fmt(fmt) "amdgpu: " fmt
42
43#include "amdgpu_ctx.h"
44
45#include <linux/atomic.h>
46#include <linux/wait.h>
47#include <linux/list.h>
48#include <linux/kref.h>
49#include <linux/rbtree.h>
50#include <linux/hashtable.h>
51#include <linux/dma-fence.h>
52#include <linux/pci.h>
53
54#include <drm/ttm/ttm_bo.h>
55#include <drm/ttm/ttm_placement.h>
56#include <drm/ttm/ttm_execbuf_util.h>
57
58#include <drm/amdgpu_drm.h>
59#include <drm/drm_gem.h>
60#include <drm/drm_ioctl.h>
61
62#include <kgd_kfd_interface.h>
63#include "dm_pp_interface.h"
64#include "kgd_pp_interface.h"
65
66#include "amd_shared.h"
67#include "amdgpu_mode.h"
68#include "amdgpu_ih.h"
69#include "amdgpu_irq.h"
70#include "amdgpu_ucode.h"
71#include "amdgpu_ttm.h"
72#include "amdgpu_psp.h"
73#include "amdgpu_gds.h"
74#include "amdgpu_sync.h"
75#include "amdgpu_ring.h"
76#include "amdgpu_vm.h"
77#include "amdgpu_dpm.h"
78#include "amdgpu_acp.h"
79#include "amdgpu_uvd.h"
80#include "amdgpu_vce.h"
81#include "amdgpu_vcn.h"
82#include "amdgpu_jpeg.h"
83#include "amdgpu_gmc.h"
84#include "amdgpu_gfx.h"
85#include "amdgpu_sdma.h"
86#include "amdgpu_lsdma.h"
87#include "amdgpu_nbio.h"
88#include "amdgpu_hdp.h"
89#include "amdgpu_dm.h"
90#include "amdgpu_virt.h"
91#include "amdgpu_csa.h"
92#include "amdgpu_mes_ctx.h"
93#include "amdgpu_gart.h"
94#include "amdgpu_debugfs.h"
95#include "amdgpu_job.h"
96#include "amdgpu_bo_list.h"
97#include "amdgpu_gem.h"
98#include "amdgpu_doorbell.h"
99#include "amdgpu_amdkfd.h"
100#include "amdgpu_discovery.h"
101#include "amdgpu_mes.h"
102#include "amdgpu_umc.h"
103#include "amdgpu_mmhub.h"
104#include "amdgpu_gfxhub.h"
105#include "amdgpu_df.h"
106#include "amdgpu_smuio.h"
107#include "amdgpu_fdinfo.h"
108#include "amdgpu_mca.h"
109#include "amdgpu_ras.h"
110
111#define MAX_GPU_INSTANCE 16
112
113struct amdgpu_gpu_instance
114{
115 struct amdgpu_device *adev;
116 int mgpu_fan_enabled;
117};
118
119struct amdgpu_mgpu_info
120{
121 struct amdgpu_gpu_instance gpu_ins[MAX_GPU_INSTANCE];
122 struct mutex mutex;
123 uint32_t num_gpu;
124 uint32_t num_dgpu;
125 uint32_t num_apu;
126
127 /* delayed reset_func for XGMI configuration if necessary */
128 struct delayed_work delayed_reset_work;
129 bool pending_reset;
130};
131
132enum amdgpu_ss {
133 AMDGPU_SS_DRV_LOAD,
134 AMDGPU_SS_DEV_D0,
135 AMDGPU_SS_DEV_D3,
136 AMDGPU_SS_DRV_UNLOAD
137};
138
139struct amdgpu_watchdog_timer
140{
141 bool timeout_fatal_disable;
142 uint32_t period; /* maxCycles = (1 << period), the number of cycles before a timeout */
143};
144
145#define AMDGPU_MAX_TIMEOUT_PARAM_LENGTH 256
146
147/*
148 * Modules parameters.
149 */
150extern int amdgpu_modeset;
151extern unsigned int amdgpu_vram_limit;
152extern int amdgpu_vis_vram_limit;
153extern int amdgpu_gart_size;
154extern int amdgpu_gtt_size;
155extern int amdgpu_moverate;
156extern int amdgpu_audio;
157extern int amdgpu_disp_priority;
158extern int amdgpu_hw_i2c;
159extern int amdgpu_pcie_gen2;
160extern int amdgpu_msi;
161extern char amdgpu_lockup_timeout[AMDGPU_MAX_TIMEOUT_PARAM_LENGTH];
162extern int amdgpu_dpm;
163extern int amdgpu_fw_load_type;
164extern int amdgpu_aspm;
165extern int amdgpu_runtime_pm;
166extern uint amdgpu_ip_block_mask;
167extern int amdgpu_bapm;
168extern int amdgpu_deep_color;
169extern int amdgpu_vm_size;
170extern int amdgpu_vm_block_size;
171extern int amdgpu_vm_fragment_size;
172extern int amdgpu_vm_fault_stop;
173extern int amdgpu_vm_debug;
174extern int amdgpu_vm_update_mode;
175extern int amdgpu_exp_hw_support;
176extern int amdgpu_dc;
177extern int amdgpu_sched_jobs;
178extern int amdgpu_sched_hw_submission;
179extern uint amdgpu_pcie_gen_cap;
180extern uint amdgpu_pcie_lane_cap;
181extern u64 amdgpu_cg_mask;
182extern uint amdgpu_pg_mask;
183extern uint amdgpu_sdma_phase_quantum;
184extern char *amdgpu_disable_cu;
185extern char *amdgpu_virtual_display;
186extern uint amdgpu_pp_feature_mask;
187extern uint amdgpu_force_long_training;
188extern int amdgpu_lbpw;
189extern int amdgpu_compute_multipipe;
190extern int amdgpu_gpu_recovery;
191extern int amdgpu_emu_mode;
192extern uint amdgpu_smu_memory_pool_size;
193extern int amdgpu_smu_pptable_id;
194extern uint amdgpu_dc_feature_mask;
195extern uint amdgpu_freesync_vid_mode;
196extern uint amdgpu_dc_debug_mask;
197extern uint amdgpu_dc_visual_confirm;
198extern uint amdgpu_dm_abm_level;
199extern int amdgpu_backlight;
200extern struct amdgpu_mgpu_info mgpu_info;
201extern int amdgpu_ras_enable;
202extern uint amdgpu_ras_mask;
203extern int amdgpu_bad_page_threshold;
204extern bool amdgpu_ignore_bad_page_threshold;
205extern struct amdgpu_watchdog_timer amdgpu_watchdog_timer;
206extern int amdgpu_async_gfx_ring;
207extern int amdgpu_mcbp;
208extern int amdgpu_discovery;
209extern int amdgpu_mes;
210extern int amdgpu_mes_kiq;
211extern int amdgpu_noretry;
212extern int amdgpu_force_asic_type;
213extern int amdgpu_smartshift_bias;
214extern int amdgpu_use_xgmi_p2p;
215#ifdef CONFIG_HSA_AMD
216extern int sched_policy;
217extern bool debug_evictions;
218extern bool no_system_mem_limit;
219extern int halt_if_hws_hang;
220#else
221static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
222static const bool __maybe_unused debug_evictions; /* = false */
223static const bool __maybe_unused no_system_mem_limit;
224static const int __maybe_unused halt_if_hws_hang;
225#endif
226#ifdef CONFIG_HSA_AMD_P2P
227extern bool pcie_p2p;
228#endif
229
230extern int amdgpu_tmz;
231extern int amdgpu_reset_method;
232
233#ifdef CONFIG_DRM_AMDGPU_SI
234extern int amdgpu_si_support;
235#endif
236#ifdef CONFIG_DRM_AMDGPU_CIK
237extern int amdgpu_cik_support;
238#endif
239extern int amdgpu_num_kcq;
240
241#define AMDGPU_VCNFW_LOG_SIZE (32 * 1024)
242extern int amdgpu_vcnfw_log;
243extern int amdgpu_sg_display;
244
245#define AMDGPU_VM_MAX_NUM_CTX 4096
246#define AMDGPU_SG_THRESHOLD (256*1024*1024)
247#define AMDGPU_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */
248#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000
249#define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */
250#define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2)
251#define AMDGPU_DEBUGFS_MAX_COMPONENTS 32
252#define AMDGPUFB_CONN_LIMIT 4
253#define AMDGPU_BIOS_NUM_SCRATCH 16
254
255#define AMDGPU_VBIOS_VGA_ALLOCATION (9 * 1024 * 1024) /* reserve 8MB for vga emulator and 1 MB for FB */
256
257/* hard reset data */
258#define AMDGPU_ASIC_RESET_DATA 0x39d5e86b
259
260/* reset flags */
261#define AMDGPU_RESET_GFX (1 << 0)
262#define AMDGPU_RESET_COMPUTE (1 << 1)
263#define AMDGPU_RESET_DMA (1 << 2)
264#define AMDGPU_RESET_CP (1 << 3)
265#define AMDGPU_RESET_GRBM (1 << 4)
266#define AMDGPU_RESET_DMA1 (1 << 5)
267#define AMDGPU_RESET_RLC (1 << 6)
268#define AMDGPU_RESET_SEM (1 << 7)
269#define AMDGPU_RESET_IH (1 << 8)
270#define AMDGPU_RESET_VMC (1 << 9)
271#define AMDGPU_RESET_MC (1 << 10)
272#define AMDGPU_RESET_DISPLAY (1 << 11)
273#define AMDGPU_RESET_UVD (1 << 12)
274#define AMDGPU_RESET_VCE (1 << 13)
275#define AMDGPU_RESET_VCE1 (1 << 14)
276
277/* max cursor sizes (in pixels) */
278#define CIK_CURSOR_WIDTH 128
279#define CIK_CURSOR_HEIGHT 128
280
281/* smart shift bias level limits */
282#define AMDGPU_SMARTSHIFT_MAX_BIAS (100)
283#define AMDGPU_SMARTSHIFT_MIN_BIAS (-100)
284
285struct amdgpu_device;
286struct amdgpu_irq_src;
287struct amdgpu_fpriv;
288struct amdgpu_bo_va_mapping;
289struct kfd_vm_fault_info;
290struct amdgpu_hive_info;
291struct amdgpu_reset_context;
292struct amdgpu_reset_control;
293
294enum amdgpu_cp_irq {
295 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP = 0,
296 AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP,
297 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP,
298 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP,
299 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP,
300 AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP,
301 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP,
302 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP,
303 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP,
304 AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP,
305
306 AMDGPU_CP_IRQ_LAST
307};
308
309enum amdgpu_thermal_irq {
310 AMDGPU_THERMAL_IRQ_LOW_TO_HIGH = 0,
311 AMDGPU_THERMAL_IRQ_HIGH_TO_LOW,
312
313 AMDGPU_THERMAL_IRQ_LAST
314};
315
316enum amdgpu_kiq_irq {
317 AMDGPU_CP_KIQ_IRQ_DRIVER0 = 0,
318 AMDGPU_CP_KIQ_IRQ_LAST
319};
320#define SRIOV_USEC_TIMEOUT 1200000 /* wait 12 * 100ms for SRIOV */
321#define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */
322#define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */
323#define MAX_KIQ_REG_TRY 1000
324
325int amdgpu_device_ip_set_clockgating_state(void *dev,
326 enum amd_ip_block_type block_type,
327 enum amd_clockgating_state state);
328int amdgpu_device_ip_set_powergating_state(void *dev,
329 enum amd_ip_block_type block_type,
330 enum amd_powergating_state state);
331void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
332 u64 *flags);
333int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
334 enum amd_ip_block_type block_type);
335bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
336 enum amd_ip_block_type block_type);
337
338#define AMDGPU_MAX_IP_NUM 16
339
340struct amdgpu_ip_block_status {
341 bool valid;
342 bool sw;
343 bool hw;
344 bool late_initialized;
345 bool hang;
346};
347
348struct amdgpu_ip_block_version {
349 const enum amd_ip_block_type type;
350 const u32 major;
351 const u32 minor;
352 const u32 rev;
353 const struct amd_ip_funcs *funcs;
354};
355
356#define HW_REV(_Major, _Minor, _Rev) \
357 ((((uint32_t) (_Major)) << 16) | ((uint32_t) (_Minor) << 8) | ((uint32_t) (_Rev)))
358
359struct amdgpu_ip_block {
360 struct amdgpu_ip_block_status status;
361 const struct amdgpu_ip_block_version *version;
362};
363
364int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
365 enum amd_ip_block_type type,
366 u32 major, u32 minor);
367
368struct amdgpu_ip_block *
369amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
370 enum amd_ip_block_type type);
371
372int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
373 const struct amdgpu_ip_block_version *ip_block_version);
374
375/*
376 * BIOS.
377 */
378bool amdgpu_get_bios(struct amdgpu_device *adev);
379bool amdgpu_read_bios(struct amdgpu_device *adev);
380bool amdgpu_soc15_read_bios_from_rom(struct amdgpu_device *adev,
381 u8 *bios, u32 length_bytes);
382/*
383 * Clocks
384 */
385
386#define AMDGPU_MAX_PPLL 3
387
388struct amdgpu_clock {
389 struct amdgpu_pll ppll[AMDGPU_MAX_PPLL];
390 struct amdgpu_pll spll;
391 struct amdgpu_pll mpll;
392 /* 10 Khz units */
393 uint32_t default_mclk;
394 uint32_t default_sclk;
395 uint32_t default_dispclk;
396 uint32_t current_dispclk;
397 uint32_t dp_extclk;
398 uint32_t max_pixel_clock;
399};
400
401/* sub-allocation manager, it has to be protected by another lock.
402 * By conception this is an helper for other part of the driver
403 * like the indirect buffer or semaphore, which both have their
404 * locking.
405 *
406 * Principe is simple, we keep a list of sub allocation in offset
407 * order (first entry has offset == 0, last entry has the highest
408 * offset).
409 *
410 * When allocating new object we first check if there is room at
411 * the end total_size - (last_object_offset + last_object_size) >=
412 * alloc_size. If so we allocate new object there.
413 *
414 * When there is not enough room at the end, we start waiting for
415 * each sub object until we reach object_offset+object_size >=
416 * alloc_size, this object then become the sub object we return.
417 *
418 * Alignment can't be bigger than page size.
419 *
420 * Hole are not considered for allocation to keep things simple.
421 * Assumption is that there won't be hole (all object on same
422 * alignment).
423 */
424
425struct amdgpu_sa_manager {
426 struct drm_suballoc_manager base;
427 struct amdgpu_bo *bo;
428 uint64_t gpu_addr;
429 void *cpu_ptr;
430};
431
432int amdgpu_fence_slab_init(void);
433void amdgpu_fence_slab_fini(void);
434
435/*
436 * IRQS.
437 */
438
439struct amdgpu_flip_work {
440 struct delayed_work flip_work;
441 struct work_struct unpin_work;
442 struct amdgpu_device *adev;
443 int crtc_id;
444 u32 target_vblank;
445 uint64_t base;
446 struct drm_pending_vblank_event *event;
447 struct amdgpu_bo *old_abo;
448 unsigned shared_count;
449 struct dma_fence **shared;
450 struct dma_fence_cb cb;
451 bool async;
452};
453
454
455/*
456 * file private structure
457 */
458
459struct amdgpu_fpriv {
460 struct amdgpu_vm vm;
461 struct amdgpu_bo_va *prt_va;
462 struct amdgpu_bo_va *csa_va;
463 struct mutex bo_list_lock;
464 struct idr bo_list_handles;
465 struct amdgpu_ctx_mgr ctx_mgr;
466};
467
468int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
469
470/*
471 * Writeback
472 */
473#define AMDGPU_MAX_WB 1024 /* Reserve at most 1024 WB slots for amdgpu-owned rings. */
474
475struct amdgpu_wb {
476 struct amdgpu_bo *wb_obj;
477 volatile uint32_t *wb;
478 uint64_t gpu_addr;
479 u32 num_wb; /* Number of wb slots actually reserved for amdgpu. */
480 unsigned long used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
481};
482
483int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb);
484void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb);
485
486/*
487 * Benchmarking
488 */
489int amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
490
491/*
492 * ASIC specific register table accessible by UMD
493 */
494struct amdgpu_allowed_register_entry {
495 uint32_t reg_offset;
496 bool grbm_indexed;
497};
498
499enum amd_reset_method {
500 AMD_RESET_METHOD_NONE = -1,
501 AMD_RESET_METHOD_LEGACY = 0,
502 AMD_RESET_METHOD_MODE0,
503 AMD_RESET_METHOD_MODE1,
504 AMD_RESET_METHOD_MODE2,
505 AMD_RESET_METHOD_BACO,
506 AMD_RESET_METHOD_PCI,
507};
508
509struct amdgpu_video_codec_info {
510 u32 codec_type;
511 u32 max_width;
512 u32 max_height;
513 u32 max_pixels_per_frame;
514 u32 max_level;
515};
516
517#define codec_info_build(type, width, height, level) \
518 .codec_type = type,\
519 .max_width = width,\
520 .max_height = height,\
521 .max_pixels_per_frame = height * width,\
522 .max_level = level,
523
524struct amdgpu_video_codecs {
525 const u32 codec_count;
526 const struct amdgpu_video_codec_info *codec_array;
527};
528
529/*
530 * ASIC specific functions.
531 */
532struct amdgpu_asic_funcs {
533 bool (*read_disabled_bios)(struct amdgpu_device *adev);
534 bool (*read_bios_from_rom)(struct amdgpu_device *adev,
535 u8 *bios, u32 length_bytes);
536 int (*read_register)(struct amdgpu_device *adev, u32 se_num,
537 u32 sh_num, u32 reg_offset, u32 *value);
538 void (*set_vga_state)(struct amdgpu_device *adev, bool state);
539 int (*reset)(struct amdgpu_device *adev);
540 enum amd_reset_method (*reset_method)(struct amdgpu_device *adev);
541 /* get the reference clock */
542 u32 (*get_xclk)(struct amdgpu_device *adev);
543 /* MM block clocks */
544 int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
545 int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
546 /* static power management */
547 int (*get_pcie_lanes)(struct amdgpu_device *adev);
548 void (*set_pcie_lanes)(struct amdgpu_device *adev, int lanes);
549 /* get config memsize register */
550 u32 (*get_config_memsize)(struct amdgpu_device *adev);
551 /* flush hdp write queue */
552 void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring);
553 /* invalidate hdp read cache */
554 void (*invalidate_hdp)(struct amdgpu_device *adev,
555 struct amdgpu_ring *ring);
556 /* check if the asic needs a full reset of if soft reset will work */
557 bool (*need_full_reset)(struct amdgpu_device *adev);
558 /* initialize doorbell layout for specific asic*/
559 void (*init_doorbell_index)(struct amdgpu_device *adev);
560 /* PCIe bandwidth usage */
561 void (*get_pcie_usage)(struct amdgpu_device *adev, uint64_t *count0,
562 uint64_t *count1);
563 /* do we need to reset the asic at init time (e.g., kexec) */
564 bool (*need_reset_on_init)(struct amdgpu_device *adev);
565 /* PCIe replay counter */
566 uint64_t (*get_pcie_replay_count)(struct amdgpu_device *adev);
567 /* device supports BACO */
568 bool (*supports_baco)(struct amdgpu_device *adev);
569 /* pre asic_init quirks */
570 void (*pre_asic_init)(struct amdgpu_device *adev);
571 /* enter/exit umd stable pstate */
572 int (*update_umd_stable_pstate)(struct amdgpu_device *adev, bool enter);
573 /* query video codecs */
574 int (*query_video_codecs)(struct amdgpu_device *adev, bool encode,
575 const struct amdgpu_video_codecs **codecs);
576};
577
578/*
579 * IOCTL.
580 */
581int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
582 struct drm_file *filp);
583
584int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
585int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
586 struct drm_file *filp);
587int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
588int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
589 struct drm_file *filp);
590
591/* VRAM scratch page for HDP bug, default vram page */
592struct amdgpu_mem_scratch {
593 struct amdgpu_bo *robj;
594 volatile uint32_t *ptr;
595 u64 gpu_addr;
596};
597
598/*
599 * CGS
600 */
601struct cgs_device *amdgpu_cgs_create_device(struct amdgpu_device *adev);
602void amdgpu_cgs_destroy_device(struct cgs_device *cgs_device);
603
604/*
605 * Core structure, functions and helpers.
606 */
607typedef uint32_t (*amdgpu_rreg_t)(struct amdgpu_device*, uint32_t);
608typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
609
610typedef uint64_t (*amdgpu_rreg64_t)(struct amdgpu_device*, uint32_t);
611typedef void (*amdgpu_wreg64_t)(struct amdgpu_device*, uint32_t, uint64_t);
612
613typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
614typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
615
616struct amdgpu_mmio_remap {
617 u32 reg_offset;
618 resource_size_t bus_addr;
619};
620
621/* Define the HW IP blocks will be used in driver , add more if necessary */
622enum amd_hw_ip_block_type {
623 GC_HWIP = 1,
624 HDP_HWIP,
625 SDMA0_HWIP,
626 SDMA1_HWIP,
627 SDMA2_HWIP,
628 SDMA3_HWIP,
629 SDMA4_HWIP,
630 SDMA5_HWIP,
631 SDMA6_HWIP,
632 SDMA7_HWIP,
633 LSDMA_HWIP,
634 MMHUB_HWIP,
635 ATHUB_HWIP,
636 NBIO_HWIP,
637 MP0_HWIP,
638 MP1_HWIP,
639 UVD_HWIP,
640 VCN_HWIP = UVD_HWIP,
641 JPEG_HWIP = VCN_HWIP,
642 VCN1_HWIP,
643 VCE_HWIP,
644 DF_HWIP,
645 DCE_HWIP,
646 OSSSYS_HWIP,
647 SMUIO_HWIP,
648 PWR_HWIP,
649 NBIF_HWIP,
650 THM_HWIP,
651 CLK_HWIP,
652 UMC_HWIP,
653 RSMU_HWIP,
654 XGMI_HWIP,
655 DCI_HWIP,
656 PCIE_HWIP,
657 MAX_HWIP
658};
659
660#define HWIP_MAX_INSTANCE 28
661
662#define HW_ID_MAX 300
663#define IP_VERSION(mj, mn, rv) (((mj) << 16) | ((mn) << 8) | (rv))
664#define IP_VERSION_MAJ(ver) ((ver) >> 16)
665#define IP_VERSION_MIN(ver) (((ver) >> 8) & 0xFF)
666#define IP_VERSION_REV(ver) ((ver) & 0xFF)
667
668struct amd_powerplay {
669 void *pp_handle;
670 const struct amd_pm_funcs *pp_funcs;
671};
672
673struct ip_discovery_top;
674
675/* polaris10 kickers */
676#define ASICID_IS_P20(did, rid) (((did == 0x67DF) && \
677 ((rid == 0xE3) || \
678 (rid == 0xE4) || \
679 (rid == 0xE5) || \
680 (rid == 0xE7) || \
681 (rid == 0xEF))) || \
682 ((did == 0x6FDF) && \
683 ((rid == 0xE7) || \
684 (rid == 0xEF) || \
685 (rid == 0xFF))))
686
687#define ASICID_IS_P30(did, rid) ((did == 0x67DF) && \
688 ((rid == 0xE1) || \
689 (rid == 0xF7)))
690
691/* polaris11 kickers */
692#define ASICID_IS_P21(did, rid) (((did == 0x67EF) && \
693 ((rid == 0xE0) || \
694 (rid == 0xE5))) || \
695 ((did == 0x67FF) && \
696 ((rid == 0xCF) || \
697 (rid == 0xEF) || \
698 (rid == 0xFF))))
699
700#define ASICID_IS_P31(did, rid) ((did == 0x67EF) && \
701 ((rid == 0xE2)))
702
703/* polaris12 kickers */
704#define ASICID_IS_P23(did, rid) (((did == 0x6987) && \
705 ((rid == 0xC0) || \
706 (rid == 0xC1) || \
707 (rid == 0xC3) || \
708 (rid == 0xC7))) || \
709 ((did == 0x6981) && \
710 ((rid == 0x00) || \
711 (rid == 0x01) || \
712 (rid == 0x10))))
713
714struct amdgpu_mqd_prop {
715 uint64_t mqd_gpu_addr;
716 uint64_t hqd_base_gpu_addr;
717 uint64_t rptr_gpu_addr;
718 uint64_t wptr_gpu_addr;
719 uint32_t queue_size;
720 bool use_doorbell;
721 uint32_t doorbell_index;
722 uint64_t eop_gpu_addr;
723 uint32_t hqd_pipe_priority;
724 uint32_t hqd_queue_priority;
725 bool hqd_active;
726};
727
728struct amdgpu_mqd {
729 unsigned mqd_size;
730 int (*init_mqd)(struct amdgpu_device *adev, void *mqd,
731 struct amdgpu_mqd_prop *p);
732};
733
734#define AMDGPU_RESET_MAGIC_NUM 64
735#define AMDGPU_MAX_DF_PERFMONS 4
736#define AMDGPU_PRODUCT_NAME_LEN 64
737struct amdgpu_reset_domain;
738
739/*
740 * Non-zero (true) if the GPU has VRAM. Zero (false) otherwise.
741 */
742#define AMDGPU_HAS_VRAM(_adev) ((_adev)->gmc.real_vram_size)
743
744struct amdgpu_device {
745 struct device *dev;
746 struct pci_dev *pdev;
747 struct drm_device ddev;
748
749#ifdef CONFIG_DRM_AMD_ACP
750 struct amdgpu_acp acp;
751#endif
752 struct amdgpu_hive_info *hive;
753 /* ASIC */
754 enum amd_asic_type asic_type;
755 uint32_t family;
756 uint32_t rev_id;
757 uint32_t external_rev_id;
758 unsigned long flags;
759 unsigned long apu_flags;
760 int usec_timeout;
761 const struct amdgpu_asic_funcs *asic_funcs;
762 bool shutdown;
763 bool need_swiotlb;
764 bool accel_working;
765 struct notifier_block acpi_nb;
766 struct amdgpu_i2c_chan *i2c_bus[AMDGPU_MAX_I2C_BUS];
767 struct debugfs_blob_wrapper debugfs_vbios_blob;
768 struct debugfs_blob_wrapper debugfs_discovery_blob;
769 struct mutex srbm_mutex;
770 /* GRBM index mutex. Protects concurrent access to GRBM index */
771 struct mutex grbm_idx_mutex;
772 struct dev_pm_domain vga_pm_domain;
773 bool have_disp_power_ref;
774 bool have_atomics_support;
775
776 /* BIOS */
777 bool is_atom_fw;
778 uint8_t *bios;
779 uint32_t bios_size;
780 uint32_t bios_scratch_reg_offset;
781 uint32_t bios_scratch[AMDGPU_BIOS_NUM_SCRATCH];
782
783 /* Register/doorbell mmio */
784 resource_size_t rmmio_base;
785 resource_size_t rmmio_size;
786 void __iomem *rmmio;
787 /* protects concurrent MM_INDEX/DATA based register access */
788 spinlock_t mmio_idx_lock;
789 struct amdgpu_mmio_remap rmmio_remap;
790 /* protects concurrent SMC based register access */
791 spinlock_t smc_idx_lock;
792 amdgpu_rreg_t smc_rreg;
793 amdgpu_wreg_t smc_wreg;
794 /* protects concurrent PCIE register access */
795 spinlock_t pcie_idx_lock;
796 amdgpu_rreg_t pcie_rreg;
797 amdgpu_wreg_t pcie_wreg;
798 amdgpu_rreg_t pciep_rreg;
799 amdgpu_wreg_t pciep_wreg;
800 amdgpu_rreg64_t pcie_rreg64;
801 amdgpu_wreg64_t pcie_wreg64;
802 /* protects concurrent UVD register access */
803 spinlock_t uvd_ctx_idx_lock;
804 amdgpu_rreg_t uvd_ctx_rreg;
805 amdgpu_wreg_t uvd_ctx_wreg;
806 /* protects concurrent DIDT register access */
807 spinlock_t didt_idx_lock;
808 amdgpu_rreg_t didt_rreg;
809 amdgpu_wreg_t didt_wreg;
810 /* protects concurrent gc_cac register access */
811 spinlock_t gc_cac_idx_lock;
812 amdgpu_rreg_t gc_cac_rreg;
813 amdgpu_wreg_t gc_cac_wreg;
814 /* protects concurrent se_cac register access */
815 spinlock_t se_cac_idx_lock;
816 amdgpu_rreg_t se_cac_rreg;
817 amdgpu_wreg_t se_cac_wreg;
818 /* protects concurrent ENDPOINT (audio) register access */
819 spinlock_t audio_endpt_idx_lock;
820 amdgpu_block_rreg_t audio_endpt_rreg;
821 amdgpu_block_wreg_t audio_endpt_wreg;
822 struct amdgpu_doorbell doorbell;
823
824 /* clock/pll info */
825 struct amdgpu_clock clock;
826
827 /* MC */
828 struct amdgpu_gmc gmc;
829 struct amdgpu_gart gart;
830 dma_addr_t dummy_page_addr;
831 struct amdgpu_vm_manager vm_manager;
832 struct amdgpu_vmhub vmhub[AMDGPU_MAX_VMHUBS];
833 unsigned num_vmhubs;
834
835 /* memory management */
836 struct amdgpu_mman mman;
837 struct amdgpu_mem_scratch mem_scratch;
838 struct amdgpu_wb wb;
839 atomic64_t num_bytes_moved;
840 atomic64_t num_evictions;
841 atomic64_t num_vram_cpu_page_faults;
842 atomic_t gpu_reset_counter;
843 atomic_t vram_lost_counter;
844
845 /* data for buffer migration throttling */
846 struct {
847 spinlock_t lock;
848 s64 last_update_us;
849 s64 accum_us; /* accumulated microseconds */
850 s64 accum_us_vis; /* for visible VRAM */
851 u32 log2_max_MBps;
852 } mm_stats;
853
854 /* display */
855 bool enable_virtual_display;
856 struct amdgpu_vkms_output *amdgpu_vkms_output;
857 struct amdgpu_mode_info mode_info;
858 /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */
859 struct delayed_work hotplug_work;
860 struct amdgpu_irq_src crtc_irq;
861 struct amdgpu_irq_src vline0_irq;
862 struct amdgpu_irq_src vupdate_irq;
863 struct amdgpu_irq_src pageflip_irq;
864 struct amdgpu_irq_src hpd_irq;
865 struct amdgpu_irq_src dmub_trace_irq;
866 struct amdgpu_irq_src dmub_outbox_irq;
867
868 /* rings */
869 u64 fence_context;
870 unsigned num_rings;
871 struct amdgpu_ring *rings[AMDGPU_MAX_RINGS];
872 struct dma_fence __rcu *gang_submit;
873 bool ib_pool_ready;
874 struct amdgpu_sa_manager ib_pools[AMDGPU_IB_POOL_MAX];
875 struct amdgpu_sched gpu_sched[AMDGPU_HW_IP_NUM][AMDGPU_RING_PRIO_MAX];
876
877 /* interrupts */
878 struct amdgpu_irq irq;
879
880 /* powerplay */
881 struct amd_powerplay powerplay;
882 struct amdgpu_pm pm;
883 u64 cg_flags;
884 u32 pg_flags;
885
886 /* nbio */
887 struct amdgpu_nbio nbio;
888
889 /* hdp */
890 struct amdgpu_hdp hdp;
891
892 /* smuio */
893 struct amdgpu_smuio smuio;
894
895 /* mmhub */
896 struct amdgpu_mmhub mmhub;
897
898 /* gfxhub */
899 struct amdgpu_gfxhub gfxhub;
900
901 /* gfx */
902 struct amdgpu_gfx gfx;
903
904 /* sdma */
905 struct amdgpu_sdma sdma;
906
907 /* lsdma */
908 struct amdgpu_lsdma lsdma;
909
910 /* uvd */
911 struct amdgpu_uvd uvd;
912
913 /* vce */
914 struct amdgpu_vce vce;
915
916 /* vcn */
917 struct amdgpu_vcn vcn;
918
919 /* jpeg */
920 struct amdgpu_jpeg jpeg;
921
922 /* firmwares */
923 struct amdgpu_firmware firmware;
924
925 /* PSP */
926 struct psp_context psp;
927
928 /* GDS */
929 struct amdgpu_gds gds;
930
931 /* KFD */
932 struct amdgpu_kfd_dev kfd;
933
934 /* UMC */
935 struct amdgpu_umc umc;
936
937 /* display related functionality */
938 struct amdgpu_display_manager dm;
939
940 /* mes */
941 bool enable_mes;
942 bool enable_mes_kiq;
943 struct amdgpu_mes mes;
944 struct amdgpu_mqd mqds[AMDGPU_HW_IP_NUM];
945
946 /* df */
947 struct amdgpu_df df;
948
949 /* MCA */
950 struct amdgpu_mca mca;
951
952 struct amdgpu_ip_block ip_blocks[AMDGPU_MAX_IP_NUM];
953 uint32_t harvest_ip_mask;
954 int num_ip_blocks;
955 struct mutex mn_lock;
956 DECLARE_HASHTABLE(mn_hash, 7);
957
958 /* tracking pinned memory */
959 atomic64_t vram_pin_size;
960 atomic64_t visible_pin_size;
961 atomic64_t gart_pin_size;
962
963 /* soc15 register offset based on ip, instance and segment */
964 uint32_t *reg_offset[MAX_HWIP][HWIP_MAX_INSTANCE];
965
966 /* delayed work_func for deferring clockgating during resume */
967 struct delayed_work delayed_init_work;
968
969 struct amdgpu_virt virt;
970
971 /* link all shadow bo */
972 struct list_head shadow_list;
973 struct mutex shadow_list_lock;
974
975 /* record hw reset is performed */
976 bool has_hw_reset;
977 u8 reset_magic[AMDGPU_RESET_MAGIC_NUM];
978
979 /* s3/s4 mask */
980 bool in_suspend;
981 bool in_s3;
982 bool in_s4;
983 bool in_s0ix;
984
985 enum pp_mp1_state mp1_state;
986 struct amdgpu_doorbell_index doorbell_index;
987
988 struct mutex notifier_lock;
989
990 int asic_reset_res;
991 struct work_struct xgmi_reset_work;
992 struct list_head reset_list;
993
994 long gfx_timeout;
995 long sdma_timeout;
996 long video_timeout;
997 long compute_timeout;
998
999 uint64_t unique_id;
1000 uint64_t df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
1001
1002 /* enable runtime pm on the device */
1003 bool in_runpm;
1004 bool has_pr3;
1005
1006 bool ucode_sysfs_en;
1007 bool psp_sysfs_en;
1008
1009 /* Chip product information */
1010 char product_number[20];
1011 char product_name[AMDGPU_PRODUCT_NAME_LEN];
1012 char serial[20];
1013
1014 atomic_t throttling_logging_enabled;
1015 struct ratelimit_state throttling_logging_rs;
1016 uint32_t ras_hw_enabled;
1017 uint32_t ras_enabled;
1018
1019 bool no_hw_access;
1020 struct pci_saved_state *pci_state;
1021 pci_channel_state_t pci_channel_state;
1022
1023 struct amdgpu_reset_control *reset_cntl;
1024 uint32_t ip_versions[MAX_HWIP][HWIP_MAX_INSTANCE];
1025
1026 bool ram_is_direct_mapped;
1027
1028 struct list_head ras_list;
1029
1030 struct ip_discovery_top *ip_top;
1031
1032 struct amdgpu_reset_domain *reset_domain;
1033
1034 struct mutex benchmark_mutex;
1035
1036 /* reset dump register */
1037 uint32_t *reset_dump_reg_list;
1038 uint32_t *reset_dump_reg_value;
1039 int num_regs;
1040#ifdef CONFIG_DEV_COREDUMP
1041 struct amdgpu_task_info reset_task_info;
1042 bool reset_vram_lost;
1043 struct timespec64 reset_time;
1044#endif
1045
1046 bool scpm_enabled;
1047 uint32_t scpm_status;
1048
1049 struct work_struct reset_work;
1050
1051 bool job_hang;
1052 bool dc_enabled;
1053};
1054
1055static inline struct amdgpu_device *drm_to_adev(struct drm_device *ddev)
1056{
1057 return container_of(ddev, struct amdgpu_device, ddev);
1058}
1059
1060static inline struct drm_device *adev_to_drm(struct amdgpu_device *adev)
1061{
1062 return &adev->ddev;
1063}
1064
1065static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_device *bdev)
1066{
1067 return container_of(bdev, struct amdgpu_device, mman.bdev);
1068}
1069
1070int amdgpu_device_init(struct amdgpu_device *adev,
1071 uint32_t flags);
1072void amdgpu_device_fini_hw(struct amdgpu_device *adev);
1073void amdgpu_device_fini_sw(struct amdgpu_device *adev);
1074
1075int amdgpu_gpu_wait_for_idle(struct amdgpu_device *adev);
1076
1077void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
1078 void *buf, size_t size, bool write);
1079size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
1080 void *buf, size_t size, bool write);
1081
1082void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
1083 void *buf, size_t size, bool write);
1084uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
1085 uint32_t reg, uint32_t acc_flags);
1086void amdgpu_device_wreg(struct amdgpu_device *adev,
1087 uint32_t reg, uint32_t v,
1088 uint32_t acc_flags);
1089void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
1090 uint32_t reg, uint32_t v);
1091void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value);
1092uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset);
1093
1094u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
1095 u32 reg_addr);
1096u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
1097 u32 reg_addr);
1098void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
1099 u32 reg_addr, u32 reg_data);
1100void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
1101 u32 reg_addr, u64 reg_data);
1102u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev);
1103bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type);
1104bool amdgpu_device_has_dc_support(struct amdgpu_device *adev);
1105
1106void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev);
1107
1108int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
1109 struct amdgpu_reset_context *reset_context);
1110
1111int amdgpu_do_asic_reset(struct list_head *device_list_handle,
1112 struct amdgpu_reset_context *reset_context);
1113
1114int emu_soc_asic_init(struct amdgpu_device *adev);
1115
1116/*
1117 * Registers read & write functions.
1118 */
1119#define AMDGPU_REGS_NO_KIQ (1<<1)
1120#define AMDGPU_REGS_RLC (1<<2)
1121
1122#define RREG32_NO_KIQ(reg) amdgpu_device_rreg(adev, (reg), AMDGPU_REGS_NO_KIQ)
1123#define WREG32_NO_KIQ(reg, v) amdgpu_device_wreg(adev, (reg), (v), AMDGPU_REGS_NO_KIQ)
1124
1125#define RREG32_KIQ(reg) amdgpu_kiq_rreg(adev, (reg))
1126#define WREG32_KIQ(reg, v) amdgpu_kiq_wreg(adev, (reg), (v))
1127
1128#define RREG8(reg) amdgpu_mm_rreg8(adev, (reg))
1129#define WREG8(reg, v) amdgpu_mm_wreg8(adev, (reg), (v))
1130
1131#define RREG32(reg) amdgpu_device_rreg(adev, (reg), 0)
1132#define DREG32(reg) printk(KERN_INFO "REGISTER: " #reg " : 0x%08X\n", amdgpu_device_rreg(adev, (reg), 0))
1133#define WREG32(reg, v) amdgpu_device_wreg(adev, (reg), (v), 0)
1134#define REG_SET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1135#define REG_GET(FIELD, v) (((v) << FIELD##_SHIFT) & FIELD##_MASK)
1136#define RREG32_PCIE(reg) adev->pcie_rreg(adev, (reg))
1137#define WREG32_PCIE(reg, v) adev->pcie_wreg(adev, (reg), (v))
1138#define RREG32_PCIE_PORT(reg) adev->pciep_rreg(adev, (reg))
1139#define WREG32_PCIE_PORT(reg, v) adev->pciep_wreg(adev, (reg), (v))
1140#define RREG64_PCIE(reg) adev->pcie_rreg64(adev, (reg))
1141#define WREG64_PCIE(reg, v) adev->pcie_wreg64(adev, (reg), (v))
1142#define RREG32_SMC(reg) adev->smc_rreg(adev, (reg))
1143#define WREG32_SMC(reg, v) adev->smc_wreg(adev, (reg), (v))
1144#define RREG32_UVD_CTX(reg) adev->uvd_ctx_rreg(adev, (reg))
1145#define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v))
1146#define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg))
1147#define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))
1148#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))
1149#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))
1150#define RREG32_SE_CAC(reg) adev->se_cac_rreg(adev, (reg))
1151#define WREG32_SE_CAC(reg, v) adev->se_cac_wreg(adev, (reg), (v))
1152#define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))
1153#define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))
1154#define WREG32_P(reg, val, mask) \
1155 do { \
1156 uint32_t tmp_ = RREG32(reg); \
1157 tmp_ &= (mask); \
1158 tmp_ |= ((val) & ~(mask)); \
1159 WREG32(reg, tmp_); \
1160 } while (0)
1161#define WREG32_AND(reg, and) WREG32_P(reg, 0, and)
1162#define WREG32_OR(reg, or) WREG32_P(reg, or, ~(or))
1163#define WREG32_PLL_P(reg, val, mask) \
1164 do { \
1165 uint32_t tmp_ = RREG32_PLL(reg); \
1166 tmp_ &= (mask); \
1167 tmp_ |= ((val) & ~(mask)); \
1168 WREG32_PLL(reg, tmp_); \
1169 } while (0)
1170
1171#define WREG32_SMC_P(_Reg, _Val, _Mask) \
1172 do { \
1173 u32 tmp = RREG32_SMC(_Reg); \
1174 tmp &= (_Mask); \
1175 tmp |= ((_Val) & ~(_Mask)); \
1176 WREG32_SMC(_Reg, tmp); \
1177 } while (0)
1178
1179#define DREG32_SYS(sqf, adev, reg) seq_printf((sqf), #reg " : 0x%08X\n", amdgpu_device_rreg((adev), (reg), false))
1180
1181#define REG_FIELD_SHIFT(reg, field) reg##__##field##__SHIFT
1182#define REG_FIELD_MASK(reg, field) reg##__##field##_MASK
1183
1184#define REG_SET_FIELD(orig_val, reg, field, field_val) \
1185 (((orig_val) & ~REG_FIELD_MASK(reg, field)) | \
1186 (REG_FIELD_MASK(reg, field) & ((field_val) << REG_FIELD_SHIFT(reg, field))))
1187
1188#define REG_GET_FIELD(value, reg, field) \
1189 (((value) & REG_FIELD_MASK(reg, field)) >> REG_FIELD_SHIFT(reg, field))
1190
1191#define WREG32_FIELD(reg, field, val) \
1192 WREG32(mm##reg, (RREG32(mm##reg) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1193
1194#define WREG32_FIELD_OFFSET(reg, offset, field, val) \
1195 WREG32(mm##reg + offset, (RREG32(mm##reg + offset) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
1196
1197/*
1198 * BIOS helpers.
1199 */
1200#define RBIOS8(i) (adev->bios[i])
1201#define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
1202#define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
1203
1204/*
1205 * ASICs macro.
1206 */
1207#define amdgpu_asic_set_vga_state(adev, state) (adev)->asic_funcs->set_vga_state((adev), (state))
1208#define amdgpu_asic_reset(adev) (adev)->asic_funcs->reset((adev))
1209#define amdgpu_asic_reset_method(adev) (adev)->asic_funcs->reset_method((adev))
1210#define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
1211#define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
1212#define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
1213#define amdgpu_get_pcie_lanes(adev) (adev)->asic_funcs->get_pcie_lanes((adev))
1214#define amdgpu_set_pcie_lanes(adev, l) (adev)->asic_funcs->set_pcie_lanes((adev), (l))
1215#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
1216#define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
1217#define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
1218#define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
1219#define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev))
1220#define amdgpu_asic_flush_hdp(adev, r) \
1221 ((adev)->asic_funcs->flush_hdp ? (adev)->asic_funcs->flush_hdp((adev), (r)) : (adev)->hdp.funcs->flush_hdp((adev), (r)))
1222#define amdgpu_asic_invalidate_hdp(adev, r) \
1223 ((adev)->asic_funcs->invalidate_hdp ? (adev)->asic_funcs->invalidate_hdp((adev), (r)) : \
1224 ((adev)->hdp.funcs->invalidate_hdp ? (adev)->hdp.funcs->invalidate_hdp((adev), (r)) : (void)0))
1225#define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev))
1226#define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev))
1227#define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1)))
1228#define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev))
1229#define amdgpu_asic_get_pcie_replay_count(adev) ((adev)->asic_funcs->get_pcie_replay_count((adev)))
1230#define amdgpu_asic_supports_baco(adev) (adev)->asic_funcs->supports_baco((adev))
1231#define amdgpu_asic_pre_asic_init(adev) (adev)->asic_funcs->pre_asic_init((adev))
1232#define amdgpu_asic_update_umd_stable_pstate(adev, enter) \
1233 ((adev)->asic_funcs->update_umd_stable_pstate ? (adev)->asic_funcs->update_umd_stable_pstate((adev), (enter)) : 0)
1234#define amdgpu_asic_query_video_codecs(adev, e, c) (adev)->asic_funcs->query_video_codecs((adev), (e), (c))
1235
1236#define amdgpu_inc_vram_lost(adev) atomic_inc(&((adev)->vram_lost_counter));
1237
1238#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
1239
1240/* Common functions */
1241bool amdgpu_device_has_job_running(struct amdgpu_device *adev);
1242bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev);
1243int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
1244 struct amdgpu_job *job,
1245 struct amdgpu_reset_context *reset_context);
1246void amdgpu_device_pci_config_reset(struct amdgpu_device *adev);
1247int amdgpu_device_pci_reset(struct amdgpu_device *adev);
1248bool amdgpu_device_need_post(struct amdgpu_device *adev);
1249bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev);
1250bool amdgpu_device_aspm_support_quirk(void);
1251
1252void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
1253 u64 num_vis_bytes);
1254int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev);
1255void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1256 const u32 *registers,
1257 const u32 array_size);
1258
1259int amdgpu_device_mode1_reset(struct amdgpu_device *adev);
1260bool amdgpu_device_supports_atpx(struct drm_device *dev);
1261bool amdgpu_device_supports_px(struct drm_device *dev);
1262bool amdgpu_device_supports_boco(struct drm_device *dev);
1263bool amdgpu_device_supports_smart_shift(struct drm_device *dev);
1264bool amdgpu_device_supports_baco(struct drm_device *dev);
1265bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
1266 struct amdgpu_device *peer_adev);
1267int amdgpu_device_baco_enter(struct drm_device *dev);
1268int amdgpu_device_baco_exit(struct drm_device *dev);
1269
1270void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
1271 struct amdgpu_ring *ring);
1272void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
1273 struct amdgpu_ring *ring);
1274
1275void amdgpu_device_halt(struct amdgpu_device *adev);
1276u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
1277 u32 reg);
1278void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
1279 u32 reg, u32 v);
1280struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
1281 struct dma_fence *gang);
1282bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev);
1283
1284/* atpx handler */
1285#if defined(CONFIG_VGA_SWITCHEROO)
1286void amdgpu_register_atpx_handler(void);
1287void amdgpu_unregister_atpx_handler(void);
1288bool amdgpu_has_atpx_dgpu_power_cntl(void);
1289bool amdgpu_is_atpx_hybrid(void);
1290bool amdgpu_atpx_dgpu_req_power_for_displays(void);
1291bool amdgpu_has_atpx(void);
1292#else
1293static inline void amdgpu_register_atpx_handler(void) {}
1294static inline void amdgpu_unregister_atpx_handler(void) {}
1295static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
1296static inline bool amdgpu_is_atpx_hybrid(void) { return false; }
1297static inline bool amdgpu_atpx_dgpu_req_power_for_displays(void) { return false; }
1298static inline bool amdgpu_has_atpx(void) { return false; }
1299#endif
1300
1301#if defined(CONFIG_VGA_SWITCHEROO) && defined(CONFIG_ACPI)
1302void *amdgpu_atpx_get_dhandle(void);
1303#else
1304static inline void *amdgpu_atpx_get_dhandle(void) { return NULL; }
1305#endif
1306
1307/*
1308 * KMS
1309 */
1310extern const struct drm_ioctl_desc amdgpu_ioctls_kms[];
1311extern const int amdgpu_max_kms_ioctl;
1312
1313int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags);
1314void amdgpu_driver_unload_kms(struct drm_device *dev);
1315void amdgpu_driver_lastclose_kms(struct drm_device *dev);
1316int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv);
1317void amdgpu_driver_postclose_kms(struct drm_device *dev,
1318 struct drm_file *file_priv);
1319void amdgpu_driver_release_kms(struct drm_device *dev);
1320
1321int amdgpu_device_ip_suspend(struct amdgpu_device *adev);
1322int amdgpu_device_suspend(struct drm_device *dev, bool fbcon);
1323int amdgpu_device_resume(struct drm_device *dev, bool fbcon);
1324u32 amdgpu_get_vblank_counter_kms(struct drm_crtc *crtc);
1325int amdgpu_enable_vblank_kms(struct drm_crtc *crtc);
1326void amdgpu_disable_vblank_kms(struct drm_crtc *crtc);
1327int amdgpu_info_ioctl(struct drm_device *dev, void *data,
1328 struct drm_file *filp);
1329
1330/*
1331 * functions used by amdgpu_encoder.c
1332 */
1333struct amdgpu_afmt_acr {
1334 u32 clock;
1335
1336 int n_32khz;
1337 int cts_32khz;
1338
1339 int n_44_1khz;
1340 int cts_44_1khz;
1341
1342 int n_48khz;
1343 int cts_48khz;
1344
1345};
1346
1347struct amdgpu_afmt_acr amdgpu_afmt_acr(uint32_t clock);
1348
1349/* amdgpu_acpi.c */
1350
1351/* ATCS Device/Driver State */
1352#define AMDGPU_ATCS_PSC_DEV_STATE_D0 0
1353#define AMDGPU_ATCS_PSC_DEV_STATE_D3_HOT 3
1354#define AMDGPU_ATCS_PSC_DRV_STATE_OPR 0
1355#define AMDGPU_ATCS_PSC_DRV_STATE_NOT_OPR 1
1356
1357#if defined(CONFIG_ACPI)
1358int amdgpu_acpi_init(struct amdgpu_device *adev);
1359void amdgpu_acpi_fini(struct amdgpu_device *adev);
1360bool amdgpu_acpi_is_pcie_performance_request_supported(struct amdgpu_device *adev);
1361bool amdgpu_acpi_is_power_shift_control_supported(void);
1362int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
1363 u8 perf_req, bool advertise);
1364int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
1365 u8 dev_state, bool drv_state);
1366int amdgpu_acpi_smart_shift_update(struct drm_device *dev, enum amdgpu_ss ss_state);
1367int amdgpu_acpi_pcie_notify_device_ready(struct amdgpu_device *adev);
1368
1369void amdgpu_acpi_get_backlight_caps(struct amdgpu_dm_backlight_caps *caps);
1370bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev);
1371void amdgpu_acpi_detect(void);
1372#else
1373static inline int amdgpu_acpi_init(struct amdgpu_device *adev) { return 0; }
1374static inline void amdgpu_acpi_fini(struct amdgpu_device *adev) { }
1375static inline bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev) { return false; }
1376static inline void amdgpu_acpi_detect(void) { }
1377static inline bool amdgpu_acpi_is_power_shift_control_supported(void) { return false; }
1378static inline int amdgpu_acpi_power_shift_control(struct amdgpu_device *adev,
1379 u8 dev_state, bool drv_state) { return 0; }
1380static inline int amdgpu_acpi_smart_shift_update(struct drm_device *dev,
1381 enum amdgpu_ss ss_state) { return 0; }
1382#endif
1383
1384#if defined(CONFIG_ACPI) && defined(CONFIG_SUSPEND)
1385bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev);
1386bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev);
1387#else
1388static inline bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev) { return false; }
1389static inline bool amdgpu_acpi_is_s3_active(struct amdgpu_device *adev) { return false; }
1390#endif
1391
1392#if defined(CONFIG_DRM_AMD_DC)
1393int amdgpu_dm_display_resume(struct amdgpu_device *adev );
1394#else
1395static inline int amdgpu_dm_display_resume(struct amdgpu_device *adev) { return 0; }
1396#endif
1397
1398
1399void amdgpu_register_gpu_instance(struct amdgpu_device *adev);
1400void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev);
1401
1402pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev,
1403 pci_channel_state_t state);
1404pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev);
1405pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev);
1406void amdgpu_pci_resume(struct pci_dev *pdev);
1407
1408bool amdgpu_device_cache_pci_state(struct pci_dev *pdev);
1409bool amdgpu_device_load_pci_state(struct pci_dev *pdev);
1410
1411bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev);
1412
1413int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
1414 enum amd_clockgating_state state);
1415int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
1416 enum amd_powergating_state state);
1417
1418static inline bool amdgpu_device_has_timeouts_enabled(struct amdgpu_device *adev)
1419{
1420 return amdgpu_gpu_recovery != 0 &&
1421 adev->gfx_timeout != MAX_SCHEDULE_TIMEOUT &&
1422 adev->compute_timeout != MAX_SCHEDULE_TIMEOUT &&
1423 adev->sdma_timeout != MAX_SCHEDULE_TIMEOUT &&
1424 adev->video_timeout != MAX_SCHEDULE_TIMEOUT;
1425}
1426
1427#include "amdgpu_object.h"
1428
1429static inline bool amdgpu_is_tmz(struct amdgpu_device *adev)
1430{
1431 return adev->gmc.tmz_enabled;
1432}
1433
1434int amdgpu_in_reset(struct amdgpu_device *adev);
1435
1436#endif
1437

source code of linux/drivers/gpu/drm/amd/amdgpu/amdgpu.h