1 | /* SPDX-License-Identifier: MIT */ |
2 | #ifndef __NVKM_FIFO_PRIV_H__ |
3 | #define __NVKM_FIFO_PRIV_H__ |
4 | #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) |
5 | #include <engine/fifo.h> |
6 | #include <core/enum.h> |
7 | struct nvkm_cctx; |
8 | struct nvkm_cgrp; |
9 | struct nvkm_chan; |
10 | struct nvkm_engn; |
11 | struct nvkm_memory; |
12 | struct nvkm_runl; |
13 | struct nvkm_runq; |
14 | struct nvkm_vctx; |
15 | |
16 | struct nvkm_fifo_func { |
17 | void (*dtor)(struct nvkm_fifo *); |
18 | |
19 | int (*chid_nr)(struct nvkm_fifo *); |
20 | int (*chid_ctor)(struct nvkm_fifo *, int nr); |
21 | int (*runq_nr)(struct nvkm_fifo *); |
22 | int (*runl_ctor)(struct nvkm_fifo *); |
23 | |
24 | void (*init)(struct nvkm_fifo *); |
25 | void (*init_pbdmas)(struct nvkm_fifo *, u32 mask); |
26 | |
27 | irqreturn_t (*intr)(struct nvkm_inth *); |
28 | void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit); |
29 | void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm); |
30 | |
31 | const struct nvkm_fifo_func_mmu_fault { |
32 | void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *); |
33 | const struct nvkm_enum *access; |
34 | const struct nvkm_enum *engine; |
35 | const struct nvkm_enum *reason; |
36 | const struct nvkm_enum *hubclient; |
37 | const struct nvkm_enum *gpcclient; |
38 | } *mmu_fault; |
39 | |
40 | void (*pause)(struct nvkm_fifo *, unsigned long *); |
41 | void (*start)(struct nvkm_fifo *, unsigned long *); |
42 | |
43 | int (*nonstall_ctor)(struct nvkm_fifo *); |
44 | const struct nvkm_event_func *nonstall; |
45 | |
46 | const struct nvkm_runl_func *runl; |
47 | const struct nvkm_runq_func *runq; |
48 | const struct nvkm_engn_func *engn; |
49 | const struct nvkm_engn_func *engn_sw; |
50 | const struct nvkm_engn_func *engn_ce; |
51 | |
52 | struct nvkm_fifo_func_cgrp { |
53 | struct nvkm_sclass user; |
54 | const struct nvkm_cgrp_func *func; |
55 | bool force; |
56 | } cgrp; |
57 | |
58 | struct nvkm_fifo_func_chan { |
59 | struct nvkm_sclass user; |
60 | const struct nvkm_chan_func *func; |
61 | } chan; |
62 | }; |
63 | |
64 | int r535_fifo_new(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, |
65 | struct nvkm_fifo **); |
66 | int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, |
67 | struct nvkm_fifo **); |
68 | |
69 | int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); |
70 | int nv04_fifo_runl_ctor(struct nvkm_fifo *); |
71 | void nv04_fifo_init(struct nvkm_fifo *); |
72 | irqreturn_t nv04_fifo_intr(struct nvkm_inth *); |
73 | void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); |
74 | void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); |
75 | extern const struct nvkm_runl_func nv04_runl; |
76 | extern const struct nvkm_engn_func nv04_engn; |
77 | extern const struct nvkm_cgrp_func nv04_cgrp; |
78 | extern const struct nvkm_chan_func_inst nv04_chan_inst; |
79 | extern const struct nvkm_chan_func_userd nv04_chan_userd; |
80 | void nv04_chan_ramfc_clear(struct nvkm_chan *); |
81 | void nv04_chan_start(struct nvkm_chan *); |
82 | void nv04_chan_stop(struct nvkm_chan *); |
83 | void nv04_eobj_ramht_del(struct nvkm_chan *, int); |
84 | |
85 | int nv10_fifo_chid_nr(struct nvkm_fifo *); |
86 | |
87 | int nv50_fifo_chid_nr(struct nvkm_fifo *); |
88 | int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); |
89 | void nv50_fifo_init(struct nvkm_fifo *); |
90 | extern const struct nvkm_runl_func nv50_runl; |
91 | int nv50_runl_update(struct nvkm_runl *); |
92 | int nv50_runl_wait(struct nvkm_runl *); |
93 | extern const struct nvkm_engn_func nv50_engn_sw; |
94 | extern const struct nvkm_chan_func_inst nv50_chan_inst; |
95 | extern const struct nvkm_chan_func_userd nv50_chan_userd; |
96 | void nv50_chan_unbind(struct nvkm_chan *); |
97 | void nv50_chan_start(struct nvkm_chan *); |
98 | void nv50_chan_stop(struct nvkm_chan *); |
99 | void nv50_chan_preempt(struct nvkm_chan *); |
100 | int nv50_eobj_ramht_add(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *); |
101 | void nv50_eobj_ramht_del(struct nvkm_chan *, int); |
102 | |
103 | extern const struct nvkm_event_func g84_fifo_nonstall; |
104 | extern const struct nvkm_engn_func g84_engn; |
105 | extern const struct nvkm_chan_func g84_chan; |
106 | |
107 | int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); |
108 | int gf100_fifo_runq_nr(struct nvkm_fifo *); |
109 | bool gf100_fifo_intr_pbdma(struct nvkm_fifo *); |
110 | void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *); |
111 | void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); |
112 | void gf100_fifo_intr_sched(struct nvkm_fifo *); |
113 | void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32); |
114 | void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *); |
115 | extern const struct nvkm_enum gf100_fifo_mmu_fault_access[]; |
116 | extern const struct nvkm_event_func gf100_fifo_nonstall; |
117 | bool gf100_runl_preempt_pending(struct nvkm_runl *); |
118 | void gf100_runq_init(struct nvkm_runq *); |
119 | bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); |
120 | void gf100_engn_mmu_fault_trigger(struct nvkm_engn *); |
121 | bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *); |
122 | extern const struct nvkm_engn_func gf100_engn_sw; |
123 | extern const struct nvkm_chan_func_inst gf100_chan_inst; |
124 | void gf100_chan_userd_clear(struct nvkm_chan *); |
125 | void gf100_chan_preempt(struct nvkm_chan *); |
126 | |
127 | int gk104_fifo_chid_nr(struct nvkm_fifo *); |
128 | int gk104_fifo_runl_ctor(struct nvkm_fifo *); |
129 | void gk104_fifo_init(struct nvkm_fifo *); |
130 | void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32); |
131 | irqreturn_t gk104_fifo_intr(struct nvkm_inth *); |
132 | void gk104_fifo_intr_runlist(struct nvkm_fifo *); |
133 | void gk104_fifo_intr_chsw(struct nvkm_fifo *); |
134 | void gk104_fifo_intr_bind(struct nvkm_fifo *); |
135 | extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; |
136 | extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; |
137 | extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; |
138 | extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; |
139 | void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); |
140 | void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int); |
141 | bool gk104_runl_pending(struct nvkm_runl *); |
142 | void gk104_runl_block(struct nvkm_runl *, u32); |
143 | void gk104_runl_allow(struct nvkm_runl *, u32); |
144 | void gk104_runl_fault_clear(struct nvkm_runl *); |
145 | extern const struct nvkm_runq_func gk104_runq; |
146 | void gk104_runq_init(struct nvkm_runq *); |
147 | bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); |
148 | extern const struct nvkm_bitfield gk104_runq_intr_0_names[]; |
149 | bool gk104_runq_idle(struct nvkm_runq *); |
150 | extern const struct nvkm_engn_func gk104_engn; |
151 | bool gk104_engn_chsw(struct nvkm_engn *); |
152 | int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); |
153 | int gk104_ectx_ctor(struct nvkm_engn *, struct nvkm_vctx *); |
154 | extern const struct nvkm_engn_func gk104_engn_ce; |
155 | extern const struct nvkm_chan_func_userd gk104_chan_userd; |
156 | extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc; |
157 | void gk104_chan_bind(struct nvkm_chan *); |
158 | void gk104_chan_bind_inst(struct nvkm_chan *); |
159 | void gk104_chan_unbind(struct nvkm_chan *); |
160 | void gk104_chan_start(struct nvkm_chan *); |
161 | void gk104_chan_stop(struct nvkm_chan *); |
162 | |
163 | int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); |
164 | extern const struct nvkm_runl_func gk110_runl; |
165 | extern const struct nvkm_cgrp_func gk110_cgrp; |
166 | void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); |
167 | extern const struct nvkm_chan_func gk110_chan; |
168 | void gk110_chan_preempt(struct nvkm_chan *); |
169 | |
170 | extern const struct nvkm_runq_func gk208_runq; |
171 | void gk208_runq_init(struct nvkm_runq *); |
172 | |
173 | void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); |
174 | extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; |
175 | extern const struct nvkm_runl_func gm107_runl; |
176 | extern const struct nvkm_chan_func gm107_chan; |
177 | |
178 | int gm200_fifo_chid_nr(struct nvkm_fifo *); |
179 | int gm200_fifo_runq_nr(struct nvkm_fifo *); |
180 | |
181 | extern const struct nvkm_enum gv100_fifo_mmu_fault_access[]; |
182 | extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[]; |
183 | extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[]; |
184 | extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[]; |
185 | void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); |
186 | void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); |
187 | void gv100_runl_preempt(struct nvkm_runl *); |
188 | extern const struct nvkm_runq_func gv100_runq; |
189 | extern const struct nvkm_engn_func gv100_engn; |
190 | void gv100_ectx_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); |
191 | extern const struct nvkm_engn_func gv100_engn_ce; |
192 | int gv100_ectx_ce_ctor(struct nvkm_engn *, struct nvkm_vctx *); |
193 | void gv100_ectx_ce_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); |
194 | extern const struct nvkm_chan_func_userd gv100_chan_userd; |
195 | extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; |
196 | |
197 | void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); |
198 | extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; |
199 | u32 tu102_chan_doorbell_handle(struct nvkm_chan *); |
200 | |
201 | int ga100_fifo_runl_ctor(struct nvkm_fifo *); |
202 | int ga100_fifo_nonstall_ctor(struct nvkm_fifo *); |
203 | extern const struct nvkm_event_func ga100_fifo_nonstall; |
204 | extern const struct nvkm_runl_func ga100_runl; |
205 | extern const struct nvkm_runq_func ga100_runq; |
206 | extern const struct nvkm_engn_func ga100_engn; |
207 | extern const struct nvkm_engn_func ga100_engn_ce; |
208 | extern const struct nvkm_cgrp_func ga100_cgrp; |
209 | extern const struct nvkm_chan_func ga100_chan; |
210 | |
211 | u32 gb202_chan_doorbell_handle(struct nvkm_chan *); |
212 | |
213 | int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, |
214 | void *argv, u32 argc, struct nvkm_object **); |
215 | int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc, |
216 | struct nvkm_object **); |
217 | #endif |
218 | |