1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * PCI Virtual Channel support |
4 | * |
5 | * Copyright (C) 2013 Red Hat, Inc. All rights reserved. |
6 | * Author: Alex Williamson <alex.williamson@redhat.com> |
7 | */ |
8 | |
9 | #include <linux/bitfield.h> |
10 | #include <linux/device.h> |
11 | #include <linux/kernel.h> |
12 | #include <linux/module.h> |
13 | #include <linux/pci.h> |
14 | #include <linux/pci_regs.h> |
15 | #include <linux/types.h> |
16 | |
17 | #include "pci.h" |
18 | |
19 | /** |
20 | * pci_vc_save_restore_dwords - Save or restore a series of dwords |
21 | * @dev: device |
22 | * @pos: starting config space position |
23 | * @buf: buffer to save to or restore from |
24 | * @dwords: number of dwords to save/restore |
25 | * @save: whether to save or restore |
26 | */ |
27 | static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos, |
28 | u32 *buf, int dwords, bool save) |
29 | { |
30 | int i; |
31 | |
32 | for (i = 0; i < dwords; i++, buf++) { |
33 | if (save) |
34 | pci_read_config_dword(dev, where: pos + (i * 4), val: buf); |
35 | else |
36 | pci_write_config_dword(dev, where: pos + (i * 4), val: *buf); |
37 | } |
38 | } |
39 | |
40 | /** |
41 | * pci_vc_load_arb_table - load and wait for VC arbitration table |
42 | * @dev: device |
43 | * @pos: starting position of VC capability (VC/VC9/MFVC) |
44 | * |
45 | * Set Load VC Arbitration Table bit requesting hardware to apply the VC |
46 | * Arbitration Table (previously loaded). When the VC Arbitration Table |
47 | * Status clears, hardware has latched the table into VC arbitration logic. |
48 | */ |
49 | static void pci_vc_load_arb_table(struct pci_dev *dev, int pos) |
50 | { |
51 | u16 ctrl; |
52 | |
53 | pci_read_config_word(dev, where: pos + PCI_VC_PORT_CTRL, val: &ctrl); |
54 | pci_write_config_word(dev, where: pos + PCI_VC_PORT_CTRL, |
55 | val: ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE); |
56 | if (pci_wait_for_pending(dev, pos: pos + PCI_VC_PORT_STATUS, |
57 | PCI_VC_PORT_STATUS_TABLE)) |
58 | return; |
59 | |
60 | pci_err(dev, "VC arbitration table failed to load\n" ); |
61 | } |
62 | |
63 | /** |
64 | * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table |
65 | * @dev: device |
66 | * @pos: starting position of VC capability (VC/VC9/MFVC) |
67 | * @res: VC resource number, ie. VCn (0-7) |
68 | * |
69 | * Set Load Port Arbitration Table bit requesting hardware to apply the Port |
70 | * Arbitration Table (previously loaded). When the Port Arbitration Table |
71 | * Status clears, hardware has latched the table into port arbitration logic. |
72 | */ |
73 | static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res) |
74 | { |
75 | int ctrl_pos, status_pos; |
76 | u32 ctrl; |
77 | |
78 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); |
79 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); |
80 | |
81 | pci_read_config_dword(dev, where: ctrl_pos, val: &ctrl); |
82 | pci_write_config_dword(dev, where: ctrl_pos, |
83 | val: ctrl | PCI_VC_RES_CTRL_LOAD_TABLE); |
84 | |
85 | if (pci_wait_for_pending(dev, pos: status_pos, PCI_VC_RES_STATUS_TABLE)) |
86 | return; |
87 | |
88 | pci_err(dev, "VC%d port arbitration table failed to load\n" , res); |
89 | } |
90 | |
91 | /** |
92 | * pci_vc_enable - Enable virtual channel |
93 | * @dev: device |
94 | * @pos: starting position of VC capability (VC/VC9/MFVC) |
95 | * @res: VC res number, ie. VCn (0-7) |
96 | * |
97 | * A VC is enabled by setting the enable bit in matching resource control |
98 | * registers on both sides of a link. We therefore need to find the opposite |
99 | * end of the link. To keep this simple we enable from the downstream device. |
100 | * RC devices do not have an upstream device, nor does it seem that VC9 do |
101 | * (spec is unclear). Once we find the upstream device, match the VC ID to |
102 | * get the correct resource, disable and enable on both ends. |
103 | */ |
104 | static void pci_vc_enable(struct pci_dev *dev, int pos, int res) |
105 | { |
106 | int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2; |
107 | u32 ctrl, , cap1, ctrl2; |
108 | struct pci_dev *link = NULL; |
109 | |
110 | /* Enable VCs from the downstream device */ |
111 | if (!pci_is_pcie(dev) || !pcie_downstream_port(dev)) |
112 | return; |
113 | |
114 | ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF); |
115 | status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF); |
116 | |
117 | pci_read_config_dword(dev, where: ctrl_pos, val: &ctrl); |
118 | id = ctrl & PCI_VC_RES_CTRL_ID; |
119 | |
120 | pci_read_config_dword(dev, where: pos, val: &header); |
121 | |
122 | /* If there is no opposite end of the link, skip to enable */ |
123 | if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 || |
124 | pci_is_root_bus(pbus: dev->bus)) |
125 | goto enable; |
126 | |
127 | pos2 = pci_find_ext_capability(dev: dev->bus->self, PCI_EXT_CAP_ID_VC); |
128 | if (!pos2) |
129 | goto enable; |
130 | |
131 | pci_read_config_dword(dev: dev->bus->self, where: pos2 + PCI_VC_PORT_CAP1, val: &cap1); |
132 | evcc = cap1 & PCI_VC_CAP1_EVCC; |
133 | |
134 | /* VC0 is hardwired enabled, so we can start with 1 */ |
135 | for (i = 1; i < evcc + 1; i++) { |
136 | ctrl_pos2 = pos2 + PCI_VC_RES_CTRL + |
137 | (i * PCI_CAP_VC_PER_VC_SIZEOF); |
138 | status_pos2 = pos2 + PCI_VC_RES_STATUS + |
139 | (i * PCI_CAP_VC_PER_VC_SIZEOF); |
140 | pci_read_config_dword(dev: dev->bus->self, where: ctrl_pos2, val: &ctrl2); |
141 | if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) { |
142 | link = dev->bus->self; |
143 | break; |
144 | } |
145 | } |
146 | |
147 | if (!link) |
148 | goto enable; |
149 | |
150 | /* Disable if enabled */ |
151 | if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) { |
152 | ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE; |
153 | pci_write_config_dword(dev: link, where: ctrl_pos2, val: ctrl2); |
154 | } |
155 | |
156 | /* Enable on both ends */ |
157 | ctrl2 |= PCI_VC_RES_CTRL_ENABLE; |
158 | pci_write_config_dword(dev: link, where: ctrl_pos2, val: ctrl2); |
159 | enable: |
160 | ctrl |= PCI_VC_RES_CTRL_ENABLE; |
161 | pci_write_config_dword(dev, where: ctrl_pos, val: ctrl); |
162 | |
163 | if (!pci_wait_for_pending(dev, pos: status_pos, PCI_VC_RES_STATUS_NEGO)) |
164 | pci_err(dev, "VC%d negotiation stuck pending\n" , id); |
165 | |
166 | if (link && !pci_wait_for_pending(dev: link, pos: status_pos2, |
167 | PCI_VC_RES_STATUS_NEGO)) |
168 | pci_err(link, "VC%d negotiation stuck pending\n" , id); |
169 | } |
170 | |
171 | /** |
172 | * pci_vc_do_save_buffer - Size, save, or restore VC state |
173 | * @dev: device |
174 | * @pos: starting position of VC capability (VC/VC9/MFVC) |
175 | * @save_state: buffer for save/restore |
176 | * @save: if provided a buffer, this indicates what to do with it |
177 | * |
178 | * Walking Virtual Channel config space to size, save, or restore it |
179 | * is complicated, so we do it all from one function to reduce code and |
180 | * guarantee ordering matches in the buffer. When called with NULL |
181 | * @save_state, return the size of the necessary save buffer. When called |
182 | * with a non-NULL @save_state, @save determines whether we save to the |
183 | * buffer or restore from it. |
184 | */ |
185 | static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos, |
186 | struct pci_cap_saved_state *save_state, |
187 | bool save) |
188 | { |
189 | u32 cap1; |
190 | char evcc, lpevcc, parb_size; |
191 | int i, len = 0; |
192 | u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL; |
193 | |
194 | /* Sanity check buffer size for save/restore */ |
195 | if (buf && save_state->cap.size != |
196 | pci_vc_do_save_buffer(dev, pos, NULL, save)) { |
197 | pci_err(dev, "VC save buffer size does not match @0x%x\n" , pos); |
198 | return -ENOMEM; |
199 | } |
200 | |
201 | pci_read_config_dword(dev, where: pos + PCI_VC_PORT_CAP1, val: &cap1); |
202 | /* Extended VC Count (not counting VC0) */ |
203 | evcc = cap1 & PCI_VC_CAP1_EVCC; |
204 | /* Low Priority Extended VC Count (not counting VC0) */ |
205 | lpevcc = FIELD_GET(PCI_VC_CAP1_LPEVCC, cap1); |
206 | /* Port Arbitration Table Entry Size (bits) */ |
207 | parb_size = 1 << FIELD_GET(PCI_VC_CAP1_ARB_SIZE, cap1); |
208 | |
209 | /* |
210 | * Port VC Control Register contains VC Arbitration Select, which |
211 | * cannot be modified when more than one LPVC is in operation. We |
212 | * therefore save/restore it first, as only VC0 should be enabled |
213 | * after device reset. |
214 | */ |
215 | if (buf) { |
216 | if (save) |
217 | pci_read_config_word(dev, where: pos + PCI_VC_PORT_CTRL, |
218 | val: (u16 *)buf); |
219 | else |
220 | pci_write_config_word(dev, where: pos + PCI_VC_PORT_CTRL, |
221 | val: *(u16 *)buf); |
222 | buf += 4; |
223 | } |
224 | len += 4; |
225 | |
226 | /* |
227 | * If we have any Low Priority VCs and a VC Arbitration Table Offset |
228 | * in Port VC Capability Register 2 then save/restore it next. |
229 | */ |
230 | if (lpevcc) { |
231 | u32 cap2; |
232 | int vcarb_offset; |
233 | |
234 | pci_read_config_dword(dev, where: pos + PCI_VC_PORT_CAP2, val: &cap2); |
235 | vcarb_offset = FIELD_GET(PCI_VC_CAP2_ARB_OFF, cap2) * 16; |
236 | |
237 | if (vcarb_offset) { |
238 | int size, vcarb_phases = 0; |
239 | |
240 | if (cap2 & PCI_VC_CAP2_128_PHASE) |
241 | vcarb_phases = 128; |
242 | else if (cap2 & PCI_VC_CAP2_64_PHASE) |
243 | vcarb_phases = 64; |
244 | else if (cap2 & PCI_VC_CAP2_32_PHASE) |
245 | vcarb_phases = 32; |
246 | |
247 | /* Fixed 4 bits per phase per lpevcc (plus VC0) */ |
248 | size = ((lpevcc + 1) * vcarb_phases * 4) / 8; |
249 | |
250 | if (size && buf) { |
251 | pci_vc_save_restore_dwords(dev, |
252 | pos: pos + vcarb_offset, |
253 | buf: (u32 *)buf, |
254 | dwords: size / 4, save); |
255 | /* |
256 | * On restore, we need to signal hardware to |
257 | * re-load the VC Arbitration Table. |
258 | */ |
259 | if (!save) |
260 | pci_vc_load_arb_table(dev, pos); |
261 | |
262 | buf += size; |
263 | } |
264 | len += size; |
265 | } |
266 | } |
267 | |
268 | /* |
269 | * In addition to each VC Resource Control Register, we may have a |
270 | * Port Arbitration Table attached to each VC. The Port Arbitration |
271 | * Table Offset in each VC Resource Capability Register tells us if |
272 | * it exists. The entry size is global from the Port VC Capability |
273 | * Register1 above. The number of phases is determined per VC. |
274 | */ |
275 | for (i = 0; i < evcc + 1; i++) { |
276 | u32 cap; |
277 | int parb_offset; |
278 | |
279 | pci_read_config_dword(dev, where: pos + PCI_VC_RES_CAP + |
280 | (i * PCI_CAP_VC_PER_VC_SIZEOF), val: &cap); |
281 | parb_offset = FIELD_GET(PCI_VC_RES_CAP_ARB_OFF, cap) * 16; |
282 | if (parb_offset) { |
283 | int size, parb_phases = 0; |
284 | |
285 | if (cap & PCI_VC_RES_CAP_256_PHASE) |
286 | parb_phases = 256; |
287 | else if (cap & (PCI_VC_RES_CAP_128_PHASE | |
288 | PCI_VC_RES_CAP_128_PHASE_TB)) |
289 | parb_phases = 128; |
290 | else if (cap & PCI_VC_RES_CAP_64_PHASE) |
291 | parb_phases = 64; |
292 | else if (cap & PCI_VC_RES_CAP_32_PHASE) |
293 | parb_phases = 32; |
294 | |
295 | size = (parb_size * parb_phases) / 8; |
296 | |
297 | if (size && buf) { |
298 | pci_vc_save_restore_dwords(dev, |
299 | pos: pos + parb_offset, |
300 | buf: (u32 *)buf, |
301 | dwords: size / 4, save); |
302 | buf += size; |
303 | } |
304 | len += size; |
305 | } |
306 | |
307 | /* VC Resource Control Register */ |
308 | if (buf) { |
309 | int ctrl_pos = pos + PCI_VC_RES_CTRL + |
310 | (i * PCI_CAP_VC_PER_VC_SIZEOF); |
311 | if (save) |
312 | pci_read_config_dword(dev, where: ctrl_pos, |
313 | val: (u32 *)buf); |
314 | else { |
315 | u32 tmp, ctrl = *(u32 *)buf; |
316 | /* |
317 | * For an FLR case, the VC config may remain. |
318 | * Preserve enable bit, restore the rest. |
319 | */ |
320 | pci_read_config_dword(dev, where: ctrl_pos, val: &tmp); |
321 | tmp &= PCI_VC_RES_CTRL_ENABLE; |
322 | tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE; |
323 | pci_write_config_dword(dev, where: ctrl_pos, val: tmp); |
324 | /* Load port arbitration table if used */ |
325 | if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT) |
326 | pci_vc_load_port_arb_table(dev, pos, res: i); |
327 | /* Re-enable if needed */ |
328 | if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE) |
329 | pci_vc_enable(dev, pos, res: i); |
330 | } |
331 | buf += 4; |
332 | } |
333 | len += 4; |
334 | } |
335 | |
336 | return buf ? 0 : len; |
337 | } |
338 | |
339 | static struct { |
340 | u16 id; |
341 | const char *name; |
342 | } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" }, |
343 | { PCI_EXT_CAP_ID_VC, "VC" }, |
344 | { PCI_EXT_CAP_ID_VC9, "VC9" } }; |
345 | |
346 | /** |
347 | * pci_save_vc_state - Save VC state to pre-allocate save buffer |
348 | * @dev: device |
349 | * |
350 | * For each type of VC capability, VC/VC9/MFVC, find the capability and |
351 | * save it to the pre-allocated save buffer. |
352 | */ |
353 | int pci_save_vc_state(struct pci_dev *dev) |
354 | { |
355 | int i; |
356 | |
357 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { |
358 | int pos, ret; |
359 | struct pci_cap_saved_state *save_state; |
360 | |
361 | pos = pci_find_ext_capability(dev, cap: vc_caps[i].id); |
362 | if (!pos) |
363 | continue; |
364 | |
365 | save_state = pci_find_saved_ext_cap(dev, cap: vc_caps[i].id); |
366 | if (!save_state) { |
367 | pci_err(dev, "%s buffer not found in %s\n" , |
368 | vc_caps[i].name, __func__); |
369 | return -ENOMEM; |
370 | } |
371 | |
372 | ret = pci_vc_do_save_buffer(dev, pos, save_state, save: true); |
373 | if (ret) { |
374 | pci_err(dev, "%s save unsuccessful %s\n" , |
375 | vc_caps[i].name, __func__); |
376 | return ret; |
377 | } |
378 | } |
379 | |
380 | return 0; |
381 | } |
382 | |
383 | /** |
384 | * pci_restore_vc_state - Restore VC state from save buffer |
385 | * @dev: device |
386 | * |
387 | * For each type of VC capability, VC/VC9/MFVC, find the capability and |
388 | * restore it from the previously saved buffer. |
389 | */ |
390 | void pci_restore_vc_state(struct pci_dev *dev) |
391 | { |
392 | int i; |
393 | |
394 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { |
395 | int pos; |
396 | struct pci_cap_saved_state *save_state; |
397 | |
398 | pos = pci_find_ext_capability(dev, cap: vc_caps[i].id); |
399 | save_state = pci_find_saved_ext_cap(dev, cap: vc_caps[i].id); |
400 | if (!save_state || !pos) |
401 | continue; |
402 | |
403 | pci_vc_do_save_buffer(dev, pos, save_state, save: false); |
404 | } |
405 | } |
406 | |
407 | /** |
408 | * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps |
409 | * @dev: device |
410 | * |
411 | * For each type of VC capability, VC/VC9/MFVC, find the capability, size |
412 | * it, and allocate a buffer for save/restore. |
413 | */ |
414 | void pci_allocate_vc_save_buffers(struct pci_dev *dev) |
415 | { |
416 | int i; |
417 | |
418 | for (i = 0; i < ARRAY_SIZE(vc_caps); i++) { |
419 | int len, pos = pci_find_ext_capability(dev, cap: vc_caps[i].id); |
420 | |
421 | if (!pos) |
422 | continue; |
423 | |
424 | len = pci_vc_do_save_buffer(dev, pos, NULL, save: false); |
425 | if (pci_add_ext_cap_save_buffer(dev, cap: vc_caps[i].id, size: len)) |
426 | pci_err(dev, "unable to preallocate %s save buffer\n" , |
427 | vc_caps[i].name); |
428 | } |
429 | } |
430 | |