1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * s390 specific pci instructions |
4 | * |
5 | * Copyright IBM Corp. 2013 |
6 | */ |
7 | |
8 | #include <linux/export.h> |
9 | #include <linux/errno.h> |
10 | #include <linux/delay.h> |
11 | #include <linux/jump_label.h> |
12 | #include <asm/asm-extable.h> |
13 | #include <asm/facility.h> |
14 | #include <asm/pci_insn.h> |
15 | #include <asm/pci_debug.h> |
16 | #include <asm/pci_io.h> |
17 | #include <asm/processor.h> |
18 | |
19 | #define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */ |
20 | |
21 | struct zpci_err_insn_data { |
22 | u8 insn; |
23 | u8 cc; |
24 | u8 status; |
25 | union { |
26 | struct { |
27 | u64 req; |
28 | u64 offset; |
29 | }; |
30 | struct { |
31 | u64 addr; |
32 | u64 len; |
33 | }; |
34 | }; |
35 | } __packed; |
36 | |
37 | static inline void zpci_err_insn_req(int lvl, u8 insn, u8 cc, u8 status, |
38 | u64 req, u64 offset) |
39 | { |
40 | struct zpci_err_insn_data data = { |
41 | .insn = insn, .cc = cc, .status = status, |
42 | .req = req, .offset = offset}; |
43 | |
44 | zpci_err_hex_level(lvl, &data, sizeof(data)); |
45 | } |
46 | |
47 | static inline void zpci_err_insn_addr(int lvl, u8 insn, u8 cc, u8 status, |
48 | u64 addr, u64 len) |
49 | { |
50 | struct zpci_err_insn_data data = { |
51 | .insn = insn, .cc = cc, .status = status, |
52 | .addr = addr, .len = len}; |
53 | |
54 | zpci_err_hex_level(lvl, &data, sizeof(data)); |
55 | } |
56 | |
57 | /* Modify PCI Function Controls */ |
58 | static inline u8 __mpcifc(u64 req, struct zpci_fib *fib, u8 *status) |
59 | { |
60 | u8 cc; |
61 | |
62 | asm volatile ( |
63 | " .insn rxy,0xe300000000d0,%[req],%[fib]\n" |
64 | " ipm %[cc]\n" |
65 | " srl %[cc],28\n" |
66 | : [cc] "=d" (cc), [req] "+d" (req), [fib] "+Q" (*fib) |
67 | : : "cc" ); |
68 | *status = req >> 24 & 0xff; |
69 | return cc; |
70 | } |
71 | |
72 | u8 zpci_mod_fc(u64 req, struct zpci_fib *fib, u8 *status) |
73 | { |
74 | bool retried = false; |
75 | u8 cc; |
76 | |
77 | do { |
78 | cc = __mpcifc(req, fib, status); |
79 | if (cc == 2) { |
80 | msleep(ZPCI_INSN_BUSY_DELAY); |
81 | if (!retried) { |
82 | zpci_err_insn_req(lvl: 1, insn: 'M', cc, status: *status, req, offset: 0); |
83 | retried = true; |
84 | } |
85 | } |
86 | } while (cc == 2); |
87 | |
88 | if (cc) |
89 | zpci_err_insn_req(lvl: 0, insn: 'M', cc, status: *status, req, offset: 0); |
90 | else if (retried) |
91 | zpci_err_insn_req(lvl: 1, insn: 'M', cc, status: *status, req, offset: 0); |
92 | |
93 | return cc; |
94 | } |
95 | EXPORT_SYMBOL_GPL(zpci_mod_fc); |
96 | |
97 | /* Refresh PCI Translations */ |
98 | static inline u8 __rpcit(u64 fn, u64 addr, u64 range, u8 *status) |
99 | { |
100 | union register_pair addr_range = {.even = addr, .odd = range}; |
101 | u8 cc; |
102 | |
103 | asm volatile ( |
104 | " .insn rre,0xb9d30000,%[fn],%[addr_range]\n" |
105 | " ipm %[cc]\n" |
106 | " srl %[cc],28\n" |
107 | : [cc] "=d" (cc), [fn] "+d" (fn) |
108 | : [addr_range] "d" (addr_range.pair) |
109 | : "cc" ); |
110 | *status = fn >> 24 & 0xff; |
111 | return cc; |
112 | } |
113 | |
114 | int zpci_refresh_trans(u64 fn, u64 addr, u64 range) |
115 | { |
116 | bool retried = false; |
117 | u8 cc, status; |
118 | |
119 | do { |
120 | cc = __rpcit(fn, addr, range, status: &status); |
121 | if (cc == 2) { |
122 | udelay(ZPCI_INSN_BUSY_DELAY); |
123 | if (!retried) { |
124 | zpci_err_insn_addr(lvl: 1, insn: 'R', cc, status, addr, len: range); |
125 | retried = true; |
126 | } |
127 | } |
128 | } while (cc == 2); |
129 | |
130 | if (cc) |
131 | zpci_err_insn_addr(lvl: 0, insn: 'R', cc, status, addr, len: range); |
132 | else if (retried) |
133 | zpci_err_insn_addr(lvl: 1, insn: 'R', cc, status, addr, len: range); |
134 | |
135 | if (cc == 1 && (status == 4 || status == 16)) |
136 | return -ENOMEM; |
137 | |
138 | return (cc) ? -EIO : 0; |
139 | } |
140 | |
141 | /* Set Interruption Controls */ |
142 | int zpci_set_irq_ctrl(u16 ctl, u8 isc, union zpci_sic_iib *iib) |
143 | { |
144 | if (!test_facility(72)) |
145 | return -EIO; |
146 | |
147 | asm volatile( |
148 | ".insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib]\n" |
149 | : : [ctl] "d" (ctl), [isc] "d" (isc << 27), [iib] "Q" (*iib)); |
150 | |
151 | return 0; |
152 | } |
153 | EXPORT_SYMBOL_GPL(zpci_set_irq_ctrl); |
154 | |
155 | /* PCI Load */ |
156 | static inline int ____pcilg(u64 *data, u64 req, u64 offset, u8 *status) |
157 | { |
158 | union register_pair req_off = {.even = req, .odd = offset}; |
159 | int cc = -ENXIO; |
160 | u64 __data; |
161 | |
162 | asm volatile ( |
163 | " .insn rre,0xb9d20000,%[data],%[req_off]\n" |
164 | "0: ipm %[cc]\n" |
165 | " srl %[cc],28\n" |
166 | "1:\n" |
167 | EX_TABLE(0b, 1b) |
168 | : [cc] "+d" (cc), [data] "=d" (__data), |
169 | [req_off] "+&d" (req_off.pair) :: "cc" ); |
170 | *status = req_off.even >> 24 & 0xff; |
171 | *data = __data; |
172 | return cc; |
173 | } |
174 | |
175 | static inline int __pcilg(u64 *data, u64 req, u64 offset, u8 *status) |
176 | { |
177 | u64 __data; |
178 | int cc; |
179 | |
180 | cc = ____pcilg(data: &__data, req, offset, status); |
181 | if (!cc) |
182 | *data = __data; |
183 | |
184 | return cc; |
185 | } |
186 | |
187 | int __zpci_load(u64 *data, u64 req, u64 offset) |
188 | { |
189 | bool retried = false; |
190 | u8 status; |
191 | int cc; |
192 | |
193 | do { |
194 | cc = __pcilg(data, req, offset, status: &status); |
195 | if (cc == 2) { |
196 | udelay(ZPCI_INSN_BUSY_DELAY); |
197 | if (!retried) { |
198 | zpci_err_insn_req(lvl: 1, insn: 'l', cc, status, req, offset); |
199 | retried = true; |
200 | } |
201 | } |
202 | } while (cc == 2); |
203 | |
204 | if (cc) |
205 | zpci_err_insn_req(lvl: 0, insn: 'l', cc, status, req, offset); |
206 | else if (retried) |
207 | zpci_err_insn_req(lvl: 1, insn: 'l', cc, status, req, offset); |
208 | |
209 | return (cc > 0) ? -EIO : cc; |
210 | } |
211 | EXPORT_SYMBOL_GPL(__zpci_load); |
212 | |
213 | static inline int zpci_load_fh(u64 *data, const volatile void __iomem *addr, |
214 | unsigned long len) |
215 | { |
216 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; |
217 | u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len); |
218 | |
219 | return __zpci_load(data, req, ZPCI_OFFSET(addr)); |
220 | } |
221 | |
222 | static inline int __pcilg_mio(u64 *data, u64 ioaddr, u64 len, u8 *status) |
223 | { |
224 | union register_pair ioaddr_len = {.even = ioaddr, .odd = len}; |
225 | int cc = -ENXIO; |
226 | u64 __data; |
227 | |
228 | asm volatile ( |
229 | " .insn rre,0xb9d60000,%[data],%[ioaddr_len]\n" |
230 | "0: ipm %[cc]\n" |
231 | " srl %[cc],28\n" |
232 | "1:\n" |
233 | EX_TABLE(0b, 1b) |
234 | : [cc] "+d" (cc), [data] "=d" (__data), |
235 | [ioaddr_len] "+&d" (ioaddr_len.pair) :: "cc" ); |
236 | *status = ioaddr_len.odd >> 24 & 0xff; |
237 | *data = __data; |
238 | return cc; |
239 | } |
240 | |
241 | int zpci_load(u64 *data, const volatile void __iomem *addr, unsigned long len) |
242 | { |
243 | u8 status; |
244 | int cc; |
245 | |
246 | if (!static_branch_unlikely(&have_mio)) |
247 | return zpci_load_fh(data, addr, len); |
248 | |
249 | cc = __pcilg_mio(data, ioaddr: (__force u64) addr, len, status: &status); |
250 | if (cc) |
251 | zpci_err_insn_addr(lvl: 0, insn: 'L', cc, status, addr: (__force u64) addr, len); |
252 | |
253 | return (cc > 0) ? -EIO : cc; |
254 | } |
255 | EXPORT_SYMBOL_GPL(zpci_load); |
256 | |
257 | /* PCI Store */ |
258 | static inline int __pcistg(u64 data, u64 req, u64 offset, u8 *status) |
259 | { |
260 | union register_pair req_off = {.even = req, .odd = offset}; |
261 | int cc = -ENXIO; |
262 | |
263 | asm volatile ( |
264 | " .insn rre,0xb9d00000,%[data],%[req_off]\n" |
265 | "0: ipm %[cc]\n" |
266 | " srl %[cc],28\n" |
267 | "1:\n" |
268 | EX_TABLE(0b, 1b) |
269 | : [cc] "+d" (cc), [req_off] "+&d" (req_off.pair) |
270 | : [data] "d" (data) |
271 | : "cc" ); |
272 | *status = req_off.even >> 24 & 0xff; |
273 | return cc; |
274 | } |
275 | |
276 | int __zpci_store(u64 data, u64 req, u64 offset) |
277 | { |
278 | bool retried = false; |
279 | u8 status; |
280 | int cc; |
281 | |
282 | do { |
283 | cc = __pcistg(data, req, offset, status: &status); |
284 | if (cc == 2) { |
285 | udelay(ZPCI_INSN_BUSY_DELAY); |
286 | if (!retried) { |
287 | zpci_err_insn_req(lvl: 1, insn: 's', cc, status, req, offset); |
288 | retried = true; |
289 | } |
290 | } |
291 | } while (cc == 2); |
292 | |
293 | if (cc) |
294 | zpci_err_insn_req(lvl: 0, insn: 's', cc, status, req, offset); |
295 | else if (retried) |
296 | zpci_err_insn_req(lvl: 1, insn: 's', cc, status, req, offset); |
297 | |
298 | return (cc > 0) ? -EIO : cc; |
299 | } |
300 | EXPORT_SYMBOL_GPL(__zpci_store); |
301 | |
302 | static inline int zpci_store_fh(const volatile void __iomem *addr, u64 data, |
303 | unsigned long len) |
304 | { |
305 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(addr)]; |
306 | u64 req = ZPCI_CREATE_REQ(READ_ONCE(entry->fh), entry->bar, len); |
307 | |
308 | return __zpci_store(data, req, ZPCI_OFFSET(addr)); |
309 | } |
310 | |
311 | static inline int __pcistg_mio(u64 data, u64 ioaddr, u64 len, u8 *status) |
312 | { |
313 | union register_pair ioaddr_len = {.even = ioaddr, .odd = len}; |
314 | int cc = -ENXIO; |
315 | |
316 | asm volatile ( |
317 | " .insn rre,0xb9d40000,%[data],%[ioaddr_len]\n" |
318 | "0: ipm %[cc]\n" |
319 | " srl %[cc],28\n" |
320 | "1:\n" |
321 | EX_TABLE(0b, 1b) |
322 | : [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) |
323 | : [data] "d" (data) |
324 | : "cc" , "memory" ); |
325 | *status = ioaddr_len.odd >> 24 & 0xff; |
326 | return cc; |
327 | } |
328 | |
329 | int zpci_store(const volatile void __iomem *addr, u64 data, unsigned long len) |
330 | { |
331 | u8 status; |
332 | int cc; |
333 | |
334 | if (!static_branch_unlikely(&have_mio)) |
335 | return zpci_store_fh(addr, data, len); |
336 | |
337 | cc = __pcistg_mio(data, ioaddr: (__force u64) addr, len, status: &status); |
338 | if (cc) |
339 | zpci_err_insn_addr(lvl: 0, insn: 'S', cc, status, addr: (__force u64) addr, len); |
340 | |
341 | return (cc > 0) ? -EIO : cc; |
342 | } |
343 | EXPORT_SYMBOL_GPL(zpci_store); |
344 | |
345 | /* PCI Store Block */ |
346 | static inline int __pcistb(const u64 *data, u64 req, u64 offset, u8 *status) |
347 | { |
348 | int cc = -ENXIO; |
349 | |
350 | asm volatile ( |
351 | " .insn rsy,0xeb00000000d0,%[req],%[offset],%[data]\n" |
352 | "0: ipm %[cc]\n" |
353 | " srl %[cc],28\n" |
354 | "1:\n" |
355 | EX_TABLE(0b, 1b) |
356 | : [cc] "+d" (cc), [req] "+d" (req) |
357 | : [offset] "d" (offset), [data] "Q" (*data) |
358 | : "cc" ); |
359 | *status = req >> 24 & 0xff; |
360 | return cc; |
361 | } |
362 | |
363 | int __zpci_store_block(const u64 *data, u64 req, u64 offset) |
364 | { |
365 | bool retried = false; |
366 | u8 status; |
367 | int cc; |
368 | |
369 | do { |
370 | cc = __pcistb(data, req, offset, status: &status); |
371 | if (cc == 2) { |
372 | udelay(ZPCI_INSN_BUSY_DELAY); |
373 | if (!retried) { |
374 | zpci_err_insn_req(lvl: 0, insn: 'b', cc, status, req, offset); |
375 | retried = true; |
376 | } |
377 | } |
378 | } while (cc == 2); |
379 | |
380 | if (cc) |
381 | zpci_err_insn_req(lvl: 0, insn: 'b', cc, status, req, offset); |
382 | else if (retried) |
383 | zpci_err_insn_req(lvl: 1, insn: 'b', cc, status, req, offset); |
384 | |
385 | return (cc > 0) ? -EIO : cc; |
386 | } |
387 | EXPORT_SYMBOL_GPL(__zpci_store_block); |
388 | |
389 | static inline int zpci_write_block_fh(volatile void __iomem *dst, |
390 | const void *src, unsigned long len) |
391 | { |
392 | struct zpci_iomap_entry *entry = &zpci_iomap_start[ZPCI_IDX(dst)]; |
393 | u64 req = ZPCI_CREATE_REQ(entry->fh, entry->bar, len); |
394 | u64 offset = ZPCI_OFFSET(dst); |
395 | |
396 | return __zpci_store_block(src, req, offset); |
397 | } |
398 | |
399 | static inline int __pcistb_mio(const u64 *data, u64 ioaddr, u64 len, u8 *status) |
400 | { |
401 | int cc = -ENXIO; |
402 | |
403 | asm volatile ( |
404 | " .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data]\n" |
405 | "0: ipm %[cc]\n" |
406 | " srl %[cc],28\n" |
407 | "1:\n" |
408 | EX_TABLE(0b, 1b) |
409 | : [cc] "+d" (cc), [len] "+d" (len) |
410 | : [ioaddr] "d" (ioaddr), [data] "Q" (*data) |
411 | : "cc" ); |
412 | *status = len >> 24 & 0xff; |
413 | return cc; |
414 | } |
415 | |
416 | int zpci_write_block(volatile void __iomem *dst, |
417 | const void *src, unsigned long len) |
418 | { |
419 | u8 status; |
420 | int cc; |
421 | |
422 | if (!static_branch_unlikely(&have_mio)) |
423 | return zpci_write_block_fh(dst, src, len); |
424 | |
425 | cc = __pcistb_mio(data: src, ioaddr: (__force u64) dst, len, status: &status); |
426 | if (cc) |
427 | zpci_err_insn_addr(lvl: 0, insn: 'B', cc, status, addr: (__force u64) dst, len); |
428 | |
429 | return (cc > 0) ? -EIO : cc; |
430 | } |
431 | EXPORT_SYMBOL_GPL(zpci_write_block); |
432 | |
433 | static inline void __pciwb_mio(void) |
434 | { |
435 | asm volatile (".insn rre,0xb9d50000,0,0\n" ); |
436 | } |
437 | |
438 | void zpci_barrier(void) |
439 | { |
440 | if (static_branch_likely(&have_mio)) |
441 | __pciwb_mio(); |
442 | } |
443 | EXPORT_SYMBOL_GPL(zpci_barrier); |
444 | |