1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Access to PCI I/O memory from user space programs. |
4 | * |
5 | * Copyright IBM Corp. 2014 |
6 | * Author(s): Alexey Ishchuk <aishchuk@linux.vnet.ibm.com> |
7 | */ |
8 | #include <linux/kernel.h> |
9 | #include <linux/syscalls.h> |
10 | #include <linux/init.h> |
11 | #include <linux/mm.h> |
12 | #include <linux/errno.h> |
13 | #include <linux/pci.h> |
14 | #include <asm/asm-extable.h> |
15 | #include <asm/pci_io.h> |
16 | #include <asm/pci_debug.h> |
17 | |
18 | static inline void zpci_err_mmio(u8 cc, u8 status, u64 offset) |
19 | { |
20 | struct { |
21 | u64 offset; |
22 | u8 cc; |
23 | u8 status; |
24 | } data = {offset, cc, status}; |
25 | |
26 | zpci_err_hex(&data, sizeof(data)); |
27 | } |
28 | |
29 | static inline int __pcistb_mio_inuser( |
30 | void __iomem *ioaddr, const void __user *src, |
31 | u64 len, u8 *status) |
32 | { |
33 | int cc = -ENXIO; |
34 | |
35 | asm volatile ( |
36 | " sacf 256\n" |
37 | "0: .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[src]\n" |
38 | "1: ipm %[cc]\n" |
39 | " srl %[cc],28\n" |
40 | "2: sacf 768\n" |
41 | EX_TABLE(0b, 2b) EX_TABLE(1b, 2b) |
42 | : [cc] "+d" (cc), [len] "+d" (len) |
43 | : [ioaddr] "a" (ioaddr), [src] "Q" (*((u8 __force *)src)) |
44 | : "cc" , "memory" ); |
45 | *status = len >> 24 & 0xff; |
46 | return cc; |
47 | } |
48 | |
49 | static inline int __pcistg_mio_inuser( |
50 | void __iomem *ioaddr, const void __user *src, |
51 | u64 ulen, u8 *status) |
52 | { |
53 | union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; |
54 | int cc = -ENXIO; |
55 | u64 val = 0; |
56 | u64 cnt = ulen; |
57 | u8 tmp; |
58 | |
59 | /* |
60 | * copy 0 < @len <= 8 bytes from @src into the right most bytes of |
61 | * a register, then store it to PCI at @ioaddr while in secondary |
62 | * address space. pcistg then uses the user mappings. |
63 | */ |
64 | asm volatile ( |
65 | " sacf 256\n" |
66 | "0: llgc %[tmp],0(%[src])\n" |
67 | "4: sllg %[val],%[val],8\n" |
68 | " aghi %[src],1\n" |
69 | " ogr %[val],%[tmp]\n" |
70 | " brctg %[cnt],0b\n" |
71 | "1: .insn rre,0xb9d40000,%[val],%[ioaddr_len]\n" |
72 | "2: ipm %[cc]\n" |
73 | " srl %[cc],28\n" |
74 | "3: sacf 768\n" |
75 | EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) |
76 | : |
77 | [src] "+a" (src), [cnt] "+d" (cnt), |
78 | [val] "+d" (val), [tmp] "=d" (tmp), |
79 | [cc] "+d" (cc), [ioaddr_len] "+&d" (ioaddr_len.pair) |
80 | :: "cc" , "memory" ); |
81 | *status = ioaddr_len.odd >> 24 & 0xff; |
82 | |
83 | /* did we read everything from user memory? */ |
84 | if (!cc && cnt != 0) |
85 | cc = -EFAULT; |
86 | |
87 | return cc; |
88 | } |
89 | |
90 | static inline int __memcpy_toio_inuser(void __iomem *dst, |
91 | const void __user *src, size_t n) |
92 | { |
93 | int size, rc = 0; |
94 | u8 status = 0; |
95 | |
96 | if (!src) |
97 | return -EINVAL; |
98 | |
99 | while (n > 0) { |
100 | size = zpci_get_max_io_size((u64 __force) dst, |
101 | (u64 __force) src, n, |
102 | ZPCI_MAX_WRITE_SIZE); |
103 | if (size > 8) /* main path */ |
104 | rc = __pcistb_mio_inuser(ioaddr: dst, src, len: size, status: &status); |
105 | else |
106 | rc = __pcistg_mio_inuser(ioaddr: dst, src, ulen: size, status: &status); |
107 | if (rc) |
108 | break; |
109 | src += size; |
110 | dst += size; |
111 | n -= size; |
112 | } |
113 | if (rc) |
114 | zpci_err_mmio(cc: rc, status, offset: (__force u64) dst); |
115 | return rc; |
116 | } |
117 | |
118 | SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, |
119 | const void __user *, user_buffer, size_t, length) |
120 | { |
121 | u8 local_buf[64]; |
122 | void __iomem *io_addr; |
123 | void *buf; |
124 | struct vm_area_struct *vma; |
125 | pte_t *ptep; |
126 | spinlock_t *ptl; |
127 | long ret; |
128 | |
129 | if (!zpci_is_enabled()) |
130 | return -ENODEV; |
131 | |
132 | if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) |
133 | return -EINVAL; |
134 | |
135 | /* |
136 | * We only support write access to MIO capable devices if we are on |
137 | * a MIO enabled system. Otherwise we would have to check for every |
138 | * address if it is a special ZPCI_ADDR and would have to do |
139 | * a pfn lookup which we don't need for MIO capable devices. Currently |
140 | * ISM devices are the only devices without MIO support and there is no |
141 | * known need for accessing these from userspace. |
142 | */ |
143 | if (static_branch_likely(&have_mio)) { |
144 | ret = __memcpy_toio_inuser(dst: (void __iomem *) mmio_addr, |
145 | src: user_buffer, |
146 | n: length); |
147 | return ret; |
148 | } |
149 | |
150 | if (length > 64) { |
151 | buf = kmalloc(size: length, GFP_KERNEL); |
152 | if (!buf) |
153 | return -ENOMEM; |
154 | } else |
155 | buf = local_buf; |
156 | |
157 | ret = -EFAULT; |
158 | if (copy_from_user(to: buf, from: user_buffer, n: length)) |
159 | goto out_free; |
160 | |
161 | mmap_read_lock(current->mm); |
162 | ret = -EINVAL; |
163 | vma = vma_lookup(current->mm, addr: mmio_addr); |
164 | if (!vma) |
165 | goto out_unlock_mmap; |
166 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) |
167 | goto out_unlock_mmap; |
168 | ret = -EACCES; |
169 | if (!(vma->vm_flags & VM_WRITE)) |
170 | goto out_unlock_mmap; |
171 | |
172 | ret = follow_pte(mm: vma->vm_mm, address: mmio_addr, ptepp: &ptep, ptlp: &ptl); |
173 | if (ret) |
174 | goto out_unlock_mmap; |
175 | |
176 | io_addr = (void __iomem *)((pte_pfn(pte: *ptep) << PAGE_SHIFT) | |
177 | (mmio_addr & ~PAGE_MASK)); |
178 | |
179 | if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) |
180 | goto out_unlock_pt; |
181 | |
182 | ret = zpci_memcpy_toio(io_addr, buf, length); |
183 | out_unlock_pt: |
184 | pte_unmap_unlock(ptep, ptl); |
185 | out_unlock_mmap: |
186 | mmap_read_unlock(current->mm); |
187 | out_free: |
188 | if (buf != local_buf) |
189 | kfree(objp: buf); |
190 | return ret; |
191 | } |
192 | |
193 | static inline int __pcilg_mio_inuser( |
194 | void __user *dst, const void __iomem *ioaddr, |
195 | u64 ulen, u8 *status) |
196 | { |
197 | union register_pair ioaddr_len = {.even = (u64 __force)ioaddr, .odd = ulen}; |
198 | u64 cnt = ulen; |
199 | int shift = ulen * 8; |
200 | int cc = -ENXIO; |
201 | u64 val, tmp; |
202 | |
203 | /* |
204 | * read 0 < @len <= 8 bytes from the PCI memory mapped at @ioaddr (in |
205 | * user space) into a register using pcilg then store these bytes at |
206 | * user address @dst |
207 | */ |
208 | asm volatile ( |
209 | " sacf 256\n" |
210 | "0: .insn rre,0xb9d60000,%[val],%[ioaddr_len]\n" |
211 | "1: ipm %[cc]\n" |
212 | " srl %[cc],28\n" |
213 | " ltr %[cc],%[cc]\n" |
214 | " jne 4f\n" |
215 | "2: ahi %[shift],-8\n" |
216 | " srlg %[tmp],%[val],0(%[shift])\n" |
217 | "3: stc %[tmp],0(%[dst])\n" |
218 | "5: aghi %[dst],1\n" |
219 | " brctg %[cnt],2b\n" |
220 | "4: sacf 768\n" |
221 | EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b) |
222 | : |
223 | [ioaddr_len] "+&d" (ioaddr_len.pair), |
224 | [cc] "+d" (cc), [val] "=d" (val), |
225 | [dst] "+a" (dst), [cnt] "+d" (cnt), [tmp] "=d" (tmp), |
226 | [shift] "+d" (shift) |
227 | :: "cc" , "memory" ); |
228 | |
229 | /* did we write everything to the user space buffer? */ |
230 | if (!cc && cnt != 0) |
231 | cc = -EFAULT; |
232 | |
233 | *status = ioaddr_len.odd >> 24 & 0xff; |
234 | return cc; |
235 | } |
236 | |
237 | static inline int __memcpy_fromio_inuser(void __user *dst, |
238 | const void __iomem *src, |
239 | unsigned long n) |
240 | { |
241 | int size, rc = 0; |
242 | u8 status; |
243 | |
244 | while (n > 0) { |
245 | size = zpci_get_max_io_size((u64 __force) src, |
246 | (u64 __force) dst, n, |
247 | ZPCI_MAX_READ_SIZE); |
248 | rc = __pcilg_mio_inuser(dst, ioaddr: src, ulen: size, status: &status); |
249 | if (rc) |
250 | break; |
251 | src += size; |
252 | dst += size; |
253 | n -= size; |
254 | } |
255 | if (rc) |
256 | zpci_err_mmio(cc: rc, status, offset: (__force u64) dst); |
257 | return rc; |
258 | } |
259 | |
260 | SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, |
261 | void __user *, user_buffer, size_t, length) |
262 | { |
263 | u8 local_buf[64]; |
264 | void __iomem *io_addr; |
265 | void *buf; |
266 | struct vm_area_struct *vma; |
267 | pte_t *ptep; |
268 | spinlock_t *ptl; |
269 | long ret; |
270 | |
271 | if (!zpci_is_enabled()) |
272 | return -ENODEV; |
273 | |
274 | if (length <= 0 || PAGE_SIZE - (mmio_addr & ~PAGE_MASK) < length) |
275 | return -EINVAL; |
276 | |
277 | /* |
278 | * We only support read access to MIO capable devices if we are on |
279 | * a MIO enabled system. Otherwise we would have to check for every |
280 | * address if it is a special ZPCI_ADDR and would have to do |
281 | * a pfn lookup which we don't need for MIO capable devices. Currently |
282 | * ISM devices are the only devices without MIO support and there is no |
283 | * known need for accessing these from userspace. |
284 | */ |
285 | if (static_branch_likely(&have_mio)) { |
286 | ret = __memcpy_fromio_inuser( |
287 | dst: user_buffer, src: (const void __iomem *)mmio_addr, |
288 | n: length); |
289 | return ret; |
290 | } |
291 | |
292 | if (length > 64) { |
293 | buf = kmalloc(size: length, GFP_KERNEL); |
294 | if (!buf) |
295 | return -ENOMEM; |
296 | } else { |
297 | buf = local_buf; |
298 | } |
299 | |
300 | mmap_read_lock(current->mm); |
301 | ret = -EINVAL; |
302 | vma = vma_lookup(current->mm, addr: mmio_addr); |
303 | if (!vma) |
304 | goto out_unlock_mmap; |
305 | if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) |
306 | goto out_unlock_mmap; |
307 | ret = -EACCES; |
308 | if (!(vma->vm_flags & VM_WRITE)) |
309 | goto out_unlock_mmap; |
310 | |
311 | ret = follow_pte(mm: vma->vm_mm, address: mmio_addr, ptepp: &ptep, ptlp: &ptl); |
312 | if (ret) |
313 | goto out_unlock_mmap; |
314 | |
315 | io_addr = (void __iomem *)((pte_pfn(pte: *ptep) << PAGE_SHIFT) | |
316 | (mmio_addr & ~PAGE_MASK)); |
317 | |
318 | if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { |
319 | ret = -EFAULT; |
320 | goto out_unlock_pt; |
321 | } |
322 | ret = zpci_memcpy_fromio(buf, io_addr, length); |
323 | |
324 | out_unlock_pt: |
325 | pte_unmap_unlock(ptep, ptl); |
326 | out_unlock_mmap: |
327 | mmap_read_unlock(current->mm); |
328 | |
329 | if (!ret && copy_to_user(to: user_buffer, from: buf, n: length)) |
330 | ret = -EFAULT; |
331 | |
332 | if (buf != local_buf) |
333 | kfree(objp: buf); |
334 | return ret; |
335 | } |
336 | |