1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Access kernel memory without faulting -- s390 specific implementation. |
4 | * |
5 | * Copyright IBM Corp. 2009, 2015 |
6 | * |
7 | */ |
8 | |
9 | #include <linux/uaccess.h> |
10 | #include <linux/kernel.h> |
11 | #include <linux/types.h> |
12 | #include <linux/errno.h> |
13 | #include <linux/gfp.h> |
14 | #include <linux/cpu.h> |
15 | #include <linux/uio.h> |
16 | #include <linux/io.h> |
17 | #include <asm/asm-extable.h> |
18 | #include <asm/abs_lowcore.h> |
19 | #include <asm/stacktrace.h> |
20 | #include <asm/maccess.h> |
21 | #include <asm/ctlreg.h> |
22 | |
23 | unsigned long __bootdata_preserved(__memcpy_real_area); |
24 | pte_t *__bootdata_preserved(memcpy_real_ptep); |
25 | static DEFINE_MUTEX(memcpy_real_mutex); |
26 | |
27 | static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) |
28 | { |
29 | unsigned long aligned, offset, count; |
30 | char tmp[8]; |
31 | |
32 | aligned = (unsigned long) dst & ~7UL; |
33 | offset = (unsigned long) dst & 7UL; |
34 | size = min(8UL - offset, size); |
35 | count = size - 1; |
36 | asm volatile( |
37 | " bras 1,0f\n" |
38 | " mvc 0(1,%4),0(%5)\n" |
39 | "0: mvc 0(8,%3),0(%0)\n" |
40 | " ex %1,0(1)\n" |
41 | " lg %1,0(%3)\n" |
42 | " lra %0,0(%0)\n" |
43 | " sturg %1,%0\n" |
44 | : "+&a" (aligned), "+&a" (count), "=m" (tmp) |
45 | : "a" (&tmp), "a" (&tmp[offset]), "a" (src) |
46 | : "cc" , "memory" , "1" ); |
47 | return size; |
48 | } |
49 | |
50 | /* |
51 | * s390_kernel_write - write to kernel memory bypassing DAT |
52 | * @dst: destination address |
53 | * @src: source address |
54 | * @size: number of bytes to copy |
55 | * |
56 | * This function writes to kernel memory bypassing DAT and possible page table |
57 | * write protection. It writes to the destination using the sturg instruction. |
58 | * Therefore we have a read-modify-write sequence: the function reads eight |
59 | * bytes from destination at an eight byte boundary, modifies the bytes |
60 | * requested and writes the result back in a loop. |
61 | */ |
62 | static DEFINE_SPINLOCK(s390_kernel_write_lock); |
63 | |
64 | notrace void *s390_kernel_write(void *dst, const void *src, size_t size) |
65 | { |
66 | void *tmp = dst; |
67 | unsigned long flags; |
68 | long copied; |
69 | |
70 | spin_lock_irqsave(&s390_kernel_write_lock, flags); |
71 | while (size) { |
72 | copied = s390_kernel_write_odd(dst: tmp, src, size); |
73 | tmp += copied; |
74 | src += copied; |
75 | size -= copied; |
76 | } |
77 | spin_unlock_irqrestore(lock: &s390_kernel_write_lock, flags); |
78 | |
79 | return dst; |
80 | } |
81 | |
82 | size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) |
83 | { |
84 | size_t len, copied, res = 0; |
85 | unsigned long phys, offset; |
86 | void *chunk; |
87 | pte_t pte; |
88 | |
89 | BUILD_BUG_ON(MEMCPY_REAL_SIZE != PAGE_SIZE); |
90 | while (count) { |
91 | phys = src & MEMCPY_REAL_MASK; |
92 | offset = src & ~MEMCPY_REAL_MASK; |
93 | chunk = (void *)(__memcpy_real_area + offset); |
94 | len = min(count, MEMCPY_REAL_SIZE - offset); |
95 | pte = mk_pte_phys(phys, PAGE_KERNEL_RO); |
96 | |
97 | mutex_lock(&memcpy_real_mutex); |
98 | if (pte_val(pte) != pte_val(pte: *memcpy_real_ptep)) { |
99 | __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); |
100 | set_pte(ptep: memcpy_real_ptep, pte); |
101 | } |
102 | copied = copy_to_iter(addr: chunk, bytes: len, i: iter); |
103 | mutex_unlock(lock: &memcpy_real_mutex); |
104 | |
105 | count -= copied; |
106 | src += copied; |
107 | res += copied; |
108 | if (copied < len) |
109 | break; |
110 | } |
111 | return res; |
112 | } |
113 | |
114 | int memcpy_real(void *dest, unsigned long src, size_t count) |
115 | { |
116 | struct iov_iter iter; |
117 | struct kvec kvec; |
118 | |
119 | kvec.iov_base = dest; |
120 | kvec.iov_len = count; |
121 | iov_iter_kvec(i: &iter, ITER_DEST, kvec: &kvec, nr_segs: 1, count); |
122 | if (memcpy_real_iter(iter: &iter, src, count) < count) |
123 | return -EFAULT; |
124 | return 0; |
125 | } |
126 | |
127 | /* |
128 | * Find CPU that owns swapped prefix page |
129 | */ |
130 | static int get_swapped_owner(phys_addr_t addr) |
131 | { |
132 | phys_addr_t lc; |
133 | int cpu; |
134 | |
135 | for_each_online_cpu(cpu) { |
136 | lc = virt_to_phys(lowcore_ptr[cpu]); |
137 | if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) |
138 | continue; |
139 | return cpu; |
140 | } |
141 | return -1; |
142 | } |
143 | |
144 | /* |
145 | * Convert a physical pointer for /dev/mem access |
146 | * |
147 | * For swapped prefix pages a new buffer is returned that contains a copy of |
148 | * the absolute memory. The buffer size is maximum one page large. |
149 | */ |
150 | void *xlate_dev_mem_ptr(phys_addr_t addr) |
151 | { |
152 | void *ptr = phys_to_virt(address: addr); |
153 | void *bounce = ptr; |
154 | struct lowcore *abs_lc; |
155 | unsigned long size; |
156 | int this_cpu, cpu; |
157 | |
158 | cpus_read_lock(); |
159 | this_cpu = get_cpu(); |
160 | if (addr >= sizeof(struct lowcore)) { |
161 | cpu = get_swapped_owner(addr); |
162 | if (cpu < 0) |
163 | goto out; |
164 | } |
165 | bounce = (void *)__get_free_page(GFP_ATOMIC); |
166 | if (!bounce) |
167 | goto out; |
168 | size = PAGE_SIZE - (addr & ~PAGE_MASK); |
169 | if (addr < sizeof(struct lowcore)) { |
170 | abs_lc = get_abs_lowcore(); |
171 | ptr = (void *)abs_lc + addr; |
172 | memcpy(bounce, ptr, size); |
173 | put_abs_lowcore(abs_lc); |
174 | } else if (cpu == this_cpu) { |
175 | ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); |
176 | memcpy(bounce, ptr, size); |
177 | } else { |
178 | memcpy(bounce, ptr, size); |
179 | } |
180 | out: |
181 | put_cpu(); |
182 | cpus_read_unlock(); |
183 | return bounce; |
184 | } |
185 | |
186 | /* |
187 | * Free converted buffer for /dev/mem access (if necessary) |
188 | */ |
189 | void unxlate_dev_mem_ptr(phys_addr_t addr, void *ptr) |
190 | { |
191 | if (addr != virt_to_phys(address: ptr)) |
192 | free_page((unsigned long)ptr); |
193 | } |
194 | |