Warning: This file is not a C or C++ file. It does not have highlighting.
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
---|---|
2 | /* |
3 | * Based on arch/arm/include/asm/io.h |
4 | * |
5 | * Copyright (C) 1996-2000 Russell King |
6 | * Copyright (C) 2012 ARM Ltd. |
7 | */ |
8 | #ifndef __ASM_IO_H |
9 | #define __ASM_IO_H |
10 | |
11 | #include <linux/types.h> |
12 | #include <linux/pgtable.h> |
13 | |
14 | #include <asm/byteorder.h> |
15 | #include <asm/barrier.h> |
16 | #include <asm/memory.h> |
17 | #include <asm/early_ioremap.h> |
18 | #include <asm/alternative.h> |
19 | #include <asm/cpufeature.h> |
20 | #include <asm/rsi.h> |
21 | |
22 | /* |
23 | * Generic IO read/write. These perform native-endian accesses. |
24 | */ |
25 | #define __raw_writeb __raw_writeb |
26 | static __always_inline void __raw_writeb(u8 val, volatile void __iomem *addr) |
27 | { |
28 | volatile u8 __iomem *ptr = addr; |
29 | asm volatile("strb %w0, %1" : : "rZ" (val), "Qo" (*ptr)); |
30 | } |
31 | |
32 | #define __raw_writew __raw_writew |
33 | static __always_inline void __raw_writew(u16 val, volatile void __iomem *addr) |
34 | { |
35 | volatile u16 __iomem *ptr = addr; |
36 | asm volatile("strh %w0, %1" : : "rZ" (val), "Qo" (*ptr)); |
37 | } |
38 | |
39 | #define __raw_writel __raw_writel |
40 | static __always_inline void __raw_writel(u32 val, volatile void __iomem *addr) |
41 | { |
42 | volatile u32 __iomem *ptr = addr; |
43 | asm volatile("str %w0, %1" : : "rZ" (val), "Qo" (*ptr)); |
44 | } |
45 | |
46 | #define __raw_writeq __raw_writeq |
47 | static __always_inline void __raw_writeq(u64 val, volatile void __iomem *addr) |
48 | { |
49 | volatile u64 __iomem *ptr = addr; |
50 | asm volatile("str %x0, %1" : : "rZ" (val), "Qo" (*ptr)); |
51 | } |
52 | |
53 | #define __raw_readb __raw_readb |
54 | static __always_inline u8 __raw_readb(const volatile void __iomem *addr) |
55 | { |
56 | u8 val; |
57 | asm volatile(ALTERNATIVE("ldrb %w0, [%1]", |
58 | "ldarb %w0, [%1]", |
59 | ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) |
60 | : "=r" (val) : "r" (addr)); |
61 | return val; |
62 | } |
63 | |
64 | #define __raw_readw __raw_readw |
65 | static __always_inline u16 __raw_readw(const volatile void __iomem *addr) |
66 | { |
67 | u16 val; |
68 | |
69 | asm volatile(ALTERNATIVE("ldrh %w0, [%1]", |
70 | "ldarh %w0, [%1]", |
71 | ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) |
72 | : "=r" (val) : "r" (addr)); |
73 | return val; |
74 | } |
75 | |
76 | #define __raw_readl __raw_readl |
77 | static __always_inline u32 __raw_readl(const volatile void __iomem *addr) |
78 | { |
79 | u32 val; |
80 | asm volatile(ALTERNATIVE("ldr %w0, [%1]", |
81 | "ldar %w0, [%1]", |
82 | ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) |
83 | : "=r" (val) : "r" (addr)); |
84 | return val; |
85 | } |
86 | |
87 | #define __raw_readq __raw_readq |
88 | static __always_inline u64 __raw_readq(const volatile void __iomem *addr) |
89 | { |
90 | u64 val; |
91 | asm volatile(ALTERNATIVE("ldr %0, [%1]", |
92 | "ldar %0, [%1]", |
93 | ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) |
94 | : "=r" (val) : "r" (addr)); |
95 | return val; |
96 | } |
97 | |
98 | /* IO barriers */ |
99 | #define __io_ar(v) \ |
100 | ({ \ |
101 | unsigned long tmp; \ |
102 | \ |
103 | dma_rmb(); \ |
104 | \ |
105 | /* \ |
106 | * Create a dummy control dependency from the IO read to any \ |
107 | * later instructions. This ensures that a subsequent call to \ |
108 | * udelay() will be ordered due to the ISB in get_cycles(). \ |
109 | */ \ |
110 | asm volatile("eor %0, %1, %1\n" \ |
111 | "cbnz %0, ." \ |
112 | : "=r" (tmp) : "r" ((unsigned long)(v)) \ |
113 | : "memory"); \ |
114 | }) |
115 | |
116 | #define __io_bw() dma_wmb() |
117 | #define __io_br(v) |
118 | #define __io_aw(v) |
119 | |
120 | /* arm64-specific, don't use in portable drivers */ |
121 | #define __iormb(v) __io_ar(v) |
122 | #define __iowmb() __io_bw() |
123 | #define __iomb() dma_mb() |
124 | |
125 | /* |
126 | * I/O port access primitives. |
127 | */ |
128 | #define arch_has_dev_port() (1) |
129 | #define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) |
130 | #define PCI_IOBASE ((void __iomem *)PCI_IO_START) |
131 | |
132 | /* |
133 | * The ARM64 iowrite implementation is intended to support drivers that want to |
134 | * use write combining. For instance PCI drivers using write combining with a 64 |
135 | * byte __iowrite64_copy() expect to get a 64 byte MemWr TLP on the PCIe bus. |
136 | * |
137 | * Newer ARM core have sensitive write combining buffers, it is important that |
138 | * the stores be contiguous blocks of store instructions. Normal memcpy |
139 | * approaches have a very low chance to generate write combining. |
140 | * |
141 | * Since this is the only API on ARM64 that should be used with write combining |
142 | * it also integrates the DGH hint which is supposed to lower the latency to |
143 | * emit the large TLP from the CPU. |
144 | */ |
145 | |
146 | static __always_inline void |
147 | __const_memcpy_toio_aligned32(volatile u32 __iomem *to, const u32 *from, |
148 | size_t count) |
149 | { |
150 | switch (count) { |
151 | case 8: |
152 | asm volatile("str %w0, [%8, #4 * 0]\n" |
153 | "str %w1, [%8, #4 * 1]\n" |
154 | "str %w2, [%8, #4 * 2]\n" |
155 | "str %w3, [%8, #4 * 3]\n" |
156 | "str %w4, [%8, #4 * 4]\n" |
157 | "str %w5, [%8, #4 * 5]\n" |
158 | "str %w6, [%8, #4 * 6]\n" |
159 | "str %w7, [%8, #4 * 7]\n" |
160 | : |
161 | : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), |
162 | "rZ"(from[3]), "rZ"(from[4]), "rZ"(from[5]), |
163 | "rZ"(from[6]), "rZ"(from[7]), "r"(to)); |
164 | break; |
165 | case 4: |
166 | asm volatile("str %w0, [%4, #4 * 0]\n" |
167 | "str %w1, [%4, #4 * 1]\n" |
168 | "str %w2, [%4, #4 * 2]\n" |
169 | "str %w3, [%4, #4 * 3]\n" |
170 | : |
171 | : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), |
172 | "rZ"(from[3]), "r"(to)); |
173 | break; |
174 | case 2: |
175 | asm volatile("str %w0, [%2, #4 * 0]\n" |
176 | "str %w1, [%2, #4 * 1]\n" |
177 | : |
178 | : "rZ"(from[0]), "rZ"(from[1]), "r"(to)); |
179 | break; |
180 | case 1: |
181 | __raw_writel(*from, to); |
182 | break; |
183 | default: |
184 | BUILD_BUG(); |
185 | } |
186 | } |
187 | |
188 | void __iowrite32_copy_full(void __iomem *to, const void *from, size_t count); |
189 | |
190 | static __always_inline void |
191 | __iowrite32_copy(void __iomem *to, const void *from, size_t count) |
192 | { |
193 | if (__builtin_constant_p(count) && |
194 | (count == 8 || count == 4 || count == 2 || count == 1)) { |
195 | __const_memcpy_toio_aligned32(to, from, count); |
196 | dgh(); |
197 | } else { |
198 | __iowrite32_copy_full(to, from, count); |
199 | } |
200 | } |
201 | #define __iowrite32_copy __iowrite32_copy |
202 | |
203 | static __always_inline void |
204 | __const_memcpy_toio_aligned64(volatile u64 __iomem *to, const u64 *from, |
205 | size_t count) |
206 | { |
207 | switch (count) { |
208 | case 8: |
209 | asm volatile("str %x0, [%8, #8 * 0]\n" |
210 | "str %x1, [%8, #8 * 1]\n" |
211 | "str %x2, [%8, #8 * 2]\n" |
212 | "str %x3, [%8, #8 * 3]\n" |
213 | "str %x4, [%8, #8 * 4]\n" |
214 | "str %x5, [%8, #8 * 5]\n" |
215 | "str %x6, [%8, #8 * 6]\n" |
216 | "str %x7, [%8, #8 * 7]\n" |
217 | : |
218 | : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), |
219 | "rZ"(from[3]), "rZ"(from[4]), "rZ"(from[5]), |
220 | "rZ"(from[6]), "rZ"(from[7]), "r"(to)); |
221 | break; |
222 | case 4: |
223 | asm volatile("str %x0, [%4, #8 * 0]\n" |
224 | "str %x1, [%4, #8 * 1]\n" |
225 | "str %x2, [%4, #8 * 2]\n" |
226 | "str %x3, [%4, #8 * 3]\n" |
227 | : |
228 | : "rZ"(from[0]), "rZ"(from[1]), "rZ"(from[2]), |
229 | "rZ"(from[3]), "r"(to)); |
230 | break; |
231 | case 2: |
232 | asm volatile("str %x0, [%2, #8 * 0]\n" |
233 | "str %x1, [%2, #8 * 1]\n" |
234 | : |
235 | : "rZ"(from[0]), "rZ"(from[1]), "r"(to)); |
236 | break; |
237 | case 1: |
238 | __raw_writeq(*from, to); |
239 | break; |
240 | default: |
241 | BUILD_BUG(); |
242 | } |
243 | } |
244 | |
245 | void __iowrite64_copy_full(void __iomem *to, const void *from, size_t count); |
246 | |
247 | static __always_inline void |
248 | __iowrite64_copy(void __iomem *to, const void *from, size_t count) |
249 | { |
250 | if (__builtin_constant_p(count) && |
251 | (count == 8 || count == 4 || count == 2 || count == 1)) { |
252 | __const_memcpy_toio_aligned64(to, from, count); |
253 | dgh(); |
254 | } else { |
255 | __iowrite64_copy_full(to, from, count); |
256 | } |
257 | } |
258 | #define __iowrite64_copy __iowrite64_copy |
259 | |
260 | /* |
261 | * I/O memory mapping functions. |
262 | */ |
263 | |
264 | typedef int (*ioremap_prot_hook_t)(phys_addr_t phys_addr, size_t size, |
265 | pgprot_t *prot); |
266 | int arm64_ioremap_prot_hook_register(const ioremap_prot_hook_t hook); |
267 | |
268 | #define ioremap_prot ioremap_prot |
269 | |
270 | #define _PAGE_IOREMAP PROT_DEVICE_nGnRE |
271 | |
272 | #define ioremap_wc(addr, size) \ |
273 | ioremap_prot((addr), (size), __pgprot(PROT_NORMAL_NC)) |
274 | #define ioremap_np(addr, size) \ |
275 | ioremap_prot((addr), (size), __pgprot(PROT_DEVICE_nGnRnE)) |
276 | |
277 | /* |
278 | * io{read,write}{16,32,64}be() macros |
279 | */ |
280 | #define ioread16be(p) ({ __u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(__v); __v; }) |
281 | #define ioread32be(p) ({ __u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(__v); __v; }) |
282 | #define ioread64be(p) ({ __u64 __v = be64_to_cpu((__force __be64)__raw_readq(p)); __iormb(__v); __v; }) |
283 | |
284 | #define iowrite16be(v,p) ({ __iowmb(); __raw_writew((__force __u16)cpu_to_be16(v), p); }) |
285 | #define iowrite32be(v,p) ({ __iowmb(); __raw_writel((__force __u32)cpu_to_be32(v), p); }) |
286 | #define iowrite64be(v,p) ({ __iowmb(); __raw_writeq((__force __u64)cpu_to_be64(v), p); }) |
287 | |
288 | #include <asm-generic/io.h> |
289 | |
290 | #define ioremap_cache ioremap_cache |
291 | static inline void __iomem *ioremap_cache(phys_addr_t addr, size_t size) |
292 | { |
293 | if (pfn_is_map_memory(__phys_to_pfn(addr))) |
294 | return (void __iomem *)__phys_to_virt(addr); |
295 | |
296 | return ioremap_prot(addr, size, __pgprot(PROT_NORMAL)); |
297 | } |
298 | |
299 | /* |
300 | * More restrictive address range checking than the default implementation |
301 | * (PHYS_OFFSET and PHYS_MASK taken into account). |
302 | */ |
303 | #define ARCH_HAS_VALID_PHYS_ADDR_RANGE |
304 | extern int valid_phys_addr_range(phys_addr_t addr, size_t size); |
305 | extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size); |
306 | |
307 | extern bool arch_memremap_can_ram_remap(resource_size_t offset, size_t size, |
308 | unsigned long flags); |
309 | #define arch_memremap_can_ram_remap arch_memremap_can_ram_remap |
310 | |
311 | static inline bool arm64_is_protected_mmio(phys_addr_t phys_addr, size_t size) |
312 | { |
313 | if (unlikely(is_realm_world())) |
314 | return __arm64_is_protected_mmio(phys_addr, size); |
315 | return false; |
316 | } |
317 | |
318 | #endif /* __ASM_IO_H */ |
319 |
Warning: This file is not a C or C++ file. It does not have highlighting.