1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * linux/arch/arm/mm/copypage-feroceon.S |
4 | * |
5 | * Copyright (C) 2008 Marvell Semiconductors |
6 | * |
7 | * This handles copy_user_highpage and clear_user_page on Feroceon |
8 | * more optimally than the generic implementations. |
9 | */ |
10 | #include <linux/init.h> |
11 | #include <linux/highmem.h> |
12 | |
13 | static void feroceon_copy_user_page(void *kto, const void *kfrom) |
14 | { |
15 | int tmp; |
16 | |
17 | asm volatile ("\ |
18 | .arch armv5te \n\ |
19 | 1: ldmia %1!, {r2 - r7, ip, lr} \n\ |
20 | pld [%1, #0] \n\ |
21 | pld [%1, #32] \n\ |
22 | pld [%1, #64] \n\ |
23 | pld [%1, #96] \n\ |
24 | pld [%1, #128] \n\ |
25 | pld [%1, #160] \n\ |
26 | pld [%1, #192] \n\ |
27 | stmia %0, {r2 - r7, ip, lr} \n\ |
28 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
29 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
30 | add %0, %0, #32 \n\ |
31 | stmia %0, {r2 - r7, ip, lr} \n\ |
32 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
33 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
34 | add %0, %0, #32 \n\ |
35 | stmia %0, {r2 - r7, ip, lr} \n\ |
36 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
37 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
38 | add %0, %0, #32 \n\ |
39 | stmia %0, {r2 - r7, ip, lr} \n\ |
40 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
41 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
42 | add %0, %0, #32 \n\ |
43 | stmia %0, {r2 - r7, ip, lr} \n\ |
44 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
45 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
46 | add %0, %0, #32 \n\ |
47 | stmia %0, {r2 - r7, ip, lr} \n\ |
48 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
49 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
50 | add %0, %0, #32 \n\ |
51 | stmia %0, {r2 - r7, ip, lr} \n\ |
52 | ldmia %1!, {r2 - r7, ip, lr} \n\ |
53 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
54 | add %0, %0, #32 \n\ |
55 | stmia %0, {r2 - r7, ip, lr} \n\ |
56 | subs %2, %2, #(32 * 8) \n\ |
57 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
58 | add %0, %0, #32 \n\ |
59 | bne 1b \n\ |
60 | mcr p15, 0, %2, c7, c10, 4 @ drain WB" |
61 | : "+&r" (kto), "+&r" (kfrom), "=&r" (tmp) |
62 | : "2" (PAGE_SIZE) |
63 | : "r2" , "r3" , "r4" , "r5" , "r6" , "r7" , "ip" , "lr" ); |
64 | } |
65 | |
66 | void feroceon_copy_user_highpage(struct page *to, struct page *from, |
67 | unsigned long vaddr, struct vm_area_struct *vma) |
68 | { |
69 | void *kto, *kfrom; |
70 | |
71 | kto = kmap_atomic(page: to); |
72 | kfrom = kmap_atomic(page: from); |
73 | flush_cache_page(vma, vmaddr: vaddr, page_to_pfn(from)); |
74 | feroceon_copy_user_page(kto, kfrom); |
75 | kunmap_atomic(kfrom); |
76 | kunmap_atomic(kto); |
77 | } |
78 | |
79 | void feroceon_clear_user_highpage(struct page *page, unsigned long vaddr) |
80 | { |
81 | void *ptr, *kaddr = kmap_atomic(page); |
82 | asm volatile ("\ |
83 | mov r1, %2 \n\ |
84 | mov r2, #0 \n\ |
85 | mov r3, #0 \n\ |
86 | mov r4, #0 \n\ |
87 | mov r5, #0 \n\ |
88 | mov r6, #0 \n\ |
89 | mov r7, #0 \n\ |
90 | mov ip, #0 \n\ |
91 | mov lr, #0 \n\ |
92 | 1: stmia %0, {r2-r7, ip, lr} \n\ |
93 | subs r1, r1, #1 \n\ |
94 | mcr p15, 0, %0, c7, c14, 1 @ clean and invalidate D line\n\ |
95 | add %0, %0, #32 \n\ |
96 | bne 1b \n\ |
97 | mcr p15, 0, r1, c7, c10, 4 @ drain WB" |
98 | : "=r" (ptr) |
99 | : "0" (kaddr), "I" (PAGE_SIZE / 32) |
100 | : "r1" , "r2" , "r3" , "r4" , "r5" , "r6" , "r7" , "ip" , "lr" ); |
101 | kunmap_atomic(kaddr); |
102 | } |
103 | |
104 | struct cpu_user_fns feroceon_user_fns __initdata = { |
105 | .cpu_clear_user_highpage = feroceon_clear_user_highpage, |
106 | .cpu_copy_user_highpage = feroceon_copy_user_highpage, |
107 | }; |
108 | |
109 | |