1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2020 ARM Ltd.
4 */
5#include <linux/linkage.h>
6
7#include <asm/asm-uaccess.h>
8#include <asm/assembler.h>
9#include <asm/mte.h>
10#include <asm/page.h>
11#include <asm/sysreg.h>
12
13 .arch armv8.5-a+memtag
14
15/*
16 * multitag_transfer_size - set \reg to the block size that is accessed by the
17 * LDGM/STGM instructions.
18 */
19 .macro multitag_transfer_size, reg, tmp
20 mrs_s \reg, SYS_GMID_EL1
21 ubfx \reg, \reg, #GMID_EL1_BS_SHIFT, #GMID_EL1_BS_WIDTH
22 mov \tmp, #4
23 lsl \reg, \tmp, \reg
24 .endm
25
26/*
27 * Clear the tags in a page
28 * x0 - address of the page to be cleared
29 */
30SYM_FUNC_START(mte_clear_page_tags)
31 multitag_transfer_size x1, x2
321: stgm xzr, [x0]
33 add x0, x0, x1
34 tst x0, #(PAGE_SIZE - 1)
35 b.ne 1b
36 ret
37SYM_FUNC_END(mte_clear_page_tags)
38
39/*
40 * Zero the page and tags at the same time
41 *
42 * Parameters:
43 * x0 - address to the beginning of the page
44 */
45SYM_FUNC_START(mte_zero_clear_page_tags)
46 and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
47 mrs x1, dczid_el0
48 tbnz x1, #4, 2f // Branch if DC GZVA is prohibited
49 and w1, w1, #0xf
50 mov x2, #4
51 lsl x1, x2, x1
52
531: dc gzva, x0
54 add x0, x0, x1
55 tst x0, #(PAGE_SIZE - 1)
56 b.ne 1b
57 ret
58
592: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2)
60 tst x0, #(PAGE_SIZE - 1)
61 b.ne 2b
62 ret
63SYM_FUNC_END(mte_zero_clear_page_tags)
64
65/*
66 * Copy the tags from the source page to the destination one
67 * x0 - address of the destination page
68 * x1 - address of the source page
69 */
70SYM_FUNC_START(mte_copy_page_tags)
71 mov x2, x0
72 mov x3, x1
73 multitag_transfer_size x5, x6
741: ldgm x4, [x3]
75 stgm x4, [x2]
76 add x2, x2, x5
77 add x3, x3, x5
78 tst x2, #(PAGE_SIZE - 1)
79 b.ne 1b
80 ret
81SYM_FUNC_END(mte_copy_page_tags)
82
83/*
84 * Read tags from a user buffer (one tag per byte) and set the corresponding
85 * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
86 * x0 - kernel address (to)
87 * x1 - user buffer (from)
88 * x2 - number of tags/bytes (n)
89 * Returns:
90 * x0 - number of tags read/set
91 */
92SYM_FUNC_START(mte_copy_tags_from_user)
93 mov x3, x1
94 cbz x2, 2f
951:
96USER(2f, ldtrb w4, [x1])
97 lsl x4, x4, #MTE_TAG_SHIFT
98 stg x4, [x0], #MTE_GRANULE_SIZE
99 add x1, x1, #1
100 subs x2, x2, #1
101 b.ne 1b
102
103 // exception handling and function return
1042: sub x0, x1, x3 // update the number of tags set
105 ret
106SYM_FUNC_END(mte_copy_tags_from_user)
107
108/*
109 * Get the tags from a kernel address range and write the tag values to the
110 * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
111 * x0 - user buffer (to)
112 * x1 - kernel address (from)
113 * x2 - number of tags/bytes (n)
114 * Returns:
115 * x0 - number of tags read/set
116 */
117SYM_FUNC_START(mte_copy_tags_to_user)
118 mov x3, x0
119 cbz x2, 2f
1201:
121 ldg x4, [x1]
122 ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
123USER(2f, sttrb w4, [x0])
124 add x0, x0, #1
125 add x1, x1, #MTE_GRANULE_SIZE
126 subs x2, x2, #1
127 b.ne 1b
128
129 // exception handling and function return
1302: sub x0, x0, x3 // update the number of tags copied
131 ret
132SYM_FUNC_END(mte_copy_tags_to_user)
133
134/*
135 * Save the tags in a page
136 * x0 - page address
137 * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
138 */
139SYM_FUNC_START(mte_save_page_tags)
140 multitag_transfer_size x7, x5
1411:
142 mov x2, #0
1432:
144 ldgm x5, [x0]
145 orr x2, x2, x5
146 add x0, x0, x7
147 tst x0, #0xFF // 16 tag values fit in a register,
148 b.ne 2b // which is 16*16=256 bytes
149
150 str x2, [x1], #8
151
152 tst x0, #(PAGE_SIZE - 1)
153 b.ne 1b
154
155 ret
156SYM_FUNC_END(mte_save_page_tags)
157
158/*
159 * Restore the tags in a page
160 * x0 - page address
161 * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes
162 */
163SYM_FUNC_START(mte_restore_page_tags)
164 multitag_transfer_size x7, x5
1651:
166 ldr x2, [x1], #8
1672:
168 stgm x2, [x0]
169 add x0, x0, x7
170 tst x0, #0xFF
171 b.ne 2b
172
173 tst x0, #(PAGE_SIZE - 1)
174 b.ne 1b
175
176 ret
177SYM_FUNC_END(mte_restore_page_tags)
178

source code of linux/arch/arm64/lib/mte.S