1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Scalar AES core transform
4 *
5 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8#include <linux/linkage.h>
9#include <asm/assembler.h>
10#include <asm/cache.h>
11
12 .text
13
14 rk .req x0
15 out .req x1
16 in .req x2
17 rounds .req x3
18 tt .req x2
19
20 .macro __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
21 .ifc \op\shift, b0
22 ubfiz \reg0, \in0, #2, #8
23 ubfiz \reg1, \in1e, #2, #8
24 .else
25 ubfx \reg0, \in0, #\shift, #8
26 ubfx \reg1, \in1e, #\shift, #8
27 .endif
28
29 /*
30 * AArch64 cannot do byte size indexed loads from a table containing
31 * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
32 * valid instruction. So perform the shift explicitly first for the
33 * high bytes (the low byte is shifted implicitly by using ubfiz rather
34 * than ubfx above)
35 */
36 .ifnc \op, b
37 ldr \reg0, [tt, \reg0, uxtw #2]
38 ldr \reg1, [tt, \reg1, uxtw #2]
39 .else
40 .if \shift > 0
41 lsl \reg0, \reg0, #2
42 lsl \reg1, \reg1, #2
43 .endif
44 ldrb \reg0, [tt, \reg0, uxtw]
45 ldrb \reg1, [tt, \reg1, uxtw]
46 .endif
47 .endm
48
49 .macro __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
50 ubfx \reg0, \in0, #\shift, #8
51 ubfx \reg1, \in1d, #\shift, #8
52 ldr\op \reg0, [tt, \reg0, uxtw #\sz]
53 ldr\op \reg1, [tt, \reg1, uxtw #\sz]
54 .endm
55
56 .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
57 ldp \out0, \out1, [rk], #8
58
59 __pair\enc \sz, \op, w12, w13, \in0, \in1, \in3, 0
60 __pair\enc \sz, \op, w14, w15, \in1, \in2, \in0, 8
61 __pair\enc \sz, \op, w16, w17, \in2, \in3, \in1, 16
62 __pair\enc \sz, \op, \t0, \t1, \in3, \in0, \in2, 24
63
64 eor \out0, \out0, w12
65 eor \out1, \out1, w13
66 eor \out0, \out0, w14, ror #24
67 eor \out1, \out1, w15, ror #24
68 eor \out0, \out0, w16, ror #16
69 eor \out1, \out1, w17, ror #16
70 eor \out0, \out0, \t0, ror #8
71 eor \out1, \out1, \t1, ror #8
72 .endm
73
74 .macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
75 __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
76 __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
77 .endm
78
79 .macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
80 __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
81 __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
82 .endm
83
84 .macro do_crypt, round, ttab, ltab, bsz
85 ldp w4, w5, [in]
86 ldp w6, w7, [in, #8]
87 ldp w8, w9, [rk], #16
88 ldp w10, w11, [rk, #-8]
89
90CPU_BE( rev w4, w4 )
91CPU_BE( rev w5, w5 )
92CPU_BE( rev w6, w6 )
93CPU_BE( rev w7, w7 )
94
95 eor w4, w4, w8
96 eor w5, w5, w9
97 eor w6, w6, w10
98 eor w7, w7, w11
99
100 adr_l tt, \ttab
101
102 tbnz rounds, #1, 1f
103
1040: \round w8, w9, w10, w11, w4, w5, w6, w7
105 \round w4, w5, w6, w7, w8, w9, w10, w11
106
1071: subs rounds, rounds, #4
108 \round w8, w9, w10, w11, w4, w5, w6, w7
109 b.ls 3f
1102: \round w4, w5, w6, w7, w8, w9, w10, w11
111 b 0b
1123: adr_l tt, \ltab
113 \round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
114
115CPU_BE( rev w4, w4 )
116CPU_BE( rev w5, w5 )
117CPU_BE( rev w6, w6 )
118CPU_BE( rev w7, w7 )
119
120 stp w4, w5, [out]
121 stp w6, w7, [out, #8]
122 ret
123 .endm
124
125SYM_FUNC_START(__aes_arm64_encrypt)
126 do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2
127SYM_FUNC_END(__aes_arm64_encrypt)
128
129 .align 5
130SYM_FUNC_START(__aes_arm64_decrypt)
131 do_crypt iround, crypto_it_tab, crypto_aes_inv_sbox, 0
132SYM_FUNC_END(__aes_arm64_decrypt)
133

source code of linux/arch/arm64/crypto/aes-cipher-core.S