| 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
|---|
| 2 | /* |
|---|
| 3 | * aes-ce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions |
|---|
| 4 | * |
|---|
| 5 | * Copyright (C) 2013 - 2017 Linaro Ltd. |
|---|
| 6 | * Copyright (C) 2024 Google LLC |
|---|
| 7 | * |
|---|
| 8 | * Author: Ard Biesheuvel <ardb@kernel.org> |
|---|
| 9 | */ |
|---|
| 10 | |
|---|
| 11 | #include <linux/linkage.h> |
|---|
| 12 | #include <asm/assembler.h> |
|---|
| 13 | |
|---|
| 14 | .text |
|---|
| 15 | .arch armv8-a+crypto |
|---|
| 16 | |
|---|
| 17 | .macro load_round_keys, rk, nr, tmp |
|---|
| 18 | sub w\tmp, \nr, #10 |
|---|
| 19 | add \tmp, \rk, w\tmp, sxtw #4 |
|---|
| 20 | ld1 {v10.4s-v13.4s}, [\rk] |
|---|
| 21 | ld1 {v14.4s-v17.4s}, [\tmp], #64 |
|---|
| 22 | ld1 {v18.4s-v21.4s}, [\tmp], #64 |
|---|
| 23 | ld1 {v3.4s-v5.4s}, [\tmp] |
|---|
| 24 | .endm |
|---|
| 25 | |
|---|
| 26 | .macro dround, va, vb, vk |
|---|
| 27 | aese \va\().16b, \vk\().16b |
|---|
| 28 | aesmc \va\().16b, \va\().16b |
|---|
| 29 | aese \vb\().16b, \vk\().16b |
|---|
| 30 | aesmc \vb\().16b, \vb\().16b |
|---|
| 31 | .endm |
|---|
| 32 | |
|---|
| 33 | .macro aes_encrypt, va, vb, nr |
|---|
| 34 | tbz \nr, #2, .L\@ |
|---|
| 35 | dround \va, \vb, v10 |
|---|
| 36 | dround \va, \vb, v11 |
|---|
| 37 | tbz \nr, #1, .L\@ |
|---|
| 38 | dround \va, \vb, v12 |
|---|
| 39 | dround \va, \vb, v13 |
|---|
| 40 | .L\@: .irp v, v14, v15, v16, v17, v18, v19, v20, v21, v3 |
|---|
| 41 | dround \va, \vb, \v |
|---|
| 42 | .endr |
|---|
| 43 | aese \va\().16b, v4.16b |
|---|
| 44 | aese \vb\().16b, v4.16b |
|---|
| 45 | .endm |
|---|
| 46 | |
|---|
| 47 | .macro aes_ccm_do_crypt,enc |
|---|
| 48 | load_round_keys x3, w4, x10 |
|---|
| 49 | |
|---|
| 50 | ld1 {v0.16b}, [x5] /* load mac */ |
|---|
| 51 | cbz x2, ce_aes_ccm_final |
|---|
| 52 | ldr x8, [x6, #8] /* load lower ctr */ |
|---|
| 53 | CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ |
|---|
| 54 | 0: /* outer loop */ |
|---|
| 55 | ld1 {v1.8b}, [x6] /* load upper ctr */ |
|---|
| 56 | prfm pldl1strm, [x1] |
|---|
| 57 | add x8, x8, #1 |
|---|
| 58 | rev x9, x8 |
|---|
| 59 | ins v1.d[1], x9 /* no carry in lower ctr */ |
|---|
| 60 | |
|---|
| 61 | aes_encrypt v0, v1, w4 |
|---|
| 62 | |
|---|
| 63 | subs w2, w2, #16 |
|---|
| 64 | bmi ce_aes_ccm_crypt_tail |
|---|
| 65 | ld1 {v2.16b}, [x1], #16 /* load next input block */ |
|---|
| 66 | .if \enc == 1 |
|---|
| 67 | eor v2.16b, v2.16b, v5.16b /* final round enc+mac */ |
|---|
| 68 | eor v6.16b, v1.16b, v2.16b /* xor with crypted ctr */ |
|---|
| 69 | .else |
|---|
| 70 | eor v2.16b, v2.16b, v1.16b /* xor with crypted ctr */ |
|---|
| 71 | eor v6.16b, v2.16b, v5.16b /* final round enc */ |
|---|
| 72 | .endif |
|---|
| 73 | eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */ |
|---|
| 74 | st1 {v6.16b}, [x0], #16 /* write output block */ |
|---|
| 75 | bne 0b |
|---|
| 76 | CPU_LE( rev x8, x8 ) |
|---|
| 77 | str x8, [x6, #8] /* store lsb end of ctr (BE) */ |
|---|
| 78 | cbnz x7, ce_aes_ccm_final |
|---|
| 79 | st1 {v0.16b}, [x5] /* store mac */ |
|---|
| 80 | ret |
|---|
| 81 | .endm |
|---|
| 82 | |
|---|
| 83 | SYM_FUNC_START_LOCAL(ce_aes_ccm_crypt_tail) |
|---|
| 84 | eor v0.16b, v0.16b, v5.16b /* final round mac */ |
|---|
| 85 | eor v1.16b, v1.16b, v5.16b /* final round enc */ |
|---|
| 86 | |
|---|
| 87 | add x1, x1, w2, sxtw /* rewind the input pointer (w2 < 0) */ |
|---|
| 88 | add x0, x0, w2, sxtw /* rewind the output pointer */ |
|---|
| 89 | |
|---|
| 90 | adr_l x8, .Lpermute /* load permute vectors */ |
|---|
| 91 | add x9, x8, w2, sxtw |
|---|
| 92 | sub x8, x8, w2, sxtw |
|---|
| 93 | ld1 {v7.16b-v8.16b}, [x9] |
|---|
| 94 | ld1 {v9.16b}, [x8] |
|---|
| 95 | |
|---|
| 96 | ld1 {v2.16b}, [x1] /* load a full block of input */ |
|---|
| 97 | tbl v1.16b, {v1.16b}, v7.16b /* move keystream to end of register */ |
|---|
| 98 | eor v7.16b, v2.16b, v1.16b /* encrypt partial input block */ |
|---|
| 99 | bif v2.16b, v7.16b, v22.16b /* select plaintext */ |
|---|
| 100 | tbx v7.16b, {v6.16b}, v8.16b /* insert output from previous iteration */ |
|---|
| 101 | tbl v2.16b, {v2.16b}, v9.16b /* copy plaintext to start of v2 */ |
|---|
| 102 | eor v0.16b, v0.16b, v2.16b /* fold plaintext into mac */ |
|---|
| 103 | |
|---|
| 104 | st1 {v7.16b}, [x0] /* store output block */ |
|---|
| 105 | cbz x7, 0f |
|---|
| 106 | |
|---|
| 107 | SYM_INNER_LABEL(ce_aes_ccm_final, SYM_L_LOCAL) |
|---|
| 108 | ld1 {v1.16b}, [x7] /* load 1st ctriv */ |
|---|
| 109 | |
|---|
| 110 | aes_encrypt v0, v1, w4 |
|---|
| 111 | |
|---|
| 112 | /* final round key cancels out */ |
|---|
| 113 | eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */ |
|---|
| 114 | 0: st1 {v0.16b}, [x5] /* store result */ |
|---|
| 115 | ret |
|---|
| 116 | SYM_FUNC_END(ce_aes_ccm_crypt_tail) |
|---|
| 117 | |
|---|
| 118 | /* |
|---|
| 119 | * void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, |
|---|
| 120 | * u8 const rk[], u32 rounds, u8 mac[], |
|---|
| 121 | * u8 ctr[], u8 const final_iv[]); |
|---|
| 122 | * void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes, |
|---|
| 123 | * u8 const rk[], u32 rounds, u8 mac[], |
|---|
| 124 | * u8 ctr[], u8 const final_iv[]); |
|---|
| 125 | */ |
|---|
| 126 | SYM_FUNC_START(ce_aes_ccm_encrypt) |
|---|
| 127 | movi v22.16b, #255 |
|---|
| 128 | aes_ccm_do_crypt 1 |
|---|
| 129 | SYM_FUNC_END(ce_aes_ccm_encrypt) |
|---|
| 130 | |
|---|
| 131 | SYM_FUNC_START(ce_aes_ccm_decrypt) |
|---|
| 132 | movi v22.16b, #0 |
|---|
| 133 | aes_ccm_do_crypt 0 |
|---|
| 134 | SYM_FUNC_END(ce_aes_ccm_decrypt) |
|---|
| 135 | |
|---|
| 136 | .section ".rodata" , "a" |
|---|
| 137 | .align 6 |
|---|
| 138 | .fill 15, 1, 0xff |
|---|
| 139 | .Lpermute: |
|---|
| 140 | .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 |
|---|
| 141 | .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf |
|---|
| 142 | .fill 15, 1, 0xff |
|---|
| 143 | |
|---|
