1use super::{arch::*, utils::*};
2use crate::{Block, Block8};
3use cipher::inout::InOut;
4use core::mem;
5
6/// AES-128 round keys
7pub(super) type RoundKeys = [__m128i; 11];
8
9#[inline]
10#[target_feature(enable = "aes")]
11pub(super) unsafe fn encrypt1(keys: &RoundKeys, block: InOut<'_, '_, Block>) {
12 let (in_ptr: *const GenericArray>, out_ptr: *mut GenericArray>) = block.into_raw();
13 let mut b: __m128i = _mm_loadu_si128(mem_addr:in_ptr as *const __m128i);
14 b = _mm_xor_si128(a:b, b:keys[0]);
15 b = _mm_aesenc_si128(a:b, round_key:keys[1]);
16 b = _mm_aesenc_si128(a:b, round_key:keys[2]);
17 b = _mm_aesenc_si128(a:b, round_key:keys[3]);
18 b = _mm_aesenc_si128(a:b, round_key:keys[4]);
19 b = _mm_aesenc_si128(a:b, round_key:keys[5]);
20 b = _mm_aesenc_si128(a:b, round_key:keys[6]);
21 b = _mm_aesenc_si128(a:b, round_key:keys[7]);
22 b = _mm_aesenc_si128(a:b, round_key:keys[8]);
23 b = _mm_aesenc_si128(a:b, round_key:keys[9]);
24 b = _mm_aesenclast_si128(a:b, round_key:keys[10]);
25 _mm_storeu_si128(mem_addr:out_ptr as *mut __m128i, a:b);
26}
27
28#[inline]
29#[target_feature(enable = "aes")]
30pub(super) unsafe fn encrypt8(keys: &RoundKeys, blocks: InOut<'_, '_, Block8>) {
31 let (in_ptr: *const GenericArray, …>, out_ptr: *mut GenericArray, …>) = blocks.into_raw();
32 let mut b: [__m128i; 8] = load8(blocks:in_ptr);
33 xor8(&mut b, key:keys[0]);
34 aesenc8(&mut b, key:keys[1]);
35 aesenc8(&mut b, key:keys[2]);
36 aesenc8(&mut b, key:keys[3]);
37 aesenc8(&mut b, key:keys[4]);
38 aesenc8(&mut b, key:keys[5]);
39 aesenc8(&mut b, key:keys[6]);
40 aesenc8(&mut b, key:keys[7]);
41 aesenc8(&mut b, key:keys[8]);
42 aesenc8(&mut b, key:keys[9]);
43 aesenclast8(&mut b, key:keys[10]);
44 store8(blocks:out_ptr, b);
45}
46
47#[inline]
48#[target_feature(enable = "aes")]
49pub(super) unsafe fn decrypt1(keys: &RoundKeys, block: InOut<'_, '_, Block>) {
50 let (in_ptr: *const GenericArray>, out_ptr: *mut GenericArray>) = block.into_raw();
51 let mut b: __m128i = _mm_loadu_si128(mem_addr:in_ptr as *const __m128i);
52 b = _mm_xor_si128(a:b, b:keys[10]);
53 b = _mm_aesdec_si128(a:b, round_key:keys[9]);
54 b = _mm_aesdec_si128(a:b, round_key:keys[8]);
55 b = _mm_aesdec_si128(a:b, round_key:keys[7]);
56 b = _mm_aesdec_si128(a:b, round_key:keys[6]);
57 b = _mm_aesdec_si128(a:b, round_key:keys[5]);
58 b = _mm_aesdec_si128(a:b, round_key:keys[4]);
59 b = _mm_aesdec_si128(a:b, round_key:keys[3]);
60 b = _mm_aesdec_si128(a:b, round_key:keys[2]);
61 b = _mm_aesdec_si128(a:b, round_key:keys[1]);
62 b = _mm_aesdeclast_si128(a:b, round_key:keys[0]);
63 _mm_storeu_si128(mem_addr:out_ptr as *mut __m128i, a:b);
64}
65
66#[inline]
67#[target_feature(enable = "aes")]
68pub(super) unsafe fn decrypt8(keys: &RoundKeys, blocks: InOut<'_, '_, Block8>) {
69 let (in_ptr: *const GenericArray, …>, out_ptr: *mut GenericArray, …>) = blocks.into_raw();
70 let mut b: [__m128i; 8] = load8(blocks:in_ptr);
71 xor8(&mut b, key:keys[10]);
72 aesdec8(&mut b, key:keys[9]);
73 aesdec8(&mut b, key:keys[8]);
74 aesdec8(&mut b, key:keys[7]);
75 aesdec8(&mut b, key:keys[6]);
76 aesdec8(&mut b, key:keys[5]);
77 aesdec8(&mut b, key:keys[4]);
78 aesdec8(&mut b, key:keys[3]);
79 aesdec8(&mut b, key:keys[2]);
80 aesdec8(&mut b, key:keys[1]);
81 aesdeclast8(&mut b, key:keys[0]);
82 store8(blocks:out_ptr, b);
83}
84
85macro_rules! expand_round {
86 ($keys:expr, $pos:expr, $round:expr) => {
87 let mut t1 = $keys[$pos - 1];
88 let mut t2;
89 let mut t3;
90
91 t2 = _mm_aeskeygenassist_si128(t1, $round);
92 t2 = _mm_shuffle_epi32(t2, 0xff);
93 t3 = _mm_slli_si128(t1, 0x4);
94 t1 = _mm_xor_si128(t1, t3);
95 t3 = _mm_slli_si128(t3, 0x4);
96 t1 = _mm_xor_si128(t1, t3);
97 t3 = _mm_slli_si128(t3, 0x4);
98 t1 = _mm_xor_si128(t1, t3);
99 t1 = _mm_xor_si128(t1, t2);
100
101 $keys[$pos] = t1;
102 };
103}
104
105#[inline]
106#[target_feature(enable = "aes")]
107pub(super) unsafe fn expand_key(key: &[u8; 16]) -> RoundKeys {
108 // SAFETY: `RoundKeys` is a `[__m128i; 11]` which can be initialized
109 // with all zeroes.
110 let mut keys: RoundKeys = mem::zeroed();
111
112 let k: __m128i = _mm_loadu_si128(mem_addr:key.as_ptr() as *const __m128i);
113 keys[0] = k;
114
115 expand_round!(keys, 1, 0x01);
116 expand_round!(keys, 2, 0x02);
117 expand_round!(keys, 3, 0x04);
118 expand_round!(keys, 4, 0x08);
119 expand_round!(keys, 5, 0x10);
120 expand_round!(keys, 6, 0x20);
121 expand_round!(keys, 7, 0x40);
122 expand_round!(keys, 8, 0x80);
123 expand_round!(keys, 9, 0x1B);
124 expand_round!(keys, 10, 0x36);
125
126 keys
127}
128
129#[inline]
130#[target_feature(enable = "aes")]
131pub(super) unsafe fn inv_expanded_keys(keys: &RoundKeys) -> RoundKeys {
132 [
133 keys[0],
134 _mm_aesimc_si128(keys[1]),
135 _mm_aesimc_si128(keys[2]),
136 _mm_aesimc_si128(keys[3]),
137 _mm_aesimc_si128(keys[4]),
138 _mm_aesimc_si128(keys[5]),
139 _mm_aesimc_si128(keys[6]),
140 _mm_aesimc_si128(keys[7]),
141 _mm_aesimc_si128(keys[8]),
142 _mm_aesimc_si128(keys[9]),
143 keys[10],
144 ]
145}
146